"""finviz_feed.py — Finviz Elite feed for the Bird's Eye dashboard (v1.17.1).

Writes finviz.json beside quotes.json (the receiver serves this folder, so the dashboard reads it directly).
Three pulls, all Finviz Elite CSV exports authenticated with the key in finviz_token.txt (never printed,
never written to finviz.json):

  stocks  every STOCK_EVERY s   one export for the whole universe (shortlist.json names + SPY/QQQ):
                                float, short float/ratio, earnings date, RVOL, gap, TTM income,
                                EPS surprise, and each name's latest headline
  news    every NEWS_EVERY s    per-ticker headlines for the top 20 shortlist names
  market  every NEWS_EVERY s    Finviz's general market news (category "Market")

Headline tone (bull/bear/neutral) is a KEYWORD read of the title, not Finviz's — Finviz carries no
sentiment. It exists so the scroll can label items bullish/bearish at a glance; treat it as a hint.

Usage
  python finviz_feed.py            # run forever (birdseye-tv.bat starts it)
  python finviz_feed.py --once     # one pass of all three pulls, then exit (first-run check)
  python finviz_feed.py --selftest # offline checks, no network, no key needed
"""
import argparse, csv, io, json, os, re, sys, time, urllib.request, urllib.error, urllib.parse
from datetime import datetime, timezone

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "finviz.json")
TOKEN_FILE = os.path.join(HERE, "finviz_token.txt")
SHORTLIST = os.path.join(os.path.dirname(HERE), "Bernie_s Shortlist", "shortlist.json")
BASE = "https://elite.finviz.com"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0 Safari/537.36"

INDEXES = ["SPY", "QQQ"]
TOP_N = 20
STOCK_EVERY = 120
NEWS_EVERY = 180
NEWS_PER_NAME = 5
MARKET_ITEMS = 12
ALL_COLS = ",".join(str(i) for i in range(160))   # ask for every column; parse by header NAME, not position

FIELDS = {  # finviz.json key -> Finviz header
    "float": "Shares Float", "shares_out": "Shares Outstanding", "float_pct": "Float %",
    "short_float": "Short Float", "short_ratio": "Short Ratio", "earnings": "Earnings Date",
    "rvol": "Relative Volume", "gap": "Gap", "chg": "Change", "price": "Price", "income": "Income",
    "eps_surprise": "EPS Surprise", "rev_surprise": "Revenue Surprise", "sector": "Sector", "industry": "Industry",
    "news_time": "News Time", "news_title": "News Title", "news_url": "News URL", "digest": "Daily Digest",
}
TEXT_FIELDS = {"earnings", "sector", "industry", "news_time", "news_title", "news_url", "digest"}
# market news: Finviz's "Market" category mixes in politics and lifestyle; keep wire/markets desks talking markets
MARKET_SOURCES = {"bloomberg", "wsj", "marketwatch", "cnbc", "reuters", "yahoo finance", "barrons.com", "financial times", "associated press"}
MARKET_WORDS = r"stock|dow|nasdaq|s&p|yield|treasur|bond|fed\b|rate|inflation|oil|crude|jobs|labor|payroll|cpi|market|selloff|rally|earnings|tariff|dollar"

BULL = r"upgrad|\bbeats?\b|tops|raises?\b|raised|surg|soar|jump|rall(y|ies)|record|\bwins?\b|awarded|approv|partnership|outperform|buy rating|overweight|higher|gains?\b|rebound|bullish|breakout|expands?|strong"
BEAR = r"(rising|rise in|surge in|jump in).{0,20}yields|yields.{0,30}(highs?\b|rising|climbing)|downgrad|\bmiss(es|ed)?\b|\bcuts?\b|falls?\b|fell|drops?\b|plung|sinks?|tumbl|slump|slid|slide|lawsuit|probe|investigat|offering|dilut|halts?\b|delay|warns?|warning|lower|sell-?off|crash|bearish|underperform|underweight|recession|weak|yields? (climb|rise|jump|spike|surge|hit)|losses?\b|layoff"


def tone_of(title):
    t = (title or "").lower()
    b, s = len(re.findall(BULL, t)), len(re.findall(BEAR, t))
    return "bull" if b > s else "bear" if s > b else "neutral"


def num(v):
    """'760.93' -> 760.93 ; '4.75%' -> 4.75 ; '' -> None"""
    v = (v or "").strip().replace(",", "")
    if not v or v == "-":
        return None
    try:
        return float(v.rstrip("%"))
    except ValueError:
        return v


def read_token(path=TOKEN_FILE):
    try:
        with open(path, encoding="utf-8-sig") as f:
            tok = f.read().strip()
    except OSError:
        raise RuntimeError("finviz_token.txt not found in the BirdsEye folder")
    tok = tok.split("auth=")[-1].split("&")[0].strip()   # tolerate a pasted export link
    if not re.fullmatch(r"[A-Za-z0-9\-_]{8,}", tok):
        raise RuntimeError("finviz_token.txt does not look like a Finviz key (paste only the key)")
    return tok


def universe(path=SHORTLIST):
    with open(path, encoding="utf-8") as f:
        d = json.load(f)
    names = d.get("names", d.get("tickers")) if isinstance(d, dict) else d
    if isinstance(names, dict):
        ranked = [k.upper() for k, _ in sorted(names.items(), key=lambda kv: (kv[1] or {}).get("priority") or 999)]
    else:
        ranked = [(x if isinstance(x, str) else x.get("ticker") or x.get("s")).upper() for x in names]
    return ranked, ranked[:TOP_N]


def get_csv(path, params, token, timeout=30):
    q = urllib.parse.urlencode(dict(params, auth=token), safe=",")
    req = urllib.request.Request(f"{BASE}{path}?{q}", headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        body = r.read().decode("utf-8-sig", "replace")
    if body.lstrip().startswith("<"):
        # Finviz answers an unauthenticated export with its HTML login page, not an HTTP error
        raise RuntimeError("Finviz returned a web page instead of CSV — the key was rejected or has expired")
    return list(csv.DictReader(io.StringIO(body)))


def parse_stocks(rows):
    out = {}
    for r in rows:
        sym = (r.get("Ticker") or "").upper()
        if not sym:
            continue
        rec = {k: (r.get(h) or None) if k in TEXT_FIELDS else num(r.get(h)) for k, h in FIELDS.items()}
        if rec.get("news_title"):
            rec["news_tone"] = tone_of(rec["news_title"])
        out[sym] = rec
    return out


def parse_news(rows, per_name=NEWS_PER_NAME):
    by = {}
    for r in rows:
        item = {"t": r.get("Date"), "title": r.get("Title"), "src": r.get("Source"),
                "url": r.get("Url"), "tone": tone_of(r.get("Title"))}
        for sym in (r.get("Ticker") or "").upper().split(","):   # one story can carry "SPY,QQQ"
            sym = sym.strip()
            if sym and len(by.setdefault(sym, [])) < per_name:
                by[sym].append(item)
    return by


def parse_market(rows, n=MARKET_ITEMS):
    out = []
    for r in rows:
        if (r.get("Category") or "").lower() != "market":
            continue
        if (r.get("Source") or "").lower() not in MARKET_SOURCES or not re.search(MARKET_WORDS, (r.get("Title") or "").lower()):
            continue
        out.append({"t": r.get("Date"), "title": r.get("Title"), "src": r.get("Source"),
                    "url": r.get("Url"), "tone": tone_of(r.get("Title"))})
        if len(out) >= n:
            break
    return out


def write_atomic(path, obj):
    tmp = path + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        json.dump(obj, f, indent=1)
    os.replace(tmp, path)


class Feed:
    def __init__(self):
        old = {}
        try:
            with open(OUT, encoding="utf-8") as f:
                old = json.load(f)
        except (OSError, ValueError):
            pass
        self.doc = {"names": old.get("names", {}), "news": old.get("news", {}), "market": old.get("market", []),
                    "stocks_asof": old.get("stocks_asof"), "news_asof": old.get("news_asof"), "error": None}
        self.next_stock = self.next_news = 0.0

    def save(self):
        self.doc["asof"] = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
        self.doc["source"] = "Finviz Elite exports (headline tone = keyword read of the title)"
        write_atomic(OUT, self.doc)

    def step(self, force=False):
        now = time.time()
        stamp = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
        try:
            token = read_token()
            allnames, top = universe()
            if force or now >= self.next_stock:
                rows = get_csv("/export.ashx", {"v": "152", "t": ",".join(INDEXES + allnames), "c": ALL_COLS}, token)
                self.doc["names"] = parse_stocks(rows)
                self.doc["stocks_asof"] = stamp
                self.next_stock = now + STOCK_EVERY
                print(f"{datetime.now():%H:%M:%S}  stocks  {len(self.doc['names'])} names", flush=True)
            if force or now >= self.next_news:
                self.doc["news"] = parse_news(get_csv("/news_export.ashx", {"v": "3", "t": ",".join(INDEXES + top)}, token))
                self.doc["market"] = parse_market(get_csv("/news_export.ashx", {"v": "1"}, token))
                self.doc["news_asof"] = stamp
                self.next_news = now + NEWS_EVERY
                print(f"{datetime.now():%H:%M:%S}  news    {sum(map(len, self.doc['news'].values()))} headlines, "
                      f"{len(self.doc['market'])} market", flush=True)
            self.doc["error"] = None
        except urllib.error.HTTPError as e:
            self.doc["error"] = f"Finviz HTTP {e.code} at {datetime.now():%H:%M}"
            print(self.doc["error"], flush=True)
            self.next_stock = self.next_news = now + (300 if e.code == 429 else 60)
        except Exception as e:
            self.doc["error"] = str(e)
            print(f"{datetime.now():%H:%M:%S}  {e}", flush=True)
            self.next_stock = self.next_news = now + 60
        self.save()

    def run(self, once=False):
        if once:
            return self.step(force=True)
        while True:
            self.step()
            time.sleep(15)


def selftest():
    ok = 0

    def check(c, what):
        nonlocal ok
        if not c:
            raise AssertionError(what)
        ok += 1

    check(tone_of("Analyst upgrades HOOD, raises target") == "bull", "bull tone")
    check(tone_of("U.S. stocks continue to fall as Treasury yields keep climbing") == "bear", "bear tone")
    check(tone_of("Stocks Fall as Long-Term Yields Hit 24-Year High") == "bear", "yields hit")
    check(tone_of("What Has The Market Missed In Coinbase Stock's Drop?") == "bear", "drop")
    check(tone_of("Robinhood to present at conference") == "neutral", "neutral")
    check(tone_of("Stock Market Today: Relentless Rise in Treasury Yields Continues") == "bear", "rise in yields")
    check(tone_of("Nasdaq opens higher as chip stocks lead") == "bull", "opens higher")
    check(num("4.75%") == 4.75 and num("760.93") == 760.93 and num("") is None and num("103,749.60") == 103749.6, "num")

    hdr = '"No.","Ticker","Shares Float","Short Float","Earnings Date","Relative Volume","Income","News Title","News URL","News Time","Daily Digest"\r\n'
    body = hdr + '1,"HOOD",760.93,4.75%,7/29/2026 4:30:00 PM,0.73,2068.00,"Robinhood shares jump on record volumes",https://x,2026-09-29 11:50:07,"CEO sold 259,166 shares"\r\n'
    rows = list(csv.DictReader(io.StringIO(body)))
    s = parse_stocks(rows)["HOOD"]
    check(s["float"] == 760.93 and s["short_float"] == 4.75 and s["rvol"] == 0.73 and s["income"] == 2068.0, "stock fields")
    check(s["earnings"] == "7/29/2026 4:30:00 PM" and s["news_tone"] == "bull", "earnings + headline tone")
    check(s["gap"] is None, "missing column is None, not an error")
    check(s["digest"] == "CEO sold 259,166 shares", "digest kept as text")

    nb = '"Title","Source","Date","Url","Category","Ticker"\r\n' + "".join(
        f'"HOOD item {i} falls","Src",2026-09-29 11:0{i}:00,"https://u/{i}",Stock,"HOOD"\r\n' for i in range(7))
    n = parse_news(list(csv.DictReader(io.StringIO(nb))))
    check(len(n["HOOD"]) == NEWS_PER_NAME and n["HOOD"][0]["tone"] == "bear", "news cap + tone")
    multi = '"Title","Source","Date","Url","Category","Ticker"\r\n"Pre-Markets Rebound","Zacks",2026-09-29 09:33:00,"https://z",Stock,"SPY,QQQ"\r\n'
    mn = parse_news(list(csv.DictReader(io.StringIO(multi))))
    check(set(mn) == {"SPY", "QQQ"} and mn["SPY"][0]["title"] == "Pre-Markets Rebound", "multi-ticker story goes to each name")

    mb = ('"Title","Source","Date","Url","Category"\r\n"Stocks rally","Bloomberg",2026-09-29 12:00:00,"https://m",Market\r\n'
          '"A blog","B",2026-09-29 12:00:00,"https://b",Blogs\r\n'
          '"Voting groups prepare for the midterms","The New York Times",2026-09-29 12:00:00,"https://n",Market\r\n'
          '"Vessel became stuck on Rhine","Bloomberg",2026-09-29 12:00:00,"https://v",Market\r\n')
    m = parse_market(list(csv.DictReader(io.StringIO(mb))))
    check(len(m) == 1 and m[0]["tone"] == "bull", "market filter keeps markets desks talking markets")

    import tempfile
    d = tempfile.mkdtemp()
    p = os.path.join(d, "t.txt")
    for text, good in [("abcDEF123456", True), ("https://elite.finviz.com/export.ashx?v=111&auth=abcDEF123456", True),
                       ("  abcDEF123456\n", True), ("paste here", False), ("", False)]:
        with open(p, "w") as f:
            f.write(text)
        try:
            check(read_token(p) == "abcDEF123456" and good, f"token {text!r}")
        except RuntimeError:
            check(not good, f"token {text!r} should be accepted")
    print(f"selftest: {ok} assertions passed")


if __name__ == "__main__":
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("--once", action="store_true")
    ap.add_argument("--selftest", action="store_true")
    a = ap.parse_args()
    if a.selftest:
        selftest()
        sys.exit(0)
    print(f"finviz_feed: writing {OUT}. Ctrl+C to stop.", flush=True)
    try:
        Feed().run(once=a.once)
    except KeyboardInterrupt:
        pass