"""Which XBRL tags do large filers actually use for the two numbers free cash
flow is made of, and what happens if you take the first tag that answers.

This is the evidence behind /guides/free-cash-flow-xbrl-tag/. It is kept in the
repo and published so the claims on that page can be checked rather than taken
on trust. Re-run it and the JSON regenerates; if a number on the page and a
number in the JSON disagree, the JSON is right and the page is out of date.

For every company it asks SEC EDGAR's companyconcept endpoint for each candidate
tag, keeps every annual (10-K) fact, and records:
  * which candidate carries the newest annual fact (the live tag),
  * which candidates still answer but stopped years ago (the trap),
  * which candidates 404 (never used),
  * and, where companyconcept returns 200 with no facts at all, whether
    companyfacts for the same company and the same tag disagrees.

Usage:  python research/fcf-tag-survey.py
Writes: research/fcf-tag-survey.json
"""
import io
import json
import os
import time
import urllib.error
import urllib.request

UA = "Fairvalue Radar (privacy@fairvalueradar.com)"
HERE = os.path.dirname(os.path.abspath(__file__))
ANNUAL = ("10-K", "20-F", "40-F")

COMPANIES = [
    ("AAPL", 320193), ("MSFT", 789019), ("NVDA", 1045810), ("AMZN", 1018724),
    ("GOOGL", 1652044), ("META", 1326801), ("TSLA", 1318605), ("JPM", 19617),
    ("XOM", 34088), ("JNJ", 200406), ("WMT", 104169), ("PG", 80424),
    ("KO", 21344), ("PFE", 78003), ("INTC", 50863), ("CVX", 93410),
    ("T", 732717), ("BA", 12927), ("GE", 40545), ("MRK", 310158),
]

CANDIDATES = {
    "ocf": ["NetCashProvidedByUsedInOperatingActivities",
            "NetCashProvidedByUsedInOperatingActivitiesContinuingOperations"],
    "capex": ["PaymentsToAcquirePropertyPlantAndEquipment",
              "PaymentsToAcquireProductiveAssets",
              "PaymentsToAcquireOtherPropertyPlantAndEquipment",
              "PaymentsForCapitalImprovements"],
}


def fetch(url, tries=4):
    """Return (parsed, error). A 404 is an answer, not a failure: it means the
    company has never used that tag. Anything else is retried, because a dropped
    request that looks like 'no data' is exactly the mistake this file exists to
    document."""
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "application/json"})
    for attempt in range(tries):
        try:
            with urllib.request.urlopen(req, timeout=30) as r:
                return json.loads(r.read().decode("utf-8")), None
        except urllib.error.HTTPError as e:
            if e.code == 404:
                return None, "404"
            time.sleep(1.5 * (attempt + 1))
        except Exception:
            time.sleep(1.5 * (attempt + 1))
    return None, "unreachable after %d tries" % tries


def annual_facts(payload):
    rows = [f for f in (payload or {}).get("units", {}).get("USD", [])
            if f.get("form") in ANNUAL and f.get("end")]
    rows.sort(key=lambda f: (f["end"], f.get("accn", "")))
    return rows


def concept(cik, tag):
    time.sleep(0.2)
    return fetch("https://data.sec.gov/api/xbrl/companyconcept/CIK%010d/us-gaap/%s.json" % (cik, tag))


def facts_crosscheck(cik, tag):
    """companyconcept has been seen answering 200 with an empty unit array while
    companyfacts, for the same company and tag, holds decades of filings. Only
    called when that happens, because companyfacts is tens of megabytes."""
    time.sleep(0.3)
    payload, err = fetch("https://data.sec.gov/api/xbrl/companyfacts/CIK%010d.json" % cik, tries=2)
    if err or not payload:
        return {"checked": False, "error": err}
    node = payload.get("facts", {}).get("us-gaap", {}).get(tag)
    rows = annual_facts(node)
    return {"checked": True, "annualFacts": len(rows),
            "last": {"end": rows[-1]["end"], "val": rows[-1]["val"]} if rows else None}


def main():
    out = {"readOn": time.strftime("%Y-%m-%d"), "source": "SEC EDGAR XBRL companyconcept",
           "companies": []}
    for ticker, cik in COMPANIES:
        rec = {"ticker": ticker, "cik": cik}
        for field, tags in CANDIDATES.items():
            seen = {}
            for tag in tags:
                payload, err = concept(cik, tag)
                if err == "404":
                    seen[tag] = {"state": "never used"}
                    continue
                if err:
                    seen[tag] = {"state": "fetch failed", "error": err}
                    continue
                rows = annual_facts(payload)
                if not rows:
                    cross = facts_crosscheck(cik, tag)
                    seen[tag] = {"state": "empty answer", "companyfacts": cross}
                    continue
                seen[tag] = {"state": "has facts", "annualFacts": len(rows),
                             "last": {"end": rows[-1]["end"], "val": rows[-1]["val"]}}
            live = [(t, s) for t, s in seen.items() if s.get("state") == "has facts"]
            live.sort(key=lambda ts: ts[1]["last"]["end"])
            rec[field] = {
                "tags": seen,
                "live": live[-1][0] if live else None,
                "lastEnd": live[-1][1]["last"]["end"] if live else None,
                "abandonedButStillAnswering": [t for t, s in live[:-1]],
            }
        out["companies"].append(rec)
        print("%-6s ocf=%-62s capex=%s" % (ticker, rec["ocf"]["live"], rec["capex"]["live"]))
    io.open(os.path.join(HERE, "fcf-tag-survey.json"), "w", encoding="utf-8").write(
        json.dumps(out, indent=1))
    print("\nwritten: research/fcf-tag-survey.json")


if __name__ == "__main__":
    main()