#!/usr/bin/env python3
"""
Regenerate every number in the TextSight false-positive study from the
released per-document CSV. No dependencies beyond the standard library.

    python3 reproduce_stats.py textsight-fpr-2026-per-document.csv

Why this file exists: the study's claim is "here is our false-positive rate,
including the parts that do not flatter us". A reader who cannot rebuild the
numbers from the released data has to take that on trust, which is exactly
what the study is arguing against.

On the confidence interval for the group difference
---------------------------------------------------
Two defensible intervals exist and they disagree about significance, so both
are printed rather than one being chosen silently:

  * Document-level: treats all 1,180 abstracts as independent draws.
  * Country-cluster: resamples the 13 affiliation countries, not the
    documents. Papers from one country share journals, fields and
    copy-editing conventions, so they are not independent; ignoring that
    understates the standard error.

The cluster interval is the honest one for a group claim, and it is wider.
"""

import csv
import math
import random
import sys
from collections import defaultdict

Z = 1.96
BOOTSTRAP_N = 20000
SEED = 20260901  # fixed so two runs of this script agree


def wilson(k, n, z=Z):
    """Wilson score interval — behaves at the small counts this study has,
    where the normal approximation would run below zero."""
    if n == 0:
        return (0.0, 0.0)
    p = k / n
    d = 1 + z * z / n
    centre = (p + z * z / (2 * n)) / d
    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
    return (100 * (centre - half), 100 * (centre + half))


def load(path):
    with open(path, newline="", encoding="utf-8") as fh:
        return list(csv.DictReader(fh))


def rate(rows):
    n = len(rows)
    k = sum(int(r["flagged_ai"]) for r in rows)
    return k, n, (100 * k / n if n else 0.0)


def group_diff(rows):
    """ESL FPR minus native FPR, in percentage points."""
    g = defaultdict(lambda: [0, 0])
    for r in rows:
        g[r["group"]][0] += 1
        g[r["group"]][1] += int(r["flagged_ai"])
    if g["esl"][0] == 0 or g["native"][0] == 0:
        return None
    return (g["esl"][1] / g["esl"][0] - g["native"][1] / g["native"][0]) * 100


def pct(lo, hi):
    return f"[{lo:+.1f}, {hi:+.1f}]"


def main(path):
    rows = load(path)
    rng = random.Random(SEED)

    print("=" * 68)
    print("TextSight false-positive study — reproduced from the released CSV")
    print("=" * 68)

    k, n, r = rate(rows)
    lo, hi = wilson(k, n)
    print(f"\nDocuments: {n}")
    print(f"Overall false-positive rate: {k}/{n} = {r:.2f}%  95% CI [{lo:.2f}, {hi:.2f}]")

    print("\nVerdict distribution")
    verdicts = defaultdict(int)
    for row in rows:
        verdicts[row["verdict"]] += 1
    for v, c in sorted(verdicts.items(), key=lambda x: -x[1]):
        vlo, vhi = wilson(c, n)
        print(f"  {v:<20} {c:>5}  {100*c/n:>6.2f}%  95% CI [{vlo:.1f}, {vhi:.1f}]")

    print("\nBy language group")
    groups = defaultdict(list)
    for row in rows:
        groups[row["group"]].append(row)
    for g in sorted(groups):
        gk, gn, gr = rate(groups[g])
        glo, ghi = wilson(gk, gn)
        print(f"  {g:<10} {gk:>4}/{gn:<5} = {gr:>5.2f}%  95% CI [{glo:.1f}, {ghi:.1f}]")

    point = group_diff(rows)
    print(f"\nDifference (ESL - native): {point:+.2f} percentage points")

    # Document-level bootstrap.
    doc = []
    for _ in range(BOOTSTRAP_N):
        v = group_diff([rows[rng.randrange(len(rows))] for _ in rows])
        if v is not None:
            doc.append(v)
    doc.sort()
    d_lo, d_hi = doc[int(0.025 * len(doc))], doc[int(0.975 * len(doc))]

    # Country-cluster bootstrap.
    by_country = defaultdict(list)
    for row in rows:
        by_country[row["affiliation_country"]].append(row)
    countries = sorted(by_country)
    clus = []
    for _ in range(BOOTSTRAP_N):
        samp = []
        for _ in countries:
            samp.extend(by_country[countries[rng.randrange(len(countries))]])
        v = group_diff(samp)
        if v is not None:
            clus.append(v)
    clus.sort()
    c_lo, c_hi = clus[int(0.025 * len(clus))], clus[int(0.975 * len(clus))]

    print(f"  document-level bootstrap   95% CI {pct(d_lo, d_hi)}"
          f"   excludes zero: {not (d_lo < 0 < d_hi)}")
    print(f"  country-cluster bootstrap  95% CI {pct(c_lo, c_hi)}"
          f"   excludes zero: {not (c_lo < 0 < c_hi)}")
    print(f"  clusters resampled: {len(countries)} countries")
    print("\n  The cluster interval is the one a group-level claim should quote.")

    print("\nFalse-positive rate at score cut-offs (threshold chosen by the reader)")
    print(f"  {'cut-off':<9}{'ESL':>22}{'native':>22}")
    for cut in (50, 60, 70, 80):
        line = f"  >= {cut:<6}"
        for g in ("esl", "native"):
            sub = groups[g]
            kk = sum(1 for x in sub if float(x["ai_score"]) >= cut)
            klo, khi = wilson(kk, len(sub))
            line += f"{100*kk/len(sub):>10.1f}% [{klo:.1f}-{khi:.1f}]"
        print(line)

    print("\nBy affiliation country")
    by = defaultdict(list)
    for row in rows:
        by[row["affiliation_country"] or "(none)"].append(row)
    for c in sorted(by, key=lambda x: -len(by[x])):
        ck, cn, cr = rate(by[c])
        clo2, chi2 = wilson(ck, cn)
        print(f"  {c:<16} {ck:>3}/{cn:<4} = {cr:>5.2f}%  95% CI [{clo2:.1f}, {chi2:.1f}]")

    print("\nProvenance check — every document must predate the models")
    years = sorted({row["publication_year"] for row in rows})
    print(f"  publication years present: {', '.join(years)}")
    print(f"  all <= 2019 (pre-GPT-3 API): {all(int(y) <= 2019 for y in years)}")
    print()


if __name__ == "__main__":
    main(sys.argv[1] if len(sys.argv) > 1 else "textsight-fpr-2026-per-document.csv")