#!/usr/bin/env python3
"""Ask Claude (with web search) a list of buyer questions, then count who gets named.

Claude does the work: it answers each question and lists the companies it named. Plain code then
checks the work: it recounts every name from the answer text itself, and reports any name Claude
listed that the answer does not contain, so a near-miss never counts.

    pip install anthropic
    export ANTHROPIC_API_KEY=...            # or LOOP_ANTHROPIC_API_KEY
    python3 ai_mention_check.py                                # live run, writes the files below
    python3 ai_mention_check.py --replay ai_mention_check-answers.json            # no key, no cost
    python3 ai_mention_check.py --replay ai_mention_check-answers.json --plant-error
    add --strict to exit 1 when any answer is caught (for a scheduled job); add --out <folder> to write elsewhere

Everything the script writes goes to --out (default: the folder this file is in):
  ai_mention_check-answers.json   every request and answer (model id, date, prompt, raw answer, tokens)
  ai_mention_check-counts.csv     one row per company: questions that named it
  run-view.html                   the run, played one question at a time
"""
import argparse
import csv
import datetime as dt
import html
import json
import os
import pathlib
import re
import sys

# ---- EDIT THIS BLOCK TO RUN IT ON YOUR OWN COMPANY ------------------------------------------
# "Matomo" and the four tools below are a public stand-in for your company and your competitors.
# Matomo is not a client and has no part in this guide.
COMPANY = "Matomo"
# Every spelling that counts as your company. Matched whole-word and case-sensitive, so "Plausible"
# counts but the ordinary word "plausible" does not. Assistants shorten names: add the short form.
COMPANY_NAMES = ["Matomo", "matomo.org"]
# Competitors: display name -> the spellings that count as that competitor.
COMPETITORS = {
    "Plausible Analytics": ["Plausible Analytics", "Plausible", "plausible.io"],
    "Fathom Analytics": ["Fathom Analytics", "Fathom", "usefathom.com"],
    "Simple Analytics": ["Simple Analytics", "simpleanalytics.com"],
    "Google Analytics": ["Google Analytics", "GA4"],
}
# The questions a buyer would type. Do not put your own company name in them. At most 10.
QUESTIONS = [
    "What are the best privacy-friendly website analytics tools?",
    "Which website analytics tool works without a cookie consent banner?",
    "What is a good self-hosted web analytics platform?",
    "Which web analytics tools are GDPR friendly for a European website?",
    "What can I use to replace Google Analytics on a small business website?",
    "Which open source web analytics tool should a nonprofit use?",
]
# ---------------------------------------------------------------------------------------------

MODEL = "claude-haiku-5-5"        # cheapest current model; it is enough to answer and list names
WEB_SEARCH_TOOL = "web_search_20250305"
MAX_SEARCHES_PER_QUESTION = 3
MAX_QUESTIONS = 10
MAX_COST_USD = 2.00
# Anthropic's price list, read 2026-11-03: https://platform.claude.com/docs/en/about-claude/pricing
PRICE_IN_PER_MTOK, PRICE_OUT_PER_MTOK, PRICE_PER_SEARCH = 0.10, 0.50, 0.01
PLANTED_AT = (0, 1, 3)            # --plant-error damages questions 1, 2 and 4
SLUG = "track-whether-ai-assistants-name-your-company"
STEM = "ai_mention_check"

SYSTEM = (
    "You answer a buyer's question the way an AI assistant would. Search the web if it helps. "
    "Reply with ONLY a JSON object, no markdown fence: "
    '{"answer": "<your answer to the buyer in plain text, under 180 words>", '
    '"companies_named": ["<every company or product name your answer recommends or mentions>"]}'
)


def aliases_of(name):
    return COMPANY_NAMES if name == COMPANY else COMPETITORS[name]


def mentions(text, names):
    """Whole-word, case-sensitive match of any spelling in `names`. 'Plausibly' is not 'Plausible'."""
    return any(re.search(r"(?<![\w.])" + re.escape(n) + r"(?![\w])", text) for n in names)


def cost_of(tokens_in, tokens_out, searches):
    return tokens_in * PRICE_IN_PER_MTOK / 1e6 + tokens_out * PRICE_OUT_PER_MTOK / 1e6 + searches * PRICE_PER_SEARCH


def ask_claude(client, question):
    resp = client.messages.create(
        # no temperature: claude-haiku-5-5 answers 400 "temperature is deprecated for this model"
        model=MODEL, max_tokens=1500, system=SYSTEM,
        tools=[{"type": WEB_SEARCH_TOOL, "name": "web_search", "max_uses": MAX_SEARCHES_PER_QUESTION}],
        messages=[{"role": "user", "content": question}],
    )
    raw = "".join(b.text for b in resp.content if b.type == "text").strip()
    queries = [b.input.get("query", "") for b in resp.content if b.type == "server_tool_use"]
    searches = getattr(getattr(resp.usage, "server_tool_use", None), "web_search_requests", None) or len(queries)
    return {"question": question, "model": MODEL, "date": dt.date.today().isoformat(),
            "prompt": question, "system": SYSTEM, "raw_answer": raw, "searches": searches,
            "tokens_in": resp.usage.input_tokens, "tokens_out": resp.usage.output_tokens}


def parse(raw):
    """Return (answer, companies_named, problem). A reply that does not parse is reported, never skipped."""
    if not isinstance(raw, str) or not raw.strip():
        return "", [], "no reply text"
    text = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw.strip())
    try:
        obj = json.loads(text)
        answer, named = obj["answer"], obj["companies_named"]
        if isinstance(answer, str):
            answer = re.sub(r"</?cite[^>]*>", "", answer)  # web search wraps cited sentences in <cite> tags
        if not isinstance(answer, str) or not answer.strip():
            return "", [], "answer is empty"
        if not isinstance(named, list) or not all(isinstance(n, str) for n in named):
            return "", [], "companies_named is not a list of names"
        return answer, named, ""
    except (ValueError, KeyError, TypeError) as e:
        return "", [], f"reply is not the expected JSON ({e.__class__.__name__})"


def plant_error(records):
    """Run 2: damage the saved answers the way a careless model or a bad copy would.
    1. Question 1: the answer's spelling of your company becomes a near-miss ("Matomoo"), but Claude's
       list still names the company. Trips: a listed name the answer text does not contain.
    2. Question 2: Claude's list gains a tracked company its answer never mentions.
       Trips: the same check, for a name that was never there.
    3. Question 4: the reply is cut off, so it is not valid JSON. Trips: the parse check."""
    out = [dict(r) for r in records]
    a, n, _ = parse(out[0]["raw_answer"])
    for spelling in COMPANY_NAMES:
        a = a.replace(spelling, spelling[:-1] + spelling[-1] * 2)
    out[0]["raw_answer"] = json.dumps({"answer": a, "companies_named": n + ([COMPANY] if COMPANY not in n else [])})
    a, n, _ = parse(out[1]["raw_answer"])
    absent = next((c for c in [COMPANY, *COMPETITORS] if not mentions(a, aliases_of(c))), None)
    out[1]["raw_answer"] = json.dumps({"answer": a, "companies_named": n + ([absent] if absent else [])})
    out[3]["raw_answer"] = out[3]["raw_answer"][: len(out[3]["raw_answer"]) // 2]
    return out


def check(records):
    """The plain-code check. Returns per-question results and the company tallies."""
    results = []
    for i, r in enumerate(records, 1):
        answer, named, problem = parse(r["raw_answer"])
        res = {"n": i, "question": r["question"], "problem": problem, "flags": [], "found": [], "answer": answer}
        if not problem:
            for c in [COMPANY, *COMPETITORS]:
                if mentions(answer, aliases_of(c)):
                    res["found"].append(c)
            for listed in named:
                known = next((c for c in [COMPANY, *COMPETITORS] if listed == c or listed in aliases_of(c)), None)
                if known and known not in res["found"]:
                    res["flags"].append(f"Claude listed {listed!r} but the answer text never names it: not counted")
                elif not known and not re.search(r"(?<![\w.])" + re.escape(listed) + r"(?![\w])", answer):
                    res["flags"].append(f"Claude listed {listed!r} but the answer text never says it")
        results.append(res)
    return results


def tally(results):
    ok = [r for r in results if not r["problem"]]
    counts = {c: sum(c in r["found"] for r in ok) for c in [COMPANY, *COMPETITORS]}
    return ok, counts


def view_row(r, label):
    bad = bool(r["problem"] or r["flags"])
    verdict = ("CAUGHT: " + (r["problem"] or "; ".join(r["flags"]))) if bad else \
              ("OK: names " + (", ".join(r["found"]) or "none of the tracked companies"))
    return {"label": label, "q": r["question"], "bad": bad, "verdict": verdict,
            "a": (r["answer"] or "(no usable answer: the reply was cut off)")[:380]}


def write_view(out_dir, results, planted, counts, ok, total_cost, searches):
    """The run, one item at a time. Items 1 to N are the real answers; the last three show the same
    answers damaged by --plant-error, so the item the check caught can be filmed with #step-<n>."""
    rows = [view_row(r, f"Question {r['n']} of {len(results)}") for r in results]
    rows += [view_row(r, f"Planted error {k} of 3, on question {r['n']}")
             for k, r in enumerate([planted[i] for i in PLANTED_AT if i < len(planted)], 1)]
    data = json.dumps({"rows": rows, "counts": counts, "answered": len(ok), "total": len(results),
                       "bad": sum(bool(r["problem"] or r["flags"]) for r in results), "company": COMPANY,
                       "cost": f"${total_cost:.4f}", "searches": searches}).replace("</", "<\\/")
    page = VIEW_TEMPLATE.replace("__DATA__", data).replace("__SLUG__", SLUG)
    (out_dir / "run-view.html").write_text(page, encoding="utf-8")


VIEW_TEMPLATE = r"""<!doctype html>
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="robots" content="noindex"><link rel="canonical" href="/guides/__SLUG__/">
<title>Run view: who gets named</title>
<style>
*{box-sizing:border-box}html,body{margin:0;height:100%;color:#0F172A;font-family:Inter,system-ui,sans-serif}
body{display:flex;justify-content:center;background:#F8FAFC}
#stage{--u:min(100vw,56.25vh);width:var(--u);height:calc(var(--u)*16/9);background:#fff;padding:5% 6%;display:flex;flex-direction:column;overflow:hidden;font-size:calc(var(--u)/1080*34)}
h1{font-family:'Space Grotesk',system-ui,sans-serif;font-size:1.3em;margin:0 0 .4em;line-height:1.15}
.tag{font-size:.85em;color:#64748B;margin:0 0 .5em;min-height:1.2em}
.card{border:2px solid #E2E8F0;border-radius:10px;padding:.7em .9em;margin:0 0 .7em}
.card h2{font-size:.8em;letter-spacing:.1em;text-transform:uppercase;color:#64748B;margin:0 0 .3em}
.card p{margin:0;line-height:1.35}
.card.ok{border-color:#2E63C6}.card.ok h2{color:#2E63C6}
.card.bad{border-color:#0F172A;background:#F1F5F9}.card.bad h2{color:#0F172A}
.step{display:none}.step.on{display:block;animation:fade .4s}
@keyframes fade{from{opacity:0}to{opacity:1}}
#sum{display:none}.bar{height:.6em;background:#2E63C6;border-radius:10px;margin:.15em 0 .5em}
.row{display:flex;justify-content:space-between;gap:1em}.row span:last-child{font-weight:700}
.bd{margin-top:auto;font-size:.8em;color:#64748B}
@media (prefers-reduced-motion: reduce){.step.on{animation:none}#stage{overflow:auto}}
</style></head><body><div id="stage"><div class="tag" id="tag"></div><h1>Who gets named when buyers ask?</h1><div id="main"></div><div id="sum"></div><div class="bd">A live run may word answers differently.</div></div>
<script>
var D=__DATA__,main=document.getElementById('main'),tag=document.getElementById('tag'),sum=document.getElementById('sum');
function esc(s){return String(s).replace(/[&<>"]/g,function(c){return{'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c]})}
main.innerHTML=D.rows.map(function(r){return '<div class="step"><div class="card"><h2>'+esc(r.label)+'</h2><p>'+esc(r.q)+'</p></div>'+
 '<div class="card"><h2>What Claude returned</h2><p>'+esc(r.a)+'</p></div>'+
 '<div class="card '+(r.bad?'bad':'ok')+'"><h2>The check</h2><p>'+esc(r.verdict)+'</p></div></div>'}).join('');
var steps=main.children;
var h='<div class="card"><h2>Questions that named each company, of '+D.answered+' answers checked</h2>';
Object.keys(D.counts).forEach(function(k){var w=D.answered?Math.round(100*D.counts[k]/D.answered):0;
 h+='<div class="row"><span>'+esc(k)+(k===D.company?' (yours)':'')+'</span><span>'+D.counts[k]+'</span></div><div class="bar" style="width:'+w+'%"></div>'});
sum.innerHTML=h+'</div><div class="card ok"><h2>Run total</h2><p>Web searches: '+D.searches+'. Cost: '+esc(D.cost)+'.</p></div>';
function only(i){for(var j=0;j<steps.length;j++)steps[j].className=j===i?'step on':'step';sum.style.display='none';tag.textContent='Step '+(i+1)+' of '+D.rows.length}
function summary(){for(var j=0;j<steps.length;j++)steps[j].className='step';sum.style.display='block';tag.textContent='Summary'}
var reduce=window.matchMedia&&window.matchMedia('(prefers-reduced-motion: reduce)').matches;
var m=/^#step-(\d+)$/.exec(location.hash||'');
if(reduce){for(var j=0;j<steps.length;j++)steps[j].className='step on';sum.style.display='block';tag.textContent='Every step'}
else if(m){only(Math.min(Math.max(parseInt(m[1],10),1),D.rows.length)-1)}
else{var per=Math.max(2000,Math.min(4000,30000/(D.rows.length+1))),i=0;
 (function next(){if(i>=D.rows.length){summary();return}only(i++);setTimeout(next,per)})()}
</script></body></html>
"""


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--replay", help="re-run every check from a saved answers file: no key, no cost")
    ap.add_argument("--plant-error", action="store_true", help="damage three saved answers, to show the check catching them")
    ap.add_argument("--strict", action="store_true", help="exit 1 when any answer is caught (for a scheduled job)")
    ap.add_argument("--out", default=str(pathlib.Path(__file__).resolve().parent), help="folder for every file written")
    a = ap.parse_args()
    out_dir = pathlib.Path(a.out)
    out_dir.mkdir(parents=True, exist_ok=True)
    if len(QUESTIONS) > MAX_QUESTIONS:
        sys.exit(f"{len(QUESTIONS)} questions; the limit is {MAX_QUESTIONS}")
    leaks = [q for q in QUESTIONS if mentions(q, COMPANY_NAMES)]
    if leaks:
        sys.exit("a question names your company, so the answer would be led to it: " + leaks[0])

    if a.replay:
        saved = json.loads(pathlib.Path(a.replay).read_text(encoding="utf-8"))
        records = saved["records"]
    else:
        key = os.environ.get("LOOP_ANTHROPIC_API_KEY") or os.environ.get("ANTHROPIC_API_KEY")
        if not key:
            sys.exit("set ANTHROPIC_API_KEY, or use --replay <answers file>")
        import anthropic
        worst = len(QUESTIONS) * (MAX_SEARCHES_PER_QUESTION * PRICE_PER_SEARCH + cost_of(60000, 1500, 0))
        if worst > MAX_COST_USD:
            sys.exit(f"worst-case cost ${worst:.2f} is over the ${MAX_COST_USD:.2f} limit")
        # an empty ANTHROPIC_BASE_URL in the environment would break the client, so fall back to the default
        client = anthropic.Anthropic(api_key=key, base_url=os.environ.get("ANTHROPIC_BASE_URL") or "https://api.anthropic.com")
        records = [ask_claude(client, q) for q in QUESTIONS]
        (out_dir / f"{STEM}-answers.json").write_text(
            json.dumps({"model": MODEL, "date": dt.date.today().isoformat(), "records": records}, indent=2), encoding="utf-8")
    clean_records = records
    can_plant = len(clean_records) >= 4   # the planted errors sit on questions 1, 2 and 4
    if a.plant_error and not can_plant:
        sys.exit("--plant-error needs at least 4 questions")
    planted_results = check(plant_error(clean_records)) if can_plant else []
    if a.plant_error:
        records = plant_error(clean_records)

    tokens_in = sum(r["tokens_in"] for r in records)
    tokens_out = sum(r["tokens_out"] for r in records)
    searches = sum(r["searches"] for r in records)
    total_cost = cost_of(tokens_in, tokens_out, searches)
    results = check(records)
    ok, counts = tally(results)

    print(f"model {records[0]['model']}, run date {records[0]['date']}, {len(records)} questions")
    print(f"tokens in {tokens_in}, tokens out {tokens_out}, web searches {searches}, cost ${total_cost:.4f}")
    print(f"tracking: {COMPANY}; competitors: {', '.join(COMPETITORS)}")
    print()
    for r in results:
        line = "REPLY PROBLEM: " + r["problem"] if r["problem"] else "names: " + (", ".join(r["found"]) or "none of the tracked companies")
        print(f"Q{r['n']}: {r['question']}\n    {line}")
        for f in r["flags"]:
            print(f"    CAUGHT: {f}")
    print()
    print(f"{'company':<22}{'questions naming it':>20}{'share':>8}")
    rows = []
    for c, n in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])):
        share = (100 * n / len(ok)) if ok else 0.0
        rows.append((c, n, share))
        print(f"{c + (' (yours)' if c == COMPANY else ''):<22}{n:>14} of {len(ok):<3}{share:>7.1f}%")
    bad = [r for r in results if r["problem"] or r["flags"]]
    # self-check: a count may never exceed the answers that parsed
    assert all(n <= len(ok) for n in counts.values()), "a count is larger than the answers checked"
    print()
    print(f"answers that parsed: {len(ok)} of {len(results)}; answers caught for a person to read: {len(bad)}")
    print("RESULT: " + ("CHECK FAILED, read the caught answers before using these counts" if bad else "PASSED, every name in every list is in its answer"))
    with open(out_dir / f"{STEM}-counts.csv", "w", encoding="utf-8", newline="") as f:
        w = csv.writer(f, lineterminator="\n")
        w.writerow(["company", "questions_naming_it", "answers_checked", "share_percent"])
        for c, n, s in rows:
            w.writerow([c, n, len(ok), f"{s:.1f}"])
    # the run view always shows the real answers, then the three planted errors, whichever mode ran
    clean_results = check(clean_records)
    clean_ok, clean_counts = tally(clean_results)
    write_view(out_dir, clean_results, planted_results, clean_counts, clean_ok, total_cost, searches)
    return 1 if (bad and a.strict) else 0


if __name__ == "__main__":
    sys.exit(main())