#!/usr/bin/env python3
"""Check a classic Purview glossary saved through the Data Map Atlas API, before the switch-off.

Reads the answers of the Data Map glossary API (api-version 2023-09-01), saved one file per page:

    <export>/glossaries.json                              GET /datamap/api/atlas/v2/glossary
    <export>/<glossaryGuid>/terms-<offset>.json           GET /datamap/api/atlas/v2/glossary/{glossaryId}/terms
    <export>/<glossaryGuid>/categories-<offset>.json      GET /datamap/api/atlas/v2/glossary/{glossaryId}/categories
    <export>/page-size.txt                                 the limit used for every page (for example 100)

    python3 check_classic_glossary.py <export folder> [--out <folder>]
    python3 check_classic_glossary.py --make-demo <folder> [--plant-errors]

Writes findings.csv and terms.csv to --out (default: this script's folder).
Guide: https://www.datameaning.com/guides/export-classic-purview-glossary-with-the-atlas-api/
Needs Python 3.9+ and duckdb (pip install duckdb).
"""
import argparse
import csv
import json
import os
import pathlib
import random
import re
import sys
import tempfile
import unicodedata
import uuid

import duckdb

plural = lambda n, one, many: f"{n} {one if n == 1 else many}"

# "A single term can only have 100 relationships in Unified Catalog": Microsoft Learn, "Migrate Classic Glossary Terms
# and Enable Asset Curation in Unified Catalog (Preview)" (read 2026-11-21). The page does not define what counts, so
# this script counts related-term links plus assigned assets and columns. A setting, so you can change it.
MAX_RELATIONSHIPS = 100
RELATED = ["seeAlso", "synonyms", "antonyms", "preferredTerms", "preferredToTerms", "replacedBy", "replacementTerms",
           "translatedTerms", "translationTerms", "isA", "classifies", "validValues", "validValuesFor"]
CHECKS = [
    ("PAGES_MISSING", "Every page of terms and categories was saved"),
    ("DUPLICATE_GUID", "No term guid appears twice"),
    ("UNKNOWN_CATEGORY", "Every category a term names is in the export"),
    ("DANGLING_RELATED", "Every related term is in the export"),
    ("DUPLICATE_NAME", "No two terms in one glossary share a name, ignoring case and spaces"),
    ("NO_STEWARD", "Every approved term has a steward or an expert"),
    ("OVER_RELATIONSHIP_LIMIT", f"No term has more than {MAX_RELATIONSHIPS} relationships"),
]


def stop(msg):
    print(f"CANNOT READ THE EXPORT: {msg}")
    sys.exit(2)


def read_json(path):
    try:
        with open(path, encoding="utf-8-sig") as fh:
            return json.load(fh)
    except (OSError, ValueError) as e:
        stop(f"{path} is not readable JSON ({e.__class__.__name__}: {e})")


def name_key(name):
    return "".join(ch for ch in unicodedata.normalize("NFKC", name or "").casefold() if not ch.isspace())


def offset_pages(folder, prefix, size, findings, label):
    """Every <prefix>-<offset>.json in order; a full page must be followed by the next offset."""
    files = {}
    for p in folder.glob(f"{prefix}-*.json"):
        m = re.fullmatch(rf"{prefix}-(\d+)\.json", p.name)
        if m:
            files[int(m.group(1))] = p
    items, offset = [], 0
    while offset in files:
        doc = read_json(files[offset])
        if not isinstance(doc, list):
            stop(f"{files[offset]} is not a list: the API answers a list for this call")
        items += [x for x in doc if isinstance(x, dict)]
        if len(doc) < size:
            break
        offset += size
    else:
        if offset or not files:
            findings.append(("PAGES_MISSING", label, "", f"{prefix}-{offset}.json was not saved after a full page"))
    extra = sorted(o for o in files if o > offset)
    if extra:
        findings.append(("PAGES_MISSING", label, "", f"{prefix} pages after offset {offset} were saved but a page before them was not"))
    return items


def load(folder):
    folder = pathlib.Path(folder)
    if not folder.is_dir():
        stop(f"{folder} is not a folder")
    try:
        size = int((folder / "page-size.txt").read_text().strip())
    except (OSError, ValueError):
        stop("page-size.txt is missing or not a number: write the limit you used for every page")
    glossaries = read_json(folder / "glossaries.json")
    if not isinstance(glossaries, list):
        stop("glossaries.json is not a list")
    findings, terms, cats = [], [], []
    for g in glossaries:
        gid, gname = str(g.get("guid") or ""), str(g.get("name") or "")
        if not gid:
            stop("a glossary in glossaries.json has no guid")
        gdir = folder / gid
        if not gdir.is_dir():
            findings.append(("PAGES_MISSING", gname, "", f"no folder {gid} for this glossary's terms"))
            continue
        for t in offset_pages(gdir, "terms", size, findings, gname):
            if not t.get("guid") or not isinstance(t.get("name"), str):
                stop(f"a term in glossary {gname} has no guid or name")
            t["guid"] = str(t["guid"]).lower()   # guids are compared without case
            terms.append((gid, gname, t))
        # The glossary list names each glossary's terms; a term deleted or moved while the export paged by offset
        # shifts the pages, and a term can fall between two pages without leaving a gap in the page numbers.
        listed = {str(x.get("termGuid")).lower() for x in g.get("terms") or [] if isinstance(x, dict) and x.get("termGuid")}
        lost = listed - {t["guid"] for g2, _, t in terms if g2 == gid}
        if lost:
            findings.append(("PAGES_MISSING", gname, "", f"{plural(len(lost), 'term', 'terms')} the glossary lists "
                             f"{'is' if len(lost) == 1 else 'are'} in no saved page"))
        cats += [(gid, c) for c in offset_pages(gdir, "categories", size, findings, gname) if c.get("guid")]
    return glossaries, terms, cats, findings


def check(folder, out, tag=""):
    glossaries, terms, cats, findings = load(folder)
    con = duckdb.connect()
    rows, rels, tcats = [], [], []
    for gid, gname, t in terms:
        contacts = t.get("contacts") if isinstance(t.get("contacts"), dict) else {}
        stewards = len(contacts.get("Steward") or []) + len(contacts.get("Expert") or [])
        attrs = t.get("attributes") if isinstance(t.get("attributes"), dict) else {}
        n_rel = 0
        for kind in RELATED:
            for r in t.get(kind) or []:
                if isinstance(r, dict) and r.get("termGuid"):
                    rels.append((t["guid"], kind, str(r["termGuid"]).lower()))
                    n_rel += 1
        assigned = len(t.get("assignedEntities") or [])
        for c in t.get("categories") or []:
            if isinstance(c, dict) and c.get("categoryGuid"):
                tcats.append((t["guid"], str(c["categoryGuid"]).lower()))
        rows.append((t["guid"], t["name"], name_key(t["name"]), gid, gname, str(t.get("status") or ""), stewards,
                     n_rel, assigned, n_rel + assigned,
                     "; ".join(map(str, t.get("templateName"))) if isinstance(t.get("templateName"), list) else str(t.get("templateName") or ""), sum(len(v) if isinstance(v, dict) else 1 for v in attrs.values())))
    con.execute("""CREATE TABLE term (guid VARCHAR, name VARCHAR, key VARCHAR, gid VARCHAR, gname VARCHAR, status VARCHAR,
                   stewards INTEGER, related INTEGER, assigned INTEGER, relationships INTEGER, template VARCHAR, attributes INTEGER)""")
    con.executemany("INSERT INTO term VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", rows) if rows else None
    con.execute("CREATE TABLE rel (guid VARCHAR, kind VARCHAR, target VARCHAR)")
    con.executemany("INSERT INTO rel VALUES (?,?,?)", rels) if rels else None
    con.execute("CREATE TABLE tcat (guid VARCHAR, cat VARCHAR)")
    con.executemany("INSERT INTO tcat VALUES (?,?)", tcats) if tcats else None
    con.execute("CREATE TABLE cat (gid VARCHAR, guid VARCHAR, name VARCHAR)")
    con.executemany("INSERT INTO cat VALUES (?,?,?)", [(g, str(c["guid"]).lower(), str(c.get("name") or "")) for g, c in cats]) if cats else None
    n = con.execute("SELECT count(*), count(DISTINCT guid) FROM term").fetchone()
    print(f"Export: {folder}")
    print(f"Read {plural(len(glossaries), 'glossary', 'glossaries')}, {n[0]} term records ({n[1]} distinct guids), "
          f"{plural(len(cats), 'category', 'categories')}, {len(rels)} related-term links")

    def add(code, q, args=()):
        findings.extend((code, *r) for r in con.execute(q, list(args)).fetchall())
    add("DUPLICATE_GUID", "SELECT min(gname), min(name), 'guid ' || guid || ' on ' || count(*) || ' records' FROM term GROUP BY guid HAVING count(*) > 1 ORDER BY 1, 2")
    add("UNKNOWN_CATEGORY", """SELECT DISTINCT t.gname, t.name, 'category ' || c.cat || ' is not in the export' FROM tcat c
        JOIN term t ON t.guid = c.guid WHERE c.cat NOT IN (SELECT guid FROM cat) ORDER BY 1, 2""")
    add("DANGLING_RELATED", """SELECT DISTINCT t.gname, t.name, r.kind || ' link to term ' || r.target || ', which is not in the export'
        FROM rel r JOIN term t ON t.guid = r.guid WHERE r.target NOT IN (SELECT guid FROM term) ORDER BY 1, 2""")
    add("DUPLICATE_NAME", """WITH k AS (SELECT DISTINCT guid, name, key, gid, gname FROM term)
        SELECT k.gname, k.name, 'same name as ' || (SELECT string_agg('"' || o.name || '"', ', ' ORDER BY o.guid) FROM k o
          WHERE o.gid = k.gid AND o.key = k.key AND o.guid <> k.guid)
        FROM k WHERE (SELECT count(*) FROM k o WHERE o.gid = k.gid AND o.key = k.key) > 1 ORDER BY k.gname, k.key, k.guid""")
    add("NO_STEWARD", """SELECT DISTINCT gname, name, 'approved with no steward or expert' FROM term
        WHERE upper(status) = 'APPROVED' AND stewards = 0 ORDER BY 1, 2""")
    add("OVER_RELATIONSHIP_LIMIT", f"""SELECT DISTINCT gname, name, relationships || ' relationships (related terms, assets and columns); Unified Catalog allows {MAX_RELATIONSHIPS}'
        FROM term WHERE relationships > {MAX_RELATIONSHIPS} ORDER BY 1, 2""")

    print()
    print("Terms by glossary and status")
    for g, st, c in con.execute("SELECT gname, status, count(DISTINCT guid) FROM term GROUP BY 1, 2 ORDER BY 1, 2").fetchall():
        print(f"  {g:<22} {st or '(none)':<10} {c:>5}")
    tmpl = con.execute("SELECT count(DISTINCT guid) FILTER (WHERE template <> ''), count(DISTINCT guid) FILTER (WHERE attributes > 0) FROM term").fetchone()
    print(f"Terms built from a term template: {tmpl[0]}; terms with template attribute values: {tmpl[1]}")
    print()
    print("Checks")
    for code, text in CHECKS:
        k = sum(1 for f in findings if f[0] == code)
        print(f"  {'PASS' if k == 0 else 'FAIL'}  {text}" + (f"  ({k})" if k else ""))
    for code, g, name, detail in findings:
        print(f"  {code:<24} {g}: " + (f"{name}: " if name else "") + detail)

    out = pathlib.Path(out)
    out.mkdir(parents=True, exist_ok=True)
    suffix = f"-{tag}" if tag else ""
    safe = lambda v: "'" + str(v) if str(v)[:1] in ("=", "+", "-", "@") else v
    with open(out / f"findings{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh); w.writerow(["check", "glossary", "term", "detail"]); w.writerows([[safe(x) for x in f] for f in findings])
    trows = con.execute("""SELECT guid, name, gname, status, stewards, related, assigned, relationships, template, attributes
                           FROM (SELECT DISTINCT * FROM term) ORDER BY gname, name, guid""").fetchall()
    with open(out / f"terms{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["term_guid", "name", "glossary", "status", "stewards_and_experts", "related_terms", "assigned_entities",
                    "relationships", "term_template", "template_attribute_values"])
        w.writerows([[safe(x) for x in r] for r in trows])
    try:
        where = os.path.relpath(out)
    except ValueError:
        where = str(out)
    print()
    print(f"Wrote findings{suffix}.csv ({len(findings)} rows) and terms{suffix}.csv ({len(trows)} rows) to {where}")
    print(f"RESULT: {len(findings)} findings." + (" Fix them in Purview, export again and re-run." if findings else
                                                 " No check blocks the move."))
    return len(findings)


# ---------------------------------------------------------------------------------------------
def make_demo(folder, plant):
    rng = random.Random(262)
    gid = lambda: str(uuid.UUID(int=rng.getrandbits(128), version=4))
    folder = pathlib.Path(folder)
    marker = folder / ".made-by-make-demo"
    if folder.exists() and any(folder.rglob("*.json")) and not marker.exists():
        sys.exit(f"{folder} already holds JSON files that --make-demo did not write; choose a new folder.")
    folder.mkdir(parents=True, exist_ok=True)
    for old in folder.rglob("*.json"):
        old.unlink()
    marker.write_text("Written by check_classic_glossary.py --make-demo: generated records, not an export.\n")
    size = 25
    (folder / "page-size.txt").write_text(f"{size}\n")
    people = [gid() for _ in range(5)]
    g = {"guid": gid(), "name": "Enterprise Glossary", "qualifiedName": "Glossary"}
    cats = [{"guid": gid(), "name": n, "anchor": {"glossaryGuid": g["guid"]}} for n in ("Finance", "Sales", "Supply Chain")]
    names = ["Net Revenue", "Gross Margin", "Operating Expense", "Accounts Receivable", "Cost Center", "Fiscal Period",
             "Accrual", "Budget Variance", "Deferred Revenue", "Invoice", "Customer", "Active Customer", "Opportunity",
             "Pipeline Value", "Win Rate", "Sales Territory", "Quote", "Order", "Booking", "Renewal", "Supplier",
             "Purchase Order", "Lead Time", "Safety Stock", "Warehouse", "Fill Rate", "Backorder", "Shipment", "Carrier",
             "Demand Forecast", "Reorder Point", "Goods Receipt", "Unit of Measure", "Return Authorization", "Price List"]
    terms = []
    for i, n in enumerate(names):
        c = cats[min(i // 10, 2)]
        t = {"guid": gid(), "name": n, "qualifiedName": f"{n}@Glossary", "status": "Approved" if i % 6 else "Draft",
             "anchor": {"glossaryGuid": g["guid"]}, "shortDescription": f"Invented definition of {n}.",
             "contacts": {"Steward": [{"id": rng.choice(people), "info": ""}]},
             "categories": [{"categoryGuid": c["guid"], "displayText": c["name"]}],
             "assignedEntities": [{"guid": gid(), "typeName": "azure_sql_table"} for _ in range(rng.randint(0, 3))]}
        if i % 7 == 0:
            t["templateName"] = ["Business glossary template"]
            t["attributes"] = {"Business glossary template": {"Data owner": "Finance office", "Retention": "7 years"}}
        terms.append(t)
    for i, t in enumerate(terms):
        if i % 4 == 1 and i + 1 < len(terms):
            t["seeAlso"] = [{"termGuid": terms[i + 1]["guid"], "displayText": terms[i + 1]["name"]}]
        if i % 9 == 2:
            t["synonyms"] = [{"termGuid": terms[i - 1]["guid"], "displayText": terms[i - 1]["name"]}]
    planted = []
    if plant:
        by = {t["name"]: t for t in terms}
        by["Accrual"]["categories"].append({"categoryGuid": gid(), "displayText": "Treasury"}); planted.append("UNKNOWN_CATEGORY")
        by["Win Rate"]["seeAlso"] = [{"termGuid": gid(), "displayText": "Close Rate"}]; planted.append("DANGLING_RELATED")
        dup = dict(by["Invoice"], guid=gid(), name="invoice ", status="Draft"); terms.insert(terms.index(by["Invoice"]) + 1, dup)
        planted.append("DUPLICATE_NAME")
        by["Shipment"]["contacts"] = {"Steward": [], "Expert": []}; planted.append("NO_STEWARD")
        by["Customer"]["assignedEntities"] = [{"guid": gid(), "typeName": "azure_sql_table"} for _ in range(MAX_RELATIONSHIPS + 1)]
        planted.append("OVER_RELATIONSHIP_LIMIT")
    (folder / g["guid"]).mkdir(exist_ok=True)
    pages = [terms[i:i + size] for i in range(0, len(terms), size)]
    if plant:
        pages[1] = [pages[0][-1]] + pages[1]; planted.append("DUPLICATE_GUID")
    for n, page in enumerate(pages):
        (folder / g["guid"] / f"terms-{n * size}.json").write_text(json.dumps(page, indent=1))
    (folder / g["guid"] / "categories-0.json").write_text(json.dumps(cats, indent=1))
    glos = [dict(g, terms=[{"termGuid": t["guid"], "displayText": t["name"]} for t in terms],
                 categories=[{"categoryGuid": c["guid"]} for c in cats])]
    if plant:
        g2 = {"guid": gid(), "name": "Regional Glossary", "qualifiedName": "Regional"}
        regional = [{"guid": gid(), "name": f"Regional term {i + 1}", "status": "Draft", "anchor": {"glossaryGuid": g2["guid"]}}
                    for i in range(size + 1)]
        glos.append(dict(g2, terms=[{"termGuid": t["guid"]} for t in regional]))
        (folder / g2["guid"]).mkdir(exist_ok=True)
        (folder / g2["guid"] / "terms-0.json").write_text(json.dumps(regional[:size], indent=1))
        # a full first page, and the page with the 26th term was never saved
        (folder / g2["guid"] / "categories-0.json").write_text("[]")
        planted.append("PAGES_MISSING")
    (folder / "glossaries.json").write_text(json.dumps(glos, indent=1))
    print(f"Wrote a generated classic glossary export to {folder}: {plural(len(glos), 'glossary', 'glossaries')}, "
          f"{sum(len(p) for p in pages)} term records, {len(cats)} categories, page size {size} (seed 262)")
    if plant:
        assert set(planted) == {c for c, _ in CHECKS}, planted
        print("Planted one error for each check: " + ", ".join(sorted(planted)))


def main():
    ap = argparse.ArgumentParser(description="Check a classic Purview glossary saved through the Atlas API.")
    ap.add_argument("export", nargs="?")
    ap.add_argument("--out", default=str(pathlib.Path(__file__).resolve().parent))
    ap.add_argument("--make-demo", metavar="FOLDER")
    ap.add_argument("--plant-errors", action="store_true")
    ap.add_argument("--tag", default="")
    a = ap.parse_args()
    if a.make_demo:
        make_demo(a.make_demo, a.plant_errors)
        return 0
    if not a.export:
        ap.error("give the export folder, or --make-demo FOLDER")
    print(f"Python {sys.version.split()[0]} duckdb {duckdb.__version__}")
    check(a.export, a.out, a.tag)
    return 0


if __name__ == "__main__":
    sys.exit(main())
