#!/usr/bin/env python3
"""Check a Microsoft Purview Unified Catalog glossary export before you move it.

Reads the files the Unified Catalog REST API returns (api-version 2026-03-20-preview), saved as the
guide shows, and checks that the export is complete and that every term can be moved:

    <export>/terms/page-001.json, page-002.json, ...       GET /terms, one file per page, in order
    <export>/relationships/<termId>.<ENTITYTYPE>.json        GET /terms/{termId}/relationships, one
                                                             file per term and entity type (add .2,
                                                             .3 before .json for further pages)
    <export>/domains/page-001.json, ...                      GET /businessdomains (optional: names)

Usage:
    python3 check_glossary_export.py <export folder> [--out <folder>]
    python3 check_glossary_export.py --make-demo <folder> [--plant-errors]

Writes findings.csv and terms.csv to --out (default: this script's folder). Exit code 2 when the
export cannot be read; with --fail-on-findings, exit code 1 when a check blocks the move (for a
pipeline step), otherwise 0.
Guide: https://www.datameaning.com/guides/check-purview-glossary-export-before-moving-it/
Needs Python 3.9+ and duckdb (pip install duckdb).
"""
import argparse
import csv
import hashlib
import json
import os
import pathlib
import random
import re
import sys
import tempfile
import unicodedata
import uuid

import duckdb

# Settings. Microsoft changes limits; each one sits here with its source.
# A Fabric item or workspace holds at most 10 tags at a time:
# https://learn.microsoft.com/en-us/fabric/governance/tags-overview (read 2026-11-21)
MAX_TAGS_PER_ITEM = 10
# Entity types whose relationships the export must hold for every term (the API's EntityCategory).
ENTITY_TYPES = ["TERM", "DATAASSET", "DATACOLUMN", "DATAPRODUCT", "CRITICALDATAELEMENT"]
# Term fields the API marks as required: Term in the 2026-03-20-preview specification.
REQUIRED_TERM_FIELDS = ["id", "name", "domain", "status"]
STATUSES = {"DRAFT", "PUBLISHED", "EXPIRED"}
REL_TYPES = {"Related", "Synonym", "Parent"}

CHECKS = [
    ("EXPORT_INCOMPLETE", "Every page of terms was saved, and the last has no nextLink"),
    ("DUPLICATE_ID", "No term id appears twice in the export"),
    ("MISSING_RELATIONSHIPS", "Every term has every page of its links saved, for every entity type"),
    ("ORPHAN_PARENT", "Every parentId is a term in the export"),
    ("PARENT_CYCLE", "No term is its own ancestor"),
    ("DANGLING_TERM_LINK", "Every term-to-term relationship points to a term in the export"),
    ("DUPLICATE_NAME_IN_DOMAIN", "No two terms in one domain share a name, ignoring case and spaces"),
    ("NO_OWNER", "Every published term has an owner"),
    ("OVER_TAG_LIMIT", f"No data asset is linked to more than {MAX_TAGS_PER_ITEM} terms"),
]


def stop(msg):
    print(f"CANNOT READ THE EXPORT: {msg}")
    sys.exit(2)


def read_json(path):
    try:
        with open(path, encoding="utf-8-sig") as fh:
            return json.load(fh)
    except (OSError, ValueError) as e:
        stop(f"{path} is not readable JSON ({e.__class__.__name__}: {e})")


def load_export(folder):
    folder = pathlib.Path(folder)
    if not folder.is_dir():
        stop(f"{folder} is not a folder")
    def page_no(p):
        m = re.fullmatch(r"page-(\d+)\.json", p.name)
        return int(m.group(1)) if m else -1

    pages = sorted((p for p in (folder / "terms").glob("page-*.json") if page_no(p) >= 0), key=page_no)
    if not pages:
        stop(f"no terms/page-*.json files in {folder}")
    nums = [page_no(p) for p in pages]
    term_gaps = [n for n in range(1, nums[-1] + 1) if n not in nums]
    terms, last_next = [], None
    for p in pages:
        i = page_no(p)
        doc = read_json(p)
        if not isinstance(doc, dict) or not isinstance(doc.get("value"), list):
            stop(f"{p} has no \"value\" list: it is not a page of GET /terms")
        for t in doc["value"]:
            if not isinstance(t, dict):
                stop(f"{p} holds a term that is not an object")
            missing = [f for f in REQUIRED_TERM_FIELDS if not isinstance(t.get(f), str) or not t.get(f).strip()]
            if missing:
                stop(f"{p}: a term has no {', '.join(missing)} (term id {t.get('id')!r})")
            if t.get("contacts") is not None and not isinstance(t.get("contacts"), dict):
                stop(f"{p}: term {t['id']!r} has contacts that are not an object")
            terms.append((i, t))
        last_next = doc.get("nextLink")
    rels, rel_files, rel_pages = [], [], {}
    pat = re.compile(r"^([0-9a-fA-F-]{36})\.([A-Z]+)(?:\.(\d+))?\.json$")
    for p in sorted((folder / "relationships").glob("*.json")):
        m = pat.match(p.name)
        if not m:
            stop(f"relationships/{p.name} is not named <termId>.<ENTITYTYPE>.json")
        term_id, etype = m.group(1).lower(), m.group(2)
        if etype not in ENTITY_TYPES:
            stop(f"relationships/{p.name}: entity type {etype} is not one of {', '.join(ENTITY_TYPES)}")
        doc = read_json(p)
        if not isinstance(doc, dict) or not isinstance(doc.get("value"), list):
            stop(f"relationships/{p.name} has no \"value\" list")
        rel_files.append((term_id, etype))
        rel_pages.setdefault((term_id, etype), {})[int(m.group(3) or 1)] = bool(doc.get("nextLink"))
        for r in doc["value"]:
            if not isinstance(r, dict) or not r.get("entityId"):
                stop(f"relationships/{p.name} holds a relationship with no entityId")
            rtype = r.get("relationshipType") or ""
            if rtype and rtype not in REL_TYPES:
                stop(f"relationships/{p.name}: relationshipType {rtype!r} is not Related, Synonym or Parent")
            rels.append((term_id, etype, str(r["entityId"]).lower(), rtype))
    domains = {}
    for p in sorted((folder / "domains").glob("page-*.json")):
        doc = read_json(p)
        for d in (doc.get("value") or []) if isinstance(doc, dict) else []:
            if isinstance(d, dict) and d.get("id"):
                domains[str(d["id"]).lower()] = d.get("name") or ""
    # Pages of links: page 1 is <termId>.<TYPE>.json, page n is <termId>.<TYPE>.<n>.json.
    rel_gaps = []
    for (term_id, etype), got in sorted(rel_pages.items()):
        last = max(got)
        for n in range(1, last):
            if n not in got:
                rel_gaps.append((term_id, f"no page {n} of {etype} links"))
        if got[last]:
            rel_gaps.append((term_id, f"page {last} of {etype} links has a nextLink; page {last + 1} was not saved"))
    return pages, terms, last_next, term_gaps, rels, rel_files, rel_gaps, domains


def safe(v):
    """A CSV cell Excel will not run as a formula."""
    s = "" if v is None else str(v)
    return "'" + s if s[:1] in ("=", "+", "-", "@", "\t", "\r") else s


def write_csv(path, header, rows):
    with open(path, "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(header)
        for r in rows:
            w.writerow([safe(x) for x in r])


def name_key(name):
    """A name as a reader sees it: Unicode-normalized, case-folded, every kind of space removed."""
    return "".join(ch for ch in unicodedata.normalize("NFKC", name).casefold() if not ch.isspace())


def parent_cycles(terms):
    """Ids of the terms that sit on a loop of parents. Each term has at most one parent, so one walk per term,
    marking what was seen, finds every loop in time proportional to the number of terms."""
    parent = {}
    for _, t in terms:
        parent.setdefault(t["id"].lower(), (t.get("parentId") or "").lower() or None)
    state, on_cycle = {}, set()
    for start in parent:
        path, node = [], start
        while node in parent and state.get(node) is None:
            state[node] = "walking"
            path.append(node)
            node = parent[node]
        if node in parent and state.get(node) == "walking":
            on_cycle.update(path[path.index(node):])
        for n in path:
            state[n] = "done"
    return sorted(on_cycle)


def bulk_insert(con, table, rows):
    """Load rows through a temporary CSV: much faster than one INSERT per row on a large export."""
    if not rows:
        return
    with tempfile.TemporaryDirectory() as tmp:
        path = pathlib.Path(tmp) / f"{table}.csv"
        with open(path, "w", newline="", encoding="utf-8") as fh:
            csv.writer(fh).writerows(["" if v is None else v for v in r] for r in rows)
        con.execute(f"INSERT INTO {table} SELECT * FROM read_csv(?, header = false, all_varchar = true, "
                    f"quote = '\"', escape = '\"', nullstr = '')", [str(path)])


def check(folder, out, suffix=""):
    pages, terms, last_next, term_gaps, rels, rel_files, rel_gaps, domains = load_export(folder)
    con = duckdb.connect()
    con.execute("""CREATE TABLE term (page INTEGER, id VARCHAR, name VARCHAR, domain VARCHAR, status VARCHAR,
        parent_id VARCHAR, owners INTEGER, experts INTEGER, has_description BOOLEAN, acronyms INTEGER,
        resources INTEGER, attributes INTEGER, name_key VARCHAR)""")
    bulk_insert(con, "term", [
        (pg, t["id"].lower(), t["name"], t["domain"].lower(), t["status"].upper(),
         (t.get("parentId") or "").lower() or None,
         len((t.get("contacts") or {}).get("owner") or []), len((t.get("contacts") or {}).get("expert") or []),
         isinstance(t.get("description"), str) and bool(t["description"].strip()), len(t.get("acronyms") or []),
         len(t.get("resources") or []), len(t.get("managedAttributes") or []), name_key(t["name"])) for pg, t in terms])
    con.execute("CREATE TABLE rel (term_id VARCHAR, entity_type VARCHAR, entity_id VARCHAR, rel_type VARCHAR)")
    bulk_insert(con, "rel", rels)
    con.execute("UPDATE rel SET rel_type = '' WHERE rel_type IS NULL")
    con.execute("CREATE TABLE rel_file (term_id VARCHAR, entity_type VARCHAR)")
    bulk_insert(con, "rel_file", sorted(set(rel_files)))
    con.execute("CREATE TABLE rel_gap (term_id VARCHAR, detail VARCHAR)")
    bulk_insert(con, "rel_gap", rel_gaps)
    con.execute("CREATE TABLE etype (entity_type VARCHAR)")
    con.executemany("INSERT INTO etype VALUES (?)", [(e,) for e in ENTITY_TYPES])
    con.execute("CREATE TABLE dom (id VARCHAR, name VARCHAR)")
    if domains:
        con.executemany("INSERT INTO dom VALUES (?,?)", list(domains.items()))

    # Self-checks: every row read is a row loaded.
    n_terms = con.execute("SELECT count(*) FROM term").fetchone()[0]
    n_rels = con.execute("SELECT count(*) FROM rel").fetchone()[0]
    assert n_terms == len(terms) and n_rels == len(rels), "rows read and rows loaded differ"
    ids = sorted({t["id"].lower() for _, t in terms})
    print(f"Export: {folder}")
    print(f"Read {len(pages)} pages of terms ({n_terms} terms, {len(ids)} distinct ids), "
          f"{len(rel_files)} relationship files ({n_rels} relationships), {len(domains)} domain names")
    print("Term ids fingerprint (SHA-256 of the sorted ids): "
          + hashlib.sha256("\n".join(ids).encode()).hexdigest()[:16])
    stale = sorted({tid for tid, _ in rel_files} - set(ids))
    if stale:
        print(f"Note: {len(stale)} terms have relationship files but are not in the pages of terms. Files left "
              "from an earlier export? Save each export to a new, empty folder.")

    findings = []

    def add(code, rows):
        for r in rows:
            findings.append((code, *r))

    name_of = "coalesce((SELECT max(name) FROM term x WHERE x.id = {c}), '')"
    add("EXPORT_INCOMPLETE", [("", "", "", f"terms/page-{n:03d}.json was not saved") for n in term_gaps])
    if last_next:
        add("EXPORT_INCOMPLETE", [("", "", "", f"{pages[-1].name} still has a nextLink: later pages were not saved")])
    add("DUPLICATE_ID", con.execute("""SELECT id, min(name), min(domain), 'id on ' || count(*) || ' rows (pages '
        || string_agg(DISTINCT page::VARCHAR, ', ' ORDER BY page::VARCHAR) || ')'
        FROM term GROUP BY id HAVING count(*) > 1 ORDER BY id""").fetchall())
    add("MISSING_RELATIONSHIPS", con.execute("""SELECT id, name, domain, detail FROM (
          SELECT t.id, t.name, t.domain, 'no file for ' || string_agg(e.entity_type, ', ' ORDER BY e.entity_type) AS detail
          FROM (SELECT DISTINCT id, name, domain FROM term) t CROSS JOIN etype e
          LEFT JOIN rel_file f ON f.term_id = t.id AND f.entity_type = e.entity_type
          WHERE f.term_id IS NULL GROUP BY t.id, t.name, t.domain
          UNION ALL
          SELECT g.term_id, coalesce((SELECT max(name) FROM term x WHERE x.id = g.term_id), ''),
                 coalesce((SELECT max(domain) FROM term x WHERE x.id = g.term_id), ''), g.detail
          FROM rel_gap g)
        ORDER BY domain, name, id, detail""").fetchall())
    add("ORPHAN_PARENT", con.execute("""SELECT DISTINCT t.id, t.name, t.domain, 'parentId ' || t.parent_id || ' is not in the export'
        FROM term t WHERE t.parent_id IS NOT NULL AND t.parent_id NOT IN (SELECT id FROM term)
        ORDER BY t.domain, t.name, t.id""").fetchall())
    for tid in parent_cycles(terms):
        add("PARENT_CYCLE", con.execute(f"SELECT id, min(name), min(domain), 'its parent chain leads back to it' "
                                        f"FROM term WHERE id = ? GROUP BY id", [tid]).fetchall())
    add("DANGLING_TERM_LINK", con.execute(f"""SELECT r.term_id, {name_of.format(c='r.term_id')},
        coalesce((SELECT max(domain) FROM term x WHERE x.id = r.term_id), ''),
        coalesce(nullif(r.rel_type, ''), 'Related') || ' link to term ' || r.entity_id || ', which is not in the export'
        FROM rel r WHERE r.entity_type = 'TERM' AND r.entity_id NOT IN (SELECT id FROM term)
        ORDER BY 3, 2, 1, r.entity_id""").fetchall())
    add("DUPLICATE_NAME_IN_DOMAIN", con.execute("""WITH k AS (SELECT DISTINCT id, name, domain, name_key AS key FROM term)
        SELECT k.id, k.name, k.domain, 'same name as ' || (SELECT string_agg('"' || o.name || '"', ', ' ORDER BY o.id)
            FROM k o WHERE o.domain = k.domain AND o.key = k.key AND o.id <> k.id)
        FROM k WHERE (SELECT count(*) FROM k o WHERE o.domain = k.domain AND o.key = k.key) > 1
        ORDER BY k.domain, k.key, k.id""").fetchall())
    add("NO_OWNER", con.execute("""SELECT DISTINCT id, name, domain, 'published with no owner'
        FROM term WHERE status = 'PUBLISHED' AND owners = 0 ORDER BY domain, name, id""").fetchall())
    add("OVER_TAG_LIMIT", con.execute(f"""SELECT '', '', '', 'data asset ' || entity_id || ' is linked to ' ||
        count(DISTINCT term_id) || ' terms; a Fabric item holds at most {MAX_TAGS_PER_ITEM} tags'
        FROM rel WHERE entity_type = 'DATAASSET' GROUP BY entity_id
        HAVING count(DISTINCT term_id) > {MAX_TAGS_PER_ITEM} ORDER BY entity_id""").fetchall())

    # What the glossary holds, and what a flat tag cannot carry.
    print()
    print("Terms by domain and status")
    for d, name, dr, pu, ex, tot in con.execute("""SELECT t.domain, coalesce(max(d.name), ''),
            count(DISTINCT t.id) FILTER (WHERE status = 'DRAFT'), count(DISTINCT t.id) FILTER (WHERE status = 'PUBLISHED'),
            count(DISTINCT t.id) FILTER (WHERE status = 'EXPIRED'), count(DISTINCT t.id)
            FROM term t LEFT JOIN dom d ON d.id = t.domain GROUP BY t.domain ORDER BY 2, 1""").fetchall():
        print(f"  {(name or d):<24} draft {dr:>3}  published {pu:>3}  expired {ex:>3}  total {tot:>3}")
    print("Relationships by entity type")
    for et, rt, n in con.execute("""SELECT e.entity_type, coalesce(nullif(r.rel_type, ''), '-'), count(r.term_id)
            FROM etype e LEFT JOIN rel r ON r.entity_type = e.entity_type GROUP BY 1, 2 ORDER BY 1, 2""").fetchall():
        print(f"  {et:<20} {rt:<8} {n:>5}")
    rich = con.execute("""SELECT count(DISTINCT id) FILTER (WHERE has_description),
        count(DISTINCT id) FILTER (WHERE acronyms > 0), count(DISTINCT id) FILTER (WHERE resources > 0),
        count(DISTINCT id) FILTER (WHERE attributes > 0), count(DISTINCT id) FILTER (WHERE parent_id IS NOT NULL)
        FROM term""").fetchone()
    syn = con.execute("SELECT count(DISTINCT term_id) FROM rel WHERE entity_type = 'TERM' AND rel_type = 'Synonym'").fetchone()[0]
    print("Content a Fabric tag cannot carry (a tag is a name only)")
    for label, n in zip(["description", "acronyms", "resources (links)", "managed attributes", "a parent term"], rich):
        print(f"  terms with {label:<22} {n:>4}")
    print(f"  terms with {'synonyms':<22} {syn:>4}")

    print()
    print("Checks")
    blocked = 0
    for code, text in CHECKS:
        n = sum(1 for f in findings if f[0] == code)
        blocked += n
        print(f"  {'PASS' if n == 0 else 'FAIL'}  {text}" + (f"  ({n})" if n else ""))
    dname = lambda d: domains.get(d, d)
    if findings:
        print()
        print("Findings")
        for code, tid, name, dom, detail in findings:
            who = f"{name} [{dname(dom)}] " if name else ""
            print(f"  {code:<25} {who}{detail}")

    out = pathlib.Path(out)
    out.mkdir(parents=True, exist_ok=True)
    write_csv(out / f"findings{suffix}.csv", ["check", "term_id", "term_name", "domain", "detail"],
              [(c, t, n, dname(d), x) for c, t, n, d, x in findings])
    rows = con.execute("""SELECT t.id, t.name, t.domain, t.status, coalesce(p.name, ''), t.owners, t.experts,
        (SELECT count(*) FROM rel r WHERE r.term_id = t.id AND r.entity_type = 'TERM' AND r.rel_type = 'Related'),
        (SELECT count(*) FROM rel r WHERE r.term_id = t.id AND r.entity_type = 'TERM' AND r.rel_type = 'Synonym'),
        (SELECT count(*) FROM rel r WHERE r.term_id = t.id AND r.entity_type = 'DATAASSET'),
        (SELECT count(*) FROM rel r WHERE r.term_id = t.id AND r.entity_type = 'DATACOLUMN'),
        (SELECT count(*) FROM rel r WHERE r.term_id = t.id AND r.entity_type = 'DATAPRODUCT'),
        (SELECT count(*) FROM rel r WHERE r.term_id = t.id AND r.entity_type = 'CRITICALDATAELEMENT'),
        t.has_description, t.acronyms, t.resources, t.attributes
        FROM (SELECT DISTINCT * EXCLUDE (page) FROM term) t
        LEFT JOIN (SELECT id, max(name) AS name FROM term GROUP BY id) p ON p.id = t.parent_id
        ORDER BY t.domain, t.name, t.id""").fetchall()
    write_csv(out / f"terms{suffix}.csv", ["term_id", "name", "domain", "status", "parent", "owners", "experts",
              "related_terms", "synonyms", "data_assets", "data_columns", "data_products",
              "critical_data_elements", "has_description", "acronyms", "resources", "managed_attributes"],
              [(r[0], r[1], dname(r[2]), *r[3:]) for r in rows])
    print()
    try:
        where = os.path.relpath(out)
    except ValueError:   # another drive on Windows
        where = str(out)
    print(f"Wrote findings{suffix}.csv ({len(findings)} rows) and terms{suffix}.csv ({len(rows)} rows) to {where}")
    if blocked:
        print(f"RESULT: {blocked} findings block the move. Fix them in Purview, export again and re-run.")
        return 1
    print("RESULT: no check blocks the move.")
    return 0


# ---------------------------------------------------------------------------------------------
# Demo export: generated records in the API's documented shape. Invented terms, ids and owners.
FINANCE = ["Net Revenue", "Gross Margin", "Operating Expense", "Accounts Receivable", "Days Sales Outstanding",
           "Cost Center", "General Ledger Account", "Fiscal Period", "Accrual", "Write-off", "Budget Variance",
           "Cash Conversion Cycle", "Capital Expenditure", "Deferred Revenue", "Intercompany Transfer",
           "Chart of Accounts", "Revenue Recognition Date", "Invoice", "Credit Memo", "Exchange Rate"]
SALES = ["Customer", "Active Customer", "Churned Customer", "Opportunity", "Pipeline Value", "Win Rate",
         "Sales Territory", "Account Owner", "Quote", "Discount Rate", "Order", "Order Line", "Booking",
         "Annual Recurring Revenue", "Customer Segment", "Lead", "Qualified Lead", "Renewal", "Price List",
         "Sales Channel"]
SUPPLY = ["Supplier", "Purchase Order", "Lead Time", "Safety Stock", "Inventory On Hand", "Stock Keeping Unit",
          "Warehouse", "Fill Rate", "Backorder", "Bill of Materials", "Reorder Point", "Shipment",
          "Carrier", "On-Time Delivery", "Demand Forecast", "Goods Receipt", "Inventory Turnover",
          "Product Category", "Unit of Measure", "Return Authorization"]
PAGE_SIZE = 25


def make_demo(folder, plant):
    rng = random.Random(273)
    new_id = lambda: str(uuid.UUID(int=rng.getrandbits(128), version=4))
    folder = pathlib.Path(folder)
    marker = folder / ".made-by-make-demo"
    if folder.exists() and any(folder.rglob("*.json")) and not marker.exists():
        sys.exit(f"{folder} already holds JSON files that --make-demo did not write. They may be a real export; "
                 "choose a new folder for the demo.")
    folder.mkdir(parents=True, exist_ok=True)
    marker.write_text("Written by check_glossary_export.py --make-demo: generated records, not an export.\n")
    for sub in ("terms", "relationships", "domains"):
        (folder / sub).mkdir(parents=True, exist_ok=True)
        for old in (folder / sub).glob("*.json"):
            old.unlink()
    domains = [(new_id(), "Finance", FINANCE), (new_id(), "Sales", SALES), (new_id(), "Supply Chain", SUPPLY)]
    people = [new_id() for _ in range(8)]
    assets = [new_id() for _ in range(30)]
    columns = [new_id() for _ in range(60)]
    products = [new_id() for _ in range(6)]
    cdes = [new_id() for _ in range(10)]
    terms, rels = [], {}
    for did, _, names in domains:
        ids = [new_id() for _ in names]
        for i, (tid, name) in enumerate(zip(ids, names)):
            parent = ids[0] if 1 <= i <= 4 else (ids[5] if 6 <= i <= 8 else None)
            status = "PUBLISHED" if i % 7 else ("DRAFT" if i % 2 else "EXPIRED")
            t = {"id": tid, "name": name, "domain": did, "status": status,
                 "systemData": {"createdAt": "2025-06-02T14:00:00.000Z", "createdBy": rng.choice(people),
                                "lastModifiedAt": "2026-09-15T09:30:00.000Z", "lastModifiedBy": rng.choice(people)},
                 "description": f"Invented definition of {name} for this demo.",
                 "contacts": {"owner": [{"id": rng.choice(people), "description": "Data steward"}],
                              "expert": [{"id": rng.choice(people), "description": "Subject expert"}] if i % 3 == 0 else []},
                 "acronyms": [], "resources": []}
            if parent:
                t["parentId"] = parent
            if i % 4 == 0:
                t["acronyms"] = ["".join(w[0] for w in name.split())]
            if i % 6 == 0:
                t["resources"] = [{"name": "Policy page", "url": "https://example.com/policies/" + str(i)}]
            terms.append(t)
            r = {e: [] for e in ENTITY_TYPES}
            if parent:
                r["TERM"].append({"entityId": parent, "relationshipType": "Parent"})
            if i % 5 == 2 and i + 1 < len(ids):
                r["TERM"].append({"entityId": ids[i + 1], "relationshipType": "Related"})
            if i % 9 == 3:
                r["TERM"].append({"entityId": ids[i - 1], "relationshipType": "Synonym"})
            for a in rng.sample(assets, rng.randint(0, 3)):
                r["DATAASSET"].append({"entityId": a, "relationshipType": "Related"})
            for c in rng.sample(columns, rng.randint(0, 2)):
                r["DATACOLUMN"].append({"entityId": c, "relationshipType": "Related"})
            if i % 4 == 1:
                r["DATAPRODUCT"].append({"entityId": rng.choice(products), "relationshipType": "Related"})
            if i % 5 == 0:
                r["CRITICALDATAELEMENT"].append({"entityId": rng.choice(cdes), "relationshipType": "Related"})
            rels[tid] = r
    counts = {}   # keep every data asset well under the tag limit in the clean export
    for tid, r in rels.items():
        kept = []
        for x in r["DATAASSET"]:
            if counts.get(x["entityId"], 0) < MAX_TAGS_PER_ITEM - 2:
                counts[x["entityId"]] = counts.get(x["entityId"], 0) + 1
                kept.append(x)
        r["DATAASSET"] = kept

    planted = []
    if plant:
        all_ids = {t["id"] for t in terms}
        fresh = lambda: next(x for x in iter(new_id, None) if x not in all_ids and x not in assets)
        by_name = {t["name"]: t for t in terms}
        # DUPLICATE_NAME_IN_DOMAIN: a second "Net Revenue" in Finance, cased and spaced differently.
        dup = dict(by_name["Net Revenue"], id=fresh(), name="net  revenue", status="DRAFT")
        terms.insert(terms.index(by_name["Net Revenue"]) + 1, dup)
        rels[dup["id"]] = {e: [] for e in ENTITY_TYPES}
        planted.append("DUPLICATE_NAME_IN_DOMAIN")
        # ORPHAN_PARENT: a parent id that no term has.
        by_name["Write-off"]["parentId"] = fresh()
        planted.append("ORPHAN_PARENT")
        # PARENT_CYCLE: two terms that are each other's parent.
        a, b = by_name["Booking"], by_name["Order"]
        a["parentId"], b["parentId"] = b["id"], a["id"]
        planted.append("PARENT_CYCLE")
        # DANGLING_TERM_LINK: a Related link to a term id the export does not hold.
        rels[by_name["Win Rate"]["id"]]["TERM"].append({"entityId": fresh(), "relationshipType": "Related"})
        planted.append("DANGLING_TERM_LINK")
        # NO_OWNER: a published term with its owner list emptied.
        assert by_name["Gross Margin"]["status"] == "PUBLISHED"
        by_name["Gross Margin"]["contacts"] = dict(by_name["Gross Margin"]["contacts"], owner=[])
        planted.append("NO_OWNER")
        # OVER_TAG_LIMIT: one data asset linked to one more term than a Fabric item can tag.
        hot = fresh()
        for t in terms[:MAX_TAGS_PER_ITEM + 1]:
            rels[t["id"]]["DATAASSET"].append({"entityId": hot, "relationshipType": "Related"})
        planted.append("OVER_TAG_LIMIT")
        # MISSING_RELATIONSHIPS: Lead Time's data asset links point to a second page that was never saved.
        cut_rel = by_name["Lead Time"]["id"]
        planted.append("MISSING_RELATIONSHIPS")
    else:
        cut_rel = None

    pages = [terms[i:i + PAGE_SIZE] for i in range(0, len(terms), PAGE_SIZE)]
    if plant:
        # DUPLICATE_ID: page 2 repeats the last term of page 1 (what overlapping pages look like).
        pages[1] = [pages[0][-1]] + pages[1]
        planted.append("DUPLICATE_ID")
    base = "https://api.purview-service.microsoft.com/datagovernance/catalog/terms?api-version=2026-03-20-preview"
    for n, page in enumerate(pages, 1):
        doc = {"value": page}
        # EXPORT_INCOMPLETE (planted): the last saved page still points to a page that was never saved.
        if n < len(pages) or plant:
            doc["nextLink"] = f"{base}&skip={PAGE_SIZE * n}&top={PAGE_SIZE}"
        (folder / "terms" / f"page-{n:03d}.json").write_text(json.dumps(doc, indent=1))
    if plant:
        planted.append("EXPORT_INCOMPLETE")
    nrel = 0
    rel_base = "https://api.purview-service.microsoft.com/datagovernance/catalog/terms/{}/relationships"
    for tid, r in rels.items():
        for e in ENTITY_TYPES:
            doc = {"value": r[e]}
            if tid == cut_rel and e == "DATAASSET":
                doc["nextLink"] = rel_base.format(tid) + "?api-version=2026-03-20-preview&entityType=DATAASSET&skip=50"
            (folder / "relationships" / f"{tid}.{e}.json").write_text(json.dumps(doc, indent=1))
            nrel += len(r[e])
    (folder / "domains" / "page-001.json").write_text(json.dumps(
        {"value": [{"id": d, "name": n, "status": "PUBLISHED", "type": "FunctionalUnit"} for d, n, _ in domains]}, indent=1))
    kept_terms = sum(len(p) for p in pages)
    print(f"Wrote a generated export to {folder}: {len(pages)} pages, {kept_terms} term records, "
          f"{nrel} relationships, {len(domains)} domains (seed 273)")
    if plant:
        assert len(planted) == len(CHECKS) and set(planted) == {c for c, _ in CHECKS}, planted
        print("Planted one error for each check: " + ", ".join(sorted(planted)))


def main():
    ap = argparse.ArgumentParser(description="Check a Purview Unified Catalog glossary export before you move it.")
    ap.add_argument("export", nargs="?", help="folder holding terms/, relationships/ and domains/")
    ap.add_argument("--out", default=str(pathlib.Path(__file__).resolve().parent),
                    help="folder for findings.csv and terms.csv (default: this script's folder)")
    ap.add_argument("--make-demo", metavar="FOLDER", help="write a generated export to FOLDER and stop")
    ap.add_argument("--plant-errors", action="store_true", help="with --make-demo: plant one error per check")
    ap.add_argument("--fail-on-findings", action="store_true", help="exit with code 1 when a check fails")
    ap.add_argument("--tag", default="", help="added to the CSV names: --tag planted writes findings-planted.csv")
    a = ap.parse_args()
    if a.make_demo:
        make_demo(a.make_demo, a.plant_errors)
        return 0
    if not a.export:
        ap.error("give the export folder, or --make-demo FOLDER")
    print(f"Python {sys.version.split()[0]} duckdb {duckdb.__version__}")
    blocked = check(a.export, a.out, "-" + a.tag if a.tag else "")
    return 1 if blocked and a.fail_on_findings else 0


if __name__ == "__main__":
    sys.exit(main())
