#!/usr/bin/env python3
"""Reconcile a Purview glossary export with what landed in Microsoft Fabric, and write the sheet each steward signs.

Reads three saved inputs:

    <purview>/terms/page-*.json, relationships/<termId>.<TYPE>.json, domains/page-*.json
        the Unified Catalog export (api-version 2026-03-20-preview), saved and checked as in
        https://www.datameaning.com/guides/check-purview-glossary-export-before-moving-it/
    <fabric>/tags/page-*.json      GET https://api.fabric.microsoft.com/v1/admin/tags     (AdminTagsInfo)
    <fabric>/items/page-*.json     GET https://api.fabric.microsoft.com/v1/admin/items    (Items, with each item's tags)
    <fabric>/domains/page-*.json   GET https://api.fabric.microsoft.com/v1/admin/domains  (DomainsResponse)
    asset map CSV                  purview_asset_id,fabric_item_id: your migration's own record of which Fabric
                                   item each Purview data asset became

Usage:
    python3 reconcile_catalog.py --purview <folder> --fabric <folder> --asset-map <csv> [--out <folder>]
    python3 reconcile_catalog.py --make-demo <folder> [--plant-errors]

Writes differences.csv and one signoff-<domain>.md per Purview governance domain to --out (default: this
script's folder). Exit code 2 when an input cannot be read; with --fail-on-differences, 1 when a check fails.
Guide: https://www.datameaning.com/guides/reconcile-migrated-catalog-against-purview-export/
Needs Python 3.9+ and duckdb (pip install duckdb).
"""
import argparse
import csv
import json
import os
import pathlib
import random
import re
import sys
import tempfile
import unicodedata
import uuid

import duckdb

# Settings.
MOVE_STATUSES = {"PUBLISHED"}   # terms expected to land in Fabric; drafts and expired terms are not moved
SAMPLE_SIZE = 10                # terms per domain the steward checks by hand
SAMPLE_SEED = 278               # fixed, so the same export always gives the same sample

CHECKS = [
    ("DOMAIN_NOT_MAPPED", "Every Purview domain has a Fabric domain of the same name"),
    ("TERM_WITHOUT_TAG", "Every published term has a Fabric tag of the same name"),
    ("TAG_RENAMED", "No tag differs from its term only in case or spacing"),
    ("ASSET_NOT_MAPPED", "Every data asset linked to a published term is in the asset map"),
    ("ITEM_MISSING", "Every Fabric item in the asset map is in the Fabric export"),
    ("LINK_NOT_TAGGED", "Every term-to-asset link is a tag on the mapped Fabric item"),
    ("EXTRA_TAG", "Every tag in a mapped Fabric domain comes from a Purview term"),
]


def stop(msg):
    print(f"CANNOT READ THE INPUT: {msg}")
    sys.exit(2)


def read_json(path):
    try:
        with open(path, encoding="utf-8-sig") as fh:
            return json.load(fh)
    except (OSError, ValueError) as e:
        stop(f"{path} is not readable JSON ({e.__class__.__name__}: {e})")


def pages(folder, sub, key):
    """Every page-NNN.json in folder/sub, in page order, as one list of the objects under key."""
    def no(p):
        m = re.fullmatch(r"page-(\d+)\.json", p.name)
        return int(m.group(1)) if m else -1
    files = sorted((p for p in (folder / sub).glob("page-*.json") if no(p) >= 0), key=no)
    if not files:
        stop(f"no {sub}/page-*.json files in {folder}" + (" (save the domain names too: the domains are matched by name)"
                                                          if sub == "domains" else ""))
    missing = sorted(set(range(1, no(files[-1]) + 1)) - {no(p) for p in files})
    if missing:
        stop(f"{sub}/page-{missing[0]:03d}.json was not saved: save every page first")
    out = []
    for p in files:
        doc = read_json(p)
        if not isinstance(doc, dict) or not isinstance(doc.get(key), list):
            stop(f"{p} has no \"{key}\" list")
        out += [x for x in doc[key] if isinstance(x, dict)]
    last = read_json(files[-1])
    if last.get("nextLink") or last.get("continuationToken") or last.get("continuationUri"):
        stop(f"{files[-1]} still points to a next page: save every page first")
    return out


def name_key(name):
    """A name as a reader sees it: Unicode-normalized, case-folded, every kind of space removed."""
    return "".join(ch for ch in unicodedata.normalize("NFKC", name or "").casefold() if not ch.isspace())


def bulk_insert(con, table, rows):
    if not rows:
        return
    with tempfile.TemporaryDirectory() as tmp:
        path = pathlib.Path(tmp) / f"{table}.csv"
        with open(path, "w", newline="", encoding="utf-8") as fh:
            csv.writer(fh).writerows(["" if v is None else v for v in r] for r in rows)
        con.execute(f"INSERT INTO {table} SELECT * FROM read_csv(?, header = false, all_varchar = true, "
                    f"quote = '\"', escape = '\"', nullstr = '')", [str(path)])


def safe(v):
    s = "" if v is None else str(v)
    return "'" + s if s[:1] in ("=", "+", "-", "@", "\t", "\r") else s


def slug(name):
    return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") or "domain"


def load(purview, fabric, asset_map):
    purview, fabric = pathlib.Path(purview), pathlib.Path(fabric)
    for f in (purview, fabric):
        if not f.is_dir():
            stop(f"{f} is not a folder")
    terms = pages(purview, "terms", "value")
    for t in terms:
        if not all(isinstance(t.get(k), str) and t[k].strip() for k in ("id", "name", "domain", "status")):
            stop(f"a Purview term has no id, name, domain or status (term id {t.get('id')!r})")
    if len({t["id"].lower() for t in terms}) != len(terms):
        stop("a Purview term id appears twice: run the export check first")
    links = []
    for p in sorted((purview / "relationships").glob("*.DATAASSET*.json")):
        m = re.match(r"^([0-9a-fA-F-]{36})\.DATAASSET(?:\.\d+)?\.json$", p.name)
        if not m:
            continue
        doc = read_json(p)
        for r in (doc.get("value") or []) if isinstance(doc, dict) else []:
            if isinstance(r, dict) and r.get("entityId"):
                links.append((m.group(1).lower(), str(r["entityId"]).lower()))
    pdomains = {str(d["id"]).lower(): d.get("name") or "" for d in pages(purview, "domains", "value") if d.get("id")}
    unknown = sorted({t["domain"].lower() for t in terms} - set(pdomains))
    if unknown:
        stop(f"terms belong to {len(unknown)} domain ids that are not in domains/ (for example {unknown[0]}): "
             "save the domain names again, so every term can be checked")
    tags = pages(fabric, "tags", "value")
    items = pages(fabric, "items", "itemEntities")
    fdomains = pages(fabric, "domains", "domains")
    amap = []
    try:
        with open(asset_map, newline="", encoding="utf-8-sig") as fh:
            rd = csv.DictReader(fh)
            need = {"purview_asset_id", "fabric_item_id"}
            if not need <= set(rd.fieldnames or []):
                stop(f"{asset_map} needs the columns purview_asset_id and fabric_item_id")
            for row in rd:
                a, i = (row["purview_asset_id"] or "").strip().lower(), (row["fabric_item_id"] or "").strip().lower()
                if a and i:
                    amap.append((a, i))
    except OSError as e:
        stop(f"{asset_map} is not readable ({e})")
    return terms, links, pdomains, tags, items, fdomains, amap


def reconcile(purview, fabric, asset_map, out, tag=""):
    terms, links, pdomains, tags, items, fdomains, amap = load(purview, fabric, asset_map)
    con = duckdb.connect()
    con.execute("CREATE TABLE term (id VARCHAR, name VARCHAR, name_key VARCHAR, domain VARCHAR, status VARCHAR, "
                "owners VARCHAR, has_description BOOLEAN, acronyms INTEGER, parent BOOLEAN)")
    bulk_insert(con, "term", [(t["id"].lower(), t["name"], name_key(t["name"]), t["domain"].lower(), t["status"].upper(),
                               ";".join(sorted(str(o.get("id")) for o in ((t.get("contacts") or {}).get("owner") or [])
                                               if isinstance(o, dict) and o.get("id"))),
                               isinstance(t.get("description"), str) and bool(t["description"].strip()),
                               len(t.get("acronyms") or []), bool(t.get("parentId"))) for t in terms])
    con.execute("CREATE TABLE link (term_id VARCHAR, asset_id VARCHAR)")
    bulk_insert(con, "link", sorted(set(links)))
    con.execute("CREATE TABLE pdom (id VARCHAR, name VARCHAR, name_key VARCHAR)")
    bulk_insert(con, "pdom", [(k, v, name_key(v)) for k, v in pdomains.items()])
    con.execute("CREATE TABLE fdom (id VARCHAR, name VARCHAR, name_key VARCHAR)")
    bulk_insert(con, "fdom", [(str(d.get("id", "")).lower(), d.get("displayName") or "", name_key(d.get("displayName")))
                              for d in fdomains])
    con.execute("CREATE TABLE ftag (id VARCHAR, name VARCHAR, name_key VARCHAR, scope VARCHAR, domain_id VARCHAR)")
    bulk_insert(con, "ftag", [(str(t.get("id", "")).lower(), t.get("displayName") or "", name_key(t.get("displayName")),
                               (t.get("scope") or {}).get("type") or "",
                               str((t.get("scope") or {}).get("domainId") or "").lower() or None) for t in tags])
    con.execute("CREATE TABLE item (id VARCHAR, name VARCHAR)")
    bulk_insert(con, "item", [(str(i.get("id", "")).lower(), i.get("name") or "") for i in items])
    con.execute("CREATE TABLE item_tag (item_id VARCHAR, tag_id VARCHAR)")
    bulk_insert(con, "item_tag", sorted({(str(i.get("id", "")).lower(), str(t.get("id", "")).lower())
                                          for i in items for t in (i.get("tags") or []) if isinstance(t, dict)}))
    con.execute("CREATE TABLE amap (asset_id VARCHAR, item_id VARCHAR)")
    bulk_insert(con, "amap", sorted(set(amap)))
    # The Fabric domain each Purview domain landed in: same name, ignoring case and spacing.
    # Exactly one match counts; none, or more than one, is a difference.
    con.execute("""CREATE TABLE dmap AS SELECT p.id AS pdom, p.name AS pname,
                   CASE WHEN count(f.id) = 1 THEN min(f.id) END AS fdom, CASE WHEN count(f.id) = 1 THEN min(f.name) END AS fname,
                   count(f.id) AS matches, string_agg(f.name, ', ' ORDER BY f.name) AS fnames
                   FROM pdom p LEFT JOIN fdom f ON f.name_key = p.name_key GROUP BY p.id, p.name""")
    # Every in-scope tag with a term's exact name: a tenant tag, or a tag in the term's Fabric domain.
    con.execute("""CREATE TABLE term_tag AS SELECT DISTINCT t.id AS term_id, g.id AS tag_id
                   FROM term t JOIN dmap d ON d.pdom = t.domain JOIN ftag g ON g.name = t.name
                   AND (g.scope = 'Tenant' OR (g.scope = 'Domain' AND g.domain_id = d.fdom))""")
    # Moved terms and the tag each should have: one in its Fabric domain, or a tenant tag.
    con.execute("""CREATE TABLE moved AS
        SELECT t.*, d.fdom, coalesce(d.pname, t.domain) AS dname,
               (SELECT min(g.id) FROM ftag g WHERE g.name = t.name AND (g.scope = 'Tenant'
                  OR (g.scope = 'Domain' AND g.domain_id = d.fdom))) AS tag_id,
               (SELECT min(g.name) FROM ftag g WHERE g.name_key = t.name_key AND g.name <> t.name AND (g.scope = 'Tenant'
                  OR (g.scope = 'Domain' AND g.domain_id = d.fdom))) AS near_name
        FROM term t LEFT JOIN dmap d ON d.pdom = t.domain
        WHERE t.status IN (SELECT unnest(?))""", [sorted(MOVE_STATUSES)])

    print(f"Purview: {len(terms)} terms ({con.execute('SELECT count(*) FROM moved').fetchone()[0]} published, "
          f"so expected in Fabric), {len(links)} links to data assets, {len(pdomains)} domains")
    print(f"Fabric: {len(tags)} tags, {len(items)} items, {len(fdomains)} domains; asset map: {len(amap)} rows")

    diffs = []

    def add(code, rows):
        diffs.extend((code, *r) for r in rows)

    add("DOMAIN_NOT_MAPPED", con.execute("""SELECT pname, '', CASE WHEN matches = 0 THEN 'no Fabric domain named "' || pname || '"'
        ELSE 'the name matches more than one Fabric domain: ' || fnames END, pdom
        FROM dmap WHERE fdom IS NULL ORDER BY pname""").fetchall())
    add("TERM_WITHOUT_TAG", con.execute("""SELECT dname, name, 'no tag named "' || name || '" in the domain or the tenant', domain
        FROM moved WHERE tag_id IS NULL AND near_name IS NULL AND fdom IS NOT NULL ORDER BY dname, name""").fetchall())
    add("TAG_RENAMED", con.execute("""SELECT dname, name, 'the tag is named "' || near_name || '"', domain
        FROM moved WHERE tag_id IS NULL AND near_name IS NOT NULL ORDER BY dname, name""").fetchall())
    add("ASSET_NOT_MAPPED", con.execute("""SELECT m.dname, m.name, 'data asset ' || l.asset_id || ' has no row in the asset map',
        m.domain FROM moved m JOIN link l ON l.term_id = m.id WHERE l.asset_id NOT IN (SELECT asset_id FROM amap)
        ORDER BY m.dname, m.name, l.asset_id""").fetchall())
    add("ITEM_MISSING", con.execute("""SELECT dname, name, detail, pdom FROM (
          SELECT m.dname, m.name, 'Fabric item ' || a.item_id || ' (from data asset ' || a.asset_id ||
                 ') is not in the Fabric export' AS detail, m.domain AS pdom
          FROM moved m JOIN link l ON l.term_id = m.id JOIN amap a ON a.asset_id = l.asset_id
          WHERE a.item_id NOT IN (SELECT id FROM item)
          UNION ALL
          SELECT '', '', 'asset map row ' || asset_id || ' -> ' || item_id || ': no Fabric item ' || item_id || ' in the export', ''
          FROM amap WHERE item_id NOT IN (SELECT id FROM item)
            AND asset_id NOT IN (SELECT l.asset_id FROM link l JOIN moved m ON m.id = l.term_id))
        ORDER BY dname, name, detail""").fetchall())
    add("LINK_NOT_TAGGED", con.execute("""SELECT m.dname, m.name, 'Fabric item ' || coalesce(nullif(i.name, ''), a.item_id)
        || ' does not carry the tag', m.domain
        FROM moved m JOIN link l ON l.term_id = m.id JOIN amap a ON a.asset_id = l.asset_id
        LEFT JOIN item i ON i.id = a.item_id
        WHERE m.tag_id IS NOT NULL AND i.id IS NOT NULL
          AND NOT EXISTS (SELECT 1 FROM item_tag x JOIN term_tag tt ON tt.tag_id = x.tag_id
                          WHERE x.item_id = a.item_id AND tt.term_id = m.id)
        ORDER BY m.dname, m.name, i.name""").fetchall())
    add("EXTRA_TAG", con.execute("""SELECT d.pname, '', 'tag "' || g.name || '" has no published term of that name', d.pdom
        FROM ftag g JOIN dmap d ON d.fdom = g.domain_id
        WHERE g.scope = 'Domain' AND g.name_key NOT IN (SELECT name_key FROM moved m WHERE m.fdom = g.domain_id)
        ORDER BY d.pname, g.name""").fetchall())

    print()
    print("Checks")
    for code, text in CHECKS:
        n = sum(1 for d in diffs if d[0] == code)
        print(f"  {'PASS' if n == 0 else 'FAIL'}  {text}" + (f"  ({n})" if n else ""))
    if diffs:
        print()
        print("Differences")
        for code, dom, name, detail, _ in diffs:
            print(f"  {code:<18} " + (f"[{dom}] " if dom else "") + (f"{name}: " if name else "") + detail)

    out = pathlib.Path(out)
    out.mkdir(parents=True, exist_ok=True)
    suffix = f"-{tag}" if tag else ""
    with open(out / f"differences{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["check", "domain", "term", "detail"])
        w.writerows([[safe(x) for x in d[:4]] for d in diffs])

    rows = con.execute("""SELECT m.dname, m.name, m.status, coalesce(g.name, ''),
          (SELECT count(*) FROM link l WHERE l.term_id = m.id),
          (SELECT string_agg(coalesce(nullif(i.name, ''), a.item_id), '; ' ORDER BY i.name) FROM link l
             JOIN amap a ON a.asset_id = l.asset_id LEFT JOIN item i ON i.id = a.item_id WHERE l.term_id = m.id),
          (SELECT string_agg(i.name, '; ' ORDER BY i.name) FROM link l JOIN amap a ON a.asset_id = l.asset_id
             JOIN item i ON i.id = a.item_id WHERE l.term_id = m.id AND EXISTS (SELECT 1 FROM item_tag x
               JOIN term_tag tt ON tt.tag_id = x.tag_id WHERE x.item_id = i.id AND tt.term_id = m.id))
        FROM moved m LEFT JOIN ftag g ON g.id = m.tag_id ORDER BY m.dname, m.name""").fetchall()
    with open(out / f"moved-terms{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["domain", "term", "status", "fabric_tag", "asset_links", "items_expected", "items_tagged"])
        w.writerows([[safe(x) for x in r] for r in rows])

    # One sign-off sheet per Purview domain.
    sheets, used = [], set()
    outside = [d for d in diffs if not d[4]]
    for pdom, pname, fdom, fname in con.execute("SELECT pdom, pname, fdom, fname FROM dmap ORDER BY pname").fetchall():
        c = con.execute("""SELECT count(*), count(tag_id),
              (SELECT count(*) FROM link l JOIN moved m ON m.id = l.term_id WHERE m.domain = ?),
              (SELECT count(*) FROM link l JOIN moved m ON m.id = l.term_id JOIN amap a ON a.asset_id = l.asset_id
                 WHERE m.domain = ? AND EXISTS (SELECT 1 FROM item_tag x JOIN term_tag tt ON tt.tag_id = x.tag_id
                   WHERE x.item_id = a.item_id AND tt.term_id = m.id)),
              count(*) FILTER (WHERE has_description), count(*) FILTER (WHERE acronyms > 0),
              count(*) FILTER (WHERE parent)
            FROM moved WHERE domain = ?""", [pdom, pdom, pdom]).fetchone()
        held = con.execute("SELECT count(*) FROM term WHERE domain = ? AND status NOT IN (SELECT unnest(?))",
                           [pdom, sorted(MOVE_STATUSES)]).fetchone()[0]
        owners = con.execute("""SELECT o, count(*) FROM (SELECT unnest(string_split(owners, ';')) AS o FROM moved
            WHERE domain = ? AND owners <> '') GROUP BY o ORDER BY count(*) DESC, o""", [pdom]).fetchall()
        dd = [d for d in diffs if d[4] == pdom]
        names = [r[0] for r in con.execute("SELECT name FROM moved WHERE domain = ? ORDER BY name", [pdom]).fetchall()]
        sample = sorted(random.Random(f"{SAMPLE_SEED}:{pname}").sample(names, min(SAMPLE_SIZE, len(names))))
        lines = [f"# Sign-off: {pname} domain", "",
                 f"Purview governance domain {pname} ({pdom}) moved to Fabric domain "
                 + (f"{fname} ({fdom})." if fdom else "(none found)."), "",
                 "| What | Purview | Fabric |", "|---|---|---|",
                 f"| Published terms / tags of the same name | {c[0]} | {c[1]} |",
                 f"| Term-to-asset links / tags on the mapped items | {c[2]} | {c[3]} |", "",
                 f"Differences found by the check: {len(dd)}" + ("" if dd else " (none)") + ".", ""]
        lines += [f"- {code}: " + (f"{name}: " if name else "") + detail for code, _, name, detail, _ in dd]
        if outside:
            lines += ["", f"Differences outside any one domain, which may still affect this one: {len(outside)}."]
            lines += [f"- {code}: {detail}" for code, _, _, detail, _ in outside]
        lines += ["", "Not carried by a Fabric tag (keep in the archive or another home):",
                  f"- {c[4]} term descriptions, {c[5]} terms with acronyms, {c[6]} parent links",
                  f"- draft or expired terms, not moved: {held}", "",
                  f"Terms to check by hand ({len(sample)}, chosen with seed {SAMPLE_SEED}): open each tag in Fabric and "
                  f"confirm the tag exists and sits on the items listed for the term in moved-terms{suffix}.csv.", ""]
        lines += [f"- [ ] {n}" for n in sample]
        lines += ["", "Owners of the moved terms (Microsoft Entra object ids, number of terms):"]
        lines += [f"- {o}: {n}" for o, n in owners] or ["- none recorded"]
        lines += ["", "Signed means: the counts above were compared, every difference is fixed or accepted in writing, "
                  "and the terms above were checked by hand.", "",
                  "Steward: ______________________  Date: ____________  Signature: ______________________", ""]
        name = slug(pname)
        if name == "domain" or name in used:
            name = f"{name}-{pdom[:8]}"   # names that reduce to the same file name keep a sheet each
        used.add(name)
        path = out / f"signoff-{name}{suffix}.md"
        path.write_text("\n".join(lines), encoding="utf-8")
        sheets.append(path.name)

    try:
        where = os.path.relpath(out)
    except ValueError:
        where = str(out)
    print()
    print(f"Wrote differences{suffix}.csv ({len(diffs)} rows), moved-terms{suffix}.csv ({len(rows)} rows) and "
          f"{len(sheets)} sign-off sheets to {where}: "
          + ", ".join(sheets))
    if diffs:
        print(f"RESULT: {len(diffs)} differences. Fix them in Fabric, or accept each one in writing, before a steward signs.")
    else:
        print("RESULT: no differences. Each domain's steward can check the sample and sign.")
    return len(diffs)


# ---------------------------------------------------------------------------------------------
# Demo: a generated Purview export and a generated Fabric landing in the APIs' documented shapes.
DOMAINS = {"Finance": ["Net Revenue", "Gross Margin", "Operating Expense", "Accounts Receivable", "Cost Center",
                       "Fiscal Period", "Accrual", "Budget Variance", "Deferred Revenue", "Invoice", "Days Sales Outstanding",
                       "General Ledger Account", "Write-off", "Cash Conversion Cycle", "Capital Expenditure",
                       "Intercompany Transfer", "Chart of Accounts", "Credit Memo", "Exchange Rate", "Revenue Recognition Date"],
           "Sales": ["Customer", "Active Customer", "Opportunity", "Pipeline Value", "Win Rate", "Sales Territory",
                     "Quote", "Order", "Booking", "Renewal", "Churned Customer", "Account Owner", "Discount Rate",
                     "Order Line", "Annual Recurring Revenue", "Customer Segment", "Lead", "Qualified Lead", "Price List",
                     "Sales Channel"],
           "Supply Chain": ["Supplier", "Purchase Order", "Lead Time", "Safety Stock", "Warehouse", "Fill Rate",
                            "Backorder", "Shipment", "Carrier", "Demand Forecast", "Inventory On Hand", "Stock Keeping Unit",
                            "Bill of Materials", "Reorder Point", "On-Time Delivery", "Goods Receipt", "Inventory Turnover",
                            "Product Category", "Unit of Measure", "Return Authorization"]}


def make_demo(folder, plant):
    rng = random.Random(278)
    new_id = lambda: str(uuid.UUID(int=rng.getrandbits(128), version=4))
    folder = pathlib.Path(folder)
    marker = folder / ".made-by-make-demo"
    if folder.exists() and any(folder.rglob("*.json")) and not marker.exists():
        sys.exit(f"{folder} already holds JSON files that --make-demo did not write; choose a new folder.")
    for sub in ("purview/terms", "purview/relationships", "purview/domains", "fabric/tags", "fabric/items",
                "fabric/domains"):
        (folder / sub).mkdir(parents=True, exist_ok=True)
        for old in (folder / sub).glob("*.json"):
            old.unlink()
    marker.write_text("Written by reconcile_catalog.py --make-demo: generated records, not an export.\n")
    people = [new_id() for _ in range(6)]
    terms, rels, pdoms, fdoms, tags, items, amap = [], {}, [], [], [], {}, []
    for dname, names in DOMAINS.items():
        pid, fid = new_id(), new_id()
        pdoms.append({"id": pid, "name": dname, "status": "PUBLISHED", "type": "FunctionalUnit"})
        fdoms.append({"id": fid, "displayName": dname, "description": f"Invented {dname} domain for this demo."})
        assets = [new_id() for _ in range(8)]
        for a in assets:
            iid = new_id()
            items[iid] = {"id": iid, "name": f"{dname.lower().replace(' ', '_')}_table_{len(items) + 1}", "type": "Lakehouse",
                          "workspaceId": fid[:-4] + "0000", "state": "Active", "tags": []}
            amap.append((a, iid))
        for i, name in enumerate(names):
            tid = new_id()
            status = "DRAFT" if i in (9, 17) else ("EXPIRED" if i == 18 else "PUBLISHED")
            terms.append({"id": tid, "name": name, "domain": pid, "status": status,
                          "description": f"Invented definition of {name}.",
                          "contacts": {"owner": [{"id": people[i % 3 + (3 if dname == 'Sales' else 0)],
                                                  "description": "Data steward"}]},
                          "acronyms": ["".join(w[0] for w in name.split())] if i % 4 == 0 else []})
            if i in (2, 3):
                terms[-1]["parentId"] = terms[-(i - 1) - 1]["id"] if i == 3 else terms[-3]["id"]
            linked = rng.sample(assets, 2)
            rels[tid] = linked
            if status == "PUBLISHED":
                gid = new_id()
                tags.append({"id": gid, "displayName": name, "scope": {"type": "Domain", "domainId": fid}})
                for a in linked:
                    items[dict(amap)[a]]["tags"].append({"id": gid, "displayName": name})
    planted = []
    if plant:
        by = {t["name"]: t for t in terms}
        tag_of = {g["displayName"]: g for g in tags}
        # TERM_WITHOUT_TAG: the Win Rate tag was never created (and so never applied).
        g = tag_of["Win Rate"]
        tags.remove(g)
        for it in items.values():
            it["tags"] = [x for x in it["tags"] if x["id"] != g["id"]]
        planted.append("TERM_WITHOUT_TAG")
        # TAG_RENAMED: the Lead Time tag was created as "lead time".
        tag_of["Lead Time"]["displayName"] = "lead time"
        for it in items.values():
            for x in it["tags"]:
                if x["id"] == tag_of["Lead Time"]["id"]:
                    x["displayName"] = "lead time"
        planted.append("TAG_RENAMED")
        # LINK_NOT_TAGGED: one item lost the Gross Margin tag.
        gm = tag_of["Gross Margin"]["id"]
        it = next(i for i in items.values() if any(x["id"] == gm for x in i["tags"]))
        it["tags"] = [x for x in it["tags"] if x["id"] != gm]
        planted.append("LINK_NOT_TAGGED")
        # ASSET_NOT_MAPPED: Quote is linked to a data asset the asset map does not hold.
        rels[by["Quote"]["id"]].append(new_id())
        planted.append("ASSET_NOT_MAPPED")
        # ITEM_MISSING: one asset map row points to an item that is not in the Fabric export.
        a0, i0 = amap[0]
        amap.append((new_id(), new_id()))
        planted.append("ITEM_MISSING")
        # EXTRA_TAG: a test tag left in the Finance domain.
        tags.append({"id": new_id(), "displayName": "Test tag", "scope": {"type": "Domain", "domainId": fdoms[0]["id"]}})
        planted.append("EXTRA_TAG")
        # DOMAIN_NOT_MAPPED: a fourth Purview domain with one draft term and no Fabric domain.
        hr = new_id()
        pdoms.append({"id": hr, "name": "Human Resources", "status": "PUBLISHED", "type": "FunctionalUnit"})
        planted.append("DOMAIN_NOT_MAPPED")
    w = lambda p, doc: (folder / p).write_text(json.dumps(doc, indent=1))
    w("purview/terms/page-001.json", {"value": terms[:30], "nextLink": "https://api.purview-service.microsoft.com/datagovernance/catalog/terms?api-version=2026-03-20-preview&skip=30"})
    w("purview/terms/page-002.json", {"value": terms[30:]})
    for t in terms:
        for e in ("TERM", "DATAASSET", "DATACOLUMN", "DATAPRODUCT", "CRITICALDATAELEMENT"):
            v = [{"entityId": a, "relationshipType": "Related"} for a in rels[t["id"]]] if e == "DATAASSET" else []
            w(f"purview/relationships/{t['id']}.{e}.json", {"value": v})
    w("purview/domains/page-001.json", {"value": pdoms})
    w("fabric/tags/page-001.json", {"value": tags})
    w("fabric/items/page-001.json", {"itemEntities": list(items.values())})
    w("fabric/domains/page-001.json", {"domains": fdoms})
    with open(folder / "asset-map.csv", "w", newline="", encoding="utf-8") as fh:
        cw = csv.writer(fh)
        cw.writerow(["purview_asset_id", "fabric_item_id"])
        cw.writerows(amap)
    print(f"Wrote a generated Purview export and Fabric landing to {folder}: {len(terms)} terms in {len(pdoms)} "
          f"Purview domains, {len(tags)} tags, {len(items)} items, {len(amap)} asset map rows (seed 278)")
    if plant:
        assert set(planted) == {c for c, _ in CHECKS}, planted
        print("Planted one difference for each check: " + ", ".join(sorted(planted)))


def main():
    ap = argparse.ArgumentParser(description="Reconcile a Purview glossary export with what landed in Fabric.")
    ap.add_argument("--purview", help="the saved Unified Catalog export folder")
    ap.add_argument("--fabric", help="the saved Fabric admin export folder (tags/, items/, domains/)")
    ap.add_argument("--asset-map", help="CSV with purview_asset_id,fabric_item_id")
    ap.add_argument("--out", default=str(pathlib.Path(__file__).resolve().parent),
                    help="folder for differences.csv and the sign-off sheets (default: this script's folder)")
    ap.add_argument("--make-demo", metavar="FOLDER", help="write a generated export and landing to FOLDER and stop")
    ap.add_argument("--plant-errors", action="store_true", help="with --make-demo: plant one difference per check")
    ap.add_argument("--fail-on-differences", action="store_true", help="exit with code 1 when a check fails")
    ap.add_argument("--tag", default="", help="added to the output names: --tag planted writes differences-planted.csv")
    a = ap.parse_args()
    if a.make_demo:
        make_demo(a.make_demo, a.plant_errors)
        return 0
    if not (a.purview and a.fabric and a.asset_map):
        ap.error("give --purview, --fabric and --asset-map, or --make-demo FOLDER")
    print(f"Python {sys.version.split()[0]} duckdb {duckdb.__version__}")
    n = reconcile(a.purview, a.fabric, a.asset_map, a.out, a.tag)
    return 1 if n and a.fail_on_differences else 0


if __name__ == "__main__":
    sys.exit(main())
