#!/usr/bin/env python3
"""Inventory a Purview Unified Catalog export and give every object a first decision: move, bridge, redesign or retire.

Reads the Unified Catalog API's own answers (api-version 2026-03-20-preview), one folder per object type and one
file per page, in page order:

    <export>/businessdomains/page-001.json     GET /businessdomains
    <export>/terms/page-001.json               GET /terms
    <export>/dataProducts/page-001.json        GET /dataProducts
    <export>/criticalDataElements/page-001.json GET /criticalDataElements
    <export>/objectives/page-001.json          GET /objectives
    <export>/dataAssets/page-001.json          GET /dataAssets
    <export>/policies/page-001.json            GET /policies (answers "values", not "value")

Every object gets the decision the DECISIONS table below gives its type: a first cut taken from Microsoft's
transition guide, for people to confirm or change. Writes objects.csv (one row per object) and inventory.csv
(counts by domain, type and decision) to --out (default: this script's folder).

Usage:
    python3 inventory_catalog.py <export folder> [--out <folder>]
    python3 inventory_catalog.py --make-demo <folder> [--plant-errors]
Guide: https://www.datameaning.com/guides/inventory-purview-governance-catalog-and-decide-move-or-retire/
Needs Python 3.9+ and duckdb (pip install duckdb).
"""
import argparse
import csv
import json
import os
import pathlib
import random
import re
import sys
import tempfile
import uuid

import duckdb

# The first decision for each object type, from the row of Microsoft's transition guide the type belongs to
# ("From Purview Data Governance to the Fabric OneLake catalog", read 2026-11-21). Written out so you can change it.
DECISIONS = {
    "businessdomains": ("move", "Collections, domains, ownership -> Fabric domains and subdomains"),
    "terms": ("redesign", "Glossary -> tags, descriptions, endorsement; the guide: do not claim full glossary parity"),
    "criticalDataElements": ("redesign", "Critical data elements -> tags, descriptions, endorsement; no CDE parity"),
    "dataProducts": ("redesign", "Data products -> domains, endorsement, sharing; define the product contract outside the catalog where needed"),
    "objectives": ("retire", "Objectives and key results: not in the transition guide's table; keep them in an archive of the export"),
    "policies": ("bridge", "Access policy -> Fabric Policies (preview) and permissions; the list also holds governance-domain role assignments, which map to domain admins and owners; do not assume universal coverage"),
    "dataAssets:AzureSqlTable": ("bridge", "Catalog search -> OneLake catalog for Fabric items; keep discovery for sources outside Fabric, or mirror the source"),
    "dataAssets:ADLSGen2Path": ("bridge", "Catalog search -> OneLake catalog for Fabric items; keep discovery for sources outside Fabric, or add a shortcut"),
    "dataAssets:General": ("bridge", "Catalog search -> OneLake catalog for Fabric items; keep discovery for sources outside Fabric"),
}
LIST_KEY = {"policies": "values"}
TYPES = ["businessdomains", "terms", "dataProducts", "criticalDataElements", "objectives", "dataAssets", "policies"]
CHECKS = [
    ("PAGES_MISSING", "Every page of every object type was saved"),
    ("DUPLICATE_ID", "No object id appears twice within its type"),
    ("UNKNOWN_DOMAIN", "Every object's domain is a domain in the export"),
    ("NO_DECISION", "Every object type in the export has a decision"),
]


def stop(msg):
    print(f"CANNOT READ THE EXPORT: {msg}")
    sys.exit(2)


def read_json(path):
    try:
        with open(path, encoding="utf-8-sig") as fh:
            return json.load(fh)
    except (OSError, ValueError) as e:
        stop(f"{path} is not readable JSON ({e.__class__.__name__}: {e})")


def page_no(p):
    m = re.fullmatch(r"page-(\d+)\.json", p.name)
    return int(m.group(1)) if m else -1


def load(folder):
    folder = pathlib.Path(folder)
    if not folder.is_dir():
        stop(f"{folder} is not a folder")
    objects, findings, found_types = [], [], []
    for sub in sorted(p for p in folder.iterdir() if p.is_dir()):
        pages = sorted((p for p in sub.glob("page-*.json") if page_no(p) >= 0), key=page_no)
        if not pages:
            continue
        found_types.append(sub.name)
        key = LIST_KEY.get(sub.name, "value")
        nums = [page_no(p) for p in pages]
        for n in range(1, nums[-1] + 1):
            if n not in nums:
                findings.append(("PAGES_MISSING", sub.name, "", "", f"{sub.name}/page-{n:03d}.json was not saved"))
        last = None
        for p in pages:
            doc = read_json(p)
            if not isinstance(doc, dict) or not isinstance(doc.get(key), list):
                stop(f"{p} has no \"{key}\" list")
            for o in doc[key]:
                if not isinstance(o, dict) or not o.get("id"):
                    stop(f"{p} holds an object with no id")
                name = o.get("name") or o.get("definition") or ""
                atype = o.get("type") if sub.name == "dataAssets" else ""
                objects.append((sub.name, str(o["id"]).lower(), str(name), str(o.get("domain") or "").lower(),
                                str(atype or ""), page_no(p)))
            last = doc
        if last.get("nextLink") or last.get("skipToken"):
            findings.append(("PAGES_MISSING", sub.name, "", "", f"{pages[-1].name} still points to a next page"))
    if "businessdomains" not in found_types:
        stop("no businessdomains/page-*.json: save the domains too, every object is counted by domain")
    for t in TYPES:
        if t not in found_types:   # a type with no objects still answers a page with an empty list: save it
            findings.append(("PAGES_MISSING", t, "", "", f"{t}/page-001.json was not saved"))
    return objects, findings, found_types


def bulk_insert(con, table, rows):
    if not rows:
        return
    with tempfile.TemporaryDirectory() as tmp:
        path = pathlib.Path(tmp) / f"{table}.csv"
        with open(path, "w", newline="", encoding="utf-8") as fh:
            csv.writer(fh).writerows(["" if v is None else v for v in r] for r in rows)
        con.execute(f"INSERT INTO {table} SELECT * FROM read_csv(?, header = false, all_varchar = true, "
                    f"quote = '\"', escape = '\"', nullstr = '')", [str(path)])


def safe(v):
    s = "" if v is None else str(v)
    return "'" + s if s[:1] in ("=", "+", "-", "@", "\t", "\r") else s


def inventory(folder, out, tag=""):
    objects, findings, found_types = load(folder)
    con = duckdb.connect()
    con.execute("CREATE TABLE obj (otype VARCHAR, id VARCHAR, name VARCHAR, domain VARCHAR, atype VARCHAR, page VARCHAR)")
    bulk_insert(con, "obj", objects)
    con.execute("CREATE TABLE decision (dkey VARCHAR, decision VARCHAR, reason VARCHAR)")
    bulk_insert(con, "decision", [(k, d, r) for k, (d, r) in DECISIONS.items()])
    con.execute("""CREATE TABLE keyed AS SELECT o.*, CASE WHEN o.otype = 'dataAssets' THEN 'dataAssets:' || coalesce(o.atype, '')
                   ELSE o.otype END AS dkey FROM obj o""")   # a data asset with no type gets 'dataAssets:', so NO_DECISION reports it
    assert con.execute("SELECT count(*) FROM obj").fetchone()[0] == len(objects), "rows read and rows loaded differ"
    print(f"Export: {folder}")
    print(f"Read {len(objects)} objects of {len(found_types)} types: " + ", ".join(found_types))

    findings += [("DUPLICATE_ID", t, i, n, f"id on {c} rows") for t, i, n, c in con.execute(
        "SELECT otype, id, min(name), count(*) FROM obj GROUP BY otype, id HAVING count(*) > 1 ORDER BY otype, id").fetchall()]
    findings += [("UNKNOWN_DOMAIN", t, i, n, f"domain {d} is not in businessdomains") for t, i, n, d in con.execute(
        """SELECT DISTINCT otype, id, name, domain FROM obj WHERE otype <> 'businessdomains' AND domain <> ''
           AND domain NOT IN (SELECT id FROM obj WHERE otype = 'businessdomains') ORDER BY otype, name, id""").fetchall()]
    findings += [("NO_DECISION", k, "", "", f"objects of type {k} with no row in DECISIONS: {n}") for k, n in con.execute(
        """SELECT dkey, count(*) FROM keyed
           GROUP BY dkey HAVING dkey NOT IN (SELECT dkey FROM decision) ORDER BY dkey""").fetchall()]

    print()
    print("Objects by type and first decision")
    for dkey, n, dec in con.execute("""SELECT k.dkey, count(*), coalesce(d.decision, '(none)') FROM keyed k
            LEFT JOIN decision d ON d.dkey = k.dkey GROUP BY k.dkey, d.decision ORDER BY k.dkey""").fetchall():
        print(f"  {dkey:<26} {n:>5}  {dec}")
    print("Objects by domain and decision")
    rows = con.execute("""SELECT coalesce(nullif(dn.name, ''), k.domain, '') AS dom,
            count(*) FILTER (WHERE d.decision = 'move'), count(*) FILTER (WHERE d.decision = 'bridge'),
            count(*) FILTER (WHERE d.decision = 'redesign'), count(*) FILTER (WHERE d.decision = 'retire'), count(*)
        FROM keyed k LEFT JOIN decision d ON d.dkey = k.dkey
        LEFT JOIN (SELECT id, min(name) AS name FROM obj WHERE otype = 'businessdomains' GROUP BY id) dn
          ON dn.id = CASE WHEN k.otype = 'businessdomains' THEN k.id ELSE k.domain END
        GROUP BY dom ORDER BY dom""").fetchall()
    print(f"  {'Domain':<24} {'move':>5} {'bridge':>7} {'redesign':>9} {'retire':>7} {'total':>6}")
    for dom, mv, br, rd, rt, tot in rows:
        print(f"  {(dom or '(no domain field)'):<24} {mv:>5} {br:>7} {rd:>9} {rt:>7} {tot:>6}")

    print()
    print("Checks")
    for code, text in CHECKS:
        n = sum(1 for f in findings if f[0] == code)
        print(f"  {'PASS' if n == 0 else 'FAIL'}  {text}" + (f"  ({n})" if n else ""))
    if findings:
        print()
        print("Findings")
        for code, t, i, n, d in findings:
            print(f"  {code:<15} {t}: " + (f"{n} " if n else "") + d)

    out = pathlib.Path(out)
    out.mkdir(parents=True, exist_ok=True)
    suffix = f"-{tag}" if tag else ""
    objs = con.execute("""SELECT k.otype, k.id, k.name, coalesce(nullif(dn.name, ''), k.domain, ''), k.atype,
            coalesce(d.decision, ''), coalesce(d.reason, 'no row in DECISIONS')
        FROM keyed k LEFT JOIN decision d ON d.dkey = k.dkey
        LEFT JOIN (SELECT id, min(name) AS name FROM obj WHERE otype = 'businessdomains' GROUP BY id) dn
          ON dn.id = CASE WHEN k.otype = 'businessdomains' THEN k.id ELSE k.domain END
        ORDER BY k.otype, k.name, k.id""").fetchall()
    with open(out / f"objects{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["object_type", "id", "name", "domain", "asset_type", "first_decision", "reason"])
        w.writerows([[safe(x) for x in r] for r in objs])
    with open(out / f"inventory{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["domain", "move", "bridge", "redesign", "retire", "total"])
        w.writerows([[safe(x) for x in r] for r in rows])
    try:
        where = os.path.relpath(out)
    except ValueError:
        where = str(out)
    print()
    print(f"Wrote objects{suffix}.csv ({len(objs)} rows) and inventory{suffix}.csv ({len(rows)} rows) to {where}")
    print(f"RESULT: {len(findings)} findings." + (" Fix the export or DECISIONS, then re-run." if findings else
                                                 " Every object has a first decision for people to confirm."))
    return len(findings)


# ---------------------------------------------------------------------------------------------
def make_demo(folder, plant):
    rng = random.Random(274)
    new_id = lambda: str(uuid.UUID(int=rng.getrandbits(128), version=4))
    folder = pathlib.Path(folder)
    marker = folder / ".made-by-make-demo"
    if folder.exists() and any(folder.rglob("*.json")) and not marker.exists():
        sys.exit(f"{folder} already holds JSON files that --make-demo did not write; choose a new folder.")
    folder.mkdir(parents=True, exist_ok=True)
    for old in folder.rglob("*.json"):
        old.unlink()
    marker.write_text("Written by inventory_catalog.py --make-demo: generated records, not an export.\n")
    doms = [(new_id(), n) for n in ("Finance", "Sales", "Supply Chain")]
    data = {t: [] for t in TYPES}
    data["businessdomains"] = [{"id": d, "name": n, "status": "PUBLISHED", "type": "FunctionalUnit"} for d, n in doms]
    counts = {"terms": 20, "dataProducts": 4, "criticalDataElements": 6, "objectives": 2}
    for d, n in doms:
        for t, k in counts.items():
            for i in range(k):
                o = {"id": new_id(), "domain": d, "status": "PUBLISHED"}
                if t == "objectives":
                    o["definition"] = f"Invented {n} objective {i + 1}"
                else:
                    o["name"] = f"{n} {t[:-1] if t.endswith('s') else t} {i + 1}"
                data[t].append(o)
    for i, atype in enumerate(["AzureSqlTable"] * 18 + ["ADLSGen2Path"] * 9 + ["General"] * 3):
        data["dataAssets"].append({"id": new_id(), "name": f"asset_{i + 1:02d}", "type": atype,
                                   "source": {"type": atype, "fqn": f"invented://asset_{i + 1:02d}"}})
    data["policies"] = [{"id": new_id(), "name": f"Invented access policy {i + 1}"} for i in range(3)]
    planted = []
    extra = {}
    if plant:
        data["terms"].append(dict(data["terms"][4]))                      # DUPLICATE_ID
        planted.append("DUPLICATE_ID")
        data["dataProducts"][0]["domain"] = new_id()                      # UNKNOWN_DOMAIN
        planted.append("UNKNOWN_DOMAIN")
        extra["customMetadata"] = [{"id": new_id(), "name": "Invented attribute group"}]   # NO_DECISION
        planted.append("NO_DECISION")
        planted.append("PAGES_MISSING")                                   # terms page 2 never saved
    for t, objs in list(data.items()) + list(extra.items()):
        (folder / t).mkdir(parents=True, exist_ok=True)
        size = 25
        pages = [objs[i:i + size] for i in range(0, len(objs), size)] or [[]]
        key = LIST_KEY.get(t, "value")
        for n, page in enumerate(pages, 1):
            if plant and t == "terms" and n == 2:
                continue
            doc = {key: page}
            if n < len(pages):
                doc["nextLink"] = f"https://api.purview-service.microsoft.com/datagovernance/catalog/{t}?api-version=2026-03-20-preview&skip={size * n}"
            (folder / t / f"page-{n:03d}.json").write_text(json.dumps(doc, indent=1))
    total = sum(len(v) for v in data.values()) + sum(len(v) for v in extra.values())
    print(f"Wrote a generated export to {folder}: {total} objects of {len(data) + len(extra)} types (seed 274)")
    if plant:
        assert set(planted) == {c for c, _ in CHECKS}, planted
        print("Planted one error for each check: " + ", ".join(sorted(planted)))


def main():
    ap = argparse.ArgumentParser(description="Inventory a Purview Unified Catalog export with a first decision per object.")
    ap.add_argument("export", nargs="?")
    ap.add_argument("--out", default=str(pathlib.Path(__file__).resolve().parent))
    ap.add_argument("--make-demo", metavar="FOLDER")
    ap.add_argument("--plant-errors", action="store_true")
    ap.add_argument("--fail-on-findings", action="store_true")
    ap.add_argument("--tag", default="")
    a = ap.parse_args()
    if a.make_demo:
        make_demo(a.make_demo, a.plant_errors)
        return 0
    if not a.export:
        ap.error("give the export folder, or --make-demo FOLDER")
    print(f"Python {sys.version.split()[0]} duckdb {duckdb.__version__}")
    n = inventory(a.export, a.out, a.tag)
    return 1 if n and a.fail_on_findings else 0


if __name__ == "__main__":
    sys.exit(main())
