#!/usr/bin/env python3
"""Check a saved Purview lineage graph before the switch-off, and join it into one edge list.

Reads the Data Map API's own answers (api-version 2023-09-01), saved one file per call:

    <export>/search/page-001.json      POST /datamap/api/search/query (the assets to save lineage for)
    <export>/lineage/<guid>.json       GET  /datamap/api/atlas/v2/lineage/{guid}?depth=1&direction=BOTH

Lineage in the Data Map is a graph of source and target entities connected by processes ("Data lineage in classic
Microsoft Purview Data Catalog", read 2026-11-21). With depth 1 for every asset, each file holds an asset's immediate
neighbors, and the files together hold the whole graph; the checks test that the files agree.

    python3 check_lineage_archive.py <export folder> [--out <folder>]
    python3 check_lineage_archive.py --make-demo <folder> [--plant-errors]

Writes nodes.csv, edges.csv (one row per relation, from every file, without repeats) and findings.csv to --out
(default: this script's folder).
Guide: https://www.datameaning.com/guides/archive-purview-lineage-before-switch-off/
Needs Python 3.9+ and duckdb (pip install duckdb).
"""
import argparse
import csv
import json
import os
import pathlib
import random
import re
import sys
import uuid

import duckdb

CHECKS = [
    ("PAGES_MISSING", "Every page of the asset search was saved"),
    ("NO_LINEAGE_FILE", "Every asset in the search has a saved lineage file"),
    ("DANGLING_RELATION", "Every relation's two ends are in the same file's entity map"),
    ("ONE_SIDED_EDGE", "Every relation between two saved assets appears in both assets' files"),
    ("SHELL_ENTITY", "No entity in the graph is a shell entity (isIncomplete)"),
    ("DELETED_ENTITY", "No entity in the graph is marked DELETED"),
]


def stop(msg):
    print(f"CANNOT READ THE EXPORT: {msg}")
    sys.exit(2)


def read_json(path):
    try:
        with open(path, encoding="utf-8-sig") as fh:
            return json.load(fh)
    except (OSError, ValueError) as e:
        stop(f"{path} is not readable JSON ({e.__class__.__name__}: {e})")


def page_no(p):
    m = re.fullmatch(r"page-(\d+)\.json", p.name)
    return int(m.group(1)) if m else -1


def check(folder, out, tag=""):
    folder = pathlib.Path(folder)
    if not folder.is_dir():
        stop(f"{folder} is not a folder")
    findings = []
    pages = sorted((p for p in (folder / "search").glob("page-*.json") if page_no(p) >= 0), key=page_no)
    if not pages:
        stop("no search/page-*.json: save the asset search first")
    nums = [page_no(p) for p in pages]
    for n in range(1, nums[-1] + 1):
        if n not in nums:
            findings.append(("PAGES_MISSING", "search", f"page-{n:03d}.json was not saved"))
    assets, last = [], None
    for p in pages:
        doc = read_json(p)
        if not isinstance(doc, dict) or not isinstance(doc.get("value"), list):
            stop(f"{p} has no \"value\" list")
        assets += [(str(a.get("id") or "").lower(), str(a.get("name") or ""), str(a.get("entityType") or ""))
                   for a in doc["value"] if isinstance(a, dict) and a.get("id")]
        last = doc
    if last.get("continuationToken"):
        findings.append(("PAGES_MISSING", "search", f"{pages[-1].name} still holds a continuationToken for the next page"))

    files = {}
    for p in sorted((folder / "lineage").glob("*.json")):
        doc = read_json(p)
        if not isinstance(doc, dict):
            stop(f"{p} is not a lineage answer")
        files[p.stem.lower()] = doc
    con = duckdb.connect()
    con.execute("CREATE TABLE asset (guid VARCHAR, name VARCHAR, type VARCHAR)")
    con.executemany("INSERT INTO asset VALUES (?,?,?)", assets) if assets else None
    con.execute("CREATE TABLE ent (file VARCHAR, guid VARCHAR, type VARCHAR, name VARCHAR, incomplete BOOLEAN, status VARCHAR)")
    con.execute("CREATE TABLE rel (file VARCHAR, src VARCHAR, dst VARCHAR, rid VARCHAR)")
    ents, rels = [], []
    for f, doc in files.items():
        base = str(doc.get("baseEntityGuid") or "").lower()
        if base and base != f:
            stop(f"lineage/{f}.json holds the lineage of {base}: name each file after its asset's guid")
        for g, e in (doc.get("guidEntityMap") or {}).items():
            e = e if isinstance(e, dict) else {}
            attrs = e.get("attributes") if isinstance(e.get("attributes"), dict) else {}
            ents.append((f, str(g).lower(), str(e.get("typeName") or ""), str(e.get("displayText") or attrs.get("name") or g),
                         bool(e.get("isIncomplete")), str(e.get("status") or "ACTIVE").upper()))
        for r in doc.get("relations") or []:
            if isinstance(r, dict):
                rels.append((f, str(r.get("fromEntityId") or "").lower(), str(r.get("toEntityId") or "").lower(),
                             str(r.get("relationshipId") or "").lower()))
    con.executemany("INSERT INTO ent VALUES (?,?,?,?,?,?)", ents) if ents else None
    con.executemany("INSERT INTO rel VALUES (?,?,?,?)", rels) if rels else None
    print(f"Export: {folder}")
    print(f"Read {len(assets)} assets from {len(pages)} search page{'s' if len(pages) != 1 else ''}, "
          f"{len(files)} lineage files holding {len(rels)} relations (a relation between two saved assets is in both files)")

    def add(code, q):
        findings.extend((code, *r) for r in con.execute(q).fetchall())
    name = "coalesce((SELECT max(name) FROM ent WHERE ent.guid = {g}), (SELECT max(name) FROM asset WHERE asset.guid = {g}), {g})"
    con.execute("CREATE TABLE saved (guid VARCHAR)")
    con.executemany("INSERT INTO saved VALUES (?)", [(f,) for f in files]) if files else None
    add("NO_LINEAGE_FILE", """SELECT 'asset ' || name, type || ' ' || guid || ' has no saved lineage file'
        FROM asset WHERE guid NOT IN (SELECT guid FROM saved) ORDER BY name""")
    add("DANGLING_RELATION", f"""SELECT DISTINCT 'file of ' || {name.format(g='r.file')}, 'relation '
            || {name.format(g='r.src')} || ' -> ' || {name.format(g='r.dst')} || ': '
            || CASE WHEN r.src NOT IN (SELECT guid FROM ent e WHERE e.file = r.file) THEN r.src ELSE r.dst END
            || ' is not in the file''s entity map'
        FROM rel r WHERE r.src NOT IN (SELECT guid FROM ent e WHERE e.file = r.file)
                      OR r.dst NOT IN (SELECT guid FROM ent e WHERE e.file = r.file) ORDER BY 1, 2""")
    add("ONE_SIDED_EDGE", f"""SELECT DISTINCT 'relation ' || {name.format(g='r.src')} || ' -> ' || {name.format(g='r.dst')},
            'in the file of ' || {name.format(g='r.file')} || ', missing from the file of ' || {name.format(g='o.guid')}
        FROM rel r JOIN saved o ON o.guid IN (r.src, r.dst) AND o.guid <> r.file
        WHERE NOT EXISTS (SELECT 1 FROM rel x WHERE x.file = o.guid AND x.src = r.src AND x.dst = r.dst) ORDER BY 1, 2""")
    add("SHELL_ENTITY", """SELECT DISTINCT 'entity ' || name, type || ' ' || guid || ' is a shell entity (isIncomplete), likely an asset that was never scanned'
        FROM ent WHERE incomplete ORDER BY 1""")
    add("DELETED_ENTITY", """SELECT DISTINCT 'entity ' || name, type || ' ' || guid || ' is marked DELETED'
        FROM ent WHERE status = 'DELETED' ORDER BY 1""")

    nodes = con.execute("""SELECT guid, max(name), max(type), bool_or(incomplete), max(status),
            guid IN (SELECT guid FROM saved), guid IN (SELECT guid FROM asset)
        FROM ent GROUP BY guid ORDER BY max(type), max(name), guid""").fetchall()
    edges = con.execute(f"""SELECT r.src, {name.format(g='r.src')}, r.dst, {name.format(g='r.dst')}, max(r.rid), count(DISTINCT r.file)
        FROM rel r GROUP BY r.src, r.dst ORDER BY 2, 4""").fetchall()
    types = con.execute("SELECT typ, count(*) FROM (SELECT guid, max(type) AS typ FROM ent GROUP BY guid) GROUP BY typ ORDER BY 1").fetchall()
    print()
    print("Graph: " + ", ".join(f"{n} {t or '(no type)'}" for t, n in types) + f"; {len(edges)} relations without repeats")
    sources = con.execute("""SELECT count(*) FROM (SELECT guid FROM ent GROUP BY guid) n
        WHERE guid NOT IN (SELECT dst FROM rel) AND guid IN (SELECT src FROM rel)""").fetchone()[0]
    sinks = con.execute("""SELECT count(*) FROM (SELECT guid FROM ent GROUP BY guid) n
        WHERE guid NOT IN (SELECT src FROM rel) AND guid IN (SELECT dst FROM rel)""").fetchone()[0]
    print(f"Entities with no upstream: {sources}; entities with no downstream: {sinks}")
    print()
    print("Checks")
    for code, text in CHECKS:
        k = sum(1 for f in findings if f[0] == code)
        print(f"  {'PASS' if k == 0 else 'FAIL'}  {text}" + (f"  ({k})" if k else ""))
    for code, where, detail in findings:
        print(f"  {code:<18} {where}: {detail}")

    out = pathlib.Path(out)
    out.mkdir(parents=True, exist_ok=True)
    suffix = f"-{tag}" if tag else ""
    safe = lambda v: "'" + str(v) if str(v)[:1] and str(v)[0] in "=+-@" else v
    with open(out / f"nodes{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["guid", "name", "type", "shell_entity", "status", "lineage_file_saved", "in_search"])
        w.writerows([[safe(x) for x in r] for r in nodes])
    with open(out / f"edges{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["from_guid", "from_name", "to_guid", "to_name", "relationship_id", "files_holding_it"])
        w.writerows([[safe(x) for x in r] for r in edges])
    with open(out / f"findings{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["check", "where", "detail"])
        w.writerows([[safe(x) for x in f] for f in findings])
    try:
        where = os.path.relpath(out)
    except ValueError:
        where = str(out)
    print()
    print(f"Wrote nodes{suffix}.csv ({len(nodes)} rows), edges{suffix}.csv ({len(edges)} rows) and findings{suffix}.csv "
          f"({len(findings)} rows) to {where}")
    print(f"RESULT: {len(findings)} findings." + (" Save the missing lineage again and re-run." if findings else
                                                 " The saved graph is whole: every file agrees with its neighbors."))
    return len(findings)


# ---------------------------------------------------------------------------------------------
def make_demo(folder, plant):
    rng = random.Random(268)
    gid = lambda: str(uuid.UUID(int=rng.getrandbits(128), version=4))
    folder = pathlib.Path(folder)
    marker = folder / ".made-by-make-demo"
    if folder.exists() and any(folder.rglob("*.json")) and not marker.exists():
        sys.exit(f"{folder} already holds JSON files that --make-demo did not write; choose a new folder.")
    folder.mkdir(parents=True, exist_ok=True)
    for old in folder.rglob("*.json"):
        old.unlink()
    marker.write_text("Written by check_lineage_archive.py --make-demo: generated records, not an export.\n")
    # sources -> copy -> landing -> transform -> curated -> report: the kind of flow Microsoft's lineage page describes
    E = {}
    def ent(name, typ):
        E[name] = {"guid": gid(), "typeName": typ, "displayText": name, "attributes": {"name": name}, "status": "ACTIVE"}
        return name
    for n in ("erp_orders", "erp_customers", "crm_accounts"):
        ent(n, "azure_sql_table")
    for n in ("landing/orders", "landing/customers", "landing/accounts"):
        ent(n, "azure_datalake_gen2_path")
    for n in ("curated/sales_fact", "curated/customer_dim"):
        ent(n, "azure_datalake_gen2_path")
    ent("Sales dashboard", "powerbi_dashboard")
    for n in ("copy_orders", "copy_customers", "copy_accounts"):
        ent(n, "adf_copy_activity")
    for n in ("build_sales_fact", "build_customer_dim", "refresh_sales_dashboard"):
        ent(n, "synapse_notebook")
    flows = [("erp_orders", "copy_orders"), ("copy_orders", "landing/orders"),
             ("erp_customers", "copy_customers"), ("copy_customers", "landing/customers"),
             ("crm_accounts", "copy_accounts"), ("copy_accounts", "landing/accounts"),
             ("landing/orders", "build_sales_fact"), ("landing/customers", "build_sales_fact"), ("build_sales_fact", "curated/sales_fact"),
             ("landing/customers", "build_customer_dim"), ("landing/accounts", "build_customer_dim"), ("build_customer_dim", "curated/customer_dim"),
             ("curated/sales_fact", "refresh_sales_dashboard"), ("curated/customer_dim", "refresh_sales_dashboard"),
             ("refresh_sales_dashboard", "Sales dashboard")]
    R = {f: gid() for f in flows}
    planted = []
    names = list(E)
    if plant:
        ent("legacy_stage", "azure_sql_table"); E["legacy_stage"]["isIncomplete"] = True
        flows.append(("legacy_stage", "copy_orders")); R[flows[-1]] = gid(); planted.append("SHELL_ENTITY")
        E["copy_accounts"]["status"] = "DELETED"; planted.append("DELETED_ENTITY")

    def lineage(n):
        near = [f for f in flows if n in f]
        m = {E[x]["guid"]: E[x] for f in near for x in f}
        m[E[n]["guid"]] = E[n]
        return {"baseEntityGuid": E[n]["guid"], "lineageDirection": "BOTH", "lineageDepth": 1, "guidEntityMap": m,
                "relations": [{"fromEntityId": E[a]["guid"], "toEntityId": E[b]["guid"], "relationshipId": R[(a, b)]} for a, b in near]}
    files = {n: lineage(n) for n in names}
    if plant:
        del files["crm_accounts"]; planted.append("NO_LINEAGE_FILE")
        files["build_customer_dim"]["relations"] = [r for r in files["build_customer_dim"]["relations"]
                                                    if r["toEntityId"] != E["curated/customer_dim"]["guid"]]
        planted.append("ONE_SIDED_EDGE")
        g = E["landing/orders"]["guid"]
        files["copy_orders"]["guidEntityMap"].pop(g); planted.append("DANGLING_RELATION")
    hits = [{"id": E[n]["guid"], "name": n, "entityType": E[n]["typeName"]} for n in names]
    size = 10
    pages = [hits[i:i + size] for i in range(0, len(hits), size)]
    for i, pg in enumerate(pages, 1):
        doc = {"@search.count": len(hits), "value": pg}
        if i < len(pages):
            doc["continuationToken"] = f"token-{i + 1}"
        (folder / "search").mkdir(exist_ok=True)
        if plant and i == len(pages):
            doc["continuationToken"] = "token-extra"; planted.append("PAGES_MISSING")
        (folder / "search" / f"page-{i:03d}.json").write_text(json.dumps(doc, indent=1))
    (folder / "lineage").mkdir(exist_ok=True)
    for n, doc in files.items():
        (folder / "lineage" / f"{E[n]['guid']}.json").write_text(json.dumps(doc, indent=1))
    print(f"Wrote a generated lineage export to {folder}: {len(hits)} assets in {len(pages)} search pages, "
          f"{len(files)} lineage files, {len(flows)} relations (seed 268)")
    if plant:
        assert set(planted) == {c for c, _ in CHECKS}, planted
        print("Planted one error for each check: " + ", ".join(sorted(planted)))


def main():
    ap = argparse.ArgumentParser(description="Check a saved Purview lineage graph and join it into one edge list.")
    ap.add_argument("export", nargs="?")
    ap.add_argument("--out", default=str(pathlib.Path(__file__).resolve().parent))
    ap.add_argument("--make-demo", metavar="FOLDER")
    ap.add_argument("--plant-errors", action="store_true")
    ap.add_argument("--tag", default="")
    a = ap.parse_args()
    if a.make_demo:
        make_demo(a.make_demo, a.plant_errors)
        return 0
    if not a.export:
        ap.error("give the export folder, or --make-demo FOLDER")
    print(f"Python {sys.version.split()[0]} duckdb {duckdb.__version__}")
    check(a.export, a.out, a.tag)
    return 0


if __name__ == "__main__":
    sys.exit(main())
