#!/usr/bin/env python3
"""What Microsoft's published APIs can export from Purview Data Governance and load into Microsoft Fabric.

Reads Microsoft's own REST API specifications, pinned to one commit each, counts every operation, sorts each
one as a read (it can export) or a write (it can load), and matches each kind of Purview governance object to
the Fabric target Microsoft's transition guide names for it. Writes operations.csv (every operation) and
landing.csv (one row per Purview object type) next to this script.

    python3 count_api_operations.py [--cache <folder>] [--out <folder>] [--plant-fault]

Sources (both MIT licensed):
  Azure/azure-rest-api-specs at commit be93aca8 (Purview Unified Catalog, Data Map and Scanning data planes)
  microsoft/fabric-rest-api-specs at commit ec85f052 (every Fabric REST API specification)
Guide: https://www.datameaning.com/guides/what-microsoft-apis-can-export-from-purview-and-load-into-onelake/
Needs Python 3.9+ and duckdb (pip install duckdb). Downloads about 4 MB the first time; --cache keeps the files.
"""
import argparse
import csv
import hashlib
import json
import os
import pathlib
import re
import sys
import urllib.request

import duckdb

AZURE = ("Azure/azure-rest-api-specs", "be93aca8c35ada896dd7fd6c7f65b1a9f766e84c")
FABRIC = ("microsoft/fabric-rest-api-specs", "ec85f0522fec7fb897456aa5caaa070969dd460f")
PURVIEW_SPECS = {
    "Unified Catalog": "specification/purviewdatagovernance/data-plane/Azure.Analytics.Purview.UnifiedCatalog/"
                       "preview/2026-03-20-preview/CatalogApiService.json",
    "Data Map": "specification/purview/data-plane/datamap/stable/2023-09-01/purviewdatamap.json",
    "Scanning": "specification/purview/data-plane/scanning/stable/2023-09-01/scanningService.json",
}
UA = "count_api_operations.py (https://www.datameaning.com/guides/what-microsoft-apis-can-export-from-purview-and-load-into-onelake/; info@datameaning.com)"
# A POST that only reads: its operationId says so.
READ_POST = re.compile(r"^(List|Get|Query|Search|Count|Facets|Find|Read|Export)")

# The judgment this measurement rests on, written out so a reader can disagree with it. Each Purview object type,
# the API tag(s) that hold it, the row of Microsoft's transition guide (read 2026-11-21) it belongs to, the
# target that row names, and the Fabric API tag(s) that would load into that target ("" = the guide names
# no Fabric target, or the target is not an API object).
LANDING = [
    # (object type, Purview API, Purview tags, transition-guide row, target the row names, Fabric tags)
    ("Business domains", "Unified Catalog", ["BusinessDomain"], "Collections, domains, ownership",
     "Fabric domains and subdomains, workspace assignment, domain admins, owners", ["Domains"]),
    ("Glossary terms", "Unified Catalog", ["Term"], "Glossary, critical data elements, custom context",
     "Tags, descriptions, endorsement", ["Tags"]),
    ("Critical data elements", "Unified Catalog", ["CriticalDataElements"],
     "Glossary, critical data elements, custom context", "Tags, descriptions, endorsement", ["Tags"]),
    ("Data products", "Unified Catalog", ["DataProduct"], "Data products and publishing",
     "Domains, endorsement, sharing, discoverable semantic models", ["Domains"]),
    ("Objectives and key results", "Unified Catalog", ["OKR"], "(not in the table)", "", []),
    ("Data access policies", "Unified Catalog", ["Policy"], "Access policy and enforcement",
     "Fabric Policies (preview); workspace, item, OneLake, and source permissions", ["PolicySets", "PolicyRules"]),
    ("Data assets", "Unified Catalog", ["DataAsset"], "Catalog search and browse",
     "OneLake catalog Explore, item details, filters, Catalog Search API", ["Items"]),
    ("Data columns", "Unified Catalog", ["DataColumn"], "Catalog search and browse",
     "OneLake catalog Explore, item details, filters, Catalog Search API", []),
    ("Classic glossary", "Data Map", ["Glossary"], "Glossary, critical data elements, custom context",
     "Tags, descriptions, endorsement", ["Tags"]),
    ("Lineage", "Data Map", ["Lineage"], "Lineage and impact analysis",
     "Fabric lineage and impact analysis for supported items", []),
    ("Classification rules", "Scanning", ["ClassificationRules"], "Classification and sensitive-data protection",
     "Purview sensitivity labels and DLP on supported Fabric items", []),
    ("Scan rule sets and scans", "Scanning", ["ScanRulesets", "Scan"], "External metadata scanning",
     "Shortcuts, mirroring, external sharing, scanner/admin APIs", []),
]


def fetch(repo, commit, path, cache):
    """One specification file at a pinned commit, from the local cache when it is there."""
    local = pathlib.Path(cache) / repo.replace("/", "_") / commit[:8] / path
    if not local.is_file():
        url = f"https://raw.githubusercontent.com/{repo}/{commit}/{path}"
        req = urllib.request.Request(url, headers={"User-Agent": UA})
        with urllib.request.urlopen(req, timeout=60) as r:
            data = r.read()
        local.parent.mkdir(parents=True, exist_ok=True)
        local.write_bytes(data)
    return local.read_bytes()


def fabric_spec_paths(cache):
    """Every swagger.json in the Fabric specification repository at the pinned commit."""
    local = pathlib.Path(cache) / f"fabric-tree-{FABRIC[1][:8]}.json"   # one list per pinned commit
    if not local.is_file():
        url = f"https://api.github.com/repos/{FABRIC[0]}/git/trees/{FABRIC[1]}?recursive=1"
        req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "application/vnd.github+json"})
        with urllib.request.urlopen(req, timeout=60) as r:
            local.parent.mkdir(parents=True, exist_ok=True)
            local.write_bytes(r.read())
    tree = json.loads(local.read_text())
    if tree.get("truncated"):
        sys.exit("the Fabric file list came back truncated; cannot count every API")
    return sorted(t["path"] for t in tree["tree"] if t["path"].endswith("/swagger.json"))


def operations(api, spec):
    """(api, tag, method, path, operationId, kind) for every operation in an OpenAPI 2.0 file."""
    out = []
    for path, ops in spec.get("paths", {}).items():
        for method, op in ops.items():
            if method not in ("get", "post", "put", "patch", "delete") or not isinstance(op, dict):
                continue
            opid = op.get("operationId", "")
            tag = (op.get("tags") or [opid.split("_")[0] or path.strip("/").split("/")[0]])[0]
            name = opid.split("_", 1)[1] if "_" in opid else opid   # "Terms_Query" -> "Query"
            kind = "read" if method == "get" or (method == "post" and READ_POST.match(name)) else "write"
            out.append((api, tag, method.upper(), (spec.get("basePath") or "") + path, opid, kind))
    return out


def main():
    ap = argparse.ArgumentParser(description="Count what Purview's APIs can export and Fabric's APIs can load.")
    ap.add_argument("--cache", default=str(pathlib.Path(__file__).resolve().parent / "spec-cache"),
                    help="folder that keeps the downloaded specifications (default: spec-cache beside the script)")
    ap.add_argument("--out", default=str(pathlib.Path(__file__).resolve().parent), help="folder for the CSVs")
    ap.add_argument("--plant-fault", action="store_true",
                    help="add a landing row whose Purview tag is not in any specification, to show the self-check")
    a = ap.parse_args()
    landing = list(LANDING)
    if a.plant_fault:
        landing.append(("Glossary term templates", "Unified Catalog", ["TermTemplate"],
                        "Glossary, critical data elements, custom context", "Tags, descriptions, endorsement", ["Tags"]))
    print(f"Python {sys.version.split()[0]} duckdb {duckdb.__version__}")
    rows, files = [], []
    for api, path in PURVIEW_SPECS.items():
        data = fetch(*AZURE, path, a.cache)
        files.append((f"Purview {api}", f"{AZURE[0]}@{AZURE[1][:8]}/{path}", data))
        rows += operations(f"Purview {api}", json.loads(data))
    for path in fabric_spec_paths(a.cache):
        data = fetch(*FABRIC, path, a.cache)
        files.append(("Fabric", f"{FABRIC[0]}@{FABRIC[1][:8]}/{path}", data))
        rows += operations("Fabric", json.loads(data))
    print(f"Read {len(files)} specification files, {sum(len(d) for _, _, d in files):,} bytes:")
    for api, where, data in files[:3]:
        print(f"  {api:<26} {len(data):>8,} bytes  sha256 {hashlib.sha256(data).hexdigest()[:16]}  {where.split('/')[-1]}")
    fab = [d for api, _, d in files if api == "Fabric"]
    print(f"  {'Fabric (' + str(len(fab)) + ' files)':<26} {sum(len(d) for d in fab):>8,} bytes  sha256 "
          + hashlib.sha256(b"".join(hashlib.sha256(d).digest() for d in fab)).hexdigest()[:16] + "  (of the file hashes)")

    con = duckdb.connect()
    con.execute("CREATE TABLE op (api VARCHAR, tag VARCHAR, method VARCHAR, path VARCHAR, opid VARCHAR, kind VARCHAR)")
    con.executemany("INSERT INTO op VALUES (?,?,?,?,?,?)", rows)
    print()
    print("Operations by API")
    for api, n, r, w in con.execute("""SELECT api, count(*), count(*) FILTER (WHERE kind = 'read'),
            count(*) FILTER (WHERE kind = 'write') FROM op GROUP BY api ORDER BY api""").fetchall():
        print(f"  {api:<26} {n:>5} operations  {r:>5} read  {w:>5} write")
    lin = con.execute("SELECT count(*) FROM op WHERE api = 'Fabric' AND (lower(path) LIKE '%lineage%' "
                      "OR lower(opid) LIKE '%lineage%')").fetchone()[0]
    print(f"  Fabric operations whose path or name mentions lineage: {lin}")

    # Self-checks: every tag the landing table names exists in its specification; counts add up.
    problems = []
    for obj, api, ptags, _, _, ftags in landing:
        for t in ptags:
            if not con.execute("SELECT count(*) FROM op WHERE api = ? AND tag = ?", [f"Purview {api}", t]).fetchone()[0]:
                problems.append(f"{obj}: no tag {t!r} in the Purview {api} specification")
        for t in ftags:
            if not con.execute("SELECT count(*) FROM op WHERE api = 'Fabric' AND tag = ?", [t]).fetchone()[0]:
                problems.append(f"{obj}: no tag {t!r} in any Fabric specification")
    total = con.execute("SELECT count(*), count(*) FILTER (WHERE kind IN ('read', 'write')) FROM op").fetchone()
    if total[0] != total[1] or total[0] != len(rows):
        problems.append("operation counts do not add up")

    out_rows = []
    for obj, api, ptags, row, target, ftags in landing:
        q = "SELECT count(*) FILTER (WHERE kind = 'read'), count(*) FILTER (WHERE kind = 'write') FROM op WHERE api = ? AND tag IN (SELECT unnest(?))"
        pr, pw = con.execute(q, [f"Purview {api}", ptags]).fetchone()
        fr, fw = con.execute(q, ["Fabric", ftags]).fetchone() if ftags else (0, 0)
        if pr == 0:
            verdict = "no export operation"
        elif not ftags:
            verdict = "export only: no Fabric API object to load into"
        elif fw == 0:
            verdict = "export only: the Fabric API has no write operation"
        else:
            verdict = "export by API; the named Fabric target has write operations"
        out_rows.append((obj, f"Purview {api}", "; ".join(ptags), pr, row, target, "; ".join(ftags) or "-", fw, verdict))
    print()
    print("Each Purview object type, and where Microsoft's transition guide says it lands")
    print(f"  {'Object type':<28} {'Export ops':>10}  {'Fabric API tags':<24} {'Load ops':>8}  Result")
    for obj, api, ptags, pr, row, target, ftags, fw, verdict in out_rows:
        print(f"  {obj:<28} {pr:>10}  {ftags:<24} {fw:>8}  {verdict}")
    counts = {}
    for r in out_rows:
        counts[r[-1]] = counts.get(r[-1], 0) + 1
    print("Summary")
    for v, n in sorted(counts.items()):
        print(f"  {n:>2} object types: {v}")

    if problems:
        print()
        print("SELF-CHECK FAILED (no CSV written, so earlier results are kept):")
        for p in problems:
            print("  " + p)
        return 1
    out = pathlib.Path(a.out)
    out.mkdir(parents=True, exist_ok=True)
    tag = "-planted" if a.plant_fault else ""
    with open(out / f"operations{tag}.csv", "w", newline="", encoding="utf-8") as fh:
        w = csv.writer(fh)
        w.writerow(["api", "tag", "method", "path", "operation_id", "kind"])
        w.writerows(sorted(rows))
    with open(out / f"landing{tag}.csv", "w", newline="", encoding="utf-8") as fh:
        w = csv.writer(fh)
        w.writerow(["purview_object", "purview_api", "purview_tags", "export_operations", "transition_guide_row",
                    "target_the_row_names", "fabric_api_tags", "fabric_write_operations", "result"])
        w.writerows(out_rows)
    try:
        where = os.path.relpath(out)
    except ValueError:
        where = str(out)
    print()
    print(f"Wrote operations{tag}.csv ({len(rows)} rows) and landing{tag}.csv ({len(out_rows)} rows) to {where}")
    print("Self-checks passed: every tag in the landing table exists in its specification, and the counts add up.")
    return 0


if __name__ == "__main__":
    sys.exit(main())
