#!/usr/bin/env python3
"""Check that Fabric items carry the sensitivity labels they should after the move off Purview Data Governance.

Reads three inputs from one folder:

    <folder>/scan/*.json      scanner API results: GET https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/{scanId}
                              (each workspace lists reports, dashboards, datasets, dataflows, datamarts and other items;
                              an item's label is sensitivityLabel.labelId)
    <folder>/labels.json      the labels: GET https://graph.microsoft.com/beta/me/security/informationProtection/sensitivityLabels
    <folder>/expected.csv     the label each item should carry: item_id, expected_label (a label id or name)

    python3 check_fabric_labels.py <folder> [--out <folder>]
    python3 check_fabric_labels.py --make-demo <folder> [--plant-errors]

Writes labels-by-item.csv (one row per scanned item) and findings.csv to --out (default: this script's folder).
Microsoft's transition guide keeps classification and sensitive-data protection in Purview and says to validate item
types, inheritance, export behavior and licensing ("From Purview Data Governance to the Fabric OneLake catalog",
read 2026-11-21). This script does the item-by-item part of that validation.
Guide: https://www.datameaning.com/guides/check-sensitivity-labels-after-moving-to-fabric/
Needs Python 3.9+ and duckdb (pip install duckdb).
"""
import argparse
import csv
import json
import os
import pathlib
import random
import sys
import uuid

import duckdb

CHECKS = [
    ("NOT_SCANNED", "Every item in expected.csv is in the scan results"),
    ("NO_LABEL", "Every item that should carry a label carries one"),
    ("DIFFERENT_LABEL", "Every item carries the label expected.csv names"),
    ("UNKNOWN_LABEL", "Every label on an item is in labels.json"),
    ("INACTIVE_LABEL", "No item carries a label the tenant has turned off"),
    ("UNLABELED_ITEM", "Every scanned item carries a label"),
]
SKIP = {"users", "tiles", "datasourceUsages", "upstreamDataflows", "upstreamDatasets", "relations", "tables",
        "expressions", "roles", "endorsementDetails", "sensitivityLabel"}


def stop(msg):
    print(f"CANNOT READ THE INPUT: {msg}")
    sys.exit(2)


def read_json(path):
    try:
        with open(path, encoding="utf-8-sig") as fh:
            return json.load(fh)
    except (OSError, ValueError) as e:
        stop(f"{path} is not readable JSON ({e.__class__.__name__}: {e})")


def items_of(ws):
    """Every object with an id in one of the workspace's lists: reports, dashboards, datasets, and the rest.
    The scan result names a dataflow by objectId rather than id."""
    for k, v in ws.items():
        if k in SKIP or not isinstance(v, list):
            continue
        for it in v:
            if isinstance(it, dict) and (it.get("id") or it.get("objectId")):
                yield k, dict(it, id=it.get("id") or it.get("objectId"))


def check(folder, out, tag=""):
    folder = pathlib.Path(folder)
    if not folder.is_dir():
        stop(f"{folder} is not a folder")
    scans = sorted((folder / "scan").glob("*.json"))
    if not scans:
        stop("no scan/*.json: save the scanner API's scan results first")
    rows = []
    for f in scans:
        doc = read_json(f)
        for ws in doc.get("workspaces") or []:
            if not isinstance(ws, dict):
                continue
            for kind, it in items_of(ws):
                lab = (it.get("sensitivityLabel") or {}).get("labelId") if isinstance(it.get("sensitivityLabel"), dict) else None
                rows.append((str(it["id"]).lower(), str(it.get("name") or it.get("displayName") or it["id"]),
                             kind, str(ws.get("name") or ws.get("id") or ""), str(lab).lower() if lab else None))
    labels = read_json(folder / "labels.json") if (folder / "labels.json").is_file() else stop("labels.json is missing")
    labels = labels.get("value") if isinstance(labels, dict) else labels
    if not isinstance(labels, list):
        stop("labels.json has no \"value\" list")
    if not (folder / "expected.csv").is_file():
        stop("expected.csv is missing")
    with open(folder / "expected.csv", newline="", encoding="utf-8-sig") as fh:
        rd = csv.DictReader(fh)
        if not {"item_id", "expected_label"} <= set(rd.fieldnames or []):
            stop("expected.csv needs the columns item_id and expected_label")
        expected = [(r["item_id"].strip().lower(), (r["expected_label"] or "").strip()) for r in rd if (r["item_id"] or "").strip()]

    con = duckdb.connect()
    con.execute("CREATE TABLE item (id VARCHAR, name VARCHAR, kind VARCHAR, ws VARCHAR, label VARCHAR)")
    con.executemany("INSERT INTO item VALUES (?,?,?,?,?)", rows) if rows else None
    con.execute("CREATE TABLE lab (id VARCHAR, name VARCHAR, active BOOLEAN, full_name VARCHAR)")
    con.executemany("INSERT INTO lab VALUES (?,?,?,?)", [
        (str(l.get("id") or "").lower(), str(l.get("name") or ""), l.get("isActive") is not False,
         (str(l["parent"].get("name") or "") + "\\" if isinstance(l.get("parent"), dict) else "") + str(l.get("name") or ""))
        for l in labels if isinstance(l, dict) and l.get("id")])
    con.execute("CREATE TABLE exp (id VARCHAR, want VARCHAR)")
    con.executemany("INSERT INTO exp VALUES (?,?)", expected) if expected else None
    # an expected label may be given by id, by "Parent\\Label" or by a name only one label has
    con.execute("""CREATE TABLE expl AS SELECT e.id, e.want, coalesce(
            (SELECT min(l.id) FROM lab l WHERE l.id = lower(e.want)),
            (SELECT min(l.id) FROM lab l WHERE lower(l.full_name) = lower(e.want) HAVING count(*) = 1),
            (SELECT min(l.id) FROM lab l WHERE lower(l.name) = lower(e.want) HAVING count(*) = 1)) AS want_id,
            (SELECT count(*) FROM lab l WHERE lower(l.name) = lower(e.want)) AS same_name FROM exp e""")
    dup = con.execute("SELECT id, count(*) FROM item GROUP BY id HAVING count(*) > 1").fetchall()
    print(f"Input: {folder}")
    print(f"Read {len(scans)} scan result file{'s' if len(scans) != 1 else ''}: {con.execute('SELECT count(DISTINCT ws) FROM item').fetchone()[0]} "
          f"workspaces, {len(rows)} items; {len(labels)} labels; {len(expected)} expected labels")
    if dup:
        stop(f"{len(dup)} item ids appear in more than one scan result: keep one scan result per workspace")

    findings = []
    lname = "coalesce((SELECT max(name) FROM lab WHERE lab.id = {x}), {x})"
    def add(code, q):
        findings.extend((code, *r) for r in con.execute(q).fetchall())
    add("NOT_SCANNED", "SELECT 'item ' || e.id, 'expected ' || e.want || ', but the item is in no scan result' FROM expl e WHERE e.id NOT IN (SELECT id FROM item) ORDER BY 1")
    add("NO_LABEL", "SELECT i.kind || ' ' || i.name, 'no label; expected ' || e.want FROM expl e JOIN item i ON i.id = e.id WHERE i.label IS NULL ORDER BY 1")
    add("DIFFERENT_LABEL", f"""SELECT i.kind || ' ' || i.name, 'carries ' || {lname.format(x='i.label')} || '; expected ' || e.want
        FROM expl e JOIN item i ON i.id = e.id WHERE i.label IS NOT NULL AND i.label IS DISTINCT FROM coalesce(e.want_id, lower(e.want)) ORDER BY 1""")
    add("UNKNOWN_LABEL", "SELECT i.kind || ' ' || i.name, 'label ' || i.label || ' is not in labels.json' FROM item i WHERE i.label IS NOT NULL AND i.label NOT IN (SELECT id FROM lab) ORDER BY 1")
    add("INACTIVE_LABEL", "SELECT i.kind || ' ' || i.name, 'label ' || l.name || ' is turned off (isActive false)' FROM item i JOIN lab l ON l.id = i.label WHERE NOT l.active ORDER BY 1")
    add("UNLABELED_ITEM", "SELECT i.kind || ' ' || i.name, 'in workspace ' || i.ws || ', no label and not in expected.csv' FROM item i WHERE i.label IS NULL AND i.id NOT IN (SELECT id FROM exp) ORDER BY 1")
    amb = con.execute("SELECT want FROM expl WHERE want_id IS NULL AND same_name > 1").fetchall()
    if amb:
        stop(f"expected.csv names {amb[0][0]!r}, a name more than one label has: give the label id, or the parent "
             f"and label as Parent\\Label")
    bad_exp = con.execute("SELECT want FROM expl WHERE want_id IS NULL").fetchall()
    if bad_exp:
        stop(f"expected.csv names {len(bad_exp)} label(s) that are not in labels.json, for example {bad_exp[0][0]!r}")

    print()
    print("Items by kind and label")
    for kind, lab, n in con.execute(f"""SELECT kind, CASE WHEN label IS NULL THEN '(no label)' ELSE {lname.format(x='label')} END, count(*)
            FROM item GROUP BY ALL ORDER BY 1, 2""").fetchall():
        print(f"  {kind:<12} {lab[:30]:<30} {n:>4}")
    print()
    print("Checks")
    for code, text in CHECKS:
        k = sum(1 for f in findings if f[0] == code)
        print(f"  {'PASS' if k == 0 else 'FAIL'}  {text}" + (f"  ({k})" if k else ""))
    for code, where, detail in findings:
        print(f"  {code:<15} {where}: {detail}")

    out = pathlib.Path(out)
    out.mkdir(parents=True, exist_ok=True)
    suffix = f"-{tag}" if tag else ""
    safe = lambda v: "'" + str(v) if str(v)[:1] and str(v)[0] in "=+-@" else ("" if v is None else v)
    table = con.execute(f"""SELECT i.ws, i.kind, i.id, i.name, coalesce(i.label, ''),
            CASE WHEN i.label IS NULL THEN '' ELSE {lname.format(x='i.label')} END, coalesce(e.want, '')
        FROM item i LEFT JOIN expl e ON e.id = i.id ORDER BY 1, 2, 4""").fetchall()
    with open(out / f"labels-by-item{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["workspace", "kind", "item_id", "item", "label_id", "label", "expected_label"])
        w.writerows([[safe(x) for x in r] for r in table])
    with open(out / f"findings{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["check", "where", "detail"])
        w.writerows([[safe(x) for x in f] for f in findings])
    try:
        where = os.path.relpath(out)
    except ValueError:
        where = str(out)
    print()
    print(f"Wrote labels-by-item{suffix}.csv ({len(table)} items) and findings{suffix}.csv ({len(findings)} rows) to {where}")
    print(f"RESULT: {len(findings)} findings." + (" Fix the labels in Fabric, scan again and re-run." if findings else
                                                 " Every item carries the label it should."))
    return len(findings)


# ---------------------------------------------------------------------------------------------
def make_demo(folder, plant):
    rng = random.Random(272)
    gid = lambda: str(uuid.UUID(int=rng.getrandbits(128), version=4))
    folder = pathlib.Path(folder)
    marker = folder / ".made-by-make-demo"
    if folder.exists() and any(folder.iterdir()) and not marker.exists():
        sys.exit(f"{folder} already holds files that --make-demo did not write; choose a new folder.")
    (folder / "scan").mkdir(parents=True, exist_ok=True)
    for old in list(folder.rglob("*.json")) + list(folder.rglob("*.csv")):
        old.unlink()
    marker.write_text("Written by check_fabric_labels.py --make-demo: generated records, not a scan.\n")
    labels = [{"id": gid(), "name": n, "sensitivity": i, "isActive": True, "parent": None}
              for i, n in enumerate(["Public", "General", "Confidential", "Highly Confidential"])]
    L = {l["name"]: l["id"] for l in labels}
    plan = {"Finance": [("reports", "Revenue report", "Confidential"), ("datasets", "Revenue model", "Confidential"),
                        ("dashboards", "CFO dashboard", "Highly Confidential"), ("dataflows", "Ledger dataflow", "Confidential")],
            "Sales": [("reports", "Pipeline report", "General"), ("datasets", "Pipeline model", "General"),
                      ("reports", "Territory report", "General"), ("datamarts", "Sales datamart", "Confidential")],
            "People": [("datasets", "Headcount model", "Highly Confidential"), ("reports", "Headcount report", "Highly Confidential"),
                       ("dataflows", "HR feed", "Highly Confidential")],
            "Shared": [("reports", "Holiday calendar", "Public"), ("datasets", "Calendar model", "Public")]}
    expected, planted = [], []
    workspaces = []
    for wsn, items in plan.items():
        ws = {"id": gid(), "name": wsn, "type": "Workspace", "state": "Active"}
        for kind, name, lab in items:
            it = {"objectId" if kind == "dataflows" else "id": gid(), "name": name, "sensitivityLabel": {"labelId": L[lab]}}
            ws.setdefault(kind, []).append(it)
            expected.append((it.get("id") or it["objectId"], lab))
        workspaces.append(ws)
    if plant:
        get = lambda n: next(it for ws in workspaces for k in ("reports", "datasets", "dashboards", "dataflows", "datamarts")
                             for it in ws.get(k, []) if it["name"] == n)
        expected.append((gid(), "Confidential")); planted.append("NOT_SCANNED")
        del get("Ledger dataflow")["sensitivityLabel"]; planted.append("NO_LABEL")
        get("Headcount report")["sensitivityLabel"]["labelId"] = L["General"]; planted.append("DIFFERENT_LABEL")
        get("Territory report")["sensitivityLabel"]["labelId"] = gid()
        expected = [e for e in expected if e[0] != get("Territory report")["id"]]; planted.append("UNKNOWN_LABEL")
        labels.append({"id": gid(), "name": "Internal (turned off)", "sensitivity": 1, "isActive": False, "parent": None})
        get("Holiday calendar")["sensitivityLabel"]["labelId"] = labels[-1]["id"]
        expected = [e for e in expected if e[0] != get("Holiday calendar")["id"]]; planted.append("INACTIVE_LABEL")
        workspaces[1]["reports"].append({"id": gid(), "name": "Ad hoc export"}); planted.append("UNLABELED_ITEM")
    half = len(workspaces) // 2
    for n, part in enumerate((workspaces[:half], workspaces[half:]), 1):
        (folder / "scan" / f"scan-{n}.json").write_text(json.dumps({"workspaces": part, "datasourceInstances": []}, indent=1))
    (folder / "labels.json").write_text(json.dumps({"value": labels}, indent=1))
    with open(folder / "expected.csv", "w", newline="") as fh:
        w = csv.writer(fh); w.writerow(["item_id", "expected_label"]); w.writerows(expected)
    n_items = sum(len(v) for ws in workspaces for k, v in ws.items() if isinstance(v, list))
    print(f"Wrote generated inputs to {folder}: {len(workspaces)} workspaces with {n_items} items in 2 scan results, "
          f"{len(labels)} labels, {len(expected)} expected labels (seed 272)")
    if plant:
        assert set(planted) == {c for c, _ in CHECKS}, planted
        print("Planted one problem for each check: " + ", ".join(sorted(planted)))


def main():
    ap = argparse.ArgumentParser(description="Check Fabric items' sensitivity labels against the labels they should carry.")
    ap.add_argument("folder", nargs="?")
    ap.add_argument("--out", default=str(pathlib.Path(__file__).resolve().parent))
    ap.add_argument("--make-demo", metavar="FOLDER")
    ap.add_argument("--plant-errors", action="store_true")
    ap.add_argument("--tag", default="")
    a = ap.parse_args()
    if a.make_demo:
        make_demo(a.make_demo, a.plant_errors)
        return 0
    if not a.folder:
        ap.error("give the input folder, or --make-demo FOLDER")
    print(f"Python {sys.version.split()[0]} duckdb {duckdb.__version__}")
    check(a.folder, a.out, a.tag)
    return 0


if __name__ == "__main__":
    sys.exit(main())
