#!/usr/bin/env python3
"""Check Purview scan rule sets and custom classification rules saved through the Scanning API, and test every rule.

Reads the Scanning API's own answers (api-version 2023-09-01), one file per page, in page order:

    <export>/classificationrules/page-001.json        GET /scan/classificationrules
    <export>/scanrulesets/page-001.json               GET /scan/scanrulesets
    <export>/datasources/page-001.json                GET /scan/datasources
    <export>/scans/<data source>/page-001.json        GET /scan/datasources/{dataSourceName}/scans
    <export>/samples.csv                              your test values: rule, column_name, value, expect (match or no match)

    python3 check_scan_rules.py <export folder> [--out <folder>]
    python3 check_scan_rules.py --make-demo <folder> [--plant-errors]

Writes rules.csv (one row per custom classification rule, with the rule sets and scans that use it) and
findings.csv to --out (default: this script's folder).
A custom classification rule holds data patterns and column patterns (regular expressions); Microsoft's page says the
column pattern is checked as an AND condition with the data pattern ("Classification best practices for Microsoft
Purview Data Map", read 2026-11-21). The script tests each sample with Python's re module, searching the value and the
column name; Purview's own engine can differ on rare syntax.
Guide: https://www.datameaning.com/guides/export-purview-scan-rules-and-custom-classifications/
Needs Python 3.9+ and duckdb (pip install duckdb).
"""
import argparse
import csv
import json
import os
import pathlib
import re
import sys

import duckdb

CHECKS = [
    ("PAGES_MISSING", "Every page of every list was saved"),
    ("UNKNOWN_RULESET", "Every scan's custom rule set is in the export"),
    ("UNKNOWN_RULE", "Every custom classification rule a rule set includes is in the export"),
    ("DISABLED_RULE_IN_USE", "No rule set includes a disabled classification rule"),
    ("UNUSED_RULE", "Every enabled custom classification rule is in at least one rule set"),
    ("BAD_PATTERN", "Every pattern can be read as a regular expression"),
    ("NO_SAMPLE", "Every custom classification rule in use has a test value that should match"),
    ("SAMPLE_FAILS", "Every test value gives the result samples.csv expects"),
]


def stop(msg):
    print(f"CANNOT READ THE EXPORT: {msg}")
    sys.exit(2)


def read_json(path):
    try:
        with open(path, encoding="utf-8-sig") as fh:
            return json.load(fh)
    except (OSError, ValueError) as e:
        stop(f"{path} is not readable JSON ({e.__class__.__name__}: {e})")


def page_no(p):
    m = re.fullmatch(r"page-(\d+)\.json", p.name)
    return int(m.group(1)) if m else -1


def pages(folder, label, findings, required=True):
    """The objects of every page-NNN.json in folder, in order, with a finding for every gap or leftover nextLink."""
    files = sorted((p for p in folder.glob("page-*.json") if page_no(p) >= 0), key=page_no) if folder.is_dir() else []
    if not files:
        if required:
            findings.append(("PAGES_MISSING", label, f"{label}/page-001.json was not saved"))
        return []
    nums = [page_no(p) for p in files]
    for n in range(1, nums[-1] + 1):
        if n not in nums:
            findings.append(("PAGES_MISSING", label, f"page-{n:03d}.json was not saved"))
    out, last = [], None
    for p in files:
        doc = read_json(p)
        if not isinstance(doc, dict) or not isinstance(doc.get("value"), list):
            stop(f"{p} has no \"value\" list")
        out += [o for o in doc["value"] if isinstance(o, dict)]
        last = doc
    if last.get("nextLink"):
        findings.append(("PAGES_MISSING", label, f"{files[-1].name} still points to a next page"))
    return out


def patterns(rule, key):
    return [str(p.get("pattern") or "") for p in (rule.get("properties") or {}).get(key) or []
            if isinstance(p, dict) and str(p.get("kind") or "Regex") == "Regex"]


def check(folder, out, tag=""):
    folder = pathlib.Path(folder)
    if not folder.is_dir():
        stop(f"{folder} is not a folder")
    findings = []
    rules = pages(folder / "classificationrules", "classificationrules", findings)
    rulesets = pages(folder / "scanrulesets", "scanrulesets", findings)
    sources = pages(folder / "datasources", "datasources", findings)
    scans = []
    for ds in sources:
        name = str(ds.get("name") or "")
        for sc in pages(folder / "scans" / name, f"scans/{name}", findings):
            scans.append((name, str(ds.get("kind") or ""), sc))

    con = duckdb.connect()
    con.execute("""CREATE TABLE rule (name VARCHAR, classification VARCHAR, status VARCHAR, threshold DOUBLE,
                   column_patterns VARCHAR, data_patterns VARCHAR)""")
    custom = [r for r in rules if str(r.get("kind")) == "Custom" and r.get("name")]
    con.executemany("INSERT INTO rule VALUES (?,?,?,?,?,?)", [
        (str(r["name"]), str((r.get("properties") or {}).get("classificationName") or ""),
         str((r.get("properties") or {}).get("ruleStatus") or ""), (r.get("properties") or {}).get("minimumPercentageMatch"),
         json.dumps(patterns(r, "columnPatterns")), json.dumps(patterns(r, "dataPatterns"))) for r in custom]) if custom else None
    con.execute("CREATE TABLE ruleset (name VARCHAR, kind VARCHAR, type VARCHAR)")
    con.execute("CREATE TABLE includes (ruleset VARCHAR, rule VARCHAR)")
    for rs in rulesets:
        props = rs.get("properties") or {}
        con.execute("INSERT INTO ruleset VALUES (?,?,?)", [str(rs.get("name") or ""), str(rs.get("kind") or ""),
                                                           str(rs.get("scanRulesetType") or "Custom")])
        for rn in props.get("includedCustomClassificationRuleNames") or []:
            con.execute("INSERT INTO includes VALUES (?,?)", [str(rs.get("name") or ""), str(rn)])
    con.execute("CREATE TABLE scan (source VARCHAR, source_kind VARCHAR, name VARCHAR, ruleset VARCHAR, ruleset_type VARCHAR)")
    for src, kind, sc in scans:
        props = sc.get("properties") or {}
        con.execute("INSERT INTO scan VALUES (?,?,?,?,?)", [src, kind, str(sc.get("name") or ""),
                                                            str(props.get("scanRulesetName") or ""),
                                                            str(props.get("scanRulesetType") or "")])
    print(f"Export: {folder}")
    print(f"Read {len(custom)} custom classification rules ({len(rules) - len(custom)} system), {len(rulesets)} scan rule sets, "
          f"{len(sources)} data sources, {len(scans)} scans")

    def add(code, q):
        findings.extend((code, *r) for r in con.execute(q).fetchall())
    add("UNKNOWN_RULESET", """SELECT 'scan ' || source || '/' || name, 'uses custom rule set ' || ruleset || ', which is not in the export'
        FROM scan WHERE ruleset_type = 'Custom' AND ruleset NOT IN (SELECT name FROM ruleset) ORDER BY 1""")
    add("UNKNOWN_RULE", """SELECT 'rule set ' || ruleset, 'includes ' || rule || ', which is not in the export'
        FROM includes WHERE rule NOT IN (SELECT name FROM rule) ORDER BY 1, 2""")
    add("DISABLED_RULE_IN_USE", """SELECT 'rule set ' || i.ruleset, 'includes ' || i.rule || ', which is disabled'
        FROM includes i JOIN rule r ON r.name = i.rule WHERE lower(r.status) = 'disabled' ORDER BY 1, 2""")
    add("UNUSED_RULE", """SELECT 'rule ' || name, 'enabled, but in no scan rule set' FROM rule
        WHERE lower(status) <> 'disabled' AND name NOT IN (SELECT rule FROM includes) ORDER BY 1""")

    compiled = {}
    for r in custom:
        if not patterns(r, "columnPatterns") and not patterns(r, "dataPatterns"):
            findings.append(("BAD_PATTERN", f"rule {r['name']}", "no regular expression pattern at all, so the rule can never match"))
        for key in ("columnPatterns", "dataPatterns"):
            for pat in patterns(r, key):
                try:
                    compiled[(r["name"], key, pat)] = re.compile(pat)
                except re.error as e:
                    findings.append(("BAD_PATTERN", f"rule {r['name']}", f"{key[:-8]} pattern '{pat}': {e}"))

    sample_rows = []
    sp = folder / "samples.csv"
    if sp.is_file():
        with open(sp, newline="", encoding="utf-8-sig") as fh:
            rd = csv.DictReader(fh)
            if not {"rule", "column_name", "value", "expect"} <= set(rd.fieldnames or []):
                stop("samples.csv needs the columns rule, column_name, value and expect")
            sample_rows = [{k: (v or "") for k, v in row.items() if k} for row in rd]
    by_name = {r["name"]: r for r in custom}
    results = []
    for s in sample_rows:
        r = by_name.get(s["rule"].strip())
        expect = s["expect"].strip().lower()
        if r is None or expect not in ("match", "no match"):
            findings.append(("SAMPLE_FAILS", f"sample for {s['rule']!r}",
                             "names no custom rule in the export" if r is None else f"expect is {s['expect']!r}, not match or no match"))
            continue
        cols = [compiled.get((r["name"], "columnPatterns", p)) for p in patterns(r, "columnPatterns")]
        data = [compiled.get((r["name"], "dataPatterns", p)) for p in patterns(r, "dataPatterns")]
        if None in cols or None in data or not (cols or data):
            continue                                   # a bad or missing pattern is already a finding
        col_ok = not cols or any(c.search(s["column_name"]) for c in cols)
        data_ok = not data or any(d.search(s["value"]) for d in data)
        got = "match" if col_ok and data_ok else "no match"
        results.append((r["name"], s["column_name"], s["value"], expect, got))
        if got != expect:
            findings.append(("SAMPLE_FAILS", f"rule {r['name']}",
                             f"{s['value']!r} in column {s['column_name']!r}: expected {expect}, got {got}"))
    tested = {n for n, _, _, e, _ in results if e == "match"} | {
        s["rule"].strip() for s in sample_rows if s["expect"].strip().lower() == "match"}
    for (name,) in con.execute("SELECT DISTINCT rule FROM includes WHERE rule IN (SELECT name FROM rule) ORDER BY 1").fetchall():
        if name not in tested:
            findings.append(("NO_SAMPLE", f"rule {name}", "in use, with no test value that should match"))

    print()
    print(f"  {'Custom classification rule':<28} {'Status':<9} {'Rule sets':>9} {'Scans':>6} {'Tests':>6}")
    rows = con.execute("""SELECT r.name, r.classification, r.status, r.threshold, r.column_patterns, r.data_patterns,
            (SELECT string_agg(i.ruleset, '; ' ORDER BY i.ruleset) FROM includes i WHERE i.rule = r.name),
            (SELECT count(*) FROM scan s JOIN includes i ON i.ruleset = s.ruleset AND s.ruleset_type = 'Custom' WHERE i.rule = r.name)
        FROM rule r ORDER BY r.name""").fetchall()
    for name, cls, st, thr, cp, dp, used, nscan in rows:
        ntest = sum(1 for x in results if x[0] == name)
        print(f"  {name[:28]:<28} {st:<9} {len(used.split('; ')) if used else 0:>9} {nscan:>6} {ntest:>6}")
    print(f"Scans by rule set type: " + ", ".join(f"{t or '(none)'} {n}" for t, n in con.execute(
        "SELECT ruleset_type, count(*) FROM scan GROUP BY 1 ORDER BY 1").fetchall()))
    print(f"Test values: {len(results)} run, {sum(1 for x in results if x[3] == x[4])} gave the expected result")
    print()
    print("Checks")
    for code, text in CHECKS:
        k = sum(1 for f in findings if f[0] == code)
        print(f"  {'PASS' if k == 0 else 'FAIL'}  {text}" + (f"  ({k})" if k else ""))
    for code, where, detail in findings:
        print(f"  {code:<21} {where}: {detail}")

    out = pathlib.Path(out)
    out.mkdir(parents=True, exist_ok=True)
    suffix = f"-{tag}" if tag else ""
    safe = lambda v: "'" + str(v) if str(v)[:1] and str(v)[0] in "=+-@" else ("" if v is None else v)
    with open(out / f"rules{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["rule", "classification", "status", "minimum_percentage_match", "column_patterns", "data_patterns",
                    "rule_sets", "scans", "test_values"])
        for name, cls, st, thr, cp, dp, used, nscan in rows:
            w.writerow([safe(x) for x in (name, cls, st, "" if thr is None else f"{thr:g}", cp, dp, used or "", nscan,
                                          sum(1 for x in results if x[0] == name))])
    with open(out / f"findings{suffix}.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.writer(fh)
        w.writerow(["check", "where", "detail"])
        w.writerows([[safe(x) for x in f] for f in findings])
    try:
        where = os.path.relpath(out)
    except ValueError:
        where = str(out)
    print()
    print(f"Wrote rules{suffix}.csv ({len(rows)} rules) and findings{suffix}.csv ({len(findings)} rows) to {where}")
    print(f"RESULT: {len(findings)} findings." + (" Fix them in Purview or in samples.csv, export again and re-run."
                                                 if findings else " Every rule is saved, in use and tested."))
    return len(findings)


# ---------------------------------------------------------------------------------------------
def make_demo(folder, plant):
    folder = pathlib.Path(folder)
    marker = folder / ".made-by-make-demo"
    if folder.exists() and any(folder.rglob("*.json")) and not marker.exists():
        sys.exit(f"{folder} already holds JSON files that --make-demo did not write; choose a new folder.")
    folder.mkdir(parents=True, exist_ok=True)
    for old in folder.rglob("*.json"):
        old.unlink()
    marker.write_text("Written by check_scan_rules.py --make-demo: generated records, not an export.\n")

    def w(path, doc):
        f = folder / path
        f.parent.mkdir(parents=True, exist_ok=True)
        f.write_text(json.dumps(doc, indent=1))

    def rule(name, cls, cols, data, status="Enabled", thr=60):
        return {"name": name, "kind": "Custom", "properties": {
            "classificationName": cls, "ruleStatus": status, "minimumPercentageMatch": thr, "classificationAction": "Keep",
            "description": f"Invented rule for {cls}.", "columnPatterns": [{"kind": "Regex", "pattern": c} for c in cols],
            "dataPatterns": [{"kind": "Regex", "pattern": d} for d in data]}}
    rules = [rule("EMPLOYEE_ID", "EXAMPLE.EMPLOYEE_ID", [r"emp.*id"], [r"^E\d{6}$"]),
             rule("POLICY_NUMBER", "EXAMPLE.POLICY_NUMBER", [r"polic(y|ies)"], [r"^POL-\d{8}$"]),
             rule("COST_CENTER", "EXAMPLE.COST_CENTER", [r"cost_?cent(er|re)"], [r"^CC\d{4}$"]),
             rule("PROJECT_CODE", "EXAMPLE.PROJECT_CODE", [], [r"^PRJ-[A-Z]{3}-\d{3}$"]),
             rule("SUPPLIER_CODE", "EXAMPLE.SUPPLIER_CODE", [r"supplier|vendor"], [r"^SUP\d{5}$"])]
    system = [{"name": n, "kind": "System", "properties": {"classificationName": "MICROSOFT." + n}}
              for n in ("EMAIL_ADDRESS", "US_PHONE_NUMBER", "CREDIT_CARD_NUMBER")]
    rulesets = [{"name": "sql-finance-rules", "kind": "AzureSqlDatabase", "scanRulesetType": "Custom", "properties": {
                    "includedCustomClassificationRuleNames": ["EMPLOYEE_ID", "COST_CENTER", "SUPPLIER_CODE"],
                    "excludedSystemClassifications": ["MICROSOFT.US_PHONE_NUMBER"]}},
                {"name": "lake-claims-rules", "kind": "AdlsGen2", "scanRulesetType": "Custom", "properties": {
                    "includedCustomClassificationRuleNames": ["POLICY_NUMBER", "PROJECT_CODE"],
                    "excludedSystemClassifications": []}}]
    sources = [{"name": "sql-finance", "kind": "AzureSqlDatabase"}, {"name": "lake-claims", "kind": "AdlsGen2"},
               {"name": "pbi-tenant", "kind": "PowerBI"}]
    scans = {"sql-finance": [("weekly-full", "sql-finance-rules", "Custom"), ("daily-ledger", "sql-finance-rules", "Custom")],
             "lake-claims": [("raw-zone", "lake-claims-rules", "Custom")],
             "pbi-tenant": [("tenant-metadata", "PowerBI", "System")]}
    samples = [("EMPLOYEE_ID", "employee_id", "E104233", "match"), ("EMPLOYEE_ID", "employee_id", "E10423", "no match"),
               ("POLICY_NUMBER", "policy_no", "POL-20261121", "match"), ("POLICY_NUMBER", "policy_no", "POL-2026", "no match"),
               ("COST_CENTER", "cost_center", "CC4410", "match"), ("COST_CENTER", "region", "CC4410", "no match"),
               ("PROJECT_CODE", "project", "PRJ-FIN-042", "match"), ("PROJECT_CODE", "project", "PRJ-42", "no match"),
               ("SUPPLIER_CODE", "vendor_code", "SUP20931", "match"), ("SUPPLIER_CODE", "vendor_code", "SUP-20931", "no match")]
    planted = []
    if plant:
        rules.append(rule("TAX_REGION", "EXAMPLE.TAX_REGION", [r"tax_?region"], [r"^TR-\d{2}$"]))
        planted.append("UNUSED_RULE")
        rules[2]["properties"]["ruleStatus"] = "Disabled"; planted.append("DISABLED_RULE_IN_USE")
        rules[3]["properties"]["dataPatterns"] = [{"kind": "Regex", "pattern": r"^PRJ-([A-Z]{3}-\d{3}$"}]; planted.append("BAD_PATTERN")
        rulesets[1]["properties"]["includedCustomClassificationRuleNames"].append("CLAIM_ID"); planted.append("UNKNOWN_RULE")
        scans["lake-claims"].append(("curated-zone", "lake-claims-rules-v2", "Custom")); planted.append("UNKNOWN_RULESET")
        samples = [s for s in samples if s[0] != "POLICY_NUMBER"]; planted.append("NO_SAMPLE")
        samples[0] = ("EMPLOYEE_ID", "employee_id", "E-104233", "match"); planted.append("SAMPLE_FAILS")
    w("classificationrules/page-001.json", {"value": rules[:4], "nextLink": "https://example.com/scan/classificationrules?page=2"})
    w("classificationrules/page-002.json", {"value": rules[4:] + system, "count": len(rules) + len(system)})
    w("scanrulesets/page-001.json", {"value": rulesets})
    w("datasources/page-001.json", {"value": sources})
    for src, items in scans.items():
        val = [{"name": n, "kind": "Custom", "properties": {"scanRulesetName": rs, "scanRulesetType": t}} for n, rs, t in items]
        if plant and src == "pbi-tenant":
            w(f"scans/{src}/page-001.json", {"value": val, "nextLink": "https://example.com/scan/datasources/pbi-tenant/scans?page=2"})
            planted.append("PAGES_MISSING")
        else:
            w(f"scans/{src}/page-001.json", {"value": val})
    with open(folder / "samples.csv", "w", newline="") as fh:
        cw = csv.writer(fh)
        cw.writerow(["rule", "column_name", "value", "expect"])
        cw.writerows(samples)
    print(f"Wrote a generated Scanning API export to {folder}: {len(rules)} custom and {len(system)} system classification "
          f"rules, {len(rulesets)} scan rule sets, {len(sources)} data sources, {sum(len(v) for v in scans.values())} scans, "
          f"{len(samples)} test values")
    if plant:
        assert set(planted) == {c for c, _ in CHECKS}, planted
        print("Planted one error for each check: " + ", ".join(sorted(planted)))


def main():
    ap = argparse.ArgumentParser(description="Check Purview scan rule sets and custom classification rules.")
    ap.add_argument("export", nargs="?")
    ap.add_argument("--out", default=str(pathlib.Path(__file__).resolve().parent))
    ap.add_argument("--make-demo", metavar="FOLDER")
    ap.add_argument("--plant-errors", action="store_true")
    ap.add_argument("--tag", default="")
    a = ap.parse_args()
    if a.make_demo:
        make_demo(a.make_demo, a.plant_errors)
        return 0
    if not a.export:
        ap.error("give the export folder, or --make-demo FOLDER")
    print(f"Python {sys.version.split()[0]} duckdb {duckdb.__version__}")
    check(a.export, a.out, a.tag)
    return 0


if __name__ == "__main__":
    sys.exit(main())
