#!/usr/bin/env python3 """NYC Local Law 144 and Claude in hiring: sort your Claude use cases by the City's three tests, and work out selection rates and impact ratios by the City's formulas, in DuckDB. python3 nyc_ll144_claude_check.py # clean generated records python3 nyc_ll144_claude_check.py --plant # generated records with planted faults python3 nyc_ll144_claude_check.py --wrong-denominator # clean records, impact ratio divided by the wrong rate python3 nyc_ll144_claude_check.py --strict # exit 1 when anything needs follow-up python3 nyc_ll144_claude_check.py --work-dir some/folder # where generated files and results are written Builds two lists, each printed with its filters, row count and a SHA-256 of its rows: use-case sort every row of your Claude use-case register: is it a hiring or promotion use, does Claude's output rely alone, weigh most or overrule (the three tests in 6 RCNY 5-300), and is it used "in the city" as the City's FAQ defines it. A use that may meet a test, or whose answer is empty or not recognized, is FOR LEGAL TO CHECK. A use is never called an AEDT and never called exempt. impact ratios selection rate and impact ratio for each category, by sex, by race/ethnicity and by both together, the way 6 RCNY 5-301(b) defines them, with the number of people in an unknown category, and the 2% option of 5-301(d) shown beside the full calculation Before it builds them it pins the City's rule (size and SHA-256), checks its own formulas against the rule's worked example, and runs input checks; the lists are marked INCOMPLETE if one fails. It prints numbers only: it does not say a tool passes or fails, and it applies no threshold, because Local Law 144 sets none. This is NOT a bias audit. Local Law 144 requires an independent auditor, within one year before use, a published summary and notice to candidates. Legal decides whether a use is covered. The generated records are NOT a real organization's data. The rule: https://rules.cityofnewyork.us/wp-content/uploads/2023/04/DCWP-NOA-for-Use-of-Automated-Employment-Decisionmaking-Tools-2.pdf """ import csv import hashlib import http.client import pathlib import random import sys import urllib.request import duckdb # ---- CHANGE THESE FOR YOUR OWN DATA --------------------------------------------------------- REGISTER_FILE = None # your Claude use-case register (CSV with a header row); None = generate SELECTIONS_FILE = None # your selection records, one row per candidate or employee considered (CSV with a header row); None = generate # the column of YOUR register and YOUR selection file that holds each field REG_COLS = {"use_case_id": "use_case_id", "use_case": "use_case", "decision": "decision", "output_use": "output_use", "nyc": "nyc_affected", "owner": "owner"} SEL_COLS = {"candidate_id": "candidate_id", "sex": "sex", "race_ethnicity": "race_ethnicity", "selected": "selected"} # YOUR answers (lower case, no spaces at the ends) -> the meaning the script gives them. An answer in your files that is # in none of these lists is counted and fails an input check; it is never guessed. DECISION_WORDS = {"hiring": "hiring", "promotion": "promotion", "neither": "neither"} # how the use's output enters the decision: i, ii, iii are the three tests of 6 RCNY 5-300; none is outside them as recorded OUTPUT_USE_WORDS = { "decides alone": "i", # test i: relies solely on a simplified output, no other factors considered "weighted most": "ii", # test ii: one of a set of criteria, weighted more than any other "overrules other factors": "iii", # test iii: overrules conclusions from other factors, including a person's decision "weighted equal or less": "none", # one criterion, weighted no more than another "not a criterion": "none", # read for background, no weight in the decision "no score, tag, recommendation or ranking": "none", # no simplified output at all "translates or transcribes": "none", # the rule excludes tools that translate or transcribe existing text } # the use is "in the city" (the City's FAQ): the job's location is a New York City office, at least part time; or the # job is fully remote but the location tied to it is a New York City office; or the employment agency using the tool is # in New York City (or, if it is outside the city, one of the first two is true). Any of the three is yes. NYC_WORDS = {"yes": "yes", "no": "no"} # None as a meaning: the person declined to self-identify; the row goes to the unknown count of 5-301(b)(5), never to a category SEX_WORDS = {"male": "Male", "female": "Female", "i do not wish to self-identify": None} RACE_WORDS = { # the EEO-1 categories, as the City's rule lists them "hispanic or latino": "Hispanic or Latino", "white": "White (Not Hispanic or Latino)", "black or african american": "Black or African American (Not Hispanic or Latino)", "native hawaiian or pacific islander": "Native Hawaiian or Pacific Islander (Not Hispanic or Latino)", "asian": "Asian (Not Hispanic or Latino)", "native american or alaska native": "Native American or Alaska Native (Not Hispanic or Latino)", "two or more races": "Two or More Races (Not Hispanic or Latino)", "i do not wish to self-identify": None, } SELECTED_WORDS = {"1": 1, "yes": 1, "0": 0, "no": 0} # selected to move forward (1) or not (0) # ---------------------------------------------------------------------------------------------- MIN_SHARE = 0.02 # 6 RCNY 5-301(d): an independent auditor may exclude a category under 2% of the data from the impact ratio RULE = ("6 RCNY 5-300 to 5-304 as adopted (DCWP, effective 2023-07-05)", "https://rules.cityofnewyork.us/wp-content/uploads/2023/04/DCWP-NOA-for-Use-of-Automated-Employment-Decisionmaking-Tools-2.pdf", 462675, "4ecc4463b29440f9b876b0b7319faf2d4c4710b8ad28db9cfa8ae15a123ece2c") # the worked example printed in 6 RCNY 5-301(b): category, applicants, selected, published selection rate (%), published impact ratio RULE_EXAMPLE = { "sex": [("Male", 1390, 667, "48", "1.00"), ("Female", 1181, 555, "47", "0.979")], "race/ethnicity": [("Hispanic or Latino", 408, 204, "50", "0.97"), ("White", 797, 412, "52", "1.00"), ("Black or African American", 390, 170, "44", "0.84"), ("Native Hawaiian or Pacific Islander", 119, 52, "44", "0.85"), ("Asian", 616, 302, "49", "0.95"), ("Native American or Alaska Native", 41, 18, "44", "0.85"), ("Two or More Races", 213, 96, "45", "0.87")], } USER_AGENT = ("nyc_ll144_claude_check.py (https://www.datameaning.com/guides/nyc-local-law-144-and-claude-in-hiring/; " "info@datameaning.com)") TESTS = {"i": "relies solely on a simplified output", "ii": "weights it more than any other criterion", "iii": "uses it to overrule other factors"} def qcol(name): return '"' + name.replace('"', '""') + '"' def generate(mode, gen_dir): """Seeded stand-in for an organization's register and selection records. mode: clean or plant.""" plant = mode == "plant" rnd = random.Random(225) gen_dir.mkdir(parents=True, exist_ok=True) # category weight and base selection rate: invented for the demo, not a measurement of anyone races = [("Hispanic or Latino", 0.17, 0.30), ("White", 0.38, 0.33), ("Black or African American", 0.19, 0.27), ("Native Hawaiian or Pacific Islander", 0.03, 0.28), ("Asian", 0.20, 0.31), ("Native American or Alaska Native", 0.01, 0.60), ("Two or More Races", 0.02, 0.35)] rows = [] for n in range(1, 2001): race = rnd.choices(races, weights=[r[1] for r in races])[0] sex = rnd.choices(["Male", "Female"], weights=[0.52, 0.48])[0] pick = rnd.random() < race[2] + (0.02 if sex == "Male" else -0.01) blank_sex, blank_race = rnd.random() < 0.04, rnd.random() < 0.03 declined = "I do not wish to self-identify" if n % 2 == 0 else "" # half the blanks are a stated decline rows.append([f"cand_{n:04d}", declined if blank_sex else sex, declined if blank_race else race[0], "1" if pick else "0"]) if plant: rows[9][2] = "Hisp." # planted: a race answer that is in no list rows[19][3] = "maybe" # planted: a selected value that is not 0 or 1 rows.append(list(rows[29])) # planted: a duplicated candidate rows.append(["", "Female", "Asian", "0"]) # planted: a row with no candidate id with open(gen_dir / "selections.csv", "w", newline="") as f: w = csv.writer(f) w.writerow(["candidate_id", "sex", "race_ethnicity", "selected"]) w.writerows(rows) reg = [["UC-101", "Summarize resumes for recruiters", "hiring", "no score, tag, recommendation or ranking", "yes", "recruiting.lead@example.test"], ["UC-102", "Rank resumes against a job description", "hiring", "weighted most", "yes", "recruiting.lead@example.test"], ["UC-103", "Draft job postings", "neither", "no score, tag, recommendation or ranking", "yes", "hr.ops@example.test"], ["UC-104", "Translate applications from other languages", "hiring", "translates or transcribes", "yes", "recruiting.lead@example.test"], ["UC-105", "Suggest a promotion shortlist", "promotion", "weighted equal or less", "yes", "people.ops@example.test"], ["UC-106", "Answer staff questions about HR policy", "neither", "not a criterion", "yes", "hr.ops@example.test"], ["UC-107", "Screen applicants for a role open only outside New York City", "hiring", "decides alone", "no", "recruiting.lead@example.test"]] if plant: reg += [["UC-108", "Score interview notes", "hiring", "", "yes", "recruiting.lead@example.test"], # planted: no answer on output use [" uc-109 ", "Rate candidates on culture fit", "hiring", " Weighted Most ", "Yes", "hr.lead@example.test"], # planted: other case and spaces; still read ["UC-110", "Pick finalists", "hiring", "overrides", "yes", "hr.lead@example.test"], # planted: an answer in no list ["UC-111", "Choose who is put forward for promotion", "promotion", "decides alone", "", "people.ops@example.test"], # planted: New York City not recorded ["UC-102", "Rank resumes against a job description", "hiring", "weighted most", "yes", "recruiting.lead@example.test"], # planted: duplicate id ["", "Chat helper for recruiters", "", "", "", "hr.ops@example.test"]] # planted: no id, no answers with open(gen_dir / "register.csv", "w", newline="") as f: w = csv.writer(f) w.writerow(["use_case_id", "use_case", "decision", "output_use", "nyc_affected", "owner"]) w.writerows(reg) return str(gen_dir / "register.csv"), str(gen_dir / "selections.csv") def fingerprint(rows): body = "\n".join("|".join("" if v is None else str(v) for v in r) for r in rows) return hashlib.sha256(body.encode()).hexdigest() def population(out_dir, name, control, filters, rows, header, complete): out_dir.mkdir(parents=True, exist_ok=True) with open(out_dir / f"{name}.csv", "w", newline="") as f: w = csv.writer(f) w.writerow(header) w.writerows(rows) print(f"\n== {control}: {name}" + ("" if complete else " ** INCOMPLETE: see input checks **")) print(f"filters : {filters}") print(f"rows : {len(rows)}") print(f"sha256 : {fingerprint(rows)}") def pin_rule(): """Download the rule once, print its size, Last-Modified and SHA-256, return 1 when it is not the pinned file.""" label, url, size, sha = RULE try: req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) with urllib.request.urlopen(req, timeout=60) as r: body, modified = r.read(), r.headers.get("Last-Modified", "not sent") except (OSError, http.client.HTTPException) as e: # a cut-off download is a failed pin, not a crash print(f" {label}: could not be fetched ({e.__class__.__name__}); the pin cannot be checked") return 1 got = hashlib.sha256(body).hexdigest() same = len(body) == size and got == sha print(f" {label}: {len(body)} bytes, Last-Modified {modified}, sha256 {got}" + ("" if same else " ** DIFFERENT FROM THE PIN **")) return 0 if same else 1 def impact_ratios(rows, wrong=False): """6 RCNY 5-300: impact ratio = the selection rate of a category / the selection rate of the most selected category. rows: (category, applicants, selected). Returns {category: (selection rate, impact ratio or None)}. wrong=True divides by the LOWEST rate instead, only to show the self-check below catching it.""" rates = {c: s / n for c, n, s in rows if n > 0} ref = (min if wrong else max)(rates.values()) if rates else 0 return {c: (r, r / ref if ref > 0 else None) for c, r in rates.items()} def check_formulas(wrong): """Recompute the City's own worked example and compare each figure with the rule's printed one.""" off = [] for dim, table in RULE_EXAMPLE.items(): got = impact_ratios([(c, n, s) for c, n, s, _, _ in table], wrong) for c, n, s, pct, ratio in table: rate, ir = got[c] if f"{rate * 100:.0f}" != pct or ir is None or f"{ir:.{len(ratio.split('.')[1])}f}" != ratio: off.append(f"{dim}: {c}") return off def ratio_text(v): return "n/a" if v is None else f"{v:.3f}" def main(): strict, wrong = "--strict" in sys.argv, "--wrong-denominator" in sys.argv mode = "plant" if "--plant" in sys.argv else "clean" base = pathlib.Path(".") if "--work-dir" in sys.argv: at = sys.argv.index("--work-dir") + 1 if at >= len(sys.argv) or sys.argv[at].startswith("--"): sys.exit("usage: --work-dir needs a folder after it, for example --work-dir some/folder") base = pathlib.Path(sys.argv[at]) out_dir, gen_dir = base / "evidence-out", base / "generated" reg_file, sel_file = REGISTER_FILE, SELECTIONS_FILE for name, words in [("DECISION_WORDS", DECISION_WORDS), ("OUTPUT_USE_WORDS", OUTPUT_USE_WORDS), ("NYC_WORDS", NYC_WORDS), ("SEX_WORDS", SEX_WORDS), ("RACE_WORDS", RACE_WORDS), ("SELECTED_WORDS", SELECTED_WORDS)]: for k in words: if k != k.strip().lower(): sys.exit(f"{name}: the answer {k!r} must be lower case with no spaces at the ends") for v in OUTPUT_USE_WORDS.values(): if v not in ("i", "ii", "iii", "none"): sys.exit(f"OUTPUT_USE_WORDS: {v!r} must be i, ii, iii or none") for v in DECISION_WORDS.values(): if v not in ("hiring", "promotion", "neither"): sys.exit(f"DECISION_WORDS: {v!r} must be hiring, promotion or neither") for v in NYC_WORDS.values(): if v not in ("yes", "no"): sys.exit(f"NYC_WORDS: {v!r} must be yes or no") for name, words in [("SEX_WORDS", SEX_WORDS), ("RACE_WORDS", RACE_WORDS)]: for v in words.values(): if v is not None and not (isinstance(v, str) and v.strip()): sys.exit(f"{name}: {v!r} must be a category name, or None for a declined answer") for v in SELECTED_WORDS.values(): if type(v) is not int or v not in (0, 1): sys.exit(f"SELECTED_WORDS: {v!r} must be 0 or 1") if reg_file is None and sel_file is None: reg_file, sel_file = generate(mode, gen_dir) print(f"input : GENERATED records (not a real organization), seed 225, mode={mode}") elif reg_file is None or sel_file is None: sys.exit("set REGISTER_FILE and SELECTIONS_FILE together") else: print("input : " + reg_file + ", " + sel_file) print("\n== The City's rule (pinned: a different size or hash means the City re-published the file)") rule_bad = pin_rule() off = check_formulas(wrong) con = duckdb.connect() # every column your settings name must exist in your files; a wrong name stops the run with the names it did find for label, path, cols in [("register", reg_file, REG_COLS), ("selection file", sel_file, SEL_COLS)]: have = {r[0].lower() for r in con.execute("DESCRIBE SELECT * FROM read_csv(?, header=true, all_varchar=true)", [path]).fetchall()} lost = [c for c in cols.values() if c.lower() not in have] if lost: sys.exit(f"{label}: no column named {', '.join(lost)} in {path}; its columns are {', '.join(sorted(have))}") for table, words in [("decision_map", DECISION_WORDS), ("output_map", OUTPUT_USE_WORDS), ("nyc_map", NYC_WORDS), ("sex_map", SEX_WORDS), ("race_map", RACE_WORDS), ("selected_map", SELECTED_WORDS)]: con.execute(f"CREATE TABLE {table}(word VARCHAR, meaning VARCHAR)") con.executemany(f"INSERT INTO {table} VALUES (?, ?)", [(k, None if v is None else str(v)) for k, v in words.items()]) word = lambda c: f"lower(trim(coalesce({qcol(c)}, '')))" con.execute(f"""CREATE TABLE register AS SELECT row_number() OVER () AS row_no, nullif(lower(trim(coalesce({qcol(REG_COLS['use_case_id'])}, ''))), '') AS use_case_id, nullif(trim(coalesce({qcol(REG_COLS['use_case_id'])}, '')), '') AS use_case_id_shown, {word(REG_COLS['decision'])} AS decision_word, {word(REG_COLS['output_use'])} AS output_word, {word(REG_COLS['nyc'])} AS nyc_word FROM read_csv(?, header=true, all_varchar=true)""", [reg_file]) con.execute(f"""CREATE TABLE selections AS SELECT row_number() OVER () AS row_no, nullif(lower(trim(coalesce({qcol(SEL_COLS['candidate_id'])}, ''))), '') AS candidate_id, {word(SEL_COLS['sex'])} AS sex_word, {word(SEL_COLS['race_ethnicity'])} AS race_word, {word(SEL_COLS['selected'])} AS selected_word FROM read_csv(?, header=true, all_varchar=true)""", [sel_file]) con.execute("""CREATE TABLE s AS SELECT t.row_no, t.candidate_id, t.sex_word, t.race_word, t.selected_word, sm.meaning AS sex, rm.meaning AS race, try_cast(lm.meaning AS INTEGER) AS selected, sm.word IS NOT NULL AS sex_matched, rm.word IS NOT NULL AS race_matched FROM selections t LEFT JOIN sex_map sm ON sm.word = t.sex_word LEFT JOIN race_map rm ON rm.word = t.race_word LEFT JOIN selected_map lm ON lm.word = t.selected_word""") q = lambda s: con.execute(s).fetchone()[0] n_uc, n_sel = q("SELECT count(*) FROM register"), q("SELECT count(*) FROM selections") # ---- input checks: each says which planted fault trips it unread = lambda col, tbl, w: q(f"SELECT count(*) FROM {tbl} WHERE {col} <> '' AND {col} NOT IN (SELECT word FROM {w})") parts = [("sex", q("SELECT count(*) FROM s WHERE NOT sex_matched AND sex_word <> ''")), ("race/ethnicity", q("SELECT count(*) FROM s WHERE NOT race_matched AND race_word <> ''"))] checks = [ ("City rule that is not the pinned file", rule_bad, "the City re-published the rule, or the fetch failed"), ("the City's worked example recomputed to different figures" + (": " + ", ".join(off[:3]) if off else ""), len(off), "an impact ratio divided by anything but the most selected category's rate (run with --wrong-denominator)"), ("register ids listed more than once", q("SELECT count(*) FROM (SELECT use_case_id FROM register WHERE use_case_id IS NOT NULL GROUP BY 1 HAVING count(*) > 1)"), "a duplicated register row"), ("register rows with no use case id", q("SELECT count(*) FROM register WHERE use_case_id IS NULL"), "a register row with no id"), ("register answers in no list (decision, output use, New York City)", unread("decision_word", "register", "decision_map") + unread("output_word", "register", "output_map") + unread("nyc_word", "register", "nyc_map"), "an answer such as 'overrides' that is not one of your words"), ("selection candidates listed more than once", q("SELECT count(*) FROM (SELECT candidate_id FROM s WHERE candidate_id IS NOT NULL GROUP BY 1 HAVING count(*) > 1)"), "a duplicated candidate row"), ("selection rows with no candidate id", q("SELECT count(*) FROM s WHERE candidate_id IS NULL"), "a row with no candidate id"), ("selected values that are not in SELECTED_WORDS", q("SELECT count(*) FROM s WHERE selected IS NULL"), "a selected value such as 'maybe'"), ("sex and race/ethnicity answers in no list", sum(n for _, n in parts), "an answer such as 'Hisp.' that is not one of your words"), ] print(f"\n== Input checks (register rows: {n_uc}; selection rows: {n_sel})") ok = True for label, n, fault in checks: ok &= n == 0 print(f"[{'ok ' if n == 0 else 'FAIL'}] {label}: {n}" + ("" if n == 0 else f" (trips on: {fault})")) # a word list entry that matches nothing in your files is reported, never silently ignored for tbl, col, src, w in [("register", "decision_word", "decision", "decision_map"), ("register", "output_word", "output_use", "output_map"), ("register", "nyc_word", "nyc", "nyc_map"), ("s", "sex_word", "sex", "sex_map"), ("s", "race_word", "race/ethnicity", "race_map")]: dead = [r[0] for r in con.execute(f"SELECT word FROM {w} WHERE word NOT IN (SELECT {col} FROM {tbl}) ORDER BY 1").fetchall()] if dead: print(f"[info] {src} answers in your word list that no row uses: " + "; ".join(dead)) # ---- the use-case sort: the City's three tests, as recorded con.execute("""CREATE TABLE sorted_uses AS SELECT r.row_no, coalesce(r.use_case_id_shown, '-') AS use_case_id, CASE WHEN d.meaning IS NULL THEN 'FOR LEGAL TO CHECK' WHEN d.meaning = 'neither' THEN 'NOT A HIRING OR PROMOTION USE AS RECORDED' WHEN o.meaning IS NULL THEN 'FOR LEGAL TO CHECK' WHEN n.meaning IS NULL THEN 'FOR LEGAL TO CHECK' WHEN n.meaning = 'no' THEN 'NOT USED IN THE CITY AS RECORDED' WHEN o.meaning IN ('i', 'ii', 'iii') THEN 'FOR LEGAL TO CHECK' ELSE 'OUTSIDE THE THREE TESTS AS RECORDED' END AS status, CASE WHEN d.meaning IS NULL THEN 'decision not recorded or not recognized' WHEN d.meaning = 'neither' THEN '' WHEN o.meaning IS NULL THEN 'output use not recorded or not recognized' WHEN n.meaning IS NULL THEN 'in the city not recorded or not recognized' WHEN n.meaning = 'no' THEN '' WHEN o.meaning IN ('i', 'ii', 'iii') THEN 'may meet test ' || o.meaning ELSE '' END AS reason FROM register r LEFT JOIN decision_map d ON d.word = r.decision_word LEFT JOIN output_map o ON o.word = r.output_word LEFT JOIN nyc_map n ON n.word = r.nyc_word""") r_uc = con.execute("SELECT row_no, use_case_id, status, reason FROM sorted_uses ORDER BY row_no").fetchall() population(out_dir, "use_case_sort", "6 RCNY 5-300: the three tests", "every register row, none dropped; first applicable status; an empty or unrecognized answer is FOR LEGAL TO CHECK, never outside", r_uc, ["row_no", "use_case_id", "status", "reason"], ok) for r in r_uc: print(" " + " | ".join(str(v) for v in r)) counts = {k: sum(1 for r in r_uc if r[2] == k) for k in sorted({r[2] for r in r_uc})} print(" " + "; ".join(f"{k}: {v}" for k, v in counts.items())) print(" This sorts uses by what the register says. It does not say a use is or is not covered: Legal decides.") # ---- selection rates and impact ratios, by the rule's formulas declined = {"sex": "sex_matched AND sex IS NULL", "race": "race_matched AND race IS NULL"} dims = [("sex", "sex", "sex IS NOT NULL", declined["sex"]), ("race/ethnicity", "race", "race IS NOT NULL", declined["race"]), ("sex and race/ethnicity", "sex || ' / ' || race", "sex IS NOT NULL AND race IS NOT NULL", f"({declined['sex']}) OR ({declined['race']})")] result, accounted, exact_shares = [], True, {} for dim, label_sql, known_sql, declined_sql in dims: cats = con.execute(f"""SELECT {label_sql} AS category, count(*), sum(selected) FROM s WHERE selected IS NOT NULL AND {known_sql} GROUP BY 1 ORDER BY 1""").fetchall() unknown = q(f"SELECT count(*) FROM s WHERE selected IS NOT NULL AND NOT ({known_sql})") n_declined = q(f"SELECT count(*) FROM s WHERE selected IS NOT NULL AND NOT ({known_sql}) AND ({declined_sql})") no_sel = q("SELECT count(*) FROM s WHERE selected IS NULL") known_n = sum(c[1] for c in cats) exact_shares[dim] = sum(c[1] / known_n for c in cats) if known_n else 0 accounted &= known_n + unknown + no_sel == n_sel full = impact_ratios(cats, wrong) kept = [c for c in cats if c[1] / known_n >= MIN_SHARE] if known_n else [] less = impact_ratios(kept, wrong) print(f"\n-- {dim}: applicants in the calculation {known_n}; in an unknown category (left out, as 5-301(b)(5) asks you to count): {unknown}" + f" (of them, declined to self-identify: {n_declined})" + (f"; selection value unreadable (left out): {no_sel}" if no_sel else "")) print(" " + f"{'category':<66}{'applicants':>11}{'selected':>9}{'rate':>8}{'share':>8}{'impact ratio':>14}{'if <2% left out':>17}") for c, n, sel in cats: rate, ir = full[c] share = n / known_n if c in less: alt = ratio_text(less[c][1]) else: alt = "excluded" print(" " + f"{c:<66}{n:>11}{int(sel):>9}{rate * 100:>7.1f}%{share * 100:>7.1f}%{ratio_text(ir):>14}{alt:>17}") result.append((dim, c, n, int(sel), f"{rate * 100:.1f}%", f"{share * 100:.1f}%", ratio_text(ir), alt)) left = [(c, n, int(sel)) for c, n, sel in cats if c not in less] if left: print(" under 2% of the data, so an independent auditor may leave them out of the impact ratio; the summary must still " "give their applicants and rate: " + "; ".join(f"{c} {n} applicants, rate {full[c][0] * 100:.1f}%" for c, n, _ in left)) sums_ok = all(abs(v - 1) < 1e-9 for v in exact_shares.values()) # exact shares, not the one-decimal ones printed ok &= accounted and sums_ok print(f"\n[{'ok ' if accounted else 'FAIL'}] every selection row is in a category, in an unknown category or left out as unreadable, in each of the three tables" + ("" if accounted else " (trips on: a row counted twice or in no bucket)")) print(f"[{'ok ' if sums_ok else 'FAIL'}] shares of the data add to 100% in each of the three tables" + ("" if sums_ok else " (trips on: a category dropped from, or counted twice in, a table)")) population(out_dir, "impact_ratios", "6 RCNY 5-301(b): selection rate and impact ratio", "every category with at least one applicant; selection rate = selected / applicants; impact ratio = rate / highest rate in the table; " "unknown category left out and counted; no threshold applied", result, ["table", "category", "applicants", "selected", "selection_rate", "share_of_data", "impact_ratio", "impact_ratio_if_under_2pct_left_out"], ok) follow = sum(1 for r in r_uc if r[2] == "FOR LEGAL TO CHECK") print("\n== Result") print(f"input checks : {'all passed' if ok else 'FAILED, the lists above are not complete'}") print(f"for Legal : {follow} use(s) marked FOR LEGAL TO CHECK") print("This is not a bias audit and it does not say any tool passes or fails: Local Law 144 sets no threshold. Legal decides coverage.") return 1 if strict and (not ok or follow) else 0 if __name__ == "__main__": sys.exit(main())