Legacy Code Characterization skill for Codex

Gives the Codex app a display name, a one-line description and a starter prompt for this skill.

interface:
  display_name: "Legacy Code Characterization"
  short_description: "Pin what old code does before you touch it"
  default_prompt: "$legacy-code-characterization I need to change this 15-year-old billing script that has no tests, get it under test first"

#!/usr/bin/env python3
"""Golden-master (approval) testing for any command-line entry point.

Record what legacy code does today for a set of inputs, then verify that later
changes keep the output identical. Captures stdout, stderr and exit code.

  python3 golden_master.py record --cmd "python3 legacy/report.py {input}" \
      --inputs "fixtures/*.csv" --out golden/ --runs 2
  python3 golden_master.py verify --cmd "python3 legacy/report.py {input}" \
      --inputs "fixtures/*.csv" --out golden/

Options:
  --cmd        command template; {input} is replaced by the quoted input path.
               Without {input} the path is appended as the last argument.
  --inputs     glob (repeatable); recursive ** is supported
  --stdin      feed the input file on stdin instead of passing its path
  --scrub      REGEX or REGEX=>REPLACEMENT applied to stdout and stderr before
               storing or comparing (repeatable); use for timestamps, PIDs, temp paths
  --runs N     record only: run each input N times and refuse to record if the
               outputs differ (detects nondeterminism you must pin first)
  --timeout S  per-run timeout in seconds (default 60)

Exit status: 0 all match / recorded, 1 mismatch, missing golden or nondeterminism,
2 usage error. Standard library only (Python 3.10+).
"""
from __future__ import annotations

import argparse
import difflib
import glob
import hashlib
import json
import os
import re
import shlex
import subprocess
import sys


def parse_scrubs(items: list[str]) -> list[tuple[re.Pattern[str], str]]:
    rules = []
    for item in items:
        pattern, sep, repl = item.partition("=>")
        try:
            rules.append((re.compile(pattern), repl if sep else "<SCRUBBED>"))
        except re.error as exc:
            sys.exit(f"error: bad --scrub regex {pattern!r}: {exc}")
    return rules


def scrub(text: str, rules) -> str:
    for rx, repl in rules:
        text = rx.sub(repl, text)
    return text


def golden_name(path: str) -> str:
    rel = os.path.normpath(path).replace(os.sep, "__")
    safe = re.sub(r"[^A-Za-z0-9._-]", "_", rel)
    if len(safe) > 120:
        safe = safe[:100] + "-" + hashlib.sha1(rel.encode()).hexdigest()[:12]
    return safe + ".golden.json"


def run_one(cmd_tpl: str, path: str, use_stdin: bool, timeout: float, rules) -> dict:
    if use_stdin:
        argv = shlex.split(cmd_tpl.replace("{input}", ""))
    elif "{input}" in cmd_tpl:
        argv = shlex.split(cmd_tpl.replace("{input}", shlex.quote(path)))
    else:
        argv = shlex.split(cmd_tpl) + [path]
    stdin_data = None
    if use_stdin:
        with open(path, "rb") as fh:
            stdin_data = fh.read()
    try:
        proc = subprocess.run(argv, input=stdin_data, capture_output=True, timeout=timeout)
    except FileNotFoundError:
        sys.exit(f"error: command not found: {argv[0]}")
    except subprocess.TimeoutExpired:
        return {"exit_code": "timeout", "stdout": "", "stderr": f"timed out after {timeout}s"}
    decode = lambda b: b.decode("utf-8", errors="backslashreplace")
    return {
        "exit_code": proc.returncode,
        "stdout": scrub(decode(proc.stdout), rules),
        "stderr": scrub(decode(proc.stderr), rules),
    }


def show_diff(label: str, old: str, new: str, limit: int = 40) -> None:
    lines = list(difflib.unified_diff(old.splitlines(), new.splitlines(),
                                      f"golden/{label}", f"current/{label}", lineterm=""))
    for line in lines[:limit]:
        print("    " + line)
    if len(lines) > limit:
        print(f"    ... {len(lines) - limit} more diff lines")


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("mode", choices=["record", "verify"])
    ap.add_argument("--cmd", required=True)
    ap.add_argument("--inputs", action="append", required=True)
    ap.add_argument("--out", required=True, help="directory holding golden files")
    ap.add_argument("--stdin", action="store_true")
    ap.add_argument("--scrub", action="append", default=[])
    ap.add_argument("--runs", type=int, default=1)
    ap.add_argument("--timeout", type=float, default=60.0)
    args = ap.parse_args()

    paths = sorted({p for g in args.inputs for p in glob.glob(g, recursive=True) if os.path.isfile(p)})
    if not paths:
        print("error: --inputs matched no files", file=sys.stderr)
        return 2
    rules = parse_scrubs(args.scrub)
    if args.mode == "record":
        os.makedirs(args.out, exist_ok=True)

    problems = 0
    for path in paths:
        gfile = os.path.join(args.out, golden_name(path))
        result = run_one(args.cmd, path, args.stdin, args.timeout, rules)

        if args.mode == "record":
            unstable = False
            for _ in range(max(args.runs, 1) - 1):
                again = run_one(args.cmd, path, args.stdin, args.timeout, rules)
                if again != result:
                    unstable = True
                    print(f"UNSTABLE {path}: output differs between runs; pin time/random/order or add --scrub")
                    for key in ("stdout", "stderr"):
                        if again[key] != result[key]:
                            show_diff(key, result[key], again[key])
                    break
            if unstable:
                problems += 1
                continue
            record = {"input": path, "cmd": args.cmd, **result}
            with open(gfile, "w", encoding="utf-8") as fh:
                json.dump(record, fh, indent=2, ensure_ascii=False)
                fh.write("\n")
            print(f"recorded {path} (exit {result['exit_code']}, {len(result['stdout'])} bytes stdout)")
            continue

        if not os.path.exists(gfile):
            print(f"MISSING  {path}: no golden file; run record first")
            problems += 1
            continue
        with open(gfile, encoding="utf-8") as fh:
            golden = json.load(fh)
        diffs = [k for k in ("exit_code", "stdout", "stderr") if golden.get(k) != result[k]]
        if not diffs:
            print(f"ok       {path}")
            continue
        problems += 1
        print(f"CHANGED  {path}: {', '.join(diffs)}")
        if "exit_code" in diffs:
            print(f"    exit code {golden.get('exit_code')} -> {result['exit_code']}")
        for key in ("stdout", "stderr"):
            if key in diffs:
                show_diff(key, golden.get(key, ""), result[key])

    verb = "recorded" if args.mode == "record" else "verified"
    print(f"\n{len(paths) - problems}/{len(paths)} inputs {verb} cleanly")
    return 1 if problems else 0


if __name__ == "__main__":
    sys.exit(main())