Skill Evals skill for Codex

Gives the Codex app a display name, a one-line description and a starter prompt for this skill.

interface:
  display_name: "Skill Evals"
  short_description: "Prove a skill triggers when it should, and only then"
  default_prompt: "$skill-evals Does my pdf-tools skill actually trigger? Build an eval for it"

#!/usr/bin/env python3
"""Check whether a Codex skill is visible and whether it triggers on the right prompts.

Subcommands:
  catalog NAME [--input FILE] [--skill-md PATH]
      Is NAME in the skills catalog Codex sends to the model, and how much of its
      description survives? Input is `codex debug prompt-input` JSON (file or - for
      stdin); without --input the script runs `codex debug prompt-input probe`.
  runs NAME --cases CASES.json --runs-dir DIR [--threshold 0.66]
      Score `codex exec --json` logs. CASES.json is a list of
      {"id": "...", "prompt": "...", "expect": true|false, "output_regex": "optional"}.
      Logs are DIR/<id>.jsonl or DIR/<id>.<n>.jsonl (repetitions). A run counts as
      triggered when any command or tool call references NAME/SKILL.md.
Exit codes: 0 pass, 1 failures, 2 bad input.
"""
import argparse
import glob
import json
import os
import re
import subprocess
import sys


def fail_input(msg):
    print(f"error: {msg}", file=sys.stderr)
    sys.exit(2)


def description_from_skill_md(path):
    try:
        text = open(path, encoding="utf-8").read()
    except OSError as err:
        fail_input(f"cannot read {path}: {err}")
    match = re.search(r"^description:\s*(.+)$", text, re.M)
    if not match:
        return None
    return match.group(1).strip().strip('"').strip("'")


def strings(node):
    """Yield every string value in a JSON structure."""
    if isinstance(node, str):
        yield node
    elif isinstance(node, dict):
        for value in node.values():
            yield from strings(value)
    elif isinstance(node, list):
        for value in node:
            yield from strings(value)


def cmd_catalog(args):
    if args.input == "-":
        raw = sys.stdin.read()
    elif args.input:
        try:
            raw = open(args.input, encoding="utf-8").read()
        except OSError as err:
            fail_input(f"cannot read {args.input}: {err}")
    else:
        try:
            raw = subprocess.run(["codex", "debug", "prompt-input", "probe"], capture_output=True,
                                 text=True, check=True, timeout=120).stdout
        except (OSError, subprocess.SubprocessError) as err:
            fail_input(f"could not run codex debug prompt-input: {err}")
    try:
        items = json.loads(raw)
    except json.JSONDecodeError as err:
        fail_input(f"prompt input is not JSON: {err}")
    text = "\n".join(strings(items))
    entries = re.findall(r"^- ([^\n:]+): (.*?) \(file: ([^)]+)\)$", text, re.M)
    print(f"catalog entries: {len(entries)}")
    hits = [e for e in entries if e[0] == args.name or e[0].endswith(":" + args.name)]
    if not hits:
        print(f"FAIL: '{args.name}' is not in the skills catalog (wrong folder, name mismatch, disabled, or dropped for budget)")
        return 1
    status = 0
    for name, shown, path in hits:
        print(f"found: {name} -> {path}")
        print(f"shown description ({len(shown)} chars): {shown}")
        if args.skill_md:
            full = description_from_skill_md(args.skill_md)
            if full and len(shown) < len(full):
                print(f"note: truncated from {len(full)} chars; keep trigger words in the first {len(shown)} chars")
    if len(hits) > 1:
        print("warn: more than one skill with this name is installed")
    return status


def run_triggered(path, needle):
    triggered = False
    final = ""
    with open(path, encoding="utf-8") as handle:
        for line in handle:
            try:
                event = json.loads(line)
            except json.JSONDecodeError:
                continue
            item = event.get("item") or {}
            kind = item.get("type")
            if kind == "agent_message":
                final = item.get("text", "")
                continue
            if kind in ("command_execution", "mcp_tool_call", "file_change"):
                blob = json.dumps({k: item.get(k) for k in ("command", "arguments", "changes")})
                if needle in blob:
                    triggered = True
    return triggered, final


def cmd_runs(args):
    try:
        cases = json.load(open(args.cases, encoding="utf-8"))
    except (OSError, json.JSONDecodeError) as err:
        fail_input(f"cannot load cases: {err}")
    needle = f"{args.name}/SKILL.md"
    tp = fp = fn = tn = 0
    failures = 0
    print(f"{'case':28} {'expect':>6} {'rate':>7} {'out':>5}  result")
    for case in cases:
        cid = case["id"]
        files = sorted(set(glob.glob(os.path.join(args.runs_dir, f"{cid}.jsonl")))
                       | set(glob.glob(os.path.join(args.runs_dir, f"{cid}.*.jsonl"))))
        if not files:
            print(f"{cid:28} {'-':>6} {'-':>7} {'-':>5}  MISSING (no logs)")
            failures += 1
            continue
        results = [run_triggered(f, needle) for f in files]
        rate = sum(1 for t, _ in results if t) / len(results)
        expect = bool(case["expect"])
        trig_ok = rate >= args.threshold if expect else rate <= 1 - args.threshold
        out_ok = True
        pattern = case.get("output_regex")
        if pattern:
            out_ok = all(re.search(pattern, final, re.S) for _, final in results)
        majority = rate >= 0.5
        if expect and majority:
            tp += 1
        elif expect:
            fn += 1
        elif majority:
            fp += 1
        else:
            tn += 1
        ok = trig_ok and out_ok
        failures += 0 if ok else 1
        out_txt = "-" if not pattern else ("ok" if out_ok else "bad")
        print(f"{cid[:28]:28} {str(expect):>6} {rate:>6.0%} {out_txt:>5}  {'pass' if ok else 'FAIL'} ({len(files)} runs)")
    precision = tp / (tp + fp) if tp + fp else 1.0
    recall = tp / (tp + fn) if tp + fn else 1.0
    print(f"\nprecision {precision:.2f}  recall {recall:.2f}  (tp={tp} fp={fp} fn={fn} tn={tn})  failures={failures}")
    return 1 if failures else 0


def main():
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    sub = parser.add_subparsers(dest="cmd", required=True)
    cat = sub.add_parser("catalog", help="check catalog visibility")
    cat.add_argument("name")
    cat.add_argument("--input", help="prompt-input JSON file, or - for stdin")
    cat.add_argument("--skill-md", help="path to the skill's SKILL.md to measure truncation")
    runs = sub.add_parser("runs", help="score codex exec --json logs")
    runs.add_argument("name")
    runs.add_argument("--cases", required=True)
    runs.add_argument("--runs-dir", required=True)
    runs.add_argument("--threshold", type=float, default=0.66)
    args = parser.parse_args()
    return cmd_catalog(args) if args.cmd == "catalog" else cmd_runs(args)


if __name__ == "__main__":
    sys.exit(main())