Codex Output Schemas skill for Codex

Gives the Codex app a display name, a one-line description and a starter prompt for this skill.

interface:
  display_name: "Codex Output Schemas"
  short_description: "JSON results your pipeline can actually trust"
  default_prompt: "$codex-output-schemas Make codex exec return review findings as JSON our CI can gate on"

#!/usr/bin/env python3
"""Lint a JSON Schema for `codex exec --output-schema` and optionally validate output.

Usage:
  python3 check_schema.py schema.json                  # lint for strict structured output
  python3 check_schema.py schema.json --validate out.json   # also validate a result file
  codex exec ... -o out.json && python3 check_schema.py schema.json --validate out.json

Lint errors (exit 1): root not an object, objects without additionalProperties: false,
properties missing from required, unsupported composition keywords.
Warnings (exit 0): deep nesting, keywords that strict mode may ignore or reject.
Validation covers the strict subset: type (incl. null unions), properties, required,
additionalProperties, items, enum, const, anyOf, and local $ref to #/$defs or #/definitions.
Exit codes: 0 ok, 1 lint or validation errors, 2 unreadable or invalid JSON.
"""
import argparse
import json
import sys

UNSUPPORTED = ("allOf", "oneOf", "not", "if", "then", "else", "patternProperties",
               "dependentRequired", "dependentSchemas", "unevaluatedProperties")
MAYBE_IGNORED = ("default", "examples", "minProperties", "maxProperties", "contains")
MAX_DEPTH = 10


def load(path):
    try:
        with open(path, encoding="utf-8") as handle:
            return json.load(handle)
    except (OSError, json.JSONDecodeError) as err:
        print(f"error: cannot load {path}: {err}", file=sys.stderr)
        sys.exit(2)


def types_of(node):
    t = node.get("type")
    if t is None:
        return set()
    return set(t) if isinstance(t, list) else {t}


def lint(schema):
    errors, warnings = [], []

    def walk(node, path, depth):
        if not isinstance(node, dict):
            return
        for key in UNSUPPORTED:
            if key in node:
                errors.append(f"{path}: '{key}' is not supported in strict mode; use anyOf or restructure")
        for key in MAYBE_IGNORED:
            if key in node:
                warnings.append(f"{path}: '{key}' may be ignored or rejected in strict mode")
        if depth > MAX_DEPTH:
            warnings.append(f"{path}: nesting depth {depth} exceeds {MAX_DEPTH}")
        kinds = types_of(node)
        if "object" in kinds or "properties" in node:
            props = node.get("properties")
            if props is None:
                errors.append(f"{path}: object without 'properties' (free-form objects are not allowed)")
                props = {}
            if node.get("additionalProperties") is not False:
                errors.append(f"{path}: set \"additionalProperties\": false")
            required = node.get("required", [])
            missing = [name for name in props if name not in required]
            if missing:
                errors.append(f"{path}: every property must be in 'required' (missing: {', '.join(missing)}); "
                              "make optional fields nullable instead, e.g. \"type\": [\"string\", \"null\"]")
            extra = [name for name in required if name not in props]
            if extra:
                errors.append(f"{path}: 'required' names unknown properties: {', '.join(extra)}")
            for name, sub in props.items():
                walk(sub, f"{path}.properties.{name}", depth + 1)
        if "array" in kinds:
            if "items" not in node:
                errors.append(f"{path}: array without 'items'")
            else:
                walk(node["items"], f"{path}.items", depth + 1)
        for i, sub in enumerate(node.get("anyOf", [])):
            walk(sub, f"{path}.anyOf[{i}]", depth + 1)
        for defs in ("$defs", "definitions"):
            for name, sub in node.get(defs, {}).items():
                walk(sub, f"{path}.{defs}.{name}", depth)

    if not isinstance(schema, dict) or "object" not in types_of(schema):
        errors.append("$: root schema must be {\"type\": \"object\", ...}")
    if isinstance(schema, dict) and "anyOf" in schema:
        errors.append("$: root must not be anyOf; wrap alternatives in a property")
    walk(schema, "$", 0)
    return errors, warnings


def resolve(ref, root):
    if not ref.startswith("#/"):
        raise ValueError(f"only local refs are supported: {ref}")
    node = root
    for part in ref[2:].split("/"):
        node = node[part]
    return node


TYPE_CHECK = {
    "string": lambda v: isinstance(v, str),
    "integer": lambda v: isinstance(v, int) and not isinstance(v, bool),
    "number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
    "boolean": lambda v: isinstance(v, bool),
    "null": lambda v: v is None,
    "object": lambda v: isinstance(v, dict),
    "array": lambda v: isinstance(v, list),
}


def validate(value, node, root, path, problems):
    if "$ref" in node:
        node = resolve(node["$ref"], root)
    if "anyOf" in node:
        for option in node["anyOf"]:
            trial = []
            validate(value, option, root, path, trial)
            if not trial:
                return
        problems.append(f"{path}: matches no anyOf option")
        return
    kinds = types_of(node)
    if kinds and not any(TYPE_CHECK.get(k, lambda v: False)(value) for k in kinds):
        problems.append(f"{path}: expected {'/'.join(sorted(kinds))}, got {type(value).__name__}")
        return
    if "enum" in node and value not in node["enum"]:
        problems.append(f"{path}: {value!r} not in enum {node['enum']}")
    if "const" in node and value != node["const"]:
        problems.append(f"{path}: expected const {node['const']!r}")
    if isinstance(value, dict):
        props = node.get("properties", {})
        for name in node.get("required", []):
            if name not in value:
                problems.append(f"{path}: missing required '{name}'")
        for name, item in value.items():
            if name in props:
                validate(item, props[name], root, f"{path}.{name}", problems)
            elif node.get("additionalProperties") is False:
                problems.append(f"{path}: unexpected property '{name}'")
    if isinstance(value, list) and "items" in node:
        for i, item in enumerate(value):
            validate(item, node["items"], root, f"{path}[{i}]", problems)


def main():
    parser = argparse.ArgumentParser(description="Lint an output schema and validate results.")
    parser.add_argument("schema")
    parser.add_argument("--validate", metavar="OUTPUT_JSON", help="result file to validate")
    args = parser.parse_args()

    schema = load(args.schema)
    errors, warnings = lint(schema)
    for w in warnings:
        print(f"warn:  {w}")
    for e in errors:
        print(f"error: {e}")
    status = 1 if errors else 0
    if not errors:
        print(f"schema ok: {args.schema}")

    if args.validate:
        output = load(args.validate)
        problems = []
        try:
            validate(output, schema, schema, "$", problems)
        except (KeyError, ValueError) as err:
            problems.append(f"cannot resolve reference: {err}")
        for p in problems:
            print(f"invalid: {p}")
        if problems:
            status = 1
        else:
            print(f"output ok: {args.validate}")
    return status


if __name__ == "__main__":
    sys.exit(main())