Localization Workflow skill for Codex

Gives the Codex app a display name, a one-line description and a starter prompt for this skill.

interface:
  display_name: "Localization Workflow"
  short_description: "Strings in, translations back, no broken placeholders"
  default_prompt: "$localization-workflow Set up Crowdin sync for our locales folder with a CI check for missing keys"

#!/usr/bin/env python3
"""Compare JSON translation files against a source locale.

Reports missing and extra keys, empty values, placeholder mismatches, ICU plurals
without an `other` branch, markup tag mismatches and values left identical to the
source. Standard library only.

Usage:
  python3 check_locales.py SOURCE.json TARGET.json [TARGET.json ...]
  python3 check_locales.py --dir locales --source en
      (finds locales/en.json + locales/de.json, or locales/en/*.json + locales/de/*.json)

Options:
  --style auto|icu|i18next|printf   placeholder syntax (default: auto)
  --allow-same KEY_PATTERN          regex of keys allowed to equal the source (repeatable)
  --strict                          treat warnings as errors

Exit code 0 when no errors, 1 when errors were found, 2 on bad input.
"""
from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path

PLURAL_SUFFIX = re.compile(r"_(zero|one|two|few|many|other)$")
TAG = re.compile(r"</?([A-Za-z0-9]+)[^>]*>")
I18NEXT_VAR = re.compile(r"\{\{\s*-?\s*([\w.]+)[^}]*\}\}")
I18NEXT_NEST = re.compile(r"\$t\(([^)]+)\)")
PRINTF = re.compile(r"%(?:\((\w+)\))?(?:(\d+)\$)?[-+ #0]*\d*(?:\.\d+)?[sdifuxXeEgGc@]")


def flatten(obj, prefix=""):
    out = {}
    if isinstance(obj, dict):
        for k, v in obj.items():
            key = f"{prefix}.{k}" if prefix else str(k)
            out.update(flatten(v, key))
    elif isinstance(obj, list):
        for i, v in enumerate(obj):
            out.update(flatten(v, f"{prefix}[{i}]"))
    else:
        out[prefix] = obj
    return out


def icu_args(msg: str):
    """Return (argument names, plural/select args missing an `other` branch).

    Minimal ICU MessageFormat walker: handles nested plural/select branches and
    apostrophe-quoted literals well enough for linting, not for formatting.
    """
    names, missing_other = set(), []
    i, n = 0, len(msg)

    def parse_message(i, depth):
        while i < n:
            c = msg[i]
            if c == "'":
                if i + 1 < n and msg[i + 1] == "'":
                    i += 2
                    continue
                if i + 1 < n and msg[i + 1] in "{}#|":
                    end = msg.find("'", i + 1)
                    i = n if end == -1 else end + 1
                    continue
                i += 1
            elif c == "{":
                i = parse_argument(i + 1, depth)
            elif c == "}":
                if depth:
                    return i + 1
                i += 1  # stray closing brace at top level: ignore
            else:
                i += 1
        return i

    def parse_argument(i, depth):
        m = re.match(r"\s*([\w.]+)\s*(?:,\s*(\w+)\s*)?", msg[i:])
        if not m:
            # not an argument; skip to matching brace
            level = 1
            while i < n and level:
                level += {"{": 1, "}": -1}.get(msg[i], 0)
                i += 1
            return i
        names.add(m.group(1))
        kind = m.group(2)
        i += m.end()
        if i < n and msg[i] == "}":
            return i + 1
        if kind in ("plural", "select", "selectordinal"):
            if i < n and msg[i] == ",":
                i += 1
            selectors = []
            while i < n:
                sm = re.match(r"\s*(offset:\d+\s*)?(=?[\w-]+)\s*\{", msg[i:])
                if not sm:
                    break
                selectors.append(sm.group(2))
                i = parse_message(i + sm.end(), depth + 1)
            if "other" not in selectors:
                missing_other.append(m.group(1))
            while i < n and msg[i] != "}":
                i += 1
            return i + 1
        # simple formatted arg such as {d, date, short}
        level = 1
        while i < n and level:
            level += {"{": 1, "}": -1}.get(msg[i], 0)
            i += 1
        return i

    parse_message(0, 0)
    return names, missing_other


def placeholders(value: str, style: str):
    if style == "i18next":
        return {f"{{{{{m}}}}}" for m in I18NEXT_VAR.findall(value)} | {
            f"$t({m})" for m in I18NEXT_NEST.findall(value)
        }, []
    if style == "printf":
        found = set()
        for named, pos in PRINTF.findall(value):
            found.add(named or pos or "%")
        # unnamed %s counts by multiplicity
        unnamed = sum(1 for named, pos in PRINTF.findall(value) if not (named or pos))
        found.discard("%")
        if unnamed:
            found.add(f"%x{unnamed}")
        return found, []
    names, missing = icu_args(value)
    return names, missing


def detect_style(values) -> str:
    text = "\n".join(v for v in values if isinstance(v, str))
    if "{{" in text:
        return "i18next"
    if PRINTF.search(text) and "{" not in text:
        return "printf"
    return "icu"


def base_key(key: str, style: str) -> str:
    return PLURAL_SUFFIX.sub("", key) if style == "i18next" else key


def compare(src: dict, tgt: dict, style: str, allow_same):
    errors, warnings = [], []
    src_bases = {base_key(k, style) for k in src}
    tgt_bases = {base_key(k, style) for k in tgt}

    for b in sorted(src_bases - tgt_bases):
        errors.append(f"missing key: {b}")
    for b in sorted(tgt_bases - src_bases):
        warnings.append(f"extra key (not in source): {b}")
    if style == "i18next":
        for b in sorted(src_bases & tgt_bases):
            src_plural = any(PLURAL_SUFFIX.search(k) and base_key(k, style) == b for k in src)
            if src_plural and f"{b}_other" not in tgt:
                errors.append(f"plural key missing _other form: {b}")

    src_by_base = {}
    for k, v in src.items():
        src_by_base.setdefault(base_key(k, style), []).append(v)

    for key, value in tgt.items():
        b = base_key(key, style)
        if b not in src_by_base:
            continue
        if value is None or (isinstance(value, str) and not value.strip()):
            errors.append(f"empty value: {key}")
            continue
        if not isinstance(value, str):
            continue
        source_val = src.get(key, src_by_base[b][-1])
        if not isinstance(source_val, str):
            continue
        s_ph, _ = placeholders(source_val, style)
        t_ph, t_missing_other = placeholders(value, style)
        if style == "i18next" and key != b:
            # plural forms may legitimately drop {{count}} in e.g. the _one form
            s_ph.discard("{{count}}")
            t_ph.discard("{{count}}")
        if s_ph != t_ph:
            lost = sorted(s_ph - t_ph)
            added = sorted(t_ph - s_ph)
            detail = []
            if lost:
                detail.append(f"missing {', '.join(lost)}")
            if added:
                detail.append(f"unexpected {', '.join(added)}")
            errors.append(f"placeholder mismatch: {key} ({'; '.join(detail)})")
        for arg in t_missing_other:
            errors.append(f"ICU plural/select without 'other' branch: {key} ({arg})")
        if sorted(TAG.findall(source_val)) != sorted(TAG.findall(value)):
            warnings.append(f"markup tags differ: {key}")
        if (
            value == source_val
            and len(re.sub(r"\W", "", value)) > 3
            and not any(re.search(p, key) for p in allow_same)
        ):
            warnings.append(f"identical to source (untranslated?): {key}")
    return errors, warnings


def load(path: Path) -> dict:
    try:
        with path.open(encoding="utf-8") as f:
            return flatten(json.load(f))
    except (OSError, json.JSONDecodeError) as exc:
        print(f"error: cannot read {path}: {exc}", file=sys.stderr)
        sys.exit(2)


def pairs_from_dir(root: Path, source: str):
    if (root / f"{source}.json").is_file():
        src = root / f"{source}.json"
        return [(src, p) for p in sorted(root.glob("*.json")) if p != src]
    src_dir = root / source
    if src_dir.is_dir():
        out = []
        for lang_dir in sorted(d for d in root.iterdir() if d.is_dir() and d != src_dir):
            for src in sorted(src_dir.glob("*.json")):
                out.append((src, lang_dir / src.name))
        return out
    print(f"error: no {source}.json or {source}/ directory in {root}", file=sys.stderr)
    sys.exit(2)


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("files", nargs="*", type=Path)
    ap.add_argument("--dir", type=Path)
    ap.add_argument("--source")
    ap.add_argument("--style", default="auto", choices=["auto", "icu", "i18next", "printf"])
    ap.add_argument("--allow-same", action="append", default=[])
    ap.add_argument("--strict", action="store_true")
    args = ap.parse_args()

    if args.dir:
        if not args.source:
            ap.error("--dir requires --source")
        pairs = pairs_from_dir(args.dir, args.source)
    elif len(args.files) >= 2:
        pairs = [(args.files[0], t) for t in args.files[1:]]
    else:
        ap.error("give SOURCE and at least one TARGET, or --dir and --source")

    total_err = total_warn = 0
    for src_path, tgt_path in pairs:
        src = load(src_path)
        if not tgt_path.is_file():
            print(f"\n{tgt_path}: MISSING FILE (source {src_path} has {len(src)} keys)")
            total_err += 1
            continue
        tgt = load(tgt_path)
        style = args.style if args.style != "auto" else detect_style(src.values())
        errors, warnings = compare(src, tgt, style, args.allow_same)
        translated = len({base_key(k, style) for k in tgt} & {base_key(k, style) for k in src})
        coverage = 100 * translated / max(1, len({base_key(k, style) for k in src}))
        print(f"\n{tgt_path} vs {src_path} [{style}] coverage {coverage:.1f}%: "
              f"{len(errors)} error(s), {len(warnings)} warning(s)")
        for e in errors:
            print(f"  ERROR {e}")
        for w in warnings:
            print(f"  warn  {w}")
        total_err += len(errors)
        total_warn += len(warnings)

    print(f"\nTotal: {total_err} error(s), {total_warn} warning(s) across {len(pairs)} file pair(s)")
    failed = total_err or (args.strict and total_warn)
    return 1 if failed else 0


if __name__ == "__main__":
    sys.exit(main())