Sitemaps and Robots.txt skill for Codex

Gives the Codex app a display name, a one-line description and a starter prompt for this skill.

interface:
  display_name: "Sitemaps and Robots.txt"
  short_description: "Crawl rules Google actually reads the way you meant"
  default_prompt: "$sitemaps-robots Generate a sitemap index for our 300k product pages from the products table"

#!/usr/bin/env python3
"""Validate XML sitemaps and test URLs against robots.txt with Google's matching rules.

Standard library only (Python 3.10+).

  python3 scripts/seo_files.py sitemap <file|url> [--host www.example.com] [--robots robots.txt]
  python3 scripts/seo_files.py robots <robots.txt file|url> <url-or-path> [...] [--agent Googlebot]

Python's urllib.robotparser uses first-match prefix rules and ignores * and $, so it
disagrees with Google on many real files. This script implements the RFC 9309 / Google
behaviour: most specific user-agent group, * and $ wildcards, longest match wins,
allow wins ties.

Exit codes: 0 ok, 1 problems found (errors, or a tested URL is disallowed), 2 bad input.
"""
from __future__ import annotations

import argparse
import gzip
import re
import sys
import urllib.parse
import urllib.request
from collections import Counter
from datetime import datetime, timezone
from xml.etree import ElementTree as ET

NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}"
MAX_URLS = 50_000
MAX_BYTES = 50 * 1024 * 1024
W3C_DATE = re.compile(
    r"^\d{4}(-\d{2}(-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2}))?)?)?$"
)


def load(source: str) -> bytes:
    if re.match(r"^https?://", source):
        req = urllib.request.Request(source, headers={"User-Agent": "seo-files-check/1.0"})
        with urllib.request.urlopen(req, timeout=30) as resp:
            data = resp.read()
    else:
        with open(source, "rb") as fh:
            data = fh.read()
    if data[:2] == b"\x1f\x8b":
        data = gzip.decompress(data)
    return data


# ---------------------------------------------------------------- robots.txt

def parse_robots(text: str) -> tuple[dict[str, list[tuple[str, str]]], list[str]]:
    groups: dict[str, list[tuple[str, str]]] = {}
    sitemaps: list[str] = []
    current: list[str] = []
    last_was_agent = False
    for raw in text.splitlines():
        line = raw.split("#", 1)[0].strip()
        if ":" not in line:
            continue
        key, value = (p.strip() for p in line.split(":", 1))
        key = key.lower()
        if key == "user-agent":
            if not last_was_agent:
                current = []
            agent = value.lower()
            current.append(agent)
            groups.setdefault(agent, [])
            last_was_agent = True
        elif key in ("allow", "disallow"):
            last_was_agent = False
            for agent in current:
                groups[agent].append((key, value))
        elif key == "sitemap":
            sitemaps.append(value)
        else:
            last_was_agent = False
    return groups, sitemaps


def pick_group(groups: dict, agent: str) -> tuple[str, list[tuple[str, str]]]:
    agent = agent.lower()
    if agent in groups:
        return agent, groups[agent]
    prefixes = [g for g in groups if g != "*" and agent.startswith(g)]
    if prefixes:
        best = max(prefixes, key=len)
        return best, groups[best]
    return "*", groups.get("*", [])


def rule_regex(path: str) -> re.Pattern:
    anchored = path.endswith("$")
    body = path[:-1] if anchored else path
    pattern = "".join(".*" if ch == "*" else re.escape(ch) for ch in body)
    return re.compile("^" + pattern + ("$" if anchored else ""))


def is_allowed(rules: list[tuple[str, str]], url_path: str) -> tuple[bool, str]:
    best: tuple[int, int, str] | None = None  # (length, allow_flag, rule)
    for kind, path in rules:
        if not path:
            continue  # empty Disallow allows everything; empty Allow is a no-op
        if rule_regex(path).match(url_path):
            cand = (len(path), 1 if kind == "allow" else 0, f"{kind.title()}: {path}")
            if best is None or cand[:2] > best[:2]:
                best = cand
    if best is None:
        return True, "no matching rule"
    return bool(best[1]), best[2]


def path_of(url: str) -> str:
    parts = urllib.parse.urlsplit(url)
    path = parts.path or "/"
    return path + ("?" + parts.query if parts.query else "")


def cmd_robots(args) -> int:
    data = load(args.robots)
    if len(data) > 500 * 1024:
        print(f"WARN robots.txt is {len(data)} bytes; Google ignores content after 500 KiB")
    groups, sitemaps = parse_robots(data.decode("utf-8", errors="replace"))
    name, rules = pick_group(groups, args.agent)
    print(f"Agent '{args.agent}' obeys group: user-agent: {name} ({len(rules)} rules)")
    for s in sitemaps:
        print(f"Sitemap line: {s}")
    blocked = 0
    for target in args.urls:
        ok, why = is_allowed(rules, path_of(target) if "://" in target else target)
        blocked += not ok
        print(f"{'ALLOW ' if ok else 'BLOCK '} {target}  ({why})")
    return 1 if blocked else 0


# ---------------------------------------------------------------- sitemaps

def cmd_sitemap(args) -> int:
    errors: list[str] = []
    warns: list[str] = []
    try:
        data = load(args.sitemap)
    except Exception as exc:
        print(f"ERROR cannot load {args.sitemap}: {exc}")
        return 2
    if len(data) > MAX_BYTES:
        errors.append(f"uncompressed size {len(data)} bytes exceeds 50 MB")
    if data.lstrip()[:5].lower() == b"<html" or b"<!doctype html" in data[:200].lower():
        print("ERROR the sitemap URL returned HTML, not XML")
        return 1
    try:
        root = ET.fromstring(data)
    except ET.ParseError as exc:
        print(f"ERROR invalid XML: {exc} (check unescaped & in URLs)")
        return 1

    kind = root.tag.replace(NS, "")
    if not root.tag.startswith(NS):
        errors.append(f"root <{root.tag}> is not in the sitemaps.org 0.9 namespace")
    child_tag = "url" if kind == "urlset" else "sitemap" if kind == "sitemapindex" else None
    if child_tag is None:
        print(f"ERROR root element must be urlset or sitemapindex, got {kind}")
        return 1

    entries = root.findall(f"{NS}{child_tag}")
    if len(entries) > MAX_URLS:
        errors.append(f"{len(entries)} entries exceeds the 50,000 limit; split into a sitemap index")

    rules = None
    if args.robots:
        groups, _ = parse_robots(load(args.robots).decode("utf-8", errors="replace"))
        rules = pick_group(groups, "googlebot")[1]

    locs: list[str] = []
    lastmods: list[str] = []
    future = 0
    ignored_tags = Counter()
    now = datetime.now(timezone.utc)
    for entry in entries:
        loc_el = entry.find(f"{NS}loc")
        loc = (loc_el.text or "").strip() if loc_el is not None else ""
        if not loc:
            errors.append(f"<{child_tag}> without <loc>")
            continue
        locs.append(loc)
        parts = urllib.parse.urlsplit(loc)
        if parts.scheme not in ("http", "https") or not parts.netloc:
            errors.append(f"relative or invalid loc: {loc}")
        elif args.host and parts.netloc.lower() != args.host.lower():
            errors.append(f"loc on another host ({parts.netloc}): {loc}")
        if parts.fragment:
            warns.append(f"loc contains a #fragment: {loc}")
        if rules is not None:
            ok, why = is_allowed(rules, path_of(loc))
            if not ok:
                errors.append(f"listed but blocked by robots.txt ({why}): {loc}")
        lm_el = entry.find(f"{NS}lastmod")
        if lm_el is not None and lm_el.text:
            lm = lm_el.text.strip()
            lastmods.append(lm)
            if not W3C_DATE.match(lm):
                errors.append(f"lastmod not W3C Datetime ({lm}): {loc}")
            else:
                try:
                    stamp = datetime.fromisoformat(lm.replace("Z", "+00:00"))
                    if stamp.tzinfo is None:
                        stamp = stamp.replace(tzinfo=timezone.utc)
                    if stamp > now:
                        future += 1
                except ValueError:
                    pass
        for tag in ("priority", "changefreq"):
            if entry.find(f"{NS}{tag}") is not None:
                ignored_tags[tag] += 1

    dupes = [u for u, n in Counter(locs).items() if n > 1]
    if dupes:
        warns.append(f"{len(dupes)} duplicate loc values, e.g. {dupes[0]}")
    if future:
        errors.append(f"{future} lastmod values are in the future")
    if lastmods and len(lastmods) > 20 and len(set(lastmods)) == 1:
        warns.append("every lastmod is identical; Google ignores lastmod it cannot trust")
    if entries and not lastmods:
        warns.append("no lastmod values; add accurate ones for changed pages")
    for tag, n in ignored_tags.items():
        warns.append(f"{n} entries use <{tag}>, which Google ignores (harmless)")
    if kind == "sitemapindex":
        nested = [u for u in locs if "sitemap" in u.lower() and u.lower().endswith("index.xml")]
        if nested:
            warns.append("index appears to reference another index; Google does not support nesting")

    print(f"{kind}: {len(entries)} entries, {len(data):,} bytes uncompressed")
    for e in errors[:50]:
        print(f"ERROR {e}")
    if len(errors) > 50:
        print(f"ERROR ... {len(errors) - 50} more")
    for w in warns:
        print(f"WARN  {w}")
    if kind == "sitemapindex":
        print("Child sitemaps (validate each one separately):")
        for u in locs:
            print(f"  {u}")
    if not errors:
        print("OK")
    return 1 if errors else 0


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = ap.add_subparsers(dest="cmd", required=True)
    s = sub.add_parser("sitemap", help="validate a sitemap or sitemap index (.xml or .xml.gz)")
    s.add_argument("sitemap")
    s.add_argument("--host", help="expected host for every loc, e.g. www.example.com")
    s.add_argument("--robots", help="robots.txt file or URL to cross-check each loc against Googlebot")
    r = sub.add_parser("robots", help="test URLs or paths against robots.txt")
    r.add_argument("robots")
    r.add_argument("urls", nargs="+")
    r.add_argument("--agent", default="Googlebot")
    args = ap.parse_args()
    try:
        return cmd_sitemap(args) if args.cmd == "sitemap" else cmd_robots(args)
    except (OSError, ValueError) as exc:
        print(f"ERROR {exc}", file=sys.stderr)
        return 2


if __name__ == "__main__":
    sys.exit(main())