Hreflang and International SEO skill for Codex

Gives the Codex app a display name, a one-line description and a starter prompt for this skill.

interface:
  display_name: "Hreflang and International SEO"
  short_description: "The right country version in every market"
  default_prompt: "$hreflang-international-seo We're launching German and French versions under /de and /fr. Add hreflang and tell me what else needs to change"

#!/usr/bin/env python3
"""Validate hreflang clusters from XML sitemaps, HTML files, a CSV export or live URLs.

Checks: language/region/script codes, fully qualified URLs, self-reference, return
links, conflicting codes, x-default, hreflang on non-canonical pages, cluster
consistency and (live mode) alternates that redirect or do not return 200.

Usage:
  python3 hreflang_check.py sitemap.xml [more.xml | sitemap.xml.gz ...]
  python3 hreflang_check.py page.html [other.html@https://example.com/de/ ...]
  python3 hreflang_check.py --csv annotations.csv        # columns: page,hreflang,href
  python3 hreflang_check.py https://example.com/ --follow  # live; needs network
Options: --json, --strict (warnings also fail), --follow, --max-fetch N, --limit N

Exit codes: 0 clean, 1 errors (or warnings with --strict), 2 bad input.
Standard library only (Python 3.10+).
"""
from __future__ import annotations

import argparse
import csv
import gzip
import json
import re
import sys
import urllib.error
import urllib.request
import xml.etree.ElementTree as ET
from html.parser import HTMLParser
from urllib.parse import urljoin, urlsplit, urlunsplit

ISO_639_1 = set("""
aa ab ae af ak am an ar as av ay az ba be bg bh bi bm bn bo br bs ca ce ch co cr cs cu
cv cy da de dv dz ee el en eo es et eu fa ff fi fj fo fr fy ga gd gl gn gu gv ha he hi
ho hr ht hu hy hz ia id ie ig ii ik io is it iu ja jv ka kg ki kj kk kl km kn ko kr ks
ku kv kw ky la lb lg li ln lo lt lu lv mg mh mi mk ml mn mr ms mt my na nb nd ne ng nl
nn no nr nv ny oc oj om or os pa pi pl ps pt qu rm rn ro ru rw sa sc sd se sg si sk sl
sm sn so sq sr ss st su sv sw ta te tg th ti tk tl tn to tr ts tt tw ty ug uk ur uz ve
vi vo wa wo xh yi yo za zh zu
""".split())

ISO_3166_1 = set("""
AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ BA BB BD BE BF BG BH BI BJ BL BM BN BO
BQ BR BS BT BV BW BY BZ CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ DE DJ
DK DM DO DZ EC EE EG EH ER ES ET FI FJ FK FM FO FR GA GB GD GE GF GG GH GI GL GM GN GP
GQ GR GS GT GU GW GY HK HM HN HR HT HU ID IE IL IM IN IO IQ IR IS IT JE JM JO JP KE KG
KH KI KM KN KP KR KW KY KZ LA LB LC LI LK LR LS LT LU LV LY MA MC MD ME MF MG MH MK ML
MM MN MO MP MQ MR MS MT MU MV MW MX MY MZ NA NC NE NF NG NI NL NO NP NR NU NZ OM PA PE
PF PG PH PK PL PM PN PR PS PT PW PY QA RE RO RS RU RW SA SB SC SD SE SG SH SI SJ SK SL
SM SN SO SR SS ST SV SX SY SZ TC TD TF TG TH TJ TK TL TM TN TO TR TT TV TW TZ UA UG UM
US UY UZ VA VC VE VG VI VN VU WF WS YE YT ZA ZM ZW
""".split())

REGION_FIXES = {"UK": "use GB", "EU": "list each country", "UN": "list each country",
                "419": "list each country (UN M.49 codes are not supported)"}
DEPRECATED_LANG = {"iw": "he", "in": "id", "ji": "yi"}
HEAD_OK = {"title", "base", "link", "meta", "style", "script", "noscript", "template"}
SITEMAP_NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}"
XHTML_NS = "{http://www.w3.org/1999/xhtml}"


def norm(url: str) -> str:
    """Lowercase scheme and host, drop fragment; keep path and query exactly."""
    p = urlsplit(url.strip())
    return urlunsplit((p.scheme.lower(), p.netloc.lower(), p.path, p.query, ""))


def loose(url: str) -> str:
    """Key used only to explain near misses (protocol, www, trailing slash)."""
    p = urlsplit(url)
    host = p.netloc.lower().removeprefix("www.")
    return host + (p.path.rstrip("/") or "/") + ("?" + p.query if p.query else "")


def check_code(code: str) -> tuple[str, str] | None:
    """Return (severity, message) for a bad hreflang value, else None."""
    if code.lower() == "x-default":
        return None
    if "_" in code:
        return ("error", f"'{code}' uses an underscore; use a hyphen ({code.replace('_', '-')})")
    parts = code.split("-")
    lang = parts[0].lower()
    if lang.upper() in ISO_3166_1 and lang not in ISO_639_1 and len(parts) == 1:
        return ("error", f"'{code}' looks like a region alone; hreflang must start with a language")
    if lang in DEPRECATED_LANG:
        return ("warn", f"'{code}' uses deprecated language code '{lang}'; use '{DEPRECATED_LANG[lang]}'")
    if lang not in ISO_639_1:
        return ("error", f"'{code}': '{lang}' is not an ISO 639-1 language code")
    rest = parts[1:]
    if rest and len(rest[0]) == 4 and rest[0].isalpha():
        rest = rest[1:]  # ISO 15924 script, e.g. zh-Hant
    if len(rest) > 1:
        return ("error", f"'{code}' has too many subtags (language[-Script][-REGION])")
    if rest:
        region = rest[0].upper()
        if region in REGION_FIXES:
            return ("error", f"'{code}': region '{rest[0]}' is not supported; {REGION_FIXES[region]}")
        if region not in ISO_3166_1:
            return ("error", f"'{code}': '{rest[0]}' is not an ISO 3166-1 alpha-2 region")
    if len(parts) == 1 and lang in {"be"}:
        return ("warn", f"'{code}' is Belarusian; for Belgium use nl-BE, fr-BE or de-BE")
    return None


class HeadParser(HTMLParser):
    """Collect hreflang links and canonical, noting links after <head> implicitly closed."""

    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.alts: list[tuple[str, str]] = []
        self.late: list[str] = []
        self.canonical: str | None = None
        self.in_head = True
        self.head_broken_by: str | None = None
        self.done = False
        self.raw: str | None = None  # inside title/script/style etc.

    def handle_starttag(self, tag, attrs):
        if self.done:
            return
        a = {k.lower(): (v or "") for k, v in attrs}
        if tag == "body":
            self.done = True
            return
        if tag in {"html", "head"}:
            return
        if self.in_head and tag not in HEAD_OK:
            self.in_head = False
            self.head_broken_by = f"<{tag}>"
        if tag in HEAD_OK - {"link", "meta", "base"}:
            self.raw = tag
        if tag == "link":
            rel = a.get("rel", "").lower().split()
            if "alternate" in rel and "hreflang" in a:
                if self.in_head:
                    self.alts.append((a["hreflang"].strip(), a.get("href", "").strip()))
                else:
                    self.late.append(a["hreflang"].strip())
            elif "canonical" in rel and self.canonical is None:
                self.canonical = a.get("href", "").strip()

    def handle_endtag(self, tag):
        if tag == self.raw:
            self.raw = None
        if tag == "head":
            self.done = True

    def handle_data(self, data):
        if not self.done and self.in_head and self.raw is None and data.strip():
            self.in_head = False
            self.head_broken_by = "text content"


def parse_link_header(value: str) -> list[tuple[str, str]]:
    out = []
    for m in re.finditer(r'<([^>]+)>\s*((?:;\s*[^;,]+)*)', value or ""):
        params = dict((k.strip().lower(), v.strip().strip('"'))
                      for k, _, v in (p.partition("=") for p in m.group(2).split(";") if "=" in p))
        if params.get("rel", "").lower() == "alternate" and "hreflang" in params:
            out.append((params["hreflang"], m.group(1)))
    return out


class Store:
    def __init__(self) -> None:
        self.pages: dict[str, dict] = {}
        self.issues: list[dict] = []

    def page(self, url: str, source: str) -> dict:
        return self.pages.setdefault(norm(url), {"url": url, "source": source, "alts": [],
                                                 "canonical": None, "status": None, "final": None})

    def add(self, sev: str, url: str, msg: str) -> None:
        self.issues.append({"severity": sev, "page": url, "message": msg})


def load_sitemap(path: str, st: Store) -> None:
    raw = gzip.open(path).read() if path.endswith(".gz") else open(path, "rb").read()
    root = ET.fromstring(raw)
    if root.tag == SITEMAP_NS + "sitemapindex":
        raise SystemExit(f"{path} is a sitemap index; pass the child sitemaps instead")
    for u in root.iter(SITEMAP_NS + "url"):
        loc = (u.findtext(SITEMAP_NS + "loc") or "").strip()
        if not loc:
            continue
        p = st.page(loc, path)
        for link in u.findall(XHTML_NS + "link"):
            if link.get("rel") == "alternate" and link.get("hreflang"):
                p["alts"].append((link.get("hreflang").strip(), (link.get("href") or "").strip()))


def load_html(arg: str, st: Store) -> None:
    path, _, url = arg.partition("@")
    hp = HeadParser()
    hp.feed(open(path, encoding="utf-8", errors="replace").read())
    url = url or hp.canonical
    if not url:
        st.add("error", path, "no canonical found; pass the page URL as file.html@https://...")
        return
    p = st.page(url, path)
    p["alts"] += hp.alts
    p["canonical"] = hp.canonical
    if hp.late:
        st.add("error", url, f"{len(hp.late)} hreflang link(s) appear after {hp.head_broken_by} "
                             "closed <head>; Google ignores them")


def load_csv(path: str, st: Store) -> None:
    with open(path, newline="", encoding="utf-8-sig") as fh:
        for row in csv.DictReader(fh):
            row = {k.strip().lower(): (v or "").strip() for k, v in row.items() if k}
            if row.get("page"):
                st.page(row["page"], path)["alts"].append((row.get("hreflang", ""), row.get("href", "")))


def fetch(url: str, st: Store) -> None:
    p = st.page(url, "live")
    req = urllib.request.Request(url, headers={"User-Agent": "hreflang-check/1.0"})
    try:
        with urllib.request.urlopen(req, timeout=20) as resp:
            p["status"], p["final"] = resp.status, resp.geturl()
            p["alts"] += parse_link_header(resp.headers.get("Link", ""))
            ctype = resp.headers.get("Content-Type", "")
            body = resp.read(2_000_000).decode("utf-8", "replace") if "html" in ctype else ""
    except urllib.error.HTTPError as exc:
        p["status"], p["final"] = exc.code, url
        return
    except (urllib.error.URLError, TimeoutError, OSError) as exc:
        st.add("error", url, f"fetch failed: {exc}")
        p["status"] = 0
        return
    if body:
        hp = HeadParser()
        hp.feed(body)
        p["alts"] += [(c, urljoin(url, h) if h and not urlsplit(h).scheme else h) for c, h in hp.alts]
        if any(not urlsplit(h).scheme for _, h in hp.alts):
            st.add("error", url, "relative hreflang href(s) in HTML; use fully qualified URLs")
        p["canonical"] = urljoin(url, hp.canonical) if hp.canonical else None
        if hp.late:
            st.add("error", url, f"{len(hp.late)} hreflang link(s) appear after {hp.head_broken_by} "
                                 "closed <head>; Google ignores them")


def analyse(st: Store, limit: int) -> None:
    unverified: list[str] = []
    parent = {}

    def find(x):
        parent.setdefault(x, x)
        while parent[x] != x:
            parent[x] = parent[parent[x]]
            x = parent[x]
        return x

    for key, p in st.pages.items():
        url, alts = p["url"], p["alts"]
        if not alts:
            if p["source"] == "live" and p["status"] not in (0, 200):
                st.add("error", url, f"returned HTTP {p['status']}")
            elif p["source"] == "live" and p["status"] == 200:
                st.add("warn", url, "no hreflang found in <head> or the Link header")
            continue
        if p["status"] not in (None, 200):
            st.add("error", url, f"page returns HTTP {p['status']}")
        if p["final"] and norm(p["final"]) != key:
            st.add("error", url, f"redirects to {p['final']}; hreflang must use final URLs")
        if p["canonical"] and norm(p["canonical"]) != key:
            st.add("error", url, f"carries hreflang but canonicalizes to {p['canonical']}")
        by_code: dict[str, set[str]] = {}
        for code, href in alts:
            bad = check_code(code)
            if bad:
                st.add(bad[0], url, bad[1])
            if not href or not urlsplit(href).scheme or not urlsplit(href).netloc:
                st.add("error", url, f"hreflang '{code}' href '{href}' is not a fully qualified URL")
                continue
            by_code.setdefault(code.lower(), set()).add(norm(href))
            find(key), find(norm(href))
            parent[find(norm(href))] = find(key)
        for code, hrefs in by_code.items():
            if len(hrefs) > 1:
                st.add("error", url, f"hreflang '{code}' points to {len(hrefs)} different URLs: "
                                     + ", ".join(sorted(hrefs)))
        targets = {h for hs in by_code.values() for h in hs}
        if key not in targets:
            near = [t for t in targets if loose(t) == loose(key)]
            hint = f" (near miss: {near[0]})" if near else ""
            st.add("error", url, "missing self-referencing hreflang" + hint)
        for code, hrefs in by_code.items():
            for t in hrefs:
                if t == key:
                    continue
                other = st.pages.get(t)
                if other is not None and (other["status"] not in (None, 200)
                                          or (other["final"] and norm(other["final"]) != t)):
                    st.add("error", url, f"'{code}' -> {t} is not a 200 final URL")
                    continue
                if other is None or not other["alts"]:
                    if other is None:
                        unverified.append(t)
                    else:
                        st.add("error", url, f"'{code}' -> {t} has no hreflang annotations (no return link)")
                    continue
                back = {norm(h): c.lower() for c, h in other["alts"] if h}
                if key not in back:
                    near = [b for b in back if loose(b) == loose(key)]
                    hint = f"; it links to {near[0]} instead" if near else ""
                    st.add("error", url, f"no return link from {t} ('{code}'){hint}")
                if code != "x-default":
                    own = {c.lower() for c, h in other["alts"] if h and norm(h) == t} - {"x-default"}
                    if own and code not in own:
                        st.add("warn", url, f"labels {t} as '{code}' but that page labels itself "
                                            f"{', '.join(sorted(own))}")
                if other["canonical"] and norm(other["canonical"]) != t:
                    st.add("error", url, f"'{code}' -> {t}, which canonicalizes to {other['canonical']}")

    clusters: dict[str, list[str]] = {}
    for key, p in st.pages.items():
        if p["alts"]:
            clusters.setdefault(find(key), []).append(key)
    for members in clusters.values():
        sets = {m: frozenset((c.lower(), norm(h)) for c, h in st.pages[m]["alts"] if h) for m in members}
        if len(set(sets.values())) > 1:
            st.add("warn", st.pages[members[0]]["url"],
                   f"cluster of {len(members)} pages declares {len(set(sets.values()))} different "
                   "hreflang sets; every member should list the identical set")
        codes = {c for s in sets.values() for c, _ in s}
        if "x-default" not in codes:
            st.add("warn", st.pages[members[0]]["url"], "cluster has no x-default annotation")
        langs = {c.split("-")[0] for c in codes if "-" in c}
        if "x-default" not in codes:
            for lang in sorted(langs):
                if lang not in codes:
                    st.add("info", st.pages[members[0]]["url"],
                           f"'{lang}' has only regional variants and no '{lang}' or x-default fallback")
    if unverified:
        uniq = sorted(set(unverified))
        st.add("warn", "(input)", f"{len(uniq)} alternate URL(s) not in the input, return links unverified: "
                                  + ", ".join(uniq[:limit]) + (" ..." if len(uniq) > limit else ""))


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("inputs", nargs="*", help="sitemap .xml/.xml.gz, .html files, or http(s) URLs")
    ap.add_argument("--csv", action="append", default=[], help="CSV with page,hreflang,href columns")
    ap.add_argument("--follow", action="store_true", help="live mode: also fetch every alternate URL")
    ap.add_argument("--max-fetch", type=int, default=200)
    ap.add_argument("--limit", type=int, default=20, help="max URLs listed per aggregated message")
    ap.add_argument("--json", action="store_true")
    ap.add_argument("--strict", action="store_true", help="exit 1 on warnings too")
    args = ap.parse_args()
    if not args.inputs and not args.csv:
        ap.print_usage(sys.stderr)
        return 2

    st = Store()
    live: list[str] = []
    try:
        for item in args.inputs:
            if re.match(r"https?://", item):
                live.append(item)
            elif re.search(r"\.html?(@|$)", item):
                load_html(item, st)
            else:
                load_sitemap(item, st)
        for path in args.csv:
            load_csv(path, st)
    except (OSError, ET.ParseError, csv.Error) as exc:
        print(f"input error: {exc}", file=sys.stderr)
        return 2

    fetched: set[str] = set()
    while live and len(fetched) < args.max_fetch:
        url = live.pop(0)
        if norm(url) in fetched:
            continue
        fetched.add(norm(url))
        fetch(url, st)
        if args.follow:
            live += [h for _, h in st.pages[norm(url)]["alts"] if h and norm(h) not in fetched]
    if live and args.follow:
        st.add("warn", "(input)", f"stopped after --max-fetch {args.max_fetch}; {len(live)} URL(s) not fetched")

    analyse(st, args.limit)
    order = {"error": 0, "warn": 1, "info": 2}
    st.issues.sort(key=lambda i: (order[i["severity"]], i["page"]))
    errors = sum(i["severity"] == "error" for i in st.issues)
    warns = sum(i["severity"] == "warn" for i in st.issues)
    annotated = sum(1 for p in st.pages.values() if p["alts"])
    if args.json:
        print(json.dumps({"pages": len(st.pages), "annotated": annotated, "errors": errors,
                          "warnings": warns, "issues": st.issues}, indent=2))
    else:
        for i in st.issues:
            print(f"{i['severity'].upper():5} {i['page']}: {i['message']}")
        print(f"\n{annotated} annotated page(s), {errors} error(s), {warns} warning(s)")
    return 1 if errors or (args.strict and warns) else 0


if __name__ == "__main__":
    sys.exit(main())