Technical SEO Audit skill for Codex

Gives the Codex app a display name, a one-line description and a starter prompt for this skill.

interface:
  display_name: "Technical SEO Audit"
  short_description: "Find what stops Google indexing your pages"
  default_prompt: "$technical-seo-audit Our organic traffic halved after Tuesday's deploy, audit the site for indexing problems"

#!/usr/bin/env python3
"""Same-host crawler for technical SEO audits. Standard library only.

Usage:
  python3 scripts/crawl_audit.py https://example.com/ [--max-pages 500] [--delay 0.5]
                                 [--out crawl.csv] [--user-agent UA] [--ignore-robots]

Records status codes, redirect chains, canonicals, meta robots / X-Robots-Tag,
titles, H1 counts and internal inlinks, then prints a grouped issue summary.
Exit code: 0 = no error-level issues, 1 = error-level issues found, 2 = could not start.
"""
from __future__ import annotations

import argparse
import csv
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import urllib.robotparser
from collections import defaultdict, deque
from html.parser import HTMLParser

DEFAULT_UA = "Mozilla/5.0 (compatible; seo-crawl-audit/1.0)"
MAX_HOPS = 10
MAX_BODY = 5 * 1024 * 1024


class PageParser(HTMLParser):
    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.title = ""
        self._in_title = False
        self.meta_robots: list[str] = []
        self.meta_description = None
        self.canonicals: list[str] = []
        self.links: list[str] = []
        self.h1 = 0
        self.base = None

    def handle_starttag(self, tag, attrs):
        a = {k.lower(): (v or "") for k, v in attrs}
        if tag == "title":
            self._in_title = True
        elif tag == "meta":
            name = a.get("name", "").lower()
            if name in ("robots", "googlebot"):
                self.meta_robots.append(a.get("content", "").lower())
            elif name == "description":
                self.meta_description = a.get("content", "")
        elif tag == "link" and "canonical" in a.get("rel", "").lower().split():
            self.canonicals.append(a.get("href", "").strip())
        elif tag == "a" and a.get("href"):
            rel = a.get("rel", "").lower().split()
            if "nofollow" not in rel:
                self.links.append(a["href"].strip())
        elif tag == "h1":
            self.h1 += 1
        elif tag == "base" and a.get("href"):
            self.base = a["href"]

    def handle_endtag(self, tag):
        if tag == "title":
            self._in_title = False

    def handle_data(self, data):
        if self._in_title:
            self.title += data


class NoRedirect(urllib.request.HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):
        return None


def normalize(url: str, base: str) -> str | None:
    joined = urllib.parse.urljoin(base, url)
    parts = urllib.parse.urlsplit(joined)
    if parts.scheme not in ("http", "https"):
        return None
    path = parts.path or "/"
    return urllib.parse.urlunsplit((parts.scheme, parts.netloc.lower(), path, parts.query, ""))


class Crawler:
    def __init__(self, start: str, ua: str, delay: float, max_pages: int, ignore_robots: bool):
        self.start = start
        self.host = urllib.parse.urlsplit(start).netloc.lower()
        self.scheme = urllib.parse.urlsplit(start).scheme
        self.ua = ua
        self.delay = delay
        self.max_pages = max_pages
        self.opener = urllib.request.build_opener(NoRedirect())
        self.resolved: dict[str, dict] = {}
        self.pages: dict[str, dict] = {}
        self.inlinks: dict[str, set[str]] = defaultdict(set)
        self.depth: dict[str, int] = {}
        self.blocked: set[str] = set()
        self.http_links: set[tuple[str, str]] = set()
        self.robots = None
        if not ignore_robots:
            rp = urllib.robotparser.RobotFileParser()
            rp.set_url(f"{self.scheme}://{self.host}/robots.txt")
            try:
                rp.read()
                self.robots = rp
            except Exception as exc:  # robots.txt unreachable: note it, crawl anyway
                print(f"note: could not read robots.txt ({exc}); crawling without it", file=sys.stderr)

    def fetch(self, url: str):
        req = urllib.request.Request(url, headers={"User-Agent": self.ua, "Accept": "text/html,*/*"})
        try:
            resp = self.opener.open(req, timeout=20)
            status, headers = resp.status, resp.headers
        except urllib.error.HTTPError as err:
            resp, status, headers = err, err.code, err.headers
        except Exception as exc:
            return 0, {}, None, str(exc)
        body = None
        ctype = (headers.get("Content-Type") or "").lower()
        if status == 200 and "html" in ctype:
            raw = resp.read(MAX_BODY)
            charset = headers.get_content_charset() or "utf-8"
            body = raw.decode(charset, errors="replace")
        resp.close()
        if self.delay:
            time.sleep(self.delay)
        return status, headers, body, None

    def resolve(self, url: str) -> dict:
        if url in self.resolved:
            return self.resolved[url]
        hops, current, seen = [], url, set()
        while True:
            status, headers, body, error = self.fetch(current)
            hops.append((current, status))
            if 300 <= status < 400 and headers.get("Location"):
                nxt = normalize(headers["Location"], current)
                if nxt is None or nxt in seen or len(hops) > MAX_HOPS:
                    result = {"final": current, "status": status, "hops": hops, "headers": headers,
                              "body": None, "error": "redirect loop or too many hops"}
                    break
                seen.add(current)
                current = nxt
                continue
            result = {"final": current, "status": status, "hops": hops, "headers": headers,
                      "body": body, "error": error}
            break
        self.resolved[url] = result
        return result

    def allowed(self, url: str) -> bool:
        return self.robots is None or self.robots.can_fetch(self.ua, url)

    def crawl(self) -> None:
        queue = deque([self.start])
        self.depth[self.start] = 0
        while queue and len(self.pages) < self.max_pages:
            url = queue.popleft()
            if not self.allowed(url):
                self.blocked.add(url)
                continue
            res = self.resolve(url)
            final = res["final"]
            page = {"url": url, "status": res["status"], "final": final, "hops": len(res["hops"]) - 1,
                    "error": res["error"], "title": "", "canonical": "", "robots": "",
                    "h1": "", "description": None}
            xrt = (res["headers"].get("X-Robots-Tag", "") if res["headers"] else "").lower()
            same_host = urllib.parse.urlsplit(final).netloc.lower() == self.host
            if page["hops"] and same_host:
                # Audit the redirect target as its own URL instead of under the redirecting one.
                if final not in self.depth:
                    self.depth[final] = self.depth[url]
                    queue.append(final)
                page["robots"] = xrt
            elif res["body"] is not None and same_host:
                parser = PageParser()
                try:
                    parser.feed(res["body"])
                except Exception:
                    pass
                base = normalize(parser.base, final) if parser.base else final
                page.update(title=" ".join(parser.title.split()), h1=parser.h1,
                            description=parser.meta_description,
                            canonical=";".join(normalize(c, base) or c for c in parser.canonicals),
                            robots=",".join(filter(None, parser.meta_robots + [xrt])))
                for href in parser.links:
                    target = normalize(href, base)
                    if not target:
                        continue
                    parts = urllib.parse.urlsplit(target)
                    if parts.netloc != self.host:
                        continue
                    if self.scheme == "https" and parts.scheme == "http":
                        self.http_links.add((url, target))
                    self.inlinks[target].add(url)
                    if target not in self.depth:
                        self.depth[target] = self.depth[url] + 1
                        queue.append(target)
            else:
                page["robots"] = xrt
            self.pages[url] = page


def issues_for(c: Crawler) -> dict[tuple[str, str], list[str]]:
    found: dict[tuple[str, str], list[str]] = defaultdict(list)
    titles: dict[str, list[str]] = defaultdict(list)
    for url, p in c.pages.items():
        linked = bool(c.inlinks.get(url))
        if p["error"]:
            found[("ERROR", f"fetch failed: {p['error']}")].append(url)
        elif p["status"] >= 400 or p["status"] == 0:
            found[("ERROR", f"internal link target returns {p['status']}")].append(url)
        if p["hops"] > 1:
            found[("WARN", "redirect chain longer than one hop")].append(url)
        elif p["hops"] == 1 and linked:
            found[("INFO", "internal link points at a redirect (link to the final URL)")].append(url)
        is_html_ok = p["status"] == 200 and p["h1"] != ""
        if not is_html_ok or p["hops"]:
            continue
        if "noindex" in p["robots"]:
            found[("WARN", "noindex page is linked internally")].append(url)
            continue
        canon = p["canonical"].split(";") if p["canonical"] else []
        if not canon:
            found[("WARN", "missing rel=canonical")].append(url)
        elif len(canon) > 1:
            found[("ERROR", "multiple rel=canonical tags (Google ignores all of them)")].append(url)
        elif canon[0] != url:
            target = canon[0]
            if urllib.parse.urlsplit(target).netloc != c.host:
                found[("WARN", "canonical points to another host")].append(url)
            else:
                res = c.resolve(target)
                if res["status"] != 200 or len(res["hops"]) > 1:
                    found[("ERROR", "canonical target is not a 200 page")].append(url)
                else:
                    found[("INFO", "canonicalised to a different URL")].append(url)
        if not p["title"]:
            found[("WARN", "missing <title>")].append(url)
        else:
            titles[p["title"].lower()].append(url)
        if p["h1"] == 0:
            found[("INFO", "no <h1>")].append(url)
        if not p["description"]:
            found[("INFO", "missing meta description")].append(url)
        if c.depth.get(url, 0) > 3:
            found[("INFO", "more than 3 clicks from the start URL")].append(url)
    for title, urls in titles.items():
        if len(urls) > 1:
            found[("WARN", "duplicate <title> across pages")].extend(urls)
    for url in sorted(c.blocked):
        found[("INFO", "linked but disallowed by robots.txt")].append(url)
    for src, target in sorted(c.http_links):
        found[("WARN", "https page links to http:// URL")].append(f"{src} -> {target}")
    return found


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("start")
    ap.add_argument("--max-pages", type=int, default=500)
    ap.add_argument("--delay", type=float, default=0.5, help="seconds between requests (be polite)")
    ap.add_argument("--out", default="crawl.csv")
    ap.add_argument("--user-agent", default=DEFAULT_UA)
    ap.add_argument("--ignore-robots", action="store_true", help="only for sites you own, e.g. staging")
    args = ap.parse_args()

    start = normalize(args.start, args.start)
    if not start:
        print("error: start URL must be http(s)", file=sys.stderr)
        return 2
    c = Crawler(start, args.user_agent, args.delay, args.max_pages, args.ignore_robots)
    c.crawl()
    first = c.pages.get(start)
    if not first or first["status"] == 0:
        print(f"error: could not fetch start URL {start}: {first and first['error']}", file=sys.stderr)
        return 2

    found = issues_for(c)
    per_url: dict[str, list[str]] = defaultdict(list)
    for (sev, label), urls in found.items():
        for u in urls:
            per_url[u.split(" -> ")[0]].append(f"{sev}: {label}")
    with open(args.out, "w", newline="", encoding="utf-8") as fh:
        w = csv.writer(fh)
        w.writerow(["url", "status", "final_url", "hops", "depth", "inlinks", "title", "canonical",
                    "robots", "h1_count", "issues"])
        for url, p in c.pages.items():
            w.writerow([url, p["status"], p["final"], p["hops"], c.depth.get(url, ""),
                        len(c.inlinks.get(url, ())), p["title"], p["canonical"], p["robots"], p["h1"],
                        " | ".join(per_url.get(url, []))])

    order = {"ERROR": 0, "WARN": 1, "INFO": 2}
    print(f"Crawled {len(c.pages)} URLs on {c.host} (limit {args.max_pages}); details in {args.out}\n")
    for (sev, label), urls in sorted(found.items(), key=lambda kv: (order[kv[0][0]], -len(kv[1]))):
        print(f"[{sev}] {label}: {len(urls)}")
        for u in urls[:5]:
            print(f"    {u}")
        if len(urls) > 5:
            print(f"    ... {len(urls) - 5} more")
    if not found:
        print("No issues found.")
    errors = sum(len(v) for (sev, _), v in found.items() if sev == "ERROR")
    return 1 if errors else 0


if __name__ == "__main__":
    sys.exit(main())