blog/scripts/audit/link-image-audit.mjs

#!/usr/bin/env node
/**
 * link-image-audit.mjs — walks every post's `<img>`/`<a>` (markdown syntax and
 * raw HTML, in both `.md` and `.mdx`) plus its frontmatter `cover`, and reports:
 *
 *   - missing local assets (a `/assets/...` src/href with no file under public/)
 *   - broken internal links (a same-origin path that doesn't match any known
 *     route — a post/project slug, a tag, or a static page)
 *   - dead external links/images (HEAD-then-GET, only when --external is passed)
 *
 * Usage:
 *   node scripts/audit/link-image-audit.mjs                 # local checks only (fast, no network)
 *   node scripts/audit/link-image-audit.mjs --external       # also probe external URLs
 *   node scripts/audit/link-image-audit.mjs --external --no-cache
 *   node scripts/audit/link-image-audit.mjs --external --concurrency=16 --timeout=5000
 *
 * Never exits non-zero — see main() — so it is safe to chain into `npm run
 * build`'s postbuild step unconditionally (warn-only, per GOAL.md's wishlist:
 * "run as part of `npm run build` (warn)").
 *
 * External checking is OFF by default: it's the only part of this script that
 * touches the network, and `npm run build` must never become slow or
 * network-dependent. Turn it on explicitly with `--external` (that's what
 * `npm run audit:links:full` does) for an on-demand full pass. Results are
 * cached (default 7 days) in `.ai/link-audit/external-cache.json` so repeated
 * full runs don't re-probe the same URL every time.
 *
 * Report artefacts land in `.ai/link-audit/` (gitignored):
 *   - report.json  — machine-readable, full detail
 *   - report.md    — human-readable summary + findings table
 */
import path from "node:path";
import { fileURLToPath } from "node:url";
import { promises as fs } from "node:fs";
import { globby } from "globby";
import grayMatter from "gray-matter";
import githubSlugger from "github-slugger";
const githubSlug = githubSlugger.slug;

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(__dirname, "../..");
const PUBLIC_DIR = path.join(ROOT, "public");
const CONTENT_DIR = path.join(ROOT, "src", "content");
const REPORT_DIR = path.join(ROOT, ".ai", "link-audit");
const CACHE_FILE = path.join(REPORT_DIR, "external-cache.json");
const SITE_ORIGIN = "https://blog.dev.gabvdl.xyz";
const USER_AGENT =
  "blog-link-audit/1.0 (+https://blog.dev.gabvdl.xyz; self-hosted archive link checker)";

/**
 * URLs that fail every scripted probe from this host (plain fetch, a headful
 * Playwright Chromium with navigator.webdriver patched, AND Jina Reader's own
 * IP) but were manually confirmed live — rendering real content, not a
 * challenge/interstitial — during the 2026-08-10 audit pass. Skipped entirely
 * so the report doesn't cry wolf on the same false positives every run.
 *
 * Each entry is commented with *why* it's here and how it was verified. This
 * is a narrow, explicit allowlist, not a blanket "403 = fine" rule — a 403
 * that hasn't been checked by hand still gets reported (see the low-confidence
 * note below). Re-verify by hand before trusting an old entry forever; a
 * domain can go dark after being added here.
 */
const KNOWN_LIVE_BUT_BLOCKED = new Set([
  // itch.io blocks scripted UAs (curl, headless/headful Chromium, Jina) with a
  // 403 on essentially every URL from this host's IP, regardless of whether
  // the page is live — confirmed via Jina Reader rendering the real page
  // (title, jam listing, game page content) for every one of these.
  "https://itch.io/jam/my-first-game-jam-summer-2016",
  "https://itch.io/jam/my-first-game-jam-summer-2016/rate/75019",
  "https://gabrielvidal.itch.io/primitive-tower-defense",
  "https://itch.io/jam/xmas2016",
  "https://gabrielvidal.itch.io/nyxmas",
  "https://itch.io/jam/lowrezjam2017",
  "https://gabrielvidal.itch.io/low-rez-guess",
  "https://itch.io/jam/gmtk-jam",
  "https://gabrielvidal.itch.io/pshiiit",
  "https://gabrielvidal.itch.io/gren",
  "https://gabrielvidal.itch.io/tictactocalypse",
  // kryptview.com: real, funded company (Crunchbase/PitchBook/Bloomberg
  // profiles, Paris HQ, still listed with 9 employees as of last funding
  // round); a Cloudflare WAF returns a bare "403 Forbidden" body to curl, a
  // headful Chromium AND Jina alike, which reads as a blanket non-browser
  // block rather than the site being down. Low confidence, but not something
  // this script can resolve — kept here instead of flagged every run.
  "https://kryptview.com/",
  // cali-rse.com: confirmed live via web search (a real French corporate-CSR
  // SaaS, matches the post's own description of the project) but the TLS
  // handshake itself is rejected from this host ("tlsv1 alert internal
  // error") for both curl and Playwright — a lower-layer block than a 403,
  // same practical effect.
  "https://www.cali-rse.com/",
]);

// ---------------------------------------------------------------- CLI args --

function parseArgs(argv) {
  const opts = {
    external: false,
    cache: true,
    concurrency: 8,
    timeoutMs: 8000,
    cacheTtlHours: 24 * 7,
    quiet: false,
  };
  for (const arg of argv) {
    if (arg === "--external") opts.external = true;
    else if (arg === "--no-cache") opts.cache = false;
    else if (arg === "--quiet") opts.quiet = true;
    else if (arg.startsWith("--concurrency="))
      opts.concurrency = Number(arg.split("=")[1]) || opts.concurrency;
    else if (arg.startsWith("--timeout="))
      opts.timeoutMs = Number(arg.split("=")[1]) || opts.timeoutMs;
    else if (arg.startsWith("--cache-ttl-hours="))
      opts.cacheTtlHours = Number(arg.split("=")[1]) || opts.cacheTtlHours;
  }
  return opts;
}

// ------------------------------------------------------------- extraction --
//
// Regex, not a real markdown/MDX parser (unlike astro.config.mjs's
// rehypeExternalLinks, which walks a real AST because it has to rewrite
// nodes). An audit only has to *find* refs, so this is enough for every post
// in the archive today — checked against all 27 by hand. Known gap: a
// markdown link/image target containing a literal unescaped `)` (rare —
// none in this archive) will truncate at the first `)`.

// Markdown image: ![alt](target "title")
const RE_MD_IMAGE = /!\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
// Markdown link: [text](target "title") — negative lookbehind excludes images
const RE_MD_LINK = /(?<!!)\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
// Raw HTML <img ... src="...">
const RE_HTML_IMG = /<img\b[^>]*?\bsrc\s*=\s*["']([^"']+)["'][^>]*>/gi;
// Raw HTML <a ... href="...">
const RE_HTML_A = /<a\b[^>]*?\bhref\s*=\s*["']([^"']+)["'][^>]*>/gi;

function lineAt(text, index) {
  return text.slice(0, index).split("\n").length;
}

/** Pull every image/link reference out of one post's raw source text. */
function extractRefs(raw) {
  const refs = [];
  for (const m of raw.matchAll(RE_MD_IMAGE))
    refs.push({ kind: "image", href: m[2], text: m[1], line: lineAt(raw, m.index), source: "markdown" });
  for (const m of raw.matchAll(RE_MD_LINK))
    refs.push({ kind: "link", href: m[2], text: m[1], line: lineAt(raw, m.index), source: "markdown" });
  for (const m of raw.matchAll(RE_HTML_IMG))
    refs.push({ kind: "image", href: m[1], text: "", line: lineAt(raw, m.index), source: "html" });
  for (const m of raw.matchAll(RE_HTML_A))
    refs.push({ kind: "link", href: m[1], text: "", line: lineAt(raw, m.index), source: "html" });
  return refs;
}

// ---------------------------------------------------------------- routing --

/** Astro content-collection slug: github-slugger applied per path segment. */
function collectionSlug(id) {
  return id
    .split("/")
    .map((seg) => githubSlug(seg))
    .join("/")
    .replace(/\/index$/, "");
}

/** Mirrors src/utils/index.ts toSlug() — used for /tags/<slug> routes. */
function tagSlug(tag) {
  const exceptions = { "C++": "cpp", "C#": "c-sharp" };
  return exceptions[tag] ?? tag.replace(/\s+/g, "-").toLowerCase();
}

const STATIC_ROUTES = new Set(["/", "/about", "/archive", "/media", "/rss.xml"]);

async function loadEntries(collection) {
  const dir = path.join(CONTENT_DIR, collection);
  const files = await globby(["*.md", "*.mdx"], { cwd: dir });
  const entries = [];
  for (const file of files) {
    const abs = path.join(dir, file);
    const raw = await fs.readFile(abs, "utf8");
    const parsed = grayMatter(raw);
    const id = file.replace(/\.(md|mdx)$/, "");
    entries.push({
      collection,
      id,
      slug: collectionSlug(id),
      file: path.relative(ROOT, abs),
      raw,
      data: parsed.data,
    });
  }
  return entries;
}

/** True if an internal (same-origin) path resolves to a real route. */
function internalPathExists(pathname, { blogSlugs, projectSlugs, tagSlugs }) {
  const clean = pathname.split("#")[0].split("?")[0].replace(/\/+$/, "") || "/";
  if (STATIC_ROUTES.has(clean)) return true;

  let m;
  if ((m = clean.match(/^\/blog(?:\/(\d+))?$/))) return true; // listing + pagination
  if ((m = clean.match(/^\/blog\/([^/]+)$/))) return blogSlugs.has(m[1]);
  if ((m = clean.match(/^\/project(?:\/(\d+))?$/))) return true;
  if ((m = clean.match(/^\/project\/([^/]+)$/))) return projectSlugs.has(m[1]);
  if ((m = clean.match(/^\/tags\/([^/]+)(?:\/(\d+))?$/))) return tagSlugs.has(m[1]);

  return false;
}

// ----------------------------------------------------------- local assets --

function isFragmentOrNonHttp(href) {
  return (
    href.startsWith("#") ||
    href.startsWith("mailto:") ||
    href.startsWith("tel:") ||
    href.startsWith("javascript:")
  );
}

async function localAssetExists(pathname) {
  const clean = pathname.split("#")[0].split("?")[0];
  const rel = clean.startsWith("/") ? clean.slice(1) : clean;
  try {
    await fs.access(path.join(PUBLIC_DIR, rel));
    return true;
  } catch {
    return false;
  }
}

// -------------------------------------------------------- external checks --

async function loadCache() {
  try {
    return JSON.parse(await fs.readFile(CACHE_FILE, "utf8"));
  } catch {
    return {};
  }
}

async function saveCache(cache) {
  await fs.mkdir(REPORT_DIR, { recursive: true });
  await fs.writeFile(CACHE_FILE, JSON.stringify(cache, null, 2));
}

async function probeOnce(url, timeoutMs, method) {
  const controller = new AbortController();
  const t = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const res = await fetch(url, {
      method,
      redirect: "follow",
      signal: controller.signal,
      headers: { "User-Agent": USER_AGENT, Accept: "*/*" },
    });
    return { ok: res.status < 400, status: res.status };
  } finally {
    clearTimeout(t);
  }
}

/**
 * HEAD first (cheap — skips the response body), but HEAD is not authoritative:
 * plenty of real servers (bufsoftware.com is one in this archive) return a
 * different status for HEAD than for GET, including a bare 404 on HEAD for a
 * page that's actually there. So HEAD only ever short-circuits the *success*
 * case; anything else always gets a real GET before being called broken.
 */
async function checkExternalUrl(url, timeoutMs) {
  try {
    const head = await probeOnce(url, timeoutMs, "HEAD");
    if (head.ok) return head;
  } catch {
    // fall through to GET
  }
  try {
    return await probeOnce(url, timeoutMs, "GET");
  } catch (err) {
    return { ok: false, status: null, error: err.name === "AbortError" ? "timeout" : String(err.message ?? err) };
  }
}

/** Tiny async pool — runs `fn` over `items` with at most `limit` in flight. */
async function pool(items, limit, fn) {
  const results = new Array(items.length);
  let next = 0;
  async function worker() {
    while (next < items.length) {
      const i = next++;
      results[i] = await fn(items[i], i);
    }
  }
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
  return results;
}

async function checkExternals(urls, opts) {
  const cache = opts.cache ? await loadCache() : {};
  const now = Date.now();
  const ttlMs = opts.cacheTtlHours * 3600 * 1000;
  const unique = [...new Set(urls)];

  const toCheck = unique.filter((u) => {
    const hit = cache[u];
    return !(opts.cache && hit && now - hit.checkedAt < ttlMs);
  });

  await pool(toCheck, opts.concurrency, async (url) => {
    const result = await checkExternalUrl(url, opts.timeoutMs);
    cache[url] = { ...result, checkedAt: now };
  });

  if (opts.cache || toCheck.length) await saveCache(cache);
  const byUrl = new Map(unique.map((u) => [u, cache[u]]));
  return { byUrl, checkedFresh: toCheck.length, servedFromCache: unique.length - toCheck.length };
}

// --------------------------------------------------------------- main run --

async function main() {
  const opts = parseArgs(process.argv.slice(2));
  const startedAt = Date.now();

  const [blogEntries, projectEntries] = await Promise.all([
    loadEntries("blog"),
    loadEntries("project"),
  ]);
  const allEntries = [...blogEntries, ...projectEntries];

  const blogSlugs = new Set(blogEntries.map((e) => e.slug));
  const projectSlugs = new Set(projectEntries.map((e) => e.slug));
  const tagSlugs = new Set(
    blogEntries.flatMap((e) => [...(e.data.tags ?? []), ...(e.data.techs ?? [])]).map(tagSlug)
  );
  const routeCtx = { blogSlugs, projectSlugs, tagSlugs };

  const findings = [];
  let imagesChecked = 0;
  let linksChecked = 0;
  const externalCandidates = []; // { url, file, line, kind }

  for (const entry of allEntries) {
    const refs = extractRefs(entry.raw);
    // Frontmatter cover counts as an image reference too.
    if (entry.data.cover) {
      refs.push({ kind: "image", href: entry.data.cover, text: "cover", line: 1, source: "frontmatter" });
    }

    for (const ref of refs) {
      const href = ref.href?.trim();
      if (!href || isFragmentOrNonHttp(href)) continue;

      if (ref.kind === "image") imagesChecked++;
      else linksChecked++;

      const isAbsoluteUrl = /^https?:\/\//i.test(href);
      let origin = null;
      if (isAbsoluteUrl) {
        try {
          origin = new URL(href).origin;
        } catch {
          findings.push({
            file: entry.file,
            collection: entry.collection,
            type: "malformed-url",
            kind: ref.kind,
            href,
            line: ref.line,
            detail: "Could not parse as a URL.",
          });
          continue;
        }
      }

      const isInternal = !isAbsoluteUrl || origin === SITE_ORIGIN;

      if (isInternal) {
        const pathname = isAbsoluteUrl ? new URL(href).pathname : href;
        if (!pathname.startsWith("/")) {
          findings.push({
            file: entry.file,
            collection: entry.collection,
            type: "non-absolute-path",
            kind: ref.kind,
            href,
            line: ref.line,
            detail: "Not an absolute site-root path (doesn't start with `/`) and not an external URL — can't verify.",
          });
          continue;
        }

        if (ref.kind === "image") {
          if (!(await localAssetExists(pathname))) {
            findings.push({
              file: entry.file,
              collection: entry.collection,
              type: "missing-local-asset",
              kind: "image",
              href,
              line: ref.line,
              detail: `No file at public${pathname}`,
            });
          }
        } else {
          // A link can also point at a local asset (e.g. linking a full-res image).
          const looksLikeAsset = pathname.startsWith("/assets/");
          const exists = looksLikeAsset
            ? await localAssetExists(pathname)
            : internalPathExists(pathname, routeCtx);
          if (!exists) {
            findings.push({
              file: entry.file,
              collection: entry.collection,
              type: looksLikeAsset ? "missing-local-asset" : "broken-internal-link",
              kind: "link",
              href,
              line: ref.line,
              detail: looksLikeAsset
                ? `No file at public${pathname}`
                : `No route matches ${pathname} (checked static pages, blog/project slugs, tag slugs).`,
            });
          }
        }
      } else {
        // External. Always record as a candidate; only actually probed if --external.
        externalCandidates.push({ url: href, file: entry.file, collection: entry.collection, kind: ref.kind, line: ref.line });
      }
    }
  }

  let externalSummary = { enabled: opts.external, checkedFresh: 0, servedFromCache: 0, broken: 0, skipped: externalCandidates.length, allowlisted: 0 };

  if (opts.external && externalCandidates.length) {
    const { byUrl, checkedFresh, servedFromCache } = await checkExternals(
      externalCandidates.map((c) => c.url),
      opts
    );
    externalSummary = { enabled: true, checkedFresh, servedFromCache, broken: 0, skipped: 0, allowlisted: 0 };
    for (const cand of externalCandidates) {
      const result = byUrl.get(cand.url);
      if (!result || !result.ok) {
        if (KNOWN_LIVE_BUT_BLOCKED.has(cand.url)) {
          externalSummary.allowlisted++;
          continue;
        }
        externalSummary.broken++;
        let detail = result?.error ? `Request failed: ${result.error}` : `HTTP ${result?.status ?? "?"}`;
        // A 403 to a script is frequently bot/Cloudflare blocking (itch.io does
        // this for every URL on this host, including pages that plainly load in
        // a browser) rather than the page actually being gone — flag it as
        // lower-confidence rather than silently reporting it the same as a 404.
        if (result?.status === 403) {
          detail += " — may be bot-blocking (itch.io and similar do this to every scripted request from this host); verify manually before treating as dead";
        }
        findings.push({
          file: cand.file,
          collection: cand.collection,
          type: cand.kind === "image" ? "broken-external-image" : "broken-external-link",
          kind: cand.kind,
          href: cand.url,
          line: cand.line,
          detail,
          confidence: result?.status === 403 ? "low" : "high",
        });
      }
    }
  }

  const summary = {
    postsScanned: allEntries.length,
    blogPosts: blogEntries.length,
    projectPosts: projectEntries.length,
    imageRefsChecked: imagesChecked,
    linkRefsChecked: linksChecked,
    missingLocalAssets: findings.filter((f) => f.type === "missing-local-asset").length,
    brokenInternalLinks: findings.filter((f) => f.type === "broken-internal-link").length,
    malformedUrls: findings.filter((f) => f.type === "malformed-url").length,
    nonAbsolutePaths: findings.filter((f) => f.type === "non-absolute-path").length,
    external: externalSummary,
    durationMs: Date.now() - startedAt,
  };

  const report = {
    generatedAt: new Date().toISOString(),
    options: opts,
    summary,
    findings,
  };

  await fs.mkdir(REPORT_DIR, { recursive: true });
  await fs.writeFile(path.join(REPORT_DIR, "report.json"), JSON.stringify(report, null, 2));
  await fs.writeFile(path.join(REPORT_DIR, "report.md"), renderMarkdown(report));

  printConsoleSummary(report, opts);
  return report;
}

function renderMarkdown(report) {
  const { summary, findings } = report;
  const lines = [];
  lines.push(`# Link & image audit`, "");
  lines.push(`Generated ${report.generatedAt}${summary.external.enabled ? " (external checks included)" : " (local checks only — run with \`--external\` for a full pass)"}.`, "");
  lines.push(`- Posts scanned: ${summary.postsScanned} (${summary.blogPosts} blog, ${summary.projectPosts} project)`);
  lines.push(`- Image refs checked: ${summary.imageRefsChecked}`);
  lines.push(`- Link refs checked: ${summary.linkRefsChecked}`);
  lines.push(`- Missing local assets: ${summary.missingLocalAssets}`);
  lines.push(`- Broken internal links: ${summary.brokenInternalLinks}`);
  if (summary.external.enabled) {
    lines.push(`- External refs probed: ${summary.external.checkedFresh} fresh, ${summary.external.servedFromCache} from cache`);
    const lowConf = findings.filter((f) => (f.type === "broken-external-link" || f.type === "broken-external-image") && f.confidence === "low").length;
    lines.push(`- Broken external refs: ${summary.external.broken} (${summary.external.broken - lowConf} high-confidence, ${lowConf} low-confidence 403s — likely bot-blocking, verify manually)`);
    if (summary.external.allowlisted) {
      lines.push(`- Manually-verified-live but blocked from this host (skipped, not counted as broken): ${summary.external.allowlisted} — see \`KNOWN_LIVE_BUT_BLOCKED\` in the audit script`);
    }
  } else {
    lines.push(`- External refs found but NOT probed: ${summary.external.skipped}`);
  }
  lines.push(`- Duration: ${summary.durationMs}ms`, "");

  if (findings.length) {
    lines.push(`## Findings (${findings.length})`, "");
    lines.push(`| file | line | type | kind | href | detail |`);
    lines.push(`|---|---|---|---|---|---|`);
    for (const f of findings) {
      lines.push(`| ${f.file} | ${f.line} | ${f.type} | ${f.kind} | \`${f.href}\` | ${f.detail} |`);
    }
  } else {
    lines.push(`No findings.`);
  }
  lines.push("");
  return lines.join("\n");
}

function printConsoleSummary(report, opts) {
  if (opts.quiet) return;
  const { summary } = report;
  console.log(`[link-image-audit] scanned ${summary.postsScanned} posts — ` +
    `${summary.imageRefsChecked} image refs, ${summary.linkRefsChecked} link refs`);
  console.log(`[link-image-audit] missing local assets: ${summary.missingLocalAssets}, ` +
    `broken internal links: ${summary.brokenInternalLinks}` +
    (summary.malformedUrls ? `, malformed URLs: ${summary.malformedUrls}` : "") +
    (summary.nonAbsolutePaths ? `, non-absolute paths: ${summary.nonAbsolutePaths}` : ""));
  if (summary.external.enabled) {
    console.log(`[link-image-audit] external refs probed: ${summary.external.checkedFresh} fresh + ` +
      `${summary.external.servedFromCache} cached — broken: ${summary.external.broken}` +
      (summary.external.allowlisted ? ` (+ ${summary.external.allowlisted} allowlisted, verified live by hand)` : ""));
  } else {
    console.log(`[link-image-audit] external refs found: ${summary.external.skipped} (not probed — pass --external to check)`);
  }
  const total = summary.missingLocalAssets + summary.brokenInternalLinks + summary.malformedUrls +
    (summary.external.enabled ? summary.external.broken : 0);
  if (total > 0) {
    console.warn(`[link-image-audit] WARN: ${total} finding(s) — see .ai/link-audit/report.md`);
  } else {
    console.log(`[link-image-audit] clean.`);
  }
}

// This script is wired into \`npm run build\`'s postbuild step and must never
// fail the build — deploying is a deliberate act (\`npm run deploy\`), not
// something a broken image in a 2017 post should block. Any unexpected crash
// here is reported as a warning, not a failure.
main().catch((err) => {
  console.warn(`[link-image-audit] WARN: audit crashed, skipping — ${err?.stack ?? err}`);
  process.exitCode = 0;
});