| author | Alan Dipert
<alan@tailrecursion.com> 2026-07-16 19:52:48 UTC |
| committer | Alan Dipert
<alan@tailrecursion.com> 2026-07-16 19:52:48 UTC |
| parent | e4f7f83e0b88dafe0ee830b290f42cdbbac31774 |
| .containerignore | +1 | -0 |
| AGENTS.md | +2 | -2 |
| Containerfile | +6 | -0 |
| Makefile | +62 | -53 |
| README.md | +7 | -2 |
| md/CoffeeTime.md | +1 | -1 |
| md/Home.md | +2 | -5 |
| md/Home/donuts_alan_float_right_392px.jpg | +0 | -0 |
| md/Home/headshot_2018_float_right_200px.jpg | +0 | -0 |
| source-assets/Home/donuts_alan_2x_blended.png | +0 | -0 |
| tests/test_sitegen.py | +162 | -0 |
| tools/build_page.py | +0 | -235 |
| tools/buildinfo.py | +3 | -0 |
| tools/check_links.py | +30 | -88 |
| tools/gen_index.py | +0 | -140 |
| tools/gen_updates_rss.py | +0 | -287 |
| tools/index_list.awk | +0 | -91 |
| tools/mdlink2html.py | +0 | -71 |
| tools/normalize_output.py | +7 | -3 |
| tools/sitegen.py | +562 | -0 |
| tools/validate_rss.py | +0 | -54 |
| tools/verify_reproducible.py | +4 | -2 |
| tools/xml2md.py | +0 | -337 |
| tpl/style.css | +116 | -17 |
diff --git a/.containerignore b/.containerignore index 4281de1..2c31441 100644 --- a/.containerignore +++ b/.containerignore @@ -2,6 +2,7 @@ out OUT drafts +source-assets vendor __pycache__ *.pyc diff --git a/AGENTS.md b/AGENTS.md index 2009b63..6edebcd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,8 +14,8 @@ Welcome! This document describes how automated helpers should operate when worki - The repo targets standard POSIX tooling (`make`, `awk`, `rsync`, `cmark-gfm`); avoid introducing extra build dependencies. - Always run `make assets && make` after touching Markdown, templates, or build scripts unless instructed otherwise. - `make deploy` is reserved for publishing—never call it unless explicitly asked and the git worktree is clean. -- Ensure `tools/mdlink2html.awk` and build helpers continue to support wiki-style intra-site links. -- Keep the generated `out/Index.html` in sync; the `tools/gen_index.sh` helper runs automatically from `make`. +- Ensure `tools/sitegen.py` continues to support wiki-style intra-site links. +- Keep the generated `out/Index.html` in sync; the site-wide generator runs automatically from `make`. - Keep updates idempotent: don’t delete generated output or vendor directories unless instructed. - When in doubt about improvements, have the AI tour the site like a new reader (browser pass, no edits) and capture TODOs for confusing spots or visual glitches before proposing changes. - Backlinks: every page renders a “Backlinks” block built from the cmark AST; keep it working whenever you touch build tooling. diff --git a/Containerfile b/Containerfile index 94a235d..e05e25d 100644 --- a/Containerfile +++ b/Containerfile @@ -1,6 +1,10 @@ FROM docker.io/library/debian:13-slim@sha256:020c0d20b9880058cbe785a9db107156c3c75c2ac944a6aa7ab59f2add76a7bd ARG DEBIAN_SNAPSHOT=20260715T000000Z +ARG DEBIAN_FRONTEND=noninteractive + +LABEL org.opencontainers.image.title="Alan Dipert homepage builder" \ + org.opencontainers.image.source="https://tailrecursion.com/git-arr/r/homepage.git/" RUN rm -f /etc/apt/sources.list /etc/apt/sources.list.d/debian.sources \ && printf '%s\n' \ @@ -16,6 +20,8 @@ RUN rm -f /etc/apt/sources.list /etc/apt/sources.list.d/debian.sources \ make \ python3 \ rsync \ + && cmark-gfm --version \ + && python3 --version \ && rm -rf /var/lib/apt/lists/* ENV LANG=C.UTF-8 \ diff --git a/Makefile b/Makefile index 1d6db48..8abeb39 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,10 @@ -.PHONY: all assets clean deploy help tree check-git-clean check-links check-tools \ +.PHONY: all site assets clean deploy help test tree check-git-clean check-output-path check-pushed check-links check-tools \ fresh container-image container-build container-check-links \ verify-reproducible verify-reproducible-inner +.DELETE_ON_ERROR: +.NOTPARALLEL: + SRC := md OUT ?= out HEAD := tpl/head.html @@ -10,66 +13,52 @@ CSS := tpl/style.css MD2HTML ?= cmark-gfm CMARK_FLAGS := --to html --extension table --validate-utf8 --unsafe PYTHON ?= python3 -BUILDINFO := tools/buildinfo.py -BUILD_PAGE := tools/build_page.py -GEN_INDEX := tools/gen_index.py +SITEGEN := tools/sitegen.py NORMALIZE_OUTPUT := tools/normalize_output.py VERIFY_OUTPUT := tools/verify_reproducible.py -INDEX_HTML := $(OUT)/Index.html DEPLOY_HOST ?= arsien23i2@pdx1-shared-a2-13.dreamhost.com:tailrecursion.com/~alan SITE_BASE_URL ?= https://tailrecursion.com/~alan/ CONTAINER_ENGINE ?= podman BUILDER_IMAGE ?= localhost/homepage-builder:2026-07-15 SOURCE_DATE_EPOCH ?= $(shell git log -1 --format=%ct HEAD 2>/dev/null) -STAGE_OUT ?= /tmp/homepage-out export SOURCE_DATE_EPOCH -MD_FILES := $(shell find $(SRC) -type f -name '*.md' | LC_ALL=C sort) -HTML := $(patsubst $(SRC)/%.md,$(OUT)/%.html,$(MD_FILES)) -HOME_MD := md/Home.md -UPDATES_FEED := $(OUT)/updates.xml - -all: check-tools assets $(OUT)/style.css $(HTML) $(INDEX_HTML) $(UPDATES_FEED) +all: check-tools assets $(OUT)/style.css site check-tools: - @command -v "$(MD2HTML)" >/dev/null 2>&1 || { \ - echo 'Missing Markdown renderer: $(MD2HTML)' >&2; \ - echo 'Use make container-build, or install cmark-gfm and set MD2HTML.' >&2; \ - exit 1; \ - } + @for command in "$(MD2HTML)" "$(PYTHON)" git rsync; do \ + command -v "$$command" >/dev/null 2>&1 || { \ + echo "Missing build tool: $$command" >&2; \ + echo 'Use make container-build or install the documented local tools.' >&2; \ + exit 1; \ + }; \ + done + @$(PYTHON) -c 'import sys; assert sys.version_info >= (3, 11), "Python 3.11 or newer is required"' $(OUT)/style.css: $(CSS) mkdir -p $(OUT) cp $(CSS) $@ -$(OUT): - mkdir -p $(OUT) - -$(OUT)/%.html: $(SRC)/%.md $(HEAD) $(FOOT) tools/mdlink2html.py $(BUILD_PAGE) $(BUILDINFO) | $(OUT) - OUT_DIR="$(OUT)" MD2HTML="$(MD2HTML)" CMARK_FLAGS="$(CMARK_FLAGS)" \ - $(PYTHON) $(BUILD_PAGE) "$@" "$<" "$(HEAD)" "$(FOOT)" tools/mdlink2html.py - assets: mkdir -p $(OUT) rsync -a --include='*/' --exclude='*.md' --exclude='*.MD' \ --exclude='*~' --exclude='.#*' --exclude='#*#' --exclude='*.swp' \ --exclude='.DS_Store' --prune-empty-dirs $(SRC)/ $(OUT)/ - -$(INDEX_HTML): $(HTML) $(GEN_INDEX) $(HEAD) $(FOOT) $(BUILDINFO) | $(OUT) - $(PYTHON) $(GEN_INDEX) $@ $(SRC) - -$(UPDATES_FEED): $(HOME_MD) tools/gen_updates_rss.py tools/mdlink2html.py tools/validate_rss.py | $(OUT) - SITE_BASE_URL="$(SITE_BASE_URL)" $(PYTHON) tools/gen_updates_rss.py "$(HOME_MD)" "$@" - $(PYTHON) tools/validate_rss.py "$@" - -fresh: - rm -rf "$(STAGE_OUT)" - $(MAKE) OUT="$(STAGE_OUT)" all - $(PYTHON) $(NORMALIZE_OUTPUT) "$(STAGE_OUT)" "$(SOURCE_DATE_EPOCH)" - mkdir -p "$(OUT)" - rsync -a --delete --chmod=F644,D755 "$(STAGE_OUT)/" "$(OUT)/" - rm -rf "$(STAGE_OUT)" +site: + OUT_DIR="$(OUT)" MD2HTML="$(MD2HTML)" CMARK_FLAGS="$(CMARK_FLAGS)" \ + SITE_BASE_URL="$(SITE_BASE_URL)" $(PYTHON) $(SITEGEN) \ + --src "$(SRC)" --out "$(OUT)" --head "$(HEAD)" --foot "$(FOOT)" + +fresh: check-output-path test + @set -eu; \ + test -n "$(SOURCE_DATE_EPOCH)" || { echo 'SOURCE_DATE_EPOCH is empty' >&2; exit 1; }; \ + stage=$$(mktemp -d "$${TMPDIR:-/tmp}/homepage-out.XXXXXX"); \ + trap 'rm -rf "$$stage"' EXIT HUP INT TERM; \ + $(MAKE) OUT="$$stage" all; \ + $(PYTHON) $(NORMALIZE_OUTPUT) "$$stage" "$(SOURCE_DATE_EPOCH)"; \ + mkdir -p "$(OUT)"; \ + rsync -a --delete --checksum --chmod=F644,D755 "$$stage/" "$(OUT)/" container-image: $(CONTAINER_ENGINE) build --pull=missing --tag "$(BUILDER_IMAGE)" --file Containerfile . @@ -92,29 +81,34 @@ verify-reproducible: container-image --env SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" \ "$(BUILDER_IMAGE)" make verify-reproducible-inner -verify-reproducible-inner: - rm -rf /tmp/homepage-out-a /tmp/homepage-out-b - $(MAKE) OUT=/tmp/homepage-out-a all - $(MAKE) OUT=/tmp/homepage-out-b all - $(PYTHON) $(NORMALIZE_OUTPUT) /tmp/homepage-out-a "$(SOURCE_DATE_EPOCH)" - $(PYTHON) $(NORMALIZE_OUTPUT) /tmp/homepage-out-b "$(SOURCE_DATE_EPOCH)" - $(PYTHON) $(VERIFY_OUTPUT) /tmp/homepage-out-a /tmp/homepage-out-b - rm -rf /tmp/homepage-out-a /tmp/homepage-out-b - -deploy: check-git-clean +verify-reproducible-inner: test + @set -eu; \ + first=$$(mktemp -d "$${TMPDIR:-/tmp}/homepage-a.XXXXXX"); \ + second=$$(mktemp -d "$${TMPDIR:-/tmp}/homepage-b.XXXXXX"); \ + trap 'rm -rf "$$first" "$$second"' EXIT HUP INT TERM; \ + $(MAKE) OUT="$$first" all; \ + $(MAKE) OUT="$$second" all; \ + $(PYTHON) $(NORMALIZE_OUTPUT) "$$first" "$(SOURCE_DATE_EPOCH)"; \ + $(PYTHON) $(NORMALIZE_OUTPUT) "$$second" "$(SOURCE_DATE_EPOCH)"; \ + $(PYTHON) $(VERIFY_OUTPUT) "$$first" "$$second" + +deploy: check-git-clean check-pushed $(MAKE) container-build @if [ -z "$(DEPLOY_HOST)" ]; then \ echo "DEPLOY_HOST is not set"; \ exit 1; \ fi - rsync -a --chmod=F644,D755 $(OUT)/ $(DEPLOY_HOST) - git push + rsync -a --delete --delay-updates --chmod=F644,D755 $(OUT)/ $(DEPLOY_HOST) tree: printf 'Markdown sources: ' && find $(SRC) -type f -name '*.md' | wc -l printf 'HTML outputs: ' && if [ -d $(OUT) ]; then find $(OUT) -type f -name '*.html' | wc -l; else echo 0; fi -clean: +test: + PYTHONPATH=tools $(PYTHON) -m unittest discover -s tests -v + $(PYTHON) -m compileall -q tools tests + +clean: check-output-path rm -rf $(OUT) help: @@ -126,8 +120,9 @@ help: @echo ' make - build locally (requires cmark-gfm)' @echo ' make deploy - container-build, then publish from the host' @echo ' make tree - count source and generated files' + @echo ' make test - run standard-library unit tests' @echo ' make clean - remove $(OUT)/' - @echo ' (deploy refuses to run with a dirty git worktree)' + @echo ' (deploy requires a clean worktree matching its upstream branch)' check-git-clean: @if [ -n "$$(git status --porcelain)" ]; then \ @@ -136,5 +131,19 @@ check-git-clean: exit 1; \ fi +check-output-path: + @case "$(OUT)" in ''|'/'|'.'|'..') \ + echo "Refusing unsafe output path: $(OUT)" >&2; exit 1 ;; \ + esac + +check-pushed: + @upstream=$$(git rev-parse --abbrev-ref '@{upstream}' 2>/dev/null) || { \ + echo 'Current branch has no upstream.' >&2; exit 1; \ + }; \ + test "$$(git rev-parse HEAD)" = "$$(git rev-parse "$$upstream")" || { \ + echo "Refusing to deploy: HEAD does not match $$upstream. Push first." >&2; \ + exit 1; \ + } + check-links: @python3 tools/check_links.py diff --git a/README.md b/README.md index bc547bb..8834e51 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,22 @@ xdg-open out/Index.html # or open on macOS The builder uses rootless Podman and leaves `out/` owned by the invoking user. It builds into a fresh staging directory, normalizes timestamps and permissions, and then replaces the generated tree so stale files cannot survive between builds. +`tools/sitegen.py` performs one deterministic site-wide pass: it parses each Markdown document once for backlinks, then writes the pages, index, and RSS feed. The build fails when generated HTML refers to a missing local page or asset. + +Lossless masters for optimized site images live in `source-assets/`. They are retained in Git but excluded from both the container context and published site. + To prove that two isolated builds produce identical files and metadata: ```sh make verify-reproducible +make test ``` The `Containerfile` pins both the Debian base-image digest and a dated Debian package snapshot. Update those two values together when intentionally refreshing the toolchain, rebuild with `make container-image`, and rerun the reproducibility check. ## Local fallback -The Make pipeline can still run without Podman when `cmark-gfm`, Python 3, rsync, Make, and Git are installed locally: +The Make pipeline can still run without Podman when `cmark-gfm`, Python 3.11 or newer, rsync, Make, and Git are installed locally: ```sh make assets && make @@ -49,4 +54,4 @@ This check contacts external sites and is diagnostic; it is not part of the byte make deploy ``` -Deploy first runs the reproducible container build, then uses host-side rsync and Git credentials to publish to `DEPLOY_HOST`. It refuses to run unless the Git worktree is clean. The container never receives SSH keys or Git credentials. +Deploy first runs the reproducible container build, then uses host-side rsync and Git credentials to publish to `DEPLOY_HOST`. It refuses to run unless the Git worktree is clean and `HEAD` matches its upstream branch. The container never receives SSH keys or Git credentials. diff --git a/md/CoffeeTime.md b/md/CoffeeTime.md index 75eed65..937b297 100644 --- a/md/CoffeeTime.md +++ b/md/CoffeeTime.md @@ -5,7 +5,7 @@ Created Monday 23 November 2020 Would you like to meet with me? Let's do it! I try to make time to reconnect with old friends and to make new ones. -You can book me at Calendly: <https://calendly.com/alan-dipert/30min> +You can book me at Calendly: <https://calendly.com/alan-dipert/meet-with-alan> Zoom and Google Meet are my go-to tools but I'm open to trying others. diff --git a/md/Home.md b/md/Home.md index ec35355..4ddc91c 100644 --- a/md/Home.md +++ b/md/Home.md @@ -1,14 +1,11 @@ # Alan's Homepage - + Hello, and welcome to the personal [homepage](https://en.wikipedia.org/wiki/Home_page) and [wiki](https://en.wikipedia.org/wiki/Wiki) of Alan Dipert! ## Explore -* [Consulting](./ConsultingPractice.md) -* [Background](./PersonalBackground.md) -* [Technical work](./TechWorks.md) -* [Essays](./Writings.md) +<!-- latest-works:6 --> ## Connect diff --git a/md/Home/donuts_alan_float_right_392px.jpg b/md/Home/donuts_alan_float_right_392px.jpg new file mode 100644 index 0000000..b62b711 Binary files /dev/null and b/md/Home/donuts_alan_float_right_392px.jpg differ diff --git a/md/Home/headshot_2018_float_right_200px.jpg b/md/Home/headshot_2018_float_right_200px.jpg deleted file mode 100644 index dc43b54..0000000 Binary files a/md/Home/headshot_2018_float_right_200px.jpg and /dev/null differ diff --git a/source-assets/Home/donuts_alan_2x_blended.png b/source-assets/Home/donuts_alan_2x_blended.png new file mode 100644 index 0000000..373b335 Binary files /dev/null and b/source-assets/Home/donuts_alan_2x_blended.png differ diff --git a/tests/test_sitegen.py b/tests/test_sitegen.py new file mode 100644 index 0000000..74cf0a5 --- /dev/null +++ b/tests/test_sitegen.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from xml.etree import ElementTree as ET + +import sitegen +import normalize_output +import verify_reproducible + + +class SitegenTests(unittest.TestCase): + def test_rewrite_uses_filesystem_to_distinguish_pages_and_assets(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "Page.md" + source.write_text("", encoding="utf-8") + (root / "Target.md").write_text("", encoding="utf-8") + (root / "Dockerfile").write_text("", encoding="utf-8") + + text = ( + '<a href="./Target">page</a> <a href="./Dockerfile">asset</a> ' + '<a href="./Target.md">explicit</a>' + ) + self.assertEqual( + sitegen.rewrite_html_links(text, source), + '<a href="./Target.html">page</a> <a href="./Dockerfile">asset</a> ' + '<a href="./Target.html">explicit</a>', + ) + + def test_rewrite_preserves_external_links_and_fragments(self) -> None: + source = Path("Page.md") + text = '<a href="https://example.com/a.md">web</a> <a href="#hello">section</a>' + self.assertEqual(sitegen.rewrite_html_links(text, source), text) + + def test_backlinks_are_sorted_and_deduplicated(self) -> None: + with tempfile.TemporaryDirectory() as directory: + src = Path(directory) + first = src / "First.md" + second = src / "Second.md" + target = src / "Target.md" + for path in (first, second, target): + path.write_text("", encoding="utf-8") + + def document(*destinations: str) -> ET.Element: + root = ET.Element(f"{sitegen.NS}document") + for destination in destinations: + ET.SubElement(root, f"{sitegen.NS}link", destination=destination) + return root + + result = sitegen.backlink_map( + src, + { + first: document("./Target.md", "./Target.md"), + second: document("./Target.md"), + target: document(), + }, + ) + self.assertEqual(result, {"Target.md": ["First.md", "Second.md"]}) + + def test_internal_reference_validation_rejects_missing_files(self) -> None: + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) + page = out / "Page.html" + page.write_text('<a href="Missing.html">missing</a>', encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, "Missing.html does not exist"): + sitegen.validate_internal_references(out) + + def test_rss_safely_splits_cdata_terminator(self) -> None: + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) + update = sitegen.Update( + datetime(2026, 1, 1, tzinfo=timezone.utc), + "Example", + "https://example.com/", + "<p>]]></p>", + "urn:test", + ) + sitegen.write_rss(out, [update], "https://example.com/") + ET.parse(out / "updates.xml") + + def test_latest_works_are_local_unique_existing_pages(self) -> None: + with tempfile.TemporaryDirectory() as directory: + src = Path(directory) + home = src / "Home.md" + home.write_text("", encoding="utf-8") + for name in ("First.md", "Second.md"): + (src / name).write_text("", encoding="utf-8") + + def update(day: int, destination: str, title: str = "Work") -> sitegen.Update: + return sitegen.Update( + datetime(2026, 1, day, tzinfo=timezone.utc), + title, + "https://example.com/", + "<p>Work</p>", + f"urn:{day}:{destination}", + source_destination=destination, + work_title=title, + ) + + updates = [ + update(7, "https://example.com/external.md"), + update(6, "./First.md", "First"), + update(5, "./First.md#again", "Duplicate"), + update(4, "./missing.md"), + update(3, "./image.jpg"), + update(2, "./Second.md", "Second"), + ] + self.assertEqual( + [item.work_title for item in sitegen.latest_local_works(updates, home, src)], + ["First", "Second"], + ) + + def test_homepage_tiles_preserve_title_date_and_link_to_updates(self) -> None: + with tempfile.TemporaryDirectory() as directory: + src = Path(directory) + home = src / "Home.md" + home.write_text("", encoding="utf-8") + (src / "Work.md").write_text("", encoding="utf-8") + update = sitegen.Update( + datetime(2026, 7, 15, tzinfo=timezone.utc), + "Work", + "https://example.com/Work.html", + "<p>Work</p>", + "urn:work", + source_destination="./Work.md", + work_title="Lisp:Work & Play", + ) + body = "<h2>Explore</h2>\n<!-- latest-works:6 -->\n<h2>Updates</h2>" + rendered = sitegen.enhance_homepage(body, [update], home, src) + self.assertIn('href="./Work.html"', rendered) + self.assertIn("Lisp:Work & Play", rendered) + self.assertIn('<time datetime="2026-07-15">15 Jul 2026</time>', rendered) + self.assertIn('<span class="work-excerpt">Work…</span>', rendered) + self.assertIn('<p class="explore-more"><a href="#updates">More</a></p>', rendered) + self.assertIn('<h2 id="updates">Updates</h2>', rendered) + + def test_normalized_manifests_include_root_metadata(self) -> None: + with tempfile.TemporaryDirectory() as first_dir, tempfile.TemporaryDirectory() as second_dir: + first = Path(first_dir) + second = Path(second_dir) + (first / "file").write_text("same", encoding="utf-8") + (second / "file").write_text("same", encoding="utf-8") + normalize_output.normalize(first, 123) + normalize_output.normalize(second, 123) + manifest = verify_reproducible.manifest(first) + self.assertEqual(manifest, verify_reproducible.manifest(second)) + self.assertIn(".", manifest) + + def test_normalization_rejects_symbolic_links(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "target").write_text("target", encoding="utf-8") + (root / "link").symlink_to("target") + with self.assertRaisesRegex(SystemExit, "symbolic link"): + normalize_output.normalize(root, 123) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/build_page.py b/tools/build_page.py deleted file mode 100755 index 634ee94..0000000 --- a/tools/build_page.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python3 -"""Render a Markdown file into HTML using project templates.""" - -from __future__ import annotations - -import argparse -import html -import os -import re -import shlex -import subprocess -import sys -import xml.etree.ElementTree as ET -from pathlib import Path -from typing import Dict, Iterable, List, Set - -TOOLS_DIR = Path(__file__).resolve().parent -if str(TOOLS_DIR) not in sys.path: - sys.path.insert(0, str(TOOLS_DIR)) - -import buildinfo # noqa: E402 -import mdlink2html # noqa: E402 - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Render Markdown to HTML") - parser.add_argument("output") - parser.add_argument("input_md") - parser.add_argument("head_tpl") - parser.add_argument("foot_tpl") - parser.add_argument( - "mdlink_script", - nargs="?", - default=str(TOOLS_DIR / "mdlink2html.py"), - help="Path to the Markdown link rewriter script (informational)", - ) - return parser.parse_args() - - -NS_URI = "http://commonmark.org/xml/1.0" -NS = f"{{{NS_URI}}}" - - -def find_md_root(input_md: Path) -> Path: - for parent in input_md.parents: - if parent.name == "md": - return parent.resolve() - return input_md.parent.resolve() - - -def is_temp_path(path: Path) -> bool: - for part in path.parts: - if part.startswith(".#") or part.endswith("~"): - return True - return False - - -def _strip_to_flags() -> List[str]: - env_flags = shlex.split(os.environ.get("CMARK_FLAGS", "")) - - flags: list[str] = [] - skip_next = False - for flag in env_flags: - if skip_next: - skip_next = False - continue - if flag in {"--to", "-t"}: - skip_next = True - continue - flags.append(flag) - return flags - - -def run_cmark_ast(md_path: Path, md2html: str) -> ET.Element: - flags = ["--to", "xml", "--extension", "table", "--validate-utf8", *_strip_to_flags()] - cmd = [md2html, *flags, str(md_path)] - result = subprocess.run(cmd, stdout=subprocess.PIPE, text=True, check=True) - return ET.fromstring(result.stdout) - - -def iter_link_destinations(node: ET.Element) -> Iterable[str]: - if node.tag == f"{NS}link": - dest = node.attrib.get("destination", "") - if dest: - yield dest - for child in node: - yield from iter_link_destinations(child) - - -def normalize_target(current_md: Path, raw_url: str, md_root: Path) -> Path | None: - if "://" in raw_url or raw_url.startswith("#") or not raw_url: - return None - - frag_split = raw_url.split("#", 1) - raw_path = frag_split[0] - if not raw_path: - return None - - candidate = Path(raw_path) - suffix = candidate.suffix.lower() - if not suffix: - candidate = candidate.with_suffix(".md") - elif suffix == ".html": - candidate = candidate.with_suffix(".md") - elif suffix != ".md": - return None - - resolved = (current_md.parent / candidate).resolve() - try: - rel = resolved.relative_to(md_root) - except ValueError: - return None - - if is_temp_path(rel): - return None - - if not (md_root / rel).exists(): - return None - - return rel - - -def compute_backlinks(md_root: Path) -> Dict[str, List[str]]: - backlinks: Dict[str, Set[str]] = {} - md2html = os.environ.get("MD2HTML", "/usr/bin/cmark-gfm") - for md_path in md_root.rglob("*.md"): - rel_source = md_path.relative_to(md_root) - if is_temp_path(rel_source): - continue - ast_root = run_cmark_ast(md_path, md2html) - for raw_url in iter_link_destinations(ast_root): - target = normalize_target(md_path, raw_url, md_root) - if target is None or target == rel_source: - continue - backlinks.setdefault(target.as_posix(), set()).add(rel_source.as_posix()) - - return {target: sorted(sources) for target, sources in backlinks.items()} - - -def run_cmark(md_content: str, md2html: str, cmark_flags: List[str]) -> str: - result = subprocess.run( - [md2html, *cmark_flags], - input=md_content, - text=True, - stdout=subprocess.PIPE, - check=True, - ) - return result.stdout - - -def compute_root_prefix(output_path: Path, out_dir: Path) -> str: - try: - relative_path = output_path.relative_to(out_dir) - except ValueError: - return "" - - relative_dir = relative_path.parent - if not relative_dir or relative_dir == Path('.'): - return "" - - depth = len([part for part in relative_dir.parts if part]) - return "../" * depth - - -def page_class(page_title: str) -> str: - slug = re.sub(r"[^a-z0-9]+", "-", page_title.lower()).strip("-") - return f"page-{slug or 'untitled'}" - - -def render_template(template_path: Path, root_prefix: str, page_title: str, build_info_html: str) -> str: - template = template_path.read_text() - rendered = template.replace("@ROOT@", root_prefix) - rendered = rendered.replace("@TITLE@", html.escape(page_title)) - rendered = rendered.replace("@PAGE_CLASS@", page_class(page_title)) - return rendered.replace("@BUILDINFO@", build_info_html) - - -def main() -> None: - args = parse_args() - - output_path = Path(args.output) - input_md = Path(args.input_md) - md_root = find_md_root(input_md) - head_tpl = Path(args.head_tpl) - foot_tpl = Path(args.foot_tpl) - mdlink_script = Path(args.mdlink_script) - if not mdlink_script.exists(): - raise FileNotFoundError(f"Markdown link rewriter not found: {mdlink_script}") - - md2html = os.environ.get("MD2HTML", "/usr/bin/cmark-gfm") - cmark_flags = shlex.split(os.environ.get("CMARK_FLAGS", "--to html --extension table --validate-utf8 --unsafe")) - out_dir = Path(os.environ.get("OUT_DIR", "out")) - - output_path.parent.mkdir(parents=True, exist_ok=True) - - rewritten_md = mdlink2html.rewrite_file(input_md) - body_html = run_cmark(rewritten_md, md2html, cmark_flags) - - page_title = input_md.stem - build_info_html = buildinfo.build_info(input_md.as_posix()) - root_prefix = compute_root_prefix(output_path, out_dir) - - backlinks = compute_backlinks(md_root) - target_key = input_md.resolve().relative_to(md_root).as_posix() - backlink_sources = backlinks.get(target_key, []) - - head_html = render_template(head_tpl, root_prefix, page_title, build_info_html) - foot_html = render_template(foot_tpl, root_prefix, page_title, build_info_html) - - backlinks_html = "" - if backlink_sources: - items = [] - for rel_source in backlink_sources: - href = html.escape(f"{root_prefix}{rel_source[:-3]}.html", quote=True) - display = html.escape(rel_source[:-3]) - items.append(f"<li><a href=\"{href}\">{display}</a></li>") - joined_items = "\n ".join(items) - backlinks_html = ( - '<div class="backlinks">\n' - " <h3>Backlinks</h3>\n" - " <ul>\n" - f" {joined_items}\n" - " </ul>\n" - "</div>\n" - ) - - with output_path.open("w") as outf: - outf.write(head_html) - outf.write(body_html) - outf.write(backlinks_html) - outf.write(foot_html) - - -if __name__ == "__main__": - main() diff --git a/tools/buildinfo.py b/tools/buildinfo.py index 66a599b..b7db265 100755 --- a/tools/buildinfo.py +++ b/tools/buildinfo.py @@ -8,6 +8,7 @@ import html import os import subprocess from datetime import datetime, timezone +from functools import lru_cache from pathlib import Path from typing import Optional @@ -36,6 +37,7 @@ def run_git(args: list[str], cwd: Path, check: bool = True) -> str: return result.stdout.strip() +@lru_cache(maxsize=None) def resolve_repo_root(start_dir: Path) -> Path: output = run_git(["rev-parse", "--show-toplevel"], cwd=start_dir) if not output: @@ -43,6 +45,7 @@ def resolve_repo_root(start_dir: Path) -> Path: return Path(output) +@lru_cache(maxsize=None) def resolve_commit(repo_root: Path, target: Optional[str]) -> str: if target: rel = Path(target) diff --git a/tools/check_links.py b/tools/check_links.py index b49fe63..188723c 100755 --- a/tools/check_links.py +++ b/tools/check_links.py @@ -3,76 +3,17 @@ from __future__ import annotations import argparse -import os import pathlib -import shutil import subprocess import sys -import xml.etree.ElementTree as ET from collections import defaultdict -from typing import Iterable + +import sitegen ROOT = pathlib.Path(__file__).resolve().parent.parent MD_DIR = ROOT / "md" -CMARK_CANDIDATES: Iterable[pathlib.Path | str] = ( - ROOT / "bin" / "cmark-gfm", - ROOT / "bin" / "cmark", - "cmark-gfm", - "cmark", -) - - -def _cmark_cmd() -> list[str]: - configured = os.environ.get("MD2HTML") - candidates: Iterable[pathlib.Path | str] - if configured: - candidates = (configured, *CMARK_CANDIDATES) - else: - candidates = CMARK_CANDIDATES - - for candidate in candidates: - if isinstance(candidate, pathlib.Path): - try: - if candidate.is_file() and os.access(candidate, os.X_OK): - return [str(candidate)] - except OSError: - continue - else: - resolved = shutil.which(candidate) - if resolved: - return [resolved] - raise RuntimeError( - "Unable to locate cmark-gfm; use make container-check-links or set MD2HTML" - ) - - def collect_links(md_path: pathlib.Path) -> set[str]: - proc = subprocess.run( - _cmark_cmd() - + [ - "--to", - "xml", - "--extension", - "table", - "--unsafe", - ], - input=md_path.read_bytes(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"cmark failed for {md_path}: {proc.stderr.decode('utf-8', 'ignore')}" - ) - - try: - root = ET.fromstring(proc.stdout.decode("utf-8")) - except ET.ParseError as exc: # pragma: no cover - raise RuntimeError( - f"Unable to parse cmark output for {md_path}: {exc}" - ) from exc - + root = sitegen.parse_markdown(md_path) urls: set[str] = set() for node in root.iter(): tag = node.tag.split('}')[-1] @@ -84,39 +25,37 @@ def collect_links(md_path: pathlib.Path) -> set[str]: def check_url(url: str) -> tuple[bool, str, str]: - try: + def request(extra: list[str]) -> tuple[int, str]: proc = subprocess.run( [ - "curl", - "-Is", - "--max-time", - "10", - url, + "curl", "--location", "--silent", "--show-error", "--output", "/dev/null", + "--connect-timeout", "5", "--max-time", "15", + "--user-agent", "homepage-link-checker/1.0", "--write-out", "%{http_code}", + *extra, url, ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, encoding="utf-8", errors="replace", check=False, ) - except Exception as exc: # pragma: no cover - defensive - return False, "error", f"error: {exc}" - - status_lines = [line for line in proc.stdout.splitlines() if line.startswith("HTTP/")] - status_line = status_lines[0] if status_lines else "" - if not status_line: - detail = proc.stderr.strip() or "no response" - return False, "error", detail - - ok_prefixes = (" 200 ", " 301 ", " 302 ", " 303 ", " 307 ", " 308 ") - if any(code in status_line for code in ok_prefixes): - parts = status_line.split() - code = parts[1] if len(parts) > 1 else "200" - return True, code, status_line.strip() - - parts = status_line.split() - code = parts[1] if len(parts) > 1 else "error" - return False, code, status_line.strip() + try: + return int(proc.stdout[-3:]), proc.stderr.strip() + except ValueError: + return 0, proc.stderr.strip() or "no response" + + code, detail = request(["--head"]) + if 200 <= code < 400: + return True, str(code), "" + if code in {0, 401, 403, 405, 429, 999}: + get_code, get_detail = request(["--range", "0-0"]) + if 200 <= get_code < 400: + return True, str(get_code), "HEAD rejected; GET succeeded" + if code in {401, 403, 405, 429, 999}: + return True, f"blocked-{code}", get_detail or detail + code, detail = get_code, get_detail or detail + return False, str(code or "error"), detail or f"HTTP {code}" def _resolve_target(path_arg: str | None) -> pathlib.Path: @@ -166,12 +105,15 @@ def main(argv: list[str] | None = None) -> int: return 2 broken: dict[str, list[tuple[str, str]]] = defaultdict(list) + results: dict[str, tuple[bool, str, str]] = {} try: for md_file in md_files: links = collect_links(md_file) for url in sorted(links): - ok, code, detail = check_url(url) + if url not in results: + results[url] = check_url(url) + ok, code, detail = results[url] print(f"{code}\t{md_file}\t{url}") sys.stdout.flush() if not ok: @@ -187,7 +129,7 @@ def main(argv: list[str] | None = None) -> int: print(f" {file_path}: {url} -> {detail}") return 1 - print("All referenced links responded with HTTP 2xx/3xx.") + print("No definitively broken external links found.") return 0 diff --git a/tools/gen_index.py b/tools/gen_index.py deleted file mode 100755 index 1e53781..0000000 --- a/tools/gen_index.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python3 -"""Generate the site index page.""" - -from __future__ import annotations - -import argparse -import html -import os -import sys -from collections import OrderedDict -from pathlib import Path -from typing import Iterable - -TOOLS_DIR = Path(__file__).resolve().parent -if str(TOOLS_DIR) not in sys.path: - sys.path.insert(0, str(TOOLS_DIR)) - -import buildinfo # noqa: E402 - - -class Node: - def __init__(self, name: str) -> None: - self.name = name - self.href: str | None = None - self.children: "OrderedDict[str, Node]" = OrderedDict() - - def child(self, part: str) -> "Node": - if part not in self.children: - self.children[part] = Node(part) - return self.children[part] - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Generate the index HTML page") - parser.add_argument("output") - parser.add_argument("src_dir", nargs="?", default="md") - parser.add_argument("commit_target", nargs="?", default=None) - return parser.parse_args() - - -def is_temp_path(path: Path) -> bool: - """Return True for files that shouldn't appear in the index.""" - for part in path.parts: - if part.startswith(".#"): - return True - if part.startswith("_"): - return True - if part.endswith("~"): - return True - return False - - -def collect_markdown(src_dir: Path) -> Iterable[str]: - return sorted( - ( - path.relative_to(src_dir).as_posix() - for path in src_dir.rglob("*.md") - if not is_temp_path(path.relative_to(src_dir)) - ), - key=lambda p: p, - ) - - -def build_tree(paths: Iterable[str]) -> Node: - root = Node("") - for rel_path in paths: - node = root - parts = rel_path[:-3].split("/") # strip .md - for part in parts: - node = node.child(part) - node.href = rel_path - return root - - -def render_children(node: Node, depth: int) -> list[str]: - indent = " " * depth - lines: list[str] = [] - for child in node.children.values(): - display = html.escape(child.name) - if child.href: - href = html.escape("./" + child.href[:-3] + ".html") - line = f"{indent}<li><a href=\"{href}\">{display}</a>" - else: - line = f"{indent}<li>{display}" - - if child.children: - lines.append(line) - lines.append(f"{indent} <ul>") - lines.extend(render_children(child, depth + 1)) - lines.append(f"{indent} </ul>") - lines.append(f"{indent}</li>") - else: - lines.append(f"{line}</li>") - return lines - - -def render_index_list(root: Node) -> str: - lines = ["<ul class=\"site-index\">"] - lines.extend(render_children(root, 0)) - lines.append("</ul>") - return "\n".join(lines) - - -def render_template(template_path: Path, build_info_html: str) -> str: - template = template_path.read_text() - template = template.replace("@ROOT@", "") - template = template.replace("@TITLE@", "Index") - template = template.replace("@PAGE_CLASS@", "page-index") - return template.replace("@BUILDINFO@", build_info_html) - - -def main() -> None: - args = parse_args() - - output_path = Path(args.output) - src_dir = Path(args.src_dir) - commit_target = args.commit_target if args.commit_target is not None else args.src_dir - - root_dir = TOOLS_DIR.parent - os.chdir(root_dir) - - build_info_html = buildinfo.build_info(commit_target) - head_html = render_template(root_dir / "tpl" / "head.html", build_info_html) - foot_html = render_template(root_dir / "tpl" / "foot.html", build_info_html) - - body_lines = ["<h1>Index of Alan's Homepage</h1>"] - - markdown_paths = list(collect_markdown(src_dir)) - tree = build_tree(markdown_paths) - body_lines.append(render_index_list(tree)) - - output_path.parent.mkdir(parents=True, exist_ok=True) - with output_path.open("w") as outf: - outf.write(head_html) - outf.write("\n".join(body_lines) + "\n") - outf.write(foot_html) - - -if __name__ == "__main__": - main() diff --git a/tools/gen_updates_rss.py b/tools/gen_updates_rss.py deleted file mode 100644 index 7e1622e..0000000 --- a/tools/gen_updates_rss.py +++ /dev/null @@ -1,287 +0,0 @@ -#!/usr/bin/env python3 -"""Generate an RSS feed from the Updates section of Home.md using the cmark AST.""" - -from __future__ import annotations - -import argparse -import hashlib -import html -import os -import re -import shlex -import subprocess -from dataclasses import dataclass -from datetime import datetime, timezone -from email.utils import format_datetime -from pathlib import Path -from typing import Iterable, Iterator, List, Optional -from urllib.parse import urljoin -from xml.etree import ElementTree as ET -from xml.sax.saxutils import escape as xml_escape - -DEFAULT_BASE_URL = os.environ.get("SITE_BASE_URL", "https://tailrecursion.com/~alan/") -DEFAULT_FEED_TITLE = "Alan Dipert — Updates" -DEFAULT_FEED_DESCRIPTION = "Recent updates from Alan's homepage." - -NS_URI = "http://commonmark.org/xml/1.0" -NS = f"{{{NS_URI}}}" -KNOWN_EXTENSIONS = (".html", ".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".css", ".js", ".zip") -TAG_PATTERN = re.compile(r"<[^>]+>") - - -@dataclass -class Update: - date: datetime - title: str - link: str - html: str - guid: str - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("home_markdown", help="Path to md/Home.md (or equivalent source)") - parser.add_argument("output", help="Destination RSS file (e.g. out/updates.xml)") - parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="Public site root used to absolutize links") - parser.add_argument("--feed-title", default=DEFAULT_FEED_TITLE, help="Title element for the RSS channel") - parser.add_argument( - "--feed-description", - default=DEFAULT_FEED_DESCRIPTION, - help="Description element for the RSS channel", - ) - parser.add_argument( - "--channel-link", - default=None, - help="Homepage URL for the channel link element (defaults to base-url + Home.html)", - ) - return parser.parse_args() - - -def run_cmark_ast(home_md: Path) -> ET.Element: - md2html = os.environ.get("MD2HTML", "/usr/bin/cmark-gfm") - env_flags = shlex.split(os.environ.get("CMARK_FLAGS", "")) - - flags: list[str] = [] - skip_next = False - for flag in env_flags: - if skip_next: - skip_next = False - continue - if flag in {"--to", "-t"}: - skip_next = True - continue - flags.append(flag) - - cmd = [md2html, "--to", "xml", "--extension", "table", "--validate-utf8", *flags, str(home_md)] - result = subprocess.run(cmd, stdout=subprocess.PIPE, text=True, check=True) - return ET.fromstring(result.stdout) - - -def local_name(tag: str) -> str: - return tag.split("}", 1)[-1] - - -def find_updates_table(root: ET.Element) -> ET.Element: - children = list(root) - for idx, child in enumerate(children): - if child.tag == f"{NS}heading": - text = "".join(c.text or "" for c in child.findall(f"{NS}text")) - if text.strip().lower() == "updates": - for sibling in children[idx + 1 :]: - if sibling.tag == f"{NS}table": - return sibling - break - raise RuntimeError("Unable to locate Updates table in Home.md") - - -def iter_rows(table: ET.Element) -> Iterator[tuple[ET.Element, ET.Element]]: - for node in table.findall(f"{NS}table_row"): - cells = node.findall(f"{NS}table_cell") - if len(cells) != 2: - continue - yield cells[0], cells[1] - - -def collect_links(node: ET.Element, base_url: str) -> Iterable[str]: - if node.tag == f"{NS}link": - dest = node.attrib.get("destination", "") - yield absolutize_url(rewrite_destination(dest), base_url) - for child in node: - yield from collect_links(child, base_url) - - -def absolutize_url(url: str, base_url: str) -> str: - lowered = url.lower() - if "://" in url or lowered.startswith("mailto:") or url.startswith("#"): - return url - return urljoin(base_url, url) - - -def rewrite_destination(url: str) -> str: - if not url: - return url - frag = "" - if "#" in url: - path, frag = url.split("#", 1) - frag = "#" + frag - else: - path = url - - lowered = path.lower() - if lowered.endswith(".md"): - path = path[:-3] + ".html" - elif lowered.endswith(KNOWN_EXTENSIONS): - pass - elif "." not in path and path and not path.endswith("/"): - path = f"{path}.html" - - return f"{path}{frag}" - - -def render_children(node: ET.Element, base_url: str) -> str: - parts: list[str] = [] - if node.text and node.text.strip(): - parts.append(html.escape(node.text)) - for child in node: - parts.append(render_node(child, base_url)) - if child.tail and child.tail.strip(): - parts.append(html.escape(child.tail)) - return "".join(parts) - - -def render_node(node: ET.Element, base_url: str) -> str: - name = local_name(node.tag) - - if name == "text": - return html.escape(node.text or "") - if name in {"softbreak", "linebreak"}: - return "<br />" - if name == "code": - return f"<code>{html.escape(node.text or '')}</code>" - if name == "strong": - return f"<strong>{render_children(node, base_url)}</strong>" - if name == "emph": - return f"<em>{render_children(node, base_url)}</em>" - if name == "link": - href = absolutize_url(rewrite_destination(node.attrib.get("destination", "")), base_url) - inner = render_children(node, base_url) - title = node.attrib.get("title", "") - title_attr = f' title="{html.escape(title)}"' if title else "" - return f'<a href="{html.escape(href)}"{title_attr}>{inner}</a>' - if name == "code_span": - return f"<code>{html.escape(node.text or '')}</code>" - if name == "paragraph": - return f"<p>{render_children(node, base_url)}</p>" - - return render_children(node, base_url) - - -def render_cell_html(cell: ET.Element, base_url: str) -> str: - paragraphs = [child for child in cell if local_name(child.tag) == "paragraph"] - if paragraphs: - return "".join(render_node(p, base_url) for p in paragraphs) - - body = render_children(cell, base_url) - return f"<p>{body}</p>" if body else "<p></p>" - - -def parse_date(cell: ET.Element) -> datetime: - text = "".join(part.strip() for part in cell.itertext()).strip() - try: - return datetime.strptime(text, "%Y-%m-%d").replace(tzinfo=timezone.utc) - except ValueError as exc: - raise ValueError(f"Invalid date in Updates table: {text!r}") from exc - - -def build_updates(home_md: Path, base_url: str) -> List[Update]: - ast_root = run_cmark_ast(home_md) - table = find_updates_table(ast_root) - - updates: list[Update] = [] - for date_cell, note_cell in iter_rows(table): - pub_date = parse_date(date_cell) - html_body = render_cell_html(note_cell, base_url) - plain = html_to_text(html_body) - guid_source = f"{pub_date.date().isoformat()}\n{plain}".encode("utf-8") - guid = f"urn:homepage-updates:{hashlib.sha1(guid_source).hexdigest()}" - - links = list(collect_links(note_cell, base_url)) - link = links[0] if links else urljoin(base_url, "Home.html") - - updates.append(Update(pub_date, plain, link, html_body, guid)) - - return updates - - -def render_feed( - updates: List[Update], - base_url: str, - feed_title: str, - feed_description: str, - channel_link: Optional[str], -) -> str: - if not updates: - raise RuntimeError("No updates found; RSS feed would be empty.") - - updates.sort(key=lambda u: u.date, reverse=True) - - last_modified = updates[0].date - resolved_channel_link = channel_link or urljoin(base_url, "Home.html") - - lines = [ - '<?xml version="1.0" encoding="UTF-8"?>', - '<rss version="2.0">', - " <channel>", - f" <title>{xml_escape(feed_title)}</title>", - f" <link>{xml_escape(resolved_channel_link)}</link>", - f" <description>{xml_escape(feed_description)}</description>", - f" <lastBuildDate>{format_datetime(last_modified)}</lastBuildDate>", - ] - - for update in updates: - lines.extend( - [ - " <item>", - f" <title>{xml_escape(update.title)}</title>", - f" <link>{xml_escape(update.link)}</link>", - f" <guid isPermaLink=\"false\">{xml_escape(update.guid)}</guid>", - f" <pubDate>{format_datetime(update.date)}</pubDate>", - " <description><![CDATA[", - f"{update.html}", - " ]]></description>", - " </item>", - ] - ) - - lines.append(" </channel>") - lines.append("</rss>") - lines.append("") - return "\n".join(lines) - - -def html_to_text(fragment: str) -> str: - stripped = TAG_PATTERN.sub("", fragment) - return " ".join(stripped.split()) - - -def main() -> None: - args = parse_args() - base_url = args.base_url.rstrip("/") + "/" - - home_md = Path(args.home_markdown) - updates = build_updates(home_md, base_url) - feed_xml = render_feed( - updates, - base_url, - args.feed_title, - args.feed_description, - args.channel_link, - ) - - output_path = Path(args.output) - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(feed_xml, encoding="utf-8") - - -if __name__ == "__main__": - main() diff --git a/tools/index_list.awk b/tools/index_list.awk deleted file mode 100755 index bda3856..0000000 --- a/tools/index_list.awk +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env awk -f -# Generate a nested unordered list of site pages from a list of Markdown files. - -function escape_regex(s, i, c, out, specials) { - specials = ".^$[]()|*+?{}\\" - out = "" - for (i = 1; i <= length(s); i++) { - c = substr(s, i, 1) - if (index(specials, c)) { - out = out "\\" c - } else { - out = out c - } - } - return out -} - -function indent(depth, i, s) { - s = "" - for (i = 1; i < depth; i++) { - s = s " " - } - return s -} - -function render(node, depth, i, child, display, link, pad, link_tmp) { - for (i = 1; i <= child_count[node]; i++) { - child = children[node, i] - display = name[child] - link = href[child] - pad = indent(depth) - - if (link != "") { - link_tmp = link - sub(/\.md$/, ".html", link_tmp) - link_tmp = "./" link_tmp - printf("%s<li><a href=\"%s\">%s</a>", pad, link_tmp, display) - } else { - printf("%s<li>%s", pad, display) - } - - if (child_count[child] > 0) { - printf("\n%s <ul>\n", pad) - render(child, depth + 1) - printf("%s </ul>\n", pad) - printf("%s</li>\n", pad) - } else { - printf("</li>\n") - } - } -} - -BEGIN { - FS = "/" - pref = prefix - if (pref != "" && substr(pref, length(pref), 1) != "/") { - pref = pref "/" - } - pref_pattern = escape_regex(pref) -} -{ - rel = $0 - if (pref != "") { - sub("^" pref_pattern, "", rel) - } - base = rel - gsub(/\.md$/, "", base) - - n = split(base, parts, "/") - full = "" - for (i = 1; i <= n; i++) { - parent = full - full = (full == "" ? parts[i] : full "/" parts[i]) - - if (!(full in seen)) { - seen[full] = 1 - child_count[parent]++ - children[parent, child_count[parent]] = full - name[full] = parts[i] - } - - if (i == n) { - href[full] = rel - } - } -} -END { - print "<ul class=\"site-index\">" - render("", 1) - print "</ul>" -} diff --git a/tools/mdlink2html.py b/tools/mdlink2html.py deleted file mode 100755 index 4462e79..0000000 --- a/tools/mdlink2html.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -"""Rewrite Markdown links to point at generated HTML pages.""" - -from __future__ import annotations - -import argparse -import re -import sys -from pathlib import Path - -_LINK_PATTERN = re.compile(r"\]\(([^)]+)\)") -_ALLOWED_PATH_PATTERN = re.compile(r"^[A-Za-z0-9._\-/]+$") -_KNOWN_EXTENSIONS = (".html", ".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".css", ".js", ".zip") - - -def _rewrite_url(url: str) -> str: - if "://" in url: - return url - if url.startswith("#"): - return url - - frag = "" - path = url - hash_pos = url.find("#") - if hash_pos != -1: - path = url[:hash_pos] - frag = url[hash_pos:] - - lower = path.lower() - if lower.endswith(".md"): - path = path[:-3] + ".html" - elif path: - if lower.endswith(_KNOWN_EXTENSIONS): - pass - elif not re.search(r"\.[A-Za-z0-9]+$", path) and _ALLOWED_PATH_PATTERN.match(path): - path = f"{path}.html" - - return f"{path}{frag}" - - -def rewrite_text(text: str) -> str: - def _replace(match: re.Match[str]) -> str: - url = match.group(1) - rewritten = _rewrite_url(url) - return f"]({rewritten})" - - return _LINK_PATTERN.sub(_replace, text) - - -def rewrite_file(path: Path) -> str: - text = path.read_text() - return rewrite_text(text) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("input", nargs="?", help="Markdown file to process (defaults to stdin)") - return parser.parse_args() - - -def main() -> None: - args = parse_args() - if args.input: - path = Path(args.input) - sys.stdout.write(rewrite_file(path)) - else: - sys.stdout.write(rewrite_text(sys.stdin.read())) - - -if __name__ == "__main__": - main() diff --git a/tools/normalize_output.py b/tools/normalize_output.py index af3e525..5b9680f 100644 --- a/tools/normalize_output.py +++ b/tools/normalize_output.py @@ -18,17 +18,21 @@ def parse_args() -> argparse.Namespace: def normalize(root: Path, epoch: int) -> None: if not root.is_dir(): raise SystemExit(f"Generated output does not exist: {root}") + if epoch < 0: + raise SystemExit(f"Invalid negative epoch: {epoch}") paths = sorted(root.rglob("*"), key=lambda path: (len(path.parts), path.as_posix()), reverse=True) for path in paths: if path.is_symlink(): - os.utime(path, (epoch, epoch), follow_symlinks=False) - elif path.is_dir(): + raise SystemExit(f"Generated output contains a symbolic link: {path}") + if path.is_dir(): path.chmod(0o755) os.utime(path, (epoch, epoch)) - else: + elif path.is_file(): path.chmod(0o644) os.utime(path, (epoch, epoch)) + else: + raise SystemExit(f"Generated output contains a special file: {path}") root.chmod(0o755) os.utime(root, (epoch, epoch)) diff --git a/tools/sitegen.py b/tools/sitegen.py new file mode 100644 index 0000000..bfaee64 --- /dev/null +++ b/tools/sitegen.py @@ -0,0 +1,562 @@ +#!/usr/bin/env python3 +"""Build all HTML pages, the site index, and the updates RSS feed.""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import os +import re +import shlex +import subprocess +import sys +from collections import OrderedDict +from dataclasses import dataclass +from datetime import datetime, timezone +from email.utils import format_datetime +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import unquote, urljoin, urlsplit +from xml.etree import ElementTree as ET +from xml.sax.saxutils import escape as xml_escape + +TOOLS_DIR = Path(__file__).resolve().parent +if str(TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(TOOLS_DIR)) + +import buildinfo # noqa: E402 + +NS_URI = "http://commonmark.org/xml/1.0" +NS = f"{{{NS_URI}}}" +KNOWN_EXTENSIONS = ( + ".html", ".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".css", ".js", ".zip" +) +HTML_HREF_PATTERN = re.compile(r'(<a\b[^>]*\bhref=")([^"]+)(")') +ALLOWED_PATH_PATTERN = re.compile(r"^[A-Za-z0-9._\-/]+$") +TAG_PATTERN = re.compile(r"<[^>]+>") +LATEST_WORKS_MARKER = "<!-- latest-works:6 -->" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--src", type=Path, default=Path("md")) + parser.add_argument("--out", type=Path, default=Path("out")) + parser.add_argument("--head", type=Path, default=Path("tpl/head.html")) + parser.add_argument("--foot", type=Path, default=Path("tpl/foot.html")) + parser.add_argument( + "--base-url", + default=os.environ.get("SITE_BASE_URL", "https://tailrecursion.com/~alan/"), + ) + return parser.parse_args() + + +def is_temp_path(path: Path) -> bool: + return any(part.startswith((".#", "_")) or part.endswith("~") for part in path.parts) + + +def markdown_files(src: Path) -> list[Path]: + return sorted( + (path for path in src.rglob("*.md") if not is_temp_path(path.relative_to(src))), + key=lambda path: path.as_posix(), + ) + + +def cmark_flags(output: str) -> list[str]: + configured = shlex.split( + os.environ.get("CMARK_FLAGS", "--to html --extension table --validate-utf8 --unsafe") + ) + flags: list[str] = [] + skip_next = False + for flag in configured: + if skip_next: + skip_next = False + elif flag in {"--to", "-t"}: + skip_next = True + else: + flags.append(flag) + return ["--to", output, *flags] + + +def run_cmark(*, output: str, source: Path | None = None, text: str | None = None) -> str: + if (source is None) == (text is None): + raise ValueError("provide exactly one of source or text") + command = [os.environ.get("MD2HTML", "cmark-gfm"), *cmark_flags(output)] + if source is not None: + command.append(str(source)) + result = subprocess.run( + command, + input=text, + text=True, + encoding="utf-8", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if result.returncode: + raise RuntimeError(f"{' '.join(command)} failed: {result.stderr.strip()}") + return result.stdout + + +def parse_markdown(path: Path) -> ET.Element: + try: + return ET.fromstring(run_cmark(output="xml", source=path)) + except ET.ParseError as exc: + raise RuntimeError(f"Invalid cmark XML for {path}: {exc}") from exc + + +def rewrite_destination(url: str, source: Path | None = None) -> str: + if not url or "://" in url or url.startswith("#"): + return url + path, separator, fragment = url.partition("#") + lower = path.lower() + if lower.endswith(".md"): + path = path[:-3] + ".html" + elif source is not None and path and not lower.endswith(KNOWN_EXTENSIONS): + if not re.search(r"\.[A-Za-z0-9]+$", path) and ALLOWED_PATH_PATTERN.fullmatch(path): + local = source.parent / path + if not local.is_file() and local.with_suffix(".md").is_file(): + path += ".html" + return path + (separator + fragment if separator else "") + + +def rewrite_html_links(text: str, source: Path) -> str: + def replace(match: re.Match[str]) -> str: + destination = html.unescape(match.group(2)) + rewritten = html.escape(rewrite_destination(destination, source), quote=True) + return f"{match.group(1)}{rewritten}{match.group(3)}" + + return HTML_HREF_PATTERN.sub(replace, text) + + +def iter_links(node: ET.Element): + if node.tag == f"{NS}link": + destination = node.attrib.get("destination", "") + if destination: + yield destination + for child in node: + yield from iter_links(child) + + +def iter_link_nodes(node: ET.Element): + if node.tag == f"{NS}link": + yield node + for child in node: + yield from iter_link_nodes(child) + + +def normalize_target(source: Path, destination: str, src: Path) -> str | None: + if not destination or "://" in destination or destination.startswith("#"): + return None + raw_path = destination.split("#", 1)[0] + if not raw_path: + return None + candidate = Path(raw_path) + if not candidate.suffix: + candidate = candidate.with_suffix(".md") + elif candidate.suffix.lower() == ".html": + candidate = candidate.with_suffix(".md") + elif candidate.suffix.lower() != ".md": + return None + resolved = (source.parent / candidate).resolve() + try: + relative = resolved.relative_to(src.resolve()) + except ValueError: + return None + if is_temp_path(relative) or not resolved.is_file(): + return None + return relative.as_posix() + + +def backlink_map(src: Path, documents: dict[Path, ET.Element]) -> dict[str, list[str]]: + backlinks: dict[str, set[str]] = {} + for source, ast in documents.items(): + relative_source = source.relative_to(src).as_posix() + for destination in iter_links(ast): + target = normalize_target(source, destination, src) + if target and target != relative_source: + backlinks.setdefault(target, set()).add(relative_source) + return {target: sorted(sources) for target, sources in backlinks.items()} + + +def root_prefix(relative: Path) -> str: + return "../" * len(relative.parent.parts) if relative.parent != Path(".") else "" + + +def page_class(title: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") + return f"page-{slug or 'untitled'}" + + +def template(text: str, *, root: str, title: str, page_css: str, info: str) -> str: + return ( + text.replace("@ROOT@", root) + .replace("@TITLE@", html.escape(title)) + .replace("@PAGE_CLASS@", page_css) + .replace("@BUILDINFO@", info) + ) + + +def backlinks_html(sources: list[str], root: str) -> str: + if not sources: + return "" + items = "\n ".join( + f'<li><a href="{html.escape(root + source[:-3] + ".html", quote=True)}">' + f"{html.escape(source[:-3])}</a></li>" + for source in sources + ) + return ( + '<div class="backlinks">\n' + " <h3>Backlinks</h3>\n" + " <ul>\n" + f" {items}\n" + " </ul>\n" + "</div>\n" + ) + + +def write_page( + source: Path, + relative: Path, + out: Path, + head: str, + foot: str, + backlinks: dict[str, list[str]], + updates: list[Update], + src: Path, +) -> None: + output_relative = relative.with_suffix(".html") + root = root_prefix(output_relative) + title = source.stem + info = buildinfo.build_info(source.as_posix()) + body = rewrite_html_links( + run_cmark(output="html", source=source), + source, + ) + if relative == Path("Home.md"): + body = enhance_homepage(body, updates, source, src) + rendered = "".join( + ( + template(head, root=root, title=title, page_css=page_class(title), info=info), + body, + backlinks_html(backlinks.get(relative.as_posix(), []), root), + template(foot, root=root, title=title, page_css=page_class(title), info=info), + ) + ) + destination = out / output_relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(rendered, encoding="utf-8") + + +class IndexNode: + def __init__(self, name: str) -> None: + self.name = name + self.href: str | None = None + self.children: OrderedDict[str, IndexNode] = OrderedDict() + + def child(self, name: str) -> "IndexNode": + if name not in self.children: + self.children[name] = IndexNode(name) + return self.children[name] + + +def index_tree(paths: list[Path]) -> IndexNode: + root = IndexNode("") + for path in paths: + node = root + for part in path.with_suffix("").parts: + node = node.child(part) + node.href = path.as_posix() + return root + + +def render_index_children(node: IndexNode, depth: int = 0) -> list[str]: + indent = " " * depth + lines: list[str] = [] + for child in node.children.values(): + display = html.escape(child.name) + line = ( + f'{indent}<li><a href="./{html.escape(child.href[:-3])}.html">{display}</a>' + if child.href + else f"{indent}<li>{display}" + ) + if child.children: + lines.extend((line, f"{indent} <ul>")) + lines.extend(render_index_children(child, depth + 1)) + lines.extend((f"{indent} </ul>", f"{indent}</li>")) + else: + lines.append(f"{line}</li>") + return lines + + +def write_index(out: Path, src: Path, paths: list[Path], head: str, foot: str) -> None: + info = buildinfo.build_info(src.as_posix()) + values = {"root": "", "title": "Index", "page_css": "page-index", "info": info} + index_lines = ['<ul class="site-index">', *render_index_children(index_tree(paths)), "</ul>"] + rendered = "".join( + ( + template(head, **values), + "<h1>Index of Alan's Homepage</h1>\n", + "\n".join(index_lines) + "\n", + template(foot, **values), + ) + ) + (out / "Index.html").write_text(rendered, encoding="utf-8") + + +@dataclass +class Update: + date: datetime + title: str + link: str + body: str + guid: str + source_destination: str = "" + work_title: str = "" + + +def local_name(tag: str) -> str: + return tag.split("}", 1)[-1] + + +def render_children(node: ET.Element, base_url: str) -> str: + parts = [html.escape(node.text)] if node.text and node.text.strip() else [] + for child in node: + parts.append(render_node(child, base_url)) + if child.tail and child.tail.strip(): + parts.append(html.escape(child.tail)) + return "".join(parts) + + +def absolute(destination: str, base_url: str) -> str: + lowered = destination.lower() + if "://" in destination or lowered.startswith("mailto:") or destination.startswith("#"): + return destination + return urljoin(base_url, rewrite_destination(destination)) + + +def render_node(node: ET.Element, base_url: str) -> str: + name = local_name(node.tag) + if name == "text": + return html.escape(node.text or "") + if name in {"softbreak", "linebreak"}: + return "<br />" + if name in {"code", "code_span"}: + return f"<code>{html.escape(node.text or '')}</code>" + if name in {"strong", "emph", "paragraph"}: + tag = {"strong": "strong", "emph": "em", "paragraph": "p"}[name] + return f"<{tag}>{render_children(node, base_url)}</{tag}>" + if name == "link": + href = html.escape(absolute(node.attrib.get("destination", ""), base_url)) + title = node.attrib.get("title", "") + title_attr = f' title="{html.escape(title)}"' if title else "" + return f'<a href="{href}"{title_attr}>{render_children(node, base_url)}</a>' + return render_children(node, base_url) + + +def updates_from_home(home: ET.Element, base_url: str) -> list[Update]: + children = list(home) + table = None + for index, child in enumerate(children): + heading = "".join(child.itertext()).strip().lower() + if child.tag == f"{NS}heading" and heading == "updates": + for sibling in children[index + 1 :]: + if sibling.tag == f"{NS}heading": + break + if sibling.tag == f"{NS}table": + table = sibling + break + break + if table is None: + raise RuntimeError("Unable to locate Updates table in Home.md") + + updates: list[Update] = [] + for row in table.findall(f"{NS}table_row"): + cells = row.findall(f"{NS}table_cell") + if len(cells) != 2: + continue + date_text = "".join(part.strip() for part in cells[0].itertext()).strip() + try: + date = datetime.strptime(date_text, "%Y-%m-%d").replace(tzinfo=timezone.utc) + except ValueError as exc: + raise RuntimeError(f"Invalid update date: {date_text!r}") from exc + paragraphs = [node for node in cells[1] if local_name(node.tag) == "paragraph"] + body = "".join(render_node(node, base_url) for node in paragraphs) + if not paragraphs: + content = render_children(cells[1], base_url) + body = f"<p>{content}</p>" if content else "<p></p>" + title = " ".join(TAG_PATTERN.sub("", body).split()) + link_nodes = list(iter_link_nodes(cells[1])) + links = [absolute(node.attrib.get("destination", ""), base_url) for node in link_nodes] + link = links[0] if links else urljoin(base_url, "Home.html") + source_destination = link_nodes[0].attrib.get("destination", "") if link_nodes else "" + work_title = "".join(link_nodes[0].itertext()).strip() if link_nodes else "" + guid_input = f"{date.date().isoformat()}\n{title}".encode("utf-8") + guid = f"urn:homepage-updates:{hashlib.sha1(guid_input).hexdigest()}" + updates.append(Update(date, title, link, body, guid, source_destination, work_title)) + if not updates: + raise RuntimeError("Updates feed would be empty") + return sorted(updates, key=lambda update: update.date, reverse=True) + + +def latest_local_works(updates: list[Update], home: Path, src: Path, limit: int = 6) -> list[Update]: + selected: list[Update] = [] + seen: set[str] = set() + source_root = src.resolve() + for update in updates: + destination = update.source_destination.split("#", 1)[0] + if not destination or "://" in destination or Path(destination).suffix.lower() != ".md": + continue + target = (home.parent / destination).resolve() + try: + key = target.relative_to(source_root).as_posix() + except ValueError: + continue + if not target.is_file() or key in seen: + continue + seen.add(key) + selected.append(update) + if len(selected) == limit: + break + return selected + + +def render_latest_works(updates: list[Update], home: Path, src: Path) -> str: + works = latest_local_works(updates, home, src) + if not works: + raise RuntimeError("No local works found for the homepage Explore section") + items: list[str] = [] + for update in works: + title = update.work_title or Path(update.source_destination).stem + href = rewrite_destination(update.source_destination, home) + visible_date = f"{update.date.day} {update.date.strftime('%b')} {update.date.year}" + plain = html.unescape(TAG_PATTERN.sub("", update.body)) + plain = " ".join(plain.split()) + prefix = re.compile(rf"^Added\s+{re.escape(title)}\s*[,.:;-]?\s*", re.IGNORECASE) + excerpt = prefix.sub("", plain) + words = excerpt.split() + excerpt = " ".join(words[:16]).rstrip(".,;:—–- ") + "…" + excerpt = excerpt[:1].upper() + excerpt[1:] + items.append( + " <li>" + f'<a href="{html.escape(href, quote=True)}">' + f'<time datetime="{update.date.date().isoformat()}">{visible_date}</time>' + f'<span class="work-title">{html.escape(title)}</span>' + f'<span class="work-excerpt">{html.escape(excerpt)}</span>' + "</a></li>" + ) + return "\n".join( + [ + '<ul class="latest-work-tiles">', + *items, + "</ul>", + '<p class="explore-more"><a href="#updates">More</a></p>', + ] + ) + + +def enhance_homepage(body: str, updates: list[Update], home: Path, src: Path) -> str: + if body.count(LATEST_WORKS_MARKER) != 1: + raise RuntimeError(f"Home.md must contain exactly one {LATEST_WORKS_MARKER!r} marker") + updates_heading = "<h2>Updates</h2>" + if body.count(updates_heading) != 1: + raise RuntimeError("Home.md must contain exactly one level-two Updates heading") + body = body.replace(LATEST_WORKS_MARKER, render_latest_works(updates, home, src), 1) + return body.replace(updates_heading, '<h2 id="updates">Updates</h2>', 1) + + +def write_rss(out: Path, updates: list[Update], base_url: str) -> None: + lines = [ + '<?xml version="1.0" encoding="UTF-8"?>', + '<rss version="2.0">', + " <channel>", + " <title>Alan Dipert — Updates</title>", + f" <link>{xml_escape(urljoin(base_url, 'Home.html'))}</link>", + " <description>Recent updates from Alan's homepage.</description>", + f" <lastBuildDate>{format_datetime(updates[0].date)}</lastBuildDate>", + ] + for update in updates: + body = update.body.replace("]]>", "]]]]><![CDATA[>") + lines.extend( + ( + " <item>", + f" <title>{xml_escape(update.title)}</title>", + f" <link>{xml_escape(update.link)}</link>", + f' <guid isPermaLink="false">{xml_escape(update.guid)}</guid>', + f" <pubDate>{format_datetime(update.date)}</pubDate>", + " <description><![CDATA[", + body, + " ]]></description>", + " </item>", + ) + ) + lines.extend((" </channel>", "</rss>", "")) + destination = out / "updates.xml" + destination.write_text("\n".join(lines), encoding="utf-8") + ET.parse(destination) + + +class LocalReferenceParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.references: list[str] = [] + + def handle_starttag(self, _tag: str, attrs: list[tuple[str, str | None]]) -> None: + self.references.extend( + value for name, value in attrs if name in {"href", "src"} and value + ) + + +def validate_internal_references(out: Path) -> None: + root = out.resolve() + failures: list[str] = [] + for page in sorted(out.rglob("*.html")): + parser = LocalReferenceParser() + parser.feed(page.read_text(encoding="utf-8")) + for reference in parser.references: + parsed = urlsplit(reference) + if parsed.scheme or parsed.netloc or not parsed.path: + continue + target = (page.parent / unquote(parsed.path)).resolve() + try: + target.relative_to(root) + except ValueError: + failures.append(f"{page}: {reference} escapes the output directory") + continue + if not target.exists(): + failures.append(f"{page}: {reference} does not exist") + if failures: + raise RuntimeError("Invalid internal references:\n" + "\n".join(failures)) + + +def build(src: Path, out: Path, head_path: Path, foot_path: Path, base_url: str) -> None: + if not src.is_dir(): + raise RuntimeError(f"Markdown source directory not found: {src}") + paths = markdown_files(src) + if not paths: + raise RuntimeError(f"No Markdown files found under {src}") + out.mkdir(parents=True, exist_ok=True) + head = head_path.read_text(encoding="utf-8") + foot = foot_path.read_text(encoding="utf-8") + documents = {path: parse_markdown(path) for path in paths} + backlinks = backlink_map(src, documents) + relatives = [path.relative_to(src) for path in paths] + home = src / "Home.md" + if home not in documents: + raise RuntimeError(f"Updates source not found: {home}") + normalized_base = base_url.rstrip("/") + "/" + updates = updates_from_home(documents[home], normalized_base) + for source, relative in zip(paths, relatives): + write_page(source, relative, out, head, foot, backlinks, updates, src) + write_index(out, src, relatives, head, foot) + write_rss(out, updates, normalized_base) + validate_internal_references(out) + + +def main() -> None: + args = parse_args() + build(args.src, args.out, args.head, args.foot, args.base_url) + + +if __name__ == "__main__": + main() diff --git a/tools/validate_rss.py b/tools/validate_rss.py deleted file mode 100644 index 6462173..0000000 --- a/tools/validate_rss.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -"""Ensure an RSS feed is well-formed and has the expected top-level structure.""" - -from __future__ import annotations - -import argparse -import sys -import xml.etree.ElementTree as ET -from pathlib import Path - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("feed", help="Path to the RSS XML file to validate") - return parser.parse_args() - - -def validate_feed(path: Path) -> None: - try: - tree = ET.parse(path) - except ET.ParseError as exc: - raise SystemExit(f"{path}: XML parse error: {exc}") from exc - - root = tree.getroot() - tag = root.tag.lower() - if not tag.endswith("rss"): - raise SystemExit(f"{path}: expected root <rss>, found <{root.tag}>") - - channel = None - for child in root: - child_tag = child.tag.lower() - if child_tag.endswith("channel"): - channel = child - break - - if channel is None: - raise SystemExit(f"{path}: missing <channel> element under <rss>") - - items = [elem for elem in channel if elem.tag.lower().endswith("item")] - if not items: - raise SystemExit(f"{path}: RSS feed has no <item> entries") - - -def main() -> None: - args = parse_args() - path = Path(args.feed) - if not path.is_file(): - raise SystemExit(f"{path}: file not found") - - validate_feed(path) - - -if __name__ == "__main__": - main() diff --git a/tools/verify_reproducible.py b/tools/verify_reproducible.py index cfcc633..0f51419 100644 --- a/tools/verify_reproducible.py +++ b/tools/verify_reproducible.py @@ -34,9 +34,11 @@ def digest(path: Path) -> str: def manifest(root: Path) -> dict[str, Entry]: + if not root.is_dir(): + raise SystemExit(f"Output tree does not exist: {root}") entries: dict[str, Entry] = {} - for path in sorted(root.rglob("*")): - relative = path.relative_to(root).as_posix() + for path in [root, *sorted(root.rglob("*"), key=lambda item: item.as_posix())]: + relative = "." if path == root else path.relative_to(root).as_posix() metadata = path.lstat() mode = stat.S_IMODE(metadata.st_mode) if path.is_symlink(): diff --git a/tools/xml2md.py b/tools/xml2md.py deleted file mode 100755 index 58ea4de..0000000 --- a/tools/xml2md.py +++ /dev/null @@ -1,337 +0,0 @@ -#!/usr/bin/env python3 -"""Convert CommonMark XML (from cmark-gfm --to xml) back into Markdown.""" - -from __future__ import annotations - -import argparse -import re -import sys -import xml.etree.ElementTree as ET -from pathlib import Path -from typing import Iterable, List - -NAMESPACE = "{http://commonmark.org/xml/1.0}" - -INLINE_NEEDS_ESCAPES = re.compile(r'[\\`*_{}\[\]()#+\-!>~|\"]') - - -def strip_tag(tag: str) -> str: - if tag.startswith("{"): - return tag.split("}", 1)[1] - return tag - - -def escape_text(text: str) -> str: - if not text: - return "" - # Escape characters that frequently cause formatting when emitted raw. - def esc(match: re.Match[str]) -> str: - char = match.group(0) - if char in "\\`*_{}[]()": - return f"\\{char}" - return char - - return INLINE_NEEDS_ESCAPES.sub(esc, text) - - -def fence_for_code(text: str) -> str: - longest = 0 - for match in re.finditer(r"`+", text): - longest = max(longest, len(match.group(0))) - return "`" * (longest + 1 or 3) - - -def render_inline(node: ET.Element) -> str: - tag = strip_tag(node.tag) - - if tag == "text": - return escape_text(node.text or "") - if tag == "softbreak": - return "\n" - if tag == "linebreak": - return " \n" - if tag == "code": - code_text = node.text or "" - fence = fence_for_code(code_text) - return f"{fence}{code_text}{fence}" - if tag == "link": - label = render_inlines(node) - destination = node.attrib.get("destination", "") - title = node.attrib.get("title", "") - title_part = f' "{title}"' if title else "" - return f"[{label}]({destination}{title_part})" - if tag == "image": - alt = render_inlines(node) - destination = node.attrib.get("destination", "") - title = node.attrib.get("title", "") - title_part = f' "{title}"' if title else "" - return f"" - if tag == "strong": - return f"**{render_inlines(node)}**" - if tag == "emph": - return f"*{render_inlines(node)}*" - if tag == "strikethrough": - return f"~~{render_inlines(node)}~~" - if tag == "html_inline": - return node.text or "" - if tag == "emoji": - return node.attrib.get("alias", "") - if tag == "code_span": - # compatibility alias - return render_inline(ET.Element(NAMESPACE + "code", text=node.text)) - - # For any unhandled inline, render its children directly. - parts: List[str] = [] - if node.text: - parts.append(node.text) - for child in list(node): - parts.append(render_inline(child)) - tail = child.tail or "" - if tail.strip(): - parts.append(tail) - return "".join(parts) - - -def render_inlines(parent: ET.Element) -> str: - parts: List[str] = [] - if parent.text and parent.text.strip(): - parts.append(escape_text(parent.text)) - for child in list(parent): - parts.append(render_inline(child)) - tail = child.tail or "" - if tail.strip(): - parts.append(escape_text(tail)) - return "".join(parts) - - -def render_code_block(node: ET.Element) -> str: - info = (node.attrib.get("info") or "").strip() - info = info.split()[0] if info else "" - code_text = node.text or "" - code_text = code_text.rstrip("\n") - - fence = "```" - if "```" in code_text: - fence = "````" - - header = f"{fence}{info}" if info else fence - return f"{header}\n{code_text}\n{fence}" - - -def render_html_block(node: ET.Element) -> str: - parts = [] - if node.text: - parts.append(node.text) - for child in list(node): - parts.append(ET.tostring(child, encoding="unicode")) - if child.tail: - parts.append(child.tail) - return "".join(parts).rstrip() - - -def render_blockquote(node: ET.Element) -> str: - content = render_children_as_blocks(node) - lines = [] - for block in content.splitlines(): - if block.strip(): - lines.append(f"> {block}") - else: - lines.append(">") - return "\n".join(lines) - - -def render_list(node: ET.Element, indent: str = "") -> str: - list_type = node.attrib.get("type", "bullet") - tight = node.attrib.get("tight", "false") == "true" - start = int(node.attrib.get("start", "1")) - delimiter = node.attrib.get("delimiter", "period") - suffix = "." if delimiter != "paren" else ")" - - lines: List[str] = [] - index = start - - for item in node.findall(NAMESPACE + "item"): - marker = "- " if list_type == "bullet" else f"{index}{suffix} " - if list_type == "ordered": - index += 1 - - item_blocks = [] - for child in list(item): - block = render_block(child) - if block: - item_blocks.append(block) - - if tight: - content = " ".join(block.strip() for block in item_blocks).strip() - content_lines = content.splitlines() if content else [""] - else: - content = "\n\n".join(item_blocks).strip() - content_lines = content.splitlines() if content else [""] - - indent_spaces = indent + " " * len(marker) - first_line = content_lines[0] if content_lines else "" - lines.append(f"{indent}{marker}{first_line}") - for extra in content_lines[1:]: - lines.append(f"{indent_spaces}{extra}") - - if not tight: - lines.append("") - - if not tight and lines and lines[-1] == "": - lines.pop() - - return "\n".join(lines) - - -def render_children_as_blocks(parent: ET.Element) -> str: - blocks: List[str] = [] - for child in list(parent): - block = render_block(child) - if block: - blocks.append(block.rstrip()) - return "\n\n".join(blocks) - - -def render_block(node: ET.Element) -> str: - tag = strip_tag(node.tag) - - if tag == "paragraph": - return render_inlines(node).strip("\n") - if tag == "heading": - level = int(node.attrib.get("level", "1")) - content = render_inlines(node).strip() - hashes = "#" * level - return f"{hashes} {content}" if content else hashes - if tag == "code_block": - return render_code_block(node) - if tag == "html_block": - return render_html_block(node) - if tag == "block_quote": - return render_blockquote(node) - if tag == "thematic_break": - return "---" - if tag == "list": - return render_list(node) - if tag == "custom_block": - literal = node.attrib.get("on_enter", "") + (node.text or "") - for child in list(node): - literal += ET.tostring(child, encoding="unicode") - literal += child.tail or "" - literal += node.attrib.get("on_exit", "") - return literal.strip() - if tag == "table": - return render_table(node) - - # Fallback: treat as container and render children. - if list(node): - return render_children_as_blocks(node) - return node.text or "" - - -def render_table(node: ET.Element) -> str: - headers: List[str] = [] - aligns: List[str] = [] - rows: List[List[str]] = [] - - for child in list(node): - tag = strip_tag(child.tag) - if tag == "table_header": - headers, aligns = _render_table_row(child) - elif tag in {"table_body", "table_row"}: - target_rows = child.findall(NAMESPACE + "table_row") if tag == "table_body" else [child] - for row in target_rows: - cells, _ = _render_table_row(row) - rows.append(cells) - - if not headers and rows: - width = len(rows[0]) - headers = ["" for _ in range(width)] - aligns = ["" for _ in range(width)] - - widths = [max(len(headers[i]), *(len(row[i]) for row in rows)) if headers else max(len(row[i]) for row in rows) - for i in range(len(headers or rows[0]))] if (headers or rows) else [] - - def align_marker(align: str, width: int) -> str: - dash_count = max(width, 3) - dashes = "-" * dash_count - if align == "left": - return ":" + dashes[1:] - if align == "right": - return dashes[:-1] + ":" - if align == "center": - if dash_count <= 2: - return ":" - return ":" + dashes[1:-1] + ":" - return dashes - - lines: List[str] = [] - if headers: - header_line = "| " + " | ".join(cell.ljust(widths[i]) for i, cell in enumerate(headers)) + " |" - align_line = "| " + " | ".join(align_marker(aligns[i], widths[i]) for i in range(len(headers))) + " |" - lines.extend([header_line, align_line]) - - for row in rows: - padded = [row[i].ljust(widths[i]) for i in range(len(row))] - lines.append("| " + " | ".join(padded) + " |") - - return "\n".join(lines).rstrip() - - -def _render_table_row(row: ET.Element) -> tuple[list[str], list[str]]: - cells: List[str] = [] - aligns: List[str] = [] - for cell in row.findall(NAMESPACE + "table_cell"): - cells.append(render_table_cell(cell)) - aligns.append(cell.attrib.get("align", "")) - return cells, aligns - - -def render_table_cell(cell: ET.Element) -> str: - content = render_inlines(cell).strip() - return content.replace("|", "\\|") - - -def render_document(root: ET.Element) -> str: - blocks: List[str] = [] - for child in list(root): - block = render_block(child) - if block: - blocks.append(block.rstrip()) - markdown = "\n\n".join(blocks) - return markdown.rstrip() + "\n" - - -def xml_to_markdown(xml_text: str) -> str: - root = ET.fromstring(xml_text) - if strip_tag(root.tag) != "document": - for child in list(root): - if strip_tag(child.tag) == "document": - root = child - break - return render_document(root) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("input", nargs="?", help="XML file to read (defaults to stdin)") - parser.add_argument("output", nargs="?", help="Destination markdown file (defaults to stdout)") - return parser.parse_args() - - -def main() -> None: - args = parse_args() - if args.input: - xml_data = Path(args.input).read_text() - else: - xml_data = sys.stdin.read() - - markdown = xml_to_markdown(xml_data) - - if args.output: - Path(args.output).write_text(markdown) - else: - sys.stdout.write(markdown) - - -if __name__ == "__main__": - main() diff --git a/tpl/style.css b/tpl/style.css index d6bd08c..cdd215d 100644 --- a/tpl/style.css +++ b/tpl/style.css @@ -309,32 +309,122 @@ img[src*="200px"]{width:200px;} font-size:1.05em; } -.page-home .content>h2:nth-of-type(1){grid-column:1;grid-row:3;} -.page-home .content>ul:nth-of-type(1){grid-column:1;grid-row:4;} -.page-home .content>h2:nth-of-type(2){grid-column:2;grid-row:3;} -.page-home .content>ul:nth-of-type(2){grid-column:2;grid-row:4;} -.page-home .content>h2:nth-of-type(3){grid-column:1/-1;grid-row:5;} -.page-home .content>p:nth-of-type(3){grid-column:1/-1;grid-row:6;} -.page-home .content>table{grid-column:1/-1;grid-row:7;} +.page-home .content>h2:nth-of-type(1){grid-row:3;} +.page-home .content>.latest-work-tiles{grid-row:4;} +.page-home .content>.explore-more{grid-row:5;} +.page-home .content>h2:nth-of-type(2){grid-row:6;} +.page-home .content>ul:not(.latest-work-tiles){grid-row:7;} +.page-home .content>h2#updates{grid-row:8;} +.page-home .content>p:nth-of-type(4){grid-row:9;} +.page-home .content>table{grid-row:10;} + +.page-home .content>h2{margin-top:.9em;} + +.page-home .content>.latest-work-tiles{ + display:grid; + grid-template-columns:repeat(2,minmax(0,1fr)); + gap:.7rem; + margin:.05rem 0 0; + padding:0; + list-style:none; +} + +.page-home .latest-work-tiles>li, +.page-home .content>ul:not(.latest-work-tiles)>li{margin:0;} + +.page-home .latest-work-tiles a{ + display:flex; + min-height:124px; + flex-direction:column; + align-items:flex-start; + gap:.38rem; + padding:.82rem .9rem .88rem; + border:1px solid #ddd4c4; + border-top:3px solid #b45a3c; + border-radius:12px 9px; + background:linear-gradient(145deg,#fffaf2,#f6f1e7); + box-shadow:0 2px 6px rgba(42,48,44,.06); + color:var(--ink); + text-decoration:none; + transition:transform .15s ease,box-shadow .15s ease,border-color .15s ease; +} + +.page-home .latest-work-tiles .work-title{ + font-family:ui-serif,Charter,"Bitstream Charter",Georgia,serif; + font-size:1.08rem; + font-weight:750; + line-height:1.18; + overflow-wrap:anywhere; +} +.page-home .latest-work-tiles time{ + color:var(--muted); + font-size:.7rem; + font-weight:750; + font-variant-numeric:tabular-nums; + letter-spacing:.07em; + line-height:1; + text-transform:uppercase; +} +.page-home .latest-work-tiles .work-excerpt{ + margin-top:.55rem; + color:#545e5a; + font-size:.82rem; + line-height:1.35; +} -.page-home .content>h2:nth-of-type(-n+2){ - margin-top:1em; +.page-home .explore-more{ + margin:.35rem 0 0; + text-align:right; + font-size:.86rem; + font-weight:700; } -.page-home .content>ul{ - margin:.1rem 0 .65rem; +.page-home .content>ul:not(.latest-work-tiles){ + display:grid; + grid-template-columns:repeat(6,minmax(0,1fr)); + gap:.45rem; + margin:.05rem 0 .25rem; padding:0; list-style:none; - border-top:1px solid var(--line); } -.page-home .content>ul>li{ - margin:0; - padding:.3rem .1rem; - border-bottom:1px solid var(--line); +.page-home .content>ul:not(.latest-work-tiles) a{ + display:flex; + min-height:48px; + align-items:center; + justify-content:center; + padding:.45rem .35rem; + border:1px solid #d6d9cf; + border-radius:11px 7px 10px 8px; + background:#f5f3ed; + color:var(--link); + font-size:.82rem; + font-weight:750; + line-height:1.15; + text-align:center; + text-decoration:none; + transition:transform .15s ease,box-shadow .15s ease,border-color .15s ease; +} + +.page-home .content>ul:not(.latest-work-tiles)>li:nth-child(even) a{ + border-radius:7px 11px 8px 10px; + background:#f5f3ed; +} + +.page-home .latest-work-tiles a:hover, +.page-home .content>ul:not(.latest-work-tiles) a:hover{ + transform:translateY(-2px); + border-color:#b7b2a6; + box-shadow:0 5px 13px rgba(42,48,44,.11); + color:var(--link-hover); +} + +.page-home .latest-work-tiles a:focus-visible, +.page-home .content>ul:not(.latest-work-tiles) a:focus-visible{ + outline:3px solid var(--focus); + outline-offset:2px; } -.page-home .content>ul>li>a:first-child{font-weight:750;} .page-home .content>table td:first-child{color:var(--muted);font-variant-numeric:tabular-nums;} .site-index{ @@ -403,6 +493,7 @@ img[src*="200px"]{width:200px;} } .page-home .content>p:nth-of-type(2){margin-bottom:.8rem;} .page-home .content>h2:nth-of-type(1){clear:both;} + .page-home .content>ul:not(.latest-work-tiles){grid-template-columns:repeat(3,minmax(0,1fr));} } @media (max-width:560px){ @@ -427,6 +518,9 @@ img[src*="200px"]{width:200px;} max-width:100%; margin:0; } + .page-home .content>ul:not(.latest-work-tiles){grid-template-columns:repeat(2,minmax(0,1fr));} + .page-home .content>.latest-work-tiles{grid-template-columns:1fr;} + .page-home .latest-work-tiles a{min-height:0;} .page-home .content>table, .page-home .content>table tbody, .page-home .content>table tr, @@ -458,3 +552,8 @@ img[src*="200px"]{width:200px;} } .content table:not(.definition-table){overflow-x:auto;} } + +@media (prefers-reduced-motion:reduce){ + .page-home .latest-work-tiles a, + .page-home .content>ul:not(.latest-work-tiles) a{transition:none;} +}