| author | Alan Dipert
<alan@tailrecursion.com> 2026-08-09 19:47:50 UTC |
| committer | Alan Dipert
<alan@tailrecursion.com> 2026-08-09 19:47:50 UTC |
| parent | 08df659bd951667907537abdfa0055e216a60b23 |
| .gitattributes | +13 | -0 |
| Containerfile | +2 | -0 |
| Makefile | +20 | -2 |
| README.md | +2 | -0 |
| archive/techworks/README.md | +17 | -0 |
| archive/techworks/SHA256SUMS | +68 | -0 |
| archive/techworks/manifest.tsv | +69 | -0 |
| tests/test_archive_techworks.py | +77 | -0 |
| tools/archive_techworks.py | +255 | -0 |
diff --git a/.gitattributes b/.gitattributes index 5389347..c1fb152 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,14 @@ md/TechWorks/media/video/*.mp4 filter=lfs diff=lfs merge=lfs -text +md/TechWorks/media/audio/*.flac filter=lfs diff=lfs merge=lfs -text +md/TechWorks/media/audio/*.m4a filter=lfs diff=lfs merge=lfs -text +md/TechWorks/media/audio/*.mp3 filter=lfs diff=lfs merge=lfs -text +md/TechWorks/media/audio/*.wav filter=lfs diff=lfs merge=lfs -text +archive/techworks/**/*.bundle filter=lfs diff=lfs merge=lfs -text +archive/techworks/**/*.flac filter=lfs diff=lfs merge=lfs -text +archive/techworks/**/*.m4a filter=lfs diff=lfs merge=lfs -text +archive/techworks/**/*.mkv filter=lfs diff=lfs merge=lfs -text +archive/techworks/**/*.mp3 filter=lfs diff=lfs merge=lfs -text +archive/techworks/**/*.mp4 filter=lfs diff=lfs merge=lfs -text +archive/techworks/**/*.warc.gz filter=lfs diff=lfs merge=lfs -text +archive/techworks/**/*.wav filter=lfs diff=lfs merge=lfs -text +archive/techworks/**/*.webm filter=lfs diff=lfs merge=lfs -text diff --git a/Containerfile b/Containerfile index 87fc22f..cd1215f 100644 --- a/Containerfile +++ b/Containerfile @@ -22,8 +22,10 @@ RUN rm -f /etc/apt/sources.list /etc/apt/sources.list.d/debian.sources \ git-lfs \ openssh-client \ make \ + poppler-utils \ python3 \ rsync \ + wget \ && cmark-gfm --version \ && ffmpeg -version | head -n 1 \ && git-lfs version \ diff --git a/Makefile b/Makefile index 8abeb39..9b6130c 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -.PHONY: all site assets clean deploy help test tree check-git-clean check-output-path check-pushed check-links check-tools \ +.PHONY: all site assets clean deploy help test tree check-git-clean check-output-path check-pushed check-links check-tools archive-audit archive-checksums archive-export \ fresh container-image container-build container-check-links \ - verify-reproducible verify-reproducible-inner + container-archive-audit verify-reproducible verify-reproducible-inner .DELETE_ON_ERROR: .NOTPARALLEL: @@ -75,6 +75,11 @@ container-check-links: container-image --volume "$(CURDIR):/workspace:Z" --workdir /workspace \ "$(BUILDER_IMAGE)" make check-links +container-archive-audit: container-image + $(CONTAINER_ENGINE) run --rm --userns=keep-id \ + --volume "$(CURDIR):/workspace:Z" --workdir /workspace \ + "$(BUILDER_IMAGE)" make archive-audit + verify-reproducible: container-image $(CONTAINER_ENGINE) run --rm --userns=keep-id \ --volume "$(CURDIR):/workspace:Z" --workdir /workspace \ @@ -116,11 +121,14 @@ help: @echo ' make container-build - reproducibly build $(OUT)/ with Podman' @echo ' make verify-reproducible - compare two isolated container builds' @echo ' make container-check-links - check outbound links with container tools' + @echo ' make container-archive-audit - verify preserved TechWorks artifacts' @echo ' make assets - copy non-Markdown assets into $(OUT)/' @echo ' make - build locally (requires cmark-gfm)' @echo ' make deploy - container-build, then publish from the host' @echo ' make tree - count source and generated files' @echo ' make test - run standard-library unit tests' + @echo ' make archive-audit - verify preserved TechWorks artifacts' + @echo ' make archive-export DEST= - create a portable preservation copy' @echo ' make clean - remove $(OUT)/' @echo ' (deploy requires a clean worktree matching its upstream branch)' @@ -147,3 +155,13 @@ check-pushed: check-links: @python3 tools/check_links.py + +archive-audit: + $(PYTHON) tools/archive_techworks.py audit + +archive-checksums: + $(PYTHON) tools/archive_techworks.py checksums + +archive-export: + @test -n "$(DEST)" || { echo 'DEST must be an absolute path' >&2; exit 1; } + $(PYTHON) tools/archive_techworks.py export --dest "$(DEST)" diff --git a/README.md b/README.md index 38b8a6a..e237a26 100644 --- a/README.md +++ b/README.md @@ -71,3 +71,5 @@ git lfs fsck ``` The DreamHost upstream uses Git LFS's pure-SSH transport. Maintainer remotes must use standard `ssh://user@host/absolute/path` syntax; a colon between the host and absolute path prevents Git LFS from identifying the endpoint. + +Preservation-only TechWorks material and its provenance manifest live under `archive/techworks/`, outside the generated website. Verify the corpus with `make container-archive-audit` (or `make archive-audit` when FFmpeg and Poppler are installed locally). To prepare a checksummed recovery directory containing Git history, LFS objects, repository bundles, and preserved files, run `make archive-export DEST=/absolute/path`. diff --git a/archive/techworks/README.md b/archive/techworks/README.md new file mode 100644 index 0000000..bdf391e --- /dev/null +++ b/archive/techworks/README.md @@ -0,0 +1,17 @@ +# TechWorks preservation data + +This directory stores preservation-only artifacts and provenance records. It is +not copied into the generated website. Visitor-facing downloads remain under +`md/TechWorks/`. + +`manifest.tsv` contains one row per artifact. A preserved row records the exact +source URL, local path, retrieval time, byte count, SHA-256 digest, and whether +the artifact is intended for the public site. Rows without retrievable files +record why only metadata or a missing-item disposition remains. + +Run `make archive-audit` after adding or replacing an artifact. Generate a +portable recovery copy with an explicit destination: + +```sh +make archive-export DEST=/absolute/path/to/empty-directory +``` diff --git a/archive/techworks/SHA256SUMS b/archive/techworks/SHA256SUMS new file mode 100644 index 0000000..ba52d9c --- /dev/null +++ b/archive/techworks/SHA256SUMS @@ -0,0 +1,68 @@ +0262515e872a123dfe8d026dd00464f221a4c980261be636c5bfa557fa6872f8 md/TechWorks/media/video/vimeo-144997124.oembed.json +03c08f2783b560406102e762f3763fc4ae31ffd75d13d941c01583ba85160cd3 md/TechWorks/authored/jlt/index.html +06b57f9625fa07b949d4620e81fe6ad739e6d77c23737af0ef616d9420e7e342 md/TechWorks/authored/wondr/posts/querying-trees-in-r.html +0adfe2c4588a6fc2f1fc39863924cc1a4a01c56c80c2e23982e3c79dee2b259a md/TechWorks/media/video/xaxF5RDdVRE.mp4 +0e8252a4757f6fe252b6a2122f0bc15a8818ef12cfa47164c70f4d99caab9433 md/TechWorks/authored/jlt/style.css +16c40b3f183bd656dda0303c53a56a85ec20e2426909b60b4b7904e920002208 md/TechWorks/flapjax-demo-b719782.tar.gz +1e876de98c65f221e3b437892e93a5c1ef911b023e85e78975769e015d95152e md/TechWorks/media/video/tSw3x0rVh88.info.json +1e9951de2d98335c45976f7dce0b5429469343eab20a5c17e5d3ed597f1fd7b4 md/TechWorks/2018-12-17-CommonLispOrangeCombinator.pdf +23aa4b7e40ecb6f2cf4e3835ed2e0742361c247fc27b4b5c919ddf65215787e5 md/TechWorks/media/video/wVXjExRiFy0.jpg +295dcd7f2ea293bb9e3bb46417d787a934e739739a98bccf7031276c6366a280 md/TechWorks/ocrug-2018-11-27-71dd0b1.tar.gz +2a2b25998617233beb48a5e213dc22fe80a561f0d576c625beeb34f9feae1c4d md/TechWorks/media/video/HGuTqsVh59w.mp4 +2fa46c61ffce23593ef18c65bdeea96bff452d233c0b75f348709ee58f85a040 md/TechWorks/media/video/wVXjExRiFy0.info.json +36efb38dd1f1fc8682b6f2e70e4ce21bad65e0b235bc236fb105d79911fd337b md/TechWorks/barcamp2012-jsonscript-7e4dcf7.tar.gz +37eb2a8885fae16fc6f45b7910a272d9564246cf140ecd72db756fbdc3efa852 md/TechWorks/oscon2012-clojure-5ac233f.tar.gz +39d6df4a9fa92d1c897cf40b79f79efaba56d0862ed8bcf80a7c7aa65ef73b63 md/TechWorks/media/audio/s4-e19-hoplon-with-alan-dipert-source.html +3a1a72783cd199b05cfb3073931edba2742a3f45264af748d09142e4ff6a4e90 md/TechWorks/authored/wondr/posts/tail-recursion-in-r.html +3aad0c50ebc6588f3a86aa4dde39e312f72c80680a6781ea7afb52dfc9780499 md/TechWorks/media/video/wVXjExRiFy0.en-orig.vtt +3aad0c50ebc6588f3a86aa4dde39e312f72c80680a6781ea7afb52dfc9780499 md/TechWorks/media/video/wVXjExRiFy0.en.vtt +3c2821ff1e6fd6306e4a47397a545248546faae0814fb1bb1ae1b44d1fe4109d md/TechWorks/authored/wondr/style.css +446379435661cd8a75647936a7b4ca130dfdfffb7e5fcc4f767aa11d2f922d13 md/TechWorks/2020-01-28-RStudio-Conf-Integration-Testing-ePoster.pdf +49b6ec526b72e3c2a8655557e715b51ca8c999b2ebef8bad5a0efb19fc69b94c md/TechWorks/media/video/44Q9ew9JH_U.en-orig.vtt +49b6ec526b72e3c2a8655557e715b51ca8c999b2ebef8bad5a0efb19fc69b94c md/TechWorks/media/video/44Q9ew9JH_U.en.vtt +4a2815c9d0b1cf0de5cba85b0b1cf22944a910fb28de1f90afb1e9c50717d987 md/TechWorks/Dipert-FRP_in_ClojureScript_with_Javelin.pdf +4b54dcb163aa3c146042c83d71c28f7464ee8bb849b7f26c8814ed6467bb95c1 md/TechWorks/jacl-demo-els-2020.pdf +4bc7e11ed9b4041cbd374ef91f90aee394b10a93c463bd881b58eb41e47a9154 md/TechWorks/Dipert-ProgrammingWithValues.pdf +4fa2b18b4df657cc4fe0b1dcc61fac3a179bb7da636cf90c06de98b38efe7e6f md/TechWorks/wombat-dbd34f4.tar.gz +5330706e824804a97059a9ea78e04ec5b30f3b35cde21e51ca9c999fb9203854 md/TechWorks/media/video/bmHTFo2Rf2w.info.json +66326358232546ee1a467dc983d144d777d8f039f1b4377a140554fe3b527781 md/TechWorks/media/video/HGuTqsVh59w.en-orig.vtt +66326358232546ee1a467dc983d144d777d8f039f1b4377a140554fe3b527781 md/TechWorks/media/video/HGuTqsVh59w.en.vtt +718dea32362c630c4756f539298cddfdc7cb3f941ebd25e303c55e174edd5aa3 md/TechWorks/media/video/44Q9ew9JH_U.webp +746135f37df7aa246fa753ce0a39d2e7f347424ee65e26eb1ec4343a1d13addd md/TechWorks/authored/wondr/posts/assertions-and-postconditions.html +76fe6f5dcebea899924ed50454fd1e8a83fc1ebcb3833a1996b14ac22d8d442f md/TechWorks/media/video/xaxF5RDdVRE.en-orig.vtt +76fe6f5dcebea899924ed50454fd1e8a83fc1ebcb3833a1996b14ac22d8d442f md/TechWorks/media/video/xaxF5RDdVRE.en.vtt +7792adfab4495d154d22bcc34d14ea5690dba64ebdebaaf3b3b6105b9d2f2de4 md/TechWorks/authored/wondr/atom.xml +779d526c75f997ad3c12419c9ad9e46c8ea9d905d8181f3c39c790270371a71b md/TechWorks/media/video/TcnzB2tB-8Q.webp +7df0a6c61f415cb3f43edb5ac551badb5cb6463e3d2d3ea9167af30f2355ca8b md/TechWorks/media/video/HGuTqsVh59w.webp +7f3712a90c6a608ebdb452fcaf94272babd93a66a93b38c801ba19cfab126fd9 md/TechWorks/media/video/TcnzB2tB-8Q.en-orig.vtt +7f3712a90c6a608ebdb452fcaf94272babd93a66a93b38c801ba19cfab126fd9 md/TechWorks/media/video/TcnzB2tB-8Q.en.vtt +83272bbfc8c1576d7695152609e9bad5ca2441b563405aa6f6da5ba147b73b7e md/TechWorks/authored/wondr/index.html +8bb253146b27ce8a8b08520b0064b0b34659b1460296ffcd69439c53a401ceab md/TechWorks/authored/jlt/atom.xml +934c2bd9997b9e6603f28308b6fccf0a72d2e30c3bb422effbcf4224ee8d7282 md/TechWorks/gherkin-76dc034.tar.gz +94d65c64f26cc5ba9bfd7377b2ffa8dd7bdb27a93db1cf47a33fc013283312a0 md/TechWorks/media/video/vimeo-144696304.oembed.json +9b5b40432bc71b1da7aeb72a106c524136112e12be1da250e8b01a8d482a262b md/TechWorks/media/video/tSw3x0rVh88.mp4 +9ffc964563d278119bf8a0feec54ab7cd41c1dbb1c4e0de058d7d370603cc0d3 md/TechWorks/media/video/44Q9ew9JH_U.info.json +a15dd034225897ddf87849d027c48c7d3ef0979b2bb17329172c5103e5b7e794 md/TechWorks/media/video/bmHTFo2Rf2w.en-orig.vtt +a15dd034225897ddf87849d027c48c7d3ef0979b2bb17329172c5103e5b7e794 md/TechWorks/media/video/bmHTFo2Rf2w.en.vtt +a4f289f053abcdb02ac7ff433f4f1aacf279e4470c999cb5e292c31a1f28f4e8 md/TechWorks/2015-04-20_ClojureWestBoot.pdf +a73794c8dd4a4550f2098db3a854959449c0adf623ea4877d3dc13e959b7cacc md/TechWorks/media/video/xaxF5RDdVRE.webp +a7d709bd9848acd265ee0a8837f1045cc29522ec544467b1922d926bbab7a0a7 md/TechWorks/media/video/wVXjExRiFy0.mp4 +a906f29ababff5e39751e14dcd9e14f74545dd3571d5a0f5c41a3c7641af25d2 md/TechWorks/authored/jlt/code_highlight.css +a906f29ababff5e39751e14dcd9e14f74545dd3571d5a0f5c41a3c7641af25d2 md/TechWorks/authored/wondr/code_highlight.css +b439b4b8993805b0163a467d8f18090e3044526ed7e50cd97bb5a32b09cbbc8a md/TechWorks/media/video/HGuTqsVh59w.info.json +c0b942c1ec79ed851fc8810fc224b3e357c01bce98f957bf63786f4e467a2ed8 md/TechWorks/media/video/TcnzB2tB-8Q.mp4 +c318fdba4c6c5caa69baefa238a7b6ecd94afefc8d2c8826a7931028d989fa51 md/TechWorks/media/video/bmHTFo2Rf2w.mp4 +cbdb7e0a11b8566c73453ca401b7cc4ef9d2a424eff8d00dee4f1a6e435d89c3 md/TechWorks/media/SHA256SUMS +d04ff1f53296811517fd27fb2ed11b4092709558723c36d07f9719a9a4850ca4 md/TechWorks/media/video/bmHTFo2Rf2w.jpg +d3bc604551b9f39837f4d619d8b61c324b95e6e26afed30a291092bc8d975d02 md/TechWorks/authored/wondr/posts/package-development-windows.html +d82843df799752ff13fef3f75599bc683cf22ce5957590aa199a2bbbd018c315 md/TechWorks/authored/jlt/posts/collecting.html +d9feaebbf7dd5f8037be7b60226c1e773f304b479f331a2bea3c1a16338684d4 md/TechWorks/media/video/tSw3x0rVh88.en-orig.vtt +d9feaebbf7dd5f8037be7b60226c1e773f304b479f331a2bea3c1a16338684d4 md/TechWorks/media/video/tSw3x0rVh88.en.vtt +dd9a11ba5b00ffa08f88a10dce60f10eb61ac83d5439f0a95e7637a8f76d084b md/TechWorks/media/video/TcnzB2tB-8Q.info.json +e5e57746ea11987744b1acacd6255d24951fddd939b57f1ceaf1b873a3b786bc md/TechWorks/media/video/xaxF5RDdVRE.info.json +ea6820d653957fcc86d231b37fab642418a9f8a8991003a30d2a4d70adbd6ce3 md/TechWorks/fast-shiny-da0ea10.tar.gz +eb519fb5da6646d808dc1801803b3d8e35d5d90bf164775c10833cc43be9b340 md/TechWorks/authored/wondr/posts/shiny-inline-markdown.html +ef306bbebe80ee481cf012cf446c563828d93902ee927b0a14dc86e60e7b5efc md/TechWorks/authored/jlt/posts/collecting-macro-edition.html +f50e8de09c6a01a8ac8178eec634fb42151ecd83e9fa1de49ee76e862b8b0507 md/TechWorks/media/video/tSw3x0rVh88.jpg +f93ac829f7ae73c768197eb4f47c634a38f123a1f7bacd3b4065927ac1c86c9a md/TechWorks/media/video/44Q9ew9JH_U.mp4 +fcdc53d3785a9eef7989b3fd4b4cb3ec1715015ba61aaa3cf6a8887aa74822c1 md/TechWorks/2018-09-06-load-testing-shiny-archive.html diff --git a/archive/techworks/manifest.tsv b/archive/techworks/manifest.tsv new file mode 100644 index 0000000..443717c --- /dev/null +++ b/archive/techworks/manifest.tsv @@ -0,0 +1,69 @@ +work_id artifact_id title kind source_url local_path retrieved_utc sha256 bytes visibility status notes +2015-04-20_ClojureWestBoot 2015-04-20_clojurewestboot-pdf 2015-04-20_ClojureWestBoot.pdf pdf repository-preserved-source md/TechWorks/2015-04-20_ClojureWestBoot.pdf 2026-08-08T00:00:00Z a4f289f053abcdb02ac7ff433f4f1aacf279e4470c999cb5e292c31a1f28f4e8 604933 public preserved +2018-09-06-load-testing-shiny-archive 2018-09-06-load-testing-shiny-archive-html 2018-09-06-load-testing-shiny-archive.html html repository-preserved-source md/TechWorks/2018-09-06-load-testing-shiny-archive.html 2026-08-08T00:00:00Z fcdc53d3785a9eef7989b3fd4b4cb3ec1715015ba61aaa3cf6a8887aa74822c1 114298 public preserved +2018-12-17-CommonLispOrangeCombinator 2018-12-17-commonlisporangecombinator-pdf 2018-12-17-CommonLispOrangeCombinator.pdf pdf repository-preserved-source md/TechWorks/2018-12-17-CommonLispOrangeCombinator.pdf 2026-08-08T00:00:00Z 1e9951de2d98335c45976f7dce0b5429469343eab20a5c17e5d3ed597f1fd7b4 183290 public preserved +2020-01-28-RStudio-Conf-Integration-Testing-ePoster 2020-01-28-rstudio-conf-integration-testing-eposter-pdf 2020-01-28-RStudio-Conf-Integration-Testing-ePoster.pdf pdf repository-preserved-source md/TechWorks/2020-01-28-RStudio-Conf-Integration-Testing-ePoster.pdf 2026-08-08T00:00:00Z 446379435661cd8a75647936a7b4ca130dfdfffb7e5fcc4f767aa11d2f922d13 313663 public preserved +Dipert-FRP_in_ClojureScript_with_Javelin dipert-frp_in_clojurescript_with_javelin-pdf Dipert-FRP_in_ClojureScript_with_Javelin.pdf pdf repository-preserved-source md/TechWorks/Dipert-FRP_in_ClojureScript_with_Javelin.pdf 2026-08-08T00:00:00Z 4a2815c9d0b1cf0de5cba85b0b1cf22944a910fb28de1f90afb1e9c50717d987 361367 public preserved +Dipert-ProgrammingWithValues dipert-programmingwithvalues-pdf Dipert-ProgrammingWithValues.pdf pdf repository-preserved-source md/TechWorks/Dipert-ProgrammingWithValues.pdf 2026-08-08T00:00:00Z 4bc7e11ed9b4041cbd374ef91f90aee394b10a93c463bd881b58eb41e47a9154 683331 public preserved +just-lisp-things authored-jlt-atom-xml atom.xml feed https://tailrecursion.com/jlt/ md/TechWorks/authored/jlt/atom.xml 2026-08-08T00:00:00Z 8bb253146b27ce8a8b08520b0064b0b34659b1460296ffcd69439c53a401ceab 1068 public preserved +just-lisp-things authored-jlt-code_highlight-css code_highlight.css css https://tailrecursion.com/jlt/ md/TechWorks/authored/jlt/code_highlight.css 2026-08-08T00:00:00Z a906f29ababff5e39751e14dcd9e14f74545dd3571d5a0f5c41a3c7641af25d2 3489 public preserved +just-lisp-things authored-jlt-index-html index.html html https://tailrecursion.com/jlt/ md/TechWorks/authored/jlt/index.html 2026-08-08T00:00:00Z 03c08f2783b560406102e762f3763fc4ae31ffd75d13d941c01583ba85160cd3 872 public preserved +just-lisp-things authored-jlt-posts-collecting-macro-edition-html collecting-macro-edition.html html https://tailrecursion.com/jlt/ md/TechWorks/authored/jlt/posts/collecting-macro-edition.html 2026-08-08T00:00:00Z ef306bbebe80ee481cf012cf446c563828d93902ee927b0a14dc86e60e7b5efc 8925 public preserved +just-lisp-things authored-jlt-posts-collecting-html collecting.html html https://tailrecursion.com/jlt/ md/TechWorks/authored/jlt/posts/collecting.html 2026-08-08T00:00:00Z d82843df799752ff13fef3f75599bc683cf22ce5957590aa199a2bbbd018c315 15631 public preserved +just-lisp-things authored-jlt-style-css style.css css https://tailrecursion.com/jlt/ md/TechWorks/authored/jlt/style.css 2026-08-08T00:00:00Z 0e8252a4757f6fe252b6a2122f0bc15a8818ef12cfa47164c70f4d99caab9433 234 public preserved +wondr authored-wondr-atom-xml atom.xml feed https://tailrecursion.com/wondr/ md/TechWorks/authored/wondr/atom.xml 2026-08-08T00:00:00Z 7792adfab4495d154d22bcc34d14ea5690dba64ebdebaaf3b3b6105b9d2f2de4 2768 public preserved +wondr authored-wondr-code_highlight-css code_highlight.css css https://tailrecursion.com/wondr/ md/TechWorks/authored/wondr/code_highlight.css 2026-08-08T00:00:00Z a906f29ababff5e39751e14dcd9e14f74545dd3571d5a0f5c41a3c7641af25d2 3489 public preserved +wondr authored-wondr-index-html index.html html https://tailrecursion.com/wondr/ md/TechWorks/authored/wondr/index.html 2026-08-08T00:00:00Z 83272bbfc8c1576d7695152609e9bad5ca2441b563405aa6f6da5ba147b73b7e 4347 public preserved +wondr authored-wondr-posts-assertions-and-postconditions-html assertions-and-postconditions.html html https://tailrecursion.com/wondr/ md/TechWorks/authored/wondr/posts/assertions-and-postconditions.html 2026-08-08T00:00:00Z 746135f37df7aa246fa753ce0a39d2e7f347424ee65e26eb1ec4343a1d13addd 18893 public preserved +wondr authored-wondr-posts-package-development-windows-html package-development-windows.html html https://tailrecursion.com/wondr/ md/TechWorks/authored/wondr/posts/package-development-windows.html 2026-08-08T00:00:00Z d3bc604551b9f39837f4d619d8b61c324b95e6e26afed30a291092bc8d975d02 9219 public preserved +wondr authored-wondr-posts-querying-trees-in-r-html querying-trees-in-r.html html https://tailrecursion.com/wondr/ md/TechWorks/authored/wondr/posts/querying-trees-in-r.html 2026-08-08T00:00:00Z 06b57f9625fa07b949d4620e81fe6ad739e6d77c23737af0ef616d9420e7e342 17569 public preserved +wondr authored-wondr-posts-shiny-inline-markdown-html shiny-inline-markdown.html html https://tailrecursion.com/wondr/ md/TechWorks/authored/wondr/posts/shiny-inline-markdown.html 2026-08-08T00:00:00Z eb519fb5da6646d808dc1801803b3d8e35d5d90bf164775c10833cc43be9b340 5890 public preserved +wondr authored-wondr-posts-tail-recursion-in-r-html tail-recursion-in-r.html html https://tailrecursion.com/wondr/ md/TechWorks/authored/wondr/posts/tail-recursion-in-r.html 2026-08-08T00:00:00Z 3a1a72783cd199b05cfb3073931edba2742a3f45264af748d09142e4ff6a4e90 18883 public preserved +wondr authored-wondr-style-css style.css css https://tailrecursion.com/wondr/ md/TechWorks/authored/wondr/style.css 2026-08-08T00:00:00Z 3c2821ff1e6fd6306e4a47397a545248546faae0814fb1bb1ae1b44d1fe4109d 16607 public preserved +barcamp2012-jsonscript barcamp2012-jsonscript-7e4dcf7-tar-gz barcamp2012-jsonscript-7e4dcf7.tar.gz git-snapshot https://github.com/alandipert/barcamp2012-jsonscript md/TechWorks/barcamp2012-jsonscript-7e4dcf7.tar.gz 2026-08-08T00:00:00Z 36efb38dd1f1fc8682b6f2e70e4ce21bad65e0b235bc236fb105d79911fd337b 391524 public preserved +fast-shiny fast-shiny-da0ea10-tar-gz fast-shiny-da0ea10.tar.gz git-snapshot https://github.com/alandipert/fast-shiny md/TechWorks/fast-shiny-da0ea10.tar.gz 2026-08-08T00:00:00Z ea6820d653957fcc86d231b37fab642418a9f8a8991003a30d2a4d70adbd6ce3 3528960 public preserved +flapjax-demo flapjax-demo-b719782-tar-gz flapjax-demo-b719782.tar.gz git-snapshot https://github.com/alandipert/flapjax-demo md/TechWorks/flapjax-demo-b719782.tar.gz 2026-08-08T00:00:00Z 16c40b3f183bd656dda0303c53a56a85ec20e2426909b60b4b7904e920002208 15133 public preserved +gherkin gherkin-76dc034-tar-gz gherkin-76dc034.tar.gz git-snapshot https://github.com/alandipert/gherkin md/TechWorks/gherkin-76dc034.tar.gz 2026-08-08T00:00:00Z 934c2bd9997b9e6603f28308b6fccf0a72d2e30c3bb422effbcf4224ee8d7282 12720 public preserved +jacl-demo-els-2020 jacl-demo-els-2020-pdf jacl-demo-els-2020.pdf pdf repository-preserved-source md/TechWorks/jacl-demo-els-2020.pdf 2026-08-08T00:00:00Z 4b54dcb163aa3c146042c83d71c28f7464ee8bb849b7f26c8814ed6467bb95c1 412722 public preserved +SHA256SUMS media-sha256sums SHA256SUMS checksum repository-preserved-source md/TechWorks/media/SHA256SUMS 2026-08-08T00:00:00Z cbdb7e0a11b8566c73453ca401b7cc4ef9d2a424eff8d00dee4f1a6e435d89c3 3594 public preserved +clojurestream-s4e19 media-audio-s4-e19-hoplon-with-alan-dipert-source-html s4-e19-hoplon-with-alan-dipert-source.html html https://soundcloud.com/user-959992602/s4-e19-hoplon-with-alan-dipert md/TechWorks/media/audio/s4-e19-hoplon-with-alan-dipert-source.html 2026-08-08T00:00:00Z 39d6df4a9fa92d1c897cf40b79f79efaba56d0862ed8bcf80a7c7aa65ef73b63 65568 public preserved +youtube-44Q9ew9JH_U media-video-44q9ew9jh_u-en-orig-vtt 44Q9ew9JH_U.en-orig.vtt captions https://www.youtube.com/watch?v=44Q9ew9JH_U md/TechWorks/media/video/44Q9ew9JH_U.en-orig.vtt 2026-08-08T00:00:00Z 49b6ec526b72e3c2a8655557e715b51ca8c999b2ebef8bad5a0efb19fc69b94c 946736 public preserved +youtube-44Q9ew9JH_U media-video-44q9ew9jh_u-en-vtt 44Q9ew9JH_U.en.vtt captions https://www.youtube.com/watch?v=44Q9ew9JH_U md/TechWorks/media/video/44Q9ew9JH_U.en.vtt 2026-08-08T00:00:00Z 49b6ec526b72e3c2a8655557e715b51ca8c999b2ebef8bad5a0efb19fc69b94c 946736 public preserved +youtube-44Q9ew9JH_U media-video-44q9ew9jh_u-info-json 44Q9ew9JH_U.info.json metadata https://www.youtube.com/watch?v=44Q9ew9JH_U md/TechWorks/media/video/44Q9ew9JH_U.info.json 2026-08-08T00:00:00Z 9ffc964563d278119bf8a0feec54ab7cd41c1dbb1c4e0de058d7d370603cc0d3 503789 public preserved +youtube-44Q9ew9JH_U media-video-44q9ew9jh_u-mp4 44Q9ew9JH_U.mp4 video https://www.youtube.com/watch?v=44Q9ew9JH_U md/TechWorks/media/video/44Q9ew9JH_U.mp4 2026-08-08T00:00:00Z f93ac829f7ae73c768197eb4f47c634a38f123a1f7bacd3b4065927ac1c86c9a 564843565 public preserved +youtube-44Q9ew9JH_U media-video-44q9ew9jh_u-webp 44Q9ew9JH_U.webp image https://www.youtube.com/watch?v=44Q9ew9JH_U md/TechWorks/media/video/44Q9ew9JH_U.webp 2026-08-08T00:00:00Z 718dea32362c630c4756f539298cddfdc7cb3f941ebd25e303c55e174edd5aa3 36258 public preserved +youtube-HGuTqsVh59w media-video-hgutqsvh59w-en-orig-vtt HGuTqsVh59w.en-orig.vtt captions https://www.youtube.com/watch?v=HGuTqsVh59w md/TechWorks/media/video/HGuTqsVh59w.en-orig.vtt 2026-08-08T00:00:00Z 66326358232546ee1a467dc983d144d777d8f039f1b4377a140554fe3b527781 182549 public preserved +youtube-HGuTqsVh59w media-video-hgutqsvh59w-en-vtt HGuTqsVh59w.en.vtt captions https://www.youtube.com/watch?v=HGuTqsVh59w md/TechWorks/media/video/HGuTqsVh59w.en.vtt 2026-08-08T00:00:00Z 66326358232546ee1a467dc983d144d777d8f039f1b4377a140554fe3b527781 182549 public preserved +youtube-HGuTqsVh59w media-video-hgutqsvh59w-info-json HGuTqsVh59w.info.json metadata https://www.youtube.com/watch?v=HGuTqsVh59w md/TechWorks/media/video/HGuTqsVh59w.info.json 2026-08-08T00:00:00Z b439b4b8993805b0163a467d8f18090e3044526ed7e50cd97bb5a32b09cbbc8a 493441 public preserved +youtube-HGuTqsVh59w media-video-hgutqsvh59w-mp4 HGuTqsVh59w.mp4 video https://www.youtube.com/watch?v=HGuTqsVh59w md/TechWorks/media/video/HGuTqsVh59w.mp4 2026-08-08T00:00:00Z 2a2b25998617233beb48a5e213dc22fe80a561f0d576c625beeb34f9feae1c4d 106983957 public preserved +youtube-HGuTqsVh59w media-video-hgutqsvh59w-webp HGuTqsVh59w.webp image https://www.youtube.com/watch?v=HGuTqsVh59w md/TechWorks/media/video/HGuTqsVh59w.webp 2026-08-08T00:00:00Z 7df0a6c61f415cb3f43edb5ac551badb5cb6463e3d2d3ea9167af30f2355ca8b 74264 public preserved +youtube-TcnzB2tB-8Q media-video-tcnzb2tb-8q-en-orig-vtt TcnzB2tB-8Q.en-orig.vtt captions https://www.youtube.com/watch?v=TcnzB2tB-8Q md/TechWorks/media/video/TcnzB2tB-8Q.en-orig.vtt 2026-08-08T00:00:00Z 7f3712a90c6a608ebdb452fcaf94272babd93a66a93b38c801ba19cfab126fd9 337167 public preserved +youtube-TcnzB2tB-8Q media-video-tcnzb2tb-8q-en-vtt TcnzB2tB-8Q.en.vtt captions https://www.youtube.com/watch?v=TcnzB2tB-8Q md/TechWorks/media/video/TcnzB2tB-8Q.en.vtt 2026-08-08T00:00:00Z 7f3712a90c6a608ebdb452fcaf94272babd93a66a93b38c801ba19cfab126fd9 337167 public preserved +youtube-TcnzB2tB-8Q media-video-tcnzb2tb-8q-info-json TcnzB2tB-8Q.info.json metadata https://www.youtube.com/watch?v=TcnzB2tB-8Q md/TechWorks/media/video/TcnzB2tB-8Q.info.json 2026-08-08T00:00:00Z dd9a11ba5b00ffa08f88a10dce60f10eb61ac83d5439f0a95e7637a8f76d084b 503359 public preserved +youtube-TcnzB2tB-8Q media-video-tcnzb2tb-8q-mp4 TcnzB2tB-8Q.mp4 video https://www.youtube.com/watch?v=TcnzB2tB-8Q md/TechWorks/media/video/TcnzB2tB-8Q.mp4 2026-08-08T00:00:00Z c0b942c1ec79ed851fc8810fc224b3e357c01bce98f957bf63786f4e467a2ed8 360992615 public preserved +youtube-TcnzB2tB-8Q media-video-tcnzb2tb-8q-webp TcnzB2tB-8Q.webp image https://www.youtube.com/watch?v=TcnzB2tB-8Q md/TechWorks/media/video/TcnzB2tB-8Q.webp 2026-08-08T00:00:00Z 779d526c75f997ad3c12419c9ad9e46c8ea9d905d8181f3c39c790270371a71b 33298 public preserved +youtube-bmHTFo2Rf2w media-video-bmhtfo2rf2w-en-orig-vtt bmHTFo2Rf2w.en-orig.vtt captions https://www.youtube.com/watch?v=bmHTFo2Rf2w md/TechWorks/media/video/bmHTFo2Rf2w.en-orig.vtt 2026-08-08T00:00:00Z a15dd034225897ddf87849d027c48c7d3ef0979b2bb17329172c5103e5b7e794 332801 public preserved +youtube-bmHTFo2Rf2w media-video-bmhtfo2rf2w-en-vtt bmHTFo2Rf2w.en.vtt captions https://www.youtube.com/watch?v=bmHTFo2Rf2w md/TechWorks/media/video/bmHTFo2Rf2w.en.vtt 2026-08-08T00:00:00Z a15dd034225897ddf87849d027c48c7d3ef0979b2bb17329172c5103e5b7e794 332801 public preserved +youtube-bmHTFo2Rf2w media-video-bmhtfo2rf2w-info-json bmHTFo2Rf2w.info.json metadata https://www.youtube.com/watch?v=bmHTFo2Rf2w md/TechWorks/media/video/bmHTFo2Rf2w.info.json 2026-08-08T00:00:00Z 5330706e824804a97059a9ea78e04ec5b30f3b35cde21e51ca9c999fb9203854 503509 public preserved +youtube-bmHTFo2Rf2w media-video-bmhtfo2rf2w-jpg bmHTFo2Rf2w.jpg image https://www.youtube.com/watch?v=bmHTFo2Rf2w md/TechWorks/media/video/bmHTFo2Rf2w.jpg 2026-08-08T00:00:00Z d04ff1f53296811517fd27fb2ed11b4092709558723c36d07f9719a9a4850ca4 64841 public preserved +youtube-bmHTFo2Rf2w media-video-bmhtfo2rf2w-mp4 bmHTFo2Rf2w.mp4 video https://www.youtube.com/watch?v=bmHTFo2Rf2w md/TechWorks/media/video/bmHTFo2Rf2w.mp4 2026-08-08T00:00:00Z c318fdba4c6c5caa69baefa238a7b6ecd94afefc8d2c8826a7931028d989fa51 126468297 public preserved +youtube-tSw3x0rVh88 media-video-tsw3x0rvh88-en-orig-vtt tSw3x0rVh88.en-orig.vtt captions https://www.youtube.com/watch?v=tSw3x0rVh88 md/TechWorks/media/video/tSw3x0rVh88.en-orig.vtt 2026-08-08T00:00:00Z d9feaebbf7dd5f8037be7b60226c1e773f304b479f331a2bea3c1a16338684d4 288703 public preserved +youtube-tSw3x0rVh88 media-video-tsw3x0rvh88-en-vtt tSw3x0rVh88.en.vtt captions https://www.youtube.com/watch?v=tSw3x0rVh88 md/TechWorks/media/video/tSw3x0rVh88.en.vtt 2026-08-08T00:00:00Z d9feaebbf7dd5f8037be7b60226c1e773f304b479f331a2bea3c1a16338684d4 288703 public preserved +youtube-tSw3x0rVh88 media-video-tsw3x0rvh88-info-json tSw3x0rVh88.info.json metadata https://www.youtube.com/watch?v=tSw3x0rVh88 md/TechWorks/media/video/tSw3x0rVh88.info.json 2026-08-08T00:00:00Z 1e876de98c65f221e3b437892e93a5c1ef911b023e85e78975769e015d95152e 491056 public preserved +youtube-tSw3x0rVh88 media-video-tsw3x0rvh88-jpg tSw3x0rVh88.jpg image https://www.youtube.com/watch?v=tSw3x0rVh88 md/TechWorks/media/video/tSw3x0rVh88.jpg 2026-08-08T00:00:00Z f50e8de09c6a01a8ac8178eec634fb42151ecd83e9fa1de49ee76e862b8b0507 57563 public preserved +youtube-tSw3x0rVh88 media-video-tsw3x0rvh88-mp4 tSw3x0rVh88.mp4 video https://www.youtube.com/watch?v=tSw3x0rVh88 md/TechWorks/media/video/tSw3x0rVh88.mp4 2026-08-08T00:00:00Z 9b5b40432bc71b1da7aeb72a106c524136112e12be1da250e8b01a8d482a262b 149330671 public preserved +vimeo-144696304 media-video-vimeo-144696304-oembed-json vimeo-144696304.oembed.json metadata https://vimeo.com/144696304 md/TechWorks/media/video/vimeo-144696304.oembed.json 2026-08-08T00:00:00Z 94d65c64f26cc5ba9bfd7377b2ffa8dd7bdb27a93db1cf47a33fc013283312a0 2289 public preserved +vimeo-144997124 media-video-vimeo-144997124-oembed-json vimeo-144997124.oembed.json metadata https://vimeo.com/144997124 md/TechWorks/media/video/vimeo-144997124.oembed.json 2026-08-08T00:00:00Z 0262515e872a123dfe8d026dd00464f221a4c980261be636c5bfa557fa6872f8 2270 public preserved +youtube-wVXjExRiFy0 media-video-wvxjexrify0-en-orig-vtt wVXjExRiFy0.en-orig.vtt captions https://www.youtube.com/watch?v=wVXjExRiFy0 md/TechWorks/media/video/wVXjExRiFy0.en-orig.vtt 2026-08-08T00:00:00Z 3aad0c50ebc6588f3a86aa4dde39e312f72c80680a6781ea7afb52dfc9780499 399905 public preserved +youtube-wVXjExRiFy0 media-video-wvxjexrify0-en-vtt wVXjExRiFy0.en.vtt captions https://www.youtube.com/watch?v=wVXjExRiFy0 md/TechWorks/media/video/wVXjExRiFy0.en.vtt 2026-08-08T00:00:00Z 3aad0c50ebc6588f3a86aa4dde39e312f72c80680a6781ea7afb52dfc9780499 399905 public preserved +youtube-wVXjExRiFy0 media-video-wvxjexrify0-info-json wVXjExRiFy0.info.json metadata https://www.youtube.com/watch?v=wVXjExRiFy0 md/TechWorks/media/video/wVXjExRiFy0.info.json 2026-08-08T00:00:00Z 2fa46c61ffce23593ef18c65bdeea96bff452d233c0b75f348709ee58f85a040 511168 public preserved +youtube-wVXjExRiFy0 media-video-wvxjexrify0-jpg wVXjExRiFy0.jpg image https://www.youtube.com/watch?v=wVXjExRiFy0 md/TechWorks/media/video/wVXjExRiFy0.jpg 2026-08-08T00:00:00Z 23aa4b7e40ecb6f2cf4e3835ed2e0742361c247fc27b4b5c919ddf65215787e5 51858 public preserved +youtube-wVXjExRiFy0 media-video-wvxjexrify0-mp4 wVXjExRiFy0.mp4 video https://www.youtube.com/watch?v=wVXjExRiFy0 md/TechWorks/media/video/wVXjExRiFy0.mp4 2026-08-08T00:00:00Z a7d709bd9848acd265ee0a8837f1045cc29522ec544467b1922d926bbab7a0a7 196324152 public preserved +youtube-xaxF5RDdVRE media-video-xaxf5rddvre-en-orig-vtt xaxF5RDdVRE.en-orig.vtt captions https://www.youtube.com/watch?v=xaxF5RDdVRE md/TechWorks/media/video/xaxF5RDdVRE.en-orig.vtt 2026-08-08T00:00:00Z 76fe6f5dcebea899924ed50454fd1e8a83fc1ebcb3833a1996b14ac22d8d442f 294720 public preserved +youtube-xaxF5RDdVRE media-video-xaxf5rddvre-en-vtt xaxF5RDdVRE.en.vtt captions https://www.youtube.com/watch?v=xaxF5RDdVRE md/TechWorks/media/video/xaxF5RDdVRE.en.vtt 2026-08-08T00:00:00Z 76fe6f5dcebea899924ed50454fd1e8a83fc1ebcb3833a1996b14ac22d8d442f 294720 public preserved +youtube-xaxF5RDdVRE media-video-xaxf5rddvre-info-json xaxF5RDdVRE.info.json metadata https://www.youtube.com/watch?v=xaxF5RDdVRE md/TechWorks/media/video/xaxF5RDdVRE.info.json 2026-08-08T00:00:00Z e5e57746ea11987744b1acacd6255d24951fddd939b57f1ceaf1b873a3b786bc 496001 public preserved +youtube-xaxF5RDdVRE media-video-xaxf5rddvre-mp4 xaxF5RDdVRE.mp4 video https://www.youtube.com/watch?v=xaxF5RDdVRE md/TechWorks/media/video/xaxF5RDdVRE.mp4 2026-08-08T00:00:00Z 0adfe2c4588a6fc2f1fc39863924cc1a4a01c56c80c2e23982e3c79dee2b259a 196095149 public preserved +youtube-xaxF5RDdVRE media-video-xaxf5rddvre-webp xaxF5RDdVRE.webp image https://www.youtube.com/watch?v=xaxF5RDdVRE md/TechWorks/media/video/xaxF5RDdVRE.webp 2026-08-08T00:00:00Z a73794c8dd4a4550f2098db3a854959449c0adf623ea4877d3dc13e959b7cacc 45366 public preserved +ocrug-2018-11-27 ocrug-2018-11-27-71dd0b1-tar-gz ocrug-2018-11-27-71dd0b1.tar.gz git-snapshot https://github.com/alandipert/ocrug-2018-11-27 md/TechWorks/ocrug-2018-11-27-71dd0b1.tar.gz 2026-08-08T00:00:00Z 295dcd7f2ea293bb9e3bb46417d787a934e739739a98bccf7031276c6366a280 1601 public preserved +oscon2012-clojure oscon2012-clojure-5ac233f-tar-gz oscon2012-clojure-5ac233f.tar.gz git-snapshot https://github.com/alandipert/oscon2012-clojure md/TechWorks/oscon2012-clojure-5ac233f.tar.gz 2026-08-08T00:00:00Z 37eb2a8885fae16fc6f45b7910a272d9564246cf140ecd72db756fbdc3efa852 380357 public preserved +wombat wombat-dbd34f4-tar-gz wombat-dbd34f4.tar.gz git-snapshot https://github.com/alandipert/wombat md/TechWorks/wombat-dbd34f4.tar.gz 2026-08-08T00:00:00Z 4fa2b18b4df657cc4fe0b1dcc61fac3a179bb7da636cf90c06de98b38efe7e6f 11907 public preserved diff --git a/tests/test_archive_techworks.py b/tests/test_archive_techworks.py new file mode 100644 index 0000000..3fe4d8f --- /dev/null +++ b/tests/test_archive_techworks.py @@ -0,0 +1,77 @@ +import csv +import hashlib +from pathlib import Path +import tempfile +import unittest + +import archive_techworks + + +class ArchiveTechWorksTests(unittest.TestCase): + def write_manifest(self, root, rows): + path = root / "archive" / "techworks" / "manifest.tsv" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter( + stream, + fieldnames=archive_techworks.FIELDS, + dialect="excel-tab", + ) + writer.writeheader() + writer.writerows(rows) + return path + + def test_audit_accepts_a_complete_preserved_artifact(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifact = root / "md" / "TechWorks" / "article.html" + artifact.parent.mkdir(parents=True) + artifact.write_text("preserved", encoding="utf-8") + row = { + "work_id": "work", + "artifact_id": "work-page", + "title": "Work page", + "kind": "html", + "source_url": "https://example.com/work", + "local_path": "md/TechWorks/article.html", + "retrieved_utc": "2026-08-09T00:00:00Z", + "sha256": hashlib.sha256(b"preserved").hexdigest(), + "bytes": str(len(b"preserved")), + "visibility": "public", + "status": "preserved", + "notes": "", + } + manifest = self.write_manifest(root, [row]) + rows = archive_techworks.audit(root, manifest, verify_formats=False) + self.assertEqual(rows[0]["artifact_id"], "work-page") + + def test_audit_rejects_an_unmanifested_corpus_file(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifact = root / "archive" / "techworks" / "orphan.bin" + artifact.parent.mkdir(parents=True) + artifact.write_bytes(b"orphan") + manifest = self.write_manifest(root, []) + with self.assertRaisesRegex( + archive_techworks.AuditError, "unmanifested corpus file" + ): + archive_techworks.audit(root, manifest, verify_formats=False) + + def test_export_destination_must_be_absolute_and_empty(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with self.assertRaisesRegex(archive_techworks.AuditError, "absolute"): + archive_techworks.safe_destination(root, Path("relative")) + occupied = root.parent / f"{root.name}-occupied" + occupied.mkdir() + try: + (occupied / "file").write_text("data", encoding="utf-8") + with self.assertRaisesRegex(archive_techworks.AuditError, "not empty"): + archive_techworks.safe_destination(root, occupied) + finally: + (occupied / "file").unlink() + occupied.rmdir() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/archive_techworks.py b/tools/archive_techworks.py new file mode 100644 index 0000000..99816d4 --- /dev/null +++ b/tools/archive_techworks.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +"""Audit and export the repository's TechWorks preservation corpus.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +from pathlib import Path +import shutil +import subprocess +import sys + + +FIELDS = ( + "work_id", "artifact_id", "title", "kind", "source_url", "local_path", + "retrieved_utc", "sha256", "bytes", "visibility", "status", "notes", +) +STATUSES = {"preserved", "metadata-only", "missing", "rights-limited"} +VISIBILITIES = {"public", "repository-only"} +CONTROL_PATHS = { + "archive/techworks/README.md", + "archive/techworks/manifest.tsv", + "archive/techworks/SHA256SUMS", +} + + +class AuditError(Exception): + pass + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def run(command: list[str], cwd: Path) -> None: + subprocess.run(command, cwd=cwd, check=True, stdout=subprocess.DEVNULL) + + +def load_manifest(path: Path) -> list[dict[str, str]]: + with path.open(encoding="utf-8", newline="") as stream: + reader = csv.DictReader(stream, dialect="excel-tab") + if tuple(reader.fieldnames or ()) != FIELDS: + raise AuditError( + f"{path}: expected fields {FIELDS}, got {tuple(reader.fieldnames or ())}" + ) + return list(reader) + + +def check_artifact(path: Path, kind: str, root: Path) -> None: + if kind in {"audio", "video"}: + run([ + "ffprobe", "-v", "error", "-show_entries", + "format=duration:stream=codec_type,codec_name,width,height", + "-of", "default=noprint_wrappers=1", str(path), + ], root) + elif kind == "pdf": + run(["pdfinfo", str(path)], root) + elif kind == "git-bundle": + run(["git", "bundle", "verify", str(path)], root) + elif kind == "warc": + run(["gzip", "--test", str(path)], root) + + +def discover_corpus_files(root: Path) -> set[str]: + paths: set[str] = set() + for directory in (root / "md" / "TechWorks", root / "archive" / "techworks"): + if not directory.exists(): + continue + for path in directory.rglob("*"): + if path.is_file(): + relative = path.relative_to(root).as_posix() + if relative not in CONTROL_PATHS: + paths.add(relative) + return paths + + +def audit(root: Path, manifest_path: Path, verify_formats: bool = True) -> list[dict[str, str]]: + rows = load_manifest(manifest_path) + errors: list[str] = [] + artifact_ids: set[str] = set() + preserved_paths: set[str] = set() + + for line_number, row in enumerate(rows, start=2): + prefix = f"{manifest_path}:{line_number}" + artifact_id = row["artifact_id"] + if not artifact_id: + errors.append(f"{prefix}: artifact_id is empty") + elif artifact_id in artifact_ids: + errors.append(f"{prefix}: duplicate artifact_id {artifact_id}") + artifact_ids.add(artifact_id) + + if row["status"] not in STATUSES: + errors.append(f"{prefix}: invalid status {row['status']!r}") + if row["visibility"] not in VISIBILITIES: + errors.append(f"{prefix}: invalid visibility {row['visibility']!r}") + + local_path = row["local_path"] + if row["status"] != "preserved": + if local_path or row["sha256"] or row["bytes"]: + errors.append(f"{prefix}: non-preserved row contains local file data") + continue + + if not local_path: + errors.append(f"{prefix}: preserved row has no local_path") + continue + path = root / local_path + preserved_paths.add(Path(local_path).as_posix()) + if path.is_symlink(): + errors.append(f"{prefix}: symbolic links are not allowed: {local_path}") + continue + if not path.is_file(): + errors.append(f"{prefix}: missing file: {local_path}") + continue + if row["visibility"] == "public" and not local_path.startswith("md/TechWorks/"): + errors.append(f"{prefix}: public artifact is outside md/TechWorks: {local_path}") + try: + expected_size = int(row["bytes"]) + except ValueError: + errors.append(f"{prefix}: invalid byte count {row['bytes']!r}") + else: + if path.stat().st_size != expected_size: + errors.append( + f"{prefix}: size mismatch for {local_path}: expected " + f"{expected_size}, got {path.stat().st_size}" + ) + actual_hash = sha256(path) + if actual_hash != row["sha256"]: + errors.append( + f"{prefix}: SHA-256 mismatch for {local_path}: expected " + f"{row['sha256']}, got {actual_hash}" + ) + if verify_formats: + try: + check_artifact(path, row["kind"], root) + except (OSError, subprocess.CalledProcessError) as error: + errors.append(f"{prefix}: format verification failed for {local_path}: {error}") + + errors.extend( + f"unmanifested corpus file: {path}" + for path in sorted(discover_corpus_files(root) - preserved_paths) + ) + if errors: + raise AuditError("\n".join(errors)) + return rows + + +def write_checksums(rows: list[dict[str, str]], destination: Path) -> None: + lines = [ + f"{row['sha256']} {row['local_path']}" + for row in rows if row["status"] == "preserved" + ] + destination.write_text("\n".join(sorted(lines)) + "\n", encoding="utf-8") + + +def safe_destination(root: Path, destination: Path) -> Path: + if not destination.is_absolute(): + raise AuditError("export destination must be an absolute path") + resolved = destination.resolve() + if resolved in {Path("/"), root.resolve(), root.resolve().parent}: + raise AuditError(f"unsafe export destination: {resolved}") + if resolved.exists() and any(resolved.iterdir()): + raise AuditError(f"export destination is not empty: {resolved}") + return resolved + + +def export(root: Path, manifest_path: Path, destination: Path) -> None: + rows = audit(root, manifest_path) + destination = safe_destination(root, destination) + destination.mkdir(parents=True, exist_ok=True) + bundle_dir = destination / "git" + files_dir = destination / "files" + lfs_dir = destination / "lfs" / "objects" + bundle_dir.mkdir() + files_dir.mkdir() + + run(["git", "bundle", "create", str(bundle_dir / "homepage.bundle"), "--all"], root) + run(["git", "lfs", "fetch", "--all"], root) + source_lfs = root / ".git" / "lfs" / "objects" + if source_lfs.exists(): + shutil.copytree(source_lfs, lfs_dir, dirs_exist_ok=True) + + for row in rows: + if row["status"] != "preserved": + continue + source = root / row["local_path"] + target = files_dir / row["local_path"] + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + + shutil.copy2(manifest_path, destination / "manifest.tsv") + (destination / "RESTORE.md").write_text( + """# TechWorks portable preservation export + +Verify this export with `sha256sum --check SHA256SUMS`. + +Restore Git history with `git clone git/homepage.bundle homepage`. Copy +`lfs/objects/` into `homepage/.git/lfs/objects/`, then check out the desired +revision. The `files/` tree is a directly browsable copy of every preserved +artifact in the manifest. +""", + encoding="utf-8", + ) + checksum_paths = [ + path for path in destination.rglob("*") + if path.is_file() and path.name != "SHA256SUMS" + ] + (destination / "SHA256SUMS").write_text( + "\n".join(sorted( + f"{sha256(path)} {path.relative_to(destination).as_posix()}" + for path in checksum_paths + )) + "\n", + encoding="utf-8", + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--manifest", type=Path, default=Path("archive/techworks/manifest.tsv")) + subparsers = parser.add_subparsers(dest="command", required=True) + audit_parser = subparsers.add_parser("audit") + audit_parser.add_argument("--skip-format-checks", action="store_true") + subparsers.add_parser("checksums") + export_parser = subparsers.add_parser("export") + export_parser.add_argument("--dest", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + root = args.root.resolve() + manifest = args.manifest if args.manifest.is_absolute() else root / args.manifest + try: + if args.command == "audit": + rows = audit(root, manifest, not args.skip_format_checks) + print(f"TechWorks archive OK: {len(rows)} manifest rows") + elif args.command == "checksums": + rows = audit(root, manifest, verify_formats=False) + write_checksums(rows, root / "archive/techworks/SHA256SUMS") + elif args.command == "export": + export(root, manifest, args.dest) + print(f"TechWorks export created at {args.dest}") + except (AuditError, OSError, subprocess.CalledProcessError) as error: + print(error, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())