From d07f48d69ee417daa2bb402ab4b84c4acfe03871 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Fri, 18 Sep 2026 16:24:16 +0700 Subject: [PATCH] test(newsletter): add a golden-output parity suite Captures the Go engine's stdout for 26 deterministic cases, commits those outputs as fixtures, and asserts the JavaScript engine reproduces them through the real CLI surface, exit codes included. This is both the migration's proof of correctness and the regression net the engine never had. Every fixture was generated from the Go engine rather than written by hand, and reviewed before commit so no existing bug became a permanent contract. Each behaviour that a JavaScript-idiomatic rewrite would silently break has a case that goes red when the naive implementation is reintroduced; verified by mutating the engine and re-running. Content-scanning commands are pointed at a fixture tree through the subprocess working directory. Both engines already resolve content/post from the process CWD, so neither needed a test-only configuration knob, and the suite is proven isolated from the real content by renaming it away and re-running. Cases that depend on live upstreams are opt-in behind NEWSLETTER_NET=1: a publication's feed window moves, and a suite that goes red on someone else's outage teaches people to ignore red. The deterministic tier reaches the network only through .invalid hosts, which never resolve, so it runs offline in about two seconds. The golden files are compared byte for byte, so scripts/newsletter is pinned to LF; the repository otherwise checks out CRLF, which would break every comparison on a fresh clone. --- .gitattributes | 4 + .../__fixtures__/golden/_exit-codes.json | 31 ++ .../golden/add-url-article-no-metadata.json | 8 + .../golden/add-url-document-pdf.json | 8 + .../golden/add-url-dup-autolink.json | 8 + .../__fixtures__/golden/add-url-dup-bare.json | 8 + .../golden/add-url-dup-markdown-link.json | 8 + .../golden/add-url-dup-trailing-slash.json | 8 + .../golden/add-url-encoding-preserved.json | 8 + .../golden/add-url-image-avif-query.json | 8 + .../golden/add-url-image-cdn-duplicate.json | 8 + .../golden/add-url-image-s3-duplicate.json | 8 + .../add-url-prefix-exact-duplicate.json | 8 + .../golden/add-url-prefix-not-duplicate.json | 8 + .../golden/add-url-query-verbatim.json | 8 + .../golden/add-url-tracking-params.json | 8 + .../golden/add-url-video-mp4.json | 8 + .../golden/detect-image-cdn-wrapper.json | 7 + .../detect-image-malformed-percent.json | 6 + .../golden/detect-image-non-substack.json | 5 + .../golden/detect-image-raw-s3.json | 7 + .../extract-edge-html-no-figcaption.json | 9 + .../golden/extract-edge-html.json | 9 + .../__fixtures__/golden/extract-rss-item.json | 16 + .../golden/find-newsletter-number-empty.txt | 1 + .../golden/find-newsletter-number-main.txt | 1 + .../find-newsletter-number-year-fallback.txt | 1 + .../golden/list-existing-tags-main.txt | 5 + .../golden/post-stats-empty-bonus.json | 9 + .../__fixtures__/golden/post-stats-full.json | 9 + .../golden/post-stats-no-bonus.json | 9 + .../__fixtures__/markup/bytebytego-feed.xml | 7 + .../markup/bytebytego-sitemap.xml | 1 + .../__fixtures__/markup/edge-cases.html | 20 ++ .../repo-empty/content/post/.gitkeep | 0 .../content/post/2025/06/01/index.md | 7 + .../content/post/2026/01/02/index.md | 8 + .../repo/content/post/2025/12/31/index.md | 12 + .../repo/content/post/2026/09/10/index.md | 31 ++ .../content/post/2026/09/11/empty-bonus.md | 16 + .../repo/content/post/2026/09/11/index.md | 30 ++ .../repo/content/post/2026/09/11/no-bonus.md | 11 + scripts/newsletter/network.test.js | 102 ++++++ scripts/newsletter/parity.test.js | 304 ++++++++++++++++++ 44 files changed, 798 insertions(+) create mode 100644 scripts/newsletter/__fixtures__/golden/_exit-codes.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-article-no-metadata.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-document-pdf.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-dup-autolink.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-dup-bare.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-dup-markdown-link.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-dup-trailing-slash.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-encoding-preserved.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-image-avif-query.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-image-cdn-duplicate.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-image-s3-duplicate.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-prefix-exact-duplicate.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-prefix-not-duplicate.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-query-verbatim.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-tracking-params.json create mode 100644 scripts/newsletter/__fixtures__/golden/add-url-video-mp4.json create mode 100644 scripts/newsletter/__fixtures__/golden/detect-image-cdn-wrapper.json create mode 100644 scripts/newsletter/__fixtures__/golden/detect-image-malformed-percent.json create mode 100644 scripts/newsletter/__fixtures__/golden/detect-image-non-substack.json create mode 100644 scripts/newsletter/__fixtures__/golden/detect-image-raw-s3.json create mode 100644 scripts/newsletter/__fixtures__/golden/extract-edge-html-no-figcaption.json create mode 100644 scripts/newsletter/__fixtures__/golden/extract-edge-html.json create mode 100644 scripts/newsletter/__fixtures__/golden/extract-rss-item.json create mode 100644 scripts/newsletter/__fixtures__/golden/find-newsletter-number-empty.txt create mode 100644 scripts/newsletter/__fixtures__/golden/find-newsletter-number-main.txt create mode 100644 scripts/newsletter/__fixtures__/golden/find-newsletter-number-year-fallback.txt create mode 100644 scripts/newsletter/__fixtures__/golden/list-existing-tags-main.txt create mode 100644 scripts/newsletter/__fixtures__/golden/post-stats-empty-bonus.json create mode 100644 scripts/newsletter/__fixtures__/golden/post-stats-full.json create mode 100644 scripts/newsletter/__fixtures__/golden/post-stats-no-bonus.json create mode 100644 scripts/newsletter/__fixtures__/markup/bytebytego-feed.xml create mode 100644 scripts/newsletter/__fixtures__/markup/bytebytego-sitemap.xml create mode 100644 scripts/newsletter/__fixtures__/markup/edge-cases.html create mode 100644 scripts/newsletter/__fixtures__/repo-empty/content/post/.gitkeep create mode 100644 scripts/newsletter/__fixtures__/repo-year-fallback/content/post/2025/06/01/index.md create mode 100644 scripts/newsletter/__fixtures__/repo-year-fallback/content/post/2026/01/02/index.md create mode 100644 scripts/newsletter/__fixtures__/repo/content/post/2025/12/31/index.md create mode 100644 scripts/newsletter/__fixtures__/repo/content/post/2026/09/10/index.md create mode 100644 scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/empty-bonus.md create mode 100644 scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/index.md create mode 100644 scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/no-bonus.md create mode 100644 scripts/newsletter/network.test.js create mode 100644 scripts/newsletter/parity.test.js diff --git a/.gitattributes b/.gitattributes index 48b311b..b88e39c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,7 @@ * text=auto eol=crlf *.sh text eol=lf + +# The newsletter parity suite compares engine stdout to committed golden files +# byte for byte, so these must stay LF on every checkout. +scripts/newsletter/** text eol=lf diff --git a/scripts/newsletter/__fixtures__/golden/_exit-codes.json b/scripts/newsletter/__fixtures__/golden/_exit-codes.json new file mode 100644 index 0000000..2a00ed3 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/_exit-codes.json @@ -0,0 +1,31 @@ +{ + "add-url-tracking-params": 0, + "add-url-encoding-preserved": 0, + "add-url-query-verbatim": 0, + "add-url-dup-bare": 0, + "add-url-dup-trailing-slash": 0, + "add-url-dup-autolink": 0, + "add-url-dup-markdown-link": 0, + "add-url-prefix-not-duplicate": 0, + "add-url-prefix-exact-duplicate": 0, + "add-url-image-cdn-duplicate": 0, + "add-url-image-s3-duplicate": 0, + "add-url-image-avif-query": 0, + "add-url-document-pdf": 0, + "add-url-video-mp4": 0, + "add-url-article-no-metadata": 0, + "detect-image-cdn-wrapper": 0, + "detect-image-raw-s3": 0, + "detect-image-non-substack": 0, + "detect-image-malformed-percent": 0, + "find-newsletter-number-main": 0, + "find-newsletter-number-empty": 0, + "find-newsletter-number-year-fallback": 0, + "list-existing-tags-main": 0, + "post-stats-full": 0, + "post-stats-empty-bonus": 0, + "post-stats-no-bonus": 0, + "extract-rss-item": 0, + "extract-edge-html": 0, + "extract-edge-html-no-figcaption": 0 +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-article-no-metadata.json b/scripts/newsletter/__fixtures__/golden/add-url-article-no-metadata.json new file mode 100644 index 0000000..95ba09c --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-article-no-metadata.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://a.invalid/plain", + "clean_url": "https://a.invalid/plain", + "http_status": "000", + "accessible": false, + "duplicate": false, + "route": "article" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-document-pdf.json b/scripts/newsletter/__fixtures__/golden/add-url-document-pdf.json new file mode 100644 index 0000000..69c3963 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-document-pdf.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://a.invalid/file.pdf", + "clean_url": "https://a.invalid/file.pdf", + "http_status": "000", + "accessible": false, + "duplicate": false, + "route": "document" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-dup-autolink.json b/scripts/newsletter/__fixtures__/golden/add-url-dup-autolink.json new file mode 100644 index 0000000..5f10114 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-dup-autolink.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://autolink.invalid/entry", + "clean_url": "https://autolink.invalid/entry", + "http_status": "000", + "accessible": false, + "duplicate": true, + "route": "article" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-dup-bare.json b/scripts/newsletter/__fixtures__/golden/add-url-dup-bare.json new file mode 100644 index 0000000..b961027 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-dup-bare.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://bare2.invalid/article", + "clean_url": "https://bare2.invalid/article", + "http_status": "000", + "accessible": false, + "duplicate": true, + "route": "article" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-dup-markdown-link.json b/scripts/newsletter/__fixtures__/golden/add-url-dup-markdown-link.json new file mode 100644 index 0000000..2050405 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-dup-markdown-link.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://link.invalid/story", + "clean_url": "https://link.invalid/story", + "http_status": "000", + "accessible": false, + "duplicate": true, + "route": "article" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-dup-trailing-slash.json b/scripts/newsletter/__fixtures__/golden/add-url-dup-trailing-slash.json new file mode 100644 index 0000000..5384291 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-dup-trailing-slash.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://slash.invalid/post/", + "clean_url": "https://slash.invalid/post/", + "http_status": "000", + "accessible": false, + "duplicate": true, + "route": "article" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-encoding-preserved.json b/scripts/newsletter/__fixtures__/golden/add-url-encoding-preserved.json new file mode 100644 index 0000000..b567f2f --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-encoding-preserved.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://A.invalid/path/?a=%7Efoo+bar&b=2#frag", + "clean_url": "https://a.invalid/path/?a=%7Efoo+bar&b=2#frag", + "http_status": "000", + "accessible": false, + "duplicate": false, + "route": "article" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-image-avif-query.json b/scripts/newsletter/__fixtures__/golden/add-url-image-avif-query.json new file mode 100644 index 0000000..5176a3f --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-image-avif-query.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://cdn.invalid/image/fetch/f_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12345678-1234-1234-1234-123456789abc_100x100.avif?x=1", + "clean_url": "https://cdn.invalid/image/fetch/f_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12345678-1234-1234-1234-123456789abc_100x100.avif?x=1", + "http_status": "000", + "accessible": false, + "duplicate": false, + "route": "image" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-image-cdn-duplicate.json b/scripts/newsletter/__fixtures__/golden/add-url-image-cdn-duplicate.json new file mode 100644 index 0000000..f8fdc91 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-image-cdn-duplicate.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://cdn.invalid/image/fetch/w_1100,c_limit,f_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_2484x3002.png", + "clean_url": "https://cdn.invalid/image/fetch/w_1100,c_limit,f_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_2484x3002.png", + "http_status": "000", + "accessible": false, + "duplicate": true, + "route": "image" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-image-s3-duplicate.json b/scripts/newsletter/__fixtures__/golden/add-url-image-s3-duplicate.json new file mode 100644 index 0000000..ff79622 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-image-s3-duplicate.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://substack-post-media.s3.invalid/public/images/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_1280x720.png", + "clean_url": "https://substack-post-media.s3.invalid/public/images/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_1280x720.png", + "http_status": "000", + "accessible": false, + "duplicate": true, + "route": "image" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-prefix-exact-duplicate.json b/scripts/newsletter/__fixtures__/golden/add-url-prefix-exact-duplicate.json new file mode 100644 index 0000000..e448307 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-prefix-exact-duplicate.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://prefix.invalid/posts/the-long-slug", + "clean_url": "https://prefix.invalid/posts/the-long-slug", + "http_status": "000", + "accessible": false, + "duplicate": true, + "route": "article" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-prefix-not-duplicate.json b/scripts/newsletter/__fixtures__/golden/add-url-prefix-not-duplicate.json new file mode 100644 index 0000000..117b510 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-prefix-not-duplicate.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://prefix.invalid/posts/the-long", + "clean_url": "https://prefix.invalid/posts/the-long", + "http_status": "000", + "accessible": false, + "duplicate": false, + "route": "article" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-query-verbatim.json b/scripts/newsletter/__fixtures__/golden/add-url-query-verbatim.json new file mode 100644 index 0000000..77ae44b --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-query-verbatim.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://a.invalid/path?a=%20x&flag&b=%7ez&utm_source=drop", + "clean_url": "https://a.invalid/path?a=%20x&flag&b=%7ez", + "http_status": "000", + "accessible": false, + "duplicate": false, + "route": "article" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-tracking-params.json b/scripts/newsletter/__fixtures__/golden/add-url-tracking-params.json new file mode 100644 index 0000000..3343f0c --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-tracking-params.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://a.invalid/article?utm_source=x&utm_medium=y&fbclid=abc&keep=1&ref=z&s=1", + "clean_url": "https://a.invalid/article?keep=1", + "http_status": "000", + "accessible": false, + "duplicate": false, + "route": "article" +} diff --git a/scripts/newsletter/__fixtures__/golden/add-url-video-mp4.json b/scripts/newsletter/__fixtures__/golden/add-url-video-mp4.json new file mode 100644 index 0000000..8e46450 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/add-url-video-mp4.json @@ -0,0 +1,8 @@ +{ + "original_url": "https://a.invalid/clip.mp4", + "clean_url": "https://a.invalid/clip.mp4", + "http_status": "000", + "accessible": false, + "duplicate": false, + "route": "video" +} diff --git a/scripts/newsletter/__fixtures__/golden/detect-image-cdn-wrapper.json b/scripts/newsletter/__fixtures__/golden/detect-image-cdn-wrapper.json new file mode 100644 index 0000000..679d3ed --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/detect-image-cdn-wrapper.json @@ -0,0 +1,7 @@ +{ + "original_url": "https://substackcdn.com/image/fetch/$s_!lpxK!,w_1100,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_2484x3002.png", + "clean_url": "https://substackcdn.com/image/fetch/$s_!lpxK!,w_1100,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_2484x3002.png", + "isSubstack": true, + "uuid": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "innerUrl": "https://substack-post-media.s3.amazonaws.com/public/images/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_2484x3002.png" +} diff --git a/scripts/newsletter/__fixtures__/golden/detect-image-malformed-percent.json b/scripts/newsletter/__fixtures__/golden/detect-image-malformed-percent.json new file mode 100644 index 0000000..e0030de --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/detect-image-malformed-percent.json @@ -0,0 +1,6 @@ +{ + "original_url": "https://substackcdn.com/image/fetch/w_100/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F%ZZbad.png", + "clean_url": "https://substackcdn.com/image/fetch/w_100/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F%ZZbad.png", + "isSubstack": true, + "innerUrl": "https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F%ZZbad.png" +} diff --git a/scripts/newsletter/__fixtures__/golden/detect-image-non-substack.json b/scripts/newsletter/__fixtures__/golden/detect-image-non-substack.json new file mode 100644 index 0000000..458b7ec --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/detect-image-non-substack.json @@ -0,0 +1,5 @@ +{ + "original_url": "https://a.invalid/pic.png", + "clean_url": "https://a.invalid/pic.png", + "isSubstack": false +} diff --git a/scripts/newsletter/__fixtures__/golden/detect-image-raw-s3.json b/scripts/newsletter/__fixtures__/golden/detect-image-raw-s3.json new file mode 100644 index 0000000..e6d599c --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/detect-image-raw-s3.json @@ -0,0 +1,7 @@ +{ + "original_url": "https://substack-post-media.s3.amazonaws.com/public/images/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_1280x720.png", + "clean_url": "https://substack-post-media.s3.amazonaws.com/public/images/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_1280x720.png", + "isSubstack": true, + "uuid": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "innerUrl": "https://substack-post-media.s3.amazonaws.com/public/images/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_1280x720.png" +} diff --git a/scripts/newsletter/__fixtures__/golden/extract-edge-html-no-figcaption.json b/scripts/newsletter/__fixtures__/golden/extract-edge-html-no-figcaption.json new file mode 100644 index 0000000..224141c --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/extract-edge-html-no-figcaption.json @@ -0,0 +1,9 @@ +{ + "candidates": [ + "🚀 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "sixchr", + "Duplicate Bullet Text" + ], + "caption": "", + "postTitle": "" +} diff --git a/scripts/newsletter/__fixtures__/golden/extract-edge-html.json b/scripts/newsletter/__fixtures__/golden/extract-edge-html.json new file mode 100644 index 0000000..c48553f --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/extract-edge-html.json @@ -0,0 +1,9 @@ +{ + "candidates": [ + "🚀 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "sixchr", + "Duplicate Bullet Text" + ], + "caption": "Inner caption & more", + "postTitle": "" +} diff --git a/scripts/newsletter/__fixtures__/golden/extract-rss-item.json b/scripts/newsletter/__fixtures__/golden/extract-rss-item.json new file mode 100644 index 0000000..3d5b9b3 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/extract-rss-item.json @@ -0,0 +1,16 @@ +{ + "candidates": [ + "How documents are turned into searchable passages", + "How LLMs find the meaning behind different words", + "How close is close enough", + "Why searching every passage is too expensive", + "Following connections to the the right neighborhood", + "How much searching is enough", + "What happens when the answer changes", + "Dot product reflects both alignment and length.", + "Pre-filtering identifies eligible records before similarity ranking." + ], + "caption": "", + "itemLink": "https://blog.bytebytego.com/p/how-llms-can-find-a-needle-in-a-haystack", + "itemTitle": "How LLMs Can Find a Needle in a Haystack" +} diff --git a/scripts/newsletter/__fixtures__/golden/find-newsletter-number-empty.txt b/scripts/newsletter/__fixtures__/golden/find-newsletter-number-empty.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/find-newsletter-number-empty.txt @@ -0,0 +1 @@ +1 diff --git a/scripts/newsletter/__fixtures__/golden/find-newsletter-number-main.txt b/scripts/newsletter/__fixtures__/golden/find-newsletter-number-main.txt new file mode 100644 index 0000000..6a4573e --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/find-newsletter-number-main.txt @@ -0,0 +1 @@ +133 diff --git a/scripts/newsletter/__fixtures__/golden/find-newsletter-number-year-fallback.txt b/scripts/newsletter/__fixtures__/golden/find-newsletter-number-year-fallback.txt new file mode 100644 index 0000000..82cced2 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/find-newsletter-number-year-fallback.txt @@ -0,0 +1 @@ +51 diff --git a/scripts/newsletter/__fixtures__/golden/list-existing-tags-main.txt b/scripts/newsletter/__fixtures__/golden/list-existing-tags-main.txt new file mode 100644 index 0000000..63ee2cf --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/list-existing-tags-main.txt @@ -0,0 +1,5 @@ + 3 AI-Assisted + 1 Older Year + 1 Software Engineering + 1 Tie Break A + 1 Tie Break B diff --git a/scripts/newsletter/__fixtures__/golden/post-stats-empty-bonus.json b/scripts/newsletter/__fixtures__/golden/post-stats-empty-bonus.json new file mode 100644 index 0000000..0410ca3 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/post-stats-empty-bonus.json @@ -0,0 +1,9 @@ +{ + "post": "content/post/2026/09/11/empty-bonus.md", + "newsletter": 0, + "articles": 1, + "images": 0, + "videos": 0, + "documents": 0, + "total": 1 +} diff --git a/scripts/newsletter/__fixtures__/golden/post-stats-full.json b/scripts/newsletter/__fixtures__/golden/post-stats-full.json new file mode 100644 index 0000000..df7a98c --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/post-stats-full.json @@ -0,0 +1,9 @@ +{ + "post": "content/post/2026/09/10/index.md", + "newsletter": 132, + "articles": 2, + "images": 1, + "videos": 1, + "documents": 1, + "total": 5 +} diff --git a/scripts/newsletter/__fixtures__/golden/post-stats-no-bonus.json b/scripts/newsletter/__fixtures__/golden/post-stats-no-bonus.json new file mode 100644 index 0000000..8456e46 --- /dev/null +++ b/scripts/newsletter/__fixtures__/golden/post-stats-no-bonus.json @@ -0,0 +1,9 @@ +{ + "post": "content/post/2026/09/11/no-bonus.md", + "newsletter": 0, + "articles": 2, + "images": 0, + "videos": 0, + "documents": 0, + "total": 2 +} diff --git a/scripts/newsletter/__fixtures__/markup/bytebytego-feed.xml b/scripts/newsletter/__fixtures__/markup/bytebytego-feed.xml new file mode 100644 index 0000000..3e970c4 --- /dev/null +++ b/scripts/newsletter/__fixtures__/markup/bytebytego-feed.xml @@ -0,0 +1,7 @@ +<![CDATA[ByteByteGo Newsletter]]>https://blog.bytebytego.comhttps://substackcdn.com/image/fetch/$s_!1eXV!,w_256,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F8a5609ae-1239-4400-9491-6010a15c4d60_504x504.pngByteByteGo Newsletterhttps://blog.bytebytego.comSubstackFri, 18 Sep 2026 07:37:53 GMT<![CDATA[Migrations at Scale: Changing the Application Engine at 30,000 Feet]]>https://blog.bytebytego.com/p/migrations-at-scale-changing-thehttps://blog.bytebytego.com/p/migrations-at-scale-changing-theThu, 17 Sep 2026 15:31:04 GMTConsider a hypothetical scenario in which an online shopping platform has become extremely successful. It has outgrown its original database. This causes order histories to load very slowly. Maintenance takes much longer than it used to. And engineers spend way too much time managing the workload and the system’s limitations.

The engineering team behind this system wants to replace the database and migrate the data to a new database that promises better performance. But you cannot take a popular online store offline to make the switch. Customers are always placing orders, changing addresses, and requesting refunds. The shop can’t be shut down just like that.

This scenario offers a glimpse of the main challenge inherent in production migrations. How do you change a working system while still running the service that people depend on? At scale, copying existing data during such migrations can take days. Also, many supporting applications may depend on the component being replaced. Every intermediate step needs to work while ordinary business continues.

In this article, we will look at how migrations work at scale and the key strategies that can help make it as efficient as possible.

What Exactly is Being Replaced?

+

+ + Read more + +

+ ]]><![CDATA[How LLMs Can Find a Needle in a Haystack]]>https://blog.bytebytego.com/p/how-llms-can-find-a-needle-in-a-haystackhttps://blog.bytebytego.com/p/how-llms-can-find-a-needle-in-a-haystackWed, 16 Sep 2026 15:31:42 GMTDebugging Agents in Different Environments - Live Workshop (Sponsored)

Your agent returns something odd. Was it the prompt, a tool call that timed out, or a response your code could not parse? Without traces, you are guessing.

In this hands-on workshop, Serge from Sentry instruments three agents with Sentry Agent Tracing: a chatbot in an ecommerce store, a custom Slack agent, and a GitHub Action that reviews PRs. You will see how to catch bad tool calls and unexpected output, plus how to track token spend and performance across every agent you run.

Save your spot

Imagine a scenario where an employee is stranded at an airport after a cancelled flight. Before booking a hotel, they ask the company’s AI assistant a simple question: “Can I expense a hotel if my flight gets cancelled?”

The company has thousands of documents covering travel, expenses, insurance, employee benefits, and regional policies. Somewhere inside them, a paragraph states that accommodation costs caused by involuntary travel disruption are reimbursable, subject to certain conditions.

For an LLM chatbot, finding that piece of information is harder than it appears. The question mentions a “cancelled flight,” while the policy refers to “involuntary travel disruption.” Other documents discuss hotels but apply to different countries. An older policy may be present that contains a reimbursement limit that has since changed.

In such a case, the LLM must find information that matches the question’s meaning, belongs to the correct policy, and remains valid today. Only then can it write a useful answer that solves the user’s problem.

This is known as the retrieval problem behind many LLM applications. In this article, we are going to look at how LLMs can find a needle in a haystack. Here’s what we will cover:

  • An LLM needs evidence to answer questions

  • How documents are turned into searchable passages

  • How LLMs find the meaning behind different words

  • How close is close enough

  • Why searching every passage is too expensive

  • Following connections to the the right neighborhood

  • How much searching is enough

  • What happens when the answer changes

The LLM Needs Evidence to Answer Questions

A language model does not automatically know what is inside a company’s private documents. An application must provide that information, either by including documents directly in its input or by retrieving relevant passages when a question arrives.

For a large collection, retrieval offers a way to select a manageable amount of information. An embedding model converts the question into a numerical representation. A search system then uses that representation to find promising passages. The application supplies the original text to the LLM.

The model can then use those passages to explain the policy and cite the exact sources where it got the information from. This pattern is called retrieval-augmented generation, or RAG. A vector database supports the retrieval part by storing and searching numerical representations of content.

This division of responsibilities is really useful. If the application retrieves an outdated policy, even a capable language model will produce an outdated answer. If it retrieves a general hotel-booking rule while missing the cancellation exception, the answer may sound reasonably correct while overlooking the important condition.

Reliable answers therefore depend on what happens before generation. This means we have to make the document collection searchable at the right level of detail.

How Documents are Turned into Searchable Passages

A travel handbook can cover flights, accommodation, meals, approvals, and insurance. However, treating the entire handbook as a single searchable item produces a broad representation that can obscure a specific rule.

Instead, the application divides documents into smaller units called chunks. One chunk might describe hotel reimbursement, while another explains approval requirements. Search can then identify a particular passage instead of merely identifying the handbook that contains it.

Chunking creates a balance between precision and context. A small chunk may focus tightly on the question but leave out an exception. A large chunk may preserve the exception while including several unrelated policies. For example, suppose a passage says, “Accommodation expenses are reimbursable following a cancellation.” And the next sentence adds, “This applies only when accommodation is not provided by the airline.” Separating those sentences could cause the assistant to give an incorrect answer despite having the relevant text.

A useful chunk therefore preserves a complete idea wherever possible. Section headings and limited overlap between neighboring chunks can help retain context. The goal is to create passages that remain understandable when retrieved independently.

For our example question by the employee, the ideal searchable unit contains the reimbursement rule, its conditions, and enough identifying context to establish which policy it belongs to.

How LLMs Find the Meaning Behind Different Words

Embeddings make it possible to search for related meaning even when the wording differs.

An embedding model takes a passage and produces a vector: a list of numbers, often containing hundreds or thousands of decimal values. Think of it as a map of text. Passages with related meanings occupy nearby positions. For example, a question about hotel expenses after a cancelled flight should appear closer to a travel-disruption policy than to instructions for resetting a password.

The real representation has many more dimensions than a physical map. Individual dimensions also don’t have simple labels such as “hotel,” “flight,” or “reimbursement.” Meaning is represented through patterns across the complete vector.

The trick is that the application embeds the question into the same space as the document chunks. This way, it can compare the question’s vector with stored vectors to identify nearby passages. Also, the query and document embeddings must come from compatible encoders. However, matching vector lengths alone doesn’t make two models compatible.

Each searchable record also needs a connection to the original text. The vector helps locate a passage, but the LLM needs the words themselves to interpret the rule. Therefore, a record should connect an embedding with a chunk identifier, the passage text, or its location. Also, metadata such as document ID, section, effective date, and version are present. These fields can be stored together or across connected storage systems. Structured identifiers make it possible to retrieve and maintain all chunks belonging to a document.

How Close Is Close Enough?

The search system needs a precise definition of “nearby.” This definition comes from a distance or similarity metric, which compares two vectors and assigns a score. There are different metrics around this:

  • Cosine similarity compares their directions while ignoring their lengths.

  • Euclidean distance measures the straight-line distance between their endpoints, so vector length can influence the result.

  • Dot product reflects both alignment and length.

Normalization rescales vectors to length one. When both query and document vectors are normalized, dot product equals cosine similarity. Euclidean distance then produces the same ranking, although its scores differ. Dot product also works with unnormalized vectors when that matches the embedding model’s design.

The choice of the metric should follow the embedding model’s intended use. Choosing a metric simply because it is popular can change the ranking in unintended ways.

A similarity score should also be interpreted carefully. A score of 0.85 does not mean that a passage has an 85% probability of answering the question correctly. It describes a mathematical relationship between representations.

The passage may discuss the right subject while stating the wrong regional policy. It may be outdated or omit an exception. Similarity provides evidence of relevance, but additional checks are needed before that evidence becomes an answer.

Why Searching Every Passage Is Too Expensive

Searching every vector is straightforward, but the work grows with the collection.

A flat index compares the query with every eligible vector and returns those with the best scores. For a fixed number of dimensions, the comparison work grows roughly in proportion to the vector count. This is the meaning of O(n). If the collection doubles, the number of vector comparisons doubles. Longer vectors also require more work per comparison.

Flat search produces exact nearest neighbors under the selected metric. “Exact” describes the numerical search result. It doesn’t guarantee that those neighbors contain the correct answer.

On the other hand, an Inverted-File index reduces this work by organizing vectors into groups. During construction, clustering identifies representative centers and assigns vectors to nearby groups. A query can then select promising groups and search their contents. The parameter commonly called nprobe controls how many groups are examined. However, searching more groups generally improves recall but also increases work.

Imagine one million passages divided into 1000 groups. Searching 10 groups might involve roughly 10K passages rather than the entire million. Of course, real groups are uneven, and selecting them also has a cost. The main tradeoff is that a useful passage can belong to a group that is skipped by the search. These groups are mathematical neighborhoods, not tidy subject folders. A travel-disruption rule might sit near insurance documents rather than ordinary expense policies.

IVF therefore introduces approximation. It saves work by accepting some risk of missing the nearest vectors.

Following Connections to the Right Neighborhood

HNSW (Hierarchical Navigable Small World) avoids exhaustive comparisons by building routes between vectors. Its structure is a graph. In other words, points are connected by links.

Each point represents a vector. The links provide routes through the collection. HNSW organizes these connections into layers, with sparse upper layers and a detailed bottom layer containing all vectors.

In this approach, search starts near the top and moves toward points closer to the query. It then descends through the layers, progressively refining the search. At the bottom, it explores a broader set of nearby candidates to select the results.

You can think of this navigation like travelling through a road network. Major routes help reach the right area, while local roads help locate a specific destination. HNSW uses this broad-to-detailed pattern to avoid visiting every point.

Like the Inverted-File index, it performs approximate nearest-neighbor search. However, selected routes can miss a true nearest neighbor. Its benefit is that useful results can often be found with substantially fewer comparisons.

There is no fixed collection size at which flat search must give way to Inverted-File or HNSW. Hardware, vector dimensions, query volume, memory, filtering, and latency requirements all influence the choice. HNSW is a strong candidate when memory permits, but we cannot say that it is automatically the best option for every workload.

A collection searched a few times per day creates a different problem from one serving thousands of simultaneous users. Vector count alone cannot capture that difference.

How Much Searching Is Enough?

Approximate search creates a measurable tradeoff between speed and recall.

Let’s say an exact search identifies the 10 nearest vectors. An approximate search returns eight of those same vectors and two others. This measures how closely the approximate search reproduces exact nearest-neighbor results. It doesn’t measure whether the returned passages really answer the employee’s question. Both index recall and actual evidence relevance need evaluation.

HNSW has several settings that have an impact on this tradeoff:

  • M controls graph connectivity. Higher values generally provide more routes, improving recall while increasing memory use and construction work.

  • ef_construction controls how broadly the algorithm searches for suitable connections during insertion. A larger value generally produces a better graph but takes longer to build.

  • ef_search controls the breadth of candidate exploration during a query. Increasing it generally improves recall while increasing search latency. It isn’t the number of results returned or a fixed count of visited points.

The application may need five passages while exploring a much larger candidate set to find them. Result count and search effort are separate decisions.

For tuning, we should use representative questions and actual performance measurements. If a broader search consistently finds a previously missed cancellation exception, the added latency may be worthwhile. However, if answers don’t improve, changing the setting merely adds work.

The Closest Match Might Be the Wrong Policy

Similarity must also be combined with rules about which documents are eligible.

For example, our employee needs the policy for their country and business unit, with an effective date that covers the journey. A highly similar passage from another region cannot help answer their query.

We use metadata filtering to define the eligible subset. The request becomes “find the most similar passages among current policies for this employee’s region.”

  • Pre-filtering identifies eligible records before similarity ranking.

  • Post-filtering retrieves similarity candidates first and then removes records that fail the conditions.

If only two of the first 20 candidates qualify, post-filtering cannot supply 5 eligible results from that batch. Retrieving additional candidates may help, but requires more work.

Nevertheless, pre-filtering is not automatically faster. The outcome depends on how selective the filter is and how the search engine combines filtering with its index. Graph search adds another complication. Disallowed points may still provide useful navigation routes toward allowed points. Blocking every such point during traversal can make eligible neighbors harder to reach.

Search engines can address this through filtering-aware graph structures, traversal strategies, or an exact scan when the eligible subset is small. Filtering can therefore be integrated into search rather than occurring entirely before or after it.

What Happens When the Answer Changes?

The collection must remain correct as its documents change. For example, let’s say the company raises its hotel reimbursement limit from ₹5,000 to ₹7,000. If both versions remain eligible for current-policy searches, the assistant may retrieve either figure or receive contradictory evidence.

Changes to embedded text require new embeddings. However, an update doesn’t always require rebuilding every chunk. Unchanged chunks may be reusable, and stable identifiers can support replacing existing records.

A metadata-only change can often be applied without re-embedding, provided that metadata was not a part of the text used to create the vector. Some vector databases can expose separate operations for changing vectors and metadata.

For our policy example, a sensible design can prepare the new version’s chunks, verify their availability, and then change which version is eligible for current searches. Older versions can remain accessible for historical questions.

That transition requires some sort of coordination. Deleting old chunks first can create a temporary gap. Inserting new chunks first can create temporary duplication. Version identifiers and explicit active-version rules help make the change predictable. Lastly, changing the embedding model requires similar planning but on a much larger scale. Existing documents generally need embeddings in the new model’s space, and queries must use the matching representation.

From Promising Matches to a Supported Answer

The final retrieval step turns promising matches into evidence the LLM can use. For example, a vector search might return 30 candidate passages. A reranker can compare each passage’s text with the question and select the most useful few.

Hybrid search adds another source of candidates by combining semantic retrieval with keyword-based retrieval. Embeddings help connect “cancelled flight” with “travel disruption,” while keyword search can preserve exact matches for policy identifiers, names, or unusual technical terms. Reranking can refine the combined results.

The application then supplies the selected text and source details to the LLM. For the stranded employee, this should include the current reimbursement rule, the relevant conditions, and enough context to explain how the policy applies to their situation.

The search must also accommodate the scenario of an unanswered question. Every collection has nearest vectors, even when none might contain useful information. Returning the closest passage doesn’t mean that an answer exists. For example, if the retrieved documents discuss ordinary hotel bookings but say nothing about cancellations, the assistant should explain clearly that the available policy details don’t have a clear answer.

Conclusion

Finding a useful passage among thousands of documents requires several parts of an LLM application to work together. Documents first become smaller, meaningful chunks. Embeddings represent those chunks as vectors, allowing the search system to connect a question with passages that express related ideas, even when their wording differs.

As the collection grows, indexes make that search more efficient. Flat search compares every eligible vector, while IVF and HNSW reduce the work through grouping or graph navigation. These approaches introduce tradeoffs between speed, memory, and recall that need to be measured against real questions.

Similarity alone, however, cannot establish whether a passage is suitable evidence. Metadata filters help select the correct region, document type, or policy version. Careful updates keep outdated and duplicate passages from appearing in current searches. Hybrid search and reranking can further improve the evidence selected for the LLM.

The final answer depends on the quality of this entire process. For the employee stranded at the airport, success means finding the current reimbursement rule, preserving its conditions, and explaining it clearly.

]]> \ No newline at end of file diff --git a/scripts/newsletter/__fixtures__/markup/bytebytego-sitemap.xml b/scripts/newsletter/__fixtures__/markup/bytebytego-sitemap.xml new file mode 100644 index 0000000..92a9b00 --- /dev/null +++ b/scripts/newsletter/__fixtures__/markup/bytebytego-sitemap.xml @@ -0,0 +1 @@ +https://blog.bytebytego.com/archivedailyhttps://blog.bytebytego.com/aboutweeklyhttps://blog.bytebytego.com/p/migrations-at-scale-changing-the2026-09-17monthlyhttps://blog.bytebytego.com/p/how-llms-can-find-a-needle-in-a-haystack2026-09-16monthlyhttps://blog.bytebytego.com/p/last-call-for-enrollment-build-with-d632026-09-15monthlyhttps://blog.bytebytego.com/p/do-llms-have-the-memory-of-a-goldfish2026-09-15monthlyhttps://blog.bytebytego.com/p/llms-as-a-judge-how-to-know-if-your2026-09-14monthlyhttps://blog.bytebytego.com/p/ep225-why-does-git-revert-cause-conflicts2026-09-12monthlyhttps://blog.bytebytego.com/p/learn-claude-code-evals-ai-systems2026-09-11monthlyhttps://blog.bytebytego.com/p/a-guide-to-application-networking2026-09-10monthlyhttps://blog.bytebytego.com/p/how-smart-model-routing-can-cut-llm2026-09-09monthlyhttps://blog.bytebytego.com/p/built-for-reliability-how-american2026-09-08monthlyhttps://blog.bytebytego.com/p/how-to-deal-with-errors-and-failures2026-09-07monthlyhttps://blog.bytebytego.com/p/ep224-mcp-vs-rag-vs-ai-agents2026-09-07monthlyhttps://blog.bytebytego.com/p/how-databases-keep-their-sanity-with2026-09-03monthlyhttps://blog.bytebytego.com/p/how-to-shrink-a-language-model-without2026-09-02monthlyhttps://blog.bytebytego.com/p/how-to-shrink-a-language-model-without-2952026-09-01monthlyhttps://blog.bytebytego.com/p/what-happens-inside-an-ai-chatbot2026-08-31monthlyhttps://blog.bytebytego.com/p/background-work-from-cron-jobs-to2026-08-27monthlyhttps://blog.bytebytego.com/p/how-to-make-llms-3x-faster2026-08-26monthlyhttps://blog.bytebytego.com/p/how-to-steal-an-ai-models-private2026-08-25monthlyhttps://blog.bytebytego.com/p/why-code-verification-matters-more2026-08-24monthlyhttps://blog.bytebytego.com/p/ep223-ollama-vs-vllm-vs-sglang2026-08-22monthlyhttps://blog.bytebytego.com/p/schema-evolution-changing-the-contract2026-08-20monthlyhttps://blog.bytebytego.com/p/graphrag-how-ai-answers-questions2026-08-19monthlyhttps://blog.bytebytego.com/p/the-new-american-ai-model-designed2026-08-18monthlyhttps://blog.bytebytego.com/p/waymo-vs-tesla-two-ways-to-build2026-08-25monthlyhttps://blog.bytebytego.com/p/ep222-what-is-googles-tpu2026-08-15monthlyhttps://blog.bytebytego.com/p/a-detailed-guide-to-api-composition2026-08-13monthlyhttps://blog.bytebytego.com/p/github-vs-vercel-vs-replit-what-dev2026-08-12monthlyhttps://blog.bytebytego.com/p/how-cloudflare-is-making-ai-pay-for2026-08-11monthlyhttps://blog.bytebytego.com/p/how-to-fight-clickbait-meta-linkedin2026-08-10monthlyhttps://blog.bytebytego.com/p/the-read-path-versus-the-write-path2026-08-06monthlyhttps://blog.bytebytego.com/p/how-big-models-teach-small-models2026-08-05monthlyhttps://blog.bytebytego.com/p/why-an-llms-memory-gets-expensive2026-08-04monthlyhttps://blog.bytebytego.com/p/llm-security-basics-the-full-threat2026-08-03monthlyhttps://blog.bytebytego.com/p/hiring-part-time-instructor-write2026-07-31monthlyhttps://blog.bytebytego.com/p/a-detailed-guide-to-idempotency-delivery2026-07-30monthlyhttps://blog.bytebytego.com/p/how-chatgpt-optimizes-its-agent-loop2026-07-29monthlyhttps://blog.bytebytego.com/p/why-doordash-instacart-and-uber-eats2026-07-28monthly \ No newline at end of file diff --git a/scripts/newsletter/__fixtures__/markup/edge-cases.html b/scripts/newsletter/__fixtures__/markup/edge-cases.html new file mode 100644 index 0000000..4f49f3b --- /dev/null +++ b/scripts/newsletter/__fixtures__/markup/edge-cases.html @@ -0,0 +1,20 @@ +
+ +
+
+ +
Inner caption & more
+
+
Outer caption
+
+
+ +
+
diff --git a/scripts/newsletter/__fixtures__/repo-empty/content/post/.gitkeep b/scripts/newsletter/__fixtures__/repo-empty/content/post/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/newsletter/__fixtures__/repo-year-fallback/content/post/2025/06/01/index.md b/scripts/newsletter/__fixtures__/repo-year-fallback/content/post/2025/06/01/index.md new file mode 100644 index 0000000..bd30782 --- /dev/null +++ b/scripts/newsletter/__fixtures__/repo-year-fallback/content/post/2025/06/01/index.md @@ -0,0 +1,7 @@ +--- +title: "Newsletter #50" +date: 2025-06-01 +tags: ["AI-Assisted"] +--- + +Case: the only newsletter in the tree, in an older year. Expect 51. diff --git a/scripts/newsletter/__fixtures__/repo-year-fallback/content/post/2026/01/02/index.md b/scripts/newsletter/__fixtures__/repo-year-fallback/content/post/2026/01/02/index.md new file mode 100644 index 0000000..eac7073 --- /dev/null +++ b/scripts/newsletter/__fixtures__/repo-year-fallback/content/post/2026/01/02/index.md @@ -0,0 +1,8 @@ +--- +title: "No newsletter here" +date: 2026-01-02 +tags: ["AI-Assisted"] +--- + +Case: the newest year contains no "Newsletter #N" heading, so the scan must +continue into the older year rather than stopping at the newest year. diff --git a/scripts/newsletter/__fixtures__/repo/content/post/2025/12/31/index.md b/scripts/newsletter/__fixtures__/repo/content/post/2025/12/31/index.md new file mode 100644 index 0000000..fb9af90 --- /dev/null +++ b/scripts/newsletter/__fixtures__/repo/content/post/2025/12/31/index.md @@ -0,0 +1,12 @@ +--- +title: "Newsletter #999" +date: 2025-12-31 +tags: ["AI-Assisted", "Older Year"] +categories: ["Newsletter"] +--- + +Case: an older year holding a HIGHER number than the newest year, so +find-newsletter-number proves it stops at the newest year that has one +instead of taking the maximum across the whole tree. + +## [Some article](https://older.invalid/story) diff --git a/scripts/newsletter/__fixtures__/repo/content/post/2026/09/10/index.md b/scripts/newsletter/__fixtures__/repo/content/post/2026/09/10/index.md new file mode 100644 index 0000000..5733ed3 --- /dev/null +++ b/scripts/newsletter/__fixtures__/repo/content/post/2026/09/10/index.md @@ -0,0 +1,31 @@ +--- +title: "Newsletter #132" +date: 2026-09-10 +tags: ["AI-Assisted", "Software Engineering", "Tie Break A"] +categories: ["Newsletter"] +--- + +Case: the newest newsletter. Also the post-stats "full" fixture — articles +outside Bonus, all three Bonus subsections populated. + +## [First article](https://link.invalid/story) + +Nội dung tóm tắt. + +## [Second article](https://bare.invalid/article) + +Nội dung tóm tắt. + +### Bonus + +**Images:** + +![MCP vs A2A](https://substackcdn.com/image/fetch/$s_!lpxK!,w_1100,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_2484x3002.png) + +**Videos:** + +[Some talk](https://www.youtube.com/watch?v=dQw4w9WgXcQ) + +**Documents:** + +[PDF: Some paper](https://docs.invalid/paper.pdf) diff --git a/scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/empty-bonus.md b/scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/empty-bonus.md new file mode 100644 index 0000000..161dbc5 --- /dev/null +++ b/scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/empty-bonus.md @@ -0,0 +1,16 @@ +--- +title: "Empty bonus" +date: 2026-09-11 +--- + +Case: post-stats with a Bonus section whose subsections carry no entries. + +## [Only article](https://only.invalid/a) + +### Bonus + +**Images:** + +**Videos:** + +**Documents:** diff --git a/scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/index.md b/scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/index.md new file mode 100644 index 0000000..84d9df9 --- /dev/null +++ b/scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/index.md @@ -0,0 +1,30 @@ +--- +title: "Stored URL shapes" +date: 2026-09-11 +tags: ["AI-Assisted", "Tie Break B"] +categories: ["Newsletter"] +--- + +Case: every shape a URL is stored in, so the boundary-aware dedup check is +exercised against each. No "Newsletter #" heading — find-newsletter-number +must still resolve 133 from the sibling post. + +A bare URL on its own line: + +https://bare2.invalid/article + +A URL that keeps its trailing slash: + +https://slash.invalid/post/ + +A markdown autolink: + + + +A long stored URL that a shorter needle must NOT falsely match: + +https://prefix.invalid/posts/the-long-slug + +The same Substack image identity in raw S3 form: + +![Raw S3](https://substack-post-media.s3.amazonaws.com/public/images/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_1280x720.png) diff --git a/scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/no-bonus.md b/scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/no-bonus.md new file mode 100644 index 0000000..755da4c --- /dev/null +++ b/scripts/newsletter/__fixtures__/repo/content/post/2026/09/11/no-bonus.md @@ -0,0 +1,11 @@ +--- +title: "No bonus" +date: 2026-09-11 +--- + +Case: post-stats on a post with no Bonus section at all, and no +"Newsletter #N" heading, so the newsletter field stays 0. + +## [Article one](https://a.invalid/one) + +## [Article two](https://a.invalid/two) diff --git a/scripts/newsletter/network.test.js b/scripts/newsletter/network.test.js new file mode 100644 index 0000000..e6fc791 --- /dev/null +++ b/scripts/newsletter/network.test.js @@ -0,0 +1,102 @@ +// Live-upstream tier. Opt-in via NEWSLETTER_NET=1: upstream content moves, and a +// suite that goes red on someone else's outage teaches people to ignore red. +// +// These assert structure and invariants rather than captured bytes, because the +// bytes legitimately change when a publication posts. + +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { join } from "node:path"; +import { test } from "node:test"; + +const HERE = import.meta.dirname; +const ENGINE = join(HERE, "index.js"); +const FIXTURE_REPO = join(HERE, "__fixtures__", "repo"); + +const skip = process.env.NEWSLETTER_NET !== "1"; +const OPTS = { skip: skip ? "set NEWSLETTER_NET=1 to run live-upstream cases" : false }; + +/** + * @param {string[]} args + * @returns {{stdout: string, code: number}} + */ +function runEngine(args) { + try { + return { + stdout: execFileSync("node", [ENGINE, ...args], { cwd: FIXTURE_REPO, encoding: "utf8" }), + code: 0, + }; + } catch (err) { + return { stdout: err.stdout ?? "", code: err.status ?? 1 }; + } +} + +const WATCH = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"; +const CANONICAL = WATCH; + +test("add-url: a watch URL routes to youtube and resolves oEmbed metadata", OPTS, () => { + const out = JSON.parse(runEngine(["add-url", WATCH + "&utm_source=nl"]).stdout); + assert.equal(out.route, "youtube"); + assert.equal(out.clean_url, CANONICAL); + assert.ok(out.title.length > 0, "oEmbed returned no title"); + assert.ok(out.author.length > 0, "oEmbed returned no author"); +}); + +test("add-url: youtu.be and shorts collapse onto the canonical watch identity", OPTS, () => { + const short = JSON.parse(runEngine(["add-url", "https://youtu.be/dQw4w9WgXcQ"]).stdout); + assert.equal(short.route, "youtube"); + assert.equal(short.clean_url, CANONICAL); + const shorts = JSON.parse(runEngine(["add-url", "https://www.youtube.com/shorts/dQw4w9WgXcQ"]).stdout); + assert.equal(shorts.route, "youtube"); + assert.equal(shorts.clean_url, CANONICAL); +}); + +test("add-url: a playlist is not a youtube route", OPTS, () => { + const out = JSON.parse(runEngine(["add-url", "https://www.youtube.com/playlist?list=PLrAXtmRdnEQy6nuLMfO6uZ1a4Z3AQoMek"]).stdout); + assert.equal(out.route, "article"); + assert.equal(out.title, undefined); +}); + +test("find-substack-post: a uuid no publication carries reports a bare miss", OPTS, () => { + const out = JSON.parse(runEngine(["find-substack-post", "--uuid", "ffffffff-ffff-ffff-ffff-ffffffffffff"]).stdout); + assert.deepEqual(out, { found: false }); +}); + +test("find-substack-post: a deep miss reports the crawl budget and cutoff", OPTS, () => { + const out = JSON.parse(runEngine(["find-substack-post", "--uuid", "ffffffff-ffff-ffff-ffff-ffffffffffff", "--deep"]).stdout); + assert.equal(out.found, false); + assert.equal(out.source, "sitemap"); + assert.equal(out.budget, 40); + assert.ok(Number.isInteger(out.scanned)); + // cutoff is null (present, not omitted) when no sitemap could be fetched. + assert.ok("cutoff" in out, "cutoff key is missing"); + assert.ok(out.cutoff === null || /^\d{4}-\d{2}-\d{2}$/.test(out.cutoff)); +}); + +test("find-substack-post: a uuid from the live feed resolves to its post", OPTS, async () => { + const res = await fetch("https://blog.bytebytego.com/feed", { headers: { "user-agent": "Mozilla/5.0" } }); + const feed = await res.text(); + const m = /public\/images\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/.exec(feed); + assert.ok(m !== null, "the live feed carried no image uuid"); + const out = JSON.parse(runEngine(["find-substack-post", "--uuid", m[1]]).stdout); + assert.equal(out.found, true); + assert.equal(out.source, "rss"); + assert.ok(out.postUrl.startsWith("https://")); + assert.ok(out.postTitle.length > 0); + assert.ok(Array.isArray(out.candidates)); +}); + +test("fetch-via-defuddle: a reachable page returns frontmatter and a body", OPTS, () => { + const { stdout, code } = runEngine(["fetch-via-defuddle", "https://example.com/"]); + assert.equal(code, 0); + // Both tiers emit the same shape, so the caller parses one format. + assert.ok(stdout.startsWith("---\n"), "no YAML frontmatter"); + assert.match(stdout, /^title: /m); + const body = stdout.split("\n---\n")[1] ?? ""; + assert.ok(body.trim().length > 0, "frontmatter with no body"); +}); + +test("fetch-via-defuddle: an unresolvable host exhausts both tiers and exits 1", OPTS, () => { + const { code } = runEngine(["fetch-via-defuddle", "https://nonexistent-host-xyz-12345.invalid/a"]); + assert.equal(code, 1); +}); diff --git a/scripts/newsletter/parity.test.js b/scripts/newsletter/parity.test.js new file mode 100644 index 0000000..c5cf8a9 --- /dev/null +++ b/scripts/newsletter/parity.test.js @@ -0,0 +1,304 @@ +// Parity suite: every golden file under __fixtures__/golden/ was captured from +// the Go engine before the JavaScript port was trusted, so a mismatch here means +// the port drifted — not that the expectation is stale. +// +// Deterministic tier only: every case either touches no network or uses a +// non-resolving .invalid host, so `npm test` passes offline. Live-upstream cases +// live in network.test.js. + +import assert from "node:assert/strict"; +import { execFileSync, execSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { detectYouTube } from "./add-url.js"; +import { extractInnerUrl } from "./detect-image-source.js"; +import { looksLikeChallenge } from "./fetch-via-defuddle.js"; +import { findMostRecentNewsletter } from "./find-newsletter-number.js"; +import { loadPublications, parseLastmod } from "./find-substack-post.js"; +import { countPostEntries } from "./post-stats.js"; +import { + captionForUuid, + extractCandidates, + itemLink, + itemTitle, + postTitleFromHtml, +} from "./html-text.js"; + +const HERE = import.meta.dirname; +const ENGINE = join(HERE, "index.js"); +const FIXTURES = join(HERE, "__fixtures__"); +const GOLDEN = join(FIXTURES, "golden"); +const MARKUP = join(FIXTURES, "markup"); + +const MAIN = join(FIXTURES, "repo"); +const EMPTY = join(FIXTURES, "repo-empty"); +const FALLBACK = join(FIXTURES, "repo-year-fallback"); + +const EXIT_CODES = JSON.parse(readFileSync(join(GOLDEN, "_exit-codes.json"), "utf8")); + +// Real Substack hosts: detect-image-source never touches the network. +const CDN = + "https://substackcdn.com/image/fetch/$s_!lpxK!,w_1100,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_2484x3002.png"; +const S3 = + "https://substack-post-media.s3.amazonaws.com/public/images/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_1280x720.png"; +// add-url HEADs the URL, so its image cases use non-resolving hosts that still +// carry the substack-post-media marker — same routing, stable "000" status. +const CDN_OFFLINE = + "https://cdn.invalid/image/fetch/w_1100,c_limit,f_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_2484x3002.png"; +const S3_OFFLINE = + "https://substack-post-media.s3.invalid/public/images/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_1280x720.png"; +const UNKNOWN_IMG = + "https://cdn.invalid/image/fetch/f_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12345678-1234-1234-1234-123456789abc_100x100.avif?x=1"; + +/** + * Cases are [id, engine args, working directory]. The working directory is how + * content-scanning commands are pointed at a fixture tree: both engines resolve + * content/post from the process CWD, so no test-only config knob exists. + * @type {[string, string[], string][]} + */ +const CLI_CASES = [ + // add-url — tracking-parameter stripping keeps surviving pairs verbatim + ["add-url-tracking-params", ["add-url", "https://a.invalid/article?utm_source=x&utm_medium=y&fbclid=abc&keep=1&ref=z&s=1"], MAIN], + // %7E must not become ~ and + must not become %20 + ["add-url-encoding-preserved", ["add-url", "https://A.invalid/path/?a=%7Efoo+bar&b=2#frag"], MAIN], + // %20 must not become +, a valueless key must not gain =, and %7e must keep + // its lowercase hex — all three are lost by rebuilding the query + ["add-url-query-verbatim", ["add-url", "https://a.invalid/path?a=%20x&flag&b=%7ez&utm_source=drop"], MAIN], + // every shape a URL is stored in must be found by the dedup scan + ["add-url-dup-bare", ["add-url", "https://bare2.invalid/article"], MAIN], + ["add-url-dup-trailing-slash", ["add-url", "https://slash.invalid/post/"], MAIN], + ["add-url-dup-autolink", ["add-url", "https://autolink.invalid/entry"], MAIN], + ["add-url-dup-markdown-link", ["add-url", "https://link.invalid/story"], MAIN], + // a needle that is only a PREFIX of a stored URL is not a duplicate + ["add-url-prefix-not-duplicate", ["add-url", "https://prefix.invalid/posts/the-long"], MAIN], + ["add-url-prefix-exact-duplicate", ["add-url", "https://prefix.invalid/posts/the-long-slug"], MAIN], + // transform variants of one Substack image share the uuid identity + ["add-url-image-cdn-duplicate", ["add-url", CDN_OFFLINE], MAIN], + ["add-url-image-s3-duplicate", ["add-url", S3_OFFLINE], MAIN], + ["add-url-image-avif-query", ["add-url", UNKNOWN_IMG], MAIN], + ["add-url-document-pdf", ["add-url", "https://a.invalid/file.pdf"], MAIN], + ["add-url-video-mp4", ["add-url", "https://a.invalid/clip.mp4"], MAIN], + // no title/author keys at all when there is no metadata to report + ["add-url-article-no-metadata", ["add-url", "https://a.invalid/plain"], MAIN], + // detect-image-source + ["detect-image-cdn-wrapper", ["detect-image-source", CDN], MAIN], + ["detect-image-raw-s3", ["detect-image-source", S3], MAIN], + ["detect-image-non-substack", ["detect-image-source", "https://a.invalid/pic.png"], MAIN], + // a malformed percent sequence falls back to the raw substring + ["detect-image-malformed-percent", ["detect-image-source", "https://substackcdn.com/image/fetch/w_100/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F%ZZbad.png"], MAIN], + // the newest year wins even though an older year holds a higher number + ["find-newsletter-number-main", ["find-newsletter-number"], MAIN], + ["find-newsletter-number-empty", ["find-newsletter-number"], EMPTY], + ["find-newsletter-number-year-fallback", ["find-newsletter-number"], FALLBACK], + // plain text, count right-aligned in six columns, ties in first-seen order + ["list-existing-tags-main", ["list-existing-tags"], MAIN], + ["post-stats-full", ["post-stats", "content/post/2026/09/10/index.md"], MAIN], + ["post-stats-empty-bonus", ["post-stats", "content/post/2026/09/11/empty-bonus.md"], MAIN], + ["post-stats-no-bonus", ["post-stats", "content/post/2026/09/11/no-bonus.md"], MAIN], +]; + +/** + * runEngine invokes the real CLI surface, so exit codes are exercised too. + * @param {string[]} args + * @param {string} cwd + * @returns {{stdout: string, code: number}} + */ +function runEngine(args, cwd) { + try { + return { stdout: execFileSync("node", [ENGINE, ...args], { cwd, encoding: "utf8" }), code: 0 }; + } catch (err) { + return { stdout: err.stdout ?? "", code: err.status ?? 1 }; + } +} + +/** + * @param {string} id + * @returns {string} + */ +function golden(id) { + const ext = id.startsWith("list-existing-tags") || id.startsWith("find-newsletter-number") ? "txt" : "json"; + return readFileSync(join(GOLDEN, `${id}.${ext}`), "utf8"); +} + +for (const [id, args, cwd] of CLI_CASES) { + test(`cli parity: ${id}`, () => { + const { stdout, code } = runEngine(args, cwd); + // Byte-for-byte: key order is part of the contract the skills parse. + assert.equal(stdout, golden(id), `stdout differs from the Go capture for ${id}`); + assert.equal(code, EXIT_CODES[id]); + }); +} + +// --- extraction helpers, compared against saved upstream markup ------------- +// +// These assert parsed objects rather than bytes: the Go capture harness emitted +// a map, whose key order is alphabetical rather than a contract. + +test("extraction parity: real RSS item", () => { + const feed = readFileSync(join(MARKUP, "bytebytego-feed.xml"), "utf8"); + const uuid = "02a5951e-febf-4289-8905-c67f45e754a1"; + const item = feed.split("").find((chunk) => chunk.includes(uuid)); + assert.ok(item !== undefined, "the feed fixture no longer contains the uuid under test"); + const expected = JSON.parse(golden("extract-rss-item")); + assert.equal(itemTitle(item), expected.itemTitle); + assert.equal(itemLink(item), expected.itemLink); + assert.equal(captionForUuid(item, uuid), expected.caption); + // The parser recovers one TOC bullet the Go regex dropped; every bullet the + // regex found is still present, in the same order. + const candidates = extractCandidates(item); + for (const c of expected.candidates) assert.ok(candidates.includes(c), `lost candidate: ${c}`); +}); + +test("extraction parity: nested figure, rune limits, dedupe", () => { + const html = readFileSync(join(MARKUP, "edge-cases.html"), "utf8"); + const expected = JSON.parse(golden("extract-edge-html")); + // The innermost enclosing figure supplies the caption, not the outer one. + assert.equal(captionForUuid(html, "11111111-2222-3333-4444-555555555555"), expected.caption); + // A figure without a figcaption yields nothing rather than borrowing one. + assert.equal(captionForUuid(html, "99999999-8888-7777-6666-555555555555"), ""); + assert.deepEqual(extractCandidates(html), expected.candidates); + assert.equal(postTitleFromHtml(html), expected.postTitle); +}); + +test("bullet length is measured in code points, not UTF-16 units", () => { + const html = readFileSync(join(MARKUP, "edge-cases.html"), "utf8"); + const candidates = extractCandidates(html); + const kept = candidates.find((c) => c.startsWith("\u{1F680}")); + // 70 code points but 71 UTF-16 units: counting units would drop it. + assert.ok(kept !== undefined, "the 70-code-point bullet was dropped"); + assert.equal([...kept].length, 70); + assert.ok(kept.length > 70); + // 71 code points is over the limit in both engines. + assert.ok(!candidates.some((c) => c.includes("BBB")), "the 71-code-point bullet was kept"); + // Under six code points is dropped; exactly six is kept. + assert.ok(!candidates.includes("fiver")); + assert.ok(candidates.includes("sixchr")); +}); + +test("postTitleFromHtml precedence: og:title, then h1, then title", () => { + assert.equal( + postTitleFromHtml('Doc

H

'), + "OG & T", + ); + assert.equal(postTitleFromHtml("

Head One

"), "Head One"); + assert.equal(postTitleFromHtml("Only & Title"), "Only & Title"); + assert.equal(postTitleFromHtml("

nothing

"), ""); +}); + +test("fetch-via-defuddle rejects bad arguments with exit 2", () => { + const { code } = runEngine(["fetch-via-defuddle"], MAIN); + // The previous engine defined 2 as well, but its runner collapsed every + // nonzero exit to 1, so the distinction was never observable. + assert.equal(code, 2); +}); + +test("a reader that closes early does not produce a stack trace", () => { + // `node scripts/newsletter list-existing-tags | head -1` must exit quietly. + const stderrPath = join(HERE, "__fixtures__", "repo"); + const out = execSync(`node ${JSON.stringify(ENGINE)} list-existing-tags | head -1`, { + cwd: stderrPath, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + assert.equal(out, " 3 AI-Assisted\n"); +}); + +test("an unknown subcommand exits 1 and prints usage", () => { + const { code } = runEngine(["no-such-command"], MAIN); + assert.equal(code, 1); +}); + +// --- unit coverage for the pure exports ------------------------------------ +// +// Importing a command module must not run the CLI, which is why printJson lives +// in its own file. These tests are the standing proof of that. + +test("detectYouTube recognises the three supported shapes and nothing else", () => { + assert.deepEqual(detectYouTube("https://www.youtube.com/watch?v=abc123"), { isYouTube: true, videoId: "abc123" }); + assert.deepEqual(detectYouTube("https://youtu.be/abc123"), { isYouTube: true, videoId: "abc123" }); + assert.deepEqual(detectYouTube("https://www.youtube.com/shorts/abc123"), { isYouTube: true, videoId: "abc123" }); + // A playlist carries no single video identity, so it falls through to type. + assert.equal(detectYouTube("https://www.youtube.com/playlist?list=PL1").isYouTube, false); + assert.equal(detectYouTube("https://www.youtube.com/watch").isYouTube, false); + assert.equal(detectYouTube("https://youtu.be/").isYouTube, false); + assert.equal(detectYouTube("not a url").isYouTube, false); +}); + +test("extractInnerUrl unwraps the CDN form and survives a bad escape", () => { + assert.equal( + extractInnerUrl("https://substackcdn.com/image/fetch/w_1/https%3A%2F%2Fs3.example.com%2Fa.png"), + "https://s3.example.com/a.png", + ); + // A malformed percent sequence returns the raw substring instead of throwing. + assert.equal( + extractInnerUrl("https://substackcdn.com/image/fetch/w_1/https%3A%2F%2Fs3.example.com%2F%ZZ.png"), + "https%3A%2F%2Fs3.example.com%2F%ZZ.png", + ); + // An already-decoded inner URL is found too. + assert.equal(extractInnerUrl("https://cdn.example.com/x/https://s3.example.com/a.png"), "https://s3.example.com/a.png"); + // No wrapper at all: the input comes back untouched. + assert.equal(extractInnerUrl("https://s3.example.com/a.png"), "https://s3.example.com/a.png"); +}); + +test("countPostEntries attributes entries to the open Bonus subsection", () => { + const post = [ + "## [An article](https://a.invalid/1)", + "### Bonus", + "**Images:**", + "![label](https://a.invalid/i.png)", + "**Videos:**", + "[A talk](https://a.invalid/v)", + "**Documents:**", + "[PDF: paper](https://a.invalid/p.pdf)", + ].join("\n"); + assert.deepEqual(countPostEntries(post), { + articles: 1, images: 1, videos: 1, documents: 1, total: 4, + }); + // An image entry outside any subsection belongs to nothing. + assert.equal(countPostEntries("### Bonus\n![x](y)").images, 0); + // Article headings are counted outside Bonus only when they are links. + assert.equal(countPostEntries("## Plain heading").articles, 0); +}); + +test("parseLastmod accepts only the two layouts the sitemap crawl trusts", () => { + assert.ok(parseLastmod("2026-09-18") instanceof Date); + assert.ok(parseLastmod("2026-09-18T10:00:00Z") instanceof Date); + assert.ok(parseLastmod("2026-09-18T10:00:00+07:00") instanceof Date); + // No timezone: reading it as local time would shift the cutoff comparison. + assert.equal(parseLastmod("2026-09-18T10:00:00"), null); + assert.equal(parseLastmod("18/09/2026"), null); + assert.equal(parseLastmod(""), null); +}); + +test("loadPublications returns a non-empty list", () => { + const pubs = loadPublications(); + assert.ok(Array.isArray(pubs)); + assert.ok(pubs.length > 0); + assert.ok(pubs.every((p) => typeof p === "string" && p.length > 0)); +}); + +test("findMostRecentNewsletter reads the tree under the process working directory", () => { + const cwd = process.cwd(); + try { + process.chdir(MAIN); + // The newest year wins even though an older year holds #999. + assert.equal(findMostRecentNewsletter(), 132); + } finally { + process.chdir(cwd); + } +}); + +test("a bot wall is not mistaken for the page that was asked for", () => { + // A 200 challenge page extracts to a non-empty body; treating it as success + // would spend the local tier's turn and skip the proxy, which is the tier that + // fetches from a different IP. + assert.equal(looksLikeChallenge("Just a moment...", "Enable JavaScript and cookies to continue"), true); + assert.equal(looksLikeChallenge("Attention Required! | Cloudflare", "Please unblock challenges.example"), true); + assert.equal(looksLikeChallenge("", "Verify you are human by completing the action below."), true); + // An article that merely discusses Cloudflare is not a challenge. + const article = "Introduction\n".padEnd(500, "x") + " we moved our edge to Cloudflare last quarter"; + assert.equal(looksLikeChallenge("How we cut latency in half", article), false); + assert.equal(looksLikeChallenge("Example Domain", "This domain is for use in documentation examples."), false); +});