diff --git a/.claude/skills/mt-add-image/SKILL.md b/.claude/skills/mt-add-image/SKILL.md index 95e2484..b5e19c8 100644 --- a/.claude/skills/mt-add-image/SKILL.md +++ b/.claude/skills/mt-add-image/SKILL.md @@ -21,7 +21,7 @@ A clean image URL (passed by `mt-add-url`, or given directly). ```bash node .claude/skills/mt-add-image/scripts/detect-image-source.js "" ``` -→ `{ isSubstack, uuid?, innerUrl? }`. +→ `{ original_url, clean_url, isSubstack, uuid?, innerUrl? }`. When invoked **directly** (not via `mt-add-url`), first run the router to get accessibility + duplicate status and skip accordingly: ```bash @@ -34,11 +34,11 @@ Find the source post: ```bash node .claude/skills/mt-add-image/scripts/find-substack-post.js --uuid ``` -- `found: false` → retry with the deeper sitemap crawl (slower — scans ~3 months; warn the user it may take ~15s): +- `found: false` → retry with the deeper sitemap crawl (slower — fetches posts ~3 months back, capped at 40 fetches total across all publications; warn the user it may take a while): ```bash node .claude/skills/mt-add-image/scripts/find-substack-post.js --uuid --deep ``` - The result reports `scanned` + `cutoff` — mention how far back it looked. + On a miss the result reports `scanned` (posts fetched), `budget` (the 40-fetch cap), and `cutoff` (oldest date looked at) — mention how far back it looked. - `found: false` after `--deep` → no source post; go to step 3 (ask) and/or step 4 (add publication). **When `found: true` — pick the label (confirm-from-candidates):** diff --git a/.claude/skills/mt-add-image/scripts/find-substack-post.js b/.claude/skills/mt-add-image/scripts/find-substack-post.js index 90beb69..2ffd759 100644 --- a/.claude/skills/mt-add-image/scripts/find-substack-post.js +++ b/.claude/skills/mt-add-image/scripts/find-substack-post.js @@ -1,15 +1,24 @@ #!/usr/bin/env node // Find which Substack post embeds a given image UUID, and extract a label. // Usage: node find-substack-post.js --uuid [--deep] -// Output: JSON { found, source?, publication?, postTitle?, postUrl?, caption?, scanned? } +// Output on hit: JSON { found:true, source, publication, postTitle, postUrl, caption, candidates } +// Output on miss: JSON { found:false } (RSS) or { found:false, source:"sitemap", scanned, budget, cutoff } (--deep) // // Strategy: RSS feed first (fast, ~recent weeks). With --deep, fall back to a -// sitemap crawl up to ~3 months back (added in the deep-lookup phase). -// A Substack CDN URL does not encode its publication, so we search each +// heavier sitemap crawl up to ~3 months back — opt-in because it fetches many +// posts. A Substack CDN URL does not encode its publication, so we search each // publication listed in config/substack-publications.json. const fs = require("fs"); const path = require("path"); +const { fetchWithTimeout } = require("../../mt-add-url/scripts/url-utils.js"); +const { + itemTitle, + itemLink, + extractCandidates, + captionForUuid, + postTitleFromHtml, +} = require("./html-text-utils.js"); const args = process.argv.slice(2); function flag(name) { @@ -33,96 +42,10 @@ function loadPublications() { } } +// Thin wrapper over the shared fetcher: returns body text, or "" on any error. async function fetchText(url) { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 10000); - try { - const res = await fetch(url, { - redirect: "follow", - signal: controller.signal, - headers: { "User-Agent": "Mozilla/5.0" }, - }); - return res.ok ? await res.text() : ""; - } catch { - return ""; - } finally { - clearTimeout(timeout); - } -} - -function decodeEntities(s) { - return s - .replace(/&/g, "&") - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, '"') - .replace(/�?39;|'/g, "'") - .replace(/ /g, " ") - .replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCodePoint(parseInt(n, 16))) - .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n)) - .trim(); -} - -function stripTags(html) { - return decodeEntities(html.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim(); -} - -// Pull the first (CDATA or plain) and <link> from an <item> chunk. -function itemTitle(item) { - const m = item.match(/<title>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/title>/); - return m ? decodeEntities(m[1].trim()) : ""; -} -function itemLink(item) { - const m = item.match(/<link>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/link>/); - return m ? m[1].trim() : ""; -} - -// Extract candidate topic titles from a post's TOC bullet list. ByteByteGo does -// not attach captions to images — the topic titles live only in the "in this -// issue" bullets. We can't reliably map image→title automatically (sponsor / -// video items interleave and break positional order), so we surface these -// candidates for the user to pick from. Light filtering keeps the list short: -// dedupe, drop sub-point explanations ("Term: long sentence") and over-long -// lines. The correct title is always present; the user selects it. -function extractCandidates(html) { - const seen = new Set(); - const out = []; - const re = /<li[^>]*>\s*(?:<p[^>]*>)?([\s\S]*?)(?:<\/p>)?\s*<\/li>/g; - let m; - while ((m = re.exec(html))) { - const text = stripTags(m[1]); - if (!text) continue; - if (text.length < 6 || text.length > 70) continue; // titles are short; long lines are sub-point explanations - const key = text.toLowerCase(); - if (seen.has(key)) continue; // content is duplicated in the page - seen.add(key); - out.push(text); - } - return out; -} - -// If the UUID sits inside a <figure>…</figure>, return that figure's -// <figcaption> text. Cover images live in <enclosure> (no figure) → "". -function captionForUuid(item, id) { - const at = item.indexOf(id); - if (at === -1) return ""; - const figStart = item.lastIndexOf("<figure", at); - if (figStart === -1) return ""; - const figEnd = item.indexOf("</figure>", at); - if (figEnd === -1) return ""; - const figure = item.slice(figStart, figEnd); - const cap = figure.match(/<figcaption[^>]*>([\s\S]*?)<\/figcaption>/); - return cap ? stripTags(cap[1]) : ""; -} - -// Extract a post title from server-rendered post HTML (og:title preferred). -function postTitleFromHtml(html) { - const og = html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i); - if (og) return decodeEntities(og[1]); - const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i); - if (h1) return stripTags(h1[1]); - const t = html.match(/<title>([\s\S]*?)<\/title>/i); - return t ? decodeEntities(t[1].trim()) : ""; + const res = await fetchWithTimeout(url, { headers: { "User-Agent": "Mozilla/5.0" } }); + return res && res.ok ? await res.text() : ""; } // Total post fetches allowed across ALL publications during a --deep crawl. diff --git a/.claude/skills/mt-add-image/scripts/html-text-utils.js b/.claude/skills/mt-add-image/scripts/html-text-utils.js new file mode 100644 index 0000000..5b099a4 --- /dev/null +++ b/.claude/skills/mt-add-image/scripts/html-text-utils.js @@ -0,0 +1,91 @@ +// HTML / RSS text-extraction helpers for find-substack-post.js. +// Pure string functions (no network, no fs) — kept separate so the crawler +// stays focused on fetch/search flow. CommonJS so plain `node` works. + +// Decode the HTML entities that appear in Substack titles/captions, including +// numeric (') and hex (—) forms. & is decoded last-ish but +// before nothing re-encodes it; ordering here is safe for our inputs. +function decodeEntities(s) { + return s + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/�?39;|'/g, "'") + .replace(/ /g, " ") + .replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCodePoint(parseInt(n, 16))) + .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n)) + .trim(); +} + +function stripTags(html) { + return decodeEntities(html.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim(); +} + +// Pull the first <title> (CDATA or plain) and <link> from an RSS <item> chunk. +function itemTitle(item) { + const m = item.match(/<title>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/title>/); + return m ? decodeEntities(m[1].trim()) : ""; +} +function itemLink(item) { + const m = item.match(/<link>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/link>/); + return m ? m[1].trim() : ""; +} + +// Extract candidate topic titles from a post's TOC bullet list. ByteByteGo does +// not attach captions to images — the topic titles live only in the "in this +// issue" bullets. We can't reliably map image→title automatically (sponsor / +// video items interleave and break positional order), so we surface these +// candidates for the user to pick from. Light filtering keeps the list short: +// dedupe, drop sub-point explanations ("Term: long sentence") and over-long +// lines. The correct title is always present; the user selects it. +function extractCandidates(html) { + const seen = new Set(); + const out = []; + const re = /<li[^>]*>\s*(?:<p[^>]*>)?([\s\S]*?)(?:<\/p>)?\s*<\/li>/g; + let m; + while ((m = re.exec(html))) { + const text = stripTags(m[1]); + if (!text) continue; + if (text.length < 6 || text.length > 70) continue; // titles are short; long lines are sub-point explanations + const key = text.toLowerCase(); + if (seen.has(key)) continue; // content is duplicated in the page + seen.add(key); + out.push(text); + } + return out; +} + +// If the UUID sits inside a <figure>…</figure>, return that figure's +// <figcaption> text. Cover images live in <enclosure> (no figure) → "". +function captionForUuid(item, id) { + const at = item.indexOf(id); + if (at === -1) return ""; + const figStart = item.lastIndexOf("<figure", at); + if (figStart === -1) return ""; + const figEnd = item.indexOf("</figure>", at); + if (figEnd === -1) return ""; + const figure = item.slice(figStart, figEnd); + const cap = figure.match(/<figcaption[^>]*>([\s\S]*?)<\/figcaption>/); + return cap ? stripTags(cap[1]) : ""; +} + +// Extract a post title from server-rendered post HTML (og:title preferred). +function postTitleFromHtml(html) { + const og = html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i); + if (og) return decodeEntities(og[1]); + const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i); + if (h1) return stripTags(h1[1]); + const t = html.match(/<title>([\s\S]*?)<\/title>/i); + return t ? decodeEntities(t[1].trim()) : ""; +} + +module.exports = { + decodeEntities, + stripTags, + itemTitle, + itemLink, + extractCandidates, + captionForUuid, + postTitleFromHtml, +}; diff --git a/.claude/skills/mt-add-url/SKILL.md b/.claude/skills/mt-add-url/SKILL.md index 80dd8a2..32a69e8 100644 --- a/.claude/skills/mt-add-url/SKILL.md +++ b/.claude/skills/mt-add-url/SKILL.md @@ -5,7 +5,7 @@ description: 'Meta entry for adding URLs to the Hugo blog newsletter. Use whenev ## Overview -`mt-add-url` is the **router**: it classifies each URL and auto-invokes the matching handler skill. Handlers (`mt-add-post`, `mt-add-video`) own the actual content writing. Shared scripts live in `.claude/skills/mt-add-url/scripts/`; shared post mechanics in `references/newsletter-post-mechanics.md`. +`mt-add-url` is the **meta dispatcher**: it classifies each URL and auto-invokes the matching handler skill. Handlers (`mt-add-post`, `mt-add-video`, `mt-add-image`) own the actual content writing. Shared scripts live in `.claude/skills/mt-add-url/scripts/`; shared post mechanics in `references/newsletter-post-mechanics.md`. **Supported routes (this version):** - `article` → `mt-add-post` diff --git a/.claude/skills/mt-add-url/scripts/add-url.js b/.claude/skills/mt-add-url/scripts/add-url.js index fe61fcc..9c99ef5 100644 --- a/.claude/skills/mt-add-url/scripts/add-url.js +++ b/.claude/skills/mt-add-url/scripts/add-url.js @@ -9,6 +9,7 @@ const path = require("path"); const { cleanUrl, isSubstackImage, + fetchWithTimeout, checkAccessibility, checkDuplicate, classifyType, @@ -21,7 +22,7 @@ if (!url) { } const PROJECT_ROOT = path.resolve(__dirname, "../../../.."); -const CONTENT_DIR = path.join(PROJECT_ROOT, "content"); +const CONTENT_DIR = path.join(PROJECT_ROOT, "content", "post"); const YT_HOSTS = new Set(["youtube.com", "www.youtube.com", "m.youtube.com"]); @@ -60,13 +61,10 @@ function canonicalWatchUrl(videoId) { // Fetch title/author via YouTube oEmbed (no API key). Best-effort: any failure // returns {} so the route stays `youtube` and the skill can fall back. async function fetchYouTubeMeta(watchUrl) { + const endpoint = `https://www.youtube.com/oembed?url=${encodeURIComponent(watchUrl)}&format=json`; + const res = await fetchWithTimeout(endpoint); + if (!res || !res.ok) return {}; try { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 10000); - const endpoint = `https://www.youtube.com/oembed?url=${encodeURIComponent(watchUrl)}&format=json`; - const res = await fetch(endpoint, { redirect: "follow", signal: controller.signal }); - clearTimeout(timeout); - if (!res.ok) return {}; const data = await res.json(); return { title: data.title, author: data.author_name }; } catch { diff --git a/.claude/skills/mt-add-url/scripts/url-utils.js b/.claude/skills/mt-add-url/scripts/url-utils.js index cee731c..f1adbcc 100644 --- a/.claude/skills/mt-add-url/scripts/url-utils.js +++ b/.claude/skills/mt-add-url/scripts/url-utils.js @@ -21,10 +21,10 @@ function cleanUrl(rawUrl) { }); return parsed.toString(); } catch { - // If URL parsing fails, do basic string cleanup - return rawUrl.replace(/[?&](utm_[^&]*|fbclid|gclid|msclkid|mc_eid|aid|ref|ref_src|ref_url|source|ck_subscriber_id|igshid|yclid|vero_id)=[^&]*/gi, "") - .replace(/\?&/, "?") - .replace(/[?&]$/, ""); + // Unparseable input (not a real URL): the per-param string surgery above + // would mangle the query (drop the `?`, leave a dangling `&`), so leave it + // untouched rather than corrupt it. + return rawUrl; } } @@ -77,21 +77,26 @@ function bareUrl(targetUrl) { } } +// fetch() with an abort timeout. Returns the Response on success, or null on +// network error / timeout. Callers decide what to read (.text/.json/.status). +// Centralizes the AbortController + clearTimeout dance so every caller cleans +// up the timer (via finally) on both the success and failure paths. +async function fetchWithTimeout(targetUrl, { method = "GET", timeoutMs = 10000, headers } = {}) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetch(targetUrl, { method, redirect: "follow", signal: controller.signal, headers }); + } catch { + return null; + } finally { + clearTimeout(timer); + } +} + // Check if URL is accessible (returns HTTP status code as a string). async function checkAccessibility(targetUrl) { - try { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 10000); - const res = await fetch(targetUrl, { - method: "HEAD", - redirect: "follow", - signal: controller.signal, - }); - clearTimeout(timeout); - return res.status.toString(); - } catch { - return "000"; - } + const res = await fetchWithTimeout(targetUrl, { method: "HEAD" }); + return res ? res.status.toString() : "000"; } function escapeRegExp(s) { @@ -116,16 +121,21 @@ function collectMarkdown(dir, acc = []) { // Check whether a URL identity already exists in the stored markdown. // Pure JS (no external `grep` dependency, works the same from any shell) and -// boundary-aware: the identity must be followed by a delimiter (close paren, -// quote, whitespace, query/fragment, the image dimension separator `_`, etc.) -// so a URL that is merely a PREFIX of a stored longer URL (e.g. /p/foo vs -// /p/foo-bar) is NOT a false duplicate. +// boundary-aware so a needle that is merely a PREFIX of a stored longer string +// is NOT a false duplicate. The two identity kinds need different boundaries: +// - Substack image UUID: the next char must not extend the hex id, so all of +// <uuid>.png (cover image, no size suffix), <uuid>_WxH and <uuid>) match. +// - URL: must be followed by a path/punctuation delimiter so /p/foo does not +// match a stored /p/foo-bar. function checkDuplicate(targetUrl, contentDir) { const needle = bareUrl(targetUrl); if (!needle) return false; - // Allow an optional trailing slash (bareUrl strips it, stored URLs may keep - // it) before the delimiter. - const boundary = new RegExp(escapeRegExp(needle) + `/?(?:[)\\]\\s"'?#<_&,]|$)`, "m"); + const isUuid = new RegExp(`^${UUID}$`, "i").test(needle); + // For URLs, allow an optional trailing slash (bareUrl strips it, stored URLs + // may keep it) before the delimiter. + const boundary = isUuid + ? new RegExp(escapeRegExp(needle) + `(?![0-9a-f])`, "i") + : new RegExp(escapeRegExp(needle) + `/?(?:[)\\]\\s"'?#<_&,]|$)`, "m"); for (const file of collectMarkdown(contentDir)) { let text; try { text = fs.readFileSync(file, "utf-8"); } catch { continue; } @@ -149,6 +159,7 @@ module.exports = { IDENTITY_PARAMS, isSubstackImage, substackImageUuid, + fetchWithTimeout, checkAccessibility, checkDuplicate, classifyType,