refactor(skills): fix dedup edge cases and modularize mt-* scripts

- fix checkDuplicate false-negative on Substack cover images (uuid with no size suffix) - stop cleanUrl from corrupting the query string on unparseable input - extract shared fetchWithTimeout into url-utils (consistent timer cleanup) - split find-substack-post html/rss helpers into html-text-utils - scan content/post for duplicates, matching the other scripts - correct doc drift in mt-add-image/mt-add-url and drop plan-phase code comment
2026-08-03 22:24:18 +00:00 · 2026-05-30 20:31:50 +07:00
parent 83ced8a575
commit 52e68c3841
6 changed files with 150 additions and 127 deletions
@@ -21,7 +21,7 @@ A clean image URL (passed by `mt-add-url`, or given directly).
 ```bash
 node .claude/skills/mt-add-image/scripts/detect-image-source.js "<url>"
 ```
-→ `{ isSubstack, uuid?, innerUrl? }`.
+→ `{ original_url, clean_url, isSubstack, uuid?, innerUrl? }`.

 When invoked **directly** (not via `mt-add-url`), first run the router to get accessibility + duplicate status and skip accordingly:
 ```bash
@@ -34,11 +34,11 @@ Find the source post:
 ```bash
 node .claude/skills/mt-add-image/scripts/find-substack-post.js --uuid <uuid>
 ```
- `found: false` → retry with the deeper sitemap crawl (slower — scans ~3 months; warn the user it may take ~15s):
+- `found: false` → retry with the deeper sitemap crawl (slower — fetches posts ~3 months back, capped at 40 fetches total across all publications; warn the user it may take a while):
  ```bash
  node .claude/skills/mt-add-image/scripts/find-substack-post.js --uuid <uuid> --deep
  ```
-  The result reports `scanned` + `cutoff` — mention how far back it looked.
+  On a miss the result reports `scanned` (posts fetched), `budget` (the 40-fetch cap), and `cutoff` (oldest date looked at) — mention how far back it looked.
 - `found: false` after `--deep` → no source post; go to step 3 (ask) and/or step 4 (add publication).

 **When `found: true` — pick the label (confirm-from-candidates):**
@@ -1,15 +1,24 @@
 #!/usr/bin/env node
 // Find which Substack post embeds a given image UUID, and extract a label.
 // Usage: node find-substack-post.js --uuid <uuid> [--deep]
-// Output: JSON { found, source?, publication?, postTitle?, postUrl?, caption?, scanned? }
+// Output on hit:  JSON { found:true, source, publication, postTitle, postUrl, caption, candidates }
+// Output on miss: JSON { found:false } (RSS) or { found:false, source:"sitemap", scanned, budget, cutoff } (--deep)
 //
 // Strategy: RSS feed first (fast, ~recent weeks). With --deep, fall back to a
-// sitemap crawl up to ~3 months back (added in the deep-lookup phase).
-// A Substack CDN URL does not encode its publication, so we search each
+// heavier sitemap crawl up to ~3 months back — opt-in because it fetches many
+// posts. A Substack CDN URL does not encode its publication, so we search each
 // publication listed in config/substack-publications.json.

 const fs = require("fs");
 const path = require("path");
+const { fetchWithTimeout } = require("../../mt-add-url/scripts/url-utils.js");
+const {
+  itemTitle,
+  itemLink,
+  extractCandidates,
+  captionForUuid,
+  postTitleFromHtml,
+} = require("./html-text-utils.js");

 const args = process.argv.slice(2);
 function flag(name) {
@@ -33,96 +42,10 @@ function loadPublications() {
  }
 }

+// Thin wrapper over the shared fetcher: returns body text, or "" on any error.
 async function fetchText(url) {
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), 10000);
-  try {
-    const res = await fetch(url, {
-      redirect: "follow",
-      signal: controller.signal,
-      headers: { "User-Agent": "Mozilla/5.0" },
-    });
-    return res.ok ? await res.text() : "";
-  } catch {
-    return "";
-  } finally {
-    clearTimeout(timeout);
-  }
-}
-
-function decodeEntities(s) {
-  return s
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&quot;/g, '"')
-    .replace(/&#0?39;|&apos;/g, "'")
-    .replace(/&nbsp;/g, " ")
-    .replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCodePoint(parseInt(n, 16)))
-    .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n))
-    .trim();
-}
-
-function stripTags(html) {
-  return decodeEntities(html.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
-}
-
-// Pull the first <title> (CDATA or plain) and <link> from an <item> chunk.
-function itemTitle(item) {
-  const m = item.match(/<title>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/title>/);
-  return m ? decodeEntities(m[1].trim()) : "";
-}
-function itemLink(item) {
-  const m = item.match(/<link>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/link>/);
-  return m ? m[1].trim() : "";
-}
-
-// Extract candidate topic titles from a post's TOC bullet list. ByteByteGo does
-// not attach captions to images — the topic titles live only in the "in this
-// issue" bullets. We can't reliably map image→title automatically (sponsor /
-// video items interleave and break positional order), so we surface these
-// candidates for the user to pick from. Light filtering keeps the list short:
-// dedupe, drop sub-point explanations ("Term: long sentence") and over-long
-// lines. The correct title is always present; the user selects it.
-function extractCandidates(html) {
-  const seen = new Set();
-  const out = [];
-  const re = /<li[^>]*>\s*(?:<p[^>]*>)?([\s\S]*?)(?:<\/p>)?\s*<\/li>/g;
-  let m;
-  while ((m = re.exec(html))) {
-    const text = stripTags(m[1]);
-    if (!text) continue;
-    if (text.length < 6 || text.length > 70) continue;     // titles are short; long lines are sub-point explanations
-    const key = text.toLowerCase();
-    if (seen.has(key)) continue;                           // content is duplicated in the page
-    seen.add(key);
-    out.push(text);
-  }
-  return out;
-}
-
-// If the UUID sits inside a <figure>…</figure>, return that figure's
-// <figcaption> text. Cover images live in <enclosure> (no figure) → "".
-function captionForUuid(item, id) {
-  const at = item.indexOf(id);
-  if (at === -1) return "";
-  const figStart = item.lastIndexOf("<figure", at);
-  if (figStart === -1) return "";
-  const figEnd = item.indexOf("</figure>", at);
-  if (figEnd === -1) return "";
-  const figure = item.slice(figStart, figEnd);
-  const cap = figure.match(/<figcaption[^>]*>([\s\S]*?)<\/figcaption>/);
-  return cap ? stripTags(cap[1]) : "";
-}
-
-// Extract a post title from server-rendered post HTML (og:title preferred).
-function postTitleFromHtml(html) {
-  const og = html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i);
-  if (og) return decodeEntities(og[1]);
-  const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
-  if (h1) return stripTags(h1[1]);
-  const t = html.match(/<title>([\s\S]*?)<\/title>/i);
-  return t ? decodeEntities(t[1].trim()) : "";
+  const res = await fetchWithTimeout(url, { headers: { "User-Agent": "Mozilla/5.0" } });
+  return res && res.ok ? await res.text() : "";
 }

 // Total post fetches allowed across ALL publications during a --deep crawl.
@@ -0,0 +1,91 @@
+// HTML / RSS text-extraction helpers for find-substack-post.js.
+// Pure string functions (no network, no fs) — kept separate so the crawler
+// stays focused on fetch/search flow. CommonJS so plain `node` works.
+
+// Decode the HTML entities that appear in Substack titles/captions, including
+// numeric (&#39;) and hex (&#x2014;) forms. &amp; is decoded last-ish but
+// before nothing re-encodes it; ordering here is safe for our inputs.
+function decodeEntities(s) {
+  return s
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#0?39;|&apos;/g, "'")
+    .replace(/&nbsp;/g, " ")
+    .replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCodePoint(parseInt(n, 16)))
+    .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n))
+    .trim();
+}
+
+function stripTags(html) {
+  return decodeEntities(html.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
+}
+
+// Pull the first <title> (CDATA or plain) and <link> from an RSS <item> chunk.
+function itemTitle(item) {
+  const m = item.match(/<title>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/title>/);
+  return m ? decodeEntities(m[1].trim()) : "";
+}
+function itemLink(item) {
+  const m = item.match(/<link>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/link>/);
+  return m ? m[1].trim() : "";
+}
+
+// Extract candidate topic titles from a post's TOC bullet list. ByteByteGo does
+// not attach captions to images — the topic titles live only in the "in this
+// issue" bullets. We can't reliably map image→title automatically (sponsor /
+// video items interleave and break positional order), so we surface these
+// candidates for the user to pick from. Light filtering keeps the list short:
+// dedupe, drop sub-point explanations ("Term: long sentence") and over-long
+// lines. The correct title is always present; the user selects it.
+function extractCandidates(html) {
+  const seen = new Set();
+  const out = [];
+  const re = /<li[^>]*>\s*(?:<p[^>]*>)?([\s\S]*?)(?:<\/p>)?\s*<\/li>/g;
+  let m;
+  while ((m = re.exec(html))) {
+    const text = stripTags(m[1]);
+    if (!text) continue;
+    if (text.length < 6 || text.length > 70) continue;     // titles are short; long lines are sub-point explanations
+    const key = text.toLowerCase();
+    if (seen.has(key)) continue;                           // content is duplicated in the page
+    seen.add(key);
+    out.push(text);
+  }
+  return out;
+}
+
+// If the UUID sits inside a <figure>…</figure>, return that figure's
+// <figcaption> text. Cover images live in <enclosure> (no figure) → "".
+function captionForUuid(item, id) {
+  const at = item.indexOf(id);
+  if (at === -1) return "";
+  const figStart = item.lastIndexOf("<figure", at);
+  if (figStart === -1) return "";
+  const figEnd = item.indexOf("</figure>", at);
+  if (figEnd === -1) return "";
+  const figure = item.slice(figStart, figEnd);
+  const cap = figure.match(/<figcaption[^>]*>([\s\S]*?)<\/figcaption>/);
+  return cap ? stripTags(cap[1]) : "";
+}
+
+// Extract a post title from server-rendered post HTML (og:title preferred).
+function postTitleFromHtml(html) {
+  const og = html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i);
+  if (og) return decodeEntities(og[1]);
+  const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
+  if (h1) return stripTags(h1[1]);
+  const t = html.match(/<title>([\s\S]*?)<\/title>/i);
+  return t ? decodeEntities(t[1].trim()) : "";
+}
+
+module.exports = {
+  decodeEntities,
+  stripTags,
+  itemTitle,
+  itemLink,
+  extractCandidates,
+  captionForUuid,
+  postTitleFromHtml,
+};
@@ -5,7 +5,7 @@ description: 'Meta entry for adding URLs to the Hugo blog newsletter. Use whenev

 ## Overview

-`mt-add-url` is the **router**: it classifies each URL and auto-invokes the matching handler skill. Handlers (`mt-add-post`, `mt-add-video`) own the actual content writing. Shared scripts live in `.claude/skills/mt-add-url/scripts/`; shared post mechanics in `references/newsletter-post-mechanics.md`.
+`mt-add-url` is the **meta dispatcher**: it classifies each URL and auto-invokes the matching handler skill. Handlers (`mt-add-post`, `mt-add-video`, `mt-add-image`) own the actual content writing. Shared scripts live in `.claude/skills/mt-add-url/scripts/`; shared post mechanics in `references/newsletter-post-mechanics.md`.

 **Supported routes (this version):**
 - `article` → `mt-add-post`
@@ -9,6 +9,7 @@ const path = require("path");
 const {
  cleanUrl,
  isSubstackImage,
+  fetchWithTimeout,
  checkAccessibility,
  checkDuplicate,
  classifyType,
@@ -21,7 +22,7 @@ if (!url) {
 }

 const PROJECT_ROOT = path.resolve(__dirname, "../../../..");
-const CONTENT_DIR = path.join(PROJECT_ROOT, "content");
+const CONTENT_DIR = path.join(PROJECT_ROOT, "content", "post");

 const YT_HOSTS = new Set(["youtube.com", "www.youtube.com", "m.youtube.com"]);

@@ -60,13 +61,10 @@ function canonicalWatchUrl(videoId) {
 // Fetch title/author via YouTube oEmbed (no API key). Best-effort: any failure
 // returns {} so the route stays `youtube` and the skill can fall back.
 async function fetchYouTubeMeta(watchUrl) {
+  const endpoint = `https://www.youtube.com/oembed?url=${encodeURIComponent(watchUrl)}&format=json`;
+  const res = await fetchWithTimeout(endpoint);
+  if (!res || !res.ok) return {};
  try {
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 10000);
-    const endpoint = `https://www.youtube.com/oembed?url=${encodeURIComponent(watchUrl)}&format=json`;
-    const res = await fetch(endpoint, { redirect: "follow", signal: controller.signal });
-    clearTimeout(timeout);
-    if (!res.ok) return {};
    const data = await res.json();
    return { title: data.title, author: data.author_name };
  } catch {
@@ -21,10 +21,10 @@ function cleanUrl(rawUrl) {
    });
    return parsed.toString();
  } catch {
-    // If URL parsing fails, do basic string cleanup
-    return rawUrl.replace(/[?&](utm_[^&]*|fbclid|gclid|msclkid|mc_eid|aid|ref|ref_src|ref_url|source|ck_subscriber_id|igshid|yclid|vero_id)=[^&]*/gi, "")
-      .replace(/\?&/, "?")
-      .replace(/[?&]$/, "");
+    // Unparseable input (not a real URL): the per-param string surgery above
+    // would mangle the query (drop the `?`, leave a dangling `&`), so leave it
+    // untouched rather than corrupt it.
+    return rawUrl;
  }
 }

@@ -77,21 +77,26 @@ function bareUrl(targetUrl) {
  }
 }

+// fetch() with an abort timeout. Returns the Response on success, or null on
+// network error / timeout. Callers decide what to read (.text/.json/.status).
+// Centralizes the AbortController + clearTimeout dance so every caller cleans
+// up the timer (via finally) on both the success and failure paths.
+async function fetchWithTimeout(targetUrl, { method = "GET", timeoutMs = 10000, headers } = {}) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    return await fetch(targetUrl, { method, redirect: "follow", signal: controller.signal, headers });
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
 // Check if URL is accessible (returns HTTP status code as a string).
 async function checkAccessibility(targetUrl) {
-  try {
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 10000);
-    const res = await fetch(targetUrl, {
-      method: "HEAD",
-      redirect: "follow",
-      signal: controller.signal,
-    });
-    clearTimeout(timeout);
-    return res.status.toString();
-  } catch {
-    return "000";
-  }
+  const res = await fetchWithTimeout(targetUrl, { method: "HEAD" });
+  return res ? res.status.toString() : "000";
 }

 function escapeRegExp(s) {
@@ -116,16 +121,21 @@ function collectMarkdown(dir, acc = []) {

 // Check whether a URL identity already exists in the stored markdown.
 // Pure JS (no external `grep` dependency, works the same from any shell) and
-// boundary-aware: the identity must be followed by a delimiter (close paren,
-// quote, whitespace, query/fragment, the image dimension separator `_`, etc.)
-// so a URL that is merely a PREFIX of a stored longer URL (e.g. /p/foo vs
-// /p/foo-bar) is NOT a false duplicate.
+// boundary-aware so a needle that is merely a PREFIX of a stored longer string
+// is NOT a false duplicate. The two identity kinds need different boundaries:
+//   - Substack image UUID: the next char must not extend the hex id, so all of
+//     <uuid>.png (cover image, no size suffix), <uuid>_WxH and <uuid>) match.
+//   - URL: must be followed by a path/punctuation delimiter so /p/foo does not
+//     match a stored /p/foo-bar.
 function checkDuplicate(targetUrl, contentDir) {
  const needle = bareUrl(targetUrl);
  if (!needle) return false;
-  // Allow an optional trailing slash (bareUrl strips it, stored URLs may keep
-  // it) before the delimiter.
-  const boundary = new RegExp(escapeRegExp(needle) + `/?(?:[)\\]\\s"'?#<_&,]|$)`, "m");
+  const isUuid = new RegExp(`^${UUID}$`, "i").test(needle);
+  // For URLs, allow an optional trailing slash (bareUrl strips it, stored URLs
+  // may keep it) before the delimiter.
+  const boundary = isUuid
+    ? new RegExp(escapeRegExp(needle) + `(?![0-9a-f])`, "i")
+    : new RegExp(escapeRegExp(needle) + `/?(?:[)\\]\\s"'?#<_&,]|$)`, "m");
  for (const file of collectMarkdown(contentDir)) {
    let text;
    try { text = fs.readFileSync(file, "utf-8"); } catch { continue; }
@@ -149,6 +159,7 @@ module.exports = {
  IDENTITY_PARAMS,
  isSubstackImage,
  substackImageUuid,
+  fetchWithTimeout,
  checkAccessibility,
  checkDuplicate,
  classifyType,