diff --git a/.claude/skills/mt-add-image/SKILL.md b/.claude/skills/mt-add-image/SKILL.md index 83a8834..95e2484 100644 --- a/.claude/skills/mt-add-image/SKILL.md +++ b/.claude/skills/mt-add-image/SKILL.md @@ -21,7 +21,13 @@ A clean image URL (passed by `mt-add-url`, or given directly). ```bash node .claude/skills/mt-add-image/scripts/detect-image-source.js "" ``` -→ `{ isSubstack, uuid?, innerUrl? }`. Skip the URL if it's a duplicate/inaccessible (the meta skill already checks; if invoked directly, sanity-check first). +→ `{ isSubstack, uuid?, innerUrl? }`. + +When invoked **directly** (not via `mt-add-url`), first run the router to get accessibility + duplicate status and skip accordingly: +```bash +node ../mt-add-url/scripts/add-url.js "" # expect route:image; skip if duplicate/!accessible +``` +(When dispatched by `mt-add-url`, that check already ran — don't repeat it.) ### 2a. Substack image (`isSubstack: true` with `uuid`) Find the source post: diff --git a/.claude/skills/mt-add-image/scripts/detect-image-source.js b/.claude/skills/mt-add-image/scripts/detect-image-source.js index 0b4878e..4ff2827 100644 --- a/.claude/skills/mt-add-image/scripts/detect-image-source.js +++ b/.claude/skills/mt-add-image/scripts/detect-image-source.js @@ -7,8 +7,11 @@ // https://substackcdn.com/image/fetch/$s_!x!,.../https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F_WxH.png // The publication is NOT encoded in the URL — only the image identity (uuid) is. -const path = require("path"); -const { cleanUrl } = require("../../mt-add-url/scripts/url-utils.js"); +const { + cleanUrl, + isSubstackImage, + substackImageUuid, +} = require("../../mt-add-url/scripts/url-utils.js"); const url = process.argv[2]; if (!url) { @@ -16,40 +19,20 @@ if (!url) { process.exit(1); } -const SUBSTACK_HOSTS = ["substackcdn.com", "substack-post-media.s3.amazonaws.com"]; - // Pull the inner S3 URL out of a substackcdn /image/fetch/ wrapper (if present). function extractInnerUrl(targetUrl) { const marker = targetUrl.indexOf("/https%3A%2F%2F"); - if (marker !== -1) { - return decodeURIComponent(targetUrl.slice(marker + 1)); - } + if (marker !== -1) return decodeURIComponent(targetUrl.slice(marker + 1)); // Some forms embed a plain (already-decoded) inner https URL. const plain = targetUrl.indexOf("/https://", 8); if (plain !== -1) return targetUrl.slice(plain + 1); return targetUrl; } -function detect(targetUrl) { - let host = ""; - try { - host = new URL(targetUrl).host.toLowerCase(); - } catch { - /* fall through — non-URL input is non-Substack */ - } - const innerUrl = extractInnerUrl(targetUrl); - const isSubstack = - SUBSTACK_HOSTS.includes(host) || /substack-post-media/i.test(innerUrl); - - // public/images/_WxH.ext — uuid is the stable image identity. - const m = innerUrl.match(/public\/images\/([0-9a-fA-F-]{36})/); - const uuid = m ? m[1].toLowerCase() : undefined; - - return { isSubstack, uuid, innerUrl: isSubstack ? innerUrl : undefined }; -} - const cleaned = cleanUrl(url); -const { isSubstack, uuid, innerUrl } = detect(url); +const isSubstack = isSubstackImage(url); +const uuid = isSubstack ? substackImageUuid(url) : null; +const innerUrl = isSubstack ? extractInnerUrl(url) : null; console.log(JSON.stringify({ original_url: url, diff --git a/.claude/skills/mt-add-image/scripts/find-substack-post.js b/.claude/skills/mt-add-image/scripts/find-substack-post.js index c4380fb..90beb69 100644 --- a/.claude/skills/mt-add-image/scripts/find-substack-post.js +++ b/.claude/skills/mt-add-image/scripts/find-substack-post.js @@ -58,7 +58,8 @@ function decodeEntities(s) { .replace(/"/g, '"') .replace(/�?39;|'/g, "'") .replace(/ /g, " ") - .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(+n)) + .replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCodePoint(parseInt(n, 16))) + .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n)) .trim(); } @@ -124,9 +125,13 @@ function postTitleFromHtml(html) { return t ? decodeEntities(t[1].trim()) : ""; } +// Total post fetches allowed across ALL publications during a --deep crawl. +const DEEP_FETCH_BUDGET = 40; + // Deep fallback: crawl the sitemap back ~3 months, fetch posts most-recent-first -// (capped), and look for the UUID. Heavier than RSS — only used on RSS miss. -async function searchSitemap(publication, id, maxFetch = 40, monthsBack = 3) { +// (up to `maxFetch` from the shared budget), and look for the UUID. Heavier than +// RSS — only used on RSS miss. +async function searchSitemap(publication, id, maxFetch, monthsBack = 3) { const xml = await fetchText(`https://${publication}/sitemap.xml`); if (!xml) return { hit: null, scanned: 0, cutoff: null }; @@ -195,20 +200,25 @@ async function main() { if (hit) return console.log(JSON.stringify(hit, null, 2)); } - // Deep fallback: sitemap crawl up to ~3 months back. + // Deep fallback: sitemap crawl up to ~3 months back, sharing one global + // fetch budget across all publications so coverage can't blow up as the + // publications list grows. if (deep) { let totalScanned = 0; let lastCutoff = null; for (const pub of publications) { - const { hit, scanned, cutoff } = await searchSitemap(pub, uuid); + const remaining = DEEP_FETCH_BUDGET - totalScanned; + if (remaining <= 0) break; + const { hit, scanned, cutoff } = await searchSitemap(pub, uuid, remaining); totalScanned += scanned; - lastCutoff = cutoff; + lastCutoff = cutoff || lastCutoff; if (hit) return console.log(JSON.stringify(hit, null, 2)); } return console.log(JSON.stringify({ found: false, source: "sitemap", scanned: totalScanned, + budget: DEEP_FETCH_BUDGET, cutoff: lastCutoff, }, null, 2)); } diff --git a/.claude/skills/mt-add-url/SKILL.md b/.claude/skills/mt-add-url/SKILL.md index 5b09cd4..80dd8a2 100644 --- a/.claude/skills/mt-add-url/SKILL.md +++ b/.claude/skills/mt-add-url/SKILL.md @@ -1,6 +1,6 @@ --- name: mt-add-url -description: 'Meta entry for adding URLs to the Hugo blog newsletter. Use whenever the user provides one or more URLs to add to their newsletter (articles, YouTube videos, etc.). Classifies each URL and auto-dispatches to the right handler skill (mt-add-post for articles, mt-add-video for YouTube). For unsupported types it asks the user how to proceed. This is the default entry point for newsletter URL processing.' +description: 'Meta entry for adding URLs to the Hugo blog newsletter. Use whenever the user provides one or more URLs to add to their newsletter (articles, YouTube videos, images, etc.). Classifies each URL and auto-dispatches to the right handler skill (mt-add-post for articles, mt-add-video for YouTube, mt-add-image for images). For unsupported types it asks the user how to proceed. This is the default entry point for newsletter URL processing.' --- ## Overview @@ -66,6 +66,7 @@ Aggregate across all URLs: ✅ Dispatched: [count] - [count] → mt-add-post (articles) - [count] → mt-add-video (YouTube) + - [count] → mt-add-image (images) ⏭️ Skipped: [count] - [url]: duplicate / inaccessible @@ -76,5 +77,5 @@ Aggregate across all URLs: ## Notes -- Handlers (`mt-add-post`, `mt-add-video`) remain directly invocable for single-purpose use, but `mt-add-url` is the normal entry point when a user pastes a URL. +- Handlers (`mt-add-post`, `mt-add-video`, `mt-add-image`) remain directly invocable for single-purpose use, but `mt-add-url` is the normal entry point when a user pastes a URL. - Shared mechanics (numbering, post find/create, Bonus insertion, language rules) are defined once in `references/newsletter-post-mechanics.md`; handlers reference it. diff --git a/.claude/skills/mt-add-url/references/newsletter-post-mechanics.md b/.claude/skills/mt-add-url/references/newsletter-post-mechanics.md index 002204c..2090777 100644 --- a/.claude/skills/mt-add-url/references/newsletter-post-mechanics.md +++ b/.claude/skills/mt-add-url/references/newsletter-post-mechanics.md @@ -1,6 +1,6 @@ # Newsletter Post Mechanics (shared) -Shared procedure used by the newsletter handler skills (`mt-add-post`, `mt-add-video`, …). +Shared procedure used by the newsletter handler skills (`mt-add-post`, `mt-add-video`, `mt-add-image`). All shared scripts live in `.claude/skills/mt-add-url/scripts/`. **Project Context:** @@ -38,6 +38,8 @@ categories: ["Newsletter"] ## 4. Section insertion (no clobbering) +**Never rewrite the whole `index.md`.** Always insert by anchoring an Edit on an existing string (e.g. `### Bonus`, `**Videos:**`) and prepending/appending around it. Multiple URLs targeting the same day's post must be applied **sequentially** so one edit doesn't clobber another. + **Articles go before the `### Bonus` section; Bonus assets go inside it.** To insert an article safely, anchor the Edit on `### Bonus` and prepend: @@ -66,7 +68,7 @@ If the post has **no `### Bonus`** yet: **Documents:** [PDF: title](url) ``` -When a subsection (e.g. `**Videos:**`) already exists, append under it; otherwise create it. +When a subsection (e.g. `**Videos:**`) already exists, append under it; otherwise create it. Keep subsections in this order: **Images** → **Videos** → **Documents**. ## 5. Language guidelines diff --git a/.claude/skills/mt-add-url/scripts/add-url.js b/.claude/skills/mt-add-url/scripts/add-url.js index a6cfe94..fe61fcc 100644 --- a/.claude/skills/mt-add-url/scripts/add-url.js +++ b/.claude/skills/mt-add-url/scripts/add-url.js @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Meta URL router for the mt-add-post skill — the single entry per URL. +// Meta URL router for the mt-add-url skill — the single entry per URL. // Usage: node add-url.js "" // Outputs: JSON { original_url, clean_url, http_status, accessible, // duplicate, route, title?, author? } @@ -8,6 +8,7 @@ const path = require("path"); const { cleanUrl, + isSubstackImage, checkAccessibility, checkDuplicate, classifyType, @@ -80,7 +81,13 @@ async function main() { // For YouTube, dedup/store against the canonical watch URL so youtu.be and // shorts links collapse onto the same identity-param key as watch URLs. const effectiveUrl = yt.isYouTube ? canonicalWatchUrl(yt.videoId) : cleanedUrl; - const route = yt.isYouTube ? "youtube" : classifyType(cleanedUrl); + // Route order: YouTube → Substack image (by host, not extension, so f_auto / + // .avif / .heic / extensionless CDN URLs still route to the image handler) → + // file-extension classification. + let route; + if (yt.isYouTube) route = "youtube"; + else if (isSubstackImage(cleanedUrl)) route = "image"; + else route = classifyType(cleanedUrl); const httpStatus = await checkAccessibility(cleanedUrl); const accessible = httpStatus === "200"; diff --git a/.claude/skills/mt-add-url/scripts/find-newsletter-number.js b/.claude/skills/mt-add-url/scripts/find-newsletter-number.js index 393426d..3ef502c 100644 --- a/.claude/skills/mt-add-url/scripts/find-newsletter-number.js +++ b/.claude/skills/mt-add-url/scripts/find-newsletter-number.js @@ -20,23 +20,6 @@ function extractNewsletterNumber(filePath) { } } -// Get current date in Asia/Ho_Chi_Minh timezone -function getCurrentDate() { - const now = new Date(); - const formatter = new Intl.DateTimeFormat("en-CA", { - timeZone: "Asia/Ho_Chi_Minh", - year: "numeric", - month: "2-digit", - day: "2-digit", - }); - const parts = formatter.formatToParts(now); - return { - year: parts.find((p) => p.type === "year").value, - month: parts.find((p) => p.type === "month").value, - day: parts.find((p) => p.type === "day").value, - }; -} - // Scan all year/month/day directories for newsletter posts function findMostRecentNewsletter() { let maxNumber = 0; diff --git a/.claude/skills/mt-add-url/scripts/url-utils.js b/.claude/skills/mt-add-url/scripts/url-utils.js index 0101e0c..cee731c 100644 --- a/.claude/skills/mt-add-url/scripts/url-utils.js +++ b/.claude/skills/mt-add-url/scripts/url-utils.js @@ -1,8 +1,9 @@ -// Shared URL helpers for the mt-add-post skill. -// Used by add-url.js (the meta router entry point). +// Shared URL helpers for the mt-* newsletter skills. +// Owned by the mt-add-url meta router; reused by handlers (e.g. mt-add-image). // CommonJS so plain `node script.js` works without a build step. -const { execFileSync } = require("child_process"); +const fs = require("fs"); +const path = require("path"); // Remove common tracking parameters (utm_* plus a fixed set of known trackers). function cleanUrl(rawUrl) { @@ -27,6 +28,24 @@ function cleanUrl(rawUrl) { } } +// --- Substack image helpers (shared by add-url.js routing and mt-add-image) --- +const SUBSTACK_IMAGE_HOSTS = ["substackcdn.com", "substack-post-media.s3.amazonaws.com"]; +const UUID = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; + +// A Substack-hosted image (CDN wrapper or raw S3), regardless of file extension. +function isSubstackImage(targetUrl) { + let host = ""; + try { host = new URL(targetUrl).host.toLowerCase(); } catch { /* non-URL */ } + return SUBSTACK_IMAGE_HOSTS.includes(host) || /substack-post-media/i.test(targetUrl); +} + +// The stable image identity is the S3 image UUID under public/images/. +// Works whether the path separators are raw (/) or percent-encoded (%2F). +function substackImageUuid(targetUrl) { + const m = targetUrl.match(new RegExp(`images(?:%2F|/)(${UUID})`, "i")); + return m ? m[1].toLowerCase() : null; +} + // Some sites carry the resource identity in a query param, not the path // (e.g. YouTube /watch?v=ID). Stripping the query for these collapses every // item to the same bare URL, causing false-positive duplicates. Preserve the @@ -37,9 +56,15 @@ const IDENTITY_PARAMS = { "m.youtube.com": "v", }; -// Extract the bare URL (scheme + host + path) — used for stricter duplicate checks. -// Keeps the host's identity query param when one is defined above. +// Reduce a URL to a stable identity used for duplicate detection: +// - Substack image → its S3 UUID (transform/size variants share one identity) +// - YouTube → scheme+host+path + the v= video id +// - everything else → scheme + host + path function bareUrl(targetUrl) { + if (isSubstackImage(targetUrl)) { + const uuid = substackImageUuid(targetUrl); + if (uuid) return uuid; + } try { const p = new URL(targetUrl); let bare = `${p.protocol}//${p.host}${p.pathname}`.replace(/\/$/, ""); @@ -69,28 +94,52 @@ async function checkAccessibility(targetUrl) { } } -// Check if URL already exists under contentDir. -// Compare by bare URL so stored copies with different tracking params -// still register as duplicates. contentDir is passed by the caller so this -// module stays decoupled from any specific project layout. -function checkDuplicate(targetUrl, contentDir) { +function escapeRegExp(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +// Recursively collect *.md files under a directory (the content tree is small). +function collectMarkdown(dir, acc = []) { + let entries; try { - const needle = bareUrl(targetUrl); - // execFileSync (no shell) — the needle is URL-derived and could otherwise - // carry shell metacharacters; passing args directly avoids any injection. - execFileSync("grep", ["-rF", needle, contentDir], { stdio: "pipe" }); - return true; + entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { - return false; + return acc; // missing dir → nothing to compare against } + for (const e of entries) { + const full = path.join(dir, e.name); + if (e.isDirectory()) collectMarkdown(full, acc); + else if (e.isFile() && e.name.toLowerCase().endsWith(".md")) acc.push(full); + } + return acc; +} + +// Check whether a URL identity already exists in the stored markdown. +// Pure JS (no external `grep` dependency, works the same from any shell) and +// boundary-aware: the identity must be followed by a delimiter (close paren, +// quote, whitespace, query/fragment, the image dimension separator `_`, etc.) +// so a URL that is merely a PREFIX of a stored longer URL (e.g. /p/foo vs +// /p/foo-bar) is NOT a false duplicate. +function checkDuplicate(targetUrl, contentDir) { + const needle = bareUrl(targetUrl); + if (!needle) return false; + // Allow an optional trailing slash (bareUrl strips it, stored URLs may keep + // it) before the delimiter. + const boundary = new RegExp(escapeRegExp(needle) + `/?(?:[)\\]\\s"'?#<_&,]|$)`, "m"); + for (const file of collectMarkdown(contentDir)) { + let text; + try { text = fs.readFileSync(file, "utf-8"); } catch { continue; } + if (text.includes(needle) && boundary.test(text)) return true; + } + return false; } // Classify URL type by file extension. Returns image|video|document|article. function classifyType(targetUrl) { const lower = targetUrl.toLowerCase(); - if (/\.(png|jpg|jpeg|gif|webp|svg)(\?.*)?$/.test(lower)) return "image"; - if (/\.(mp4|webm|mov|avi)(\?.*)?$/.test(lower)) return "video"; - if (/\.(pdf|doc|docx|xls|xlsx)(\?.*)?$/.test(lower)) return "document"; + if (/\.(png|jpg|jpeg|gif|webp|svg|avif|heic|heif|bmp|tiff?)(\?.*)?$/.test(lower)) return "image"; + if (/\.(mp4|webm|mov|avi|mkv)(\?.*)?$/.test(lower)) return "video"; + if (/\.(pdf|docx?|xlsx?|pptx?)(\?.*)?$/.test(lower)) return "document"; return "article"; } @@ -98,6 +147,8 @@ module.exports = { cleanUrl, bareUrl, IDENTITY_PARAMS, + isSubstackImage, + substackImageUuid, checkAccessibility, checkDuplicate, classifyType, diff --git a/CLAUDE.md b/CLAUDE.md index d7d3e29..afe167c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,6 +43,7 @@ This project uses custom skills for automated workflows: - **mt-add-video**: YouTube handler — adds a YouTube link to the newsletter Bonus → Videos - **mt-add-image**: Image handler — adds an image to Bonus → Images; for Substack images finds the source post (ByteByteGo + configured publications) via RSS/sitemap to label it, else asks for a label - **mt-add-tags**: Add/update tags in Hugo post frontmatter +- **mt-webfetch**: Fallback web fetcher (defuddle proxy) — use only when the built-in WebFetch is blocked Shared scripts (`add-url.js`, `url-utils.js`, `find-newsletter-number.js`) and the shared post-mechanics reference live under `.claude/skills/mt-add-url/`. `mt-add-url` dispatches `article`/`youtube`/`image`; other types (direct video files, documents, unknown) prompt the user to add or extend a handler.