mirror of
https://github.com/tiennm99/miti99.git
synced 2026-08-26 06:27:43 +00:00
feat(tooling): run blog from Claude Code, OpenCode & Codex on shared engine
Extract newsletter scripts to a neutral scripts/newsletter/ engine and add side-by-side support for all three AI coding tools off one source of truth. - Move 9 scripts + substack config from .claude/skills/**/scripts to scripts/newsletter/ (history preserved); fix PROJECT_ROOT depth and cross-require/config paths; repoint all SKILL.md invocations - Add canonical AGENTS.md; reduce CLAUDE.md to an @AGENTS.md import - Add opencode.json (permissions, no MCP); skills auto-discovered in place - Add codex/prompts/*.md (6 prompts) + copy installers (install.sh/.ps1) - Add docs/multi-tool-usage.md (setup, per-tool invocation, teardown) + README pointer
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env node
|
||||
// Meta URL router for the mt-add-url skill — the single entry per URL.
|
||||
// Usage: node add-url.js "<url>"
|
||||
// Outputs: JSON { original_url, clean_url, http_status, accessible,
|
||||
// duplicate, route, title?, author? }
|
||||
// route ∈ youtube | image | video | document | article
|
||||
|
||||
const path = require("path");
|
||||
const {
|
||||
cleanUrl,
|
||||
isSubstackImage,
|
||||
fetchWithTimeout,
|
||||
checkAccessibility,
|
||||
checkDuplicate,
|
||||
classifyType,
|
||||
} = require("./url-utils");
|
||||
|
||||
const url = process.argv[2];
|
||||
if (!url) {
|
||||
console.error("Usage: node add-url.js <url>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const PROJECT_ROOT = path.resolve(__dirname, "../..");
|
||||
const CONTENT_DIR = path.join(PROJECT_ROOT, "content", "post");
|
||||
|
||||
const YT_HOSTS = new Set(["youtube.com", "www.youtube.com", "m.youtube.com"]);
|
||||
|
||||
// Detect a YouTube video and extract its id from the supported URL shapes:
|
||||
// youtube.com/watch?v=ID, youtu.be/ID, youtube.com/shorts/ID
|
||||
// Playlists/channels are intentionally NOT YouTube routes (fall through to type).
|
||||
function detectYouTube(targetUrl) {
|
||||
try {
|
||||
const p = new URL(targetUrl);
|
||||
const host = p.host.toLowerCase();
|
||||
if (host === "youtu.be") {
|
||||
const id = p.pathname.slice(1).split("/")[0];
|
||||
return id ? { isYouTube: true, videoId: id } : { isYouTube: false };
|
||||
}
|
||||
if (YT_HOSTS.has(host)) {
|
||||
if (p.pathname === "/watch") {
|
||||
const id = p.searchParams.get("v");
|
||||
return id ? { isYouTube: true, videoId: id } : { isYouTube: false };
|
||||
}
|
||||
if (p.pathname.startsWith("/shorts/")) {
|
||||
const id = p.pathname.split("/")[2];
|
||||
return id ? { isYouTube: true, videoId: id } : { isYouTube: false };
|
||||
}
|
||||
}
|
||||
return { isYouTube: false };
|
||||
} catch {
|
||||
return { isYouTube: false };
|
||||
}
|
||||
}
|
||||
|
||||
// Canonical watch URL — oEmbed accepts watch URLs reliably for all shapes.
|
||||
function canonicalWatchUrl(videoId) {
|
||||
return `https://www.youtube.com/watch?v=${videoId}`;
|
||||
}
|
||||
|
||||
// Fetch title/author via YouTube oEmbed (no API key). Best-effort: any failure
|
||||
// returns {} so the route stays `youtube` and the skill can fall back.
|
||||
async function fetchYouTubeMeta(watchUrl) {
|
||||
const endpoint = `https://www.youtube.com/oembed?url=${encodeURIComponent(watchUrl)}&format=json`;
|
||||
const res = await fetchWithTimeout(endpoint);
|
||||
if (!res || !res.ok) return {};
|
||||
try {
|
||||
const data = await res.json();
|
||||
return { title: data.title, author: data.author_name };
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const cleanedUrl = cleanUrl(url);
|
||||
const yt = detectYouTube(cleanedUrl);
|
||||
|
||||
// For YouTube, dedup/store against the canonical watch URL so youtu.be and
|
||||
// shorts links collapse onto the same identity-param key as watch URLs.
|
||||
const effectiveUrl = yt.isYouTube ? canonicalWatchUrl(yt.videoId) : cleanedUrl;
|
||||
// Route order: YouTube → Substack image (by host, not extension, so f_auto /
|
||||
// .avif / .heic / extensionless CDN URLs still route to the image handler) →
|
||||
// file-extension classification.
|
||||
let route;
|
||||
if (yt.isYouTube) route = "youtube";
|
||||
else if (isSubstackImage(cleanedUrl)) route = "image";
|
||||
else route = classifyType(cleanedUrl);
|
||||
|
||||
const httpStatus = await checkAccessibility(cleanedUrl);
|
||||
const accessible = httpStatus === "200";
|
||||
const duplicate = checkDuplicate(effectiveUrl, CONTENT_DIR);
|
||||
|
||||
const out = {
|
||||
original_url: url,
|
||||
clean_url: effectiveUrl,
|
||||
http_status: httpStatus,
|
||||
accessible,
|
||||
duplicate,
|
||||
route,
|
||||
};
|
||||
|
||||
if (route === "youtube") {
|
||||
const meta = await fetchYouTubeMeta(effectiveUrl);
|
||||
if (meta.title) out.title = meta.title;
|
||||
if (meta.author) out.author = meta.author;
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(out, null, 2));
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1 @@
|
||||
["blog.bytebytego.com"]
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env node
|
||||
// Detect whether an image URL is Substack-hosted and extract its S3 image UUID.
|
||||
// Usage: node detect-image-source.js "<image-url>"
|
||||
// Output: JSON { original_url, clean_url, isSubstack, uuid?, innerUrl? }
|
||||
//
|
||||
// Substack images are usually served via a CDN wrapper:
|
||||
// https://substackcdn.com/image/fetch/$s_!x!,.../https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F<uuid>_WxH.png
|
||||
// The publication is NOT encoded in the URL — only the image identity (uuid) is.
|
||||
|
||||
const {
|
||||
cleanUrl,
|
||||
isSubstackImage,
|
||||
substackImageUuid,
|
||||
} = require("./url-utils.js");
|
||||
|
||||
const url = process.argv[2];
|
||||
if (!url) {
|
||||
console.error("Usage: node detect-image-source.js <image-url>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Pull the inner S3 URL out of a substackcdn /image/fetch/ wrapper (if present).
|
||||
function extractInnerUrl(targetUrl) {
|
||||
const marker = targetUrl.indexOf("/https%3A%2F%2F");
|
||||
if (marker !== -1) return decodeURIComponent(targetUrl.slice(marker + 1));
|
||||
// Some forms embed a plain (already-decoded) inner https URL.
|
||||
const plain = targetUrl.indexOf("/https://", 8);
|
||||
if (plain !== -1) return targetUrl.slice(plain + 1);
|
||||
return targetUrl;
|
||||
}
|
||||
|
||||
const cleaned = cleanUrl(url);
|
||||
const isSubstack = isSubstackImage(url);
|
||||
const uuid = isSubstack ? substackImageUuid(url) : null;
|
||||
const innerUrl = isSubstack ? extractInnerUrl(url) : null;
|
||||
|
||||
console.log(JSON.stringify({
|
||||
original_url: url,
|
||||
clean_url: cleaned,
|
||||
isSubstack,
|
||||
...(uuid ? { uuid } : {}),
|
||||
...(innerUrl ? { innerUrl } : {}),
|
||||
}, null, 2));
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env node
|
||||
// mt-webfetch fallback fetcher via defuddle.md
|
||||
// Usage: node fetch.js <target_url>
|
||||
// Exit codes: 0 = content returned, 1 = empty/failed, 2 = bad args
|
||||
|
||||
const url = process.argv[2];
|
||||
if (!url) {
|
||||
console.error("Usage: node fetch.js <target_url>");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const defuddleUrl = `https://defuddle.md/${url}`;
|
||||
|
||||
async function main() {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 30000);
|
||||
try {
|
||||
const res = await fetch(defuddleUrl, {
|
||||
redirect: "follow",
|
||||
signal: controller.signal,
|
||||
headers: { "User-Agent": "mt-webfetch/1.0" },
|
||||
});
|
||||
clearTimeout(timeout);
|
||||
const body = await res.text();
|
||||
if (!res.ok || !body || body.trim().length === 0) {
|
||||
console.error(`mt-webfetch: defuddle returned ${res.status} / empty body for ${url}`);
|
||||
process.exit(1);
|
||||
}
|
||||
process.stdout.write(body);
|
||||
} catch (err) {
|
||||
clearTimeout(timeout);
|
||||
console.error(`mt-webfetch: fetch failed for ${url}: ${err.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env node
|
||||
// Find the most recent newsletter number and return the next one
|
||||
// Usage: node find-newsletter-number.js
|
||||
// Outputs: The next newsletter number
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const PROJECT_ROOT = path.resolve(__dirname, "../..");
|
||||
const CONTENT_DIR = path.join(PROJECT_ROOT, "content", "post");
|
||||
|
||||
// Extract newsletter number from file content
|
||||
function extractNewsletterNumber(filePath) {
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, "utf-8");
|
||||
const match = content.match(/Newsletter\s*#(\d+)/);
|
||||
return match ? parseInt(match[1], 10) : 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Scan all year/month/day directories for newsletter posts
|
||||
function findMostRecentNewsletter() {
|
||||
let maxNumber = 0;
|
||||
|
||||
// List year directories, sorted descending
|
||||
let years;
|
||||
try {
|
||||
years = fs.readdirSync(CONTENT_DIR)
|
||||
.filter((d) => /^\d{4}$/.test(d))
|
||||
.sort((a, b) => b.localeCompare(a));
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (const year of years) {
|
||||
const yearDir = path.join(CONTENT_DIR, year);
|
||||
let months;
|
||||
try {
|
||||
months = fs.readdirSync(yearDir)
|
||||
.filter((d) => /^\d{2}$/.test(d))
|
||||
.sort((a, b) => b.localeCompare(a));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const month of months) {
|
||||
const monthDir = path.join(yearDir, month);
|
||||
let days;
|
||||
try {
|
||||
days = fs.readdirSync(monthDir)
|
||||
.filter((d) => /^\d{2}$/.test(d))
|
||||
.sort((a, b) => b.localeCompare(a));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const day of days) {
|
||||
const filePath = path.join(monthDir, day, "index.md");
|
||||
const num = extractNewsletterNumber(filePath);
|
||||
if (num > maxNumber) maxNumber = num;
|
||||
}
|
||||
}
|
||||
|
||||
// Early exit: if we found a newsletter in this year, no need to go further back
|
||||
if (maxNumber > 0) break;
|
||||
}
|
||||
|
||||
return maxNumber;
|
||||
}
|
||||
|
||||
const mostRecent = findMostRecentNewsletter();
|
||||
console.log(mostRecent + 1);
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env node
|
||||
// Find which Substack post embeds a given image UUID, and extract a label.
|
||||
// Usage: node find-substack-post.js --uuid <uuid> [--deep]
|
||||
// Output on hit: JSON { found:true, source, publication, postTitle, postUrl, caption, candidates }
|
||||
// Output on miss: JSON { found:false } (RSS) or { found:false, source:"sitemap", scanned, budget, cutoff } (--deep)
|
||||
//
|
||||
// Strategy: RSS feed first (fast, ~recent weeks). With --deep, fall back to a
|
||||
// heavier sitemap crawl up to ~3 months back — opt-in because it fetches many
|
||||
// posts. A Substack CDN URL does not encode its publication, so we search each
|
||||
// publication listed in config/substack-publications.json.
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { fetchWithTimeout } = require("./url-utils.js");
|
||||
const {
|
||||
itemTitle,
|
||||
itemLink,
|
||||
extractCandidates,
|
||||
captionForUuid,
|
||||
postTitleFromHtml,
|
||||
} = require("./html-text-utils.js");
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
function flag(name) {
|
||||
const i = args.indexOf(name);
|
||||
return i !== -1 ? (args[i + 1] && !args[i + 1].startsWith("--") ? args[i + 1] : true) : undefined;
|
||||
}
|
||||
const uuid = flag("--uuid");
|
||||
const deep = !!flag("--deep");
|
||||
|
||||
if (!uuid || uuid === true) {
|
||||
console.error("Usage: node find-substack-post.js --uuid <uuid> [--deep]");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const CONFIG = path.resolve(__dirname, "./config/substack-publications.json");
|
||||
function loadPublications() {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(CONFIG, "utf-8"));
|
||||
} catch {
|
||||
return ["blog.bytebytego.com"];
|
||||
}
|
||||
}
|
||||
|
||||
// Thin wrapper over the shared fetcher: returns body text, or "" on any error.
|
||||
async function fetchText(url) {
|
||||
const res = await fetchWithTimeout(url, { headers: { "User-Agent": "Mozilla/5.0" } });
|
||||
return res && res.ok ? await res.text() : "";
|
||||
}
|
||||
|
||||
// Total post fetches allowed across ALL publications during a --deep crawl.
|
||||
const DEEP_FETCH_BUDGET = 40;
|
||||
|
||||
// Deep fallback: crawl the sitemap back ~3 months, fetch posts most-recent-first
|
||||
// (up to `maxFetch` from the shared budget), and look for the UUID. Heavier than
|
||||
// RSS — only used on RSS miss.
|
||||
async function searchSitemap(publication, id, maxFetch, monthsBack = 3) {
|
||||
const xml = await fetchText(`https://${publication}/sitemap.xml`);
|
||||
if (!xml) return { hit: null, scanned: 0, cutoff: null };
|
||||
|
||||
const cutoff = new Date();
|
||||
cutoff.setMonth(cutoff.getMonth() - monthsBack);
|
||||
|
||||
const candidates = [];
|
||||
for (const block of xml.split("<url>").slice(1)) {
|
||||
const loc = block.match(/<loc>([^<]+)<\/loc>/);
|
||||
const lastmod = block.match(/<lastmod>([^<]+)<\/lastmod>/);
|
||||
if (!loc || !lastmod) continue;
|
||||
if (!/\/p\//.test(loc[1])) continue; // posts only
|
||||
const when = new Date(lastmod[1]);
|
||||
if (isNaN(when) || when < cutoff) continue;
|
||||
candidates.push({ url: loc[1], when });
|
||||
}
|
||||
candidates.sort((a, b) => b.when - a.when);
|
||||
|
||||
let scanned = 0;
|
||||
for (const c of candidates.slice(0, maxFetch)) {
|
||||
scanned++;
|
||||
const html = await fetchText(c.url);
|
||||
if (!html || !html.includes(id)) continue;
|
||||
return {
|
||||
hit: {
|
||||
found: true,
|
||||
source: "sitemap",
|
||||
publication,
|
||||
postTitle: postTitleFromHtml(html),
|
||||
postUrl: c.url,
|
||||
caption: captionForUuid(html, id),
|
||||
candidates: extractCandidates(html),
|
||||
},
|
||||
scanned,
|
||||
cutoff: cutoff.toISOString().slice(0, 10),
|
||||
};
|
||||
}
|
||||
return { hit: null, scanned, cutoff: cutoff.toISOString().slice(0, 10) };
|
||||
}
|
||||
|
||||
async function searchRss(publication, id) {
|
||||
const xml = await fetchText(`https://${publication}/feed`);
|
||||
if (!xml) return null;
|
||||
const items = xml.split("<item>").slice(1);
|
||||
for (const item of items) {
|
||||
if (!item.includes(id)) continue;
|
||||
const postTitle = itemTitle(item);
|
||||
const caption = captionForUuid(item, id);
|
||||
return {
|
||||
found: true,
|
||||
source: "rss",
|
||||
publication,
|
||||
postTitle,
|
||||
postUrl: itemLink(item),
|
||||
caption,
|
||||
candidates: extractCandidates(item),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const publications = loadPublications();
|
||||
for (const pub of publications) {
|
||||
const hit = await searchRss(pub, uuid);
|
||||
if (hit) return console.log(JSON.stringify(hit, null, 2));
|
||||
}
|
||||
|
||||
// Deep fallback: sitemap crawl up to ~3 months back, sharing one global
|
||||
// fetch budget across all publications so coverage can't blow up as the
|
||||
// publications list grows.
|
||||
if (deep) {
|
||||
let totalScanned = 0;
|
||||
let lastCutoff = null;
|
||||
for (const pub of publications) {
|
||||
const remaining = DEEP_FETCH_BUDGET - totalScanned;
|
||||
if (remaining <= 0) break;
|
||||
const { hit, scanned, cutoff } = await searchSitemap(pub, uuid, remaining);
|
||||
totalScanned += scanned;
|
||||
lastCutoff = cutoff || lastCutoff;
|
||||
if (hit) return console.log(JSON.stringify(hit, null, 2));
|
||||
}
|
||||
return console.log(JSON.stringify({
|
||||
found: false,
|
||||
source: "sitemap",
|
||||
scanned: totalScanned,
|
||||
budget: DEEP_FETCH_BUDGET,
|
||||
cutoff: lastCutoff,
|
||||
}, null, 2));
|
||||
}
|
||||
console.log(JSON.stringify({ found: false }, null, 2));
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,91 @@
|
||||
// HTML / RSS text-extraction helpers for find-substack-post.js.
|
||||
// Pure string functions (no network, no fs) — kept separate so the crawler
|
||||
// stays focused on fetch/search flow. CommonJS so plain `node` works.
|
||||
|
||||
// Decode the HTML entities that appear in Substack titles/captions, including
|
||||
// numeric (') and hex (—) forms. & is decoded last-ish but
|
||||
// before nothing re-encodes it; ordering here is safe for our inputs.
|
||||
function decodeEntities(s) {
|
||||
return s
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/�?39;|'/g, "'")
|
||||
.replace(/ /g, " ")
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCodePoint(parseInt(n, 16)))
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n))
|
||||
.trim();
|
||||
}
|
||||
|
||||
function stripTags(html) {
|
||||
return decodeEntities(html.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
// Pull the first <title> (CDATA or plain) and <link> from an RSS <item> chunk.
|
||||
function itemTitle(item) {
|
||||
const m = item.match(/<title>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/title>/);
|
||||
return m ? decodeEntities(m[1].trim()) : "";
|
||||
}
|
||||
function itemLink(item) {
|
||||
const m = item.match(/<link>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/link>/);
|
||||
return m ? m[1].trim() : "";
|
||||
}
|
||||
|
||||
// Extract candidate topic titles from a post's TOC bullet list. ByteByteGo does
|
||||
// not attach captions to images — the topic titles live only in the "in this
|
||||
// issue" bullets. We can't reliably map image→title automatically (sponsor /
|
||||
// video items interleave and break positional order), so we surface these
|
||||
// candidates for the user to pick from. Light filtering keeps the list short:
|
||||
// dedupe, drop sub-point explanations ("Term: long sentence") and over-long
|
||||
// lines. The correct title is always present; the user selects it.
|
||||
function extractCandidates(html) {
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
const re = /<li[^>]*>\s*(?:<p[^>]*>)?([\s\S]*?)(?:<\/p>)?\s*<\/li>/g;
|
||||
let m;
|
||||
while ((m = re.exec(html))) {
|
||||
const text = stripTags(m[1]);
|
||||
if (!text) continue;
|
||||
if (text.length < 6 || text.length > 70) continue; // titles are short; long lines are sub-point explanations
|
||||
const key = text.toLowerCase();
|
||||
if (seen.has(key)) continue; // content is duplicated in the page
|
||||
seen.add(key);
|
||||
out.push(text);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// If the UUID sits inside a <figure>…</figure>, return that figure's
|
||||
// <figcaption> text. Cover images live in <enclosure> (no figure) → "".
|
||||
function captionForUuid(item, id) {
|
||||
const at = item.indexOf(id);
|
||||
if (at === -1) return "";
|
||||
const figStart = item.lastIndexOf("<figure", at);
|
||||
if (figStart === -1) return "";
|
||||
const figEnd = item.indexOf("</figure>", at);
|
||||
if (figEnd === -1) return "";
|
||||
const figure = item.slice(figStart, figEnd);
|
||||
const cap = figure.match(/<figcaption[^>]*>([\s\S]*?)<\/figcaption>/);
|
||||
return cap ? stripTags(cap[1]) : "";
|
||||
}
|
||||
|
||||
// Extract a post title from server-rendered post HTML (og:title preferred).
|
||||
function postTitleFromHtml(html) {
|
||||
const og = html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i);
|
||||
if (og) return decodeEntities(og[1]);
|
||||
const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
|
||||
if (h1) return stripTags(h1[1]);
|
||||
const t = html.match(/<title>([\s\S]*?)<\/title>/i);
|
||||
return t ? decodeEntities(t[1].trim()) : "";
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
decodeEntities,
|
||||
stripTags,
|
||||
itemTitle,
|
||||
itemLink,
|
||||
extractCandidates,
|
||||
captionForUuid,
|
||||
postTitleFromHtml,
|
||||
};
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env node
|
||||
// List existing tags in the repo ranked by frequency
|
||||
// Usage: node list-existing-tags.js
|
||||
// Outputs: tag count and name, sorted most-used first (top 40)
|
||||
//
|
||||
// NOTE: This script is NOT currently used by the mt-add-tags skill.
|
||||
// Tag normalization is disabled until existing posts have standardized tags.
|
||||
// To enable, uncomment step 4a in SKILL.md.
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const PROJECT_ROOT = path.resolve(__dirname, "../..");
|
||||
const CONTENT_DIR = path.join(PROJECT_ROOT, "content", "post");
|
||||
|
||||
// Recursively find all index.md files
|
||||
function findIndexFiles(dir) {
|
||||
const results = [];
|
||||
try {
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
results.push(...findIndexFiles(fullPath));
|
||||
} else if (entry.name === "index.md") {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// skip unreadable directories
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// Extract tags from frontmatter
|
||||
function extractTags(filePath) {
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, "utf-8");
|
||||
const match = content.match(/^tags:\s*\[([^\]]*)\]/m);
|
||||
if (!match) return [];
|
||||
// Extract quoted strings from the tags array
|
||||
return [...match[1].matchAll(/"([^"]+)"/g)].map((m) => m[1]);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Count tag frequency
|
||||
const tagCounts = new Map();
|
||||
const files = findIndexFiles(CONTENT_DIR);
|
||||
|
||||
for (const file of files) {
|
||||
for (const tag of extractTags(file)) {
|
||||
tagCounts.set(tag, (tagCounts.get(tag) || 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by frequency descending, take top 40
|
||||
const sorted = [...tagCounts.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 40);
|
||||
|
||||
for (const [tag, count] of sorted) {
|
||||
console.log(`${String(count).padStart(6)} ${tag}`);
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
// Shared URL helpers for the mt-* newsletter skills.
|
||||
// Owned by the mt-add-url meta router; reused by handlers (e.g. mt-add-image).
|
||||
// CommonJS so plain `node script.js` works without a build step.
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
// Remove common tracking parameters (utm_* plus a fixed set of known trackers).
|
||||
function cleanUrl(rawUrl) {
|
||||
const EXACT_TRACKING = new Set([
|
||||
"fbclid", "gclid", "msclkid", "mc_eid",
|
||||
"aid", "ref", "ref_src", "ref_url", "source", "s",
|
||||
"ck_subscriber_id", "igshid", "yclid", "vero_id",
|
||||
]);
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
[...parsed.searchParams.keys()].forEach((k) => {
|
||||
if (k.toLowerCase().startsWith("utm_") || EXACT_TRACKING.has(k.toLowerCase())) {
|
||||
parsed.searchParams.delete(k);
|
||||
}
|
||||
});
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
// Unparseable input (not a real URL): the per-param string surgery above
|
||||
// would mangle the query (drop the `?`, leave a dangling `&`), so leave it
|
||||
// untouched rather than corrupt it.
|
||||
return rawUrl;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Substack image helpers (shared by add-url.js routing and mt-add-image) ---
|
||||
const SUBSTACK_IMAGE_HOSTS = ["substackcdn.com", "substack-post-media.s3.amazonaws.com"];
|
||||
const UUID = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
|
||||
|
||||
// A Substack-hosted image (CDN wrapper or raw S3), regardless of file extension.
|
||||
function isSubstackImage(targetUrl) {
|
||||
let host = "";
|
||||
try { host = new URL(targetUrl).host.toLowerCase(); } catch { /* non-URL */ }
|
||||
return SUBSTACK_IMAGE_HOSTS.includes(host) || /substack-post-media/i.test(targetUrl);
|
||||
}
|
||||
|
||||
// The stable image identity is the S3 image UUID under public/images/<uuid>.
|
||||
// Works whether the path separators are raw (/) or percent-encoded (%2F).
|
||||
function substackImageUuid(targetUrl) {
|
||||
const m = targetUrl.match(new RegExp(`images(?:%2F|/)(${UUID})`, "i"));
|
||||
return m ? m[1].toLowerCase() : null;
|
||||
}
|
||||
|
||||
// Some sites carry the resource identity in a query param, not the path
|
||||
// (e.g. YouTube /watch?v=ID). Stripping the query for these collapses every
|
||||
// item to the same bare URL, causing false-positive duplicates. Preserve the
|
||||
// identity param for those hosts.
|
||||
const IDENTITY_PARAMS = {
|
||||
"youtube.com": "v",
|
||||
"www.youtube.com": "v",
|
||||
"m.youtube.com": "v",
|
||||
};
|
||||
|
||||
// Reduce a URL to a stable identity used for duplicate detection:
|
||||
// - Substack image → its S3 UUID (transform/size variants share one identity)
|
||||
// - YouTube → scheme+host+path + the v= video id
|
||||
// - everything else → scheme + host + path
|
||||
function bareUrl(targetUrl) {
|
||||
if (isSubstackImage(targetUrl)) {
|
||||
const uuid = substackImageUuid(targetUrl);
|
||||
if (uuid) return uuid;
|
||||
}
|
||||
try {
|
||||
const p = new URL(targetUrl);
|
||||
let bare = `${p.protocol}//${p.host}${p.pathname}`.replace(/\/$/, "");
|
||||
const idParam = IDENTITY_PARAMS[p.host.toLowerCase()];
|
||||
const idValue = idParam ? p.searchParams.get(idParam) : null;
|
||||
if (idValue) bare += `?${idParam}=${idValue}`;
|
||||
return bare;
|
||||
} catch {
|
||||
return targetUrl.split("?")[0].replace(/\/$/, "");
|
||||
}
|
||||
}
|
||||
|
||||
// fetch() with an abort timeout. Returns the Response on success, or null on
|
||||
// network error / timeout. Callers decide what to read (.text/.json/.status).
|
||||
// Centralizes the AbortController + clearTimeout dance so every caller cleans
|
||||
// up the timer (via finally) on both the success and failure paths.
|
||||
async function fetchWithTimeout(targetUrl, { method = "GET", timeoutMs = 10000, headers } = {}) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
return await fetch(targetUrl, { method, redirect: "follow", signal: controller.signal, headers });
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
// Check if URL is accessible (returns HTTP status code as a string).
|
||||
async function checkAccessibility(targetUrl) {
|
||||
const res = await fetchWithTimeout(targetUrl, { method: "HEAD" });
|
||||
return res ? res.status.toString() : "000";
|
||||
}
|
||||
|
||||
function escapeRegExp(s) {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
// Recursively collect *.md files under a directory (the content tree is small).
|
||||
function collectMarkdown(dir, acc = []) {
|
||||
let entries;
|
||||
try {
|
||||
entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return acc; // missing dir → nothing to compare against
|
||||
}
|
||||
for (const e of entries) {
|
||||
const full = path.join(dir, e.name);
|
||||
if (e.isDirectory()) collectMarkdown(full, acc);
|
||||
else if (e.isFile() && e.name.toLowerCase().endsWith(".md")) acc.push(full);
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
// Check whether a URL identity already exists in the stored markdown.
|
||||
// Pure JS (no external `grep` dependency, works the same from any shell) and
|
||||
// boundary-aware so a needle that is merely a PREFIX of a stored longer string
|
||||
// is NOT a false duplicate. The two identity kinds need different boundaries:
|
||||
// - Substack image UUID: the next char must not extend the hex id, so all of
|
||||
// <uuid>.png (cover image, no size suffix), <uuid>_WxH and <uuid>) match.
|
||||
// - URL: must be followed by a path/punctuation delimiter so /p/foo does not
|
||||
// match a stored /p/foo-bar.
|
||||
function checkDuplicate(targetUrl, contentDir) {
|
||||
const needle = bareUrl(targetUrl);
|
||||
if (!needle) return false;
|
||||
const isUuid = new RegExp(`^${UUID}$`, "i").test(needle);
|
||||
// For URLs, allow an optional trailing slash (bareUrl strips it, stored URLs
|
||||
// may keep it) before the delimiter.
|
||||
const boundary = isUuid
|
||||
? new RegExp(escapeRegExp(needle) + `(?![0-9a-f])`, "i")
|
||||
: new RegExp(escapeRegExp(needle) + `/?(?:[)\\]\\s"'?#<_&,]|$)`, "m");
|
||||
for (const file of collectMarkdown(contentDir)) {
|
||||
let text;
|
||||
try { text = fs.readFileSync(file, "utf-8"); } catch { continue; }
|
||||
if (text.includes(needle) && boundary.test(text)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Classify URL type by file extension. Returns image|video|document|article.
|
||||
function classifyType(targetUrl) {
|
||||
const lower = targetUrl.toLowerCase();
|
||||
if (/\.(png|jpg|jpeg|gif|webp|svg|avif|heic|heif|bmp|tiff?)(\?.*)?$/.test(lower)) return "image";
|
||||
if (/\.(mp4|webm|mov|avi|mkv)(\?.*)?$/.test(lower)) return "video";
|
||||
if (/\.(pdf|docx?|xlsx?|pptx?)(\?.*)?$/.test(lower)) return "document";
|
||||
return "article";
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
cleanUrl,
|
||||
bareUrl,
|
||||
IDENTITY_PARAMS,
|
||||
isSubstackImage,
|
||||
substackImageUuid,
|
||||
fetchWithTimeout,
|
||||
checkAccessibility,
|
||||
checkDuplicate,
|
||||
classifyType,
|
||||
};
|
||||
Reference in New Issue
Block a user