mirror of
https://github.com/tiennm99/miti99.git
synced 2026-09-02 14:21:01 +00:00
refactor(scripts): migrate newsletter engine from Node to Go
One Go binary with a subcommand per former script, invoked as 'go run ./scripts/newsletter <command>' from repo root. Stdlib only; publications config embedded via go:embed (go run recompiles on edit). Behavior parity verified side-by-side before deleting the JS — see plans/reports/parity-260818-newsletter-go-migration-report.md. Permissions: allow 'Bash(go *)', drop 'Bash(node *)' and the redundant 'Write(content)' rule (Edit rules cover all file-editing tools).
This commit is contained in:
@@ -10,10 +10,10 @@
|
||||
"Bash(find *)",
|
||||
"Bash(git add *)",
|
||||
"Bash(git commit *)",
|
||||
"Bash(go *)",
|
||||
"Bash(ls *)",
|
||||
"Bash(mkdir *)",
|
||||
"Bash(mv *)",
|
||||
"Bash(node *)",
|
||||
"Bash(npm install *)",
|
||||
"Bash(pip install *)",
|
||||
"Bash(pip3 install *)",
|
||||
@@ -27,7 +27,6 @@
|
||||
"Skill(update-config)",
|
||||
"WebFetch(*)",
|
||||
"WebSearch(*)",
|
||||
"Write(content)",
|
||||
"mcp__acp__Bash",
|
||||
"mcp__acp__Edit",
|
||||
"mcp__acp__Write",
|
||||
|
||||
@@ -21,24 +21,24 @@ A clean image URL (passed by `mt-add-url`, or given directly).
|
||||
|
||||
### 1. Detect source
|
||||
```bash
|
||||
node scripts/newsletter/detect-image-source.js "<url>"
|
||||
go run ./scripts/newsletter detect-image-source "<url>"
|
||||
```
|
||||
→ `{ original_url, clean_url, isSubstack, uuid?, innerUrl? }`.
|
||||
|
||||
When invoked **directly** (not via `mt-add-url`), first run the router to get accessibility + duplicate status and skip accordingly:
|
||||
```bash
|
||||
node scripts/newsletter/add-url.js "<url>" # expect route:image; skip if duplicate/!accessible
|
||||
go run ./scripts/newsletter add-url "<url>" # expect route:image; skip if duplicate/!accessible
|
||||
```
|
||||
(When dispatched by `mt-add-url`, that check already ran — don't repeat it.)
|
||||
|
||||
### 2a. Substack image (`isSubstack: true` with `uuid`)
|
||||
Find the source post:
|
||||
```bash
|
||||
node scripts/newsletter/find-substack-post.js --uuid <uuid>
|
||||
go run ./scripts/newsletter find-substack-post --uuid <uuid>
|
||||
```
|
||||
- `found: false` → retry with the deeper sitemap crawl (slower — fetches posts ~3 months back, capped at 40 fetches total across all publications; warn the user it may take a while):
|
||||
```bash
|
||||
node scripts/newsletter/find-substack-post.js --uuid <uuid> --deep
|
||||
go run ./scripts/newsletter find-substack-post --uuid <uuid> --deep
|
||||
```
|
||||
On a miss the result reports `scanned` (posts fetched), `budget` (the 40-fetch cap), and `cutoff` (oldest date looked at) — mention how far back it looked.
|
||||
- `found: false` after `--deep` → no source post; go to step 3 (ask) and/or step 4 (add publication).
|
||||
|
||||
@@ -13,7 +13,7 @@ Shared scripts: `scripts/newsletter/`. Shared procedure: `../mt-add-url/referenc
|
||||
|
||||
A clean article URL (passed by `mt-add-url`, or given directly). If a raw URL is provided directly, you may run the classifier to clean/dedup it first:
|
||||
```bash
|
||||
node scripts/newsletter/add-url.js "<url>"
|
||||
go run ./scripts/newsletter add-url "<url>"
|
||||
```
|
||||
Trust `route: article`; skip if `duplicate` or not `accessible`.
|
||||
|
||||
|
||||
@@ -50,7 +50,7 @@ Read the full post body. Identify:
|
||||
|
||||
Before generating, run:
|
||||
```bash
|
||||
node scripts/newsletter/list-existing-tags.js
|
||||
go run ./scripts/newsletter list-existing-tags
|
||||
```
|
||||
When a proposed tag matches an existing one case-insensitively, use the existing casing.
|
||||
-->
|
||||
|
||||
@@ -20,7 +20,7 @@ Everything else (direct `video` file, `document`, or anything unrecognized) is *
|
||||
|
||||
For every URL the user provides:
|
||||
```bash
|
||||
node scripts/newsletter/add-url.js "<url>"
|
||||
go run ./scripts/newsletter add-url "<url>"
|
||||
```
|
||||
Output (JSON): `{ original_url, clean_url, http_status, accessible, duplicate, route, title?, author? }`.
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ Get current date in `YYYY-MM-DD` (UTC+7). Check `content/post/YYYY/MM/DD/index.m
|
||||
## 2. Newsletter number
|
||||
|
||||
```bash
|
||||
node scripts/newsletter/find-newsletter-number.js
|
||||
go run ./scripts/newsletter find-newsletter-number
|
||||
```
|
||||
Searches backwards from today for the most recent newsletter and returns the next number. Only needed when **creating** a new post.
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ A clean YouTube URL (passed by `mt-add-url`, or given directly).
|
||||
|
||||
1. **Classify / fetch title** — run the router to get the canonical URL + title:
|
||||
```bash
|
||||
node scripts/newsletter/add-url.js "<url>"
|
||||
go run ./scripts/newsletter add-url "<url>"
|
||||
```
|
||||
Confirm `route: youtube`; skip if `duplicate` or not `accessible`. Use the returned `clean_url` (canonical `watch?v=ID`) and `title`.
|
||||
- If `title` is missing (oEmbed failed), fetch the title via WebFetch on the watch URL.
|
||||
|
||||
@@ -28,7 +28,7 @@ Use this skill only after a WebFetch attempt returned one of:
|
||||
1. Confirm WebFetch already failed on the target URL
|
||||
2. Run the fetch script:
|
||||
```bash
|
||||
node scripts/newsletter/fetch-via-defuddle.js "<target_url>"
|
||||
go run ./scripts/newsletter fetch-via-defuddle "<target_url>"
|
||||
```
|
||||
Alternatively, use WebFetch with the defuddle-prefixed URL:
|
||||
```
|
||||
@@ -70,7 +70,7 @@ Never loop. Never retry more than once.
|
||||
User wanted to extract content from https://example.com/article
|
||||
WebFetch returned: "Request failed with status code 403"
|
||||
→ Trigger mt-webfetch
|
||||
→ node scripts/newsletter/fetch-via-defuddle.js "https://example.com/article"
|
||||
→ go run ./scripts/newsletter fetch-via-defuddle "https://example.com/article"
|
||||
→ Parse markdown output
|
||||
→ Summarize as usual
|
||||
```
|
||||
|
||||
@@ -39,15 +39,15 @@ The site will be available at `http://localhost:1313`
|
||||
|
||||
## Shared Engine
|
||||
|
||||
All portable newsletter scripts live in **`scripts/newsletter/`** and are invoked from the repo root with plain Node (stdlib only, no deps):
|
||||
The portable newsletter engine lives in **`scripts/newsletter/`** (Go, stdlib only, no deps — one binary, one subcommand per task) and is invoked from the repo root with `go run`:
|
||||
|
||||
```bash
|
||||
node scripts/newsletter/add-url.js "<url>" # classify + dedup a URL → JSON route
|
||||
node scripts/newsletter/find-newsletter-number.js # next newsletter number
|
||||
node scripts/newsletter/list-existing-tags.js # existing tag frequencies
|
||||
node scripts/newsletter/detect-image-source.js "<url>" # detect Substack image + uuid
|
||||
node scripts/newsletter/find-substack-post.js --uuid <uuid>
|
||||
node scripts/newsletter/fetch-via-defuddle.js "<url>" # fallback fetch (defuddle proxy)
|
||||
go run ./scripts/newsletter add-url "<url>" # classify + dedup a URL → JSON route
|
||||
go run ./scripts/newsletter find-newsletter-number # next newsletter number
|
||||
go run ./scripts/newsletter list-existing-tags # existing tag frequencies
|
||||
go run ./scripts/newsletter detect-image-source "<url>" # detect Substack image + uuid
|
||||
go run ./scripts/newsletter find-substack-post --uuid <uuid>
|
||||
go run ./scripts/newsletter fetch-via-defuddle "<url>" # fallback fetch (defuddle proxy)
|
||||
```
|
||||
|
||||
These are shared by all three tools — no tool-specific copies.
|
||||
|
||||
@@ -39,7 +39,7 @@ The Stack theme is pulled in as a git submodule under `themes/hugo-theme-stack/`
|
||||
|
||||
## Working with AI tools
|
||||
|
||||
This repo runs from Claude Code, OpenCode, or Codex off one shared script engine (`scripts/newsletter/`) and one instruction file (`AGENTS.md`). Repository-scoped Codex skills use the official `.agents/skills/` format; no installer is required. Setup, invocation per tool, and how to pick one and remove the rest: see [docs/multi-tool-usage.md](docs/multi-tool-usage.md).
|
||||
This repo runs from Claude Code, OpenCode, or Codex off one shared engine (`scripts/newsletter/`, Go via `go run`) and one instruction file (`AGENTS.md`). Repository-scoped Codex skills use the official `.agents/skills/` format; no installer is required. Setup, invocation per tool, and how to pick one and remove the rest: see [docs/multi-tool-usage.md](docs/multi-tool-usage.md).
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Multi-tool usage (Claude Code · OpenCode · Codex)
|
||||
|
||||
This repo is usable from three AI coding tools off **one shared engine**. The newsletter scripts live in `scripts/newsletter/` and every tool calls them as `node scripts/newsletter/*.js` from the repo root. Project instructions live once in `AGENTS.md`.
|
||||
This repo is usable from three AI coding tools off **one shared engine**. The newsletter engine lives in `scripts/newsletter/` (Go, stdlib only) and every tool calls it as `go run ./scripts/newsletter <command>` from the repo root. Project instructions live once in `AGENTS.md`.
|
||||
|
||||
## Per-tool setup & invocation
|
||||
|
||||
@@ -10,7 +10,7 @@ This repo is usable from three AI coding tools off **one shared engine**. The ne
|
||||
| **OpenCode** | `AGENTS.md` (auto-read) | `.claude/skills/` auto-discovered via the `skill` tool, governed by `opencode.json` | None beyond `opencode.json` (committed) | Ask to add a URL; pick/`skill` `mt-add-url` |
|
||||
| **Codex** | `AGENTS.md` (auto-read) | Repository skills discovered from `.agents/skills/` | None (works as-is) | Ask to add a URL or invoke `$mt-add-url` |
|
||||
|
||||
**Shared engine:** all three call `node scripts/newsletter/*.js` from repo root — no per-tool script copies.
|
||||
**Shared engine:** all three call `go run ./scripts/newsletter <command>` from repo root — no per-tool script copies.
|
||||
|
||||
### Notes per tool
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
# Phase 1 — Port to Go
|
||||
|
||||
## Context
|
||||
|
||||
- Source: `scripts/newsletter/*.js` (8 files, ~740 LoC, Node stdlib only, CommonJS).
|
||||
- Read first: all JS files, `AGENTS.md` (Shared Engine section), `docs/multi-tool-usage.md`.
|
||||
- Go 1.26.5 available (linux/arm64). No existing `go.mod` anywhere in the repo.
|
||||
|
||||
## Files to Create
|
||||
|
||||
```
|
||||
go.mod module github.com/tiennm99/miti99, go 1.26
|
||||
scripts/newsletter/main.go subcommand dispatch + usage text
|
||||
scripts/newsletter/url_utils.go cleanUrl, bareUrl, isSubstackImage, substackImageUuid,
|
||||
httpGet/httpHead helpers, checkAccessibility,
|
||||
checkDuplicate, classifyType, collectMarkdown
|
||||
scripts/newsletter/html_text.go stripTags, itemTitle, itemLink, extractCandidates,
|
||||
captionForUuid, postTitleFromHtml
|
||||
scripts/newsletter/add_url.go add-url: YouTube detect, oEmbed meta, route JSON
|
||||
scripts/newsletter/detect_image_source.go detect-image-source: CDN unwrap + uuid JSON
|
||||
scripts/newsletter/find_substack_post.go find-substack-post: RSS search, --deep sitemap crawl
|
||||
scripts/newsletter/find_newsletter_number.go find-newsletter-number: scan YYYY/MM/DD, print max+1
|
||||
scripts/newsletter/list_existing_tags.go list-existing-tags: frontmatter tag frequency top-40
|
||||
scripts/newsletter/fetch_via_defuddle.go fetch-via-defuddle: proxy fetch, exit codes 0/1/2
|
||||
```
|
||||
|
||||
`scripts/newsletter/config/substack-publications.json` — unchanged, loaded via `go:embed` (fallback `["blog.bytebytego.com"]` on parse error, matching JS).
|
||||
|
||||
All files are one `package main`; no `internal/` packaging (KISS — this is a script bundle, not a library).
|
||||
|
||||
## Implementation Steps
|
||||
|
||||
1. `go.mod` at repo root; verify `hugo` build still works untouched (it should — no `[module]` config in use).
|
||||
2. `main.go`: `os.Args[1]` switch → handler funcs; unknown/missing subcommand prints usage to stderr, exit 1.
|
||||
3. Port `url-utils.js` → `url_utils.go`. See parity traps below — this file has all the hard ones.
|
||||
4. Port `html-text-utils.js` → `html_text.go`. Use `html.UnescapeString` (stdlib) instead of hand-rolled `decodeEntities` — strictly more complete, acceptable improvement.
|
||||
5. Port the 6 entrypoints. JSON output via structs + `json.MarshalIndent(v, "", " ")` with field order matching the JS key order; optional fields (`title`, `author`, `uuid`, `innerUrl`, `caption`…) use `omitempty` to reproduce JS's conditional key emission. **Skills parse these JSON shapes — field names are a public contract.**
|
||||
6. `gofmt` + `go vet ./scripts/newsletter/`.
|
||||
|
||||
## Parity Traps (must handle explicitly)
|
||||
|
||||
| # | JS behavior | Go trap | Resolution |
|
||||
|---|-------------|---------|------------|
|
||||
| 1 | `checkDuplicate` boundary regexes use negative lookahead `(?![0-9a-f])` | RE2 has **no lookahead** | Scan `strings.Index` occurrences of the needle; check the following byte(s) in code: uuid → next char ∉ `[0-9a-f]`; URL → optional `/` then one of `)]"'?#<_&,` whitespace or end |
|
||||
| 2 | `cleanUrl` rebuilds query via `URLSearchParams` preserving insertion order | `url.Values.Encode()` **sorts keys alphabetically** | Split `RawQuery` on `&`, drop `utm_*`/tracker keys (case-insensitive match on the key before `=`), rejoin the surviving pairs verbatim. Closer to "don't corrupt" than JS re-encoding; document any percent-encoding drift in phase 2 |
|
||||
| 3 | `new URL()` lowercases host in output | Go keeps host casing as parsed | Lowercase host manually when rebuilding (`clean_url`, `bareUrl`) |
|
||||
| 4 | `[\s\S]*?` in figure/caption/candidate regexes | Go `.` excludes newline by default | Use `(?s)` flag |
|
||||
| 5 | `new Date(lastmod)` accepts RFC3339 and date-only | `time.Parse` needs explicit layouts | Try `time.RFC3339`, then `"2006-01-02"`; unparseable → skip entry (JS `isNaN` path) |
|
||||
| 6 | `fetch` HEAD, 10s timeout, follows redirects; network error → status `"000"` | — | `http.Client{Timeout: 10s}` (follows redirects by default); any error → `"000"` |
|
||||
| 7 | defuddle: 30s timeout, UA `mt-webfetch/1.0`, exit 0/1/2, raw body to stdout, diagnostics to stderr | — | Same client pattern; `os.Exit` codes identical |
|
||||
| 8 | `--uuid <v> --deep` flag parsing | — | stdlib `flag` on a subcommand FlagSet (`--uuid v` and `--uuid=v` both accepted); missing uuid → usage + exit 1 |
|
||||
| 9 | uuid/substack regexes are case-insensitive | — | `(?i)` prefix |
|
||||
| 10 | oEmbed URL built with `encodeURIComponent` | — | `url.QueryEscape` |
|
||||
| 11 | Deep-crawl shared budget `DEEP_FETCH_BUDGET = 40` across publications; miss JSON includes `scanned`, `budget`, `cutoff` (ISO date) | — | Same constant, same miss shape; cutoff `time.Format("2006-01-02")` |
|
||||
| 12 | `classifyType` extension regexes allow trailing `?query` | — | Port regex as-is (RE2-safe) |
|
||||
|
||||
## Validation
|
||||
|
||||
- `go vet` clean; each subcommand runs without args → same usage/exit behavior as its JS counterpart.
|
||||
- Spot-run each subcommand once (real output sanity, not yet full parity — that is phase 2).
|
||||
|
||||
## Risks / Rollback
|
||||
|
||||
- No call sites change in this phase; JS remains the live engine. Rollback = delete the `.go` files and `go.mod`.
|
||||
@@ -0,0 +1,38 @@
|
||||
# Phase 2 — Parity Verification (JS vs Go side-by-side)
|
||||
|
||||
## Context
|
||||
|
||||
Both engines coexist. For each case below, run `node scripts/newsletter/<script>.js …` and `go run ./scripts/newsletter <cmd> …` with identical args and `diff` the outputs. Work from repo root. Use the scratchpad for output capture — no files in the repo.
|
||||
|
||||
## Verification Matrix
|
||||
|
||||
**Deterministic (must be byte-identical):**
|
||||
|
||||
| Command | Cases |
|
||||
|---------|-------|
|
||||
| `find-newsletter-number` | no args (repo content as-is) |
|
||||
| `list-existing-tags` | no args — full 40-line ranking identical |
|
||||
| `detect-image-source` | (a) real substackcdn wrapper URL pulled from an existing post, (b) raw S3 URL, (c) non-Substack image URL, (d) garbage non-URL |
|
||||
| `add-url` routing/dedup fields | compare all fields except `http_status`/`accessible` timing flakes — rerun on mismatch |
|
||||
|
||||
**Network-dependent (identical modulo transient upstream changes):**
|
||||
|
||||
| Command | Cases |
|
||||
|---------|-------|
|
||||
| `add-url` | (a) YouTube watch URL, (b) `youtu.be` short link, (c) `/shorts/` URL — all three must emit the same canonical `clean_url` + oEmbed `title`/`author`; (d) article URL with `utm_*`+`fbclid` params (check `clean_url`); (e) URL already present in `content/post` (must report `duplicate: true`); (f) Substack image URL (route `image`); (g) `.pdf` URL (route `document`) |
|
||||
| `find-substack-post` | (a) uuid from a recent ByteByteGo post (grep an existing post's markdown for a substack uuid) → RSS hit shape; (b) fabricated uuid → `{ found: false }`; (c) fabricated uuid with `--deep` → miss shape with `scanned`/`budget`/`cutoff` |
|
||||
| `fetch-via-defuddle` | (a) fetchable article — compare exit code + first ~40 lines of body; (b) unreachable URL — exit 1, stderr message |
|
||||
|
||||
**Duplicate-boundary regression (the lookahead rewrite):** craft two checks against real repo content — a stored URL where the probe is a strict prefix (`/p/foo` vs stored `/p/foo-bar`, must be `duplicate: false`) and a stored substack uuid probed exactly (must be `true`).
|
||||
|
||||
## Acceptable Drift (document, don't fix)
|
||||
|
||||
- `clean_url` percent-encoding differences from query-rebuild strategy (phase-1 trap #2), as long as `bareUrl`-based dedup results agree.
|
||||
- Entity decoding: `html.UnescapeString` decodes entities the JS map missed — Go output may be *more* correct in `postTitle`/`caption`/`candidates`.
|
||||
|
||||
Any other divergence is a bug: fix Go, rerun the failing case.
|
||||
|
||||
## Exit Criteria
|
||||
|
||||
- Matrix fully executed; results table (case → identical / drift-documented / fixed) recorded in `plans/reports/parity-260818-newsletter-go-migration-report.md`.
|
||||
- One real end-to-end `mt-add-url` flow executed manually against the Go engine (temporarily invoke the Go command in place of the node line) to prove the JSON contract holds for skill consumption.
|
||||
@@ -0,0 +1,41 @@
|
||||
# Phase 3 — Cutover, Docs, Cleanup
|
||||
|
||||
## Context
|
||||
|
||||
Phase 2 passed. Replace every `node scripts/newsletter/<name>.js` invocation with `go run ./scripts/newsletter <name-without-.js>`, update prose, delete JS.
|
||||
|
||||
## Call Sites to Update (verified by grep, 2026-08-18)
|
||||
|
||||
| File | Lines | Change |
|
||||
|------|-------|--------|
|
||||
| `.claude/skills/mt-add-url/SKILL.md` | 23 | `add-url` invocation |
|
||||
| `.claude/skills/mt-add-post/SKILL.md` | 16 | `add-url` invocation |
|
||||
| `.claude/skills/mt-add-video/SKILL.md` | 22 | `add-url` invocation |
|
||||
| `.claude/skills/mt-add-image/SKILL.md` | 24, 30, 37, 41 | `detect-image-source`, `add-url`, `find-substack-post` (×2, incl. `--deep`) |
|
||||
| `.claude/skills/mt-add-tags/SKILL.md` | 53 | `list-existing-tags` invocation |
|
||||
| `.claude/skills/mt-webfetch/SKILL.md` | 31, 73 | `fetch-via-defuddle` invocation |
|
||||
| `.claude/skills/mt-add-url/references/newsletter-post-mechanics.md` | 21 | `find-newsletter-number` invocation |
|
||||
| `AGENTS.md` | 42–52 | Shared Engine block: command list + prose "plain Node (stdlib only, no deps)" → "Go (stdlib only, `go run`, no deps)" |
|
||||
| `README.md` | ~42 | shared-engine sentence mentioning node |
|
||||
| `docs/multi-tool-usage.md` | 3, 13 | "all three call `node scripts/newsletter/*.js`" → Go invocation |
|
||||
|
||||
`.agents/skills/` Codex adapters reference the `.claude/skills/` SKILL.md files, not the scripts directly — verify with a final grep, no direct edits expected.
|
||||
|
||||
## Steps
|
||||
|
||||
1. Update all call sites above (mechanical string replacement per file; keep surrounding comments like `# expect route:image` intact).
|
||||
2. Add `"Bash(go *)"` to `.claude/settings.json` permissions allow (alphabetical position). Leave `Bash(node *)` pending the plan's open question 1.
|
||||
3. Delete `scripts/newsletter/*.js` (all 8). Keep `config/substack-publications.json`.
|
||||
4. Verify: `grep -rn "node scripts/newsletter\|newsletter/[a-z-]*\.js" .claude .agents AGENTS.md README.md docs scripts` → only the go:embed reference to the config JSON may remain.
|
||||
5. Run each of the 6 Go subcommands once from repo root as a smoke test.
|
||||
6. `hugo --quiet` (or `hugo server` spot check) to confirm the site build is untouched by root `go.mod`.
|
||||
7. Commit as `refactor(scripts): migrate newsletter engine from Node to Go` (conventional, no AI references). Run the pre-commit tag check per Git Workflow Rules — no post content changes expected, so it should be a no-op.
|
||||
|
||||
## Docs Policy Check
|
||||
|
||||
Per documentation-management rules this change affects commands + architecture docs → `AGENTS.md`, `README.md`, `docs/multi-tool-usage.md` updates above are required; no changelog file exists in this repo, so nothing more.
|
||||
|
||||
## Risks / Rollback
|
||||
|
||||
- Risk: an AI tool (Codex/OpenCode) with a cold Go build cache sees a slow first `go run` (~1–3 s compile). Acceptable; subsequent runs are cached.
|
||||
- Rollback: `git revert` the cutover commit — JS engine and call sites restore together (single commit contains both).
|
||||
@@ -0,0 +1,36 @@
|
||||
# Migrate Newsletter Scripts from Node.js to Go
|
||||
|
||||
**Status:** COMPLETED (2026-08-18) — all 3 phases done; parity report in `plans/reports/parity-260818-newsletter-go-migration-report.md`
|
||||
**Created:** 2026-08-18
|
||||
**Scope:** Port the 8-file shared engine in `scripts/newsletter/` (~740 LoC JS, stdlib-only) to a single Go package with 6 subcommands, verify behavior parity, cut over all skill/doc call sites, delete the JS.
|
||||
|
||||
## Design Decisions
|
||||
|
||||
- **One `package main` with subcommands** at `scripts/newsletter/`, not 6 separate binaries. Subcommand names mirror the old script names (`add-url`, `find-newsletter-number`, `list-existing-tags`, `detect-image-source`, `find-substack-post`, `fetch-via-defuddle`) so skills stay greppable.
|
||||
- **`go.mod` at repo root** (`module github.com/tiennm99/miti99`). Hugo does not use Go modules here (theme vendored in `themes/`, no `[module]` config), so this is inert to the site build. Gives the simplest documented invocation: `go run ./scripts/newsletter <cmd> [args]` from repo root. (Alternative considered: module inside `scripts/newsletter/` + `go run -C scripts/newsletter . <cmd>` — works, verified, but clunkier in 8 documented call sites.)
|
||||
- **`go run`, no committed binary.** Build cache makes reruns fast; no stale-binary risk across the 3 AI tools.
|
||||
- **`go:embed` for `config/substack-publications.json`.** `go run` recompiles on change, so editing the JSON still takes effect; the file stays the editable source of truth. Solves the `__dirname` problem (`os.Executable()` under `go run` points at the build cache, not the source dir).
|
||||
- **Repo paths resolve from CWD** (`content/post`). The documented contract is already "invoked from the repo root"; missing dir degrades gracefully exactly like the JS (`0` / no duplicates).
|
||||
|
||||
## Phases
|
||||
|
||||
| Phase | File | Depends on |
|
||||
|-------|------|------------|
|
||||
| 1. Port to Go | [phase-01-port-to-go.md](phase-01-port-to-go.md) | — |
|
||||
| 2. Parity verification | [phase-02-parity-verification.md](phase-02-parity-verification.md) | 1 |
|
||||
| 3. Cutover, docs, cleanup | [phase-03-cutover-and-docs.md](phase-03-cutover-and-docs.md) | 2 |
|
||||
|
||||
JS and Go coexist during phases 1–2 (Go toolchain ignores `.js` files in the package dir); JS is deleted only after phase 2 passes.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- All 6 subcommands match the Node output on the phase-02 verification matrix (byte-identical JSON, or drift explicitly documented as acceptable).
|
||||
- One real `mt-add-url` skill flow works end-to-end on the Go engine.
|
||||
- `grep -rn "node scripts/newsletter"` across the repo returns nothing (skills, AGENTS.md, README.md, docs/).
|
||||
- All `.js` files under `scripts/newsletter/` deleted; `config/substack-publications.json` retained.
|
||||
- `.claude/settings.json` allows `Bash(go *)`.
|
||||
|
||||
## Resolved Decisions (user, 2026-08-18)
|
||||
|
||||
1. Drop the `Bash(node *)` allow rule at cutover (phase 3).
|
||||
2. Module path: `github.com/tiennm99/miti99` confirmed.
|
||||
@@ -0,0 +1,36 @@
|
||||
# Parity Report — Newsletter Engine JS → Go (2026-08-18)
|
||||
|
||||
Method: identical args to `node scripts/newsletter/<script>.js` and `go run ./scripts/newsletter <cmd>`, `diff` on stdout, exit codes compared. Live network cases against real upstreams.
|
||||
|
||||
## Results
|
||||
|
||||
| Case | Result |
|
||||
|------|--------|
|
||||
| find-newsletter-number | IDENTICAL |
|
||||
| list-existing-tags (full 40-line ranking) | IDENTICAL |
|
||||
| detect-image-source: substackcdn wrapper (real URL from posts) | IDENTICAL |
|
||||
| detect-image-source: raw S3 URL | IDENTICAL |
|
||||
| detect-image-source: non-Substack image w/ utm param | IDENTICAL |
|
||||
| detect-image-source: garbage non-URL | IDENTICAL |
|
||||
| add-url: YouTube watch / youtu.be / shorts (same video id) | IDENTICAL ×3; all canonicalize to same watch URL, oEmbed title+author present |
|
||||
| add-url: stored article + `utm_source`+`fbclid`+`x=1` | IDENTICAL; trackers dropped, `x=1` kept, `duplicate: true` |
|
||||
| add-url: prefix probe `/p/goclaw-30-mot-buoc` vs stored `…-ngoat-lon` | IDENTICAL; `duplicate: false` — lookahead→index-scan rewrite verified |
|
||||
| add-url: stored S3 image (uuid dedup) | IDENTICAL; `route: image`, `duplicate: true` |
|
||||
| add-url: `.pdf` | IDENTICAL; `route: document` |
|
||||
| find-substack-post: live ByteByteGo RSS uuid | IDENTICAL (hit shape incl. candidates list) |
|
||||
| find-substack-post: fabricated uuid | IDENTICAL (`{found:false}`) |
|
||||
| find-substack-post: fabricated uuid `--deep` | IDENTICAL (miss shape: scanned/budget/cutoff) |
|
||||
| fetch-via-defuddle: example.com | exit 0=0, body byte-identical |
|
||||
| fetch-via-defuddle: unresolvable host | exit 1=1 |
|
||||
| fetch-via-defuddle: no args | node 2, go-run 1 — see drift |
|
||||
|
||||
## Accepted Drift
|
||||
|
||||
1. **`go run` collapses nonzero exits to 1.** In-program codes (1 fail / 2 bad-args) are correct in a compiled binary, but `go run` reports any child failure as 1. No skill distinguishes 1 vs 2 (verified mt-webfetch SKILL.md acts on stderr/body, not codes). Noted in `fetch_via_defuddle.go` header.
|
||||
2. **Entity decoding**: Go uses stdlib `html.UnescapeString` (superset of the JS hand-rolled map). No divergence observed on live data; Go can only be more correct on exotic entities.
|
||||
3. **`clean_url` query strategy**: Go keeps surviving query pairs verbatim (no re-encode) vs JS re-serialization. No divergence observed in the matrix; could differ on already-percent-encoded params. Dedup unaffected (path/identity-based).
|
||||
4. **list-existing-tags tie order**: Go walks lexically (deterministic); Node readdir order is FS-dependent. Full output was identical on current content; a future tie could order differently between engines — moot once JS is deleted.
|
||||
|
||||
## Verdict
|
||||
|
||||
All matrix cases pass. Go engine is behavior-equivalent for skill consumption. Proceed to cutover.
|
||||
@@ -1,112 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// Meta URL router for the mt-add-url skill — the single entry per URL.
|
||||
// Usage: node add-url.js "<url>"
|
||||
// Outputs: JSON { original_url, clean_url, http_status, accessible,
|
||||
// duplicate, route, title?, author? }
|
||||
// route ∈ youtube | image | video | document | article
|
||||
|
||||
const path = require("path");
|
||||
const {
|
||||
cleanUrl,
|
||||
isSubstackImage,
|
||||
fetchWithTimeout,
|
||||
checkAccessibility,
|
||||
checkDuplicate,
|
||||
classifyType,
|
||||
} = require("./url-utils");
|
||||
|
||||
const url = process.argv[2];
|
||||
if (!url) {
|
||||
console.error("Usage: node add-url.js <url>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const PROJECT_ROOT = path.resolve(__dirname, "../..");
|
||||
const CONTENT_DIR = path.join(PROJECT_ROOT, "content", "post");
|
||||
|
||||
const YT_HOSTS = new Set(["youtube.com", "www.youtube.com", "m.youtube.com"]);
|
||||
|
||||
// Detect a YouTube video and extract its id from the supported URL shapes:
|
||||
// youtube.com/watch?v=ID, youtu.be/ID, youtube.com/shorts/ID
|
||||
// Playlists/channels are intentionally NOT YouTube routes (fall through to type).
|
||||
function detectYouTube(targetUrl) {
|
||||
try {
|
||||
const p = new URL(targetUrl);
|
||||
const host = p.host.toLowerCase();
|
||||
if (host === "youtu.be") {
|
||||
const id = p.pathname.slice(1).split("/")[0];
|
||||
return id ? { isYouTube: true, videoId: id } : { isYouTube: false };
|
||||
}
|
||||
if (YT_HOSTS.has(host)) {
|
||||
if (p.pathname === "/watch") {
|
||||
const id = p.searchParams.get("v");
|
||||
return id ? { isYouTube: true, videoId: id } : { isYouTube: false };
|
||||
}
|
||||
if (p.pathname.startsWith("/shorts/")) {
|
||||
const id = p.pathname.split("/")[2];
|
||||
return id ? { isYouTube: true, videoId: id } : { isYouTube: false };
|
||||
}
|
||||
}
|
||||
return { isYouTube: false };
|
||||
} catch {
|
||||
return { isYouTube: false };
|
||||
}
|
||||
}
|
||||
|
||||
// Canonical watch URL — oEmbed accepts watch URLs reliably for all shapes.
|
||||
function canonicalWatchUrl(videoId) {
|
||||
return `https://www.youtube.com/watch?v=${videoId}`;
|
||||
}
|
||||
|
||||
// Fetch title/author via YouTube oEmbed (no API key). Best-effort: any failure
|
||||
// returns {} so the route stays `youtube` and the skill can fall back.
|
||||
async function fetchYouTubeMeta(watchUrl) {
|
||||
const endpoint = `https://www.youtube.com/oembed?url=${encodeURIComponent(watchUrl)}&format=json`;
|
||||
const res = await fetchWithTimeout(endpoint);
|
||||
if (!res || !res.ok) return {};
|
||||
try {
|
||||
const data = await res.json();
|
||||
return { title: data.title, author: data.author_name };
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const cleanedUrl = cleanUrl(url);
|
||||
const yt = detectYouTube(cleanedUrl);
|
||||
|
||||
// For YouTube, dedup/store against the canonical watch URL so youtu.be and
|
||||
// shorts links collapse onto the same identity-param key as watch URLs.
|
||||
const effectiveUrl = yt.isYouTube ? canonicalWatchUrl(yt.videoId) : cleanedUrl;
|
||||
// Route order: YouTube → Substack image (by host, not extension, so f_auto /
|
||||
// .avif / .heic / extensionless CDN URLs still route to the image handler) →
|
||||
// file-extension classification.
|
||||
let route;
|
||||
if (yt.isYouTube) route = "youtube";
|
||||
else if (isSubstackImage(cleanedUrl)) route = "image";
|
||||
else route = classifyType(cleanedUrl);
|
||||
|
||||
const httpStatus = await checkAccessibility(cleanedUrl);
|
||||
const accessible = httpStatus === "200";
|
||||
const duplicate = checkDuplicate(effectiveUrl, CONTENT_DIR);
|
||||
|
||||
const out = {
|
||||
original_url: url,
|
||||
clean_url: effectiveUrl,
|
||||
http_status: httpStatus,
|
||||
accessible,
|
||||
duplicate,
|
||||
route,
|
||||
};
|
||||
|
||||
if (route === "youtube") {
|
||||
const meta = await fetchYouTubeMeta(effectiveUrl);
|
||||
if (meta.title) out.title = meta.title;
|
||||
if (meta.author) out.author = meta.author;
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(out, null, 2));
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,138 @@
|
||||
// Meta URL router for the mt-add-url skill — the single entry per URL.
|
||||
// Usage: go run ./scripts/newsletter add-url "<url>"
|
||||
// Outputs: JSON { original_url, clean_url, http_status, accessible,
|
||||
//
|
||||
// duplicate, route, title?, author? }
|
||||
//
|
||||
// route ∈ youtube | image | video | document | article
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// contentDir is repo-root relative (invocation contract: run from repo root).
|
||||
func contentDir() string {
|
||||
return filepath.Join("content", "post")
|
||||
}
|
||||
|
||||
var ytHosts = map[string]bool{
|
||||
"youtube.com": true,
|
||||
"www.youtube.com": true,
|
||||
"m.youtube.com": true,
|
||||
}
|
||||
|
||||
// detectYouTube extracts a video id from the supported URL shapes:
|
||||
// youtube.com/watch?v=ID, youtu.be/ID, youtube.com/shorts/ID.
|
||||
// Playlists/channels are intentionally NOT YouTube routes (fall through to type).
|
||||
func detectYouTube(target string) (bool, string) {
|
||||
u, err := url.Parse(target)
|
||||
if err != nil {
|
||||
return false, ""
|
||||
}
|
||||
host := strings.ToLower(u.Host)
|
||||
if host == "youtu.be" {
|
||||
id := strings.SplitN(strings.TrimPrefix(u.Path, "/"), "/", 2)[0]
|
||||
return id != "", id
|
||||
}
|
||||
if ytHosts[host] {
|
||||
if u.Path == "/watch" {
|
||||
id := u.Query().Get("v")
|
||||
return id != "", id
|
||||
}
|
||||
if strings.HasPrefix(u.Path, "/shorts/") {
|
||||
parts := strings.Split(u.Path, "/")
|
||||
if len(parts) > 2 && parts[2] != "" {
|
||||
return true, parts[2]
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// canonicalWatchURL — oEmbed accepts watch URLs reliably for all shapes.
|
||||
func canonicalWatchURL(videoID string) string {
|
||||
return "https://www.youtube.com/watch?v=" + videoID
|
||||
}
|
||||
|
||||
// fetchYouTubeMeta fetches title/author via YouTube oEmbed (no API key).
|
||||
// Best-effort: any failure returns empty strings so the route stays `youtube`
|
||||
// and the skill can fall back.
|
||||
func fetchYouTubeMeta(watchURL string) (title, author string) {
|
||||
endpoint := "https://www.youtube.com/oembed?url=" + url.QueryEscape(watchURL) + "&format=json"
|
||||
body := fetchTextOK(endpoint, 10*time.Second, "")
|
||||
if body == "" {
|
||||
return "", ""
|
||||
}
|
||||
var data struct {
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author_name"`
|
||||
}
|
||||
if json.Unmarshal([]byte(body), &data) != nil {
|
||||
return "", ""
|
||||
}
|
||||
return data.Title, data.Author
|
||||
}
|
||||
|
||||
type addURLOutput struct {
|
||||
OriginalURL string `json:"original_url"`
|
||||
CleanURL string `json:"clean_url"`
|
||||
HTTPStatus string `json:"http_status"`
|
||||
Accessible bool `json:"accessible"`
|
||||
Duplicate bool `json:"duplicate"`
|
||||
Route string `json:"route"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Author string `json:"author,omitempty"`
|
||||
}
|
||||
|
||||
func runAddURL(args []string) {
|
||||
if len(args) < 1 || args[0] == "" {
|
||||
fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter add-url <url>")
|
||||
os.Exit(1)
|
||||
}
|
||||
target := args[0]
|
||||
|
||||
cleaned := cleanURL(target)
|
||||
isYT, videoID := detectYouTube(cleaned)
|
||||
|
||||
// For YouTube, dedup/store against the canonical watch URL so youtu.be and
|
||||
// shorts links collapse onto the same identity-param key as watch URLs.
|
||||
effectiveURL := cleaned
|
||||
if isYT {
|
||||
effectiveURL = canonicalWatchURL(videoID)
|
||||
}
|
||||
|
||||
// Route order: YouTube → Substack image (by host, not extension, so f_auto /
|
||||
// .avif / .heic / extensionless CDN URLs still route to the image handler) →
|
||||
// file-extension classification.
|
||||
var route string
|
||||
switch {
|
||||
case isYT:
|
||||
route = "youtube"
|
||||
case isSubstackImage(cleaned):
|
||||
route = "image"
|
||||
default:
|
||||
route = classifyType(cleaned)
|
||||
}
|
||||
|
||||
httpStatus := checkAccessibility(cleaned)
|
||||
|
||||
out := addURLOutput{
|
||||
OriginalURL: target,
|
||||
CleanURL: effectiveURL,
|
||||
HTTPStatus: httpStatus,
|
||||
Accessible: httpStatus == "200",
|
||||
Duplicate: checkDuplicate(effectiveURL, contentDir()),
|
||||
Route: route,
|
||||
}
|
||||
if route == "youtube" {
|
||||
out.Title, out.Author = fetchYouTubeMeta(effectiveURL)
|
||||
}
|
||||
printJSON(out)
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// Detect whether an image URL is Substack-hosted and extract its S3 image UUID.
|
||||
// Usage: node detect-image-source.js "<image-url>"
|
||||
// Output: JSON { original_url, clean_url, isSubstack, uuid?, innerUrl? }
|
||||
//
|
||||
// Substack images are usually served via a CDN wrapper:
|
||||
// https://substackcdn.com/image/fetch/$s_!x!,.../https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F<uuid>_WxH.png
|
||||
// The publication is NOT encoded in the URL — only the image identity (uuid) is.
|
||||
|
||||
const {
|
||||
cleanUrl,
|
||||
isSubstackImage,
|
||||
substackImageUuid,
|
||||
} = require("./url-utils.js");
|
||||
|
||||
const url = process.argv[2];
|
||||
if (!url) {
|
||||
console.error("Usage: node detect-image-source.js <image-url>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Pull the inner S3 URL out of a substackcdn /image/fetch/ wrapper (if present).
|
||||
function extractInnerUrl(targetUrl) {
|
||||
const marker = targetUrl.indexOf("/https%3A%2F%2F");
|
||||
if (marker !== -1) return decodeURIComponent(targetUrl.slice(marker + 1));
|
||||
// Some forms embed a plain (already-decoded) inner https URL.
|
||||
const plain = targetUrl.indexOf("/https://", 8);
|
||||
if (plain !== -1) return targetUrl.slice(plain + 1);
|
||||
return targetUrl;
|
||||
}
|
||||
|
||||
const cleaned = cleanUrl(url);
|
||||
const isSubstack = isSubstackImage(url);
|
||||
const uuid = isSubstack ? substackImageUuid(url) : null;
|
||||
const innerUrl = isSubstack ? extractInnerUrl(url) : null;
|
||||
|
||||
console.log(JSON.stringify({
|
||||
original_url: url,
|
||||
clean_url: cleaned,
|
||||
isSubstack,
|
||||
...(uuid ? { uuid } : {}),
|
||||
...(innerUrl ? { innerUrl } : {}),
|
||||
}, null, 2));
|
||||
@@ -0,0 +1,63 @@
|
||||
// Detect whether an image URL is Substack-hosted and extract its S3 image UUID.
|
||||
// Usage: go run ./scripts/newsletter detect-image-source "<image-url>"
|
||||
// Output: JSON { original_url, clean_url, isSubstack, uuid?, innerUrl? }
|
||||
//
|
||||
// Substack images are usually served via a CDN wrapper:
|
||||
//
|
||||
// https://substackcdn.com/image/fetch/$s_!x!,.../https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F<uuid>_WxH.png
|
||||
//
|
||||
// The publication is NOT encoded in the URL — only the image identity (uuid) is.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// extractInnerURL pulls the inner S3 URL out of a substackcdn /image/fetch/
|
||||
// wrapper (if present).
|
||||
func extractInnerURL(target string) string {
|
||||
if marker := strings.Index(target, "/https%3A%2F%2F"); marker != -1 {
|
||||
decoded, err := url.PathUnescape(target[marker+1:])
|
||||
if err != nil {
|
||||
return target[marker+1:]
|
||||
}
|
||||
return decoded
|
||||
}
|
||||
// Some forms embed a plain (already-decoded) inner https URL.
|
||||
if len(target) > 8 {
|
||||
if plain := strings.Index(target[8:], "/https://"); plain != -1 {
|
||||
return target[8+plain+1:]
|
||||
}
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
type detectImageOutput struct {
|
||||
OriginalURL string `json:"original_url"`
|
||||
CleanURL string `json:"clean_url"`
|
||||
IsSubstack bool `json:"isSubstack"`
|
||||
UUID string `json:"uuid,omitempty"`
|
||||
InnerURL string `json:"innerUrl,omitempty"`
|
||||
}
|
||||
|
||||
func runDetectImageSource(args []string) {
|
||||
if len(args) < 1 || args[0] == "" {
|
||||
fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter detect-image-source <image-url>")
|
||||
os.Exit(1)
|
||||
}
|
||||
target := args[0]
|
||||
|
||||
out := detectImageOutput{
|
||||
OriginalURL: target,
|
||||
CleanURL: cleanURL(target),
|
||||
IsSubstack: isSubstackImage(target),
|
||||
}
|
||||
if out.IsSubstack {
|
||||
out.UUID = substackImageUUID(target)
|
||||
out.InnerURL = extractInnerURL(target)
|
||||
}
|
||||
printJSON(out)
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// mt-webfetch fallback fetcher via defuddle.md
|
||||
// Usage: node fetch.js <target_url>
|
||||
// Exit codes: 0 = content returned, 1 = empty/failed, 2 = bad args
|
||||
|
||||
const url = process.argv[2];
|
||||
if (!url) {
|
||||
console.error("Usage: node fetch.js <target_url>");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const defuddleUrl = `https://defuddle.md/${url}`;
|
||||
|
||||
async function main() {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 30000);
|
||||
try {
|
||||
const res = await fetch(defuddleUrl, {
|
||||
redirect: "follow",
|
||||
signal: controller.signal,
|
||||
headers: { "User-Agent": "mt-webfetch/1.0" },
|
||||
});
|
||||
clearTimeout(timeout);
|
||||
const body = await res.text();
|
||||
if (!res.ok || !body || body.trim().length === 0) {
|
||||
console.error(`mt-webfetch: defuddle returned ${res.status} / empty body for ${url}`);
|
||||
process.exit(1);
|
||||
}
|
||||
process.stdout.write(body);
|
||||
} catch (err) {
|
||||
clearTimeout(timeout);
|
||||
console.error(`mt-webfetch: fetch failed for ${url}: ${err.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,51 @@
|
||||
// mt-webfetch fallback fetcher via defuddle.md.
|
||||
// Usage: go run ./scripts/newsletter fetch-via-defuddle <target_url>
|
||||
// Exit codes: 0 = content returned, 1 = empty/failed, 2 = bad args.
|
||||
// Note: `go run` collapses any nonzero program exit to 1, so callers invoking
|
||||
// through `go run` can only distinguish success from failure — the stderr
|
||||
// diagnostics carry the detail.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func runFetchViaDefuddle(args []string) {
|
||||
if len(args) < 1 || args[0] == "" {
|
||||
fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter fetch-via-defuddle <target_url>")
|
||||
os.Exit(2)
|
||||
}
|
||||
target := args[0]
|
||||
defuddleURL := "https://defuddle.md/" + target
|
||||
|
||||
fail := func(err error) {
|
||||
fmt.Fprintf(os.Stderr, "mt-webfetch: fetch failed for %s: %v\n", target, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, defuddleURL, nil)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "mt-webfetch/1.0")
|
||||
res, err := (&http.Client{Timeout: 30 * time.Second}).Do(req)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
ok := res.StatusCode >= 200 && res.StatusCode < 300
|
||||
if !ok || strings.TrimSpace(string(body)) == "" {
|
||||
fmt.Fprintf(os.Stderr, "mt-webfetch: defuddle returned %d / empty body for %s\n", res.StatusCode, target)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Stdout.Write(body)
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// Find the most recent newsletter number and return the next one
|
||||
// Usage: node find-newsletter-number.js
|
||||
// Outputs: The next newsletter number
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const PROJECT_ROOT = path.resolve(__dirname, "../..");
|
||||
const CONTENT_DIR = path.join(PROJECT_ROOT, "content", "post");
|
||||
|
||||
// Extract newsletter number from file content
|
||||
function extractNewsletterNumber(filePath) {
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, "utf-8");
|
||||
const match = content.match(/Newsletter\s*#(\d+)/);
|
||||
return match ? parseInt(match[1], 10) : 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Scan all year/month/day directories for newsletter posts
|
||||
function findMostRecentNewsletter() {
|
||||
let maxNumber = 0;
|
||||
|
||||
// List year directories, sorted descending
|
||||
let years;
|
||||
try {
|
||||
years = fs.readdirSync(CONTENT_DIR)
|
||||
.filter((d) => /^\d{4}$/.test(d))
|
||||
.sort((a, b) => b.localeCompare(a));
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (const year of years) {
|
||||
const yearDir = path.join(CONTENT_DIR, year);
|
||||
let months;
|
||||
try {
|
||||
months = fs.readdirSync(yearDir)
|
||||
.filter((d) => /^\d{2}$/.test(d))
|
||||
.sort((a, b) => b.localeCompare(a));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const month of months) {
|
||||
const monthDir = path.join(yearDir, month);
|
||||
let days;
|
||||
try {
|
||||
days = fs.readdirSync(monthDir)
|
||||
.filter((d) => /^\d{2}$/.test(d))
|
||||
.sort((a, b) => b.localeCompare(a));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const day of days) {
|
||||
const filePath = path.join(monthDir, day, "index.md");
|
||||
const num = extractNewsletterNumber(filePath);
|
||||
if (num > maxNumber) maxNumber = num;
|
||||
}
|
||||
}
|
||||
|
||||
// Early exit: if we found a newsletter in this year, no need to go further back
|
||||
if (maxNumber > 0) break;
|
||||
}
|
||||
|
||||
return maxNumber;
|
||||
}
|
||||
|
||||
const mostRecent = findMostRecentNewsletter();
|
||||
console.log(mostRecent + 1);
|
||||
@@ -1,151 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// Find which Substack post embeds a given image UUID, and extract a label.
|
||||
// Usage: node find-substack-post.js --uuid <uuid> [--deep]
|
||||
// Output on hit: JSON { found:true, source, publication, postTitle, postUrl, caption, candidates }
|
||||
// Output on miss: JSON { found:false } (RSS) or { found:false, source:"sitemap", scanned, budget, cutoff } (--deep)
|
||||
//
|
||||
// Strategy: RSS feed first (fast, ~recent weeks). With --deep, fall back to a
|
||||
// heavier sitemap crawl up to ~3 months back — opt-in because it fetches many
|
||||
// posts. A Substack CDN URL does not encode its publication, so we search each
|
||||
// publication listed in config/substack-publications.json.
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { fetchWithTimeout } = require("./url-utils.js");
|
||||
const {
|
||||
itemTitle,
|
||||
itemLink,
|
||||
extractCandidates,
|
||||
captionForUuid,
|
||||
postTitleFromHtml,
|
||||
} = require("./html-text-utils.js");
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
function flag(name) {
|
||||
const i = args.indexOf(name);
|
||||
return i !== -1 ? (args[i + 1] && !args[i + 1].startsWith("--") ? args[i + 1] : true) : undefined;
|
||||
}
|
||||
const uuid = flag("--uuid");
|
||||
const deep = !!flag("--deep");
|
||||
|
||||
if (!uuid || uuid === true) {
|
||||
console.error("Usage: node find-substack-post.js --uuid <uuid> [--deep]");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const CONFIG = path.resolve(__dirname, "./config/substack-publications.json");
|
||||
function loadPublications() {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(CONFIG, "utf-8"));
|
||||
} catch {
|
||||
return ["blog.bytebytego.com"];
|
||||
}
|
||||
}
|
||||
|
||||
// Thin wrapper over the shared fetcher: returns body text, or "" on any error.
|
||||
async function fetchText(url) {
|
||||
const res = await fetchWithTimeout(url, { headers: { "User-Agent": "Mozilla/5.0" } });
|
||||
return res && res.ok ? await res.text() : "";
|
||||
}
|
||||
|
||||
// Total post fetches allowed across ALL publications during a --deep crawl.
|
||||
const DEEP_FETCH_BUDGET = 40;
|
||||
|
||||
// Deep fallback: crawl the sitemap back ~3 months, fetch posts most-recent-first
|
||||
// (up to `maxFetch` from the shared budget), and look for the UUID. Heavier than
|
||||
// RSS — only used on RSS miss.
|
||||
async function searchSitemap(publication, id, maxFetch, monthsBack = 3) {
|
||||
const xml = await fetchText(`https://${publication}/sitemap.xml`);
|
||||
if (!xml) return { hit: null, scanned: 0, cutoff: null };
|
||||
|
||||
const cutoff = new Date();
|
||||
cutoff.setMonth(cutoff.getMonth() - monthsBack);
|
||||
|
||||
const candidates = [];
|
||||
for (const block of xml.split("<url>").slice(1)) {
|
||||
const loc = block.match(/<loc>([^<]+)<\/loc>/);
|
||||
const lastmod = block.match(/<lastmod>([^<]+)<\/lastmod>/);
|
||||
if (!loc || !lastmod) continue;
|
||||
if (!/\/p\//.test(loc[1])) continue; // posts only
|
||||
const when = new Date(lastmod[1]);
|
||||
if (isNaN(when) || when < cutoff) continue;
|
||||
candidates.push({ url: loc[1], when });
|
||||
}
|
||||
candidates.sort((a, b) => b.when - a.when);
|
||||
|
||||
let scanned = 0;
|
||||
for (const c of candidates.slice(0, maxFetch)) {
|
||||
scanned++;
|
||||
const html = await fetchText(c.url);
|
||||
if (!html || !html.includes(id)) continue;
|
||||
return {
|
||||
hit: {
|
||||
found: true,
|
||||
source: "sitemap",
|
||||
publication,
|
||||
postTitle: postTitleFromHtml(html),
|
||||
postUrl: c.url,
|
||||
caption: captionForUuid(html, id),
|
||||
candidates: extractCandidates(html),
|
||||
},
|
||||
scanned,
|
||||
cutoff: cutoff.toISOString().slice(0, 10),
|
||||
};
|
||||
}
|
||||
return { hit: null, scanned, cutoff: cutoff.toISOString().slice(0, 10) };
|
||||
}
|
||||
|
||||
async function searchRss(publication, id) {
|
||||
const xml = await fetchText(`https://${publication}/feed`);
|
||||
if (!xml) return null;
|
||||
const items = xml.split("<item>").slice(1);
|
||||
for (const item of items) {
|
||||
if (!item.includes(id)) continue;
|
||||
const postTitle = itemTitle(item);
|
||||
const caption = captionForUuid(item, id);
|
||||
return {
|
||||
found: true,
|
||||
source: "rss",
|
||||
publication,
|
||||
postTitle,
|
||||
postUrl: itemLink(item),
|
||||
caption,
|
||||
candidates: extractCandidates(item),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const publications = loadPublications();
|
||||
for (const pub of publications) {
|
||||
const hit = await searchRss(pub, uuid);
|
||||
if (hit) return console.log(JSON.stringify(hit, null, 2));
|
||||
}
|
||||
|
||||
// Deep fallback: sitemap crawl up to ~3 months back, sharing one global
|
||||
// fetch budget across all publications so coverage can't blow up as the
|
||||
// publications list grows.
|
||||
if (deep) {
|
||||
let totalScanned = 0;
|
||||
let lastCutoff = null;
|
||||
for (const pub of publications) {
|
||||
const remaining = DEEP_FETCH_BUDGET - totalScanned;
|
||||
if (remaining <= 0) break;
|
||||
const { hit, scanned, cutoff } = await searchSitemap(pub, uuid, remaining);
|
||||
totalScanned += scanned;
|
||||
lastCutoff = cutoff || lastCutoff;
|
||||
if (hit) return console.log(JSON.stringify(hit, null, 2));
|
||||
}
|
||||
return console.log(JSON.stringify({
|
||||
found: false,
|
||||
source: "sitemap",
|
||||
scanned: totalScanned,
|
||||
budget: DEEP_FETCH_BUDGET,
|
||||
cutoff: lastCutoff,
|
||||
}, null, 2));
|
||||
}
|
||||
console.log(JSON.stringify({ found: false }, null, 2));
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,77 @@
|
||||
// Find the most recent newsletter number and return the next one.
|
||||
// Usage: go run ./scripts/newsletter find-newsletter-number
|
||||
// Outputs: the next newsletter number.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
var (
|
||||
newsletterNumRe = regexp.MustCompile(`Newsletter\s*#(\d+)`)
|
||||
yearDirRe = regexp.MustCompile(`^\d{4}$`)
|
||||
twoDigitDirRe = regexp.MustCompile(`^\d{2}$`)
|
||||
)
|
||||
|
||||
// listDirsDesc returns dir's subdirectory names matching re, sorted descending.
|
||||
func listDirsDesc(dir string, re *regexp.Regexp) []string {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
if re.MatchString(e.Name()) {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
}
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(names)))
|
||||
return names
|
||||
}
|
||||
|
||||
func extractNewsletterNumber(path string) int {
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
m := newsletterNumRe.FindSubmatch(content)
|
||||
if m == nil {
|
||||
return 0
|
||||
}
|
||||
n, err := strconv.Atoi(string(m[1]))
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// findMostRecentNewsletter scans year/month/day directories newest-first for
|
||||
// the highest newsletter number.
|
||||
func findMostRecentNewsletter() int {
|
||||
maxNumber := 0
|
||||
for _, year := range listDirsDesc(contentDir(), yearDirRe) {
|
||||
yearDir := filepath.Join(contentDir(), year)
|
||||
for _, month := range listDirsDesc(yearDir, twoDigitDirRe) {
|
||||
monthDir := filepath.Join(yearDir, month)
|
||||
for _, day := range listDirsDesc(monthDir, twoDigitDirRe) {
|
||||
if n := extractNewsletterNumber(filepath.Join(monthDir, day, "index.md")); n > maxNumber {
|
||||
maxNumber = n
|
||||
}
|
||||
}
|
||||
}
|
||||
// Early exit: a newsletter found in this year — no need to go further back.
|
||||
if maxNumber > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return maxNumber
|
||||
}
|
||||
|
||||
func runFindNewsletterNumber(_ []string) {
|
||||
fmt.Println(findMostRecentNewsletter() + 1)
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
// Find which Substack post embeds a given image UUID, and extract a label.
|
||||
// Usage: go run ./scripts/newsletter find-substack-post --uuid <uuid> [--deep]
|
||||
// Output on hit: JSON { found:true, source, publication, postTitle, postUrl, caption, candidates }
|
||||
// Output on miss: JSON { found:false } (RSS) or { found:false, source:"sitemap", scanned, budget, cutoff } (--deep)
|
||||
//
|
||||
// Strategy: RSS feed first (fast, ~recent weeks). With --deep, fall back to a
|
||||
// heavier sitemap crawl up to ~3 months back — opt-in because it fetches many
|
||||
// posts. A Substack CDN URL does not encode its publication, so we search each
|
||||
// publication listed in config/substack-publications.json (embedded at build
|
||||
// time; `go run` recompiles on change, so editing the JSON still takes effect).
|
||||
package main
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
//go:embed config/substack-publications.json
|
||||
var publicationsJSON []byte
|
||||
|
||||
func loadPublications() []string {
|
||||
var pubs []string
|
||||
if err := json.Unmarshal(publicationsJSON, &pubs); err != nil || len(pubs) == 0 {
|
||||
return []string{"blog.bytebytego.com"}
|
||||
}
|
||||
return pubs
|
||||
}
|
||||
|
||||
// fetchPage: body text with a browser-ish UA, or "" on any error.
|
||||
func fetchPage(target string) string {
|
||||
return fetchTextOK(target, 10*time.Second, "Mozilla/5.0")
|
||||
}
|
||||
|
||||
// Total post fetches allowed across ALL publications during a --deep crawl.
|
||||
const deepFetchBudget = 40
|
||||
|
||||
type postHit struct {
|
||||
Found bool `json:"found"`
|
||||
Source string `json:"source"`
|
||||
Publication string `json:"publication"`
|
||||
PostTitle string `json:"postTitle"`
|
||||
PostURL string `json:"postUrl"`
|
||||
Caption string `json:"caption"`
|
||||
Candidates []string `json:"candidates"`
|
||||
}
|
||||
|
||||
var (
|
||||
locRe = regexp.MustCompile(`<loc>([^<]+)</loc>`)
|
||||
lastmodRe = regexp.MustCompile(`<lastmod>([^<]+)</lastmod>`)
|
||||
)
|
||||
|
||||
func parseLastmod(s string) (time.Time, bool) {
|
||||
for _, layout := range []string{time.RFC3339, "2006-01-02"} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
// searchSitemap is the deep fallback: crawl the sitemap back ~3 months, fetch
|
||||
// posts most-recent-first (up to maxFetch from the shared budget), and look
|
||||
// for the UUID. Heavier than RSS — only used on RSS miss.
|
||||
// cutoff is "" when the sitemap itself could not be fetched.
|
||||
func searchSitemap(publication, id string, maxFetch int) (hit *postHit, scanned int, cutoff string) {
|
||||
xml := fetchPage("https://" + publication + "/sitemap.xml")
|
||||
if xml == "" {
|
||||
return nil, 0, ""
|
||||
}
|
||||
|
||||
cutoffTime := time.Now().UTC().AddDate(0, -3, 0)
|
||||
cutoff = cutoffTime.Format("2006-01-02")
|
||||
|
||||
type candidate struct {
|
||||
url string
|
||||
when time.Time
|
||||
}
|
||||
var candidates []candidate
|
||||
for _, block := range strings.Split(xml, "<url>")[1:] {
|
||||
loc := locRe.FindStringSubmatch(block)
|
||||
lastmod := lastmodRe.FindStringSubmatch(block)
|
||||
if loc == nil || lastmod == nil {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(loc[1], "/p/") { // posts only
|
||||
continue
|
||||
}
|
||||
when, ok := parseLastmod(lastmod[1])
|
||||
if !ok || when.Before(cutoffTime) {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, candidate{url: loc[1], when: when})
|
||||
}
|
||||
sort.SliceStable(candidates, func(i, j int) bool {
|
||||
return candidates[i].when.After(candidates[j].when)
|
||||
})
|
||||
|
||||
if len(candidates) > maxFetch {
|
||||
candidates = candidates[:maxFetch]
|
||||
}
|
||||
for _, c := range candidates {
|
||||
scanned++
|
||||
html := fetchPage(c.url)
|
||||
if html == "" || !strings.Contains(html, id) {
|
||||
continue
|
||||
}
|
||||
return &postHit{
|
||||
Found: true,
|
||||
Source: "sitemap",
|
||||
Publication: publication,
|
||||
PostTitle: postTitleFromHTML(html),
|
||||
PostURL: c.url,
|
||||
Caption: captionForUUID(html, id),
|
||||
Candidates: extractCandidates(html),
|
||||
}, scanned, cutoff
|
||||
}
|
||||
return nil, scanned, cutoff
|
||||
}
|
||||
|
||||
func searchRSS(publication, id string) *postHit {
|
||||
xml := fetchPage("https://" + publication + "/feed")
|
||||
if xml == "" {
|
||||
return nil
|
||||
}
|
||||
for _, item := range strings.Split(xml, "<item>")[1:] {
|
||||
if !strings.Contains(item, id) {
|
||||
continue
|
||||
}
|
||||
return &postHit{
|
||||
Found: true,
|
||||
Source: "rss",
|
||||
Publication: publication,
|
||||
PostTitle: itemTitle(item),
|
||||
PostURL: itemLink(item),
|
||||
Caption: captionForUUID(item, id),
|
||||
Candidates: extractCandidates(item),
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runFindSubstackPost(args []string) {
|
||||
fs := flag.NewFlagSet("find-substack-post", flag.ExitOnError)
|
||||
uuid := fs.String("uuid", "", "Substack S3 image uuid to search for")
|
||||
deep := fs.Bool("deep", false, "fall back to a sitemap crawl (~3 months back)")
|
||||
_ = fs.Parse(args)
|
||||
|
||||
if *uuid == "" {
|
||||
fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter find-substack-post --uuid <uuid> [--deep]")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
publications := loadPublications()
|
||||
for _, pub := range publications {
|
||||
if hit := searchRSS(pub, *uuid); hit != nil {
|
||||
printJSON(hit)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Deep fallback: sitemap crawl up to ~3 months back, sharing one global
|
||||
// fetch budget across all publications so coverage can't blow up as the
|
||||
// publications list grows.
|
||||
if *deep {
|
||||
totalScanned := 0
|
||||
var lastCutoff *string
|
||||
for _, pub := range publications {
|
||||
remaining := deepFetchBudget - totalScanned
|
||||
if remaining <= 0 {
|
||||
break
|
||||
}
|
||||
hit, scanned, cutoff := searchSitemap(pub, *uuid, remaining)
|
||||
totalScanned += scanned
|
||||
if cutoff != "" {
|
||||
lastCutoff = &cutoff
|
||||
}
|
||||
if hit != nil {
|
||||
printJSON(hit)
|
||||
return
|
||||
}
|
||||
}
|
||||
printJSON(struct {
|
||||
Found bool `json:"found"`
|
||||
Source string `json:"source"`
|
||||
Scanned int `json:"scanned"`
|
||||
Budget int `json:"budget"`
|
||||
Cutoff *string `json:"cutoff"`
|
||||
}{false, "sitemap", totalScanned, deepFetchBudget, lastCutoff})
|
||||
return
|
||||
}
|
||||
|
||||
printJSON(struct {
|
||||
Found bool `json:"found"`
|
||||
}{false})
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
// HTML / RSS text-extraction helpers for find-substack-post.js.
|
||||
// Pure string functions (no network, no fs) — kept separate so the crawler
|
||||
// stays focused on fetch/search flow. CommonJS so plain `node` works.
|
||||
|
||||
// Decode the HTML entities that appear in Substack titles/captions, including
|
||||
// numeric (') and hex (—) forms. & is decoded last-ish but
|
||||
// before nothing re-encodes it; ordering here is safe for our inputs.
|
||||
function decodeEntities(s) {
|
||||
return s
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/�?39;|'/g, "'")
|
||||
.replace(/ /g, " ")
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCodePoint(parseInt(n, 16)))
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n))
|
||||
.trim();
|
||||
}
|
||||
|
||||
function stripTags(html) {
|
||||
return decodeEntities(html.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
// Pull the first <title> (CDATA or plain) and <link> from an RSS <item> chunk.
|
||||
function itemTitle(item) {
|
||||
const m = item.match(/<title>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/title>/);
|
||||
return m ? decodeEntities(m[1].trim()) : "";
|
||||
}
|
||||
function itemLink(item) {
|
||||
const m = item.match(/<link>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/link>/);
|
||||
return m ? m[1].trim() : "";
|
||||
}
|
||||
|
||||
// Extract candidate topic titles from a post's TOC bullet list. ByteByteGo does
|
||||
// not attach captions to images — the topic titles live only in the "in this
|
||||
// issue" bullets. We can't reliably map image→title automatically (sponsor /
|
||||
// video items interleave and break positional order), so we surface these
|
||||
// candidates for the user to pick from. Light filtering keeps the list short:
|
||||
// dedupe, drop sub-point explanations ("Term: long sentence") and over-long
|
||||
// lines. The correct title is always present; the user selects it.
|
||||
function extractCandidates(html) {
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
const re = /<li[^>]*>\s*(?:<p[^>]*>)?([\s\S]*?)(?:<\/p>)?\s*<\/li>/g;
|
||||
let m;
|
||||
while ((m = re.exec(html))) {
|
||||
const text = stripTags(m[1]);
|
||||
if (!text) continue;
|
||||
if (text.length < 6 || text.length > 70) continue; // titles are short; long lines are sub-point explanations
|
||||
const key = text.toLowerCase();
|
||||
if (seen.has(key)) continue; // content is duplicated in the page
|
||||
seen.add(key);
|
||||
out.push(text);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// If the UUID sits inside a <figure>…</figure>, return that figure's
|
||||
// <figcaption> text. Cover images live in <enclosure> (no figure) → "".
|
||||
function captionForUuid(item, id) {
|
||||
const at = item.indexOf(id);
|
||||
if (at === -1) return "";
|
||||
const figStart = item.lastIndexOf("<figure", at);
|
||||
if (figStart === -1) return "";
|
||||
const figEnd = item.indexOf("</figure>", at);
|
||||
if (figEnd === -1) return "";
|
||||
const figure = item.slice(figStart, figEnd);
|
||||
const cap = figure.match(/<figcaption[^>]*>([\s\S]*?)<\/figcaption>/);
|
||||
return cap ? stripTags(cap[1]) : "";
|
||||
}
|
||||
|
||||
// Extract a post title from server-rendered post HTML (og:title preferred).
|
||||
function postTitleFromHtml(html) {
|
||||
const og = html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i);
|
||||
if (og) return decodeEntities(og[1]);
|
||||
const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
|
||||
if (h1) return stripTags(h1[1]);
|
||||
const t = html.match(/<title>([\s\S]*?)<\/title>/i);
|
||||
return t ? decodeEntities(t[1].trim()) : "";
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
decodeEntities,
|
||||
stripTags,
|
||||
itemTitle,
|
||||
itemLink,
|
||||
extractCandidates,
|
||||
captionForUuid,
|
||||
postTitleFromHtml,
|
||||
};
|
||||
@@ -0,0 +1,116 @@
|
||||
// HTML / RSS text-extraction helpers for find-substack-post, ported from
|
||||
// html-text-utils.js. Pure string functions — no network, no fs.
|
||||
package main
|
||||
|
||||
import (
|
||||
"html"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var (
|
||||
tagRe = regexp.MustCompile(`<[^>]+>`)
|
||||
spaceRe = regexp.MustCompile(`\s+`)
|
||||
titleTagRe = regexp.MustCompile(`(?s)<title>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</title>`)
|
||||
linkTagRe = regexp.MustCompile(`(?s)<link>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</link>`)
|
||||
liRe = regexp.MustCompile(`(?s)<li[^>]*>\s*(?:<p[^>]*>)?(.*?)(?:</p>)?\s*</li>`)
|
||||
figcaptionRe = regexp.MustCompile(`(?s)<figcaption[^>]*>(.*?)</figcaption>`)
|
||||
ogTitleRe = regexp.MustCompile(`(?i)<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']`)
|
||||
h1Re = regexp.MustCompile(`(?is)<h1[^>]*>(.*?)</h1>`)
|
||||
htmlTitleRe = regexp.MustCompile(`(?is)<title>(.*?)</title>`)
|
||||
)
|
||||
|
||||
// decodeEntities decodes HTML entities (named, numeric, hex) and trims.
|
||||
// html.UnescapeString covers a superset of the entity map the JS hand-rolled.
|
||||
func decodeEntities(s string) string {
|
||||
return strings.TrimSpace(html.UnescapeString(s))
|
||||
}
|
||||
|
||||
func stripTags(s string) string {
|
||||
return strings.TrimSpace(spaceRe.ReplaceAllString(decodeEntities(tagRe.ReplaceAllString(s, "")), " "))
|
||||
}
|
||||
|
||||
// itemTitle pulls the first <title> (CDATA or plain) from an RSS <item> chunk.
|
||||
func itemTitle(item string) string {
|
||||
m := titleTagRe.FindStringSubmatch(item)
|
||||
if m == nil {
|
||||
return ""
|
||||
}
|
||||
return decodeEntities(m[1])
|
||||
}
|
||||
|
||||
// itemLink pulls the first <link> (CDATA or plain) from an RSS <item> chunk.
|
||||
func itemLink(item string) string {
|
||||
m := linkTagRe.FindStringSubmatch(item)
|
||||
if m == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(m[1])
|
||||
}
|
||||
|
||||
// extractCandidates pulls candidate topic titles from a post's TOC bullet
|
||||
// list. ByteByteGo does not attach captions to images — the topic titles live
|
||||
// only in the "in this issue" bullets, and image→title cannot be mapped
|
||||
// automatically (sponsor/video items interleave), so these are surfaced for
|
||||
// the user to pick from. Light filtering keeps the list short: dedupe, drop
|
||||
// sub-point explanations and over-long lines.
|
||||
func extractCandidates(htmlSrc string) []string {
|
||||
seen := map[string]bool{}
|
||||
out := []string{} // non-nil: marshals as [] like the JS output
|
||||
for _, m := range liRe.FindAllStringSubmatch(htmlSrc, -1) {
|
||||
text := stripTags(m[1])
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
// titles are short; long lines are sub-point explanations
|
||||
if n := utf8.RuneCountInString(text); n < 6 || n > 70 {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(text)
|
||||
if seen[key] { // content is duplicated in the page
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, text)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// captionForUUID returns the <figcaption> text of the <figure> containing the
|
||||
// UUID. Cover images live in <enclosure> (no figure) → "".
|
||||
func captionForUUID(item, id string) string {
|
||||
at := strings.Index(item, id)
|
||||
if at == -1 {
|
||||
return ""
|
||||
}
|
||||
figStart := strings.LastIndex(item[:at], "<figure")
|
||||
if figStart == -1 {
|
||||
return ""
|
||||
}
|
||||
rel := strings.Index(item[at:], "</figure>")
|
||||
if rel == -1 {
|
||||
return ""
|
||||
}
|
||||
figure := item[figStart : at+rel]
|
||||
m := figcaptionRe.FindStringSubmatch(figure)
|
||||
if m == nil {
|
||||
return ""
|
||||
}
|
||||
return stripTags(m[1])
|
||||
}
|
||||
|
||||
// postTitleFromHTML extracts a post title from server-rendered post HTML
|
||||
// (og:title preferred, then <h1>, then <title>).
|
||||
func postTitleFromHTML(htmlSrc string) string {
|
||||
if m := ogTitleRe.FindStringSubmatch(htmlSrc); m != nil {
|
||||
return decodeEntities(m[1])
|
||||
}
|
||||
if m := h1Re.FindStringSubmatch(htmlSrc); m != nil {
|
||||
return stripTags(m[1])
|
||||
}
|
||||
if m := htmlTitleRe.FindStringSubmatch(htmlSrc); m != nil {
|
||||
return decodeEntities(m[1])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// List existing tags in the repo ranked by frequency
|
||||
// Usage: node list-existing-tags.js
|
||||
// Outputs: tag count and name, sorted most-used first (top 40)
|
||||
//
|
||||
// NOTE: This script is NOT currently used by the mt-add-tags skill.
|
||||
// Tag normalization is disabled until existing posts have standardized tags.
|
||||
// To enable, uncomment step 4a in SKILL.md.
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const PROJECT_ROOT = path.resolve(__dirname, "../..");
|
||||
const CONTENT_DIR = path.join(PROJECT_ROOT, "content", "post");
|
||||
|
||||
// Recursively find all index.md files
|
||||
function findIndexFiles(dir) {
|
||||
const results = [];
|
||||
try {
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
results.push(...findIndexFiles(fullPath));
|
||||
} else if (entry.name === "index.md") {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// skip unreadable directories
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// Extract tags from frontmatter
|
||||
function extractTags(filePath) {
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, "utf-8");
|
||||
const match = content.match(/^tags:\s*\[([^\]]*)\]/m);
|
||||
if (!match) return [];
|
||||
// Extract quoted strings from the tags array
|
||||
return [...match[1].matchAll(/"([^"]+)"/g)].map((m) => m[1]);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Count tag frequency
|
||||
const tagCounts = new Map();
|
||||
const files = findIndexFiles(CONTENT_DIR);
|
||||
|
||||
for (const file of files) {
|
||||
for (const tag of extractTags(file)) {
|
||||
tagCounts.set(tag, (tagCounts.get(tag) || 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by frequency descending, take top 40
|
||||
const sorted = [...tagCounts.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 40);
|
||||
|
||||
for (const [tag, count] of sorted) {
|
||||
console.log(`${String(count).padStart(6)} ${tag}`);
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
// List existing tags in the repo ranked by frequency.
|
||||
// Usage: go run ./scripts/newsletter list-existing-tags
|
||||
// Outputs: tag count and name, sorted most-used first (top 40).
|
||||
//
|
||||
// NOTE: not currently used by the mt-add-tags skill. Tag normalization is
|
||||
// disabled until existing posts have standardized tags; to enable, uncomment
|
||||
// step 4a in that skill's SKILL.md.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
)
|
||||
|
||||
var (
|
||||
tagsLineRe = regexp.MustCompile(`(?m)^tags:\s*\[([^\]]*)\]`)
|
||||
quotedTagRe = regexp.MustCompile(`"([^"]+)"`)
|
||||
)
|
||||
|
||||
// extractTags pulls quoted tag strings from an index.md frontmatter tags array.
|
||||
func extractTags(path string) []string {
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
m := tagsLineRe.FindSubmatch(content)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
var tags []string
|
||||
for _, q := range quotedTagRe.FindAllSubmatch(m[1], -1) {
|
||||
tags = append(tags, string(q[1]))
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
func runListExistingTags(_ []string) {
|
||||
type tagCount struct {
|
||||
tag string
|
||||
count int
|
||||
}
|
||||
// Slice + index map keeps first-seen order for equal counts, so the stable
|
||||
// sort below ranks ties deterministically (walk order is lexical).
|
||||
var counts []tagCount
|
||||
index := map[string]int{}
|
||||
|
||||
_ = filepath.WalkDir(contentDir(), func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || d.Name() != "index.md" {
|
||||
return nil
|
||||
}
|
||||
for _, tag := range extractTags(p) {
|
||||
if i, ok := index[tag]; ok {
|
||||
counts[i].count++
|
||||
} else {
|
||||
index[tag] = len(counts)
|
||||
counts = append(counts, tagCount{tag: tag, count: 1})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
sort.SliceStable(counts, func(i, j int) bool {
|
||||
return counts[i].count > counts[j].count
|
||||
})
|
||||
if len(counts) > 40 {
|
||||
counts = counts[:40]
|
||||
}
|
||||
for _, tc := range counts {
|
||||
fmt.Printf("%6d %s\n", tc.count, tc.tag)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// Newsletter engine for the mt-* skills — single binary, one subcommand per
|
||||
// former Node script. Invoked from the repo root:
|
||||
//
|
||||
// go run ./scripts/newsletter <command> [args]
|
||||
//
|
||||
// Repo-relative paths (content/post) resolve from the working directory, so
|
||||
// the repo-root invocation contract from AGENTS.md still applies.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `Usage: go run ./scripts/newsletter <command> [args]
|
||||
|
||||
Commands:
|
||||
add-url <url> classify + dedup a URL, emit JSON route
|
||||
find-newsletter-number print the next newsletter number
|
||||
list-existing-tags tag frequencies, most-used first (top 40)
|
||||
detect-image-source <url> detect Substack image + uuid
|
||||
find-substack-post --uuid <uuid> [--deep] find the post embedding an image uuid
|
||||
fetch-via-defuddle <url> fallback fetch via defuddle.md proxy
|
||||
`)
|
||||
}
|
||||
|
||||
// printJSON mirrors console.log(JSON.stringify(v, null, 2)): 2-space indent,
|
||||
// no HTML escaping (URLs with & must stay readable), trailing newline.
|
||||
func printJSON(v any) {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetEscapeHTML(false)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(v); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "json encode:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
args := os.Args[2:]
|
||||
switch os.Args[1] {
|
||||
case "add-url":
|
||||
runAddURL(args)
|
||||
case "find-newsletter-number":
|
||||
runFindNewsletterNumber(args)
|
||||
case "list-existing-tags":
|
||||
runListExistingTags(args)
|
||||
case "detect-image-source":
|
||||
runDetectImageSource(args)
|
||||
case "find-substack-post":
|
||||
runFindSubstackPost(args)
|
||||
case "fetch-via-defuddle":
|
||||
runFetchViaDefuddle(args)
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown command: %s\n", os.Args[1])
|
||||
usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,166 +0,0 @@
|
||||
// Shared URL helpers for the mt-* newsletter skills.
|
||||
// Owned by the mt-add-url meta router; reused by handlers (e.g. mt-add-image).
|
||||
// CommonJS so plain `node script.js` works without a build step.
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
// Remove common tracking parameters (utm_* plus a fixed set of known trackers).
|
||||
function cleanUrl(rawUrl) {
|
||||
const EXACT_TRACKING = new Set([
|
||||
"fbclid", "gclid", "msclkid", "mc_eid",
|
||||
"aid", "ref", "ref_src", "ref_url", "source", "s",
|
||||
"ck_subscriber_id", "igshid", "yclid", "vero_id",
|
||||
]);
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
[...parsed.searchParams.keys()].forEach((k) => {
|
||||
if (k.toLowerCase().startsWith("utm_") || EXACT_TRACKING.has(k.toLowerCase())) {
|
||||
parsed.searchParams.delete(k);
|
||||
}
|
||||
});
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
// Unparseable input (not a real URL): the per-param string surgery above
|
||||
// would mangle the query (drop the `?`, leave a dangling `&`), so leave it
|
||||
// untouched rather than corrupt it.
|
||||
return rawUrl;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Substack image helpers (shared by add-url.js routing and mt-add-image) ---
|
||||
const SUBSTACK_IMAGE_HOSTS = ["substackcdn.com", "substack-post-media.s3.amazonaws.com"];
|
||||
const UUID = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
|
||||
|
||||
// A Substack-hosted image (CDN wrapper or raw S3), regardless of file extension.
|
||||
function isSubstackImage(targetUrl) {
|
||||
let host = "";
|
||||
try { host = new URL(targetUrl).host.toLowerCase(); } catch { /* non-URL */ }
|
||||
return SUBSTACK_IMAGE_HOSTS.includes(host) || /substack-post-media/i.test(targetUrl);
|
||||
}
|
||||
|
||||
// The stable image identity is the S3 image UUID under public/images/<uuid>.
|
||||
// Works whether the path separators are raw (/) or percent-encoded (%2F).
|
||||
function substackImageUuid(targetUrl) {
|
||||
const m = targetUrl.match(new RegExp(`images(?:%2F|/)(${UUID})`, "i"));
|
||||
return m ? m[1].toLowerCase() : null;
|
||||
}
|
||||
|
||||
// Some sites carry the resource identity in a query param, not the path
|
||||
// (e.g. YouTube /watch?v=ID). Stripping the query for these collapses every
|
||||
// item to the same bare URL, causing false-positive duplicates. Preserve the
|
||||
// identity param for those hosts.
|
||||
const IDENTITY_PARAMS = {
|
||||
"youtube.com": "v",
|
||||
"www.youtube.com": "v",
|
||||
"m.youtube.com": "v",
|
||||
};
|
||||
|
||||
// Reduce a URL to a stable identity used for duplicate detection:
|
||||
// - Substack image → its S3 UUID (transform/size variants share one identity)
|
||||
// - YouTube → scheme+host+path + the v= video id
|
||||
// - everything else → scheme + host + path
|
||||
function bareUrl(targetUrl) {
|
||||
if (isSubstackImage(targetUrl)) {
|
||||
const uuid = substackImageUuid(targetUrl);
|
||||
if (uuid) return uuid;
|
||||
}
|
||||
try {
|
||||
const p = new URL(targetUrl);
|
||||
let bare = `${p.protocol}//${p.host}${p.pathname}`.replace(/\/$/, "");
|
||||
const idParam = IDENTITY_PARAMS[p.host.toLowerCase()];
|
||||
const idValue = idParam ? p.searchParams.get(idParam) : null;
|
||||
if (idValue) bare += `?${idParam}=${idValue}`;
|
||||
return bare;
|
||||
} catch {
|
||||
return targetUrl.split("?")[0].replace(/\/$/, "");
|
||||
}
|
||||
}
|
||||
|
||||
// fetch() with an abort timeout. Returns the Response on success, or null on
|
||||
// network error / timeout. Callers decide what to read (.text/.json/.status).
|
||||
// Centralizes the AbortController + clearTimeout dance so every caller cleans
|
||||
// up the timer (via finally) on both the success and failure paths.
|
||||
async function fetchWithTimeout(targetUrl, { method = "GET", timeoutMs = 10000, headers } = {}) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
return await fetch(targetUrl, { method, redirect: "follow", signal: controller.signal, headers });
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
// Check if URL is accessible (returns HTTP status code as a string).
|
||||
async function checkAccessibility(targetUrl) {
|
||||
const res = await fetchWithTimeout(targetUrl, { method: "HEAD" });
|
||||
return res ? res.status.toString() : "000";
|
||||
}
|
||||
|
||||
function escapeRegExp(s) {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
// Recursively collect *.md files under a directory (the content tree is small).
|
||||
function collectMarkdown(dir, acc = []) {
|
||||
let entries;
|
||||
try {
|
||||
entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return acc; // missing dir → nothing to compare against
|
||||
}
|
||||
for (const e of entries) {
|
||||
const full = path.join(dir, e.name);
|
||||
if (e.isDirectory()) collectMarkdown(full, acc);
|
||||
else if (e.isFile() && e.name.toLowerCase().endsWith(".md")) acc.push(full);
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
// Check whether a URL identity already exists in the stored markdown.
|
||||
// Pure JS (no external `grep` dependency, works the same from any shell) and
|
||||
// boundary-aware so a needle that is merely a PREFIX of a stored longer string
|
||||
// is NOT a false duplicate. The two identity kinds need different boundaries:
|
||||
// - Substack image UUID: the next char must not extend the hex id, so all of
|
||||
// <uuid>.png (cover image, no size suffix), <uuid>_WxH and <uuid>) match.
|
||||
// - URL: must be followed by a path/punctuation delimiter so /p/foo does not
|
||||
// match a stored /p/foo-bar.
|
||||
function checkDuplicate(targetUrl, contentDir) {
|
||||
const needle = bareUrl(targetUrl);
|
||||
if (!needle) return false;
|
||||
const isUuid = new RegExp(`^${UUID}$`, "i").test(needle);
|
||||
// For URLs, allow an optional trailing slash (bareUrl strips it, stored URLs
|
||||
// may keep it) before the delimiter.
|
||||
const boundary = isUuid
|
||||
? new RegExp(escapeRegExp(needle) + `(?![0-9a-f])`, "i")
|
||||
: new RegExp(escapeRegExp(needle) + `/?(?:[)\\]\\s"'?#<_&,]|$)`, "m");
|
||||
for (const file of collectMarkdown(contentDir)) {
|
||||
let text;
|
||||
try { text = fs.readFileSync(file, "utf-8"); } catch { continue; }
|
||||
if (text.includes(needle) && boundary.test(text)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Classify URL type by file extension. Returns image|video|document|article.
|
||||
function classifyType(targetUrl) {
|
||||
const lower = targetUrl.toLowerCase();
|
||||
if (/\.(png|jpg|jpeg|gif|webp|svg|avif|heic|heif|bmp|tiff?)(\?.*)?$/.test(lower)) return "image";
|
||||
if (/\.(mp4|webm|mov|avi|mkv)(\?.*)?$/.test(lower)) return "video";
|
||||
if (/\.(pdf|docx?|xlsx?|pptx?)(\?.*)?$/.test(lower)) return "document";
|
||||
return "article";
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
cleanUrl,
|
||||
bareUrl,
|
||||
IDENTITY_PARAMS,
|
||||
isSubstackImage,
|
||||
substackImageUuid,
|
||||
fetchWithTimeout,
|
||||
checkAccessibility,
|
||||
checkDuplicate,
|
||||
classifyType,
|
||||
};
|
||||
@@ -0,0 +1,276 @@
|
||||
// Shared URL helpers, ported from url-utils.js. Owned by the add-url router;
|
||||
// reused by the other subcommands.
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Exact-match tracking params; any key starting with utm_ is also dropped.
|
||||
var exactTracking = map[string]bool{
|
||||
"fbclid": true, "gclid": true, "msclkid": true, "mc_eid": true,
|
||||
"aid": true, "ref": true, "ref_src": true, "ref_url": true, "source": true, "s": true,
|
||||
"ck_subscriber_id": true, "igshid": true, "yclid": true, "vero_id": true,
|
||||
}
|
||||
|
||||
// cleanURL removes common tracking parameters. Surviving query pairs are kept
|
||||
// verbatim (no re-encoding) and in their original order — Go's url.Values
|
||||
// would sort keys alphabetically, which must not leak into stored clean_url
|
||||
// values. Unparseable / non-absolute input is returned untouched rather than
|
||||
// corrupted.
|
||||
func cleanURL(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return raw
|
||||
}
|
||||
u.Host = strings.ToLower(u.Host)
|
||||
// WHATWG URL serializes an empty path as "/"; keep that shape so cleaned
|
||||
// URLs match what the JS engine has already written into posts.
|
||||
if u.Path == "" {
|
||||
u.Path = "/"
|
||||
}
|
||||
if u.RawQuery != "" {
|
||||
var kept []string
|
||||
for _, pair := range strings.Split(u.RawQuery, "&") {
|
||||
if pair == "" {
|
||||
continue
|
||||
}
|
||||
key := pair
|
||||
if i := strings.IndexByte(pair, '='); i != -1 {
|
||||
key = pair[:i]
|
||||
}
|
||||
k := strings.ToLower(key)
|
||||
if strings.HasPrefix(k, "utm_") || exactTracking[k] {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, pair)
|
||||
}
|
||||
u.RawQuery = strings.Join(kept, "&")
|
||||
}
|
||||
u.ForceQuery = false
|
||||
return u.String()
|
||||
}
|
||||
|
||||
// --- Substack image helpers (shared by add-url routing and detect-image-source) ---
|
||||
|
||||
var substackImageHosts = map[string]bool{
|
||||
"substackcdn.com": true,
|
||||
"substack-post-media.s3.amazonaws.com": true,
|
||||
}
|
||||
|
||||
const uuidPattern = `[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`
|
||||
|
||||
var (
|
||||
imageUUIDRe = regexp.MustCompile(`(?i)images(?:%2F|/)(` + uuidPattern + `)`)
|
||||
uuidExactRe = regexp.MustCompile(`(?i)^` + uuidPattern + `$`)
|
||||
)
|
||||
|
||||
// isSubstackImage reports a Substack-hosted image (CDN wrapper or raw S3),
|
||||
// regardless of file extension.
|
||||
func isSubstackImage(target string) bool {
|
||||
host := ""
|
||||
if u, err := url.Parse(target); err == nil {
|
||||
host = strings.ToLower(u.Host)
|
||||
}
|
||||
return substackImageHosts[host] || strings.Contains(strings.ToLower(target), "substack-post-media")
|
||||
}
|
||||
|
||||
// substackImageUUID extracts the stable image identity: the S3 image UUID
|
||||
// under public/images/<uuid>, with raw (/) or percent-encoded (%2F) separators.
|
||||
// Returns "" when absent.
|
||||
func substackImageUUID(target string) string {
|
||||
m := imageUUIDRe.FindStringSubmatch(target)
|
||||
if m == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(m[1])
|
||||
}
|
||||
|
||||
// Some sites carry the resource identity in a query param, not the path
|
||||
// (e.g. YouTube /watch?v=ID). Preserve the identity param for those hosts so
|
||||
// dedup does not collapse every video onto the same bare URL.
|
||||
var identityParams = map[string]string{
|
||||
"youtube.com": "v",
|
||||
"www.youtube.com": "v",
|
||||
"m.youtube.com": "v",
|
||||
}
|
||||
|
||||
// bareURL reduces a URL to a stable identity for duplicate detection:
|
||||
// - Substack image → its S3 UUID (transform/size variants share one identity)
|
||||
// - YouTube → scheme+host+path + the v= video id
|
||||
// - everything else → scheme + host + path
|
||||
func bareURL(target string) string {
|
||||
if isSubstackImage(target) {
|
||||
if uuid := substackImageUUID(target); uuid != "" {
|
||||
return uuid
|
||||
}
|
||||
}
|
||||
u, err := url.Parse(target)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return strings.TrimSuffix(strings.SplitN(target, "?", 2)[0], "/")
|
||||
}
|
||||
host := strings.ToLower(u.Host)
|
||||
bare := strings.TrimSuffix(u.Scheme+"://"+host+u.EscapedPath(), "/")
|
||||
if idParam, ok := identityParams[host]; ok {
|
||||
if v := u.Query().Get(idParam); v != "" {
|
||||
bare += "?" + idParam + "=" + v
|
||||
}
|
||||
}
|
||||
return bare
|
||||
}
|
||||
|
||||
// fetchTextOK GETs target and returns the body on a 2xx response, "" on any
|
||||
// error, non-2xx status, or timeout. Redirects are followed.
|
||||
func fetchTextOK(target string, timeout time.Duration, userAgent string) string {
|
||||
req, err := http.NewRequest(http.MethodGet, target, nil)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
if userAgent != "" {
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
}
|
||||
res, err := (&http.Client{Timeout: timeout}).Do(req)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return ""
|
||||
}
|
||||
body, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
|
||||
// checkAccessibility HEADs the URL and returns the final HTTP status code as a
|
||||
// string, or "000" on network error / timeout (matching the JS sentinel).
|
||||
func checkAccessibility(target string) string {
|
||||
req, err := http.NewRequest(http.MethodHead, target, nil)
|
||||
if err != nil {
|
||||
return "000"
|
||||
}
|
||||
res, err := (&http.Client{Timeout: 10 * time.Second}).Do(req)
|
||||
if err != nil {
|
||||
return "000"
|
||||
}
|
||||
res.Body.Close()
|
||||
return strconv.Itoa(res.StatusCode)
|
||||
}
|
||||
|
||||
// collectMarkdown recursively collects *.md files under dir (the content tree
|
||||
// is small). Missing or unreadable directories yield nothing.
|
||||
func collectMarkdown(dir string) []string {
|
||||
var acc []string
|
||||
_ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil // skip unreadable entries
|
||||
}
|
||||
if !d.IsDir() && strings.HasSuffix(strings.ToLower(d.Name()), ".md") {
|
||||
acc = append(acc, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return acc
|
||||
}
|
||||
|
||||
// uuidBoundaryOK: the char after a UUID match must not extend the hex id, so
|
||||
// <uuid>.png (cover image), <uuid>_WxH and <uuid>) all match.
|
||||
func uuidBoundaryOK(text string, end int) bool {
|
||||
if end >= len(text) {
|
||||
return true
|
||||
}
|
||||
c := text[end]
|
||||
return !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))
|
||||
}
|
||||
|
||||
// urlBoundaryOK: an optional trailing slash (bareURL strips it, stored URLs may
|
||||
// keep it), then a path/punctuation delimiter, whitespace, or end of text — so
|
||||
// /p/foo does not match a stored /p/foo-bar.
|
||||
func urlBoundaryOK(text string, end int) bool {
|
||||
if end < len(text) && text[end] == '/' {
|
||||
end++
|
||||
}
|
||||
if end >= len(text) {
|
||||
return true
|
||||
}
|
||||
switch c := text[end]; c {
|
||||
case ' ', '\t', '\n', '\r', '\f', '\v':
|
||||
return true
|
||||
default:
|
||||
return strings.IndexByte(`)]"'?#<_&,`, c) != -1
|
||||
}
|
||||
}
|
||||
|
||||
// hasBoundaryMatch scans every occurrence of needle and applies the boundary
|
||||
// check in code — Go's RE2 regexp has no lookahead, which the JS version used.
|
||||
func hasBoundaryMatch(text, needle string, isUUID bool) bool {
|
||||
for from := 0; ; {
|
||||
i := strings.Index(text[from:], needle)
|
||||
if i == -1 {
|
||||
return false
|
||||
}
|
||||
end := from + i + len(needle)
|
||||
if isUUID {
|
||||
if uuidBoundaryOK(text, end) {
|
||||
return true
|
||||
}
|
||||
} else if urlBoundaryOK(text, end) {
|
||||
return true
|
||||
}
|
||||
from = from + i + 1
|
||||
}
|
||||
}
|
||||
|
||||
// checkDuplicate reports whether a URL identity already exists in the stored
|
||||
// markdown, boundary-aware so a needle that is merely a PREFIX of a stored
|
||||
// longer string is NOT a false duplicate.
|
||||
func checkDuplicate(target, contentDir string) bool {
|
||||
needle := bareURL(target)
|
||||
if needle == "" {
|
||||
return false
|
||||
}
|
||||
isUUID := uuidExactRe.MatchString(needle)
|
||||
for _, file := range collectMarkdown(contentDir) {
|
||||
data, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
text := string(data)
|
||||
if strings.Contains(text, needle) && hasBoundaryMatch(text, needle, isUUID) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var (
|
||||
imageExtRe = regexp.MustCompile(`\.(png|jpg|jpeg|gif|webp|svg|avif|heic|heif|bmp|tiff?)(\?.*)?$`)
|
||||
videoExtRe = regexp.MustCompile(`\.(mp4|webm|mov|avi|mkv)(\?.*)?$`)
|
||||
documentExtRe = regexp.MustCompile(`\.(pdf|docx?|xlsx?|pptx?)(\?.*)?$`)
|
||||
)
|
||||
|
||||
// classifyType classifies a URL by file extension: image|video|document|article.
|
||||
func classifyType(target string) string {
|
||||
lower := strings.ToLower(target)
|
||||
switch {
|
||||
case imageExtRe.MatchString(lower):
|
||||
return "image"
|
||||
case videoExtRe.MatchString(lower):
|
||||
return "video"
|
||||
case documentExtRe.MatchString(lower):
|
||||
return "document"
|
||||
default:
|
||||
return "article"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user