diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..01ac3bc
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,7 @@
+# Build output
+/monkeyd-crawler
+/monkeyd-crawler.exe
+
+# Cached pages and exported books
+/.cache/
+*.pdf
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..484e04e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,114 @@
+# monkeyd-crawler
+
+Downloads every chapter of a [monkeydd.com](https://monkeydd.com) novel and exports it as a
+single PDF sized for reading on a phone.
+
+## Install
+
+Requires Go 1.22+.
+
+```sh
+go build ./cmd/monkeyd-crawler
+```
+
+## Usage
+
+Pass the novel's landing page URL:
+
+```sh
+./monkeyd-crawler -url https://monkeydd.com/tro-lai-nam-thang-cu.html
+```
+
+The PDF is named after the novel unless you pass `-out`.
+
+```sh
+# Bigger type, A5 page for a tablet
+./monkeyd-crawler -url https://monkeydd.com/tro-lai-nam-thang-cu.html \
+ -page a5 -font-size 14 -out truyen.pdf
+
+# Try the layout on 3 chapters before fetching the whole book
+./monkeyd-crawler -url https://monkeydd.com/tro-lai-nam-thang-cu.html -limit 3
+```
+
+### Flags
+
+| Flag | Default | Purpose |
+| --- | --- | --- |
+| `-url` | *required* | Novel landing page URL |
+| `-out` | novel title | Output PDF path |
+| `-page` | `phone` | Page size: `phone`, `a5`, `a4` |
+| `-font-size` | `12` | Body font size in points |
+| `-line-spacing` | `1.55` | Line height as a multiple of font size |
+| `-margin` | `6` | Page margin in mm |
+| `-font` | auto | Path to a `.ttf`; defaults to a system font with Vietnamese coverage |
+| `-workers` | `4` | Concurrent chapter fetches |
+| `-delay` | `400ms` | Minimum delay between requests |
+| `-retries` | `3` | Retries per request |
+| `-limit` | `0` | Fetch only the first N chapters (0 = all) |
+| `-cache` | `.cache` | Cache directory for raw pages (empty to disable) |
+
+## Why the default page is 90×160 mm
+
+Phone readability is governed by page *shape* more than by font size. A PDF viewer scales a
+whole page to fit the screen, so a large font on an A4 page still ends up small: the page is
+about three times wider than a phone screen and gets shrunk to match. The default page is cut
+to a 9:16 ratio so it fills the screen at 100% zoom, where 12 pt renders at a comfortable
+size with roughly 35–40 characters per line.
+
+Use `-page a5` or `-page a4` for a tablet or for printing.
+
+## How it works
+
+1. Fetch the landing page and read the chapter list from `div.list-chapters`.
+2. Cross-check that list against the `#selected_chapter` dropdown embedded in the first
+ chapter page. When the dropdown is a superset it wins; a disagreement is reported.
+3. Fetch each chapter concurrently and extract its text.
+4. Render one PDF, each chapter starting on a new page.
+
+### Two site behaviours the extractor has to handle
+
+**Chapter text is partly served through CSS.** The markup contains empty elements, and the
+stylesheet supplies the missing word:
+
+```html
+Nghe trưởng tử
+```
+
+```css
+.t-3e625e…:before { content: "vị"; }
+```
+
+Reading DOM text alone silently drops these — about 19% of the words in a sampled chapter. The
+extractor parses the `:before` rules and substitutes each word back in.
+
+**Chapter URLs cannot be generated.** Numbering has gaps (the sample novel has no chapter 4)
+and slugs are not uniform across novels (`/14.html` on one, `/chuong-12.html` on another), so
+chapter links are always parsed from the page rather than constructed from a count.
+
+## Politeness and caching
+
+Requests are spaced by `-delay` globally, so raising `-workers` does not raise the request
+rate. Raw pages are cached under `.cache/`, so re-exporting with different font or page
+settings costs no requests. Delete the directory to refetch.
+
+## Tests
+
+```sh
+go test ./...
+```
+
+Tests run against synthetic fixtures that reproduce the CSS-injected words, the newest-first
+chapter ordering, and the numbering gap. No network access required.
+
+## Layout
+
+```
+cmd/monkeyd-crawler/ CLI
+internal/monkeyd/ fetching, HTML/CSS parsing, crawl orchestration
+internal/pdfout/ PDF rendering and font discovery
+```
+
+## Scope
+
+Downloaded text stays on your machine; only fetch content you are allowed to read offline, and
+respect the site's terms.
diff --git a/cmd/monkeyd-crawler/main.go b/cmd/monkeyd-crawler/main.go
new file mode 100644
index 0000000..41c4b18
--- /dev/null
+++ b/cmd/monkeyd-crawler/main.go
@@ -0,0 +1,184 @@
+// Command monkeyd-crawler downloads every chapter of a monkeydd.com novel and
+// exports it as a PDF sized for reading on a phone.
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "net/url"
+ "os"
+ "os/signal"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/tiennm99/monkeyd-crawler/internal/monkeyd"
+ "github.com/tiennm99/monkeyd-crawler/internal/pdfout"
+)
+
+type config struct {
+ novelURL string
+ out string
+ page string
+ fontFile string
+ fontSize float64
+ lineSpacing float64
+ margin float64
+ workers int
+ delay time.Duration
+ retries int
+ limit int
+ cacheDir string
+}
+
+func main() {
+ if err := run(); err != nil {
+ fmt.Fprintln(os.Stderr, "error:", err)
+ os.Exit(1)
+ }
+}
+
+func run() error {
+ cfg, err := parseFlags()
+ if err != nil {
+ return err
+ }
+
+ // Ctrl-C cancels in-flight fetches instead of leaving a partial PDF.
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer stop()
+
+ crawler := &monkeyd.Crawler{
+ Client: monkeyd.NewClient(cfg.delay, cfg.retries),
+ CacheDir: cfg.cacheDir,
+ Workers: cfg.workers,
+ Log: func(format string, args ...any) {
+ fmt.Fprintf(os.Stderr, format+"\n", args...)
+ },
+ }
+
+ novel, err := crawler.Novel(ctx, cfg.novelURL)
+ if err != nil {
+ return err
+ }
+
+ if cfg.limit > 0 && cfg.limit < len(novel.Chapters) {
+ fmt.Fprintf(os.Stderr, "limiting to first %d of %d chapters\n", cfg.limit, len(novel.Chapters))
+ novel.Chapters = novel.Chapters[:cfg.limit]
+ }
+
+ chapters, err := crawler.Chapters(ctx, novel)
+ if err != nil {
+ return err
+ }
+
+ fontFile := cfg.fontFile
+ if fontFile == "" {
+ if fontFile, err = pdfout.FindFont(); err != nil {
+ return err
+ }
+ }
+
+ outPath := cfg.out
+ if outPath == "" {
+ outPath = safeFileName(novel.Title, novel.Slug) + ".pdf"
+ }
+
+ opts := pdfout.Options{
+ Page: pdfout.Presets[cfg.page],
+ Margin: cfg.margin,
+ FontFile: fontFile,
+ FontSize: cfg.fontSize,
+ LineSpacing: cfg.lineSpacing,
+ Title: novel.Title,
+ SourceURL: novel.URL,
+ }
+ if err := pdfout.Write(outPath, opts, toPDFChapters(chapters)); err != nil {
+ return err
+ }
+
+ fmt.Fprintf(os.Stderr, "\n%s\n", monkeyd.Describe(novel, chapters))
+ fmt.Fprintf(os.Stderr, "font: %s at %.0fpt on %s page (%.0f x %.0f mm)\n",
+ filepath.Base(fontFile), cfg.fontSize, opts.Page.Name, opts.Page.W, opts.Page.H)
+ fmt.Println(outPath)
+ return nil
+}
+
+func parseFlags() (*config, error) {
+ cfg := &config{}
+
+ flag.StringVar(&cfg.novelURL, "url", "", "novel page URL, e.g. https://monkeydd.com/tro-lai-nam-thang-cu.html")
+ flag.StringVar(&cfg.out, "out", "", "output PDF path (default: novel title)")
+ flag.StringVar(&cfg.page, "page", "phone",
+ "page size: "+strings.Join(pdfout.PresetNames(), ", "))
+ flag.StringVar(&cfg.fontFile, "font", "", "path to a .ttf font (default: a Vietnamese-capable system font)")
+ flag.Float64Var(&cfg.fontSize, "font-size", 12, "body font size in points")
+ flag.Float64Var(&cfg.lineSpacing, "line-spacing", 1.55, "line height as a multiple of font size")
+ flag.Float64Var(&cfg.margin, "margin", 6, "page margin in millimetres")
+ flag.IntVar(&cfg.workers, "workers", 4, "concurrent chapter fetches")
+ flag.DurationVar(&cfg.delay, "delay", 400*time.Millisecond, "minimum delay between requests")
+ flag.IntVar(&cfg.retries, "retries", 3, "retries per request")
+ flag.IntVar(&cfg.limit, "limit", 0, "only fetch the first N chapters (0 = all)")
+ flag.StringVar(&cfg.cacheDir, "cache", ".cache",
+ "directory for cached pages, so re-exports need no requests (empty to disable)")
+
+ flag.Usage = func() {
+ fmt.Fprintf(flag.CommandLine.Output(),
+ "Download a monkeydd.com novel and export it as a phone-friendly PDF.\n\n"+
+ "Usage:\n monkeyd-crawler -url [flags]\n\nFlags:\n")
+ flag.PrintDefaults()
+ }
+ flag.Parse()
+
+ if cfg.novelURL == "" {
+ flag.Usage()
+ return nil, fmt.Errorf("-url is required")
+ }
+ parsed, err := url.Parse(cfg.novelURL)
+ if err != nil {
+ return nil, fmt.Errorf("invalid -url: %w", err)
+ }
+ if parsed.Scheme != "http" && parsed.Scheme != "https" {
+ return nil, fmt.Errorf("invalid -url: want an http(s) URL, got %q", cfg.novelURL)
+ }
+ if _, ok := pdfout.Presets[cfg.page]; !ok {
+ return nil, fmt.Errorf("unknown -page %q: want one of %s",
+ cfg.page, strings.Join(pdfout.PresetNames(), ", "))
+ }
+ if cfg.fontSize <= 0 {
+ return nil, fmt.Errorf("-font-size must be positive")
+ }
+ if cfg.lineSpacing <= 0 {
+ return nil, fmt.Errorf("-line-spacing must be positive")
+ }
+ if cfg.margin < 0 {
+ return nil, fmt.Errorf("-margin cannot be negative")
+ }
+ if cfg.workers < 1 {
+ return nil, fmt.Errorf("-workers must be at least 1")
+ }
+ return cfg, nil
+}
+
+func toPDFChapters(chapters []*monkeyd.Chapter) []pdfout.Chapter {
+ out := make([]pdfout.Chapter, 0, len(chapters))
+ for _, ch := range chapters {
+ out = append(out, pdfout.Chapter{Heading: ch.Heading(), Paragraphs: ch.Paragraphs})
+ }
+ return out
+}
+
+var unsafeNameChars = regexp.MustCompile(`[^\p{L}\p{N}]+`)
+
+// safeFileName builds a file name from the novel title, falling back to the
+// slug when the title has no usable characters.
+func safeFileName(title, fallback string) string {
+ name := strings.Trim(unsafeNameChars.ReplaceAllString(title, "-"), "-")
+ if name == "" {
+ return fallback
+ }
+ return name
+}
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..4c5fcb7
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,9 @@
+module github.com/tiennm99/monkeyd-crawler
+
+go 1.26.4
+
+require (
+ github.com/go-pdf/fpdf v0.9.0
+ golang.org/x/net v0.57.0
+ golang.org/x/sync v0.22.0
+)
diff --git a/go.sum b/go.sum
new file mode 100644
index 0000000..5038e44
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,6 @@
+github.com/go-pdf/fpdf v0.9.0 h1:PPvSaUuo1iMi9KkaAn90NuKi+P4gwMedWPHhj8YlJQw=
+github.com/go-pdf/fpdf v0.9.0/go.mod h1:oO8N111TkmKb9D7VvWGLvLJlaZUQVPM+6V42pp3iV4Y=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
diff --git a/internal/monkeyd/chapter.go b/internal/monkeyd/chapter.go
new file mode 100644
index 0000000..40d0548
--- /dev/null
+++ b/internal/monkeyd/chapter.go
@@ -0,0 +1,153 @@
+package monkeyd
+
+import (
+ "bytes"
+ "fmt"
+ "strconv"
+ "strings"
+
+ "golang.org/x/net/html"
+)
+
+// contentElementID is the container holding a chapter's body text.
+const contentElementID = "chapter-content-render"
+
+// Chapter is one fetched chapter, reduced to plain paragraphs.
+type Chapter struct {
+ Label string
+ URL string
+ Paragraphs []string
+}
+
+// Heading is the chapter title to print. Labels are often a bare number, which
+// reads poorly as a heading, so those get the Vietnamese word for "chapter".
+func (c *Chapter) Heading() string {
+ label := strings.TrimSpace(c.Label)
+ if label == "" {
+ return "Chương"
+ }
+ if _, err := strconv.Atoi(label); err == nil {
+ return "Chương " + label
+ }
+ return label
+}
+
+// WordCount is a rough word count, used to sanity-check extraction.
+func (c *Chapter) WordCount() int {
+ n := 0
+ for _, p := range c.Paragraphs {
+ n += len(strings.Fields(p))
+ }
+ return n
+}
+
+// blockTags end the current paragraph when opened or closed.
+var blockTags = map[string]bool{
+ "p": true, "div": true, "br": true, "hr": true, "blockquote": true,
+ "h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true,
+ "li": true, "ul": true, "ol": true, "tr": true,
+}
+
+// skipTags never contribute text: scripts, styles and the ad/report widgets
+// the site injects inside the content container.
+var skipTags = map[string]bool{
+ "script": true, "style": true, "noscript": true, "iframe": true,
+ "ins": true, "form": true, "select": true, "button": true, "textarea": true,
+}
+
+// ParseChapter extracts a chapter's paragraphs, restoring the words the site
+// serves through CSS :before rules instead of markup.
+func ParseChapter(page []byte, ref ChapterRef) (*Chapter, error) {
+ doc, err := html.Parse(bytes.NewReader(page))
+ if err != nil {
+ return nil, fmt.Errorf("parse chapter %s: %w", ref.URL, err)
+ }
+ content := elementByID(doc, contentElementID)
+ if content == nil {
+ return nil, fmt.Errorf("chapter %s: no #%s container (page layout may have changed)",
+ ref.URL, contentElementID)
+ }
+
+ ch := &Chapter{
+ Label: ref.Label,
+ URL: ref.URL,
+ Paragraphs: extractParagraphs(content, ParseWordClasses(page)),
+ }
+ if len(ch.Paragraphs) == 0 {
+ return nil, fmt.Errorf("chapter %s: extracted no text", ref.URL)
+ }
+ ch.Paragraphs = dropRepeatedTitle(ch.Paragraphs, ref.Label)
+ return ch, nil
+}
+
+// extractParagraphs walks the content subtree into plain paragraphs, replacing
+// each word-carrying element with the word its CSS rule injects.
+func extractParagraphs(content *html.Node, words map[string]string) []string {
+ var b strings.Builder
+
+ var walk func(*html.Node)
+ walk = func(n *html.Node) {
+ switch n.Type {
+ case html.TextNode:
+ // The HTML parser has already decoded entities such as ư.
+ b.WriteString(n.Data)
+ return
+ case html.ElementNode:
+ if skipTags[n.Data] {
+ return
+ }
+ // These elements are empty in the markup; the CSS word replaces them.
+ if word, ok := injectedWord(n, words); ok {
+ b.WriteString(word)
+ return
+ }
+ if blockTags[n.Data] {
+ b.WriteByte('\n')
+ }
+ }
+
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ walk(c)
+ }
+
+ if n.Type == html.ElementNode && blockTags[n.Data] {
+ b.WriteByte('\n')
+ }
+ }
+ walk(content)
+
+ var paragraphs []string
+ for _, line := range strings.Split(b.String(), "\n") {
+ if p := collapseSpaces(line); p != "" {
+ paragraphs = append(paragraphs, p)
+ }
+ }
+ return paragraphs
+}
+
+// injectedWord returns the word a node's class supplies via CSS, if any.
+func injectedWord(n *html.Node, words map[string]string) (string, bool) {
+ class := attr(n, "class")
+ if class == "" {
+ return "", false
+ }
+ for _, tok := range strings.Fields(class) {
+ if word, ok := words[tok]; ok {
+ return word, true
+ }
+ }
+ return "", false
+}
+
+// dropRepeatedTitle removes a leading paragraph that only repeats the chapter
+// label, since the export prints its own heading.
+func dropRepeatedTitle(paragraphs []string, label string) []string {
+ if len(paragraphs) < 2 {
+ return paragraphs
+ }
+ first := strings.TrimSpace(paragraphs[0])
+ if strings.EqualFold(first, strings.TrimSpace(label)) {
+ return paragraphs[1:]
+ }
+ return paragraphs
+}
diff --git a/internal/monkeyd/chapter_test.go b/internal/monkeyd/chapter_test.go
new file mode 100644
index 0000000..94e64a2
--- /dev/null
+++ b/internal/monkeyd/chapter_test.go
@@ -0,0 +1,129 @@
+package monkeyd
+
+import (
+ "strings"
+ "testing"
+)
+
+// chapterFixture mirrors the real page shape: words split between markup and
+// CSS :before rules, HTML entities, spacer paragraphs and an ad script
+// inside the content container.
+const chapterFixture = `
+
+
TEN TRUYEN - 1
+
+
1
+
+
Nghe trưởng tử noi.
+
+
Một và di.
+
+
`
+
+func TestParseWordClassesDecodesEscapes(t *testing.T) {
+ words := ParseWordClasses([]byte(chapterFixture))
+
+ for class, want := range map[string]string{
+ "t-aaa": "vị",
+ "j-bbb": "trồ",
+ "z-ccc": "nàng",
+ } {
+ if got := words[class]; got != want {
+ t.Errorf("words[%q] = %q, want %q", class, got, want)
+ }
+ }
+ if len(words) != 4 {
+ t.Errorf("got %d rules, want 4", len(words))
+ }
+}
+
+func TestParseChapterRestoresCSSWords(t *testing.T) {
+ ch, err := ParseChapter([]byte(chapterFixture), ChapterRef{Label: "1", URL: "http://x/1.html"})
+ if err != nil {
+ t.Fatalf("ParseChapter: %v", err)
+ }
+
+ want := []string{
+ "Nghe vị trưởng tử noi.",
+ "Một trồ và nàng di.",
+ }
+ if len(ch.Paragraphs) != len(want) {
+ t.Fatalf("got %d paragraphs %q, want %d", len(ch.Paragraphs), ch.Paragraphs, len(want))
+ }
+ for i, w := range want {
+ if ch.Paragraphs[i] != w {
+ t.Errorf("paragraph %d = %q, want %q", i, ch.Paragraphs[i], w)
+ }
+ }
+}
+
+// The CSS-injected words are the difference between real text and text with
+// silent holes, so guard against a regression that drops them.
+func TestParseChapterWithoutCSSWouldLoseWords(t *testing.T) {
+ withoutCSS := strings.Replace(chapterFixture, `.t-aaa:before { content: "v\1ecb "; }`, "", 1)
+
+ ch, err := ParseChapter([]byte(withoutCSS), ChapterRef{Label: "1", URL: "http://x/1.html"})
+ if err != nil {
+ t.Fatalf("ParseChapter: %v", err)
+ }
+ if strings.Contains(ch.Paragraphs[0], "vị") {
+ t.Fatal("word appeared without its CSS rule; fixture no longer proves anything")
+ }
+ if want := "Nghe trưởng tử noi."; ch.Paragraphs[0] != want {
+ t.Errorf("paragraph 0 = %q, want %q", ch.Paragraphs[0], want)
+ }
+}
+
+func TestParseChapterDropsScriptsAndSpacers(t *testing.T) {
+ ch, err := ParseChapter([]byte(chapterFixture), ChapterRef{Label: "1", URL: "http://x/1.html"})
+ if err != nil {
+ t.Fatalf("ParseChapter: %v", err)
+ }
+ for _, p := range ch.Paragraphs {
+ if strings.Contains(p, "ads()") {
+ t.Errorf("script text leaked into paragraph %q", p)
+ }
+ if strings.TrimSpace(p) == "" {
+ t.Error("empty spacer paragraph was kept")
+ }
+ if strings.Contains(p, " ") {
+ t.Errorf("non-breaking space survived in %q", p)
+ }
+ }
+}
+
+// The leading "
1
" repeats the chapter label and would print twice.
+func TestParseChapterDropsRepeatedTitle(t *testing.T) {
+ ch, err := ParseChapter([]byte(chapterFixture), ChapterRef{Label: "1", URL: "http://x/1.html"})
+ if err != nil {
+ t.Fatalf("ParseChapter: %v", err)
+ }
+ if ch.Paragraphs[0] == "1" {
+ t.Error("repeated chapter label was kept as a paragraph")
+ }
+}
+
+func TestParseChapterMissingContainer(t *testing.T) {
+ if _, err := ParseChapter([]byte(`
hi
`),
+ ChapterRef{URL: "http://x/1.html"}); err == nil {
+ t.Fatal("want an error when the content container is absent")
+ }
+}
+
+func TestChapterHeading(t *testing.T) {
+ for _, tc := range []struct{ label, want string }{
+ {"14", "Chương 14"},
+ {"Chương 12", "Chương 12"},
+ {"", "Chương"},
+ } {
+ ch := &Chapter{Label: tc.label}
+ if got := ch.Heading(); got != tc.want {
+ t.Errorf("Heading(%q) = %q, want %q", tc.label, got, tc.want)
+ }
+ }
+}
diff --git a/internal/monkeyd/client.go b/internal/monkeyd/client.go
new file mode 100644
index 0000000..5dc7cab
--- /dev/null
+++ b/internal/monkeyd/client.go
@@ -0,0 +1,135 @@
+package monkeyd
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "sync"
+ "time"
+)
+
+// The site rejects requests without a browser-like User-Agent.
+const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
+ "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+
+// maxPageSize caps how much of a response we buffer; chapter pages are ~130 KB.
+const maxPageSize = 8 << 20
+
+// statusError reports an unexpected HTTP status so Get can decide whether
+// retrying is worthwhile.
+type statusError struct {
+ code int
+ status string
+}
+
+func (e *statusError) Error() string { return "unexpected status " + e.status }
+
+// retryable is true for transient failures. A 404 or 403 will not fix itself,
+// so those fail immediately instead of burning the retry budget.
+func (e *statusError) retryable() bool {
+ return e.code == http.StatusTooManyRequests || e.code >= 500
+}
+
+// Client fetches pages from monkeydd.com. It spaces requests out by a fixed
+// delay no matter how many goroutines call Get, so raising the worker count
+// never raises the request rate, and it retries transient failures with
+// exponential backoff.
+type Client struct {
+ http *http.Client
+ ua string
+ delay time.Duration
+ retries int
+
+ mu sync.Mutex
+ nextSlot time.Time
+}
+
+func NewClient(delay time.Duration, retries int) *Client {
+ return &Client{
+ http: &http.Client{Timeout: 45 * time.Second},
+ ua: defaultUserAgent,
+ delay: delay,
+ retries: retries,
+ }
+}
+
+// reserve claims the next request slot and blocks until it comes due, holding
+// the global rate at one request per delay across all callers.
+func (c *Client) reserve(ctx context.Context) error {
+ c.mu.Lock()
+ slot := c.nextSlot
+ if now := time.Now(); slot.Before(now) {
+ slot = now
+ }
+ c.nextSlot = slot.Add(c.delay)
+ c.mu.Unlock()
+
+ wait := time.Until(slot)
+ if wait <= 0 {
+ return nil
+ }
+ timer := time.NewTimer(wait)
+ defer timer.Stop()
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-timer.C:
+ return nil
+ }
+}
+
+// Get fetches url, retrying transient failures.
+func (c *Client) Get(ctx context.Context, url string) ([]byte, error) {
+ var lastErr error
+ for attempt := 0; attempt <= c.retries; attempt++ {
+ if attempt > 0 {
+ backoff := time.Duration(1< 0 {
+ c.logf("warning: %d chapter(s) appear only in the chapter dropdown and were not "+
+ "in the landing page list; verify the export is complete", len(extra))
+ }
+ if len(final) != len(novel.Chapters) {
+ c.logf("chapter list reconciled to %d chapters using the in-chapter dropdown", len(final))
+ }
+ novel.Chapters = final
+ return novel, nil
+}
+
+// Chapters fetches every chapter concurrently and returns them in reading
+// order. Any chapter that cannot be fetched or parsed fails the whole run
+// rather than yielding a book with a hole in it.
+func (c *Crawler) Chapters(ctx context.Context, novel *Novel) ([]*Chapter, error) {
+ chapters := make([]*Chapter, len(novel.Chapters))
+
+ workers := c.Workers
+ if workers < 1 {
+ workers = 1
+ }
+
+ group, groupCtx := errgroup.WithContext(ctx)
+ group.SetLimit(workers)
+
+ var mu sync.Mutex
+ done := 0
+
+ for i, ref := range novel.Chapters {
+ i, ref := i, ref
+ group.Go(func() error {
+ page, err := c.page(groupCtx, ref.URL)
+ if err != nil {
+ return err
+ }
+ chapter, err := ParseChapter(page, ref)
+ if err != nil {
+ return err
+ }
+ chapters[i] = chapter
+
+ mu.Lock()
+ done++
+ c.logf("fetched %d/%d: %s (%d words)", done, len(novel.Chapters),
+ chapter.Heading(), chapter.WordCount())
+ mu.Unlock()
+ return nil
+ })
+ }
+
+ if err := group.Wait(); err != nil {
+ return nil, err
+ }
+ return chapters, nil
+}
+
+// page returns a page from the cache when available, otherwise fetches and
+// caches it.
+func (c *Crawler) page(ctx context.Context, pageURL string) ([]byte, error) {
+ path := c.cachePath(pageURL)
+ if path != "" {
+ if body, err := os.ReadFile(path); err == nil && len(body) > 0 {
+ return body, nil
+ }
+ }
+
+ body, err := c.Client.Get(ctx, pageURL)
+ if err != nil {
+ return nil, err
+ }
+
+ if path != "" {
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err == nil {
+ // A failed cache write must not fail the crawl.
+ _ = os.WriteFile(path, body, 0o644)
+ }
+ }
+ return body, nil
+}
+
+// unsafeFileChars matches everything not allowed in a cache file name.
+var unsafeFileChars = regexp.MustCompile(`[^A-Za-z0-9._-]+`)
+
+// cachePath maps a page URL to a cache file, or "" when caching is disabled.
+func (c *Crawler) cachePath(pageURL string) string {
+ if c.CacheDir == "" {
+ return ""
+ }
+ u, err := url.Parse(pageURL)
+ if err != nil {
+ return ""
+ }
+ name := unsafeFileChars.ReplaceAllString(strings.Trim(u.Path, "/"), "_")
+ if name == "" {
+ return ""
+ }
+ if !strings.HasSuffix(name, ".html") {
+ name += ".html"
+ }
+ return filepath.Join(c.CacheDir, name)
+}
+
+// TotalWords sums the word count across chapters.
+func TotalWords(chapters []*Chapter) int {
+ n := 0
+ for _, ch := range chapters {
+ n += ch.WordCount()
+ }
+ return n
+}
+
+// Describe renders a one-line summary of a crawl result.
+func Describe(novel *Novel, chapters []*Chapter) string {
+ return fmt.Sprintf("%s — %d chapters, %d words", novel.Title, len(chapters), TotalWords(chapters))
+}
diff --git a/internal/monkeyd/css_words.go b/internal/monkeyd/css_words.go
new file mode 100644
index 0000000..6317b25
--- /dev/null
+++ b/internal/monkeyd/css_words.go
@@ -0,0 +1,50 @@
+package monkeyd
+
+import (
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+// The site hides part of every chapter behind CSS rather than putting it in the
+// markup. Chapter HTML carries empty elements such as
+//
+// Nghe trưởng tử
+//
+// and the page stylesheet supplies the missing word:
+//
+// .t-3e625e...:before { content: "vị"; }
+//
+// Reading DOM text alone therefore drops hundreds of words per chapter without
+// any visible error. wordRule finds those rules so the words can be put back.
+var wordRule = regexp.MustCompile(`\.([A-Za-z0-9_-]+)\s*::?before\s*\{[^}]*?content\s*:\s*"((?:[^"\\]|\\.)*)"`)
+
+// cssEscape matches a CSS character escape: a hex code point, optionally
+// followed by one whitespace terminator, or an escaped literal character.
+var cssEscape = regexp.MustCompile(`\\([0-9A-Fa-f]{1,6})\s?|\\(.)`)
+
+// ParseWordClasses maps CSS class name to the word its :before rule injects.
+func ParseWordClasses(page []byte) map[string]string {
+ words := make(map[string]string)
+ for _, m := range wordRule.FindAllSubmatch(page, -1) {
+ words[string(m[1])] = decodeCSSString(string(m[2]))
+ }
+ return words
+}
+
+// decodeCSSString resolves the escape sequences allowed inside a CSS string.
+func decodeCSSString(s string) string {
+ if !strings.Contains(s, `\`) {
+ return s
+ }
+ return cssEscape.ReplaceAllStringFunc(s, func(esc string) string {
+ m := cssEscape.FindStringSubmatch(esc)
+ if m[1] != "" {
+ if cp, err := strconv.ParseInt(m[1], 16, 32); err == nil && cp > 0 {
+ return string(rune(cp))
+ }
+ return ""
+ }
+ return m[2]
+ })
+}
diff --git a/internal/monkeyd/html_nodes.go b/internal/monkeyd/html_nodes.go
new file mode 100644
index 0000000..7d5d16d
--- /dev/null
+++ b/internal/monkeyd/html_nodes.go
@@ -0,0 +1,95 @@
+package monkeyd
+
+import (
+ "strings"
+
+ "golang.org/x/net/html"
+)
+
+// attr returns the value of the named attribute, or "" when absent.
+func attr(n *html.Node, name string) string {
+ for _, a := range n.Attr {
+ if a.Key == name {
+ return a.Val
+ }
+ }
+ return ""
+}
+
+// hasClass reports whether the node carries the given class token.
+func hasClass(n *html.Node, class string) bool {
+ for _, tok := range strings.Fields(attr(n, "class")) {
+ if tok == class {
+ return true
+ }
+ }
+ return false
+}
+
+// findNode returns the first node in document order satisfying match.
+func findNode(root *html.Node, match func(*html.Node) bool) *html.Node {
+ if match(root) {
+ return root
+ }
+ for c := root.FirstChild; c != nil; c = c.NextSibling {
+ if found := findNode(c, match); found != nil {
+ return found
+ }
+ }
+ return nil
+}
+
+// findAllNodes returns every node satisfying match, in document order.
+func findAllNodes(root *html.Node, match func(*html.Node) bool) []*html.Node {
+ var out []*html.Node
+ var walk func(*html.Node)
+ walk = func(n *html.Node) {
+ if match(n) {
+ out = append(out, n)
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ walk(c)
+ }
+ }
+ walk(root)
+ return out
+}
+
+// elementByID finds an element by its id attribute.
+func elementByID(root *html.Node, id string) *html.Node {
+ return findNode(root, func(n *html.Node) bool {
+ return n.Type == html.ElementNode && attr(n, "id") == id
+ })
+}
+
+// elementByTag finds the first element with the given tag name.
+func elementByTag(root *html.Node, tag string) *html.Node {
+ return findNode(root, func(n *html.Node) bool {
+ return n.Type == html.ElementNode && n.Data == tag
+ })
+}
+
+// nodeText collects the descendant text of a node with whitespace collapsed.
+func nodeText(n *html.Node) string {
+ if n == nil {
+ return ""
+ }
+ var b strings.Builder
+ var walk func(*html.Node)
+ walk = func(n *html.Node) {
+ if n.Type == html.TextNode {
+ b.WriteString(n.Data)
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ walk(c)
+ }
+ }
+ walk(n)
+ return collapseSpaces(b.String())
+}
+
+// collapseSpaces trims the string and reduces every whitespace run, including
+// the non-breaking spaces the site emits as , to a single space.
+func collapseSpaces(s string) string {
+ return strings.Join(strings.Fields(s), " ")
+}
diff --git a/internal/monkeyd/novel.go b/internal/monkeyd/novel.go
new file mode 100644
index 0000000..35a4b9e
--- /dev/null
+++ b/internal/monkeyd/novel.go
@@ -0,0 +1,189 @@
+package monkeyd
+
+import (
+ "bytes"
+ "fmt"
+ "net/url"
+ "strings"
+
+ "golang.org/x/net/html"
+)
+
+// ChapterRef points at a single chapter listed on a novel page.
+type ChapterRef struct {
+ Label string // as shown on the site, e.g. "14" or "Chương 12"
+ URL string
+}
+
+// Novel is a novel's landing page: its title and its chapters in reading order.
+type Novel struct {
+ Title string
+ Slug string
+ URL string
+ Chapters []ChapterRef
+}
+
+// ParseNovelPage reads the title and chapter list from a novel landing page.
+//
+// Chapter URLs are always taken from the anchors on the page. Slugs are not
+// uniform across novels ("/14.html" on one, "/chuong-12.html" on another) and
+// numbering has gaps, so generating URLs from a chapter count would fetch 404s
+// and miss real chapters.
+func ParseNovelPage(page []byte, pageURL string) (*Novel, error) {
+ doc, err := html.Parse(bytes.NewReader(page))
+ if err != nil {
+ return nil, fmt.Errorf("parse novel page: %w", err)
+ }
+ base, err := url.Parse(pageURL)
+ if err != nil {
+ return nil, fmt.Errorf("parse novel url: %w", err)
+ }
+
+ novel := &Novel{
+ Title: novelTitle(doc),
+ Slug: slugFromNovelURL(base),
+ URL: pageURL,
+ Chapters: chapterRefsFromList(doc, base),
+ }
+ if novel.Title == "" {
+ novel.Title = novel.Slug
+ }
+ if len(novel.Chapters) == 0 {
+ return nil, fmt.Errorf("no chapters found on %s (page layout may have changed)", pageURL)
+ }
+ return novel, nil
+}
+
+// novelTitle prefers the
heading and falls back to the document title.
+func novelTitle(doc *html.Node) string {
+ if h1 := elementByTag(doc, "h1"); h1 != nil {
+ if t := nodeText(h1); t != "" {
+ return t
+ }
+ }
+ return nodeText(elementByTag(doc, "title"))
+}
+
+// slugFromNovelURL turns https://host/tro-lai-nam-thang-cu.html into
+// "tro-lai-nam-thang-cu".
+func slugFromNovelURL(u *url.URL) string {
+ seg := strings.Trim(u.Path, "/")
+ if i := strings.LastIndex(seg, "/"); i >= 0 {
+ seg = seg[i+1:]
+ }
+ return strings.TrimSuffix(seg, ".html")
+}
+
+// chapterRefsFromList reads the "list-chapters" block on the landing page.
+// The site lists newest first, so the result is reversed into reading order.
+func chapterRefsFromList(doc *html.Node, base *url.URL) []ChapterRef {
+ list := findNode(doc, func(n *html.Node) bool {
+ return n.Type == html.ElementNode && hasClass(n, "list-chapters")
+ })
+ if list == nil {
+ return nil
+ }
+
+ titles := findAllNodes(list, func(n *html.Node) bool {
+ return n.Type == html.ElementNode && hasClass(n, "episode-title")
+ })
+
+ var refs []ChapterRef
+ for _, title := range titles {
+ link := elementByTag(title, "a")
+ if link == nil {
+ continue
+ }
+ href := strings.TrimSpace(attr(link, "href"))
+ if href == "" {
+ continue
+ }
+ abs, err := base.Parse(href)
+ if err != nil {
+ continue
+ }
+ refs = append(refs, ChapterRef{Label: nodeText(link), URL: abs.String()})
+ }
+ return reverseRefs(refs)
+}
+
+// ChapterRefsFromSelect reads the chapter dropdown embedded in every chapter
+// page, whose options hold "novel-slug,chapter-slug" pairs. This is a second,
+// independent view of the chapter list used to cross-check the landing page.
+func ChapterRefsFromSelect(page []byte, base *url.URL) ([]ChapterRef, error) {
+ doc, err := html.Parse(bytes.NewReader(page))
+ if err != nil {
+ return nil, fmt.Errorf("parse chapter page: %w", err)
+ }
+ sel := elementByID(doc, "selected_chapter")
+ if sel == nil {
+ return nil, nil
+ }
+
+ var refs []ChapterRef
+ for _, opt := range findAllNodes(sel, func(n *html.Node) bool {
+ return n.Type == html.ElementNode && n.Data == "option"
+ }) {
+ novelSlug, chapterSlug, ok := strings.Cut(attr(opt, "value"), ",")
+ if !ok || novelSlug == "" || chapterSlug == "" {
+ continue
+ }
+ abs, err := base.Parse("/" + novelSlug + "/" + chapterSlug + ".html")
+ if err != nil {
+ continue
+ }
+ refs = append(refs, ChapterRef{Label: nodeText(opt), URL: abs.String()})
+ }
+ return reverseRefs(refs), nil
+}
+
+// ReconcileChapterRefs picks the chapter list to crawl from the landing page
+// list and the in-chapter dropdown.
+//
+// Both views come from the same site ordering, so when the dropdown is a
+// superset it is preferred: that keeps the run correct even if the landing page
+// ever truncates or paginates its list. Anything the dropdown alone knows about
+// while disagreeing on order is returned as extra so the caller can warn rather
+// than silently export a short book.
+func ReconcileChapterRefs(fromList, fromSelect []ChapterRef) (final, extra []ChapterRef) {
+ if len(fromSelect) == 0 {
+ return fromList, nil
+ }
+
+ inSelect := refURLSet(fromSelect)
+ if listIsSubsetOf(fromList, inSelect) {
+ return fromSelect, nil
+ }
+
+ inList := refURLSet(fromList)
+ for _, ref := range fromSelect {
+ if !inList[ref.URL] {
+ extra = append(extra, ref)
+ }
+ }
+ return fromList, extra
+}
+
+func refURLSet(refs []ChapterRef) map[string]bool {
+ set := make(map[string]bool, len(refs))
+ for _, r := range refs {
+ set[r.URL] = true
+ }
+ return set
+}
+
+func listIsSubsetOf(refs []ChapterRef, set map[string]bool) bool {
+ for _, r := range refs {
+ if !set[r.URL] {
+ return false
+ }
+ }
+ return true
+}
+
+func reverseRefs(refs []ChapterRef) []ChapterRef {
+ for i, j := 0, len(refs)-1; i < j; i, j = i+1, j-1 {
+ refs[i], refs[j] = refs[j], refs[i]
+ }
+ return refs
+}
diff --git a/internal/monkeyd/novel_test.go b/internal/monkeyd/novel_test.go
new file mode 100644
index 0000000..112d344
--- /dev/null
+++ b/internal/monkeyd/novel_test.go
@@ -0,0 +1,119 @@
+package monkeyd
+
+import (
+ "net/url"
+ "testing"
+)
+
+// novelFixture reproduces the two traits that break naive crawlers: the list is
+// newest-first, and chapter numbering has a gap (no chapter 4).
+const novelFixture = `TEN TRUYEN
+