refactor: make crawler importable and add export pipeline

Move monkeyd and pdfout out of internal/ so other modules can import them,
and add export.Export, which holds the crawl-to-PDF sequence the CLI used to
inline. The CLI now parses flags and delegates, so an embedding program gets
the same defaults and validation.
This commit is contained in:
2026-07-29 22:49:30 +07:00
parent 07c0b125f4
commit c925f10ae0
15 changed files with 490 additions and 128 deletions
+26 -2
View File
@@ -104,10 +104,34 @@ chapter ordering, and the numbering gap. No network access required.
```
cmd/monkeyd-crawler/ CLI
internal/monkeyd/ fetching, HTML/CSS parsing, crawl orchestration
internal/pdfout/ PDF rendering and font discovery
export/ URL -> PDF in one call; shared by the CLI and importers
monkeyd/ fetching, HTML/CSS parsing, crawl orchestration
pdfout/ PDF rendering and font discovery
```
## Use as a library
The packages are importable, so another Go program can produce the same PDF
without shelling out to the binary. `export.Export` is the whole pipeline —
chapter list, fetch, font discovery, render:
```go
result, err := export.Export(ctx, export.Request{
NovelURL: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
OutDir: tmpDir,
})
```
Only `NovelURL` is required; each zero-valued field falls back to the same
default as the matching CLI flag. Because zero means "unset", ask for *no*
cache or *no* request delay with the `NoCache` and `NoDelay` fields rather than
by zeroing `CacheDir` or `Delay`. Pass a `Log` function to receive the progress
messages the CLI prints to stderr.
Callers running in a container should note that `pdfout.FindFont` searches
system font paths: a minimal image with no fonts installed needs either a font
present or an explicit `FontFile`.
## Scope
Downloaded text stays on your machine; only fetch content you are allowed to read offline, and
+33 -126
View File
@@ -6,34 +6,16 @@ import (
"context"
"flag"
"fmt"
"net/url"
"os"
"os/signal"
"path/filepath"
"regexp"
"strings"
"syscall"
"time"
"github.com/tiennm99/monkeyd-crawler/internal/monkeyd"
"github.com/tiennm99/monkeyd-crawler/internal/pdfout"
"github.com/tiennm99/monkeyd-crawler/export"
"github.com/tiennm99/monkeyd-crawler/pdfout"
)
type config struct {
novelURL string
out string
page string
fontFile string
fontSize float64
lineSpacing float64
margin float64
workers int
delay time.Duration
retries int
limit int
cacheDir string
}
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, "error:", err)
@@ -42,7 +24,7 @@ func main() {
}
func run() error {
cfg, err := parseFlags()
req, err := parseFlags()
if err != nil {
return err
}
@@ -51,78 +33,41 @@ func run() error {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
crawler := &monkeyd.Crawler{
Client: monkeyd.NewClient(cfg.delay, cfg.retries),
CacheDir: cfg.cacheDir,
Workers: cfg.workers,
Log: func(format string, args ...any) {
fmt.Fprintf(os.Stderr, format+"\n", args...)
},
req.Log = func(format string, args ...any) {
fmt.Fprintf(os.Stderr, format+"\n", args...)
}
novel, err := crawler.Novel(ctx, cfg.novelURL)
result, err := export.Export(ctx, *req)
if err != nil {
return err
}
if cfg.limit > 0 && cfg.limit < len(novel.Chapters) {
fmt.Fprintf(os.Stderr, "limiting to first %d of %d chapters\n", cfg.limit, len(novel.Chapters))
novel.Chapters = novel.Chapters[:cfg.limit]
}
chapters, err := crawler.Chapters(ctx, novel)
if err != nil {
return err
}
fontFile := cfg.fontFile
if fontFile == "" {
if fontFile, err = pdfout.FindFont(); err != nil {
return err
}
}
outPath := cfg.out
if outPath == "" {
outPath = safeFileName(novel.Title, novel.Slug) + ".pdf"
}
opts := pdfout.Options{
Page: pdfout.Presets[cfg.page],
Margin: cfg.margin,
FontFile: fontFile,
FontSize: cfg.fontSize,
LineSpacing: cfg.lineSpacing,
Title: novel.Title,
SourceURL: novel.URL,
}
if err := pdfout.Write(outPath, opts, toPDFChapters(chapters)); err != nil {
return err
}
fmt.Fprintf(os.Stderr, "\n%s\n", monkeyd.Describe(novel, chapters))
fmt.Fprintf(os.Stderr, "\n%s\n", result.Summary())
fmt.Fprintf(os.Stderr, "font: %s at %.0fpt on %s page (%.0f x %.0f mm)\n",
filepath.Base(fontFile), cfg.fontSize, opts.Page.Name, opts.Page.W, opts.Page.H)
fmt.Println(outPath)
filepath.Base(result.FontFile), req.FontSize, result.Page.Name, result.Page.W, result.Page.H)
fmt.Println(result.Path)
return nil
}
func parseFlags() (*config, error) {
cfg := &config{}
// parseFlags builds the export request from the command line. Validation of the
// resulting values lives in export.Export, so the CLI and an embedding program
// reject the same inputs.
func parseFlags() (*export.Request, error) {
req := &export.Request{}
flag.StringVar(&cfg.novelURL, "url", "", "novel page URL, e.g. https://monkeydd.com/tro-lai-nam-thang-cu.html")
flag.StringVar(&cfg.out, "out", "", "output PDF path (default: novel title)")
flag.StringVar(&cfg.page, "page", "phone",
flag.StringVar(&req.NovelURL, "url", "", "novel page URL, e.g. https://monkeydd.com/tro-lai-nam-thang-cu.html")
flag.StringVar(&req.OutPath, "out", "", "output PDF path (default: novel title)")
flag.StringVar(&req.Page, "page", export.DefaultPage,
"page size: "+strings.Join(pdfout.PresetNames(), ", "))
flag.StringVar(&cfg.fontFile, "font", "", "path to a .ttf font (default: a Vietnamese-capable system font)")
flag.Float64Var(&cfg.fontSize, "font-size", 12, "body font size in points")
flag.Float64Var(&cfg.lineSpacing, "line-spacing", 1.55, "line height as a multiple of font size")
flag.Float64Var(&cfg.margin, "margin", 6, "page margin in millimetres")
flag.IntVar(&cfg.workers, "workers", 4, "concurrent chapter fetches")
flag.DurationVar(&cfg.delay, "delay", 400*time.Millisecond, "minimum delay between requests")
flag.IntVar(&cfg.retries, "retries", 3, "retries per request")
flag.IntVar(&cfg.limit, "limit", 0, "only fetch the first N chapters (0 = all)")
flag.StringVar(&cfg.cacheDir, "cache", ".cache",
flag.StringVar(&req.FontFile, "font", "", "path to a .ttf font (default: a Vietnamese-capable system font)")
flag.Float64Var(&req.FontSize, "font-size", export.DefaultFontSize, "body font size in points")
flag.Float64Var(&req.LineSpacing, "line-spacing", export.DefaultLineSpacing, "line height as a multiple of font size")
flag.Float64Var(&req.Margin, "margin", export.DefaultMargin, "page margin in millimetres")
flag.IntVar(&req.Workers, "workers", export.DefaultWorkers, "concurrent chapter fetches")
flag.DurationVar(&req.Delay, "delay", export.DefaultDelay, "minimum delay between requests")
flag.IntVar(&req.Retries, "retries", export.DefaultRetries, "retries per request")
flag.IntVar(&req.Limit, "limit", 0, "only fetch the first N chapters (0 = all)")
flag.StringVar(&req.CacheDir, "cache", export.DefaultCacheDir,
"directory for cached pages, so re-exports need no requests (empty to disable)")
flag.Usage = func() {
@@ -133,52 +78,14 @@ func parseFlags() (*config, error) {
}
flag.Parse()
if cfg.novelURL == "" {
if req.NovelURL == "" {
flag.Usage()
return nil, fmt.Errorf("-url is required")
}
parsed, err := url.Parse(cfg.novelURL)
if err != nil {
return nil, fmt.Errorf("invalid -url: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return nil, fmt.Errorf("invalid -url: want an http(s) URL, got %q", cfg.novelURL)
}
if _, ok := pdfout.Presets[cfg.page]; !ok {
return nil, fmt.Errorf("unknown -page %q: want one of %s",
cfg.page, strings.Join(pdfout.PresetNames(), ", "))
}
if cfg.fontSize <= 0 {
return nil, fmt.Errorf("-font-size must be positive")
}
if cfg.lineSpacing <= 0 {
return nil, fmt.Errorf("-line-spacing must be positive")
}
if cfg.margin < 0 {
return nil, fmt.Errorf("-margin cannot be negative")
}
if cfg.workers < 1 {
return nil, fmt.Errorf("-workers must be at least 1")
}
return cfg, nil
}
func toPDFChapters(chapters []*monkeyd.Chapter) []pdfout.Chapter {
out := make([]pdfout.Chapter, 0, len(chapters))
for _, ch := range chapters {
out = append(out, pdfout.Chapter{Heading: ch.Heading(), Paragraphs: ch.Paragraphs})
}
return out
}
var unsafeNameChars = regexp.MustCompile(`[^\p{L}\p{N}]+`)
// safeFileName builds a file name from the novel title, falling back to the
// slug when the title has no usable characters.
func safeFileName(title, fallback string) string {
name := strings.Trim(unsafeNameChars.ReplaceAllString(title, "-"), "-")
if name == "" {
return fallback
}
return name
// The flag defaults above are already applied, so a zero value here can
// only come from the user asking for none. Say so explicitly: left as the
// zero value, Export would read it as "unset" and restore the default.
req.NoCache = req.CacheDir == ""
req.NoDelay = req.Delay == 0
return req, nil
}
+244
View File
@@ -0,0 +1,244 @@
// Package export turns a novel URL into a PDF file. It holds the sequence the
// CLI and any embedding program both need — resolve the chapter list, fetch the
// chapters, pick a font, render the PDF — so neither has to reassemble it.
package export
import (
"context"
"fmt"
"net/url"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/tiennm99/monkeyd-crawler/monkeyd"
"github.com/tiennm99/monkeyd-crawler/pdfout"
)
// Defaults for every tunable field of Request. Exported so a caller's own
// flags or config can advertise the same values instead of restating them.
const (
DefaultPage = "phone"
DefaultFontSize = 12.0
DefaultLineSpacing = 1.55
DefaultMargin = 6.0
DefaultWorkers = 4
DefaultDelay = 400 * time.Millisecond
DefaultRetries = 3
DefaultCacheDir = ".cache"
)
// Request describes one export. Only NovelURL is required; every zero-valued
// tunable falls back to its Default above, so a caller that only has a URL can
// leave the rest alone.
type Request struct {
NovelURL string
// OutPath is the exact PDF path to write. When empty the file is named
// after the novel title and placed in OutDir.
OutPath string
OutDir string
Page string // preset key: phone, a5, a4
FontFile string // path to a .ttf; empty means discover a system font
FontSize float64 // points
LineSpacing float64 // multiple of font size
Margin float64 // millimetres
Workers int
Retries int
// Delay is the minimum spacing between requests. Set NoDelay to remove
// the spacing rather than setting this to 0, which is read as "unset"
// and gets the default back.
Delay time.Duration
NoDelay bool
// Limit caps the export to the first N chapters. 0 means every chapter.
Limit int
// CacheDir stores raw pages so a re-export costs no requests. Set
// NoCache to opt out rather than clearing this field, which would be
// read as "unset" and get the default back.
CacheDir string
NoCache bool
// Log receives progress messages. Optional.
Log func(format string, args ...any)
}
// Result reports what was produced.
type Result struct {
Path string
Title string
SourceURL string
Chapters int
Words int
FontFile string
Page pdfout.PageSize
}
// Summary renders a one-line description of the exported book.
func (r *Result) Summary() string {
return fmt.Sprintf("%s — %d chapters, %d words", r.Title, r.Chapters, r.Words)
}
// Export fetches the novel at req.NovelURL and writes it as a PDF, returning
// where it landed. The context bounds the whole crawl; cancelling it abandons
// the run without leaving a partial PDF behind.
func Export(ctx context.Context, req Request) (*Result, error) {
req.applyDefaults()
if err := req.validate(); err != nil {
return nil, err
}
crawler := &monkeyd.Crawler{
Client: monkeyd.NewClient(req.Delay, req.Retries),
CacheDir: req.CacheDir,
Workers: req.Workers,
Log: req.Log,
}
novel, err := crawler.Novel(ctx, req.NovelURL)
if err != nil {
return nil, err
}
if req.Limit > 0 && req.Limit < len(novel.Chapters) {
req.logf("limiting to first %d of %d chapters", req.Limit, len(novel.Chapters))
novel.Chapters = novel.Chapters[:req.Limit]
}
chapters, err := crawler.Chapters(ctx, novel)
if err != nil {
return nil, err
}
fontFile := req.FontFile
if fontFile == "" {
if fontFile, err = pdfout.FindFont(); err != nil {
return nil, err
}
}
outPath := req.OutPath
if outPath == "" {
outPath = filepath.Join(req.OutDir, SafeFileName(novel.Title, novel.Slug)+".pdf")
}
page := pdfout.Presets[req.Page]
opts := pdfout.Options{
Page: page,
Margin: req.Margin,
FontFile: fontFile,
FontSize: req.FontSize,
LineSpacing: req.LineSpacing,
Title: novel.Title,
SourceURL: novel.URL,
}
if err := pdfout.Write(outPath, opts, toPDFChapters(chapters)); err != nil {
return nil, err
}
return &Result{
Path: outPath,
Title: novel.Title,
SourceURL: novel.URL,
Chapters: len(chapters),
Words: monkeyd.TotalWords(chapters),
FontFile: fontFile,
Page: page,
}, nil
}
func (r *Request) applyDefaults() {
if r.Page == "" {
r.Page = DefaultPage
}
if r.FontSize == 0 {
r.FontSize = DefaultFontSize
}
if r.LineSpacing == 0 {
r.LineSpacing = DefaultLineSpacing
}
if r.Margin == 0 {
r.Margin = DefaultMargin
}
if r.Workers == 0 {
r.Workers = DefaultWorkers
}
switch {
case r.NoDelay:
r.Delay = 0
case r.Delay == 0:
r.Delay = DefaultDelay
}
if r.Retries == 0 {
r.Retries = DefaultRetries
}
switch {
case r.NoCache:
r.CacheDir = ""
case r.CacheDir == "":
r.CacheDir = DefaultCacheDir
}
}
// validate rejects a Request before any request is made, so a typo costs no
// fetches. Call applyDefaults first: it checks the effective values.
func (r *Request) validate() error {
if r.NovelURL == "" {
return fmt.Errorf("novel url is required")
}
parsed, err := url.Parse(r.NovelURL)
if err != nil {
return fmt.Errorf("invalid novel url: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return fmt.Errorf("invalid novel url: want an http(s) URL, got %q", r.NovelURL)
}
if _, ok := pdfout.Presets[r.Page]; !ok {
return fmt.Errorf("unknown page %q: want one of %s",
r.Page, strings.Join(pdfout.PresetNames(), ", "))
}
if r.FontSize <= 0 {
return fmt.Errorf("font size must be positive")
}
if r.LineSpacing <= 0 {
return fmt.Errorf("line spacing must be positive")
}
if r.Margin < 0 {
return fmt.Errorf("margin cannot be negative")
}
if r.Workers < 1 {
return fmt.Errorf("workers must be at least 1")
}
return nil
}
func (r *Request) logf(format string, args ...any) {
if r.Log != nil {
r.Log(format, args...)
}
}
func toPDFChapters(chapters []*monkeyd.Chapter) []pdfout.Chapter {
out := make([]pdfout.Chapter, 0, len(chapters))
for _, ch := range chapters {
out = append(out, pdfout.Chapter{Heading: ch.Heading(), Paragraphs: ch.Paragraphs})
}
return out
}
var unsafeNameChars = regexp.MustCompile(`[^\p{L}\p{N}]+`)
// SafeFileName builds a file name from the novel title, falling back to the
// slug when the title has no usable characters. The result has no extension.
func SafeFileName(title, fallback string) string {
name := strings.Trim(unsafeNameChars.ReplaceAllString(title, "-"), "-")
if name == "" {
return fallback
}
return name
}
+187
View File
@@ -0,0 +1,187 @@
package export
import (
"testing"
"time"
)
func TestApplyDefaultsFillsUnsetFields(t *testing.T) {
req := Request{NovelURL: "https://monkeydd.com/example.html"}
req.applyDefaults()
if req.Page != DefaultPage {
t.Errorf("Page = %q, want %q", req.Page, DefaultPage)
}
if req.FontSize != DefaultFontSize {
t.Errorf("FontSize = %v, want %v", req.FontSize, DefaultFontSize)
}
if req.LineSpacing != DefaultLineSpacing {
t.Errorf("LineSpacing = %v, want %v", req.LineSpacing, DefaultLineSpacing)
}
if req.Margin != DefaultMargin {
t.Errorf("Margin = %v, want %v", req.Margin, DefaultMargin)
}
if req.Workers != DefaultWorkers {
t.Errorf("Workers = %v, want %v", req.Workers, DefaultWorkers)
}
if req.Delay != DefaultDelay {
t.Errorf("Delay = %v, want %v", req.Delay, DefaultDelay)
}
if req.Retries != DefaultRetries {
t.Errorf("Retries = %v, want %v", req.Retries, DefaultRetries)
}
if req.CacheDir != DefaultCacheDir {
t.Errorf("CacheDir = %q, want %q", req.CacheDir, DefaultCacheDir)
}
}
func TestApplyDefaultsKeepsExplicitValues(t *testing.T) {
req := Request{
NovelURL: "https://monkeydd.com/example.html",
Page: "a5",
FontSize: 14,
LineSpacing: 1.4,
Margin: 10,
Workers: 2,
Delay: time.Second,
Retries: 1,
CacheDir: "/tmp/pages",
}
req.applyDefaults()
if req.Page != "a5" || req.FontSize != 14 || req.LineSpacing != 1.4 {
t.Errorf("layout fields overwritten: %+v", req)
}
if req.Margin != 10 || req.Workers != 2 || req.Delay != time.Second || req.Retries != 1 {
t.Errorf("fetch fields overwritten: %+v", req)
}
if req.CacheDir != "/tmp/pages" {
t.Errorf("CacheDir = %q, want /tmp/pages", req.CacheDir)
}
}
// A zero Margin is indistinguishable from "unset", so it becomes the default.
// NoCache and NoDelay exist precisely because "none" must survive that rule.
func TestApplyDefaultsHonoursOptOuts(t *testing.T) {
req := Request{NovelURL: "https://monkeydd.com/example.html", NoCache: true, NoDelay: true}
req.applyDefaults()
if req.CacheDir != "" {
t.Errorf("NoCache left CacheDir = %q, want empty", req.CacheDir)
}
if req.Delay != 0 {
t.Errorf("NoDelay left Delay = %v, want 0", req.Delay)
}
}
// Opt-outs win over an explicitly set value so a caller cannot end up with both.
func TestApplyDefaultsOptOutsOverrideExplicitValues(t *testing.T) {
req := Request{
NovelURL: "https://monkeydd.com/example.html",
CacheDir: "/tmp/pages",
NoCache: true,
Delay: time.Second,
NoDelay: true,
}
req.applyDefaults()
if req.CacheDir != "" {
t.Errorf("CacheDir = %q, want empty", req.CacheDir)
}
if req.Delay != 0 {
t.Errorf("Delay = %v, want 0", req.Delay)
}
}
func TestValidate(t *testing.T) {
tests := []struct {
name string
req Request
wantErr bool
}{
{
name: "defaults are valid",
req: Request{NovelURL: "https://monkeydd.com/example.html"},
},
{
name: "missing url",
req: Request{},
wantErr: true,
},
{
name: "non-http scheme",
req: Request{NovelURL: "ftp://monkeydd.com/example.html"},
wantErr: true,
},
{
name: "scheme-less url",
req: Request{NovelURL: "monkeydd.com/example.html"},
wantErr: true,
},
{
name: "unknown page preset",
req: Request{NovelURL: "https://monkeydd.com/example.html", Page: "letter"},
wantErr: true,
},
{
name: "negative font size",
req: Request{NovelURL: "https://monkeydd.com/example.html", FontSize: -1},
wantErr: true,
},
{
name: "negative line spacing",
req: Request{NovelURL: "https://monkeydd.com/example.html", LineSpacing: -1},
wantErr: true,
},
{
name: "negative margin",
req: Request{NovelURL: "https://monkeydd.com/example.html", Margin: -1},
wantErr: true,
},
{
name: "zero workers is filled by defaults, negative is not",
req: Request{NovelURL: "https://monkeydd.com/example.html", Workers: -1},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := tt.req
req.applyDefaults()
err := req.validate()
if tt.wantErr && err == nil {
t.Error("validate() = nil, want error")
}
if !tt.wantErr && err != nil {
t.Errorf("validate() = %v, want nil", err)
}
})
}
}
func TestSafeFileName(t *testing.T) {
tests := []struct {
title string
fallback string
want string
}{
{"Trở Lại Năm Tháng Cũ", "slug", "Trở-Lại-Năm-Tháng-Cũ"},
{"Chapter: One / Two", "slug", "Chapter-One-Two"},
{" --- ", "slug", "slug"},
{"", "slug", "slug"},
}
for _, tt := range tests {
if got := SafeFileName(tt.title, tt.fallback); got != tt.want {
t.Errorf("SafeFileName(%q, %q) = %q, want %q", tt.title, tt.fallback, got, tt.want)
}
}
}
func TestResultSummary(t *testing.T) {
r := &Result{Title: "Example", Chapters: 12, Words: 3400}
want := "Example — 12 chapters, 3400 words"
if got := r.Summary(); got != want {
t.Errorf("Summary() = %q, want %q", got, want)
}
}