diff --git a/README.md b/README.md index 484e04e..02729e2 100644 --- a/README.md +++ b/README.md @@ -104,10 +104,34 @@ chapter ordering, and the numbering gap. No network access required. ``` cmd/monkeyd-crawler/ CLI -internal/monkeyd/ fetching, HTML/CSS parsing, crawl orchestration -internal/pdfout/ PDF rendering and font discovery +export/ URL -> PDF in one call; shared by the CLI and importers +monkeyd/ fetching, HTML/CSS parsing, crawl orchestration +pdfout/ PDF rendering and font discovery ``` +## Use as a library + +The packages are importable, so another Go program can produce the same PDF +without shelling out to the binary. `export.Export` is the whole pipeline — +chapter list, fetch, font discovery, render: + +```go +result, err := export.Export(ctx, export.Request{ + NovelURL: "https://monkeydd.com/tro-lai-nam-thang-cu.html", + OutDir: tmpDir, +}) +``` + +Only `NovelURL` is required; each zero-valued field falls back to the same +default as the matching CLI flag. Because zero means "unset", ask for *no* +cache or *no* request delay with the `NoCache` and `NoDelay` fields rather than +by zeroing `CacheDir` or `Delay`. Pass a `Log` function to receive the progress +messages the CLI prints to stderr. + +Callers running in a container should note that `pdfout.FindFont` searches +system font paths: a minimal image with no fonts installed needs either a font +present or an explicit `FontFile`. + ## Scope Downloaded text stays on your machine; only fetch content you are allowed to read offline, and diff --git a/cmd/monkeyd-crawler/main.go b/cmd/monkeyd-crawler/main.go index 41c4b18..2b0570a 100644 --- a/cmd/monkeyd-crawler/main.go +++ b/cmd/monkeyd-crawler/main.go @@ -6,34 +6,16 @@ import ( "context" "flag" "fmt" - "net/url" "os" "os/signal" "path/filepath" - "regexp" "strings" "syscall" - "time" - "github.com/tiennm99/monkeyd-crawler/internal/monkeyd" - "github.com/tiennm99/monkeyd-crawler/internal/pdfout" + "github.com/tiennm99/monkeyd-crawler/export" + "github.com/tiennm99/monkeyd-crawler/pdfout" ) -type config struct { - novelURL string - out string - page string - fontFile string - fontSize float64 - lineSpacing float64 - margin float64 - workers int - delay time.Duration - retries int - limit int - cacheDir string -} - func main() { if err := run(); err != nil { fmt.Fprintln(os.Stderr, "error:", err) @@ -42,7 +24,7 @@ func main() { } func run() error { - cfg, err := parseFlags() + req, err := parseFlags() if err != nil { return err } @@ -51,78 +33,41 @@ func run() error { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - crawler := &monkeyd.Crawler{ - Client: monkeyd.NewClient(cfg.delay, cfg.retries), - CacheDir: cfg.cacheDir, - Workers: cfg.workers, - Log: func(format string, args ...any) { - fmt.Fprintf(os.Stderr, format+"\n", args...) - }, + req.Log = func(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) } - novel, err := crawler.Novel(ctx, cfg.novelURL) + result, err := export.Export(ctx, *req) if err != nil { return err } - if cfg.limit > 0 && cfg.limit < len(novel.Chapters) { - fmt.Fprintf(os.Stderr, "limiting to first %d of %d chapters\n", cfg.limit, len(novel.Chapters)) - novel.Chapters = novel.Chapters[:cfg.limit] - } - - chapters, err := crawler.Chapters(ctx, novel) - if err != nil { - return err - } - - fontFile := cfg.fontFile - if fontFile == "" { - if fontFile, err = pdfout.FindFont(); err != nil { - return err - } - } - - outPath := cfg.out - if outPath == "" { - outPath = safeFileName(novel.Title, novel.Slug) + ".pdf" - } - - opts := pdfout.Options{ - Page: pdfout.Presets[cfg.page], - Margin: cfg.margin, - FontFile: fontFile, - FontSize: cfg.fontSize, - LineSpacing: cfg.lineSpacing, - Title: novel.Title, - SourceURL: novel.URL, - } - if err := pdfout.Write(outPath, opts, toPDFChapters(chapters)); err != nil { - return err - } - - fmt.Fprintf(os.Stderr, "\n%s\n", monkeyd.Describe(novel, chapters)) + fmt.Fprintf(os.Stderr, "\n%s\n", result.Summary()) fmt.Fprintf(os.Stderr, "font: %s at %.0fpt on %s page (%.0f x %.0f mm)\n", - filepath.Base(fontFile), cfg.fontSize, opts.Page.Name, opts.Page.W, opts.Page.H) - fmt.Println(outPath) + filepath.Base(result.FontFile), req.FontSize, result.Page.Name, result.Page.W, result.Page.H) + fmt.Println(result.Path) return nil } -func parseFlags() (*config, error) { - cfg := &config{} +// parseFlags builds the export request from the command line. Validation of the +// resulting values lives in export.Export, so the CLI and an embedding program +// reject the same inputs. +func parseFlags() (*export.Request, error) { + req := &export.Request{} - flag.StringVar(&cfg.novelURL, "url", "", "novel page URL, e.g. https://monkeydd.com/tro-lai-nam-thang-cu.html") - flag.StringVar(&cfg.out, "out", "", "output PDF path (default: novel title)") - flag.StringVar(&cfg.page, "page", "phone", + flag.StringVar(&req.NovelURL, "url", "", "novel page URL, e.g. https://monkeydd.com/tro-lai-nam-thang-cu.html") + flag.StringVar(&req.OutPath, "out", "", "output PDF path (default: novel title)") + flag.StringVar(&req.Page, "page", export.DefaultPage, "page size: "+strings.Join(pdfout.PresetNames(), ", ")) - flag.StringVar(&cfg.fontFile, "font", "", "path to a .ttf font (default: a Vietnamese-capable system font)") - flag.Float64Var(&cfg.fontSize, "font-size", 12, "body font size in points") - flag.Float64Var(&cfg.lineSpacing, "line-spacing", 1.55, "line height as a multiple of font size") - flag.Float64Var(&cfg.margin, "margin", 6, "page margin in millimetres") - flag.IntVar(&cfg.workers, "workers", 4, "concurrent chapter fetches") - flag.DurationVar(&cfg.delay, "delay", 400*time.Millisecond, "minimum delay between requests") - flag.IntVar(&cfg.retries, "retries", 3, "retries per request") - flag.IntVar(&cfg.limit, "limit", 0, "only fetch the first N chapters (0 = all)") - flag.StringVar(&cfg.cacheDir, "cache", ".cache", + flag.StringVar(&req.FontFile, "font", "", "path to a .ttf font (default: a Vietnamese-capable system font)") + flag.Float64Var(&req.FontSize, "font-size", export.DefaultFontSize, "body font size in points") + flag.Float64Var(&req.LineSpacing, "line-spacing", export.DefaultLineSpacing, "line height as a multiple of font size") + flag.Float64Var(&req.Margin, "margin", export.DefaultMargin, "page margin in millimetres") + flag.IntVar(&req.Workers, "workers", export.DefaultWorkers, "concurrent chapter fetches") + flag.DurationVar(&req.Delay, "delay", export.DefaultDelay, "minimum delay between requests") + flag.IntVar(&req.Retries, "retries", export.DefaultRetries, "retries per request") + flag.IntVar(&req.Limit, "limit", 0, "only fetch the first N chapters (0 = all)") + flag.StringVar(&req.CacheDir, "cache", export.DefaultCacheDir, "directory for cached pages, so re-exports need no requests (empty to disable)") flag.Usage = func() { @@ -133,52 +78,14 @@ func parseFlags() (*config, error) { } flag.Parse() - if cfg.novelURL == "" { + if req.NovelURL == "" { flag.Usage() return nil, fmt.Errorf("-url is required") } - parsed, err := url.Parse(cfg.novelURL) - if err != nil { - return nil, fmt.Errorf("invalid -url: %w", err) - } - if parsed.Scheme != "http" && parsed.Scheme != "https" { - return nil, fmt.Errorf("invalid -url: want an http(s) URL, got %q", cfg.novelURL) - } - if _, ok := pdfout.Presets[cfg.page]; !ok { - return nil, fmt.Errorf("unknown -page %q: want one of %s", - cfg.page, strings.Join(pdfout.PresetNames(), ", ")) - } - if cfg.fontSize <= 0 { - return nil, fmt.Errorf("-font-size must be positive") - } - if cfg.lineSpacing <= 0 { - return nil, fmt.Errorf("-line-spacing must be positive") - } - if cfg.margin < 0 { - return nil, fmt.Errorf("-margin cannot be negative") - } - if cfg.workers < 1 { - return nil, fmt.Errorf("-workers must be at least 1") - } - return cfg, nil -} - -func toPDFChapters(chapters []*monkeyd.Chapter) []pdfout.Chapter { - out := make([]pdfout.Chapter, 0, len(chapters)) - for _, ch := range chapters { - out = append(out, pdfout.Chapter{Heading: ch.Heading(), Paragraphs: ch.Paragraphs}) - } - return out -} - -var unsafeNameChars = regexp.MustCompile(`[^\p{L}\p{N}]+`) - -// safeFileName builds a file name from the novel title, falling back to the -// slug when the title has no usable characters. -func safeFileName(title, fallback string) string { - name := strings.Trim(unsafeNameChars.ReplaceAllString(title, "-"), "-") - if name == "" { - return fallback - } - return name + // The flag defaults above are already applied, so a zero value here can + // only come from the user asking for none. Say so explicitly: left as the + // zero value, Export would read it as "unset" and restore the default. + req.NoCache = req.CacheDir == "" + req.NoDelay = req.Delay == 0 + return req, nil } diff --git a/export/export.go b/export/export.go new file mode 100644 index 0000000..cf6fd0e --- /dev/null +++ b/export/export.go @@ -0,0 +1,244 @@ +// Package export turns a novel URL into a PDF file. It holds the sequence the +// CLI and any embedding program both need — resolve the chapter list, fetch the +// chapters, pick a font, render the PDF — so neither has to reassemble it. +package export + +import ( + "context" + "fmt" + "net/url" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/tiennm99/monkeyd-crawler/monkeyd" + "github.com/tiennm99/monkeyd-crawler/pdfout" +) + +// Defaults for every tunable field of Request. Exported so a caller's own +// flags or config can advertise the same values instead of restating them. +const ( + DefaultPage = "phone" + DefaultFontSize = 12.0 + DefaultLineSpacing = 1.55 + DefaultMargin = 6.0 + DefaultWorkers = 4 + DefaultDelay = 400 * time.Millisecond + DefaultRetries = 3 + DefaultCacheDir = ".cache" +) + +// Request describes one export. Only NovelURL is required; every zero-valued +// tunable falls back to its Default above, so a caller that only has a URL can +// leave the rest alone. +type Request struct { + NovelURL string + + // OutPath is the exact PDF path to write. When empty the file is named + // after the novel title and placed in OutDir. + OutPath string + OutDir string + + Page string // preset key: phone, a5, a4 + FontFile string // path to a .ttf; empty means discover a system font + FontSize float64 // points + LineSpacing float64 // multiple of font size + Margin float64 // millimetres + + Workers int + Retries int + + // Delay is the minimum spacing between requests. Set NoDelay to remove + // the spacing rather than setting this to 0, which is read as "unset" + // and gets the default back. + Delay time.Duration + NoDelay bool + + // Limit caps the export to the first N chapters. 0 means every chapter. + Limit int + + // CacheDir stores raw pages so a re-export costs no requests. Set + // NoCache to opt out rather than clearing this field, which would be + // read as "unset" and get the default back. + CacheDir string + NoCache bool + + // Log receives progress messages. Optional. + Log func(format string, args ...any) +} + +// Result reports what was produced. +type Result struct { + Path string + Title string + SourceURL string + Chapters int + Words int + FontFile string + Page pdfout.PageSize +} + +// Summary renders a one-line description of the exported book. +func (r *Result) Summary() string { + return fmt.Sprintf("%s — %d chapters, %d words", r.Title, r.Chapters, r.Words) +} + +// Export fetches the novel at req.NovelURL and writes it as a PDF, returning +// where it landed. The context bounds the whole crawl; cancelling it abandons +// the run without leaving a partial PDF behind. +func Export(ctx context.Context, req Request) (*Result, error) { + req.applyDefaults() + if err := req.validate(); err != nil { + return nil, err + } + + crawler := &monkeyd.Crawler{ + Client: monkeyd.NewClient(req.Delay, req.Retries), + CacheDir: req.CacheDir, + Workers: req.Workers, + Log: req.Log, + } + + novel, err := crawler.Novel(ctx, req.NovelURL) + if err != nil { + return nil, err + } + + if req.Limit > 0 && req.Limit < len(novel.Chapters) { + req.logf("limiting to first %d of %d chapters", req.Limit, len(novel.Chapters)) + novel.Chapters = novel.Chapters[:req.Limit] + } + + chapters, err := crawler.Chapters(ctx, novel) + if err != nil { + return nil, err + } + + fontFile := req.FontFile + if fontFile == "" { + if fontFile, err = pdfout.FindFont(); err != nil { + return nil, err + } + } + + outPath := req.OutPath + if outPath == "" { + outPath = filepath.Join(req.OutDir, SafeFileName(novel.Title, novel.Slug)+".pdf") + } + + page := pdfout.Presets[req.Page] + opts := pdfout.Options{ + Page: page, + Margin: req.Margin, + FontFile: fontFile, + FontSize: req.FontSize, + LineSpacing: req.LineSpacing, + Title: novel.Title, + SourceURL: novel.URL, + } + if err := pdfout.Write(outPath, opts, toPDFChapters(chapters)); err != nil { + return nil, err + } + + return &Result{ + Path: outPath, + Title: novel.Title, + SourceURL: novel.URL, + Chapters: len(chapters), + Words: monkeyd.TotalWords(chapters), + FontFile: fontFile, + Page: page, + }, nil +} + +func (r *Request) applyDefaults() { + if r.Page == "" { + r.Page = DefaultPage + } + if r.FontSize == 0 { + r.FontSize = DefaultFontSize + } + if r.LineSpacing == 0 { + r.LineSpacing = DefaultLineSpacing + } + if r.Margin == 0 { + r.Margin = DefaultMargin + } + if r.Workers == 0 { + r.Workers = DefaultWorkers + } + switch { + case r.NoDelay: + r.Delay = 0 + case r.Delay == 0: + r.Delay = DefaultDelay + } + if r.Retries == 0 { + r.Retries = DefaultRetries + } + switch { + case r.NoCache: + r.CacheDir = "" + case r.CacheDir == "": + r.CacheDir = DefaultCacheDir + } +} + +// validate rejects a Request before any request is made, so a typo costs no +// fetches. Call applyDefaults first: it checks the effective values. +func (r *Request) validate() error { + if r.NovelURL == "" { + return fmt.Errorf("novel url is required") + } + parsed, err := url.Parse(r.NovelURL) + if err != nil { + return fmt.Errorf("invalid novel url: %w", err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return fmt.Errorf("invalid novel url: want an http(s) URL, got %q", r.NovelURL) + } + if _, ok := pdfout.Presets[r.Page]; !ok { + return fmt.Errorf("unknown page %q: want one of %s", + r.Page, strings.Join(pdfout.PresetNames(), ", ")) + } + if r.FontSize <= 0 { + return fmt.Errorf("font size must be positive") + } + if r.LineSpacing <= 0 { + return fmt.Errorf("line spacing must be positive") + } + if r.Margin < 0 { + return fmt.Errorf("margin cannot be negative") + } + if r.Workers < 1 { + return fmt.Errorf("workers must be at least 1") + } + return nil +} + +func (r *Request) logf(format string, args ...any) { + if r.Log != nil { + r.Log(format, args...) + } +} + +func toPDFChapters(chapters []*monkeyd.Chapter) []pdfout.Chapter { + out := make([]pdfout.Chapter, 0, len(chapters)) + for _, ch := range chapters { + out = append(out, pdfout.Chapter{Heading: ch.Heading(), Paragraphs: ch.Paragraphs}) + } + return out +} + +var unsafeNameChars = regexp.MustCompile(`[^\p{L}\p{N}]+`) + +// SafeFileName builds a file name from the novel title, falling back to the +// slug when the title has no usable characters. The result has no extension. +func SafeFileName(title, fallback string) string { + name := strings.Trim(unsafeNameChars.ReplaceAllString(title, "-"), "-") + if name == "" { + return fallback + } + return name +} diff --git a/export/export_test.go b/export/export_test.go new file mode 100644 index 0000000..2c6a3ef --- /dev/null +++ b/export/export_test.go @@ -0,0 +1,187 @@ +package export + +import ( + "testing" + "time" +) + +func TestApplyDefaultsFillsUnsetFields(t *testing.T) { + req := Request{NovelURL: "https://monkeydd.com/example.html"} + req.applyDefaults() + + if req.Page != DefaultPage { + t.Errorf("Page = %q, want %q", req.Page, DefaultPage) + } + if req.FontSize != DefaultFontSize { + t.Errorf("FontSize = %v, want %v", req.FontSize, DefaultFontSize) + } + if req.LineSpacing != DefaultLineSpacing { + t.Errorf("LineSpacing = %v, want %v", req.LineSpacing, DefaultLineSpacing) + } + if req.Margin != DefaultMargin { + t.Errorf("Margin = %v, want %v", req.Margin, DefaultMargin) + } + if req.Workers != DefaultWorkers { + t.Errorf("Workers = %v, want %v", req.Workers, DefaultWorkers) + } + if req.Delay != DefaultDelay { + t.Errorf("Delay = %v, want %v", req.Delay, DefaultDelay) + } + if req.Retries != DefaultRetries { + t.Errorf("Retries = %v, want %v", req.Retries, DefaultRetries) + } + if req.CacheDir != DefaultCacheDir { + t.Errorf("CacheDir = %q, want %q", req.CacheDir, DefaultCacheDir) + } +} + +func TestApplyDefaultsKeepsExplicitValues(t *testing.T) { + req := Request{ + NovelURL: "https://monkeydd.com/example.html", + Page: "a5", + FontSize: 14, + LineSpacing: 1.4, + Margin: 10, + Workers: 2, + Delay: time.Second, + Retries: 1, + CacheDir: "/tmp/pages", + } + req.applyDefaults() + + if req.Page != "a5" || req.FontSize != 14 || req.LineSpacing != 1.4 { + t.Errorf("layout fields overwritten: %+v", req) + } + if req.Margin != 10 || req.Workers != 2 || req.Delay != time.Second || req.Retries != 1 { + t.Errorf("fetch fields overwritten: %+v", req) + } + if req.CacheDir != "/tmp/pages" { + t.Errorf("CacheDir = %q, want /tmp/pages", req.CacheDir) + } +} + +// A zero Margin is indistinguishable from "unset", so it becomes the default. +// NoCache and NoDelay exist precisely because "none" must survive that rule. +func TestApplyDefaultsHonoursOptOuts(t *testing.T) { + req := Request{NovelURL: "https://monkeydd.com/example.html", NoCache: true, NoDelay: true} + req.applyDefaults() + + if req.CacheDir != "" { + t.Errorf("NoCache left CacheDir = %q, want empty", req.CacheDir) + } + if req.Delay != 0 { + t.Errorf("NoDelay left Delay = %v, want 0", req.Delay) + } +} + +// Opt-outs win over an explicitly set value so a caller cannot end up with both. +func TestApplyDefaultsOptOutsOverrideExplicitValues(t *testing.T) { + req := Request{ + NovelURL: "https://monkeydd.com/example.html", + CacheDir: "/tmp/pages", + NoCache: true, + Delay: time.Second, + NoDelay: true, + } + req.applyDefaults() + + if req.CacheDir != "" { + t.Errorf("CacheDir = %q, want empty", req.CacheDir) + } + if req.Delay != 0 { + t.Errorf("Delay = %v, want 0", req.Delay) + } +} + +func TestValidate(t *testing.T) { + tests := []struct { + name string + req Request + wantErr bool + }{ + { + name: "defaults are valid", + req: Request{NovelURL: "https://monkeydd.com/example.html"}, + }, + { + name: "missing url", + req: Request{}, + wantErr: true, + }, + { + name: "non-http scheme", + req: Request{NovelURL: "ftp://monkeydd.com/example.html"}, + wantErr: true, + }, + { + name: "scheme-less url", + req: Request{NovelURL: "monkeydd.com/example.html"}, + wantErr: true, + }, + { + name: "unknown page preset", + req: Request{NovelURL: "https://monkeydd.com/example.html", Page: "letter"}, + wantErr: true, + }, + { + name: "negative font size", + req: Request{NovelURL: "https://monkeydd.com/example.html", FontSize: -1}, + wantErr: true, + }, + { + name: "negative line spacing", + req: Request{NovelURL: "https://monkeydd.com/example.html", LineSpacing: -1}, + wantErr: true, + }, + { + name: "negative margin", + req: Request{NovelURL: "https://monkeydd.com/example.html", Margin: -1}, + wantErr: true, + }, + { + name: "zero workers is filled by defaults, negative is not", + req: Request{NovelURL: "https://monkeydd.com/example.html", Workers: -1}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := tt.req + req.applyDefaults() + err := req.validate() + if tt.wantErr && err == nil { + t.Error("validate() = nil, want error") + } + if !tt.wantErr && err != nil { + t.Errorf("validate() = %v, want nil", err) + } + }) + } +} + +func TestSafeFileName(t *testing.T) { + tests := []struct { + title string + fallback string + want string + }{ + {"Trở Lại Năm Tháng Cũ", "slug", "Trở-Lại-Năm-Tháng-Cũ"}, + {"Chapter: One / Two", "slug", "Chapter-One-Two"}, + {" --- ", "slug", "slug"}, + {"", "slug", "slug"}, + } + for _, tt := range tests { + if got := SafeFileName(tt.title, tt.fallback); got != tt.want { + t.Errorf("SafeFileName(%q, %q) = %q, want %q", tt.title, tt.fallback, got, tt.want) + } + } +} + +func TestResultSummary(t *testing.T) { + r := &Result{Title: "Example", Chapters: 12, Words: 3400} + want := "Example — 12 chapters, 3400 words" + if got := r.Summary(); got != want { + t.Errorf("Summary() = %q, want %q", got, want) + } +} diff --git a/internal/monkeyd/chapter.go b/monkeyd/chapter.go similarity index 100% rename from internal/monkeyd/chapter.go rename to monkeyd/chapter.go diff --git a/internal/monkeyd/chapter_test.go b/monkeyd/chapter_test.go similarity index 100% rename from internal/monkeyd/chapter_test.go rename to monkeyd/chapter_test.go diff --git a/internal/monkeyd/client.go b/monkeyd/client.go similarity index 100% rename from internal/monkeyd/client.go rename to monkeyd/client.go diff --git a/internal/monkeyd/crawl.go b/monkeyd/crawl.go similarity index 100% rename from internal/monkeyd/crawl.go rename to monkeyd/crawl.go diff --git a/internal/monkeyd/css_words.go b/monkeyd/css_words.go similarity index 100% rename from internal/monkeyd/css_words.go rename to monkeyd/css_words.go diff --git a/internal/monkeyd/html_nodes.go b/monkeyd/html_nodes.go similarity index 100% rename from internal/monkeyd/html_nodes.go rename to monkeyd/html_nodes.go diff --git a/internal/monkeyd/novel.go b/monkeyd/novel.go similarity index 100% rename from internal/monkeyd/novel.go rename to monkeyd/novel.go diff --git a/internal/monkeyd/novel_test.go b/monkeyd/novel_test.go similarity index 100% rename from internal/monkeyd/novel_test.go rename to monkeyd/novel_test.go diff --git a/internal/pdfout/font.go b/pdfout/font.go similarity index 100% rename from internal/pdfout/font.go rename to pdfout/font.go diff --git a/internal/pdfout/pdf.go b/pdfout/pdf.go similarity index 100% rename from internal/pdfout/pdf.go rename to pdfout/pdf.go diff --git a/internal/pdfout/pdf_test.go b/pdfout/pdf_test.go similarity index 100% rename from internal/pdfout/pdf_test.go rename to pdfout/pdf_test.go