mirror of
https://github.com/tiennm99/monkeyd-crawler.git
synced 2026-08-24 03:24:33 +00:00
refactor: make crawler importable and add export pipeline
Move monkeyd and pdfout out of internal/ so other modules can import them, and add export.Export, which holds the crawl-to-PDF sequence the CLI used to inline. The CLI now parses flags and delegates, so an embedding program gets the same defaults and validation.
This commit is contained in:
@@ -0,0 +1,244 @@
|
||||
// Package export turns a novel URL into a PDF file. It holds the sequence the
|
||||
// CLI and any embedding program both need — resolve the chapter list, fetch the
|
||||
// chapters, pick a font, render the PDF — so neither has to reassemble it.
|
||||
package export
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/tiennm99/monkeyd-crawler/monkeyd"
|
||||
"github.com/tiennm99/monkeyd-crawler/pdfout"
|
||||
)
|
||||
|
||||
// Defaults for every tunable field of Request. Exported so a caller's own
|
||||
// flags or config can advertise the same values instead of restating them.
|
||||
const (
|
||||
DefaultPage = "phone"
|
||||
DefaultFontSize = 12.0
|
||||
DefaultLineSpacing = 1.55
|
||||
DefaultMargin = 6.0
|
||||
DefaultWorkers = 4
|
||||
DefaultDelay = 400 * time.Millisecond
|
||||
DefaultRetries = 3
|
||||
DefaultCacheDir = ".cache"
|
||||
)
|
||||
|
||||
// Request describes one export. Only NovelURL is required; every zero-valued
|
||||
// tunable falls back to its Default above, so a caller that only has a URL can
|
||||
// leave the rest alone.
|
||||
type Request struct {
|
||||
NovelURL string
|
||||
|
||||
// OutPath is the exact PDF path to write. When empty the file is named
|
||||
// after the novel title and placed in OutDir.
|
||||
OutPath string
|
||||
OutDir string
|
||||
|
||||
Page string // preset key: phone, a5, a4
|
||||
FontFile string // path to a .ttf; empty means discover a system font
|
||||
FontSize float64 // points
|
||||
LineSpacing float64 // multiple of font size
|
||||
Margin float64 // millimetres
|
||||
|
||||
Workers int
|
||||
Retries int
|
||||
|
||||
// Delay is the minimum spacing between requests. Set NoDelay to remove
|
||||
// the spacing rather than setting this to 0, which is read as "unset"
|
||||
// and gets the default back.
|
||||
Delay time.Duration
|
||||
NoDelay bool
|
||||
|
||||
// Limit caps the export to the first N chapters. 0 means every chapter.
|
||||
Limit int
|
||||
|
||||
// CacheDir stores raw pages so a re-export costs no requests. Set
|
||||
// NoCache to opt out rather than clearing this field, which would be
|
||||
// read as "unset" and get the default back.
|
||||
CacheDir string
|
||||
NoCache bool
|
||||
|
||||
// Log receives progress messages. Optional.
|
||||
Log func(format string, args ...any)
|
||||
}
|
||||
|
||||
// Result reports what was produced.
|
||||
type Result struct {
|
||||
Path string
|
||||
Title string
|
||||
SourceURL string
|
||||
Chapters int
|
||||
Words int
|
||||
FontFile string
|
||||
Page pdfout.PageSize
|
||||
}
|
||||
|
||||
// Summary renders a one-line description of the exported book.
|
||||
func (r *Result) Summary() string {
|
||||
return fmt.Sprintf("%s — %d chapters, %d words", r.Title, r.Chapters, r.Words)
|
||||
}
|
||||
|
||||
// Export fetches the novel at req.NovelURL and writes it as a PDF, returning
|
||||
// where it landed. The context bounds the whole crawl; cancelling it abandons
|
||||
// the run without leaving a partial PDF behind.
|
||||
func Export(ctx context.Context, req Request) (*Result, error) {
|
||||
req.applyDefaults()
|
||||
if err := req.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
crawler := &monkeyd.Crawler{
|
||||
Client: monkeyd.NewClient(req.Delay, req.Retries),
|
||||
CacheDir: req.CacheDir,
|
||||
Workers: req.Workers,
|
||||
Log: req.Log,
|
||||
}
|
||||
|
||||
novel, err := crawler.Novel(ctx, req.NovelURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if req.Limit > 0 && req.Limit < len(novel.Chapters) {
|
||||
req.logf("limiting to first %d of %d chapters", req.Limit, len(novel.Chapters))
|
||||
novel.Chapters = novel.Chapters[:req.Limit]
|
||||
}
|
||||
|
||||
chapters, err := crawler.Chapters(ctx, novel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fontFile := req.FontFile
|
||||
if fontFile == "" {
|
||||
if fontFile, err = pdfout.FindFont(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
outPath := req.OutPath
|
||||
if outPath == "" {
|
||||
outPath = filepath.Join(req.OutDir, SafeFileName(novel.Title, novel.Slug)+".pdf")
|
||||
}
|
||||
|
||||
page := pdfout.Presets[req.Page]
|
||||
opts := pdfout.Options{
|
||||
Page: page,
|
||||
Margin: req.Margin,
|
||||
FontFile: fontFile,
|
||||
FontSize: req.FontSize,
|
||||
LineSpacing: req.LineSpacing,
|
||||
Title: novel.Title,
|
||||
SourceURL: novel.URL,
|
||||
}
|
||||
if err := pdfout.Write(outPath, opts, toPDFChapters(chapters)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Result{
|
||||
Path: outPath,
|
||||
Title: novel.Title,
|
||||
SourceURL: novel.URL,
|
||||
Chapters: len(chapters),
|
||||
Words: monkeyd.TotalWords(chapters),
|
||||
FontFile: fontFile,
|
||||
Page: page,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Request) applyDefaults() {
|
||||
if r.Page == "" {
|
||||
r.Page = DefaultPage
|
||||
}
|
||||
if r.FontSize == 0 {
|
||||
r.FontSize = DefaultFontSize
|
||||
}
|
||||
if r.LineSpacing == 0 {
|
||||
r.LineSpacing = DefaultLineSpacing
|
||||
}
|
||||
if r.Margin == 0 {
|
||||
r.Margin = DefaultMargin
|
||||
}
|
||||
if r.Workers == 0 {
|
||||
r.Workers = DefaultWorkers
|
||||
}
|
||||
switch {
|
||||
case r.NoDelay:
|
||||
r.Delay = 0
|
||||
case r.Delay == 0:
|
||||
r.Delay = DefaultDelay
|
||||
}
|
||||
if r.Retries == 0 {
|
||||
r.Retries = DefaultRetries
|
||||
}
|
||||
switch {
|
||||
case r.NoCache:
|
||||
r.CacheDir = ""
|
||||
case r.CacheDir == "":
|
||||
r.CacheDir = DefaultCacheDir
|
||||
}
|
||||
}
|
||||
|
||||
// validate rejects a Request before any request is made, so a typo costs no
|
||||
// fetches. Call applyDefaults first: it checks the effective values.
|
||||
func (r *Request) validate() error {
|
||||
if r.NovelURL == "" {
|
||||
return fmt.Errorf("novel url is required")
|
||||
}
|
||||
parsed, err := url.Parse(r.NovelURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid novel url: %w", err)
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return fmt.Errorf("invalid novel url: want an http(s) URL, got %q", r.NovelURL)
|
||||
}
|
||||
if _, ok := pdfout.Presets[r.Page]; !ok {
|
||||
return fmt.Errorf("unknown page %q: want one of %s",
|
||||
r.Page, strings.Join(pdfout.PresetNames(), ", "))
|
||||
}
|
||||
if r.FontSize <= 0 {
|
||||
return fmt.Errorf("font size must be positive")
|
||||
}
|
||||
if r.LineSpacing <= 0 {
|
||||
return fmt.Errorf("line spacing must be positive")
|
||||
}
|
||||
if r.Margin < 0 {
|
||||
return fmt.Errorf("margin cannot be negative")
|
||||
}
|
||||
if r.Workers < 1 {
|
||||
return fmt.Errorf("workers must be at least 1")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Request) logf(format string, args ...any) {
|
||||
if r.Log != nil {
|
||||
r.Log(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func toPDFChapters(chapters []*monkeyd.Chapter) []pdfout.Chapter {
|
||||
out := make([]pdfout.Chapter, 0, len(chapters))
|
||||
for _, ch := range chapters {
|
||||
out = append(out, pdfout.Chapter{Heading: ch.Heading(), Paragraphs: ch.Paragraphs})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var unsafeNameChars = regexp.MustCompile(`[^\p{L}\p{N}]+`)
|
||||
|
||||
// SafeFileName builds a file name from the novel title, falling back to the
|
||||
// slug when the title has no usable characters. The result has no extension.
|
||||
func SafeFileName(title, fallback string) string {
|
||||
name := strings.Trim(unsafeNameChars.ReplaceAllString(title, "-"), "-")
|
||||
if name == "" {
|
||||
return fallback
|
||||
}
|
||||
return name
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package export
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestApplyDefaultsFillsUnsetFields(t *testing.T) {
|
||||
req := Request{NovelURL: "https://monkeydd.com/example.html"}
|
||||
req.applyDefaults()
|
||||
|
||||
if req.Page != DefaultPage {
|
||||
t.Errorf("Page = %q, want %q", req.Page, DefaultPage)
|
||||
}
|
||||
if req.FontSize != DefaultFontSize {
|
||||
t.Errorf("FontSize = %v, want %v", req.FontSize, DefaultFontSize)
|
||||
}
|
||||
if req.LineSpacing != DefaultLineSpacing {
|
||||
t.Errorf("LineSpacing = %v, want %v", req.LineSpacing, DefaultLineSpacing)
|
||||
}
|
||||
if req.Margin != DefaultMargin {
|
||||
t.Errorf("Margin = %v, want %v", req.Margin, DefaultMargin)
|
||||
}
|
||||
if req.Workers != DefaultWorkers {
|
||||
t.Errorf("Workers = %v, want %v", req.Workers, DefaultWorkers)
|
||||
}
|
||||
if req.Delay != DefaultDelay {
|
||||
t.Errorf("Delay = %v, want %v", req.Delay, DefaultDelay)
|
||||
}
|
||||
if req.Retries != DefaultRetries {
|
||||
t.Errorf("Retries = %v, want %v", req.Retries, DefaultRetries)
|
||||
}
|
||||
if req.CacheDir != DefaultCacheDir {
|
||||
t.Errorf("CacheDir = %q, want %q", req.CacheDir, DefaultCacheDir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDefaultsKeepsExplicitValues(t *testing.T) {
|
||||
req := Request{
|
||||
NovelURL: "https://monkeydd.com/example.html",
|
||||
Page: "a5",
|
||||
FontSize: 14,
|
||||
LineSpacing: 1.4,
|
||||
Margin: 10,
|
||||
Workers: 2,
|
||||
Delay: time.Second,
|
||||
Retries: 1,
|
||||
CacheDir: "/tmp/pages",
|
||||
}
|
||||
req.applyDefaults()
|
||||
|
||||
if req.Page != "a5" || req.FontSize != 14 || req.LineSpacing != 1.4 {
|
||||
t.Errorf("layout fields overwritten: %+v", req)
|
||||
}
|
||||
if req.Margin != 10 || req.Workers != 2 || req.Delay != time.Second || req.Retries != 1 {
|
||||
t.Errorf("fetch fields overwritten: %+v", req)
|
||||
}
|
||||
if req.CacheDir != "/tmp/pages" {
|
||||
t.Errorf("CacheDir = %q, want /tmp/pages", req.CacheDir)
|
||||
}
|
||||
}
|
||||
|
||||
// A zero Margin is indistinguishable from "unset", so it becomes the default.
|
||||
// NoCache and NoDelay exist precisely because "none" must survive that rule.
|
||||
func TestApplyDefaultsHonoursOptOuts(t *testing.T) {
|
||||
req := Request{NovelURL: "https://monkeydd.com/example.html", NoCache: true, NoDelay: true}
|
||||
req.applyDefaults()
|
||||
|
||||
if req.CacheDir != "" {
|
||||
t.Errorf("NoCache left CacheDir = %q, want empty", req.CacheDir)
|
||||
}
|
||||
if req.Delay != 0 {
|
||||
t.Errorf("NoDelay left Delay = %v, want 0", req.Delay)
|
||||
}
|
||||
}
|
||||
|
||||
// Opt-outs win over an explicitly set value so a caller cannot end up with both.
|
||||
func TestApplyDefaultsOptOutsOverrideExplicitValues(t *testing.T) {
|
||||
req := Request{
|
||||
NovelURL: "https://monkeydd.com/example.html",
|
||||
CacheDir: "/tmp/pages",
|
||||
NoCache: true,
|
||||
Delay: time.Second,
|
||||
NoDelay: true,
|
||||
}
|
||||
req.applyDefaults()
|
||||
|
||||
if req.CacheDir != "" {
|
||||
t.Errorf("CacheDir = %q, want empty", req.CacheDir)
|
||||
}
|
||||
if req.Delay != 0 {
|
||||
t.Errorf("Delay = %v, want 0", req.Delay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
req Request
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "defaults are valid",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html"},
|
||||
},
|
||||
{
|
||||
name: "missing url",
|
||||
req: Request{},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "non-http scheme",
|
||||
req: Request{NovelURL: "ftp://monkeydd.com/example.html"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "scheme-less url",
|
||||
req: Request{NovelURL: "monkeydd.com/example.html"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unknown page preset",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", Page: "letter"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "negative font size",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", FontSize: -1},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "negative line spacing",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", LineSpacing: -1},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "negative margin",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", Margin: -1},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "zero workers is filled by defaults, negative is not",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", Workers: -1},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := tt.req
|
||||
req.applyDefaults()
|
||||
err := req.validate()
|
||||
if tt.wantErr && err == nil {
|
||||
t.Error("validate() = nil, want error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Errorf("validate() = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeFileName(t *testing.T) {
|
||||
tests := []struct {
|
||||
title string
|
||||
fallback string
|
||||
want string
|
||||
}{
|
||||
{"Trở Lại Năm Tháng Cũ", "slug", "Trở-Lại-Năm-Tháng-Cũ"},
|
||||
{"Chapter: One / Two", "slug", "Chapter-One-Two"},
|
||||
{" --- ", "slug", "slug"},
|
||||
{"", "slug", "slug"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := SafeFileName(tt.title, tt.fallback); got != tt.want {
|
||||
t.Errorf("SafeFileName(%q, %q) = %q, want %q", tt.title, tt.fallback, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResultSummary(t *testing.T) {
|
||||
r := &Result{Title: "Example", Chapters: 12, Words: 3400}
|
||||
want := "Example — 12 chapters, 3400 words"
|
||||
if got := r.Summary(); got != want {
|
||||
t.Errorf("Summary() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user