monkeyd-crawler/monkeyd/crawl.go

package monkeyd

import (
	"context"
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"golang.org/x/sync/errgroup"
)

// Crawler fetches a novel and its chapters.
type Crawler struct {
	Client *Client

	// CacheDir, when set, stores raw pages on disk and serves later runs from
	// them. Re-exporting with different font or page settings then costs no
	// requests.
	CacheDir string

	// Workers bounds concurrent fetches. The client's delay still caps the
	// overall request rate.
	Workers int

	// Log receives progress messages. Optional.
	Log func(format string, args ...any)
}

func (c *Crawler) logf(format string, args ...any) {
	if c.Log != nil {
		c.Log(format, args...)
	}
}

// Novel fetches a novel landing page and resolves its chapter list.
//
// The chapter list is taken from the landing page and cross-checked against the
// dropdown embedded in the first chapter page, so a truncated list cannot
// silently shorten the export.
func (c *Crawler) Novel(ctx context.Context, novelURL string) (*Novel, error) {
	page, err := c.page(ctx, novelURL)
	if err != nil {
		return nil, err
	}
	novel, err := ParseNovelPage(page, novelURL)
	if err != nil {
		return nil, err
	}
	c.logf("novel: %s (%d chapters listed)", novel.Title, len(novel.Chapters))

	base, err := url.Parse(novelURL)
	if err != nil {
		return nil, err
	}
	firstPage, err := c.page(ctx, novel.Chapters[0].URL)
	if err != nil {
		return nil, err
	}
	fromSelect, err := ChapterRefsFromSelect(firstPage, base)
	if err != nil {
		return nil, err
	}

	final, extra := ReconcileChapterRefs(novel.Chapters, fromSelect)
	if len(extra) > 0 {
		c.logf("warning: %d chapter(s) appear only in the chapter dropdown and were not "+
			"in the landing page list; verify the export is complete", len(extra))
	}
	if len(final) != len(novel.Chapters) {
		c.logf("chapter list reconciled to %d chapters using the in-chapter dropdown", len(final))
	}
	novel.Chapters = final
	return novel, nil
}

// Chapters fetches every chapter concurrently and returns them in reading
// order. Any chapter that cannot be fetched or parsed fails the whole run
// rather than yielding a book with a hole in it.
func (c *Crawler) Chapters(ctx context.Context, novel *Novel) ([]*Chapter, error) {
	chapters := make([]*Chapter, len(novel.Chapters))

	workers := c.Workers
	if workers < 1 {
		workers = 1
	}

	group, groupCtx := errgroup.WithContext(ctx)
	group.SetLimit(workers)

	var mu sync.Mutex
	done := 0

	for i, ref := range novel.Chapters {
		i, ref := i, ref
		group.Go(func() error {
			page, err := c.page(groupCtx, ref.URL)
			if err != nil {
				return err
			}
			chapter, err := ParseChapter(page, ref)
			if err != nil {
				return err
			}
			chapters[i] = chapter

			mu.Lock()
			done++
			c.logf("fetched %d/%d: %s (%d words)", done, len(novel.Chapters),
				chapter.Heading(), chapter.WordCount())
			mu.Unlock()
			return nil
		})
	}

	if err := group.Wait(); err != nil {
		return nil, err
	}
	return chapters, nil
}

// page returns a page from the cache when available, otherwise fetches and
// caches it.
func (c *Crawler) page(ctx context.Context, pageURL string) ([]byte, error) {
	path := c.cachePath(pageURL)
	if path != "" {
		if body, err := os.ReadFile(path); err == nil && len(body) > 0 {
			return body, nil
		}
	}

	body, err := c.Client.Get(ctx, pageURL)
	if err != nil {
		return nil, err
	}

	if path != "" {
		if err := os.MkdirAll(filepath.Dir(path), 0o755); err == nil {
			// A failed cache write must not fail the crawl.
			_ = os.WriteFile(path, body, 0o644)
		}
	}
	return body, nil
}

// unsafeFileChars matches everything not allowed in a cache file name.
var unsafeFileChars = regexp.MustCompile(`[^A-Za-z0-9._-]+`)

// cachePath maps a page URL to a cache file, or "" when caching is disabled.
func (c *Crawler) cachePath(pageURL string) string {
	if c.CacheDir == "" {
		return ""
	}
	u, err := url.Parse(pageURL)
	if err != nil {
		return ""
	}
	name := unsafeFileChars.ReplaceAllString(strings.Trim(u.Path, "/"), "_")
	if name == "" {
		return ""
	}
	if !strings.HasSuffix(name, ".html") {
		name += ".html"
	}
	return filepath.Join(c.CacheDir, name)
}

// TotalWords sums the word count across chapters.
func TotalWords(chapters []*Chapter) int {
	n := 0
	for _, ch := range chapters {
		n += ch.WordCount()
	}
	return n
}

// Describe renders a one-line summary of a crawl result.
func Describe(novel *Novel, chapters []*Chapter) string {
	return fmt.Sprintf("%s — %d chapters, %d words", novel.Title, len(chapters), TotalWords(chapters))
}