diff --git a/monkeyd/crawl.go b/monkeyd/crawl.go index 8aa70c1..b1e6dcd 100644 --- a/monkeyd/crawl.go +++ b/monkeyd/crawl.go @@ -77,6 +77,21 @@ func (c *Crawler) Novel(ctx context.Context, novelURL string) (*Novel, error) { return novel, nil } +// NovelInfo fetches only the landing page and returns what it carries: title, +// slug, tags, and the chapter list as that page shows it. +// +// Unlike Novel it does not also fetch a chapter page to cross-check the chapter +// list, so it costs a single request. Callers that only want metadata should +// prefer it; callers about to export every chapter want Novel, whose +// reconciliation guards against a silently truncated list. +func (c *Crawler) NovelInfo(ctx context.Context, novelURL string) (*Novel, error) { + page, err := c.page(ctx, novelURL) + if err != nil { + return nil, err + } + return ParseNovelPage(page, novelURL) +} + // Chapters fetches every chapter concurrently and returns them in reading // order. Any chapter that cannot be fetched or parsed fails the whole run // rather than yielding a book with a hole in it. diff --git a/monkeyd/novel.go b/monkeyd/novel.go index 35a4b9e..52fd08a 100644 --- a/monkeyd/novel.go +++ b/monkeyd/novel.go @@ -17,9 +17,13 @@ type ChapterRef struct { // Novel is a novel's landing page: its title and its chapters in reading order. type Novel struct { - Title string - Slug string - URL string + Title string + Slug string + URL string + + // Tags are the novel's own genres, as labelled on the site, in the order + // they appear. Empty when the page lists none. + Tags []string Chapters []ChapterRef } @@ -43,6 +47,7 @@ func ParseNovelPage(page []byte, pageURL string) (*Novel, error) { Title: novelTitle(doc), Slug: slugFromNovelURL(base), URL: pageURL, + Tags: tagsFromInfo(doc), Chapters: chapterRefsFromList(doc, base), } if novel.Title == "" { @@ -64,6 +69,35 @@ func novelTitle(doc *html.Node) string { return nodeText(elementByTag(doc, "title")) } +// tagsFromInfo reads the novel's own genres out of the info block. +// +// The anchors are matched on the schema.org itemprop="genre" microdata rather +// than on their href. The page also carries a site-wide genre menu linking every +// category — 69 distinct ones against this novel's 6 on a sampled page — so +// matching the /the-loai/ URL shape would pull in the whole navigation. +func tagsFromInfo(doc *html.Node) []string { + links := findAllNodes(doc, func(n *html.Node) bool { + return n.Type == html.ElementNode && n.Data == "a" && attr(n, "itemprop") == "genre" + }) + + var tags []string + seen := make(map[string]bool, len(links)) + for _, link := range links { + // The label appears both as the link text and as its title attribute; + // prefer the text and fall back to the attribute. + name := nodeText(link) + if name == "" { + name = collapseSpaces(attr(link, "title")) + } + if name == "" || seen[name] { + continue + } + seen[name] = true + tags = append(tags, name) + } + return tags +} + // slugFromNovelURL turns https://host/tro-lai-nam-thang-cu.html into // "tro-lai-nam-thang-cu". func slugFromNovelURL(u *url.URL) string { diff --git a/monkeyd/novel_test.go b/monkeyd/novel_test.go index 112d344..c89ec8f 100644 --- a/monkeyd/novel_test.go +++ b/monkeyd/novel_test.go @@ -44,6 +44,57 @@ func TestParseNovelPageOrdersChaptersForReading(t *testing.T) { } } +// tagsFixture mirrors the real page: the novel's own genres carry +// itemprop="genre", while a site-wide menu links every category without it. A +// parser matching on the /the-loai/ href shape would swallow the whole menu. +const tagsFixture = `