diff --git a/README.md b/README.md index 5864d1e..ae3a43f 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Atlas via long polling and an in-process cron scheduler. | `gold` | Gold paper trading (opt-in; VNAppMob SJC buy/sell VND/luong) | | `coin` | Crypto paper trading in USD (Binance -> Coinbase -> CoinGecko price fallback) | | `stats` | `/stats` (top commands), `/stats users`, `/stats user `, `/stats cmd ` | -| `monkeyd` | `/monkeyd_crawl [font_size]` — export a monkeydd.com novel as a PDF | +| `monkeyd` | `/monkeyd_crawl [font_size]` export a monkeydd.com novel as a PDF, `/monkeyd_tags ` list its tags as hashtags | Disable modules with the `MODULES` environment variable. @@ -170,6 +170,33 @@ The PDF embeds a font covering Vietnamese diacritics. The crawler prefers a system font and falls back to one compiled into the binary, so the runtime image needs no fonts installed. +### Novel tags + +`/monkeyd_tags ` reports a novel's tags as a hashtag line, sent as a code +block so it can be copied in one tap: + +```text +#MonkeyD #CổĐại #GiaĐình + +https://monkeydd.com/truong-an-gwem.html +``` + +Each label becomes one hashtag with spaces and punctuation removed and every +word capitalised, because Telegram ends a hashtag at the first character that is +not a letter, digit, or underscore. Diacritics are kept. A label with no letters +is dropped rather than emitted as a bare `#`. + +The tags are the novel's own genres, identified by their `itemprop="genre"` +microdata. The same page also links every genre on the site as navigation — 69 +of them against one novel's 6 in a sampled page — so matching the category URL +shape instead would return the whole menu. + +This command costs a single request and runs inline rather than in the +background, bounded by a short timeout: handlers are dispatched one at a time, so +a stalled fetch would hold up every other command. It shares the export page +cache, so a lookup for a novel that was already exported costs no request at +all. + ## Layout ``` diff --git a/cmd/server/command_menu_test.go b/cmd/server/command_menu_test.go index 0d9f5fd..a81a002 100644 --- a/cmd/server/command_menu_test.go +++ b/cmd/server/command_menu_test.go @@ -68,6 +68,7 @@ func TestCommandDiscovery_AllPublicCommandsHaveSafeMetadata(t *testing.T) { "lol": "[date]", "loldle": "[champion]", "monkeyd_crawl": " [font_size]", + "monkeyd_tags": "", "random": "", "stats": "[users | user | cmd ]", "stock_events": " [days]", diff --git a/internal/modules/monkeyd/handlers_test.go b/internal/modules/monkeyd/handlers_test.go index 06ca029..14b1439 100644 --- a/internal/modules/monkeyd/handlers_test.go +++ b/internal/modules/monkeyd/handlers_test.go @@ -26,12 +26,22 @@ const testNovelURL = "https://monkeydd.com/tro-lai-nam-thang-cu.html" // ownerID is permitted, which /monkeyd_crawl requires: the command is // Protected, and the dispatcher drops unauthorized calls silently. func install(t *testing.T, ownerID int64) (*testutil.RecordingBot, *runner) { + t.Helper() + return installWith(t, ownerID, func(context.Context, string) ([]string, error) { + t.Helper() + t.Error("tags fetcher called by an export test") + return nil, nil + }) +} + +// installWith is install with an explicit tag fetcher, for the tags command. +func installWith(t *testing.T, ownerID int64, fetch tagsFetcher) (*testutil.RecordingBot, *runner) { t.Helper() rb := testutil.NewRecordingBot(t) r := newRunner() // Run the export inline so assertions do not race a detached goroutine. r.launch = func(job func()) { job() } - mod := newModule(r) + mod := newModule(r, fetch) reg := &modules.Registry{ Modules: []modules.Module{{Name: "monkeyd", Commands: mod.Commands}}, @@ -335,25 +345,40 @@ func TestCrawl_AvailableToAnySender(t *testing.T) { func TestRegistration(t *testing.T) { mod := New(modules.Deps{Store: storage.NewMemoryProvider().Collection("monkeyd")}) - if len(mod.Commands) != 1 { - t.Fatalf("expected 1 command, got %d", len(mod.Commands)) + + wantParameters := map[string]string{ + commandName: " [font_size]", + tagsCommandName: "", } - cmd := mod.Commands[0] - if cmd.Name != commandName { - t.Errorf("Name = %q, want %q", cmd.Name, commandName) + if len(mod.Commands) != len(wantParameters) { + t.Fatalf("expected %d commands, got %d", len(wantParameters), len(mod.Commands)) } - if cmd.Visibility != modules.VisibilityPublic { - t.Errorf("Visibility = %v, want Public", cmd.Visibility) + + for _, cmd := range mod.Commands { + want, known := wantParameters[cmd.Name] + if !known { + t.Errorf("unexpected command %q", cmd.Name) + continue + } + delete(wantParameters, cmd.Name) + + if cmd.Parameters != want { + t.Errorf("/%s parameters = %q, want %q", cmd.Name, cmd.Parameters, want) + } + if cmd.Visibility != modules.VisibilityPublic { + t.Errorf("/%s visibility = %v, want Public", cmd.Name, cmd.Visibility) + } + if cmd.Description == "" { + t.Errorf("/%s description is empty; command discovery requires one", cmd.Name) + } + if cmd.Handler == nil { + t.Errorf("/%s handler is nil", cmd.Name) + } } - if cmd.Parameters != " [font_size]" { - t.Errorf("Parameters = %q, want %q", cmd.Parameters, " [font_size]") - } - if cmd.Description == "" { - t.Error("Description is empty; command discovery requires one") - } - if cmd.Handler == nil { - t.Error("Handler is nil") + for name := range wantParameters { + t.Errorf("command %q was not registered", name) } + if len(mod.Crons) != 0 || len(mod.Callbacks) != 0 { t.Errorf("expected no crons or callbacks, got %d crons and %d callbacks", len(mod.Crons), len(mod.Callbacks)) diff --git a/internal/modules/monkeyd/hashtags.go b/internal/modules/monkeyd/hashtags.go new file mode 100644 index 0000000..871b4e3 --- /dev/null +++ b/internal/modules/monkeyd/hashtags.go @@ -0,0 +1,77 @@ +package monkeyd + +import ( + "strings" + "unicode" +) + +// sourceHashtag leads every tag line so the source is visible when the line is +// pasted somewhere on its own. +const sourceHashtag = "#MonkeyD" + +// tagsMessage renders the tag line followed by the novel URL, separated by a +// blank line. Plain text: the caller decides how to present it. +func tagsMessage(tags []string, novelURL string) string { + return hashtagLine(tags) + "\n\n" + novelURL +} + +// hashtagLine turns genre labels into a single space-separated line of +// hashtags, led by sourceHashtag. Labels that cannot form a usable hashtag are +// dropped, and duplicates collapse — two labels differing only in punctuation +// or spacing produce the same hashtag. +func hashtagLine(tags []string) string { + parts := []string{sourceHashtag} + seen := map[string]bool{sourceHashtag: true} + for _, tag := range tags { + body := hashtag(tag) + if body == "" { + continue + } + tag := "#" + body + if seen[tag] { + continue + } + seen[tag] = true + parts = append(parts, tag) + } + return strings.Join(parts, " ") +} + +// hashtag converts a genre label into a hashtag body: "Cổ Đại" -> "CổĐại". +// +// Telegram ends a hashtag at the first character that is not a letter, digit or +// underscore, so spaces and punctuation are removed rather than replaced, and +// each word is capitalised to keep the boundaries readable. Diacritics survive: +// Telegram accepts non-ASCII letters in hashtags. +// +// Returns "" for a label with no letters. A digits-only hashtag is not linkified +// by Telegram, so emitting one would only add noise. +func hashtag(tag string) string { + var b strings.Builder + b.Grow(len(tag)) + + hasLetter := false + wordStart := true + for _, r := range tag { + switch { + case unicode.IsLetter(r): + hasLetter = true + if wordStart { + r = unicode.ToUpper(r) + } + b.WriteRune(r) + wordStart = false + case unicode.IsDigit(r): + b.WriteRune(r) + wordStart = false + default: + // Dropped, but it still ends the word, so the next letter is + // capitalised: "sci-fi" -> "SciFi". + wordStart = true + } + } + if !hasLetter { + return "" + } + return b.String() +} diff --git a/internal/modules/monkeyd/hashtags_test.go b/internal/modules/monkeyd/hashtags_test.go new file mode 100644 index 0000000..b73eeb2 --- /dev/null +++ b/internal/modules/monkeyd/hashtags_test.go @@ -0,0 +1,81 @@ +package monkeyd + +import "testing" + +func TestHashtag(t *testing.T) { + tests := []struct { + tag string + want string + }{ + {"Cổ Đại", "CổĐại"}, + {"Gia Đình", "GiaĐình"}, + {"Trọng Sinh", "TrọngSinh"}, + {"Nữ Cường", "NữCường"}, + {"Vả Mặt", "VảMặt"}, + // Single word passes through untouched. + {"Ngôn", "Ngôn"}, + // A lower-case label is capitalised per word. + {"ngôn tình", "NgônTình"}, + // Punctuation is removed but still ends the word. + {"sci-fi", "SciFi"}, + {"Đông/Tây", "ĐôngTây"}, + {"Truyện (Chữ)", "TruyệnChữ"}, + // Collapsed whitespace and surrounding space. + {" Cổ Đại ", "CổĐại"}, + // Digits are kept. + {"3D", "3D"}, + {"18+ Tuổi", "18Tuổi"}, + // Nothing usable: Telegram will not linkify a digits-only hashtag. + {"18+", ""}, + {"", ""}, + {"---", ""}, + {"+", ""}, + } + for _, tt := range tests { + if got := hashtag(tt.tag); got != tt.want { + t.Errorf("hashtag(%q) = %q, want %q", tt.tag, got, tt.want) + } + } +} + +func TestHashtagLine(t *testing.T) { + got := hashtagLine([]string{"Cổ Đại", "Gia Đình"}) + want := "#MonkeyD #CổĐại #GiaĐình" + if got != want { + t.Errorf("hashtagLine() = %q, want %q", got, want) + } +} + +func TestHashtagLineWithoutTags(t *testing.T) { + if got := hashtagLine(nil); got != sourceHashtag { + t.Errorf("hashtagLine(nil) = %q, want %q", got, sourceHashtag) + } +} + +// Labels differing only in punctuation collapse to one hashtag, and an unusable +// label is dropped rather than emitted as a bare "#". +func TestHashtagLineDropsUnusableAndDuplicates(t *testing.T) { + got := hashtagLine([]string{"Cổ Đại", "Cổ-Đại", "18+", "Gia Đình"}) + want := "#MonkeyD #CổĐại #GiaĐình" + if got != want { + t.Errorf("hashtagLine() = %q, want %q", got, want) + } +} + +func TestTagsMessage(t *testing.T) { + got := tagsMessage([]string{"Cổ Đại", "Gia Đình"}, "https://monkeydd.com/truong-an-gwem.html") + want := "#MonkeyD #CổĐại #GiaĐình\n\nhttps://monkeydd.com/truong-an-gwem.html" + if got != want { + t.Errorf("tagsMessage() =\n%q\nwant\n%q", got, want) + } +} + +// The block is what makes the line copyable in one tap, and its contents must be +// escaped so a label containing markup cannot break out of it. +func TestMonospaceBlockEscapes(t *testing.T) { + got := monospaceBlock("#MonkeyD #A&") + want := "
#MonkeyD #A<b>&
" + if got != want { + t.Errorf("monospaceBlock() = %q, want %q", got, want) + } +} diff --git a/internal/modules/monkeyd/monkeyd.go b/internal/modules/monkeyd/monkeyd.go index c2092a6..d801974 100644 --- a/internal/modules/monkeyd/monkeyd.go +++ b/internal/modules/monkeyd/monkeyd.go @@ -36,12 +36,13 @@ var usage = fmt.Sprintf( // New is the module Factory. The module keeps no persistent state — an export // is a one-shot job — so deps.Store is unused. func New(_ modules.Deps) modules.Module { - return newModule(newRunner()) + return newModule(newRunner(), fetchTags) } -// newModule builds the module around a given runner, which is how tests supply -// one with a stubbed exporter and a synchronous launch. -func newModule(r *runner) modules.Module { +// newModule builds the module around a given runner and tag fetcher, which is +// how tests supply a stubbed exporter, a synchronous launch, and tags without +// network access. +func newModule(r *runner, fetch tagsFetcher) modules.Module { return modules.Module{ Commands: []modules.Command{ { @@ -55,6 +56,7 @@ func newModule(r *runner) modules.Module { Parameters: parameters, Handler: r.handle, }, + tagsCommand(fetch), }, } } diff --git a/internal/modules/monkeyd/tags_command.go b/internal/modules/monkeyd/tags_command.go new file mode 100644 index 0000000..a54942b --- /dev/null +++ b/internal/modules/monkeyd/tags_command.go @@ -0,0 +1,122 @@ +package monkeyd + +import ( + "context" + "fmt" + "html" + "os" + "path/filepath" + "strings" + "time" + + "github.com/go-telegram/bot" + "github.com/go-telegram/bot/models" + + "github.com/tiennm99/monkeyd-crawler/export" + crawler "github.com/tiennm99/monkeyd-crawler/monkeyd" + + "github.com/tiennm99/miti99bot/internal/log" + "github.com/tiennm99/miti99bot/internal/modules" + "github.com/tiennm99/miti99bot/internal/modules/util/chathelper" +) + +// tagsCommandName is the command that reports a novel's tags. +const tagsCommandName = "monkeyd_tags" + +// tagsParameters is the display syntax shared by the command menu, /help, and +// the usage text. +const tagsParameters = "" + +const tagsUsage = "Usage: /" + tagsCommandName + " " + tagsParameters + +const ( + // tagsFetchTimeout bounds the whole command. Unlike an export this is one + // request and runs inline, and handlers are dispatched one at a time, so a + // stalled fetch would hold up every other command until it gives up. + tagsFetchTimeout = 30 * time.Second + + // tagsRetries is deliberately lower than the export's: the caller is + // waiting on this reply, so failing early beats a long retry chain. + tagsRetries = 1 +) + +// tagsFetcher returns a novel's tags. A field on the module rather than a +// direct call so tests can exercise the command without network access. +type tagsFetcher func(ctx context.Context, novelURL string) ([]string, error) + +func tagsCommand(fetch tagsFetcher) modules.Command { + return modules.Command{ + Name: tagsCommandName, + Visibility: modules.VisibilityPublic, + Description: "Show a " + AllowedHostsHint + " novel's tags as hashtags", + Parameters: tagsParameters, + Handler: tagsHandler(fetch), + } +} + +func tagsHandler(fetch tagsFetcher) modules.CommandHandler { + return func(ctx context.Context, b *bot.Bot, update *models.Update) error { + msg := update.Message + if msg == nil { + return nil + } + + arg := chathelper.ArgAfterCommand(msg.Text) + if arg == "" { + return chathelper.Reply(ctx, b, msg, tagsUsage) + } + fields := strings.Fields(arg) + if len(fields) > 1 { + return chathelper.Reply(ctx, b, msg, + fmt.Sprintf("%s.\n%s", capitalize(errTooManyArgs.Error()), tagsUsage)) + } + novelURL, err := normalizeNovelURL(fields[0]) + if err != nil { + return chathelper.Reply(ctx, b, msg, + fmt.Sprintf("%s.\n%s", capitalize(err.Error()), tagsUsage)) + } + + // Reserve part of the handler's budget for the reply itself, then cap + // the fetch so it cannot outlive the command. + fetchCtx, cancel := chathelper.FetchContext(ctx) + defer cancel() + fetchCtx, cancelTimeout := context.WithTimeout(fetchCtx, tagsFetchTimeout) + defer cancelTimeout() + + tags, err := fetch(fetchCtx, novelURL) + if err != nil { + log.Error("monkeyd tags fetch failed", + "command", tagsCommandName, "url", novelURL, "err", err) + return chathelper.Reply(ctx, b, msg, "Could not read that novel page: "+err.Error()) + } + if len(tags) == 0 { + return chathelper.Reply(ctx, b, msg, "That page lists no tags.") + } + + // Sent as a code block so the line can be copied in one tap. + return chathelper.ReplyHTML(ctx, b, msg, monospaceBlock(tagsMessage(tags, novelURL))) + } +} + +// monospaceBlock wraps text in Telegram's
 block, which renders as a
+// copyable code box. The content is escaped, so a tag containing < or & cannot
+// break the markup.
+func monospaceBlock(text string) string {
+	return "
" + html.EscapeString(text) + "
" +} + +// fetchTags reads a novel's tags from the site. +// +// It shares the export cache directory, so tags requested for a novel that was +// already exported — or an export following a tags lookup — cost no request. +func fetchTags(ctx context.Context, novelURL string) ([]string, error) { + c := &crawler.Crawler{ + Client: crawler.NewClient(export.DefaultDelay, tagsRetries), + CacheDir: filepath.Join(os.TempDir(), cacheDirName), + } + novel, err := c.NovelInfo(ctx, novelURL) + if err != nil { + return nil, err + } + return novel.Tags, nil +} diff --git a/internal/modules/monkeyd/tags_command_test.go b/internal/modules/monkeyd/tags_command_test.go new file mode 100644 index 0000000..89d8cb5 --- /dev/null +++ b/internal/modules/monkeyd/tags_command_test.go @@ -0,0 +1,151 @@ +package monkeyd + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/tiennm99/miti99bot/internal/testutil" +) + +// stubTags returns a fetcher yielding fixed tags and records the URL it saw. +func stubTags(tags []string, gotURL *string) tagsFetcher { + return func(_ context.Context, novelURL string) ([]string, error) { + if gotURL != nil { + *gotURL = novelURL + } + return tags, nil + } +} + +func TestTags_RepliesWithHashtagBlock(t *testing.T) { + var gotURL string + rb, _ := installWith(t, 999, stubTags([]string{"Cổ Đại", "Gia Đình"}, &gotURL)) + + rb.Bot.ProcessUpdate(context.Background(), + testutil.NewPrivateMessage(999, "/monkeyd_tags https://monkeydd.com/truong-an-gwem.html")) + + if want := "https://monkeydd.com/truong-an-gwem.html"; gotURL != want { + t.Errorf("fetcher got %q, want %q", gotURL, want) + } + + sent := rb.LastSent() + want := "
#MonkeyD #CổĐại #GiaĐình\n\nhttps://monkeydd.com/truong-an-gwem.html
" + if got := sent.Text(); got != want { + t.Errorf("reply =\n%q\nwant\n%q", got, want) + } + // Without HTML parse mode Telegram would show the literal
 tags.
+	if got := sent.Form["parse_mode"]; got != "HTML" {
+		t.Errorf("parse_mode = %q, want HTML", got)
+	}
+}
+
+func TestTags_NoArgumentRepliesUsage(t *testing.T) {
+	rb, _ := installWith(t, 999, stubTags(nil, nil))
+	rb.Bot.ProcessUpdate(context.Background(), testutil.NewPrivateMessage(999, "/monkeyd_tags"))
+
+	if got := rb.LastSent().Text(); !strings.Contains(got, tagsUsage) {
+		t.Errorf("reply = %q, want it to contain %q", got, tagsUsage)
+	}
+}
+
+func TestTags_RejectsDisallowedHost(t *testing.T) {
+	called := false
+	rb, _ := installWith(t, 999, func(context.Context, string) ([]string, error) {
+		called = true
+		return nil, nil
+	})
+
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewPrivateMessage(999, "/monkeyd_tags https://example.com/novel.html"))
+
+	if called {
+		t.Error("fetcher ran for a disallowed host")
+	}
+	if got := rb.LastSent().Text(); !strings.Contains(got, AllowedHostsHint) {
+		t.Errorf("reply = %q, want it to name %q", got, AllowedHostsHint)
+	}
+}
+
+// A bare hostname is normalised before the fetch, matching /monkeyd_crawl, and
+// the reply quotes the normalised form rather than what was typed.
+func TestTags_NormalizesURL(t *testing.T) {
+	const wantURL = "https://monkeydd.com/truong-an-gwem.html"
+
+	var gotURL string
+	rb, _ := installWith(t, 999, stubTags([]string{"Cổ Đại"}, &gotURL))
+
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewPrivateMessage(999, "/monkeyd_tags monkeydd.com/truong-an-gwem.html"))
+
+	if gotURL != wantURL {
+		t.Errorf("fetcher got %q, want %q", gotURL, wantURL)
+	}
+	if got := rb.LastSent().Text(); !strings.Contains(got, wantURL) {
+		t.Errorf("reply = %q, want it to carry the normalised URL", got)
+	}
+}
+
+func TestTags_ReportsFetchFailure(t *testing.T) {
+	rb, _ := installWith(t, 999, func(context.Context, string) ([]string, error) {
+		return nil, errors.New("fetch failed: 404")
+	})
+
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewPrivateMessage(999, "/monkeyd_tags "+testNovelURL))
+
+	got := rb.LastSent().Text()
+	if !strings.Contains(got, "404") {
+		t.Errorf("reply = %q, want it to carry the underlying error", got)
+	}
+	if strings.Contains(got, sourceHashtag) {
+		t.Errorf("reply = %q, want no hashtag line on failure", got)
+	}
+}
+
+// An empty tag list must say so rather than send a lone "#MonkeyD".
+func TestTags_ReportsEmptyTagList(t *testing.T) {
+	rb, _ := installWith(t, 999, stubTags(nil, nil))
+
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewPrivateMessage(999, "/monkeyd_tags "+testNovelURL))
+
+	got := rb.LastSent().Text()
+	if strings.Contains(got, sourceHashtag) {
+		t.Errorf("reply = %q, want a plain explanation rather than a bare source hashtag", got)
+	}
+	if !strings.Contains(strings.ToLower(got), "no tags") {
+		t.Errorf("reply = %q, want it to say no tags were listed", got)
+	}
+}
+
+func TestTags_RejectsExtraArguments(t *testing.T) {
+	called := false
+	rb, _ := installWith(t, 999, func(context.Context, string) ([]string, error) {
+		called = true
+		return nil, nil
+	})
+
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewPrivateMessage(999, "/monkeyd_tags "+testNovelURL+" extra"))
+
+	if called {
+		t.Error("fetcher ran despite extra arguments")
+	}
+	if got := rb.LastSent().Text(); !strings.Contains(got, tagsUsage) {
+		t.Errorf("reply = %q, want usage text", got)
+	}
+}
+
+// The command is public, so a non-admin in a group must get a reply.
+func TestTags_AvailableToAnySender(t *testing.T) {
+	rb, _ := installWith(t, 999, stubTags([]string{"Cổ Đại"}, nil))
+
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewGroupMessage(-100, 12345, "/monkeyd_tags "+testNovelURL))
+
+	if got := rb.LastSent().Text(); !strings.Contains(got, "#CổĐại") {
+		t.Errorf("reply = %q, want the hashtag line", got)
+	}
+}
diff --git a/third_party/monkeyd-crawler b/third_party/monkeyd-crawler
index f400a83..d88f2a4 160000
--- a/third_party/monkeyd-crawler
+++ b/third_party/monkeyd-crawler
@@ -1 +1 @@
-Subproject commit f400a83678a513f0448e6881c61560fc9771aa22
+Subproject commit d88f2a482d84322aae0bde9b98a1d6985ef36b17