diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ba2432..145095a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,12 @@ jobs: matrix: go: ['1.26.5'] steps: + # The monkeyd module builds against third_party/monkeyd-crawler, which is + # a submodule wired in through a go.mod replace directive. Without it + # checked out, every Go step fails to resolve the package. - uses: actions/checkout@v6 + with: + submodules: true - uses: actions/setup-go@v6 with: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..2940d69 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "third_party/monkeyd-crawler"] + path = third_party/monkeyd-crawler + url = https://github.com/tiennm99/monkeyd-crawler.git diff --git a/AGENTS.md b/AGENTS.md index 8071184..f876e8a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,12 @@ `internal/modules`. Runtime storage is MongoDB when `MONGO_URL` is set and in-memory storage in tests. Read `README.md` before implementation work. +`third_party/monkeyd-crawler` is a git submodule resolved through a `go.mod` +`replace` directive, not a versioned dependency. Go commands fail until it is +checked out (`git submodule update --init --recursive`). Changes to the crawler +belong in its own repository and must be pushed before the submodule pointer is +advanced here, or fresh clones cannot resolve the pinned commit. + ## Development Rules - Keep changes scoped to the requested module or shared contract. diff --git a/Dockerfile b/Dockerfile index 1726cc0..86df086 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,18 @@ FROM golang:1.26.5-alpine AS builder WORKDIR /src +# The monkeyd module renders PDFs with an embedded TrueType font, and the +# runtime image below ships no fonts at all. DejaVuSans covers the Latin +# Extended Additional block that Vietnamese diacritics live in; it is installed +# here and copied into the final stage. +RUN apk add --no-cache font-dejavu + +# The monkeyd-crawler submodule is resolved through a `replace` directive, so +# its go.mod must be present before `go mod download` can read the build list. +# Only the module files are copied here, keeping this layer cached across +# ordinary source edits. COPY go.mod go.sum ./ +COPY third_party/monkeyd-crawler/go.mod third_party/monkeyd-crawler/go.sum ./third_party/monkeyd-crawler/ RUN go mod download COPY . . @@ -16,6 +27,8 @@ RUN CGO_ENABLED=0 GOOS=linux go build \ FROM gcr.io/distroless/static:nonroot COPY --from=builder /out/server /server +# pdfout.FindFont() probes system font paths; this is one of the paths it knows. +COPY --from=builder /usr/share/fonts/dejavu/DejaVuSans.ttf /usr/share/fonts/dejavu/DejaVuSans.ttf USER nonroot:nonroot EXPOSE 8080 ENTRYPOINT ["/server"] diff --git a/README.md b/README.md index dc477b4..13c21d1 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Atlas via long polling and an in-process cron scheduler. | `gold` | Gold paper trading (opt-in; VNAppMob SJC buy/sell VND/luong) | | `coin` | Crypto paper trading in USD (Binance -> Coinbase -> CoinGecko price fallback) | | `stats` | `/stats` (top commands), `/stats users`, `/stats user `, `/stats cmd ` | +| `monkeyd` | `/monkeyd_crawl ` — export a monkeydd.com novel as a PDF (admin-only) | Disable modules with the `MODULES` environment variable. @@ -129,6 +130,33 @@ The stock-only `assets..openedAt` marker identifies the current position lifecycle, invalidates buttons after a full sale and later repurchase, and prevents a position opened after Record date from applying an older event. +### Novel PDF export + +`/monkeyd_crawl ` downloads every chapter of a monkeydd.com novel and +sends it back as a single PDF document, sized for reading on a phone. The +crawling and rendering come from the +[monkeyd-crawler](https://github.com/tiennm99/monkeyd-crawler) submodule; the +module is the Telegram surface around it. + +The command is admin-only, because one invocation makes hundreds of outbound +requests spread over several minutes. Only `monkeydd.com` URLs are accepted — +the extractor is written against that site's markup, and the allowlist also +keeps the bot from being used to fetch arbitrary URLs. A missing scheme is +filled in, so a pasted bare hostname works. + +Exports run one at a time. The bot replies immediately that the export started, +then sends the PDF when it is ready; a second request while one is in flight is +told which novel is currently running. Requests are spaced out by the crawler, +so the run is deliberately slow rather than aggressive. + +Raw pages are cached under the system temp directory, so re-exporting the same +novel costs no requests. The cache is not pruned and a container restart clears +it. Finished PDFs are deleted after upload. Telegram caps bot uploads at 50 MB; +a larger book is reported instead of being sent. + +The runtime image installs DejaVuSans, since the PDF embeds a font that covers +Vietnamese diacritics and the distroless base ships none. + ## Layout ``` @@ -139,12 +167,27 @@ internal/cron/ in-process cron scheduler internal/modules/ Module framework, registry, dispatchers, modules internal/storage/ typed DocStore[T] (Provider + Typed); mongodb runtime + memory (tests). Values persist as flattened native BSON root documents internal/systemstate/ shared `system` collection helper for startup migration records +third_party/monkeyd-crawler/ git submodule; resolved by a go.mod replace directive compose.yml Coolify self-host stack (single bot service) docs/deploy-coolify-selfhosted.md Self-host deploy and operations guide ``` ## Run locally +Clone with submodules — the `monkeyd` module builds against +`third_party/monkeyd-crawler`, and Go resolves it through a `replace` directive +pointing at that directory: + +```sh +git clone --recurse-submodules https://github.com/tiennm99/miti99bot.git + +# already cloned without them: +git submodule update --init --recursive +``` + +Without the submodule checked out, every Go command fails to resolve +`github.com/tiennm99/monkeyd-crawler`. + In-memory storage requires no database. Set the environment variables for your shell, then run the server with Go: diff --git a/cmd/server/main.go b/cmd/server/main.go index 5842ef4..bf6bd7b 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -23,6 +23,7 @@ import ( "github.com/tiennm99/miti99bot/internal/modules/lol" "github.com/tiennm99/miti99bot/internal/modules/loldle" "github.com/tiennm99/miti99bot/internal/modules/misc" + "github.com/tiennm99/miti99bot/internal/modules/monkeyd" "github.com/tiennm99/miti99bot/internal/modules/stats" "github.com/tiennm99/miti99bot/internal/modules/stock" "github.com/tiennm99/miti99bot/internal/modules/util" @@ -80,6 +81,7 @@ func factories() map[string]modules.Factory { return map[string]modules.Factory{ "util": util.New, "misc": misc.New, + "monkeyd": monkeyd.New, "wordle": wordle.New, "loldle": loldle.New, lol.CollectionName: lol.New, diff --git a/docs/deploy-coolify-selfhosted.md b/docs/deploy-coolify-selfhosted.md index 116d28e..a737699 100644 --- a/docs/deploy-coolify-selfhosted.md +++ b/docs/deploy-coolify-selfhosted.md @@ -120,15 +120,23 @@ Successful GIF replies include the result behind Telegram spoiler formatting. 1. New resource → from this Git repo (Docker Compose), or a prebuilt image. The committed [`compose.yml`](../compose.yml) defines the single `bot` service. -2. Set the env vars above in Coolify. -3. **No public domain / port** is needed — polling is outbound-only. Do not +2. **Enable submodule checkout.** The `monkeyd` module builds against + `third_party/monkeyd-crawler`, a git submodule wired in through a `go.mod` + `replace` directive. Coolify must clone submodules, or the Docker build + fails at `go mod download` with an unresolved + `github.com/tiennm99/monkeyd-crawler`. Turn on Coolify's recursive-clone / + submodule option for the resource. If submodules cannot be enabled, drop the + module instead by setting `MODULES` to the list without `monkeyd` — the build + still needs the submodule, so this is only a runtime opt-out. +3. Set the env vars above in Coolify. +4. **No public domain / port** is needed — polling is outbound-only. Do not publish a port or attach a domain. `expose: 8080` keeps the health endpoint reachable only inside Coolify's network. -4. **Exactly one replica.** Telegram permits only one `getUpdates` consumer per +5. **Exactly one replica.** Telegram permits only one `getUpdates` consumer per bot token; a second poller gets HTTP 409, and a second in-process scheduler double-fires crons. Prefer **stop-first redeploys** so two containers never overlap near a cron time. -5. **deploynotify commit SHA:** `SOURCE_COMMIT` is a Coolify predefined +6. **deploynotify commit SHA:** `SOURCE_COMMIT` is a Coolify predefined variable. The bot reads it at startup and DMs the owner on every boot; outside Coolify (local `docker compose up`) it is unset and the DM shows `unknown`. Keep "Include Source Commit in Build" disabled: that setting @@ -136,7 +144,7 @@ Successful GIF replies include the result behind Telegram spoiler formatting. invalidate Docker cache on every commit. Do not add `SOURCE_COMMIT` to `compose.yml`; an interpolated empty value can override Coolify's runtime env-file value. -6. **Health check:** use Coolify's HTTP monitor against `GET /` (returns +7. **Health check:** use Coolify's HTTP monitor against `GET /` (returns `text/plain` `miti99bot ok`). Do **not** use a compose `healthcheck` — the distroless image has no shell/curl and `cmd/server` has no `-healthcheck` flag. Note: `/` reports healthy even if Mongo is unreachable (the driver diff --git a/go.mod b/go.mod index 80bfc07..5a2fd1f 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,15 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/testcontainers/testcontainers-go v0.43.0 github.com/testcontainers/testcontainers-go/modules/mongodb v0.43.0 + github.com/tiennm99/monkeyd-crawler v0.0.0 go.mongodb.org/mongo-driver/v2 v2.7.0 ) +require ( + github.com/go-pdf/fpdf v0.9.0 // indirect + golang.org/x/net v0.57.0 // indirect +) + require ( dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect @@ -62,9 +68,11 @@ require ( go.opentelemetry.io/otel v1.41.0 // indirect go.opentelemetry.io/otel/metric v1.41.0 // indirect go.opentelemetry.io/otel/trace v1.41.0 // indirect - golang.org/x/crypto v0.52.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/tiennm99/monkeyd-crawler => ./third_party/monkeyd-crawler diff --git a/go.sum b/go.sum index f44e5f5..3da38c3 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-pdf/fpdf v0.9.0 h1:PPvSaUuo1iMi9KkaAn90NuKi+P4gwMedWPHhj8YlJQw= +github.com/go-pdf/fpdf v0.9.0/go.mod h1:oO8N111TkmKb9D7VvWGLvLJlaZUQVPM+6V42pp3iV4Y= github.com/go-telegram/bot v1.20.0 h1:4Pea/qTidSspr4WBJw9FbHUMNhYeqszBqQUfsQEyFbc= github.com/go-telegram/bot v1.20.0/go.mod h1:i2TRs7fXWIeaceF3z7KzsMt/he0TwkVC680mvdTFYeM= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -133,16 +135,18 @@ go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -151,18 +155,18 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/internal/modules/monkeyd/export_job.go b/internal/modules/monkeyd/export_job.go new file mode 100644 index 0000000..eeb6e8c --- /dev/null +++ b/internal/modules/monkeyd/export_job.go @@ -0,0 +1,155 @@ +package monkeyd + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime/debug" + "time" + + "github.com/go-telegram/bot" + "github.com/go-telegram/bot/models" + + "github.com/tiennm99/monkeyd-crawler/export" + + "github.com/tiennm99/miti99bot/internal/log" + "github.com/tiennm99/miti99bot/internal/modules/util/chathelper" +) + +const ( + // crawlTimeout bounds one export. A long novel is roughly one request per + // 400ms plus retries, so even a thousand chapters fits well inside this; + // the ceiling exists to release the single export slot if the site stalls. + crawlTimeout = 30 * time.Minute + + // uploadTimeout is separate from crawlTimeout so a slow crawl cannot eat + // the budget needed to actually deliver the finished PDF. + uploadTimeout = 5 * time.Minute + + // statusTimeout bounds the short progress and failure replies. + statusTimeout = 30 * time.Second + + // maxDocumentBytes is the Bot API's upload ceiling for sendDocument. + // Checked before opening the upload so an oversized book fails with an + // explanation instead of a Telegram API error. + maxDocumentBytes = 50 << 20 +) + +// cacheDirName is the shared page cache under the system temp directory. +// Keeping it outside the per-run directory is what makes a repeat export of the +// same novel cost no requests. It is not pruned; a container restart clears it. +const cacheDirName = "miti99bot-monkeyd-cache" + +// export runs one crawl to completion and delivers the PDF. It is called on its +// own goroutine, detached from the Telegram handler context. +func (r *runner) export(b *bot.Bot, msg *models.Message, novelURL string) { + defer r.end() + // A panic here would reach no handler recover and would take the whole + // process down with it, so contain it. + defer func() { + if p := recover(); p != nil { + log.Error("monkeyd export panicked", + "command", commandName, "url", novelURL, "panic", p, "stack", string(debug.Stack())) + r.reportFailure(b, msg, "The export crashed. Nothing was sent.") + } + }() + + ctx, cancel := context.WithTimeout(context.Background(), crawlTimeout) + defer cancel() + + // The PDF is a temporary artefact: it is uploaded and then dropped. + outDir, err := os.MkdirTemp("", "monkeyd-export-*") + if err != nil { + log.Error("monkeyd temp dir failed", "command", commandName, "err", err) + r.reportFailure(b, msg, "Could not create a working directory for the export.") + return + } + defer func() { + if err := os.RemoveAll(outDir); err != nil { + log.Warn("monkeyd temp cleanup failed", "command", commandName, "dir", outDir, "err", err) + } + }() + + result, err := r.exporter(ctx, export.Request{ + NovelURL: novelURL, + OutDir: outDir, + CacheDir: filepath.Join(os.TempDir(), cacheDirName), + // Per-chapter progress is one line per chapter — useful when + // diagnosing a stuck export, too noisy for the default level. + Log: func(format string, args ...any) { + log.Debug("monkeyd crawl: "+fmt.Sprintf(format, args...), "command", commandName, "url", novelURL) + }, + }) + if err != nil { + log.Error("monkeyd export failed", "command", commandName, "url", novelURL, "err", err) + r.reportFailure(b, msg, "Export failed: "+err.Error()) + return + } + + log.Info("monkeyd export done", "command", commandName, "url", novelURL, + "title", result.Title, "chapters", result.Chapters, "words", result.Words) + + if err := sendPDF(b, msg, result); err != nil { + log.Error("monkeyd delivery failed", "command", commandName, "url", novelURL, "err", err) + r.reportFailure(b, msg, "The novel was exported but the PDF could not be sent: "+err.Error()) + } +} + +// sendPDF uploads the finished book as a Telegram document. +func sendPDF(b *bot.Bot, msg *models.Message, result *export.Result) error { + info, err := os.Stat(result.Path) + if err != nil { + return fmt.Errorf("stat pdf: %w", err) + } + if info.Size() > maxDocumentBytes { + return fmt.Errorf("the PDF is %.1f MB, above Telegram's %d MB limit for bots", + float64(info.Size())/(1<<20), maxDocumentBytes>>20) + } + + file, err := os.Open(result.Path) + if err != nil { + return fmt.Errorf("open pdf: %w", err) + } + defer func() { _ = file.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), uploadTimeout) + defer cancel() + + // MessageThreadID keeps the document in the forum topic the command came + // from, matching chathelper.Reply's behaviour. + _, err = b.SendDocument(ctx, &bot.SendDocumentParams{ + ChatID: msg.Chat.ID, + MessageThreadID: msg.MessageThreadID, + Document: &models.InputFileUpload{ + Filename: filepath.Base(result.Path), + Data: file, + }, + Caption: caption(result), + }) + return err +} + +// captionLimit is the Bot API's maximum caption length in characters. +const captionLimit = 1024 + +// caption describes the book under the document. Plain text, so a title +// containing markup characters needs no escaping. +func caption(result *export.Result) string { + text := fmt.Sprintf("%s\n%s page (%.0f x %.0f mm)\n%s", + result.Summary(), result.Page.Name, result.Page.W, result.Page.H, result.SourceURL) + if runes := []rune(text); len(runes) > captionLimit { + text = string(runes[:captionLimit]) + } + return text +} + +// reportFailure tells the chat the export did not produce a document. Delivery +// failures here are only logged — there is nothing further to fall back to. +func (r *runner) reportFailure(b *bot.Bot, msg *models.Message, text string) { + ctx, cancel := context.WithTimeout(context.Background(), statusTimeout) + defer cancel() + if err := chathelper.Reply(ctx, b, msg, text); err != nil { + log.Error("monkeyd failure reply failed", "command", commandName, "err", err) + } +} diff --git a/internal/modules/monkeyd/handlers_test.go b/internal/modules/monkeyd/handlers_test.go new file mode 100644 index 0000000..1dd48d2 --- /dev/null +++ b/internal/modules/monkeyd/handlers_test.go @@ -0,0 +1,286 @@ +package monkeyd + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tiennm99/monkeyd-crawler/export" + "github.com/tiennm99/monkeyd-crawler/pdfout" + + "github.com/tiennm99/miti99bot/internal/modules" + "github.com/tiennm99/miti99bot/internal/storage" + "github.com/tiennm99/miti99bot/internal/testutil" +) + +const testNovelURL = "https://monkeydd.com/tro-lai-nam-thang-cu.html" + +// install wires the module to a recording bot. The returned runner is the one +// behind the registered command, so tests can substitute its exporter and run +// the export synchronously instead of on a detached goroutine. +// +// ownerID is permitted, which /monkeyd_crawl requires: the command is +// Protected, and the dispatcher drops unauthorized calls silently. +func install(t *testing.T, ownerID int64) (*testutil.RecordingBot, *runner) { + t.Helper() + rb := testutil.NewRecordingBot(t) + r := newRunner() + // Run the export inline so assertions do not race a detached goroutine. + r.launch = func(job func()) { job() } + mod := newModule(r) + + reg := &modules.Registry{ + Modules: []modules.Module{{Name: "monkeyd", Commands: mod.Commands}}, + AllCommands: map[string]modules.Command{}, + } + for _, c := range mod.Commands { + reg.AllCommands[c.Name] = c + } + modules.Install(rb.Bot, reg, modules.Auth{BotOwnerID: ownerID}) + return rb, r +} + +// stubPDF creates a file of the given size standing in for a rendered book and +// returns a Result describing it, as export.Export would. The file is sized by +// truncation so an over-the-limit case costs no real bytes. +func stubPDF(t *testing.T, dir, name string, size int64) *export.Result { + t.Helper() + path := filepath.Join(dir, name) + file, err := os.Create(path) + if err != nil { + t.Fatalf("create stub pdf: %v", err) + } + if err := file.Truncate(size); err != nil { + _ = file.Close() + t.Fatalf("size stub pdf: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("close stub pdf: %v", err) + } + return &export.Result{ + Path: path, + Title: "Example Novel", + SourceURL: testNovelURL, + Chapters: 3, + Words: 1200, + Page: pdfout.Presets["phone"], + } +} + +func TestCrawl_NoArgumentRepliesUsage(t *testing.T) { + rb, _ := install(t, 999) + rb.Bot.ProcessUpdate(context.Background(), testutil.NewPrivateMessage(999, "/monkeyd_crawl")) + + if got := rb.LastSent().Text(); !strings.Contains(got, usage) { + t.Errorf("reply = %q, want it to contain %q", got, usage) + } +} + +func TestCrawl_RejectsDisallowedHost(t *testing.T) { + rb, r := install(t, 999) + called := false + r.exporter = func(context.Context, export.Request) (*export.Result, error) { + called = true + return nil, nil + } + + rb.Bot.ProcessUpdate(context.Background(), + testutil.NewPrivateMessage(999, "/monkeyd_crawl https://example.com/novel.html")) + + if called { + t.Error("exporter ran for a disallowed host") + } + if got := rb.LastSent().Text(); !strings.Contains(got, AllowedHostsHint) { + t.Errorf("reply = %q, want it to name %q", got, AllowedHostsHint) + } +} + +func TestCrawl_SendsPDFOnSuccess(t *testing.T) { + rb, r := install(t, 999) + dir := t.TempDir() + + var gotRequest export.Request + r.exporter = func(_ context.Context, req export.Request) (*export.Result, error) { + gotRequest = req + return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil + } + + rb.Bot.ProcessUpdate(context.Background(), + testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL)) + + if gotRequest.NovelURL != testNovelURL { + t.Errorf("exporter got NovelURL %q, want %q", gotRequest.NovelURL, testNovelURL) + } + if gotRequest.OutDir == "" { + t.Error("exporter got an empty OutDir; the PDF would land in the working directory") + } + + calls := rb.Sent() + if len(calls) < 2 { + t.Fatalf("expected an acknowledgement and a document, got %d calls: %+v", len(calls), calls) + } + if first := calls[0]; !strings.Contains(first.Text(), testNovelURL) { + t.Errorf("first reply = %q, want it to name the novel URL", first.Text()) + } + last := calls[len(calls)-1] + if last.Method != "sendDocument" { + t.Fatalf("last call = %q, want sendDocument", last.Method) + } + if caption := last.Form["caption"]; !strings.Contains(caption, "Example Novel") { + t.Errorf("caption = %q, want it to name the novel", caption) + } +} + +func TestCrawl_ReportsExportFailure(t *testing.T) { + rb, r := install(t, 999) + r.exporter = func(context.Context, export.Request) (*export.Result, error) { + return nil, errors.New("fetch failed: 404") + } + + rb.Bot.ProcessUpdate(context.Background(), + testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL)) + + last := rb.LastSent() + if last.Method == "sendDocument" { + t.Fatal("a document was sent despite the export failing") + } + if got := last.Text(); !strings.Contains(got, "404") { + t.Errorf("failure reply = %q, want it to carry the underlying error", got) + } +} + +func TestCrawl_RefusesOversizedPDF(t *testing.T) { + rb, r := install(t, 999) + dir := t.TempDir() + r.exporter = func(context.Context, export.Request) (*export.Result, error) { + return stubPDF(t, dir, "Huge-Novel.pdf", maxDocumentBytes+1), nil + } + + rb.Bot.ProcessUpdate(context.Background(), + testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL)) + + last := rb.LastSent() + if last.Method == "sendDocument" { + t.Fatal("an oversized document was uploaded instead of being refused") + } + if got := last.Text(); !strings.Contains(got, "limit") { + t.Errorf("reply = %q, want it to explain the size limit", got) + } +} + +// The single export slot must be released whether the run succeeded or failed, +// otherwise the command is dead until the process restarts. +func TestCrawl_ReleasesSlotAfterRun(t *testing.T) { + for _, tc := range []struct { + name string + exporter func(t *testing.T) func(context.Context, export.Request) (*export.Result, error) + }{ + { + name: "after success", + exporter: func(t *testing.T) func(context.Context, export.Request) (*export.Result, error) { + dir := t.TempDir() + return func(context.Context, export.Request) (*export.Result, error) { + return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil + } + }, + }, + { + name: "after failure", + exporter: func(*testing.T) func(context.Context, export.Request) (*export.Result, error) { + return func(context.Context, export.Request) (*export.Result, error) { + return nil, errors.New("boom") + } + }, + }, + { + name: "after panic", + exporter: func(*testing.T) func(context.Context, export.Request) (*export.Result, error) { + return func(context.Context, export.Request) (*export.Result, error) { + panic("boom") + } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + rb, r := install(t, 999) + r.exporter = tc.exporter(t) + + rb.Bot.ProcessUpdate(context.Background(), + testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL)) + + r.mu.Lock() + running := r.running + r.mu.Unlock() + if running { + t.Error("export slot still held after the run finished") + } + }) + } +} + +func TestCrawl_RepliesBusyWhileAnotherExportRuns(t *testing.T) { + rb, r := install(t, 999) + // Simulate an in-flight export rather than racing a real one. + if ok, _ := r.begin("https://monkeydd.com/other-novel.html"); !ok { + t.Fatal("could not claim the export slot") + } + + called := false + r.exporter = func(context.Context, export.Request) (*export.Result, error) { + called = true + return nil, nil + } + + rb.Bot.ProcessUpdate(context.Background(), + testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL)) + + if called { + t.Error("a second export started while one was already running") + } + got := rb.LastSent().Text() + if !strings.Contains(got, "other-novel") { + t.Errorf("busy reply = %q, want it to name the in-flight novel", got) + } +} + +// A Protected command must not respond at all to an unauthorized sender, or its +// existence leaks. +func TestCrawl_SilentForUnauthorizedSender(t *testing.T) { + rb, _ := install(t, 999) + rb.Bot.ProcessUpdate(context.Background(), + testutil.NewPrivateMessage(12345, "/monkeyd_crawl "+testNovelURL)) + + if calls := rb.Sent(); len(calls) != 0 { + t.Errorf("expected no reply to an unauthorized sender, got %+v", calls) + } +} + +func TestRegistration(t *testing.T) { + mod := New(modules.Deps{Store: storage.NewMemoryProvider().Collection("monkeyd")}) + if len(mod.Commands) != 1 { + t.Fatalf("expected 1 command, got %d", len(mod.Commands)) + } + cmd := mod.Commands[0] + if cmd.Name != commandName { + t.Errorf("Name = %q, want %q", cmd.Name, commandName) + } + if cmd.Visibility != modules.VisibilityProtected { + t.Errorf("Visibility = %v, want Protected", cmd.Visibility) + } + if cmd.Parameters != "" { + t.Errorf("Parameters = %q, want %q", cmd.Parameters, "") + } + if cmd.Description == "" { + t.Error("Description is empty; command discovery requires one") + } + if cmd.Handler == nil { + t.Error("Handler is nil") + } + if len(mod.Crons) != 0 || len(mod.Callbacks) != 0 { + t.Errorf("expected no crons or callbacks, got %d crons and %d callbacks", + len(mod.Crons), len(mod.Callbacks)) + } +} diff --git a/internal/modules/monkeyd/monkeyd.go b/internal/modules/monkeyd/monkeyd.go new file mode 100644 index 0000000..defac5b --- /dev/null +++ b/internal/modules/monkeyd/monkeyd.go @@ -0,0 +1,151 @@ +// Package monkeyd exports a monkeydd.com novel as a PDF and sends it back as a +// Telegram document. The crawling and rendering live in the monkeyd-crawler +// submodule (third_party/monkeyd-crawler); this module is the Telegram surface +// around it: argument validation, one-at-a-time scheduling, and delivery. +package monkeyd + +import ( + "context" + "fmt" + "strings" + "sync" + + "github.com/go-telegram/bot" + "github.com/go-telegram/bot/models" + + "github.com/tiennm99/monkeyd-crawler/export" + + "github.com/tiennm99/miti99bot/internal/modules" + "github.com/tiennm99/miti99bot/internal/modules/util/chathelper" +) + +// commandName is the single command this module exposes. +const commandName = "monkeyd_crawl" + +// usage is shown when the command arrives without a usable URL. It repeats the +// Parameters syntax so the error and the command menu agree. +const usage = "Usage: /" + commandName + " " + +// New is the module Factory. The module keeps no persistent state — an export +// is a one-shot job — so deps.Store is unused. +func New(_ modules.Deps) modules.Module { + return newModule(newRunner()) +} + +// newModule builds the module around a given runner, which is how tests supply +// one with a stubbed exporter and a synchronous launch. +func newModule(r *runner) modules.Module { + return modules.Module{ + Commands: []modules.Command{ + { + Name: commandName, + Visibility: modules.VisibilityProtected, + // One invocation makes hundreds of outbound requests over + // several minutes, so it stays off the public surface. + Description: "Export a " + AllowedHostsHint + " novel as a PDF", + Parameters: "", + Handler: r.handle, + }, + }, + } +} + +// runner serialises exports. The crawler spaces its own requests out per run, +// so two concurrent crawls would double the request rate against the site — +// and a novel is minutes of work, which makes queueing pointless. One at a +// time, globally, with a clear reply to anyone who asks meanwhile. +type runner struct { + mu sync.Mutex + running bool + current string // novel URL of the in-flight export, for the busy reply + + // exporter is the crawl-and-render step. It is a field so tests can + // exercise scheduling and delivery without network access. + exporter func(context.Context, export.Request) (*export.Result, error) + + // launch runs an export job. Production detaches it onto its own + // goroutine; tests substitute a synchronous run for determinism. + launch func(job func()) +} + +func newRunner() *runner { + return &runner{ + exporter: export.Export, + launch: func(job func()) { + // Detached from the handler context on purpose: handlers run one + // at a time (the bot is built WithNotAsyncHandlers), so crawling + // inline would block every other command for minutes. The job + // owns its own timeout and recovers its own panics. + go job() //nolint:gosec // G118: intentional; see runner.export + }, + } +} + +// begin claims the single export slot, reporting the in-flight URL when it is +// already taken. +func (r *runner) begin(novelURL string) (ok bool, inFlight string) { + r.mu.Lock() + defer r.mu.Unlock() + if r.running { + return false, r.current + } + r.running = true + r.current = novelURL + return true, "" +} + +func (r *runner) end() { + r.mu.Lock() + defer r.mu.Unlock() + r.running = false + r.current = "" +} + +func (r *runner) handle(ctx context.Context, b *bot.Bot, update *models.Update) error { + msg := update.Message + if msg == nil { + return nil + } + + arg := chathelper.ArgAfterCommand(msg.Text) + if arg == "" { + return chathelper.Reply(ctx, b, msg, usage) + } + // Telegram may hand the URL over with trailing punctuation or a stray + // second word; only the first token can be the URL. + if fields := strings.Fields(arg); len(fields) > 0 { + arg = fields[0] + } + + novelURL, err := normalizeNovelURL(arg) + if err != nil { + return chathelper.Reply(ctx, b, msg, fmt.Sprintf("%s.\n%s", capitalize(err.Error()), usage)) + } + + ok, inFlight := r.begin(novelURL) + if !ok { + return chathelper.Reply(ctx, b, msg, + "Already exporting "+inFlight+". Try again once it finishes.") + } + + // Reply before starting so the user knows the wait is expected. If the + // reply cannot be delivered, drop the slot rather than crawling for a chat + // that will never hear the result. + if err := chathelper.Reply(ctx, b, msg, + "Exporting "+novelURL+" — this takes a few minutes. I will send the PDF here when it is ready."); err != nil { + r.end() + return err + } + + r.launch(func() { r.export(b, msg, novelURL) }) + return nil +} + +// capitalize upper-cases the first letter so a lower-case error string reads as +// a sentence in a Telegram reply. +func capitalize(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} diff --git a/internal/modules/monkeyd/novel_url.go b/internal/modules/monkeyd/novel_url.go new file mode 100644 index 0000000..8cf3963 --- /dev/null +++ b/internal/modules/monkeyd/novel_url.go @@ -0,0 +1,71 @@ +package monkeyd + +import ( + "errors" + "net/url" + "strings" +) + +// allowedHosts are the hostnames the crawler's extractor understands. It is +// written against monkeydd.com's specific markup — the chapter list, the +// in-chapter dropdown, and the CSS rules that supply part of the chapter text — +// so another host would parse to nothing useful. Refusing it up front also +// keeps the command from being used to make the bot fetch arbitrary URLs. +var allowedHosts = map[string]bool{ + "monkeydd.com": true, + "www.monkeydd.com": true, +} + +// AllowedHostsHint lists the accepted hosts for user-facing usage text. +const AllowedHostsHint = "monkeydd.com" + +var ( + errNotAURL = errors.New("that does not look like a URL") + errHostNotAllow = errors.New("only " + AllowedHostsHint + " novel URLs are supported") + errNoPath = errors.New("that URL has no novel path — link the novel's own page") +) + +// normalizeNovelURL validates a user-supplied novel URL and returns the form to +// crawl. A missing scheme is filled in with https, since people paste bare +// hostnames; anything else that is not a plain http(s) URL on an allowed host +// is rejected. +func normalizeNovelURL(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", errNotAURL + } + // A bare "monkeydd.com/x.html" parses as a path with no host, so give it a + // scheme before parsing rather than trying to interpret the result. + if !strings.Contains(raw, "://") { + raw = "https://" + raw + } + + parsed, err := url.Parse(raw) + if err != nil { + return "", errNotAURL + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", errNotAURL + } + // Credentials in the URL are never needed here and would be logged with the + // crawl, so treat them as malformed input. + if parsed.User != nil { + return "", errNotAURL + } + // Hostname() drops any port, which the allowlist must not be fooled by. + if !allowedHosts[strings.ToLower(parsed.Hostname())] { + return "", errHostNotAllow + } + if strings.Trim(parsed.Path, "/") == "" { + return "", errNoPath + } + // Rebuild from the parsed parts so the crawl uses a canonical host and no + // fragment; query strings are kept because the site may need them. + canonical := url.URL{ + Scheme: parsed.Scheme, + Host: strings.ToLower(parsed.Host), + Path: parsed.Path, + RawQuery: parsed.RawQuery, + } + return canonical.String(), nil +} diff --git a/internal/modules/monkeyd/novel_url_test.go b/internal/modules/monkeyd/novel_url_test.go new file mode 100644 index 0000000..5d173cc --- /dev/null +++ b/internal/modules/monkeyd/novel_url_test.go @@ -0,0 +1,128 @@ +package monkeyd + +import "testing" + +func TestNormalizeNovelURL(t *testing.T) { + tests := []struct { + name string + raw string + want string + wantErr bool + }{ + { + name: "https url passes through", + raw: "https://monkeydd.com/tro-lai-nam-thang-cu.html", + want: "https://monkeydd.com/tro-lai-nam-thang-cu.html", + }, + { + name: "www host is allowed", + raw: "https://www.monkeydd.com/tro-lai-nam-thang-cu.html", + want: "https://www.monkeydd.com/tro-lai-nam-thang-cu.html", + }, + { + name: "missing scheme gets https", + raw: "monkeydd.com/tro-lai-nam-thang-cu.html", + want: "https://monkeydd.com/tro-lai-nam-thang-cu.html", + }, + { + name: "http is kept", + raw: "http://monkeydd.com/tro-lai-nam-thang-cu.html", + want: "http://monkeydd.com/tro-lai-nam-thang-cu.html", + }, + { + name: "surrounding whitespace is trimmed", + raw: " https://monkeydd.com/tro-lai-nam-thang-cu.html ", + want: "https://monkeydd.com/tro-lai-nam-thang-cu.html", + }, + { + name: "uppercase host is canonicalised", + raw: "https://MonkeyDD.com/tro-lai-nam-thang-cu.html", + want: "https://monkeydd.com/tro-lai-nam-thang-cu.html", + }, + { + name: "fragment is dropped", + raw: "https://monkeydd.com/tro-lai-nam-thang-cu.html#chuong-1", + want: "https://monkeydd.com/tro-lai-nam-thang-cu.html", + }, + { + name: "query is preserved", + raw: "https://monkeydd.com/novel.html?page=2", + want: "https://monkeydd.com/novel.html?page=2", + }, + { + name: "empty input", + raw: "", + wantErr: true, + }, + { + name: "other host", + raw: "https://example.com/novel.html", + wantErr: true, + }, + { + name: "host that merely ends with the allowed name", + raw: "https://evilmonkeydd.com/novel.html", + wantErr: true, + }, + { + name: "host that merely contains the allowed name", + raw: "https://monkeydd.com.evil.example/novel.html", + wantErr: true, + }, + { + name: "allowed host as a subdomain of another host", + raw: "https://monkeydd.com.attacker.test/novel.html", + wantErr: true, + }, + { + name: "non-http scheme", + raw: "file:///etc/passwd", + wantErr: true, + }, + { + name: "credentials in url", + raw: "https://user:pass@monkeydd.com/novel.html", + wantErr: true, + }, + { + name: "landing page with no novel path", + raw: "https://monkeydd.com", + wantErr: true, + }, + { + name: "root path only", + raw: "https://monkeydd.com/", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := normalizeNovelURL(tt.raw) + if tt.wantErr { + if err == nil { + t.Fatalf("normalizeNovelURL(%q) = %q, want error", tt.raw, got) + } + return + } + if err != nil { + t.Fatalf("normalizeNovelURL(%q) returned error: %v", tt.raw, err) + } + if got != tt.want { + t.Errorf("normalizeNovelURL(%q) = %q, want %q", tt.raw, got, tt.want) + } + }) + } +} + +// A port on an allowed host must not defeat the allowlist, and must survive +// canonicalisation so the crawl reaches the same place the user asked for. +func TestNormalizeNovelURLKeepsPort(t *testing.T) { + got, err := normalizeNovelURL("http://monkeydd.com:8080/novel.html") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if want := "http://monkeydd.com:8080/novel.html"; got != want { + t.Errorf("got %q, want %q", got, want) + } +} diff --git a/third_party/monkeyd-crawler b/third_party/monkeyd-crawler new file mode 160000 index 0000000..c925f10 --- /dev/null +++ b/third_party/monkeyd-crawler @@ -0,0 +1 @@ +Subproject commit c925f10ae0c4cf26f2ffbc4fcd893dedcdb81f7a