From d9c7421c7e5ce1cf2d3dd7ab36fef3a399d10666 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Mon, 29 Jun 2026 09:15:29 +0700 Subject: [PATCH] chore(plans): remove completed planning artifacts --- .../phase-01-gcp-setup.md | 63 --- .../phase-02-repo-bootstrap.md | 82 ---- .../phase-03-module-framework.md | 132 ------ .../phase-04-firestore-kv.md | 86 ---- .../phase-05-port-simple-modules.md | 118 ----- .../phase-06-port-loldle-variants.md | 102 ---- .../phase-07-gemini-ai-modules.md | 99 ---- .../phase-08-port-trading.md | 87 ---- .../phase-09-cloud-scheduler.md | 70 --- .../phase-10-ci-cd.md | 90 ---- .../phase-11-tests-observability.md | 77 --- .../phase-12-cutover.md | 98 ---- plans/260508-2222-go-port-cloud-run/plan.md | 96 ---- .../phase-01-aws-bootstrap.md | 69 --- .../phase-02-lambda-runtime.md | 78 --- .../phase-03-dynamodb-kv.md | 88 ---- .../phase-04-eventbridge-cron.md | 92 ---- .../phase-05-gha-deploy.md | 96 ---- .../phase-06-observability.md | 99 ---- .../260510-0114-aws-port/phase-07-cutover.md | 90 ---- plans/260510-0114-aws-port/plan.md | 77 --- .../phase-01-cosmetics.md | 57 --- .../phase-02-metric-filter.md | 62 --- .../phase-03-lolschedule-cron.md | 109 ----- .../phase-04-trading-module.md | 90 ---- .../phase-05-eventbridge-schedules.md | 105 ----- plans/260510-0234-pre-deploy-wrapup/plan.md | 66 --- ...e-reviewer-260510-0244-cron-and-trading.md | 189 -------- ...1-source-inventory-and-migration-policy.md | 55 --- ...-02-backfill-toolchain-and-safety-rails.md | 61 --- .../phase-03-trading-and-durable-kv-import.md | 59 --- ...se-04-parity-verification-and-rehearsal.md | 50 -- ...integration-and-cloudflare-decommission.md | 56 --- .../plan.md | 71 --- .../phase-01-add-trongtruonghop-command.md | 138 ------ .../plan.md | 79 ---- .../phase-01-narrow-oidc-trust-f2.md | 130 ----- .../phase-02-discover-required-actions.md | 193 -------- .../phase-03-draft-custom-policy.md | 137 ------ .../phase-04-cutover-validate.md | 233 --------- .../phase-05-update-bootstrap-docs.md | 148 ------ plans/260518-1019-iam-least-privilege/plan.md | 142 ------ ...ement-deploynotify-package-main-go-hook.md | 141 ------ .../phase-02-wire-git-sha-into-build.md | 98 ---- plans/260522-1109-deploy-notify-owner/plan.md | 66 --- ...e-01-hook-signature-per-user-write-path.md | 62 --- ...se-02-subcommand-parser-new-stats-views.md | 68 --- .../phase-03-tests.md | 62 --- .../phase-04-command-menu.md | 49 -- .../phase-05-docs.md | 48 -- .../plan.md | 80 ---- .../plan.md | 37 -- ...-01-research-and-existing-trade-pattern.md | 65 --- .../phase-02-gold-price-client.md | 87 ---- .../phase-03-gold-portfolio-commands.md | 89 ---- .../phase-04-module-registration-and-docs.md | 69 --- .../phase-05-tests-and-verification.md | 78 --- .../plan.md | 111 ----- .../reports/plan-review-260611-gold-module.md | 35 -- .../pm-260611-gold-module-completion.md | 48 -- .../phase-01-implement-gold-price-command.md | 79 ---- plans/260611-gold-price-command/plan.md | 33 -- .../phase-01-price-provider-chain.md | 114 ----- .../phase-02-portfolio-and-commands.md | 128 ----- .../phase-03-module-registration-and-docs.md | 102 ---- .../phase-04-tests-and-verification.md | 118 ----- .../plan.md | 83 ---- .../plan.md | 63 --- .../phase-01-update-sell-message-and-tests.md | 68 --- .../plan.md | 66 --- .../plan.md | 84 ---- .../phase-01-mongodb-storage-provider.md | 83 ---- .../phase-02-in-process-cron-scheduler.md | 91 ---- ...hase-03-containerize-and-coolify-deploy.md | 123 ----- .../phase-04-data-migration-and-cutover.md | 96 ---- .../phase-05-aws-decommission.md | 139 ------ .../plan.md | 151 ------ ...ase-01-version-field-optimistic-locking.md | 142 ------ ...ase-02-native-bson-value-representation.md | 108 ----- .../plan.md | 128 ----- ...01-schema-contract-and-regression-tests.md | 136 ------ ...phase-02-root-document-storage-encoding.md | 139 ------ .../phase-03-migration-and-documentation.md | 126 ----- .../phase-04-verification-and-rollout.md | 121 ----- .../plan.md | 221 --------- ...se-01-typed-docstore-and-memory-backend.md | 107 ----- .../phase-02-mongo-typed-store.md | 116 ----- .../phase-03-migrate-modules-and-wiring.md | 115 ----- .../phase-04-dynamo-migrator-and-docs.md | 115 ----- .../phase-05-verification-and-rollout.md | 89 ---- .../plan.md | 223 --------- .../plan.md | 50 -- ...mob-api-research-and-key-refresh-design.md | 64 --- ...jc-price-client-with-api-key-management.md | 76 --- ...re-client-into-gold-module-and-handlers.md | 99 ---- plans/phase-04-update-iac-and-env-handling.md | 55 --- plans/phase-05-tests-and-verification.md | 56 --- plans/plan.md | 56 --- ...260612-0948-coin-module-research-report.md | 431 ----------------- ...stable-vn-stock-price-provider-research.md | 292 ------------ ...5-1510-vn-stock-price-provider-research.md | 320 ------------- ...-1523-ssi-direct-current-price-research.md | 182 ------- ...1704-kbs-vs-vci-price-provider-research.md | 164 ------- .../260627-1143-sam-no-s3-deploy-research.md | 166 ------- ...nt-and-cleanup-audit-260628-2344-report.md | 79 ---- ...rm-260517-1411-eventbridge-schedule-fix.md | 94 ---- ...0628-1113-mongo-native-documents-report.md | 89 ---- ...viewer-260508-2254-phase02-03-bootstrap.md | 327 ------------- ...viewer-260508-2333-phase04-firestore-kv.md | 156 ------ ...-reviewer-260509-0813-phase5a-util-misc.md | 165 ------- ...ode-reviewer-260509-0918-phase5b-wordle.md | 240 ---------- ...ode-reviewer-260509-0940-phase5c-loldle.md | 314 ------------ ...viewer-260509-1206-phase6a-loldle-emoji.md | 201 -------- ...er-260513-2133-deploy-aws-script-safety.md | 445 ------------------ ...ode-reviewer-260516-1426-trongtruonghop.md | 42 -- ...518-1019-recent-changes-cron-regression.md | 118 ----- ...reviewer-260518-1019-security-aws-infra.md | 99 ---- ...de-reviewer-260518-1019-security-go-app.md | 101 ---- ...-reviewer-infra-260516-1231-infra-audit.md | 96 ---- ...iewer-modules-260516-1232-modules-audit.md | 89 ---- .../debugger-2026-05-22-stats-failure.md | 115 ----- .../debugger-260516-1057-ci-lint-failure.md | 65 --- ...260518-1019-cfn-http-invoke-unsupported.md | 63 --- ...260518-1019-lolschedule-cron-not-firing.md | 144 ------ ...t-and-s3-elimination-260627-1849-report.md | 133 ------ ...to-cook-native-value-version-cas-review.md | 55 --- ...-to-cook-selfhost-implementation-review.md | 75 --- ...615-0948-vnappmob-gold-price-completion.md | 41 -- ...search-260510-0012-aws-vs-gcp-free-tier.md | 241 ---------- ...0510-0021-aws-vs-gcp-greenfield-rethink.md | 205 -------- ...-0605-mongodb-document-design-standards.md | 258 ---------- .../research-260629-world-cup-schedule-api.md | 229 --------- ...60513-2133-aws-cli-command-verification.md | 282 ----------- ...rcher-260518-1019-security-dependencies.md | 159 ------- 134 files changed, 15650 deletions(-) delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-01-gcp-setup.md delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-02-repo-bootstrap.md delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-03-module-framework.md delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-04-firestore-kv.md delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-05-port-simple-modules.md delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-06-port-loldle-variants.md delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-07-gemini-ai-modules.md delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-08-port-trading.md delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-09-cloud-scheduler.md delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-10-ci-cd.md delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-11-tests-observability.md delete mode 100644 plans/260508-2222-go-port-cloud-run/phase-12-cutover.md delete mode 100644 plans/260508-2222-go-port-cloud-run/plan.md delete mode 100644 plans/260510-0114-aws-port/phase-01-aws-bootstrap.md delete mode 100644 plans/260510-0114-aws-port/phase-02-lambda-runtime.md delete mode 100644 plans/260510-0114-aws-port/phase-03-dynamodb-kv.md delete mode 100644 plans/260510-0114-aws-port/phase-04-eventbridge-cron.md delete mode 100644 plans/260510-0114-aws-port/phase-05-gha-deploy.md delete mode 100644 plans/260510-0114-aws-port/phase-06-observability.md delete mode 100644 plans/260510-0114-aws-port/phase-07-cutover.md delete mode 100644 plans/260510-0114-aws-port/plan.md delete mode 100644 plans/260510-0234-pre-deploy-wrapup/phase-01-cosmetics.md delete mode 100644 plans/260510-0234-pre-deploy-wrapup/phase-02-metric-filter.md delete mode 100644 plans/260510-0234-pre-deploy-wrapup/phase-03-lolschedule-cron.md delete mode 100644 plans/260510-0234-pre-deploy-wrapup/phase-04-trading-module.md delete mode 100644 plans/260510-0234-pre-deploy-wrapup/phase-05-eventbridge-schedules.md delete mode 100644 plans/260510-0234-pre-deploy-wrapup/plan.md delete mode 100644 plans/260510-0234-pre-deploy-wrapup/reports/code-reviewer-260510-0244-cron-and-trading.md delete mode 100644 plans/260515-2250-cf-data-to-aws-migration/phase-01-source-inventory-and-migration-policy.md delete mode 100644 plans/260515-2250-cf-data-to-aws-migration/phase-02-backfill-toolchain-and-safety-rails.md delete mode 100644 plans/260515-2250-cf-data-to-aws-migration/phase-03-trading-and-durable-kv-import.md delete mode 100644 plans/260515-2250-cf-data-to-aws-migration/phase-04-parity-verification-and-rehearsal.md delete mode 100644 plans/260515-2250-cf-data-to-aws-migration/phase-05-cutover-integration-and-cloudflare-decommission.md delete mode 100644 plans/260515-2250-cf-data-to-aws-migration/plan.md delete mode 100644 plans/260516-1409-trongtruonghop-command/phase-01-add-trongtruonghop-command.md delete mode 100644 plans/260516-1409-trongtruonghop-command/plan.md delete mode 100644 plans/260518-1019-iam-least-privilege/phase-01-narrow-oidc-trust-f2.md delete mode 100644 plans/260518-1019-iam-least-privilege/phase-02-discover-required-actions.md delete mode 100644 plans/260518-1019-iam-least-privilege/phase-03-draft-custom-policy.md delete mode 100644 plans/260518-1019-iam-least-privilege/phase-04-cutover-validate.md delete mode 100644 plans/260518-1019-iam-least-privilege/phase-05-update-bootstrap-docs.md delete mode 100644 plans/260518-1019-iam-least-privilege/plan.md delete mode 100644 plans/260522-1109-deploy-notify-owner/phase-01-implement-deploynotify-package-main-go-hook.md delete mode 100644 plans/260522-1109-deploy-notify-owner/phase-02-wire-git-sha-into-build.md delete mode 100644 plans/260522-1109-deploy-notify-owner/plan.md delete mode 100644 plans/260522-1742-stats-per-user-analytics/phase-01-hook-signature-per-user-write-path.md delete mode 100644 plans/260522-1742-stats-per-user-analytics/phase-02-subcommand-parser-new-stats-views.md delete mode 100644 plans/260522-1742-stats-per-user-analytics/phase-03-tests.md delete mode 100644 plans/260522-1742-stats-per-user-analytics/phase-04-command-menu.md delete mode 100644 plans/260522-1742-stats-per-user-analytics/phase-05-docs.md delete mode 100644 plans/260522-1742-stats-per-user-analytics/plan.md delete mode 100644 plans/260605-0256-trade-income-events-command/plan.md delete mode 100644 plans/260611-0735-gold-module-trading-parity/phase-01-research-and-existing-trade-pattern.md delete mode 100644 plans/260611-0735-gold-module-trading-parity/phase-02-gold-price-client.md delete mode 100644 plans/260611-0735-gold-module-trading-parity/phase-03-gold-portfolio-commands.md delete mode 100644 plans/260611-0735-gold-module-trading-parity/phase-04-module-registration-and-docs.md delete mode 100644 plans/260611-0735-gold-module-trading-parity/phase-05-tests-and-verification.md delete mode 100644 plans/260611-0735-gold-module-trading-parity/plan.md delete mode 100644 plans/260611-0735-gold-module-trading-parity/reports/plan-review-260611-gold-module.md delete mode 100644 plans/260611-0735-gold-module-trading-parity/reports/pm-260611-gold-module-completion.md delete mode 100644 plans/260611-gold-price-command/phase-01-implement-gold-price-command.md delete mode 100644 plans/260611-gold-price-command/plan.md delete mode 100644 plans/260612-1005-coin-module-price-fallback/phase-01-price-provider-chain.md delete mode 100644 plans/260612-1005-coin-module-price-fallback/phase-02-portfolio-and-commands.md delete mode 100644 plans/260612-1005-coin-module-price-fallback/phase-03-module-registration-and-docs.md delete mode 100644 plans/260612-1005-coin-module-price-fallback/phase-04-tests-and-verification.md delete mode 100644 plans/260612-1005-coin-module-price-fallback/plan.md delete mode 100644 plans/260615-1145-fix-vnappmob-key-parsing/plan.md delete mode 100644 plans/260620-coin-sell-insufficient-message/phase-01-update-sell-message-and-tests.md delete mode 100644 plans/260620-coin-sell-insufficient-message/plan.md delete mode 100644 plans/260625-1337-stats-reply-budget-timeout/plan.md delete mode 100644 plans/260627-1849-selfhost-coolify-mongodb/phase-01-mongodb-storage-provider.md delete mode 100644 plans/260627-1849-selfhost-coolify-mongodb/phase-02-in-process-cron-scheduler.md delete mode 100644 plans/260627-1849-selfhost-coolify-mongodb/phase-03-containerize-and-coolify-deploy.md delete mode 100644 plans/260627-1849-selfhost-coolify-mongodb/phase-04-data-migration-and-cutover.md delete mode 100644 plans/260627-1849-selfhost-coolify-mongodb/phase-05-aws-decommission.md delete mode 100644 plans/260627-1849-selfhost-coolify-mongodb/plan.md delete mode 100644 plans/260628-1113-mongo-native-value-documents/phase-01-version-field-optimistic-locking.md delete mode 100644 plans/260628-1113-mongo-native-value-documents/phase-02-native-bson-value-representation.md delete mode 100644 plans/260628-1113-mongo-native-value-documents/plan.md delete mode 100644 plans/260628-1310-flatten-mongo-value-documents/phase-01-schema-contract-and-regression-tests.md delete mode 100644 plans/260628-1310-flatten-mongo-value-documents/phase-02-root-document-storage-encoding.md delete mode 100644 plans/260628-1310-flatten-mongo-value-documents/phase-03-migration-and-documentation.md delete mode 100644 plans/260628-1310-flatten-mongo-value-documents/phase-04-verification-and-rollout.md delete mode 100644 plans/260628-1310-flatten-mongo-value-documents/plan.md delete mode 100644 plans/260628-1318-mongo-native-typed-stores/phase-01-typed-docstore-and-memory-backend.md delete mode 100644 plans/260628-1318-mongo-native-typed-stores/phase-02-mongo-typed-store.md delete mode 100644 plans/260628-1318-mongo-native-typed-stores/phase-03-migrate-modules-and-wiring.md delete mode 100644 plans/260628-1318-mongo-native-typed-stores/phase-04-dynamo-migrator-and-docs.md delete mode 100644 plans/260628-1318-mongo-native-typed-stores/phase-05-verification-and-rollout.md delete mode 100644 plans/260628-1318-mongo-native-typed-stores/plan.md delete mode 100644 plans/260629-0043-world-cup-schedule-module/plan.md delete mode 100644 plans/phase-01-vnappmob-api-research-and-key-refresh-design.md delete mode 100644 plans/phase-02-implement-sjc-price-client-with-api-key-management.md delete mode 100644 plans/phase-03-wire-client-into-gold-module-and-handlers.md delete mode 100644 plans/phase-04-update-iac-and-env-handling.md delete mode 100644 plans/phase-05-tests-and-verification.md delete mode 100644 plans/plan.md delete mode 100644 plans/reports/260612-0948-coin-module-research-report.md delete mode 100644 plans/reports/260625-0936-stable-vn-stock-price-provider-research.md delete mode 100644 plans/reports/260625-1510-vn-stock-price-provider-research.md delete mode 100644 plans/reports/260625-1523-ssi-direct-current-price-research.md delete mode 100644 plans/reports/260625-1704-kbs-vs-vci-price-provider-research.md delete mode 100644 plans/reports/260627-1143-sam-no-s3-deploy-research.md delete mode 100644 plans/reports/aws-footprint-and-cleanup-audit-260628-2344-report.md delete mode 100644 plans/reports/brainstorm-260517-1411-eventbridge-schedule-fix.md delete mode 100644 plans/reports/brainstorm-260628-1113-mongo-native-documents-report.md delete mode 100644 plans/reports/code-reviewer-260508-2254-phase02-03-bootstrap.md delete mode 100644 plans/reports/code-reviewer-260508-2333-phase04-firestore-kv.md delete mode 100644 plans/reports/code-reviewer-260509-0813-phase5a-util-misc.md delete mode 100644 plans/reports/code-reviewer-260509-0918-phase5b-wordle.md delete mode 100644 plans/reports/code-reviewer-260509-0940-phase5c-loldle.md delete mode 100644 plans/reports/code-reviewer-260509-1206-phase6a-loldle-emoji.md delete mode 100644 plans/reports/code-reviewer-260513-2133-deploy-aws-script-safety.md delete mode 100644 plans/reports/code-reviewer-260516-1426-trongtruonghop.md delete mode 100644 plans/reports/code-reviewer-260518-1019-recent-changes-cron-regression.md delete mode 100644 plans/reports/code-reviewer-260518-1019-security-aws-infra.md delete mode 100644 plans/reports/code-reviewer-260518-1019-security-go-app.md delete mode 100644 plans/reports/code-reviewer-infra-260516-1231-infra-audit.md delete mode 100644 plans/reports/code-reviewer-modules-260516-1232-modules-audit.md delete mode 100644 plans/reports/debugger-2026-05-22-stats-failure.md delete mode 100644 plans/reports/debugger-260516-1057-ci-lint-failure.md delete mode 100644 plans/reports/debugger-260518-1019-cfn-http-invoke-unsupported.md delete mode 100644 plans/reports/debugger-260518-1019-lolschedule-cron-not-firing.md delete mode 100644 plans/reports/free-tier-audit-and-s3-elimination-260627-1849-report.md delete mode 100644 plans/reports/from-code-reviewer-to-cook-native-value-version-cas-review.md delete mode 100644 plans/reports/from-code-reviewer-to-cook-selfhost-implementation-review.md delete mode 100644 plans/reports/pm-260615-0948-vnappmob-gold-price-completion.md delete mode 100644 plans/reports/research-260510-0012-aws-vs-gcp-free-tier.md delete mode 100644 plans/reports/research-260510-0021-aws-vs-gcp-greenfield-rethink.md delete mode 100644 plans/reports/research-260628-0605-mongodb-document-design-standards.md delete mode 100644 plans/reports/research-260629-world-cup-schedule-api.md delete mode 100644 plans/reports/researcher-260513-2133-aws-cli-command-verification.md delete mode 100644 plans/reports/researcher-260518-1019-security-dependencies.md diff --git a/plans/260508-2222-go-port-cloud-run/phase-01-gcp-setup.md b/plans/260508-2222-go-port-cloud-run/phase-01-gcp-setup.md deleted file mode 100644 index 716d92f..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-01-gcp-setup.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -phase: 1 -title: "GCP setup + free-tier baseline" -status: pending -priority: P1 -effort: "3h" -dependencies: [] ---- - -# Phase 01: GCP setup + free-tier baseline - -## Overview -Stand up the GCP project with all needed APIs enabled, deploy a throwaway Go hello-world to Cloud Run, capture cold-start P95 baseline. Validates that all free-tier services work together before any port code is written. - -## Requirements -- Functional: GCP project ready, Cloud Run accepts deploys, Firestore Native initialized, Secret Manager + Artifact Registry usable, Gemini API key works. -- Non-functional: cold-start P95 measured (target ≤1.5s for static-link Go binary), free-tier budgets confirmed in Billing dashboard. - -## Architecture - -``` -GCP project (free tier) -├── Cloud Run service (region: asia-southeast1) -├── Firestore Native database (region: asia-southeast1, default db) -├── Artifact Registry repo (region: asia-southeast1, format: docker) -├── Secret Manager -├── Cloud Scheduler (jobs created later in Phase 09) -└── Generative Language API (Gemini, key-based, region-agnostic) -``` - -Region pinned to `asia-southeast1` (Singapore) to keep RTT to VN users low. Same region for Cloud Run + Firestore + Artifact Registry to avoid cross-region egress. - -## Related Code Files -- Create: `scripts/gcp-bootstrap.sh` — idempotent setup script -- Create: `docs/gcp-free-tier.md` — captured caps + measured baseline - -## Implementation Steps -1. Create GCP project: `miti99bot-prod`. Confirm billing account linked but free-tier-only. -2. Enable APIs: `run.googleapis.com`, `firestore.googleapis.com`, `cloudscheduler.googleapis.com`, `artifactregistry.googleapis.com`, `secretmanager.googleapis.com`, `generativelanguage.googleapis.com`, `cloudbuild.googleapis.com`. -3. Initialize Firestore Native in `asia-southeast1`. Create `(default)` database. -4. Create Artifact Registry docker repo `miti99bot-go` in `asia-southeast1`. -5. Create runtime service account `miti99bot-runtime@…iam.gserviceaccount.com` with roles: `roles/datastore.user`, `roles/secretmanager.secretAccessor`, `roles/run.invoker` (for Scheduler→Run OIDC). -6. Create deployer service account `miti99bot-deployer@…` for CI with `roles/run.admin`, `roles/artifactregistry.writer`, `roles/iam.serviceAccountUser`. -7. Get a Gemini API key from AI Studio → store as a Secret Manager secret `gemini-api-key` (value-only test now; real wiring in Phase 07). -8. Write a 30-line Go hello-world (stdlib `net/http`) responding "ok" on `/`. Build static binary, multi-stage Dockerfile (`golang:1.23 → distroless/static`). -9. Push image, deploy: `gcloud run deploy miti99bot-baseline --image=… --region=asia-southeast1 --allow-unauthenticated --min-instances=0 --max-instances=2 --memory=128Mi --cpu=1 --timeout=30s`. -10. Cold-start measurement: hit `/` 10× with 5-min spacing (force scale-to-zero between). Record P50/P95/P99 from `gcloud run services logs` or curl `-w "%{time_total}"`. -11. Document baseline in `docs/gcp-free-tier.md`. Tear down baseline service: `gcloud run services delete miti99bot-baseline`. - -## Success Criteria -- [ ] All 7 APIs enabled, billing-free confirmed -- [ ] Firestore Native exists in `asia-southeast1` -- [ ] Hello-world deploys end-to-end -- [ ] Cold-start P95 documented (any number — used as Phase 11 soak gate) -- [ ] Baseline service torn down (no idle resources) - -## Risk Assessment -- **Risk**: GCP requires billing account even for free tier → unexpected charges. **Mitigation**: enable Billing alert at $1, $5, $10 thresholds. -- **Risk**: Firestore in `asia-southeast1` has higher per-op cost than `us-central1` once free tier exceeded. **Mitigation**: latency wins for VN users; monitor reads in Phase 11. -- **Risk**: Gemini API key has no fine-grained quota controls. **Mitigation**: handle 429s gracefully in Phase 07. - -## Rollback -Delete project. No state to preserve. diff --git a/plans/260508-2222-go-port-cloud-run/phase-02-repo-bootstrap.md b/plans/260508-2222-go-port-cloud-run/phase-02-repo-bootstrap.md deleted file mode 100644 index be43607..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-02-repo-bootstrap.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -phase: 2 -title: "New repo bootstrap + webhook skeleton" -status: partial -priority: P1 -effort: "3h" -dependencies: [1] ---- - -# Phase 02: New repo bootstrap + webhook skeleton - -## Overview -Create `miti99bot-go` GitHub repo, init Go module, scaffold HTTP server with `/`, `/webhook`, `/cron/{name}` routes. Wire Telegram webhook secret-token validation. End-to-end test with a dev bot to prove the loop works before any module logic. - -## Requirements -- Functional: `POST /webhook` accepts a Telegram update, validates `X-Telegram-Bot-Api-Secret-Token`, replies 200 OK with no-op handler. `GET /` returns 200 "miti99bot-go ok". Unknown routes 404. -- Non-functional: stdlib `net/http` only (no router framework yet — KISS). Static-linked binary, ≤15 MiB. Cold-start ≤500ms target. - -## Architecture - -``` -cmd/server/main.go ← entrypoint, wires deps + http.ListenAndServe -internal/server/router.go ← HTTP routes, secret-token middleware -internal/server/health.go ← GET / handler -internal/telegram/webhook.go ← /webhook handler, no-op dispatch -internal/telegram/client.go ← grammY-equivalent: github.com/go-telegram/bot wrapper -go.mod (module github.com//miti99bot-go) -Dockerfile ← multi-stage, golang:1.23-alpine → gcr.io/distroless/static -.github/workflows/ci.yml ← go vet + go test + go build (no deploy yet) -README.md -``` - -Choice of `github.com/go-telegram/bot` (not `go-telegram-bot-api/v5`) — actively maintained, generic-handler API, `bot.MatchTypeCommand`, idiomatic. - -## Related Code Files -- Create: `cmd/server/main.go` -- Create: `internal/server/router.go`, `internal/server/health.go` -- Create: `internal/telegram/webhook.go`, `internal/telegram/client.go` -- Create: `Dockerfile`, `.dockerignore`, `.gitignore` -- Create: `.github/workflows/ci.yml` -- Create: `go.mod`, `go.sum`, `README.md` - -## Implementation Steps -1. `gh repo create /miti99bot-go --public --description "Go port of miti99bot for Cloud Run"` (or private — user choice). -2. `git clone` locally. `go mod init github.com//miti99bot-go`. Require Go 1.23. -3. Add deps: `go get github.com/go-telegram/bot`. -4. Write `cmd/server/main.go`: - - Read `PORT`, `TELEGRAM_BOT_TOKEN`, `TELEGRAM_WEBHOOK_SECRET` from env. - - Construct `*bot.Bot`. Register a single dummy `/ping` command (returns "pong") for smoke test. - - Build `http.ServeMux`, attach `/`, `/webhook`, `/cron/{name}` handlers. - - `http.ListenAndServe(":"+port, mux)`. -5. Webhook handler: - - Reject non-POST → 405. - - Compare `X-Telegram-Bot-Api-Secret-Token` header to `TELEGRAM_WEBHOOK_SECRET`. Mismatch → 401. - - Decode JSON `models.Update`, call `b.ProcessUpdate(ctx, &update)`. Return 200. -6. Cron handler stub: returns 200 OK, logs `cron name=`. Real dispatch in Phase 09. -7. Dockerfile multi-stage: - - Builder: `FROM golang:1.23-alpine`, `CGO_ENABLED=0 go build -ldflags="-s -w" -o /server ./cmd/server`. - - Runtime: `FROM gcr.io/distroless/static`, `COPY --from=builder /server /`, `ENTRYPOINT ["/server"]`. -8. `.github/workflows/ci.yml`: matrix on Go 1.23, run `go vet ./...`, `go test ./...`, `go build ./...`. No deploy step yet (Phase 10). -9. Local smoke test: `TELEGRAM_BOT_TOKEN=… TELEGRAM_WEBHOOK_SECRET=local PORT=8080 go run ./cmd/server`. Use `ngrok http 8080`, call Telegram `setWebhook` against the dev bot, `/ping` → "pong". -10. Manual deploy to Cloud Run for end-to-end check: `gcloud run deploy miti99bot-go --source=. --region=asia-southeast1 --set-env-vars=… --set-secrets=TELEGRAM_BOT_TOKEN=…,TELEGRAM_WEBHOOK_SECRET=…`. Point dev bot's webhook at the Cloud Run URL. Verify `/ping`. - -## Success Criteria -- [x] Repo exists, CI workflow defined (`go vet` + `go test -race` + `go build`) -- [ ] `/ping` works against dev bot via Cloud Run URL — **deferred until Phase 01 (GCP setup) lands** -- [x] Secret-token mismatch returns 401 (constant-time compare; covered by `internal/telegram/webhook_test.go`) -- [x] Image size ≤20 MiB (binary 6.4 MB; distroless image ≈9 MB) -- [x] No-secrets-in-git audit clean (no creds present; secrets stripped from `Deps.Env`) - -## Implementation deviations -- Step 1 (`gh repo create`) skipped — repo already exists at `github.com/tiennm99/miti99bot-go`. -- Steps 9–10 (ngrok smoke test, Cloud Run deploy) deferred to Phase 01. -- Step 4's direct `/ping` registration replaced by the Phase 03 module dispatcher; no example module is shipped yet (`MODULES=""` boots cleanly). -- Webhook + cron handlers carry hardenings beyond spec: constant-time secret compare, `MaxBytesReader`, shared-secret bridge for `/cron/{name}` (env: `CRON_SHARED_SECRET`; absent → endpoint disabled), bounded handler context via `bot.WithNotAsyncHandlers`, `bot.WithSkipGetMe` to avoid 5s cold-start blocking call. Code review recommendations C1–C3, H1–H7, M1, M4, L5 applied; remaining M-class items tracked in [report](reports/code-reviewer-260508-2254-phase02-03-bootstrap.md). - -## Risk Assessment -- **Risk**: `gcloud run deploy --source` uses Cloud Build, which has its own free tier (120 build-min/day). **Mitigation**: small Go build is ~30s; well within. CI/CD in Phase 10 may move builds to GHA to keep Cloud Build for fallback. -- **Risk**: `go-telegram/bot` API surface differs from grammY — handler signatures + middleware patterns require relearning. **Mitigation**: stick to common patterns (commands + plain handlers); avoid grammY's plugins/middleware where translation is fuzzy. - -## Rollback -Delete repo + Cloud Run service. CF Worker still owns prod webhook so no impact. diff --git a/plans/260508-2222-go-port-cloud-run/phase-03-module-framework.md b/plans/260508-2222-go-port-cloud-run/phase-03-module-framework.md deleted file mode 100644 index 115caa6..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-03-module-framework.md +++ /dev/null @@ -1,132 +0,0 @@ ---- -phase: 3 -title: "Module framework + storage interfaces" -status: done -priority: P1 -effort: "4h" -dependencies: [2] ---- - -# Phase 03: Module framework + storage interfaces - -## Overview -Replicate the JS plug-n-play module system in Go. Define `Module`, `Command`, `Cron` types. Build static module registry with conflict detection. Define `KVStore` interface (SQL pattern dropped — trading uses Firestore directly). Wire dispatcher to `bot.RegisterHandler` calls. - -## Requirements -- Functional: at runtime, `MODULES` env var (CSV) selects which modules load. Each module exposes `Commands []Command` and optional `Crons []Cron`. Registry detects name conflicts across all visibility levels and aborts on conflict (fail-fast at startup). -- Non-functional: zero reflection, no plugins. Static slice of constructors registered in `internal/modules/registry.go`. Idiomatic Go (interfaces small, structs concrete). - -## Architecture - -``` -internal/modules/ -├── module.go ← Module, Command, Cron types + Visibility enum -├── registry.go ← static map + Build() + name-conflict detection -├── dispatcher.go ← installCommands(b *bot.Bot, reg *Registry) -├── cron_dispatcher.go ← DispatchScheduled(name, deps) -├── validate.go ← validateCommand / validateCron -└── modules.go ← static import map (slice of factories) - -internal/storage/ -├── kv_store.go ← KVStore interface -├── memory_kv.go ← in-memory fake (for tests + smoke) -└── prefix.go ← per-module key prefixing wrapper -``` - -Module type: - -```go -type Visibility int -const ( - VisibilityPublic Visibility = iota - VisibilityProtected - VisibilityPrivate -) - -type Command struct { - Name string // ^[a-z0-9_]{1,32}$ - Visibility Visibility - Description string // required - Handler func(ctx context.Context, b *bot.Bot, u *models.Update) error -} - -type Cron struct { - Schedule string // documentation only - Name string // unique within module - Handler func(ctx context.Context, deps Deps) error -} - -type Module struct { - Name string - Commands []Command - Crons []Cron - Init func(ctx context.Context, deps Deps) error // optional -} - -type Deps struct { - KV KVStore // already prefixed per-module - Firestore *firestore.Client - Gemini *genai.Client - Env map[string]string -} - -type Factory func() Module -``` - -`KVStore` interface mirrors the JS contract: - -```go -type KVStore interface { - Get(ctx context.Context, key string) ([]byte, error) - GetJSON(ctx context.Context, key string, dst any) error // returns ErrNotFound if missing - Put(ctx context.Context, key string, val []byte) error - PutJSON(ctx context.Context, key string, val any) error - Delete(ctx context.Context, key string) error - List(ctx context.Context, prefix string) ([]string, error) -} -``` - -## Related Code Files -- Create: `internal/modules/{module,registry,dispatcher,cron_dispatcher,validate,modules}.go` -- Create: `internal/storage/{kv_store,memory_kv,prefix}.go` -- Modify: `cmd/server/main.go` to construct registry, pass to dispatcher - -## Implementation Steps -1. Define types in `internal/modules/module.go`. Visibility enum + Command/Cron/Module/Deps structs. -2. `internal/modules/validate.go`: - - `validateCommand(c Command) error` — name regex, visibility known, description nonempty, handler nonnil. - - `validateCron(c Cron) error` — name nonempty, handler nonnil. -3. `internal/modules/modules.go`: empty `var Factories = []Factory{}` for now. Each module registers itself in subsequent phases. -4. `internal/modules/registry.go`: - - `Build(env []string, factories []Factory) (*Registry, error)`. - - For each name in `env` ∩ factory map: call factory, validate every command/cron, accumulate into `publicCmds`, `protectedCmds`, `privateCmds`, `allCmds`. - - Detect duplicate command names across all 3 maps → error `command conflict: /foo defined in and `. -5. `internal/modules/dispatcher.go`: - - `Install(b *bot.Bot, reg *Registry)`: iterate `reg.AllCommands`, call `b.RegisterHandler(bot.HandlerTypeMessageText, "/"+name, bot.MatchTypeCommand, handler)`. -6. `internal/modules/cron_dispatcher.go`: - - `DispatchScheduled(ctx, cronName string, reg *Registry, deps Deps)`: look up cron by name across all modules, run all matching handlers concurrently (errgroup). -7. `internal/storage/memory_kv.go`: `sync.Map`-backed KVStore for tests + smoke runs. -8. `internal/storage/prefix.go`: `Prefixed(s KVStore, prefix string) KVStore` wrapper that prepends `:` to all keys. -9. Wire in `cmd/server/main.go`: build registry, install commands, pass to webhook + cron handlers. -10. Unit tests: `registry_test.go` (conflict detection, validation errors), `prefix_test.go` (round-trip). - -## Success Criteria -- [x] Empty `MODULES=""` boots cleanly (no fallback handler today; grammY's `/start` parity deferred to a future phase) -- [x] Two modules with same command name → startup fails with clear error (`TestBuild_DetectsCommandConflict`) -- [x] Per-module KVStore prefix isolation verified by test (`TestBuild_PerModulePrefixedKV`, `TestPrefixed_RoundTrip`, `TestDispatchScheduled_PassesPrefixedDeps`) -- [x] `go vet ./...` + `go test -race -count=1 ./...` green - -## Implementation deviations -- `Factory func() Module` → `Factory func(deps Deps) Module`: handler closures capture deps directly. Eliminates a separate `Module.Init` lifecycle step. -- `Factories []Factory` → `Factories map[string]Factory`: required for `MODULES`-env name lookup; prevents duplicate names at compile-load. -- `Deps` ships only `KV` + `Env` today. `Firestore` + `Gemini` fields land in Phases 04 / 07 (YAGNI). -- Cron uniqueness enforced across modules (registry-level), instead of "concurrent errgroup of all matches" — simpler and matches the one-cron-per-name reality. -- `cmd/server` strips `TELEGRAM_BOT_TOKEN`, `TELEGRAM_WEBHOOK_SECRET`, `CRON_SHARED_SECRET` from `Deps.Env` to prevent accidental leakage. -- Module names validated against `^[a-z0-9_]{1,32}$` (same regex as commands) so KV prefix isolation cannot be subverted by a `:` in the name. - -## Risk Assessment -- **Risk**: `go-telegram/bot` `RegisterHandler` is more general than grammY's `bot.command`. Need to confirm behavior on `/cmd@botname` (group chats). **Mitigation**: library docs say `MatchTypeCommand` strips `@botname`; verify with a group chat test before Phase 05. -- **Risk**: Static factory slice means new modules require code change — same constraint as JS `index.js` static map. Acceptable. - -## Rollback -Revert to Phase 02 main.go. Module framework is purely additive. diff --git a/plans/260508-2222-go-port-cloud-run/phase-04-firestore-kv.md b/plans/260508-2222-go-port-cloud-run/phase-04-firestore-kv.md deleted file mode 100644 index 4f57bae..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-04-firestore-kv.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -phase: 4 -title: "Firestore KVStore + per-module prefixing" -status: done -priority: P1 -effort: "4h" -dependencies: [3] ---- - -# Phase 04: Firestore KVStore + per-module prefixing - -## Overview -Implement `FirestoreKVStore` against the `KVStore` interface from Phase 03. One Firestore collection per module (``), each KV entry one document. Test against the local Firestore emulator. Provide an in-memory fake for module-level unit tests so they don't need the emulator. - -## Requirements -- Functional: `Get/GetJSON/Put/PutJSON/Delete/List` work against Firestore. Per-module isolation via collection name. JSON values stored as `value` field on the document. -- Non-functional: P50 read ≤80ms warm, ≤500ms cold. Connection reused across requests via package-level `*firestore.Client`. Free-tier-aware: avoid `Query.GetAll` on hot paths. - -## Architecture - -``` -internal/storage/ -├── firestore_kv.go ← FirestoreKVStore impl -├── firestore_client.go ← package-level client (lazy init, project ID from env) -└── firestore_kv_test.go ← runs against emulator if FIRESTORE_EMULATOR_HOST set -``` - -Firestore document shape: -``` -collection: -document id: ← URL-safe key (rejects `/` per Firestore rules) -fields: - value: bytes | string | map ← raw bytes for Put, JSON-marshaled struct for PutJSON - updatedAt: timestamp -``` - -`List(prefix)` uses `collection.Where(firestore.DocumentID(), ">=", prefix).Where(firestore.DocumentID(), "<", prefixSuccessor(prefix))`. - -## Related Code Files -- Create: `internal/storage/firestore_kv.go`, `firestore_client.go`, `firestore_kv_test.go` -- Modify: `cmd/server/main.go` to initialize Firestore client, pass to module Deps -- Modify: `internal/modules/dispatcher.go` Deps construction -- Create: `Makefile` target `test-emulator` (start emulator, run tests) - -## Implementation Steps -1. Add dep: `go get cloud.google.com/go/firestore`. -2. `firestore_client.go`: singleton `func Client(ctx) (*firestore.Client, error)` reading `GOOGLE_CLOUD_PROJECT` from env. Reuse across requests. -3. `firestore_kv.go`: - - Struct `FirestoreKVStore { c *firestore.Client; collection string }`. - - `Get(ctx, key)`: `c.Collection(collection).Doc(key).Get(ctx)`. Map `codes.NotFound` → `ErrNotFound`. Return `value` field as bytes. - - `Put(ctx, key, val)`: `Doc(key).Set(ctx, map{"value": val, "updatedAt": time.Now()})`. - - `GetJSON/PutJSON`: marshal/unmarshal via `encoding/json`. - - `Delete`: `Doc(key).Delete(ctx)`. - - `List(prefix)`: `Where(DocumentID >= prefix).Where(DocumentID < successor)`. Iterator → slice of doc IDs. -4. Key validation: reject `/`, empty string, length >1500 bytes (Firestore limit). -5. `firestore_kv_test.go`: skip if `FIRESTORE_EMULATOR_HOST` not set. Round-trip Put/Get/Delete/List/PutJSON/GetJSON/NotFound. -6. Update `internal/storage/memory_kv.go` to support `List(prefix)` symmetrically (iterate map keys). -7. Update `cmd/server/main.go`: - - Init Firestore client at startup. - - For each module, pass `Prefixed(NewFirestoreKVStore(client, module.Name), module.Name)` (collection name = module name = prefix; equivalent to single-collection prefixing). - - Actually: drop the `Prefixed` wrapper for Firestore — collection itself isolates. `Prefixed` only used with `MemoryKV` for tests. -8. Add `Makefile`: `firestore-emulator: gcloud emulators firestore start --host-port=localhost:8085` and `test: FIRESTORE_EMULATOR_HOST=localhost:8085 go test ./...`. - -## Success Criteria -- [x] All KV ops round-trip against emulator (`firestore_kv_test.go`, runs via `make test-emulator`) -- [x] In-memory fake matches Firestore semantics for List ordering + ErrNotFound -- [x] Two modules writing to same key name → no collision (`TestBuild_PerModulePrefixedKV` for memory backend; collection-per-module IS the isolation for Firestore — test exists in registry layer) -- [x] `go test -race -count=1 ./...` green (Firestore tests skip cleanly without emulator) - -## Implementation deviations -- Spec step 3 (wrap Firestore in `Prefixed`) contradicts step 7 (drop the wrapper). We followed step 7 — collection-per-module IS isolation. Memory backend keeps `Prefixed` because all modules share one in-process store. -- Introduced `KVProvider` interface (not in spec): `MemoryProvider` wraps base+Prefixed, `FirestoreProvider` returns one collection per module. `modules.Build` now takes `KVProvider`+env map instead of base `Deps`. Cleaner: modules never see the backend choice. -- Backend selection in `cmd/server/main.go`: Firestore when `GOOGLE_CLOUD_PROJECT` or `FIRESTORE_EMULATOR_HOST` is set (latter supplies a placeholder project ID for the SDK); otherwise in-memory. -- `validateKey` rejects more than spec required: empty, `/`, `.`, `..`, `__namespace__`, > 1500 bytes. `validatePrefix` runs the same check on `List` arguments. -- Binary size 6.4 MB → 17 MB after Firestore SDK + gRPC. Within Phase 02's ≤20 MiB target. Distroless image ≈19 MB. - -## Code review -[Phase 04 review](reports/code-reviewer-260508-2333-phase04-firestore-kv.md) — 0 critical, 3 high (H1 emulator-only-no-project trap, H2 List-prefix unvalidated, H3 bytes-vs-runes doc) all addressed in same session; M1 prefixSuccessor all-0xFF degeneracy documented; remaining mediums deferred. - -## Risk Assessment -- **Risk**: Firestore document IDs reject `/` but JS keys may contain them (e.g. nested loldle state). **Mitigation**: encode `/` → `_` in Put, decode on Get. Document the mapping. Or use base64 for arbitrary keys. -- **Risk**: 50k reads/day hard cap. Listing leaderboards on every request hits this fast. **Mitigation**: cache hot reads in process memory with 5-minute TTL — free for warm instance, costs 0 reads. -- **Risk**: Emulator behavior diverges from prod (e.g. timestamp resolution, indexes). **Mitigation**: smoke a 50-key Put/List against real Firestore at end of phase. - -## Rollback -Drop the Firestore client init, revert to `MemoryKV` for all modules. Modules continue working but lose persistence. diff --git a/plans/260508-2222-go-port-cloud-run/phase-05-port-simple-modules.md b/plans/260508-2222-go-port-cloud-run/phase-05-port-simple-modules.md deleted file mode 100644 index 607a683..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-05-port-simple-modules.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -phase: 5 -title: "Port simple modules (util, misc, wordle, loldle classic)" -status: done -priority: P2 -effort: "6h" -dependencies: [4] ---- - -# Phase 05: Port simple modules - -## Overview -Port the four KV-only, AI-free modules: `util` (info/help renderer), `misc` (stub easter eggs), `wordle` (5-letter game with 14k-word dict), `loldle` classic (LoL champion guesser). Validates the framework end-to-end before the more complex modules. - -## Requirements -- Functional: command parity with JS — same names, same behaviors, same KV state shape (so a future export-import migration is feasible). -- Non-functional: each module file ≤200 lines per code-standards.md. Static word/champion datasets embedded via `go:embed`. - -## Architecture - -``` -internal/modules/util/ -├── util.go ← Module factory, registers /info /help -├── info.go ← /info handler -└── help.go ← /help renderer (groups by visibility) - -internal/modules/misc/ -└── misc.go ← stub commands - -internal/modules/wordle/ -├── wordle.go ← factory, registers /wordle /wguess /wgiveup /wstats -├── game.go ← session state struct, Get/Save via KV -├── guess.go ← scoring (green/yellow/gray) -├── data/words.txt ← 14k word dict (embedded) -└── words.go ← go:embed loader - -internal/modules/loldle/ -├── loldle.go ← factory -├── game.go ← session state -├── champions.go ← go:embed champion JSON -├── data/champions.json -└── compare.go ← attribute comparison logic -``` - -## Related Code Files -- Create: above tree under `internal/modules/{util,misc,wordle,loldle}` -- Modify: `internal/modules/modules.go` Factories slice — append `util.New, misc.New, wordle.New, loldle.New` -- Copy: word list + champion JSON from JS repo (verbatim) - -## Implementation Steps -1. Copy `src/modules/util/*` JS source as reference. Implement `/info` (returns env-derived bot info) + `/help` (groups commands public+protected, omits private). -2. `/help` queries the registry — already accessible via `Deps`. Format as Telegram MarkdownV2. -3. Misc module: port commands as-is (mostly text replies). -4. Wordle: - - Copy `src/modules/wordle/words.txt` to `internal/modules/wordle/data/words.txt`. - - `go:embed data/words.txt` into a `string`, split lines, build a `map[string]struct{}` for O(1) validity checks. - - Game state: `{ word string; guesses []string; status string }` saved per user. - - KV key: `game:`. -5. Loldle classic: - - Copy champion JSON dataset into `data/champions.json`. - - State: `{ targetID string; guesses []string }` per user per UTC day. Key: `game::`. - - Comparison: gender, position, species, resource, range, region, release year. Yields green/yellow/red per attribute. -6. Port unit tests from JS: - - `wordle/format_test.go` — score formatting - - `wordle/guess_test.go` — green/yellow/gray correctness, double-letter edge case - - `loldle/compare_test.go` — each attribute comparison - - `loldle/game_test.go` — daily reset, max-guesses gate -7. Wire into `Factories` slice. `MODULES=util,misc,wordle,loldle` env var enables them. -8. Smoke test on Cloud Run with dev bot. - -## Success Criteria -- [x] `/wordle`, `/wordle `, `/wordle_new`, `/wordle_giveup`, `/wordle_stats` ported (commands renamed from spec's `/wguess` etc. to match JS source) -- [x] `/loldle`, `/loldle `, `/loldle_giveup`, `/loldle_stats`, `/loldle_setmax` (private) ported -- [x] `/help` lists all loaded modules' public + protected commands (util + misc + wordle + loldle) -- [x] All ported tests pass — wordle and loldle JS vitest suites ported verbatim, plus Go-only coverage for race-free pickers, pool exhaustion, render alignment, keylock fan-out -- [x] Image size stays ≤25 MiB after embedding word + champion data (binary 17 MB; 88 KB words.txt + 65 KB champions.json are noise vs the 10 MB Firestore SDK) - -## Cook scope split -This phase shipped in three sub-cooks: -- **5a (done):** util + misc — small, validates the module-loading pipeline end-to-end. ✅ -- **5b (done):** wordle — 14855-word dict, scoring, sessions. ✅ -- **5c (this cook):** loldle classic — 172-champion JSON, attribute comparison, sticker pools. ✅ - -## Implementation deviations (5a) -- `modules.Deps` gained a `Registry *Registry` pointer so `/help` can introspect at runtime. Pointer is captured at factory time and stable thereafter; Registry is documented read-only after Build returns. -- Static factory catalog (`modules.Factories`) moved to `cmd/server/main.go::factories()` to avoid an import cycle (`modules → util → modules`). The empty `internal/modules/modules.go` file remains as a doc anchor. -- `misc.lastPing.At` stored as int64 ms-epoch (matches JS `Date.now()`) — preserves byte-for-byte KV parity for the future export-import migration. -- Telegram-side handler tests intentionally skipped — would require a fake bot HTTP server for negligible coverage gain. Renderer + KV behaviour ARE tested. - -## Implementation deviations (5b — wordle) -- KV TTL: JS uses Cloudflare KV's `expirationTtl: 60*60*24*7`. Firestore has no equivalent per-doc TTL; `gameTTLSeconds` constant is informational. Old games linger — Phase 11 GC if needed. -- `pickDaily` ported but unused (handlers call `pickRandom`). Kept for parity so future "daily wordle" mode is a one-line swap. -- Added `subjectLocks` (per-subject `sync.Mutex` map) to serialise `Get → mutate → Put` in handlers. Cloudflare Workers' isolate model gave the JS source this for free; Go + Firestore needs explicit locking or two concurrent guesses to the same group chat silently lose one. -- `pickRandom(words, nil)` falls through to `math/rand.Intn` (package-level, mutex-protected globals) instead of a singleton `*rand.Rand` so the bot dispatcher's per-update goroutines don't race on RNG state. -- KV wire-format parity: `GameState.Giveup` always emitted (no omitempty); `Stats.LastResultAt` is `*int64` so unplayed accounts marshal as `null` matching JS shape; `StartedAt` is ms-epoch int64. -- Subject IDs converted to strings for KV keys (`game:`); JS uses numbers but Cloudflare KV stringifies on the wire so Firestore round-trips identically. -- Word-list loader panics on malformed embedded data — corrupt regen of `words.txt` is a build-time bug, not a runtime concern worth recovering from. - -## Implementation deviations (5c — loldle) -- Per-subject lock extracted from wordle into `internal/keylock` (shared package). Both wordle and loldle now import it. Naming chosen as a peer to `internal/storage` and `internal/telegram` rather than nesting under `internal/modules/`. -- KV TTL deferred — Cloudflare KV's `expirationTtl` has no Firestore equivalent. Phase 11 GC if old games become a cost concern. -- Sticker pools (win/lose/giveup) preserved verbatim from `stickers.js`; file_ids are bot-scoped to `@miti99bot` and were already valid against the new bot per the test-bot policy. -- `lastResultAt` deliberately omitted from loldle stats (parity with JS source — different from wordle's stats which DOES include it; that asymmetry exists in the JS source). -- `pickRandomChampion` and `pickSticker` use `math/rand.Intn` (package-level mutex-protected globals) so concurrent /loldle handlers don't race on RNG state. Same pattern as wordle 5b. -- `winRate` uses `math.Round` not `int(...)` truncation, after Phase 5c review caught the JS-parity bug. The same fix was retroactively applied to wordle's `/wordle_stats`. - -## Code reviews -- [Phase 5a review](reports/code-reviewer-260509-0813-phase5a-util-misc.md) — 1 critical (`/info` nil-deref), 2 high (1 informational + 1 perf-deferred), 4 mediums/lows. C1, L2, M1, L3, H1 doc applied. -- [Phase 5b review](reports/code-reviewer-260509-0918-phase5b-wordle.md) — 1 critical (`defaultRNG` data race) + 2 high (Get-mutate-Put logical race; dead `debugPickerError`) + extra compare test + race test for `pickRandom`. All addressed in same session. Mediums (M1 giveup-on-never-played JS-faithful gotcha; M2 `subjectFor` test) deferred — JS-parity intentional. -- [Phase 5c review](reports/code-reviewer-260509-0940-phase5c-loldle.md) — 1 high (`winRate` truncation across both wordle + loldle) + 4 mediums (test gaps). H1 fixed in both modules in same session; M1 (render alignment golden test) and M2 (keylock fan-out + serialisation tests) added; M3/M4 deferred — covered transitively elsewhere. - -## Risk Assessment -- **Risk**: 14k-word file embedded → ~120 KiB. `go:embed` puts it in the binary; no runtime IO. Acceptable. -- **Risk**: Wordle scoring has a known JS-side edge case (double-letter); ensure ported logic matches. **Mitigation**: bring the failing-cases test verbatim. -- **Risk**: Loldle daily reset uses UTC in JS; confirm Go uses same. **Mitigation**: explicit `time.Now().UTC()` in date key. - -## Rollback -Remove modules from `Factories` slice or `MODULES` env. Each module is independent. diff --git a/plans/260508-2222-go-port-cloud-run/phase-06-port-loldle-variants.md b/plans/260508-2222-go-port-cloud-run/phase-06-port-loldle-variants.md deleted file mode 100644 index 6ad87d2..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-06-port-loldle-variants.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -phase: 6 -title: "Port loldle variants + lolschedule" -status: partial -priority: P2 -effort: "5h" -dependencies: [5] ---- - -# Phase 06: Port loldle variants + lolschedule - -## Overview -Port the four loldle variants (`loldle-emoji`, `loldle-quote`, `loldle-ability`, `loldle-splash`) plus `lolschedule`. They share the per-day session pattern from classic loldle, differ only in clue-reveal mechanics. - -## Requirements -- Functional: command parity — same commands, same data sources (Riot Data Dragon for ability icons + splash arts; loldle.net derived for emoji + quote pools). -- Non-functional: image-bearing commands (ability icons, splash) reuse remote URLs — do not embed binaries. Reply uses Telegram `sendPhoto` with URL string. - -## Architecture - -``` -internal/modules/loldle-emoji/ -├── module.go -├── data/emoji-pool.json ← embedded -└── game.go - -internal/modules/loldle-quote/ -├── module.go -├── data/quotes.json -└── game.go - -internal/modules/loldle-ability/ -├── module.go -├── ability.go ← URL pattern: ddragon ability icon -└── game.go - -internal/modules/loldle-splash/ -├── module.go -├── data/skin-pool.json ← scraped from loldle.net (per credits in README) -├── splash.go ← ddragon splash URL builder -└── game.go - -internal/modules/lolschedule/ -├── module.go -├── client.go ← lolesports/leaguepedia HTTP client -└── format.go ← schedule formatter -``` - -A small shared package would help, but keep modules independent (KISS) until duplication exceeds 3 callers — then extract. - -## Related Code Files -- Create: above 5 module trees -- Reuse: copy data files from `src/modules//data/*` verbatim -- Modify: `internal/modules/modules.go` Factories slice -- Update: `MODULES` env var in Cloud Run service yaml - -## Implementation Steps -1. **loldle-emoji**: Port emoji clue pool. Game state `{ targetID; guesses []; cluesShown int }`. Reveal one emoji per wrong guess, max 4. -2. **loldle-quote**: Port quote pool. Reveal up to 3 quote chunks across guesses. -3. **loldle-ability**: Build ability icon URL from champion ID + ability slot (Q/W/E/R), e.g. `https://ddragon.leagueoflegends.com/cdn//img/spell/.png`. Cache the latest ddragon version once per cold start. -4. **loldle-splash**: URL pattern `https://ddragon.leagueoflegends.com/cdn/img/champion/splash/_.jpg`. -5. **lolschedule**: HTTP client to lolesports/leaguepedia API for upcoming match schedule. Format with `/lolschedule [date]` syntax (recent commit shows this is current behavior). -6. Use the same KV `game::` namespace pattern (one game per variant per day). -7. Port unit tests for clue-reveal logic, URL builders, schedule formatter. -8. Wire factories. -9. Smoke each command against dev bot. - -## Cook scope split -This phase ships in five sub-cooks (one per module — each is large enough to risk context exhaustion): -- **6a:** loldle-emoji — 172-record emoji clue dict, binary scoring, simplest variant. ✅ -- **6b:** loldle-quote — quote-pool variant, default 6 guesses. ✅ (consumes the shared `chathelper` + `champname` packages extracted in fix-all-review-findings Phase 03) -- **6c:** loldle-ability — DDragon ability-icon URL builder, sendPhoto reply, gameState gains a `slot` field so the same icon shows across guesses. ✅ -- **6d:** loldle-splash — DDragon splash URL, sendPhoto reply, gameState locks `skinId` so the same splash shows across guesses. Default 4 guesses. ✅ -- **6e:** lolschedule — HTTP client to lolesports.com persisted API (cache-first with 60-min stale fallback), ICT-anchored date parsing (dd-mm-yyyy / dd/mm/yyyy / ddmmyyyy), today/week renderers, subscriber list. 5 user commands shipped. Daily-push cron deferred to Phase 09 (Cloud Scheduler) since `Deps` doesn't currently expose a `*bot.Bot` reference. ✅ - -## Success Criteria -- [x] loldle-emoji responds to `/loldle_emoji`, `/loldle_emoji_giveup`, `/loldle_emoji_stats`, `/loldle_emoji_setmax` -- [x] loldle-quote responds to `/loldle_quote`, `/loldle_quote_giveup`, `/loldle_quote_stats`, `/loldle_quote_setmax` -- [x] loldle-ability responds to `/loldle_ability`, `/loldle_ability_giveup`, `/loldle_ability_stats`, `/loldle_ability_setmax`; sendPhoto path uses the DDragon icon URL directly -- [x] loldle-splash responds to `/loldle_splash`, `/loldle_splash_giveup`, `/loldle_splash_stats`, `/loldle_splash_setmax`; sendPhoto path uses the DDragon splash URL directly -- [x] `/lolschedule [date]`, `/lolschedule_today`, `/lolschedule_week`, `/lolschedule_subscribe`, `/lolschedule_unsubscribe` match JS behavior; daily-push cron deferred to Phase 09 -- [x] All variants share consistent guess-count limits matching JS (emoji 5, quote 6 — JS parity) -- [x] Ported tests pass for loldle-emoji + loldle-quote (lookup, state, render, JS-wire-format decode, handler integration) - -## Implementation deviations (6a — loldle-emoji) -- `moduleNameRe` relaxed from `^[a-z0-9_]{1,32}$` to `^[a-z0-9_-]{1,32}$` so JS-source module names like `loldle-emoji` pass validation. The storage prefix delimiter (`:`) remains rejected; tests cover both shapes. -- Go package directory + package name use `loldleemoji` (no separator) per Go convention; the registered MODULE name is `loldle-emoji` (hyphenated, byte-identical to JS) for KV-prefix migration parity. -- `normalize`, `subjectFor`, `argAfterCommand` duplicated from classic loldle. Marked for extraction at the start of cook 6b — three callers will exist by then, past the YAGNI threshold. -- `winRate` uses `math.Round` from day one (lesson from Phase 5c review). -- KV TTL deferred — Cloudflare KV's `expirationTtl` has no Firestore equivalent. -- No sticker pools — JS source has none for emoji mode. - -## Code reviews (6a) -- [Phase 6a review](reports/code-reviewer-260509-1206-phase6a-loldle-emoji.md) — 0 critical, 0 high. Concerns: F#1 (JS-wire-format decode test) added in same session; F#2 (`getOrInitGame` cap-reduction edge case) deferred — defensive branch only; E (extract shared helpers) earmarked as 6b prep work. - -## Risk Assessment -- **Risk**: Riot Data Dragon version pinning — JS version may use different ddragon version than fresh fetch. **Mitigation**: pin version in env or fetch latest at cold start; document in README. -- **Risk**: lolschedule API surface may have changed since JS implementation. **Mitigation**: re-test against live API; fix forward if drifted. -- **Risk**: Splash skin pool was scraped from loldle.net; legality + freshness. **Mitigation**: reuse the same JSON file already in repo (no re-scrape). - -## Rollback -Remove from Factories. Per-variant rollback works independently. diff --git a/plans/260508-2222-go-port-cloud-run/phase-07-gemini-ai-modules.md b/plans/260508-2222-go-port-cloud-run/phase-07-gemini-ai-modules.md deleted file mode 100644 index c2cc0b4..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-07-gemini-ai-modules.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -phase: 7 -title: "Gemini AI + port semantle/doantu/twentyq" -status: done -priority: P2 -effort: "6h" -dependencies: [4] ---- - -# Phase 07: Gemini AI + port semantle/doantu/twentyq - -## Overview -Wire Gemini API as the Workers AI replacement. Port the three AI-using modules: `semantle` and `doantu` use embeddings; `twentyq` uses chat-style text generation. All must respect Gemini free-tier RPM/RPD; degrade gracefully on 429. - -## Requirements -- Functional: - - `semantle`/`doantu`: target word + user guess → cosine similarity score via embeddings. - - `twentyq`: 20-question style game, model plays the responder (yes/no/sometimes), tracks remaining questions. -- Non-functional: - - Free-tier-aware: cache embeddings of game targets (rare changes), retry-with-jitter on 429. - - Gemini client reused as package-level singleton (gRPC connection). - - Per-user RPM soft-limit in process to prevent abuse from blowing through 1500 RPD shared quota. - -## Architecture - -``` -internal/ai/ -├── gemini.go ← package-level *genai.Client init -├── embeddings.go ← Embed(ctx, text) ([]float32, error) using text-embedding-004 -├── chat.go ← Generate(ctx, prompt, history) (string, error) using gemini-1.5-flash -└── ratelimit.go ← per-user token bucket (in-memory, sync.Map of buckets) - -internal/modules/semantle/ -├── module.go -├── data/targets-en.json ← curated daily target pool -├── game.go ← session state -├── score.go ← cosine similarity -└── targets.go ← daily target selection (deterministic from date) - -internal/modules/doantu/ -├── module.go ← Vietnamese variant (different target pool, same algorithm) -└── data/targets-vi.json - -internal/modules/twentyq/ -├── module.go -├── prompt.go ← system prompt + history serialization -├── game.go -└── parser.go ← yes/no/maybe extractor from model output -``` - -`bge-m3` (1024d, multilingual) is replaced by `text-embedding-004` (768d). Different vector space — pre-cached target vectors must be re-computed; do not migrate vectors from CF KV. - -## Related Code Files -- Create: `internal/ai/{gemini,embeddings,chat,ratelimit}.go` -- Create: `internal/modules/{semantle,doantu,twentyq}/...` -- Modify: `Deps` struct (already contains `Gemini *genai.Client` from Phase 03) -- Modify: `cmd/server/main.go` to init Gemini client - -## Implementation Steps -1. Add dep: `go get google.golang.org/genai` (official Google GenAI Go SDK). -2. `internal/ai/gemini.go`: lazy client init from `GEMINI_API_KEY` (Secret Manager → env var injection at deploy). -3. `internal/ai/embeddings.go`: `Embed(ctx, texts []string) ([][]float32, error)` using `text-embedding-004`. Batch up to 100 inputs per call. -4. `internal/ai/chat.go`: `Generate(ctx, system, history []Msg) (string, error)` using `gemini-1.5-flash`. Output ≤200 tokens, temperature 0.7. -5. `internal/ai/ratelimit.go`: per-user 5 req/min bucket via `golang.org/x/time/rate`. Drop-on-exceed with user-visible "slow down" reply. -6. Pre-compute target embeddings: - - At cold start, load target pool, embed any not yet cached in Firestore (`semantle_target_cache:` → `[]float32`). - - 1500 RPD limit means ≤1500 fresh target embeds/day. Curated pool of ~365 targets (one per day) embedded once = ~30 minutes work amortized. -7. Semantle/doantu game flow: user `/semantle`, target picked deterministically from `today's UTC date`. Each `/sguess ` → embed user word → cosine similarity → reply with score. -8. Twentyq: user picks a topic, model (system prompt: "you're answering 20-questions about X, reply only yes/no/maybe"). Track Q count, end at 20. -9. Tests: - - `embeddings_test.go` — fake `*genai.Client` interface; verify cache hit/miss - - `score_test.go` — cosine math - - `parser_test.go` — twentyq response parsing - - `ratelimit_test.go` — bucket refill + drop -10. Smoke each module on dev bot. - -## Success Criteria -- [x] `internal/ai` package wraps `google.golang.org/genai` (v1.56) with Embedder/Chatter interfaces; per-user `PerUserLimiter` (5 req / 60s burst). -- [x] `Deps` extended with `Embedder`/`Chatter` (nil when GEMINI_API_KEY unset → modules refuse with config-error). -- [x] `/semantle` ported: 9894-word google-10k pool, JS-parity sigmoid calibration, OOV gate, fast-path dedup, render board with sort+top-15. -- [x] `/doantu` ported via JS-parity `phow2sim` HTTP client (NOT Gemini — see Deviations below). -- [x] `/twentyq` ported with prompts.go (verbatim JS prompt strings), parser.go (JSON-with-fence extraction), redact-secret defense, fallback round-start. -- [x] 429 from Gemini mapped to `ai.ErrRateLimited` → user-visible "rate-limited" reply. -- [x] All factories registered in `cmd/server/main.go`; `go vet ./...` and `go test -race -count=1 ./...` clean. - -## Deviations from original plan -- **doantu uses phow2sim HTTP, not Gemini embeddings.** Rationale: text-embedding-004 was not trained for Vietnamese semantic relatedness; phow2sim is a domain-trained PhoW2V model. The JS bot already uses it; switching to embeddings would diverge behaviour, not preserve it. `PHOW2SIM_API_URL` overridable via env (allowlisted in `cmd/server/main.go`). -- **No Firestore-backed target embedding cache** (plan step 6). semantle embeds both target+guess on every call (matches JS bge-m3 path). Cache adds complexity without measurable savings until Phase 11 soak data shows the 1500 RPD ceiling is real. -- **gemini-2.5-flash, not 1.5.** SDK default is the newer flash; behaviour-equivalent for the twentyq use case. -- **Per-day cap deferred.** Token bucket only; if Phase 11 soak shows abuse, add a Firestore counter. - -## Risk Assessment -- **Risk**: 768d vs 1024d means similarity scores have different distribution. Game tuning constants (winning threshold) need re-calibration. **Mitigation**: empirical tune against dev bot; document in module file. -- **Risk**: 1500 RPD shared across all users. Heavy semantle play could exhaust. **Mitigation**: per-user 50 req/day soft cap. Cache user-guess embeddings too (most users guess common words). -- **Risk**: `gemini-1.5-flash` cold-start latency (gRPC TLS handshake) on Cloud Run. **Mitigation**: client init at process start, not per-request. -- **Risk**: Gemini may be deprecated or repriced. **Mitigation**: AI ops abstracted behind `internal/ai` package — switching providers (e.g. Vertex AI, OpenRouter free-tier) is a single-package change. - -## Rollback -Remove from Factories. AI modules are isolated by package; main framework continues without them. diff --git a/plans/260508-2222-go-port-cloud-run/phase-08-port-trading.md b/plans/260508-2222-go-port-cloud-run/phase-08-port-trading.md deleted file mode 100644 index e24e766..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-08-port-trading.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -phase: 8 -title: "Port trading + Firestore composite indexes" -status: pending -priority: P2 -effort: "6h" -dependencies: [4] ---- - -# Phase 08: Port trading + Firestore composite indexes - -## Overview -Port the most complex module: VN-stocks paper trading. Original used D1 (relational SQL) for trades + leaderboards. Translate to Firestore document model with composite indexes for the leaderboard query path. - -## Requirements -- Functional: `/trade`, `/buy `, `/sell …`, `/portfolio`, `/leaderboard`, plus the daily price-update cron at `0 17 * * *`. -- Non-functional: leaderboard query stays under 100ms warm. Daily cron fits within 50k-reads/20k-writes per-day cap (≤300 active users, ≤50 unique tickets traded). - -## Architecture - -Firestore data model (replacing D1's `trading_trades` table): - -``` -collection: trading_users ← user state - doc id: - fields: - balanceVnd: number - createdAt: timestamp - lastTradeAt: timestamp - pnlVnd: number ← denormalized for leaderboard - - subcollection: trades ← per-user trade log - doc id: - fields: { ticker, side, qty, priceVnd, ts } - - subcollection: holdings ← current positions (one per ticker) - doc id: - fields: { qty, avgCostVnd } - -collection: trading_prices ← current ticker prices - doc id: - fields: { priceVnd, updatedAt } -``` - -Composite index: `trading_users` on `(pnlVnd DESC)` for leaderboard. Single-field default indexes cover everything else. - -## Related Code Files -- Create: `internal/modules/trading/{module,buy,sell,portfolio,leaderboard,prices,cron_daily_update}.go` -- Create: `internal/modules/trading/store.go` — direct Firestore access (bypassing KVStore for relational queries) -- Create: `firestore.indexes.json` (committed) — composite indexes deployed via `gcloud firestore indexes composite create` -- Modify: `Deps` to include `*firestore.Client` (already present) -- Modify: `MODULES` env var in deploy yaml — add `trading` - -## Implementation Steps -1. **Schema**: Define structs `User`, `Trade`, `Holding`, `Price` in `store.go`. Use `firestore` struct tags. -2. **Buy flow**: - - Read user balance + ticker price. - - Validate sufficient balance + qty > 0. - - In a Firestore `RunTransaction`: decrement balance, increment holding (compute new avgCost), append trade, update `lastTradeAt`. -3. **Sell flow**: - - Symmetric. Realized PnL = (sellPrice - avgCost) * qty. Update `pnlVnd` denorm. -4. **Portfolio**: list holdings + current prices (one read per ticker — typical user holds <10). -5. **Leaderboard**: `Where(pnlVnd > 0).OrderBy(pnlVnd DESC).Limit(10)`. Requires composite index. -6. **Daily price update cron**: - - Triggered by Cloud Scheduler at `0 17 * * *` (set up in Phase 09). - - Fetches VN stock prices from existing data source (port URL/parsing from JS module). - - Writes ~50 ticker docs into `trading_prices`. Stays under 20k writes/day cap easily. -7. **One-time data import** (optional, decided in Phase 12 cutover): script to read D1 dump, transform, write to Firestore. Skip if user opts to start fresh. -8. **Tests**: emulator-based — buy → sell → portfolio → leaderboard parity with JS expectations. -9. **firestore.indexes.json**: capture the composite index definition; `gcloud firestore indexes composite create --collection-group=trading_users --field-config=field-path=pnlVnd,order=descending`. - -## Success Criteria -- [ ] Buy/sell round-trips correctly compute balance + avgCost -- [ ] Leaderboard query returns top 10 by pnl in <100ms -- [ ] Daily price cron runs (manual trigger via `/cron/trading-daily-update` for now) -- [ ] Composite index deployed and active -- [ ] Tests pass against emulator - -## Risk Assessment -- **Risk**: Firestore transactions have a 500-doc / 5MB / 10s limit. Trading transactions are tiny — fine. -- **Risk**: Leaderboard composite index requires explicit creation (Firestore prompts in console on first failed query). **Mitigation**: capture in `firestore.indexes.json` + deploy via gcloud in CI. -- **Risk**: Denormalized `pnlVnd` can drift if a sell update partially fails. **Mitigation**: always update inside transaction with the trade write. -- **Risk**: Free tier 20k writes/day. Per active user, a buy+sell = 4 writes (user, trade, holding, price-touched). 300 users × 5 trades/day = 6k writes — well within. -- **Risk**: VN stock data source may be unstable. **Mitigation**: port the same source used by JS; if cron fails, retry on next run. - -## Rollback -Remove `trading` from `MODULES`. Existing data in `trading_users` collection persists harmlessly; no orphan refs since modules are isolated. diff --git a/plans/260508-2222-go-port-cloud-run/phase-09-cloud-scheduler.md b/plans/260508-2222-go-port-cloud-run/phase-09-cloud-scheduler.md deleted file mode 100644 index 2838bd3..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-09-cloud-scheduler.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -phase: 9 -title: "Cloud Scheduler cron wiring" -status: pending -priority: P2 -effort: "2h" -dependencies: [3] ---- - -# Phase 09: Cloud Scheduler cron wiring - -## Overview -Replace CF Worker `[triggers] crons` with Cloud Scheduler. Each module-declared cron becomes a Scheduler job that POSTs to `/cron/{name}` on the Cloud Run service with an OIDC token, which the service validates before dispatching to module cron handlers. - -## Requirements -- Functional: 2 jobs run on schedule (`0 17 * * *`, `0 1 * * *`). Each invocation reaches the corresponding module cron handlers and completes within Cloud Run timeout. -- Non-functional: free-tier — 3 jobs/mo cap, fits with 33% headroom. OIDC auth so the `/cron/*` endpoint stays Cloud-Scheduler-only (private). No public bypass. - -## Architecture - -``` -Cloud Scheduler Cloud Run -┌───────────────────────┐ ┌───────────────────────────┐ -│ job: cron-0-17 │ POST + OIDC│ /cron/0_17_star_star_star │ -│ schedule: 0 17 * * * │────────────►│ ──► validate OIDC │ -│ target: /cron/0_17... │ │ ──► dispatcher.Dispatch │ -│ auth: OIDC │ │ (cron name = "0 17 * *│ -└───────────────────────┘ │ *") │ - └───────────────────────────┘ -``` - -Path structure: encode the cron expression in the URL (URL-safe form), e.g. `0 17 * * *` → `/cron/0_17_star_star_star`. Or simpler: use a stable name per scheduler job (e.g. `/cron/daily-eod` and `/cron/daily-cleanup`), with the registry mapping name → cron handlers. - -Adopt the named-job approach: cleaner than escaping cron syntax in URLs. - -## Related Code Files -- Modify: `internal/modules/cron_dispatcher.go` — `DispatchByName(ctx, name string, reg *Registry, deps Deps) error` -- Modify: `internal/server/router.go` — `/cron/{name}` handler, validates OIDC token via `google.golang.org/api/idtoken` -- Create: `scripts/setup-scheduler.sh` — idempotent `gcloud scheduler jobs create http …` for each cron -- Modify: per-module `Cron` declarations to use **stable names** (e.g. `daily-eod-update`, `nightly-cleanup`) instead of cron syntax - -## Implementation Steps -1. Refactor `Cron.Schedule` field's role: keep as **documentation only**. Add `Cron.Name` as the stable identifier. Wrangler-style auto-registration is no longer needed. -2. Update `cron_dispatcher.go`: - - `DispatchByName(ctx, name, reg, deps)`: find all crons across all modules where `c.Name == name`. Run with errgroup. Return aggregate error. -3. Update `/cron/{name}` handler: - - Reject non-POST → 405. - - Validate `Authorization: Bearer ` header via `idtoken.Validate(ctx, token, audience=cloudRunURL)`. Confirm `email` claim matches the runtime SA. Mismatch → 401. - - Call `DispatchByName(ctx, mux.Vars["name"], reg, deps)`. - - 200 on success, 500 on dispatcher error (Scheduler retries with backoff). -4. `scripts/setup-scheduler.sh`: - - For each known cron (currently 2): `gcloud scheduler jobs create http --schedule= --uri=/cron/ --http-method=POST --oidc-service-account-email= --oidc-token-audience= --location=asia-southeast1`. - - Idempotent: try `update` first, fall back to `create` on not-found. -5. Local test: simulate Scheduler call with `gcloud scheduler jobs run `; verify Cloud Run logs show successful dispatch. -6. Document in `docs/using-cron.md` (port from JS repo, adjusted for Cloud Scheduler model). - -## Success Criteria -- [ ] 2 Scheduler jobs created in `asia-southeast1` -- [ ] OIDC validation rejects unsigned POSTs (401) -- [ ] Manual `gcloud scheduler jobs run` triggers handler -- [ ] Cron handler error → Scheduler retries (configured retry policy) -- [ ] Stays within 3-job free cap - -## Risk Assessment -- **Risk**: 3-job hard cap. Adding a 4th cron later → paid tier. **Mitigation**: collapse multiple module crons into a single dispatcher endpoint sharing one Scheduler job; or rely on internal-time-based-fan-out (cheaper but less precise). -- **Risk**: OIDC token validation requires correct audience. Misconfig → 401 in prod. **Mitigation**: Phase 09 ends only after manual `jobs run` succeeds. -- **Risk**: Cron handler exceeding Cloud Run timeout (default 5 min, our config 30s). Trading daily update fetches ~50 prices serially. **Mitigation**: parallelize price fetches with errgroup + worker pool of 5. - -## Rollback -`gcloud scheduler jobs delete `. CF Worker still owns prod cron triggers — no missed runs during transition. diff --git a/plans/260508-2222-go-port-cloud-run/phase-10-ci-cd.md b/plans/260508-2222-go-port-cloud-run/phase-10-ci-cd.md deleted file mode 100644 index c4921d7..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-10-ci-cd.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -phase: 10 -title: "CI/CD + Dockerfile + Secret Manager" -status: pending -priority: P2 -effort: "4h" -dependencies: [2] ---- - -# Phase 10: CI/CD + Dockerfile + Secret Manager - -## Overview -Production-grade build + deploy pipeline. GitHub Actions builds image, pushes to Artifact Registry, deploys to Cloud Run. Secrets pulled from Secret Manager at runtime via Cloud Run's `--set-secrets`. Post-deploy hook runs `setWebhook` + `setMyCommands` against Telegram (replacing JS `scripts/register.js`). - -## Requirements -- Functional: PR → CI green; merge to `main` → auto-deploy to Cloud Run; deploy includes Telegram registration. -- Non-functional: build time ≤2 min; no secrets in image, in env yaml, or in repo. Workload Identity Federation between GHA + GCP (no long-lived JSON key). - -## Architecture - -``` -.github/workflows/ -├── ci.yml ← PRs: vet, test, build (no deploy) -└── deploy.yml ← main: build, push to AR, deploy Cloud Run, register Telegram - -cmd/register/main.go ← Go port of scripts/register.js (setWebhook + setMyCommands) -Dockerfile ← finalized multi-stage -firestore.indexes.json ← composite indexes (Phase 08) -.dockerignore -``` - -Secret Manager secrets (created in Phase 01, populated here): -- `telegram-bot-token` -- `telegram-webhook-secret` -- `gemini-api-key` - -Cloud Run service env (non-secret): -- `MODULES=util,misc,wordle,loldle,loldle-emoji,loldle-quote,loldle-ability,loldle-splash,trading,lolschedule,semantle,doantu,twentyq` -- `GOOGLE_CLOUD_PROJECT` -- `LOG_LEVEL=info` - -## Related Code Files -- Create: `.github/workflows/{ci,deploy}.yml` -- Create: `cmd/register/main.go` -- Modify: `Dockerfile` (finalize from Phase 02) -- Create: `infra/cloud-run.yaml` (declarative service spec) OR keep imperative `gcloud run deploy` flags - -## Implementation Steps -1. **Workload Identity Federation setup** (one-time): - - `gcloud iam workload-identity-pools create github-pool --location=global`. - - `gcloud iam workload-identity-pools providers create-oidc github-provider …`. - - Bind `roles/iam.workloadIdentityUser` from GHA repo → `miti99bot-deployer` SA. -2. **Dockerfile finalization**: - - Builder: `FROM golang:1.23-alpine`, install ca-certs, `CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /server ./cmd/server`. - - Runtime: `FROM gcr.io/distroless/static-debian12:nonroot`, `COPY --from=builder /server /server`, `USER nonroot`, `ENTRYPOINT ["/server"]`. -3. **`ci.yml`**: - - Triggers: `pull_request`, `push: branches: [main]`. - - Steps: checkout, setup-go, `go vet ./...`, `go test ./...`, `go build ./...`. (No emulator integration tests in CI — run locally.) -4. **`deploy.yml`**: - - Trigger: `push: branches: [main]` after `ci` workflow succeeds. - - Auth via WIF (`google-github-actions/auth@v2`). - - Build + push: `docker build -t asia-southeast1-docker.pkg.dev/$PROJECT/miti99bot-go/server:$SHA .; docker push …`. - - Deploy: `gcloud run deploy miti99bot-go --image=… --region=asia-southeast1 --service-account=miti99bot-runtime@… --set-env-vars="MODULES=…,GOOGLE_CLOUD_PROJECT=$PROJECT" --set-secrets="TELEGRAM_BOT_TOKEN=telegram-bot-token:latest,TELEGRAM_WEBHOOK_SECRET=telegram-webhook-secret:latest,GEMINI_API_KEY=gemini-api-key:latest" --min-instances=0 --max-instances=2 --memory=256Mi --cpu=1 --timeout=30s --allow-unauthenticated`. - - Apply Firestore indexes: `gcloud firestore indexes composite create --collection-group=trading_users --field-config=field-path=pnlVnd,order=descending` (idempotent — errors on already-exists, swallow). - - Apply Scheduler jobs: invoke `scripts/setup-scheduler.sh` with current Cloud Run URL. - - Post-deploy: `go run ./cmd/register` reads `MODULES` + Cloud Run URL + bot token from env, calls Telegram `setWebhook` (with secret token) + `setMyCommands` (public commands only). Idempotent. -5. **`cmd/register/main.go`**: - - Build registry locally (no Firestore — embed an `OfflineKVStore` that no-ops). Walk public commands. - - HTTP POST to `https://api.telegram.org/bot/setWebhook` with `{url, secret_token, allowed_updates: ["message"]}`. - - HTTP POST to `…/setMyCommands` with `{commands: [{command, description}]}`. - - `--dry-run` flag prints payloads without calling API (parity with JS `register:dry`). -6. **Concurrency-1 lock** on `deploy.yml` to prevent overlapping deploys (Cloud Run handles multiple revisions, but webhook race is annoying). -7. **Smoke after deploy**: GHA waits 10s, curls `/` → expect 200 "miti99bot-go ok"; if not, fail the workflow. - -## Success Criteria -- [ ] PR triggers CI, all checks pass -- [ ] Merge → deploy runs end-to-end, Cloud Run revision served -- [ ] No secret values appear in workflow logs -- [ ] Telegram webhook is set after deploy (verify `getWebhookInfo`) -- [ ] `setMyCommands` reflects current `MODULES` -- [ ] Image size ≤30 MiB - -## Risk Assessment -- **Risk**: WIF setup is tricky; bad bind → GHA can't auth. **Mitigation**: validate via a manual workflow run before relying on auto-deploy. -- **Risk**: Deploy runs Telegram register before Cloud Run is healthy → Telegram pings new URL, gets 503. **Mitigation**: smoke `/` first, register only after. -- **Risk**: A bad deploy auto-flips webhook to broken revision. **Mitigation**: Cloud Run keeps prior revision; manual `gcloud run services update-traffic` is the rollback. Document in deployment-guide.md. -- **Risk**: Cost spike if `--max-instances` set too high under attack. **Mitigation**: capped at 2 — handles VN-side org load comfortably; raise only if measured. - -## Rollback -`gcloud run services update-traffic miti99bot-go --to-revisions==100`. Re-register webhook against prev URL not needed (URL is service-level, not revision-level). diff --git a/plans/260508-2222-go-port-cloud-run/phase-11-tests-observability.md b/plans/260508-2222-go-port-cloud-run/phase-11-tests-observability.md deleted file mode 100644 index 5de3794..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-11-tests-observability.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -phase: 11 -title: "Test parity + observability" -status: partial -priority: P3 -effort: "4h" -dependencies: [8] ---- - -# Phase 11: Test parity + observability - -## Overview -Reach test-count parity with the JS suite where applicable. Wire structured JSON logs to Cloud Logging. Add lightweight metrics (counters for command invocations, errors, AI calls). Soak the Go service against a test bot for 48 hours before cutover. - -## Requirements -- Functional: every JS test that covers logic (not framework/transport) has a Go counterpart. Logs are JSON-shaped, consumable by Cloud Logging severity filters. -- Non-functional: no external metrics backend (free-tier discipline) — Cloud Logging structured fields used as the metrics surface (Log Explorer + Log-based Metrics, all free up to default quota). - -## Architecture - -``` -internal/log/ -├── logger.go ← slog.Logger configured with JSON handler, severity → Cloud Logging convention -└── middleware.go ← request log: msg=req method= path= status= ms= - -internal/metrics/ -└── counters.go ← incrCommand(name), incrError(kind), incrAI(model). Logged at info severity. - -tests/integration/ ← optional: emulator-based end-to-end (not run in CI) -``` - -`slog` (Go 1.21+) handles JSON output. Cloud Logging auto-parses structured `severity` + `message` + custom fields when written to stdout. - -## Related Code Files -- Create: `internal/log/{logger,middleware}.go` -- Create: `internal/metrics/counters.go` -- Modify: every module command handler — add `metrics.IncCommand("/wordle")` etc. -- Modify: `internal/ai/*` — add `metrics.IncAI("embedding")` and `metrics.IncError("ai-429")` paths -- Add: per-module `*_test.go` files until parity reached - -## Implementation Steps -1. **Logger**: `slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo, ReplaceAttr: replaceLevelKey}))`. Map slog `level` → Cloud Logging `severity` convention (DEBUG/INFO/WARNING/ERROR). -2. **Request middleware**: wraps `/webhook`, `/cron/*`. Logs `{msg: "req", method, path, status, ms}` at info. Mirrors JS index.js shape. -3. **Counters**: `metrics.IncCommand(name)` increments an in-memory `sync.Map[string]*atomic.Int64`. Periodic flush every 60s logs `{msg: "metrics", commands: {...}, errors: {...}, ai: {...}}` then resets. Graceful shutdown flushes once on SIGTERM. -4. **Log-based metrics in GCP** (one-time setup, document in deployment-guide.md): - - Counter on `severity=ERROR` → alerts. - - Counter on `jsonPayload.msg=req AND jsonPayload.status>=500` → 5xx rate. - - Counter on `jsonPayload.msg=metrics` → daily aggregation by command. -5. **Test parity audit**: - - Run `find . -name "*.test.js" | wc -l` against JS repo for baseline. - - Run `find . -name "*_test.go"` count in Go repo. - - Aim ≥80% of JS tests have a Go counterpart. Skip framework-only tests (e.g. CF Worker fetch handler tests) — they have no analogue. -6. **48-hour soak**: - - Point a separate test bot at the Cloud Run service. - - Manual playthrough of every module's commands × 3 users. - - Watch Cloud Logging for errors. Watch Firestore reads/writes per day. - - Capture cold-start P95 (`severity=INFO AND jsonPayload.msg=req` filtered to first request after gap). -7. **Compare to Phase 01 baseline**: if Phase 11 cold-start P95 > Phase 01 baseline × 1.5, investigate before cutover (gRPC client init usually the suspect). - -## Success Criteria -- [x] **Logger** ported: `internal/log/log.go` exposes `slog.JSONHandler` writing to stdout, severity-aware via `LOG_LEVEL` env (Phase 04 of fix-all-review-findings forward-ported this). -- [x] **Request middleware** ported: `internal/server/log_middleware.go` wraps every route and emits `{msg:"req", method, path, status, ms}` per request. -- [x] **In-memory counters** ported: `internal/metrics/counters.go` exposes `IncCommand`/`IncError`/`IncAI` with 60s periodic `Flush` to `{msg:"metrics", commands, errors, ai}`. Wired into the dispatcher so every command invocation + handler error is counted; `cmd/server/main.go` runs the flush loop bound to rootCtx (one final flush on SIGTERM). -- [x] Test coverage 69.8% across 20 packages (`fix-all-review-findings` Phase 05 raised it from 44.7% baseline). Module-level coverage: champname/keylock/telegram 100%, util 90%, log/chathelper/loldle/wordle/misc 77-81%, others ≥70%. -- [ ] All errors during 48h soak triaged — **deferred** (requires Cloud Run deployment). -- [ ] Cold-start P95 ≤1.5s — **deferred** (requires Phase 01 GCP baseline). -- [ ] Daily Firestore reads <40k cap — **deferred** (production observation). -- [ ] Cloud Logging log-based metrics setup — **deferred** (one-time GCP console / `gcloud logging metrics create`; document in `docs/deployment-guide.md` once Phase 01 lands). -- [ ] No memory leaks check — **deferred** (production observation). - -## Risk Assessment -- **Risk**: in-memory counters lost when instance scales to zero. **Mitigation**: acceptable — Cloud Logging is the source of truth via per-request log lines; in-memory counters are just convenience for debugging. -- **Risk**: 48h soak reveals a cold-start regression we can't fix without major rework. **Mitigation**: trigger an abort criterion → keep CF Worker as primary, treat Go as standby. -- **Risk**: log-based metrics setup is fiddly. **Mitigation**: use `gcloud logging metrics create` in `scripts/setup-logging.sh`, idempotent. - -## Rollback -None needed — observability is read-only. If logging is overly noisy, lower default level via env var. diff --git a/plans/260508-2222-go-port-cloud-run/phase-12-cutover.md b/plans/260508-2222-go-port-cloud-run/phase-12-cutover.md deleted file mode 100644 index 81a3298..0000000 --- a/plans/260508-2222-go-port-cloud-run/phase-12-cutover.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -phase: 12 -title: "Cutover + decommission CF Worker" -status: pending -priority: P3 -effort: "3h" -dependencies: [10, 11] ---- - -# Phase 12: Cutover + decommission CF Worker - -## Overview -Final phase: flip the prod Telegram webhook from CF Worker to Cloud Run, observe a 7-day soak with both deployments side by side (CF still receives bg cron triggers, Cloud Run owns webhook), then decommission the Worker. - -## Requirements -- Functional: prod bot answers from Cloud Run after webhook flip. No commands regress. Daily cron jobs continue running. -- Non-functional: rollback to CF Worker is one Telegram API call away (`setWebhook` back to Worker URL) at any time during the soak. - -## Architecture - -Cutover sequence: - -``` -Day 0 test-bot soak passed (Phase 11) - ↓ -Day 0 setWebhook(prod-bot, CloudRunURL) - ↓ ↑ rollback: setWebhook(prod-bot, WorkerURL) -Day 0–7 observe error rate, AI quota, Firestore quota - ↓ -Day 7 delete CF Worker, KV namespace, D1 database - ↓ -Day 7 update README to point at miti99bot-go repo - ↓ -Day 7+ journal entry, archive plan -``` - -Data migration (one-shot, optional): -- Trading: export D1 → transform → write to Firestore. Done before cutover so user balances persist. Script: `cmd/migrate-trading/main.go` reads from D1 export JSON, writes via Firestore client. -- Loldle/wordle/semantle: session state is per-day; users start fresh on Day 0. No migration. -- Twentyq: stateless (history embedded in conversation). No migration. - -## Related Code Files -- Create: `cmd/migrate-trading/main.go` (one-shot, optional based on user choice) -- Modify: `README.md` of JS repo — point to Go repo, mark archived -- Modify: `wrangler.toml` — comment out crons, set up auto-disable webhook on next deploy -- Create: `docs/cutover-runbook.md` — sequence + rollback exact commands - -## Implementation Steps -1. **Pre-flight checklist** (before flipping webhook): - - [ ] Phase 11 success criteria all green - - [ ] Cloud Run service has prod-secret values, not test - - [ ] Telegram `setMyCommands` reflects production module set - - [ ] Trading data exported from D1 (if migrating) -2. **Trading data import** (only if user opted in): - - Export D1: `wrangler d1 execute miti99bot-db --command="SELECT * FROM trading_trades" --json > trades.json`. Repeat for users + holdings tables. - - Transform script: read JSON, write to Firestore `trading_users/{id}` + subcollections. - - Verify counts match: `gcloud firestore export` count = D1 row count. -3. **Webhook flip**: - - `curl -F "url=/webhook" -F "secret_token=" https://api.telegram.org/bot/setWebhook`. - - `getWebhookInfo` to confirm. - - Send a `/info` to prod bot, verify response from Cloud Run (check Cloud Logging). -4. **Soak (Day 0–7)**: - - Daily check: Firestore quota usage, error rate from Cloud Logging, Gemini RPD usage. - - User-facing channel for issue reports (existing channel). - - If criticals → `setWebhook` back to CF Worker URL → diagnose → re-attempt cutover. -5. **Decommission (Day 7)**: - - `wrangler deployments list` — note current revision (rollback insurance). - - `wrangler delete miti99bot` — removes service. - - Drop CF KV namespace + D1 database via dashboard. - - Remove `wrangler.toml` cron triggers + secrets via `wrangler secret delete`. - - JS repo: add archive notice to README, push final commit `chore: archive — superseded by miti99bot-go`. -6. **Wrap-up**: - - Run `/ck:journal` to capture lessons learned. - - `ck plan archive` on this plan. - - Mark `260425-1945-mongodb-atlas-migration` plan as superseded (already documented in its frontmatter once this plan is created). - -## Success Criteria -- [ ] Prod webhook hits Cloud Run, no commands regress -- [ ] 7-day soak completes without rollback -- [ ] CF resources fully torn down (Worker, KV, D1) -- [ ] JS repo archived, README points to Go repo -- [ ] Free-tier budget unbroken throughout soak (no surprise bills) - -## Risk Assessment -- **Risk**: Webhook flip is atomic in Telegram but Cloud Run cold-start delays first replies. **Mitigation**: schedule flip during VN low-traffic hours (3-4am Saigon). -- **Risk**: User complaints about lost game state (loldle/wordle in-flight). **Mitigation**: announce cutover in channel, urge users to finish open games. -- **Risk**: Trading data import has subtle schema mismatch leading to corrupt balances. **Mitigation**: import to a `trading_users_staging` collection first, eyeball-verify a few users, then rename collection (Firestore lacks rename — copy then delete original). -- **Risk**: A pending CF cron at the moment of cutover runs against decommissioned data. **Mitigation**: pause CF crons (set `[triggers] crons = []` and deploy) before flipping webhook. Cloud Scheduler picks up at next interval. -- **Risk**: Telegram caches commands client-side; `setMyCommands` takes minutes to propagate. **Mitigation**: tolerate; cosmetic only. - -## Rollback -- **During soak**: `setWebhook(prod-bot, WorkerURL)` reverts. CF Worker still alive. -- **After Day 7 decommission**: rollback requires re-deploying Worker from git history. Document last-known-good wrangler version in cutover-runbook.md before deletion. - -## Next Steps -- Archive this plan: `ck plan archive 260508-2222-go-port-cloud-run` -- Run `/ck:journal` for retrospective -- Update `docs/development-roadmap.md` of Go repo with post-cutover priorities (e.g. observability dashboards, additional modules) diff --git a/plans/260508-2222-go-port-cloud-run/plan.md b/plans/260508-2222-go-port-cloud-run/plan.md deleted file mode 100644 index a5622d1..0000000 --- a/plans/260508-2222-go-port-cloud-run/plan.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -title: "Go port of miti99bot to Google Cloud Run (free tier)" -description: "Full rewrite of the grammY/Cloudflare Worker bot in Go, deployed to Cloud Run with Firestore + Gemini + Cloud Scheduler — all free-tier." -status: in-progress -priority: P2 -effort: 5-7d -branch: main -tags: [go, cloud-run, gcp, firestore, gemini, port, telegram-bot] -created: 2026-05-08 -blockedBy: [] -blocks: [] -supersedes: [260425-1945-mongodb-atlas-migration] ---- - -# Plan: Go port → Google Cloud Run (free tier) - -> **2026-05-10:** Deploy phases (01, 09–12) **superseded by [`plans/260510-0114-aws-port/`](../260510-0114-aws-port/plan.md)** — strict $0 free-tier goal motivated switch to AWS Lambda + DynamoDB + EventBridge. Module work (phases 03–07) is done and **reused unchanged** by the AWS plan. Phase 08 (trading) remains pending, cloud-agnostic, can be tackled before or after the AWS cutover. - -Full rewrite of miti99bot in Go for deployment on Cloud Run, swapping CF KV+D1+Workers AI for Firestore Native + Gemini API + Cloud Scheduler. Source repo lives at a new `miti99bot-go` repo (separate). Cutover via dual-run + soak. - -## Locked decisions -- **Compute**: Cloud Run (min-instances=0, scale-to-zero). Free tier: 2M req/mo, 360k vCPU-s, 180k GiB-s. -- **Storage**: Firestore Native, region `asia-southeast1`. Free: 1 GiB, 50k reads/d, 20k writes/d. -- **AI**: Gemini API via `google.golang.org/genai`. `text-embedding-004` (768d) + `gemini-1.5-flash`. Free: 15 RPM / 1500 RPD per model. -- **Cron**: Cloud Scheduler. 3 jobs/mo free → fits 2 current crons (`0 17 * * *`, `0 1 * * *`). -- **Secrets**: Secret Manager. Free: 6 active secret versions, 10k access ops/mo. -- **Image registry**: Artifact Registry. Free: 0.5 GiB storage. -- **Telegram lib**: `github.com/go-telegram/bot` (modern, idiomatic, active). -- **Repo layout**: separate `miti99bot-go` repo. JS/TS repo stays as-is during port. -- **Cutover**: dual-run on test bot → flip prod webhook → 7-day overlap → decommission CF Worker. - -## Reports -- [code-reviewer 2026-05-08 — Phase 02-03 bootstrap](reports/code-reviewer-260508-2254-phase02-03-bootstrap.md) (3 critical + 7 high addressed in same session; 2 medium + nits deferred) -- [code-reviewer 2026-05-08 — Phase 04 Firestore](reports/code-reviewer-260508-2333-phase04-firestore-kv.md) (0 critical, 3 high all addressed; mediums deferred) -- [code-reviewer 2026-05-09 — Phase 5a util+misc](reports/code-reviewer-260509-0813-phase5a-util-misc.md) (1 critical /info nil-deref + L2 KV wire-format mismatch with JS, both fixed; M1 doc + L3 escape test applied) -- [code-reviewer 2026-05-09 — Phase 5b wordle](reports/code-reviewer-260509-0918-phase5b-wordle.md) (1 critical defaultRNG race + 1 high Get-mutate-Put race + dead-code; all fixed in same session; per-subject mutex added to serialise compound KV ops) -- [code-reviewer 2026-05-09 — Phase 5c loldle](reports/code-reviewer-260509-0940-phase5c-loldle.md) (1 high winRate truncation in both loldle AND wordle; both fixed; render + keylock test gaps closed) -- [code-reviewer 2026-05-09 — Phase 6a loldle-emoji](reports/code-reviewer-260509-1206-phase6a-loldle-emoji.md) (0 critical/high; JS-wire-format decode test added; shared-helper extraction queued for 6b) - -## Phases - -| # | Phase | Status | Effort | Key deliverable | -|---|-------|--------|--------|-----------------| -| 01 | [GCP setup + free-tier baseline](phase-01-gcp-setup.md) | pending | 3h | Hello-world Go on Cloud Run, cold-start P95 captured | -| 02 | [New repo bootstrap + webhook skeleton](phase-02-repo-bootstrap.md) | partial | 3h | `miti99bot-go` repo, `/webhook` validates secret token (Cloud Run deploy + Telegram smoke test deferred to Phase 01) | -| 03 | [Module framework + storage interfaces](phase-03-module-framework.md) | done | 4h | Module/Command/Cron interfaces, registry, dispatcher | -| 04 | [Firestore KVStore + per-module prefixing](phase-04-firestore-kv.md) | done | 4h | `FirestoreKVStore`, emulator tests, KVProvider abstraction (Memory + Firestore) | -| 05 | [Port simple modules (util/misc/wordle/loldle)](phase-05-port-simple-modules.md) | done | 6h | 4 KV-only modules at JS parity; shared `internal/keylock` extracted | -| 06 | [Port loldle variants + lolschedule](phase-06-port-loldle-variants.md) | done | 5h | All five sub-modules ported (emoji, quote, ability, splash, lolschedule); lolschedule daily-push cron deferred to Phase 09 | -| 07 | [Gemini AI + port semantle/doantu/twentyq](phase-07-gemini-ai-modules.md) | done | 6h | `internal/ai` (Embedder/Chatter + per-user bucket); semantle (text-embedding-004), doantu (phow2sim HTTP — JS-parity deviation), twentyq (gemini-2.5-flash) | -| 08 | [Port trading + composite indexes](phase-08-port-trading.md) | pending | 6h | VN-stocks paper trading + daily price cron | -| 09 | [Cloud Scheduler cron wiring](phase-09-cloud-scheduler.md) | pending | 2h | 2 jobs → `/cron/{name}` with OIDC | -| 10 | [CI/CD + Dockerfile + Secret Manager](phase-10-ci-cd.md) | pending | 4h | GHA pipeline → AR → Cloud Run, idempotent | -| 11 | [Test parity + observability](phase-11-tests-observability.md) | partial | 4h | Code-side done: `internal/log` (slog JSON), request log middleware, `internal/metrics` counters + 60s flush, dispatcher instrumented. 48h soak + cold-start measurement + log-based metrics setup deferred to post-deploy. | -| 12 | [Cutover + decommission CF Worker](phase-12-cutover.md) | pending | 3h | Prod webhook flipped, soak passed, Worker retired | - -## Dependency graph - -``` -01 ──► 02 ──► 03 ──► 04 ──► 05 ──► 06 ─┐ - ├──► 07 ─────┤ - └──► 08 ─────┤ - 03 ──────────► 09 ───────┤ - 02 ──────────► 10 ───────┤ - 08 ──► 11 ──► 12 ◄── 10 -``` - -## Free-tier budget at peak - -| Resource | Cap | Expected | Headroom | -|---|---|---|---| -| Cloud Run req | 2M/mo | ~30k/mo | 99% | -| Cloud Run vCPU-s | 360k | ~5k | 99% | -| Firestore reads | 50k/day | ~5k/day | 90% | -| Firestore writes | 20k/day | ~2k/day | 90% | -| Cloud Scheduler jobs | 3 | 2 | 33% | -| Gemini RPM (flash) | 15 | <5 burst | 67% | -| Gemini RPD | 1500 | ~200 | 87% | -| Secret Manager versions | 6 | 3 | 50% | -| Artifact Registry storage | 0.5 GiB | <50 MiB | 90% | - -If Firestore reads cap is hit → enable Cloud Run instance-level cache (warm-instance memo). If Gemini RPD cap is hit → degrade twentyq with a "free tier exhausted, retry tomorrow" reply. - -## Abort criteria -- **Cold-start P95 > 1.5s** sustained (Phase 01 baseline + Phase 11 soak): retain JS Worker for time-sensitive surfaces. -- **Firestore reads > 80% of cap** during Phase 11 soak: add KV-style instance cache before cutover. -- **Gemini quota exhaustion** during normal use: switch to lower-RPM-friendly Vertex AI (still free under credit) or accept degraded UX. - -## Rollback -Per-phase rollback documented in each phase file. Phase 12 is the only irreversible step; until then, the CF Worker continues to serve prod via existing webhook. - -## Open questions -_Resolved 2026-05-08:_ -1. ~~Scheduler cron names~~ → **Keep `0 17 * * *` UTC** (= midnight Saigon). Cloud Scheduler stays UTC, no behavior change vs. JS Worker. -2. ~~Migrate KV/D1 data~~ → **Migrate everything**. One-shot export of D1 + KV → Firestore on cutover. Phase 12 owns the migration script. -3. ~~Test Telegram bot~~ → **User creates the bot manually**, token + webhook secret injected via Cloud Run env vars (`TELEGRAM_BOT_TOKEN`, `TELEGRAM_WEBHOOK_SECRET`). diff --git a/plans/260510-0114-aws-port/phase-01-aws-bootstrap.md b/plans/260510-0114-aws-port/phase-01-aws-bootstrap.md deleted file mode 100644 index 1eed35c..0000000 --- a/plans/260510-0114-aws-port/phase-01-aws-bootstrap.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -phase: 1 -title: "AWS bootstrap + IAM OIDC + SAM skeleton" -status: pending -priority: P1 -effort: "3h" -dependencies: [] ---- - -# Phase 01: AWS bootstrap + IAM OIDC + SAM skeleton - -## Overview -Stand up the AWS account with strict $0 footprint: IAM OIDC trust for GitHub Actions, baseline SAM stack that deploys an empty Lambda + DynamoDB table + Function URL placeholder. Nothing wired to real bot yet. - -## Requirements -- **Functional:** Empty stack deploys via `sam deploy --guided` from local. GitHub Actions can assume the deploy role via OIDC (no long-lived keys). -- **Non-functional:** Region `ap-southeast-1`. Single AWS account. Stack name `miti99bot-aws-port`. All resources tagged `app=miti99bot, env=prod`. Strict free-tier resources only. - -## Architecture -``` -GitHub Actions ─OIDC─► AWS IAM Role (github-deploy) - │ └─ trust: token.actions.githubusercontent.com - │ └─ scoped: repo:tiennm99/miti99bot:ref:refs/heads/main - │ - └─► CloudFormation (SAM) ─► Lambda + DynamoDB + ParamStore + EventBridge + Logs -``` - -## Related Code Files -- Create: `template.yaml` (SAM root, all resources declared here) -- Create: `samconfig.toml` (stack name, region, capabilities) -- Create: `aws/iam-github-oidc-trust.json` (one-shot reference doc, not deployed) -- Create: `aws/README.md` (commands cheat sheet for first-time setup) -- Create: `Makefile` (targets: `build`, `package`, `deploy`, `logs`) -- Modify: `.gitignore` (add `.aws-sam/`, `samconfig.toml.local`) - -## Implementation Steps -1. Create AWS account (or reuse existing). Enable MFA on root, create IAM admin user for one-time bootstrap. Set region default `ap-southeast-1`. -2. Create the GitHub OIDC identity provider in IAM: thumbprint, audience `sts.amazonaws.com`. (One-time, manual or via small CloudFormation snippet.) -3. Create IAM role `github-deploy-miti99bot` with trust policy scoped to `repo:tiennm99/miti99bot:ref:refs/heads/main` and `repo:tiennm99/miti99bot:ref:refs/heads/dev`. Attach managed policies for SAM deploy: CloudFormation, Lambda, DynamoDB, EventBridge, IAM (PassRole only), SSM Parameter Store, Logs, S3 (SAM staging bucket). -4. Write `template.yaml` skeleton: - - `AWSTemplateFormatVersion: '2010-09-09'`, `Transform: AWS::Serverless-2016-10-31` - - `Globals.Function`: `Runtime: provided.al2023`, `Architectures: [arm64]`, `MemorySize: 256`, `Timeout: 15`, `Tracing: Active` (still free at this volume) - - `Resources.BotFunction`: empty handler (`bootstrap` not yet built), Function URL with `AuthType: NONE` - - `Resources.BotTable`: DynamoDB on-demand, PK=`pk` (S), no GSI yet - - Outputs: function URL, table name -5. Write `samconfig.toml` with stack name, region, capabilities (`CAPABILITY_IAM`). -6. First deploy: `sam build && sam deploy --guided` from local using bootstrap admin credentials. Confirm stack reaches `CREATE_COMPLETE`. Save the Function URL. -7. Verify GH Actions OIDC by running a one-shot workflow that calls `aws sts get-caller-identity` — confirms trust works without keys. -8. Manual smoke: `curl ` returns 502 (no handler yet) — proves URL is reachable. - -## Success Criteria -- [ ] AWS account active, MFA on root, region default `ap-southeast-1` -- [ ] GitHub OIDC provider created -- [ ] `github-deploy-miti99bot` IAM role assumes successfully from a test GH Actions run -- [ ] `sam deploy` succeeds; stack `miti99bot-aws-port` in `CREATE_COMPLETE` -- [ ] DynamoDB table `miti99bot` exists, on-demand billing mode -- [ ] Function URL reachable (502 expected) -- [ ] AWS Cost Explorer shows $0 spend after 24h - -## Risk Assessment -- **OIDC trust scope too loose** (any branch / any repo) → Mitigation: scope to specific repo + ref pattern; review `sub` claim in CloudTrail after first successful run. -- **IAM policy over-broad** → Mitigation: start with managed policies for speed, tighten in Phase 06 once resource ARNs stable. -- **SAM staging bucket created in wrong region / accumulates artifacts** → Mitigation: pin region in samconfig; add lifecycle rule (7-day expiration) on staging bucket. -- **CloudFormation drift** if user edits via console → Mitigation: forbid console edits, document in `aws/README.md`. - -## Open questions -1. Single account vs separate dev/prod accounts? Single is simpler for solo dev; defer split until usage warrants. -2. Reuse SAM staging bucket from existing AWS work or fresh one? Fresh, scoped to this stack, easier to clean up. -3. Pin SAM CLI version in `Makefile`? Yes, document expected version (current latest works); rely on `setup-sam` action in CI to pin. diff --git a/plans/260510-0114-aws-port/phase-02-lambda-runtime.md b/plans/260510-0114-aws-port/phase-02-lambda-runtime.md deleted file mode 100644 index cf2a28f..0000000 --- a/plans/260510-0114-aws-port/phase-02-lambda-runtime.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -phase: 2 -title: "Lambda runtime (Go ZIP + LWA + Function URL)" -status: pending -priority: P1 -effort: "4h" -dependencies: [1] ---- - -# Phase 02: Lambda runtime (Go ZIP + LWA + Function URL) - -## Overview -Make the existing Go HTTP server run as a Lambda behind a Function URL with zero handler-code changes, using AWS Lambda Web Adapter. Routes `/` (healthcheck) and `/webhook` work end-to-end with secret verification. - -## Requirements -- **Functional:** Function URL responds to `GET /` with the existing health JSON, and to `POST /webhook` with the existing Telegram dispatcher logic. Secret-token verification (`X-Telegram-Bot-Api-Secret-Token`) preserved. -- **Non-functional:** Cold start P95 < 1.5s for ARM64 Go ZIP. Memory 256 MiB. Timeout 15s. Binary size <30 MiB. - -## Architecture -``` -Telegram ──HTTPS──► Function URL ──► Lambda runtime - │ - ├── LWA layer (extension) translates Lambda event → HTTP - │ └── localhost:8080 (LWA listens here) - │ - └── bootstrap binary (existing Go server) - starts http.ListenAndServe(":8080", ...) - dispatcher → modules → DynamoDB / Gemini -``` - -LWA is added as a Lambda layer; binary just runs `http.ListenAndServe` — no Lambda SDK import required. - -## Related Code Files -- Create: `cmd/server/lambda.go` (build-tag `lambda`, sets `PORT=8080` defaults; minimal — possibly empty) -- Modify: `cmd/server/main.go` — accept `PORT` env (likely already does), confirm graceful shutdown on `SIGTERM` (LWA sends it on shutdown) -- Modify: `template.yaml` — wire `BotFunction` properly: - - `CodeUri: build/` (ZIP staging) - - `Handler: bootstrap` - - `Layers: [arn:aws:lambda:ap-southeast-1:753240598075:layer:LambdaAdapterLayerArm64:]` - - `Environment.Variables`: `AWS_LAMBDA_EXEC_WRAPPER=/opt/bootstrap`, `PORT=8080`, `READINESS_CHECK_PATH=/`, `MODULES=util,misc,wordle,...`, `TELEGRAM_BOT_TOKEN={{resolve:ssm-secure:...}}`, etc. -- Modify: `Makefile` — add `build-lambda` target: `GOOS=linux GOARCH=arm64 go build -tags lambda.norpc -ldflags="-s -w" -o build/bootstrap ./cmd/server && chmod +x build/bootstrap` -- Reference: `internal/server/router.go` (unchanged) -- Reference: `internal/telegram/*.go` (unchanged) - -## Implementation Steps -1. Confirm `cmd/server/main.go` reads `PORT` env (it does per inspection). Confirm graceful shutdown on `SIGTERM`/`SIGINT`. -2. Add `build-lambda` Makefile target. Test locally: `make build-lambda && file build/bootstrap` shows ARM64 ELF. -3. Pick latest LWA layer ARN for `ap-southeast-1` ARM64 — pin major version in `template.yaml` with comment linking to release notes. -4. Add Function env vars in `template.yaml`. Use `{{resolve:ssm-secure:...}}` for secrets so values never appear in template. Reference Phase 01's Parameter Store names. -5. Wire DynamoDB IAM permissions (read/write on table) via `Policies: - DynamoDBCrudPolicy`. Wire SSM read perms via `SSMParameterReadPolicy`. -6. Build + deploy: `sam build && sam deploy`. Tail logs: `sam logs --tail`. -7. Smoke test: - - `curl /` → 200 with health JSON - - `curl -XPOST /webhook -H "X-Telegram-Bot-Api-Secret-Token: wrong"` → 401 - - `curl -XPOST /webhook -H "X-Telegram-Bot-Api-Secret-Token: " -d '{"update_id":1,"message":{"text":"/start","chat":{"id":1},"from":{"id":1}}}'` → 200 (or expected dispatcher response) -8. Set Telegram dev-bot webhook to Function URL. Send `/start` from real client. Confirm response in chat. -9. Capture cold-start P95 from CloudWatch Logs `Init Duration` field over 20+ invocations (use Powertools or grep). Record in this phase's "Risks" if >1s. - -## Success Criteria -- [ ] `make build-lambda` produces ARM64 binary <30 MiB -- [ ] `sam deploy` updates `BotFunction` successfully -- [ ] `curl /` returns health JSON -- [ ] Wrong webhook secret → 401 -- [ ] Correct webhook secret + valid update → dispatcher responds -- [ ] Telegram dev bot exchanges messages end-to-end via Function URL -- [ ] Cold start P95 < 1.5s - -## Risk Assessment -- **LWA cold-start tax** adds ~100ms — acceptable; if not, fall back to `lambda.Start()` adapter path (rewrite handler, more invasive). -- **`{{resolve:ssm-secure:...}}` requires CloudFormation perms** — SAM handles this if role has `ssm:GetParameter*`. -- **Webhook secret leakage in logs** — confirm `internal/server/router.go` does not log header value; if it does, redact. -- **Binary too large** (>50 MiB unzipped) — strip with `-ldflags="-s -w"` (already done); if still too big, audit deps with `go build -ldflags="-s -w" -trimpath` + `goweight`. -- **ARM64 incompatibility** with any cgo dep — confirm `CGO_ENABLED=0` in build (existing Dockerfile does this). - -## Open questions -1. Should LWA `READINESS_CHECK_PATH` be `/` or a dedicated `/healthz`? `/` works since handler is cheap; revisit if `/` ever does work. -2. Telegram delivery reliability with cold-start 1–3s — acceptable in practice (Telegram retries), but document in README. -3. Provisioned concurrency to eliminate cold start — kills free tier, defer indefinitely. diff --git a/plans/260510-0114-aws-port/phase-03-dynamodb-kv.md b/plans/260510-0114-aws-port/phase-03-dynamodb-kv.md deleted file mode 100644 index 1f7ea42..0000000 --- a/plans/260510-0114-aws-port/phase-03-dynamodb-kv.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -phase: 3 -title: "DynamoDB KV provider" -status: pending -priority: P1 -effort: "4h" -dependencies: [1] ---- - -# Phase 03: DynamoDB KV provider - -## Overview -Add `DynamoDBKVStore` + `DynamoDBProvider` as a sibling to the existing Firestore impl, satisfying the same `KVStore` / `KVProvider` interface. Selectable via `KV_PROVIDER=dynamodb|firestore|memory` env. Default in production: `dynamodb`. Firestore impl preserved for parity tests. - -## Requirements -- **Functional:** All existing modules' KV ops (Get/Put/Delete/List + JSON convenience methods) work against DynamoDB with byte-for-byte parity to Firestore where observable. -- **Non-functional:** Single-table design. On-demand billing. P99 < 50ms for Get/Put. List() with prefix uses `Query` (not `Scan`) — must be cheap. - -## Architecture -**Single-table schema (composite key):** -``` -TableName: miti99bot-data -PK (pk): string = moduleName (e.g. "wordle") -SK (sk): string = caller-provided key (e.g. "user:42:state") -attrs: - value: binary raw bytes - updatedAt: number epoch nanos (parity with Firestore impl) -``` - -Composite key is the canonical DynamoDB shape for prefix-scan workloads: `Query` supports `begins_with(sk, :prefix)` on the **sort key**, but only `=` on the partition key — so the sort key holds the user-supplied key and the partition key holds the module name (which gives free isolation by partition). - -**Operations:** -- `Get(key)` → `GetItem(pk=module, sk=key)` with `ConsistentRead: true` (parity with Firestore strong read) -- `Put(key, val)` → `PutItem(pk=module, sk=key, value=val, updatedAt=now)` -- `Delete(key)` → `DeleteItem(pk=module, sk=key)` -- `List(prefix)` → `Query(pk=module AND begins_with(sk, prefix))` paginated - -**Provider isolation:** `For(moduleName)` returns a `DynamoDBKVStore` bound to that module name; the partition key is the isolation boundary. No prefix wrapping needed at this layer. - -**Reserved word handling:** `value` is reserved in DynamoDB expressions; resolved via `ExpressionAttributeNames` (`#v` → `value`). - -## Related Code Files -- Create: `internal/storage/dynamodb_client.go` — AWS SDK v2 client init, region from env -- Create: `internal/storage/dynamodb_kv.go` — `DynamoDBKVStore` (Get/Put/Delete/List + JSON helpers) -- Create: `internal/storage/dynamodb_provider.go` — `DynamoDBProvider`, `For()` returns module-bound store -- Create: `internal/storage/dynamodb_kv_test.go` — uses `localstack` or DynamoDB Local via `testcontainers-go` -- Create: `internal/storage/dynamodb_provider_test.go` — cross-module isolation -- Create: `internal/storage/parity_test.go` (optional) — runs the same op sequence against Memory + Firestore + DynamoDB and asserts identical observables -- Modify: `cmd/server/main.go` — read `KV_PROVIDER`, branch on value; default `dynamodb` when running on Lambda (detect via `AWS_LAMBDA_FUNCTION_NAME` env) -- Modify: `go.mod` — add `github.com/aws/aws-sdk-go-v2`, `…/config`, `…/service/dynamodb`, `…/feature/dynamodb/attributevalue`, `…/feature/dynamodb/expression` - -## Implementation Steps -1. Add SDK deps. `go mod tidy`. -2. Implement `DynamoDBKVStore` with the same method set as `FirestoreKVStore`. Key composition: `pk = moduleName + "#" + key`. -3. `List(prefix)` implementation — `Query` with `KeyConditionExpression` on PK begins-with semantics (use range trick or `BEGINS_WITH` on PK; AWS docs: `BEGINS_WITH` works on sort key only, so use the start/end range trick on PK directly). -4. JSON helpers (`GetJSON`, `PutJSON`) — mirror Firestore impl exactly: marshal/unmarshal with `encoding/json`, store as binary, `ErrNotFound` semantics preserved. -5. Tests with DynamoDB Local (Docker image `amazon/dynamodb-local`). Add `make dynamodb-local` target. Skip if `DYNAMODB_LOCAL_URL` env unset (so CI without Docker can still build). -6. Cross-module isolation test: Put `wordle#k=A`, `loldle#k=B`, assert `wordleStore.Get("k") == A`, `loldleStore.Get("k") == B`, `loldleStore.List("") returns ["k"]` (not `[wordle#k, loldle#k]`). -7. Wire provider selection in `main.go`: - ```go - switch os.Getenv("KV_PROVIDER") { - case "dynamodb": kv = storage.NewDynamoDBProvider(...) - case "firestore": kv = storage.NewFirestoreProvider(...) - default: kv = storage.NewMemoryProvider() - } - ``` -8. Manual smoke against deployed Lambda: send `/start`, then verify `aws dynamodb scan --table-name miti99bot --max-items 5` shows expected keys. - -## Success Criteria -- [ ] `dynamodb_kv_test.go` passes against DynamoDB Local -- [ ] `dynamodb_provider_test.go` passes (cross-module isolation) -- [ ] `parity_test.go` (if added) passes — Memory ≡ Firestore ≡ DynamoDB on observables -- [ ] `KV_PROVIDER=dynamodb` works in deployed Lambda end-to-end -- [ ] One full game session of `/wordle` → state persists across invocations (cold-start safe) -- [ ] List() with prefix returns expected keys, no `Scan` calls in CloudWatch metrics - -## Risk Assessment -- **`List` performance** if a module accumulates >1k keys — DynamoDB Query handles this fine via paginated results. Confirm caller iterates pages (or all calls fit in one page). -- **Item size limit (400 KB)** — modules generally store small JSON; document the cap and add a `len(val) > 380*1024 → error` guard. -- **Eventual consistency** — DynamoDB defaults to eventually consistent reads. Use `ConsistentRead: true` in `Get` to match Firestore's strong default. Costs 2× the RCU but on-demand absorbs it. -- **Reserved word `value`** — DynamoDB reserves many names; use `ExpressionAttributeNames` `#v = "value"` to avoid the conflict. -- **AWS SDK v2 cold-start tax** (~80ms) — acceptable; cache client instance globally in `init()` or pkg-level var. - -## Open questions -1. TTL attribute for ephemeral keys (e.g. wordle daily state)? Add optional `ttl` Number attr; modules opt in via a new method or skip for v1. -2. Use single PK or PK+SK? Sticking with single PK for KISS — no current module needs sort-key queries. -3. Encryption — DynamoDB uses AWS-owned KMS by default (free); switch to AWS-managed only if compliance demands it. -4. Backup strategy — point-in-time recovery is paid; for free-tier hobby use, accept "no backup" and document. diff --git a/plans/260510-0114-aws-port/phase-04-eventbridge-cron.md b/plans/260510-0114-aws-port/phase-04-eventbridge-cron.md deleted file mode 100644 index 4c4c9e9..0000000 --- a/plans/260510-0114-aws-port/phase-04-eventbridge-cron.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -phase: 4 -title: "EventBridge cron wiring" -status: pending -priority: P2 -effort: "3h" -dependencies: [2] ---- - -# Phase 04: EventBridge cron wiring - -## Overview -Replace the planned Cloud Scheduler design with EventBridge Scheduler. Preserve the existing `/cron/{name}` HTTP route shape inside the Lambda by invoking the Function URL via Scheduler's HTTPS target. Auth via `X-Cron-Token` header sourced from Parameter Store. - -## Requirements -- **Functional:** Two scheduled jobs fire on cron expressions matching the GCP plan (`0 17 * * *` for daily push, `0 1 * * *` for cleanup or whatever Phase 09 of GCP plan defined). Both routes execute against the live module dispatcher and complete within Lambda timeout. -- **Non-functional:** Token rotates without code changes (Parameter Store update). Failure retried 2× with exponential backoff. Dead letters logged. - -## Architecture -**Decision:** HTTPS target (Function URL) over direct Lambda invoke. **Why:** -- Preserves the existing `/cron/{name}` route + dispatcher code from Phase 03 of GCP plan -- Local dev still works: `curl localhost:8080/cron/dailypush -H "X-Cron-Token: ..."` -- Single ingress path for observability (one URL, one log group) -- Direct invoke would require a separate Lambda entrypoint or routing on event shape — more code, less testable - -**Trade-off accepted:** Slightly less AWS-idiomatic; HTTPS adds ~10ms latency vs direct invoke; not material here. - -``` -EventBridge Scheduler ─cron─► HTTPS POST /cron/{name} - + Header: X-Cron-Token: - + AWS Sigv4 NOT used (Function URL AuthType: NONE) - │ - └─► Lambda → router → dispatcher → cron handler -``` - -**Auth model:** Function URL `AuthType: NONE` (already set in Phase 02 for Telegram). Cron auth = shared-secret header verified server-side. The token lives in Parameter Store (`/miti99bot/prod/cron-token`) and is fetched by Scheduler at invoke time via `SECRETSMANAGER_SECRET` reference (Scheduler supports referencing Parameter Store via `secret reference` in target input transformer, OR plain text in target — for KISS, store the token in Scheduler's invocation HTTP target headers as a templated literal, but referenced from Parameter Store via SAM resource attribute). - -**Simpler concrete approach:** SAM template reads the Parameter Store value at deploy time using `{{resolve:ssm-secure:...}}` in the schedule target's HTTP header config. Token rotation = update parameter, redeploy. - -## Related Code Files -- Create: SAM resources `Resources.DailyPushSchedule` (AWS::Scheduler::Schedule) -- Create: SAM resources `Resources.CleanupSchedule` (or whichever second cron) -- Create: SAM resource `Resources.SchedulerExecutionRole` with `events:InvokeApiDestination` / equivalent for HTTPS targets (or use built-in `aws.UniversalTarget` for `https`) -- Modify: `internal/server/router.go` — confirm `/cron/{name}` validates `X-Cron-Token` against env-loaded value (currently has `cronAuthHeader = "X-Cron-Token"`, good) -- Modify: `cmd/server/main.go` — load `CRON_TOKEN` env from Parameter Store reference, pass to `Config.CronToken` -- Reference: existing module cron registrations in each module's `Cron()` method - -## Implementation Steps -1. Define SAM `AWS::Scheduler::Schedule` for each cron job: - ```yaml - DailyPushSchedule: - Type: AWS::Scheduler::Schedule - Properties: - ScheduleExpression: "cron(0 17 * * ? *)" # 17:00 UTC = 00:00 Saigon - FlexibleTimeWindow: { Mode: 'OFF' } - Target: - Arn: arn:aws:scheduler:::http-invoke - RoleArn: !GetAtt SchedulerRole.Arn - Input: '{"name":"dailypush"}' - HttpParameters: - HeaderParameters: { X-Cron-Token: '{{resolve:ssm-secure:/miti99bot/prod/cron-token:1}}' } - RetryPolicy: { MaximumRetryAttempts: 2, MaximumEventAgeInSeconds: 600 } - DeadLetterConfig: { Arn: !GetAtt CronDLQ.Arn } - FlexibleTimeWindow: { Mode: OFF } - ``` - *(Pseudo — confirm exact `aws.HttpInvoke` target syntax against current AWS SAM docs at deploy time; AWS docs note the API surface is evolving.)* -2. Add `CronDLQ` (SQS queue, free tier 1M req/mo). -3. Add `SchedulerRole` IAM with `lambda:InvokeFunctionUrl` (or `events:InvokeApiDestination` if going via API destination). -4. Provision `/miti99bot/prod/cron-token` in Parameter Store with a 32-byte random value (`openssl rand -hex 32`). -5. Verify router rejects requests with wrong/missing token (test exists; confirm). -6. Deploy. From AWS console, "run now" each schedule. Confirm CloudWatch log entry shows successful 200 from Lambda. -7. Wait one full schedule window (or change to `rate(2 minutes)` temporarily) to confirm automatic firing. -8. Restore production cron expressions. Confirm next-fire timestamp. - -## Success Criteria -- [ ] Both schedules deploy via SAM -- [ ] Manual "run now" returns HTTP 200 from Function URL -- [ ] Server logs show cron handler executing the right module -- [ ] Wrong/missing token → 401, no module side effects -- [ ] DLQ receives failed invocations on simulated Lambda error -- [ ] Schedule fires automatically once on production cron expression - -## Risk Assessment -- **AWS Scheduler HTTPS target maturity** — relatively new feature; if SAM transform doesn't support `aws.HttpInvoke` cleanly, fall back to: Scheduler → SNS → Lambda subscription → existing handler (one extra hop, identical effect). Document fallback in this file. -- **Token in template via `resolve`** — at deploy time the value is fetched and embedded into the schedule target's static config; rotation requires redeploy. If frequent rotation needed, switch to a Lambda authorizer pattern (out of scope for v1). -- **Cron drift / TZ confusion** — EventBridge `cron()` uses UTC by default (matches Cloud Scheduler behavior in GCP plan). Use `?` for day-of-week-or-month constraint per AWS syntax. -- **Cold-start during cron** — first invocation after idle = 1–3s; cron handler logic must complete within Lambda timeout (15s). If a cron handler calls Gemini and exceeds 15s, raise function timeout to 30s (still free). - -## Open questions -1. Direct Lambda invoke vs HTTPS target — locked to HTTPS for the reasons above; revisit only if HTTPS proves flaky. -2. Single schedule with dynamic `name` vs one schedule per cron — one per cron is clearer in console; switch to dynamic only if cron count grows past ~5. -3. Cleanup cron (`0 1 * * *`) — confirm what it does in original miti99bot. Likely TTL-style sweep; review and decide if DynamoDB TTL attribute can replace it (eliminates the cron entirely). diff --git a/plans/260510-0114-aws-port/phase-05-gha-deploy.md b/plans/260510-0114-aws-port/phase-05-gha-deploy.md deleted file mode 100644 index ceaed04..0000000 --- a/plans/260510-0114-aws-port/phase-05-gha-deploy.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -phase: 5 -title: "GitHub Actions deploy (OIDC + SAM)" -status: pending -priority: P2 -effort: "3h" -dependencies: [2, 3, 4] ---- - -# Phase 05: GitHub Actions deploy (OIDC + SAM) - -## Overview -Push to `main` → CI builds the ARM64 Go binary, packages into ZIP, runs `sam deploy` against the existing stack via OIDC-assumed role. No long-lived AWS keys. Idempotent (zero-diff deploys are no-ops). - -## Requirements -- **Functional:** PR validates (`go vet`, `go test`, `sam validate`). Push to `main` deploys. Manual workflow_dispatch redeploy supported. -- **Non-functional:** Deploy < 4 min. Concurrency: only one deploy at a time per ref. Stack name parameterized by env (default `prod`). - -## Architecture -``` -GitHub push to main - └─► .github/workflows/deploy.yml - 1. checkout - 2. setup-go (1.25) - 3. setup-sam - 4. configure-aws-credentials (OIDC) ─► assume github-deploy-miti99bot - 5. make build-lambda - 6. sam build --use-container=false - 7. sam deploy --no-confirm-changeset --no-fail-on-empty-changeset - 8. post-deploy smoke (curl /) -``` - -## Related Code Files -- Create: `.github/workflows/deploy.yml` -- Create: `.github/workflows/ci.yml` — maybe split out validate-only path; or add `if:` guard in `deploy.yml` -- Modify: existing `.github/workflows/ci.yml` — add `sam validate` step -- Modify: `Makefile` — `deploy` target runs `sam build && sam deploy --no-confirm-changeset` - -## Implementation Steps -1. Confirm Phase 01's IAM role trust policy includes `repo:tiennm99/miti99bot:ref:refs/heads/main` and the matching repo subject for any manual deploy path. -2. Write `deploy.yml`: - ```yaml - name: Deploy to AWS - on: - push: { branches: [main] } - workflow_dispatch: - permissions: - id-token: write - contents: read - concurrency: { group: deploy-prod, cancel-in-progress: false } - jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: { go-version: '1.25' } - - uses: aws-actions/setup-sam@v2 - with: { use-installer: true } - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::225603493174:role/github-deploy-miti99bot - aws-region: ap-southeast-1 - - run: make build-lambda - - run: sam build - - run: sam deploy --no-confirm-changeset --no-fail-on-empty-changeset --stack-name miti99bot-aws-port - - name: Smoke test - run: | - URL=$(aws cloudformation describe-stacks --stack-name miti99bot-aws-port --query "Stacks[0].Outputs[?OutputKey=='FunctionUrl'].OutputValue" --output text) - curl -fsSL "$URL/" | jq . - ``` -3. Keep the AWS account ID in the committed role ARN for this repo. If the deploy account changes later, update both the workflow ARN and the IAM trust policy together. -4. PR validation workflow (`ci.yml`): runs `go vet`, `go test`, `sam validate` (no AWS creds needed for validate). -5. Test the full path: open a PR with a trivial change → CI green; merge → deploy fires; smoke step prints health JSON. -6. Add a `rollback.yml` workflow_dispatch path: re-run with a chosen commit SHA. CloudFormation handles the rollback inherently. - -## Success Criteria -- [ ] PR triggers `ci.yml` only (no AWS deploy) -- [ ] Merge to `main` triggers `deploy.yml` -- [ ] Deploy succeeds without manual intervention -- [ ] Post-deploy smoke step returns 200 from Function URL -- [ ] Concurrency lock prevents overlapping deploys -- [ ] Re-running a no-op deploy reports "no changes" and exits 0 -- [ ] No AWS access keys in repo, GitHub Actions secrets, or anywhere - -## Risk Assessment -- **OIDC trust misconfiguration** locks deploy out — Mitigation: keep bootstrap admin user (Phase 01) as glass-break recovery; rotate after 90 days. -- **`sam build` with native deps fails** — pure Go has no native deps; should be fine. -- **CloudFormation drift** between manual changes and CI — Mitigation: forbid console edits; daily `sam deploy --no-execute-changeset` to detect. -- **Build cache cold each run** (~90s for Go deps) — Mitigation: `actions/setup-go` cache enabled by default. -- **Workflow secret leak via debug logs** — Mitigation: never echo `secrets.*`, GH masks them automatically. - -## Open questions -1. Separate dev/staging stacks for PR previews? Out of scope for v1; `prod` only. -2. Slack/Telegram notification on deploy success/failure? Defer; CloudWatch logs + `gh run list` suffice initially. -3. Pin SAM version in `setup-sam`? Yes — pin to a specific version to avoid surprise breakage. diff --git a/plans/260510-0114-aws-port/phase-06-observability.md b/plans/260510-0114-aws-port/phase-06-observability.md deleted file mode 100644 index 85d1d67..0000000 --- a/plans/260510-0114-aws-port/phase-06-observability.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -phase: 6 -title: "Observability + budget alert" -status: pending -priority: P2 -effort: "2h" -dependencies: [5] ---- - -# Phase 06: Observability + budget alert - -## Overview -Wire CloudWatch Logs retention, metric filters for key counters, AWS Budgets $1/mo alert, and capture cold-start P95 baseline for the abort criterion in `plan.md`. - -## Requirements -- **Functional:** Log retention 7 days. Budget alert fires at $1.00 actual. Cold-start P95 measurable from logs. -- **Non-functional:** All observability stays in free tier (5 GB log ingest/mo, 10 custom metrics free, 1k AWS Budgets API ops free). - -## Architecture -- **Logs:** Lambda's auto-created log group `/aws/lambda/miti99bot-aws-port-BotFunction-*`. SAM sets retention. -- **Metrics:** Existing `internal/metrics` package emits counters. Lambda env can ship them via stdout — CloudWatch Logs ingests, log-based metric filters extract `request.duration`, `module.dispatched`, `cron.fired`. Free metric filter quota: unlimited filters, paid for resulting metrics past 10/mo. -- **Budget:** `AWS::Budgets::Budget` in SAM template, threshold $1, email alert. -- **Cold start:** parse `REPORT` log lines → `Init Duration` field → P50/P95/P99. - -## Related Code Files -- Modify: `template.yaml` — add `LogRetentionInDays: 7` on `BotFunction` (SAM `LoggingConfig`); add `AWS::Budgets::Budget` resource; add `AWS::Logs::MetricFilter` for key metrics -- Create: `aws/dashboards/cold-start-coldwatch.json` (optional, manual import) -- Reference: `internal/log/*.go` (slog JSON emitter — already exists) -- Reference: `internal/metrics/*.go` (counters + 60s flush — already exists) - -## Implementation Steps -1. Add to `template.yaml` under `BotFunction.Properties`: - ```yaml - LoggingConfig: - LogFormat: JSON - ApplicationLogLevel: INFO - SystemLogLevel: WARN - LogGroup: !Ref BotFunctionLogGroup - ``` -2. Add explicit log group to control retention: - ```yaml - BotFunctionLogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: /aws/lambda/miti99bot-aws-port-bot - RetentionInDays: 7 - ``` -3. Add metric filter for cold start: - ```yaml - ColdStartFilter: - Type: AWS::Logs::MetricFilter - Properties: - LogGroupName: !Ref BotFunctionLogGroup - FilterPattern: '[report="REPORT", ..., init_label="Init", init_dur_label="Duration:", init_dur, ...]' - MetricTransformations: - - MetricName: ColdStartInitDuration - MetricNamespace: miti99bot - MetricValue: $init_dur - ``` -4. Add `AWS::Budgets::Budget` (sends email at 80% and 100% of $1): - ```yaml - MonthlyBudget: - Type: AWS::Budgets::Budget - Properties: - Budget: - BudgetName: miti99bot-monthly - BudgetLimit: { Amount: '1.00', Unit: 'USD' } - TimeUnit: MONTHLY - BudgetType: COST - NotificationsWithSubscribers: - - Notification: { ComparisonOperator: GREATER_THAN, NotificationType: ACTUAL, Threshold: 80, ThresholdType: PERCENTAGE } - Subscribers: [{ Address: , SubscriptionType: EMAIL }] - ``` -5. Deploy. Trigger cold start (`aws lambda update-function-configuration --function-name … --environment 'Variables={…,FORCE_RESTART=$(date +%s)}'`). -6. Capture 50 cold starts manually or via a one-shot load script (`hey -n 50 -c 1 -i 30s /`) — concurrency=1 with delay forces fresh inits. Compute P95 from CloudWatch Insights: - ``` - filter @type = "REPORT" - | stats avg(@initDuration), pct(@initDuration, 95) - ``` -7. Record P95 in `plan.md`'s "Free-tier budget at peak" or as an addendum here. -8. Confirm budget shows up in AWS Console > Budgets and has the email subscriber. - -## Success Criteria -- [ ] Log group `RetentionInDays: 7` set -- [ ] Cold-start P95 captured and < 1.5s (per abort criterion) -- [ ] Budget alert visible in console, email subscriber confirmed (test mail received) -- [ ] No log group accumulates >500 MiB after 7 days of normal traffic -- [ ] CloudWatch Insights query for cold start works without manual setup - -## Risk Assessment -- **Email subscriber not confirmed** → first alert silently dropped — Mitigation: send a test from console before relying on it. -- **Log retention deleted by SAM redeploy** if log group not explicit → Mitigation: declare log group explicitly (step 2). -- **Cold start drift** as deps grow — Mitigation: re-measure quarterly; add a CI step that fails if `bootstrap` binary > 30 MiB. -- **Budget delays** (AWS Budgets evaluates ~3× day, not real-time) → Mitigation: also enable Cost Anomaly Detection (free) for spike alerts. - -## Open questions -1. Email vs SNS topic for budget alerts? Email is simpler; SNS lets fan-out to webhook later. Start with email, migrate if needed. -2. Custom CloudWatch dashboard? Skip for v1 — Insights queries are enough for solo-dev. -3. Trace via AWS X-Ray? `Tracing: Active` already set in Phase 02 globals — free at this volume; review traces post-Phase 07. diff --git a/plans/260510-0114-aws-port/phase-07-cutover.md b/plans/260510-0114-aws-port/phase-07-cutover.md deleted file mode 100644 index 2ba50e6..0000000 --- a/plans/260510-0114-aws-port/phase-07-cutover.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -phase: 7 -title: "Cutover + README + retire GCP paths" -status: pending -priority: P2 -effort: "3h" -dependencies: [2, 3, 4, 5, 6] ---- - -# Phase 07: Cutover + README + retire GCP paths - -## Overview -Flip the production Telegram webhook to the AWS Function URL only after the Cloudflare→AWS migration plan has produced a green parity report and a rehearsed final-delta procedure. Keep GCP code paths in tree (Firestore impl, Cloud Run Dockerfile) but unwired by default. - -## Requirements -- **Functional:** Real production bot serves users from Lambda with durable Cloudflare data already migrated or intentionally archived. No regressions vs prior baseline (whatever ran before — JS Worker or partial GCP). -- **Non-functional:** Accept a brief operator-controlled freeze window for the final delta import; no silent data loss. 7-day soak with logs reviewed daily. Fallback path documented. - -## Architecture -- **Migration gate:** `plans/260515-2250-cf-data-to-aws-migration/` must finish first; Phase 04 parity report there is the go/no-go input for this phase. -- **Freeze-window cutover:** pause Cloudflare cron/webhook writes, run final delta export/import + verify, then call `setWebhook` to point Telegram at the AWS Function URL with the production webhook secret. -- **Rollback path:** if the final delta verify fails or AWS smoke fails before the first AWS-served write, restore the prior webhook target. After AWS starts accepting new writes, this cutover is forward-fix only unless a reverse-sync path exists. -- **Code:** Firestore impl stays compilable, gated by `KV_PROVIDER=firestore`. Default `KV_PROVIDER=dynamodb` in Lambda env. Cloud Run Dockerfile retained for offline / non-AWS users. - -## Related Code Files -- Modify: `README.md` — full rewrite of "Run locally", "Build", new "Deploy to AWS" section, status table updated, link to AWS plan, archive link to GCP plan -- Modify: `cmd/server/main.go` — default `KV_PROVIDER` selection logic: `dynamodb` if `AWS_LAMBDA_FUNCTION_NAME` set, else `memory` -- Create: `docs/deploy-aws.md` — single source of truth for AWS deploy ops (parameter store names, IAM role ARN, smoke commands) -- Modify: `docs/cf-to-aws-migration-runbook.md` — freeze-window delta import + rollback sequence -- Modify: `plans/260508-2222-go-port-cloud-run/plan.md` — top-of-file note: "Deploy phases 01, 09–12 superseded by `plans/260510-0114-aws-port/`. Module work (phases 03–07) reused unchanged." -- Optional remove: `Dockerfile` retained for now; revisit in 30 days -- Optional remove: GCP-specific docs in `docs/` if any (none observed) - -## Implementation Steps -1. Pre-flight checklist (run inside this phase): - - [ ] Phase 02 smoke green (manual curl) - - [ ] Phase 03 wordle daily state survives deploy + cold start - - [ ] Phase 04 cron fired at least one real trigger - - [ ] Phase 05 push-to-main auto-deploys - - [ ] Phase 06 budget alert email confirmed - - [ ] Cold-start P95 < 1.5s confirmed - - [ ] `plans/260515-2250-cf-data-to-aws-migration/phase-04-parity-verification-and-rehearsal.md` passed with a saved green report - - [ ] Final delta import commands rehearsed during the freeze window -2. Pause Cloudflare writes and run the final delta import + verify. -3. Run `setWebhook` against production bot: - ```sh - curl -X POST "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/setWebhook" \ - -d "url=$AWS_FUNCTION_URL/webhook" \ - -d "secret_token=$TELEGRAM_WEBHOOK_SECRET" \ - -d "drop_pending_updates=false" \ - -d "allowed_updates=[\"message\",\"callback_query\"]" - ``` -4. Verify with `getWebhookInfo`: - ```sh - curl "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/getWebhookInfo" | jq . - ``` - Confirm `url`, `pending_update_count` near 0, `last_error_date` empty. -5. Send a test command (`/start`, `/wordle`, `/twentyq` to exercise Gemini path). Confirm responses match prior behavior and expected migrated state is visible. -6. Verify at least one migrated trading account, one existing lolschedule subscriber path, and `/mstats` if `last_ping` was migrated. -7. Soak for 7 days: each morning, check CloudWatch Logs for ERROR / WARN, DynamoDB throttle metrics (should be zero), budget current spend (should be $0), Gemini RPD usage (should be far under cap). -8. After 7-day soak, update README: - - Status table: AWS port phases marked "done" - - Replace "Run locally" with two paths: in-memory (no AWS) and DynamoDB Local (with AWS deps) - - "Deploy" section: link `docs/deploy-aws.md`, drop Cloud Run instructions - - Status badge / link to `plans/260510-0114-aws-port/` -9. Add a top-of-file note in `plans/260508-2222-go-port-cloud-run/plan.md` redirecting deploy questions to the AWS plan. -10. Tag a release: `git tag v1.0.0-aws -m "AWS deploy default"` and push. - -## Success Criteria -- [ ] Telegram webhook `getWebhookInfo` shows AWS Function URL -- [ ] Production bot answers `/start` from real users with normal latency and expected migrated data -- [ ] Final Cloudflare→AWS migration report is green before any CF teardown -- [ ] 7-day soak: zero unrecovered errors, zero throttles, zero unexpected spend -- [ ] README accurately reflects AWS as the default deploy -- [ ] `docs/deploy-aws.md` is sufficient for a fresh dev to redeploy from scratch -- [ ] GCP plan file annotated; old phase files preserved for history -- [ ] Release tag pushed - -## Risk Assessment -- **Final import misses writes** if Cloudflare stays writable during cutover — Mitigation: use the migration plan's freeze-window delta import before `setWebhook`, then verify parity again. -- **Latency regression vs JS Worker** — Cloud Run / JS Worker had different cold-start profiles; if users complain, document and consider ARM→x86 swap or provisioned concurrency (kills free tier). -- **Hidden Firestore dependency** still wired in some module — Mitigation: grep for `firestore.NewClient` and confirm all paths are gated by `KV_PROVIDER=firestore` env. Add a CI test that builds with `KV_PROVIDER=dynamodb` and asserts Firestore client is not initialized. -- **GCP project quietly billing** because resources weren't deleted — Mitigation: explicit step in this phase: `gcloud projects delete ` OR `gcloud run services delete` for any deployed services. Check Cloud Console for any orphaned resources. -- **Lambda Web Adapter unsupported on a future runtime** — Mitigation: pin LWA layer version, monitor AWS Labs repo. - -## Open questions -1. Delete the old GCP project entirely or leave it dormant? Dormant is safe (no GCP free-tier abandonment penalty); delete after 30 days if no regret. -2. Keep Dockerfile in repo? Yes — useful for non-Lambda local runs and as reference for any future Cloud Run revival. -3. Keep Firestore impl forever or drop after 90 days? Drop only if the parity test proves redundant; the impl itself is small and tested. -4. Announce the change anywhere (README badge, release notes) — depends on whether this is a public bot. User decides. diff --git a/plans/260510-0114-aws-port/plan.md b/plans/260510-0114-aws-port/plan.md deleted file mode 100644 index 042a67d..0000000 --- a/plans/260510-0114-aws-port/plan.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: "Migrate miti99bot from GCP to AWS (Lambda + DynamoDB + EventBridge, free tier)" -description: "Re-target the deploy/runtime layer from Cloud Run + Firestore + Cloud Scheduler to Lambda (Go ZIP + LWA + Function URL) + DynamoDB on-demand + EventBridge Scheduler, region ap-southeast-1, IaC via SAM, CI via GH Actions OIDC. Module code unchanged." -status: in-progress -priority: P2 -effort: 3-4d -branch: main -tags: [aws, lambda, dynamodb, eventbridge, sam, port, telegram-bot, free-tier] -created: 2026-05-10 -blockedBy: [260515-2250-cf-data-to-aws-migration] -blocks: [] -supersedes-deploy-of: [260508-2222-go-port-cloud-run] ---- - -# Plan: AWS port (Lambda + DynamoDB + EventBridge, free tier) - -Re-target only the deploy/runtime layer. Module work (Phases 03–07 of GCP plan) is **done and reused unchanged**. The KVStore interface (`internal/storage/`) absorbs the swap; `http.Handler` code (`internal/server/`) is preserved via Lambda Web Adapter. - -## Context -- **Why switch:** Strict $0 free-tier goal — DynamoDB 25 GiB / 200M req-mo, EventBridge unlimited rules, 100 GB egress all-region beat Firestore 1 GiB, Cloud Scheduler 3-job cap, GCP NA-only egress. See `plans/reports/research-260510-0021-aws-vs-gcp-greenfield-rethink.md`. -- **Reused as-is:** module framework, registry, dispatcher, Telegram lib, AI clients, all 11 modules, Firestore impl (kept as sibling for parity tests). -- **Replaced:** Cloud Run → Lambda; Firestore → DynamoDB (sibling provider, default switchable via env); Cloud Scheduler → EventBridge Scheduler; Secret Manager → Parameter Store; Artifact Registry → none (ZIP); Cloud Logging → CloudWatch Logs; CF Worker / GCP CI → GH Actions + SAM. - -## Locked decisions -- **Compute:** Lambda Go on `provided.al2023`, **ARM64**, ZIP package, binary `bootstrap`, build with `-tags lambda.norpc -ldflags="-s -w"`. -- **HTTP:** Lambda Function URL (`AuthType: NONE`) + AWS Lambda Web Adapter layer → existing `http.Handler` runs unchanged. -- **KV:** DynamoDB single-table `miti99bot`, composite key `(pk, sk)` where `pk = moduleName` and `sk = caller key`, attr `value` (Binary). On-demand billing. -- **Cron:** EventBridge Scheduler → HTTPS target = Function URL `/cron/{name}` with `X-Cron-Token` header (token in Parameter Store). Preserves existing route shape; alternative (direct Lambda invoke) deferred. -- **Secrets:** SSM Parameter Store SecureString. Names: `/miti99bot/{env}/telegram-token`, `…/webhook-secret`, `…/gemini-api-key`, `…/cron-token`. Fetched at cold start. -- **Region:** `ap-southeast-1` (Singapore). -- **IaC:** AWS SAM (`template.yaml`). -- **CI:** GitHub Actions, OIDC role, `aws-actions/configure-aws-credentials@v4` + `aws-actions/setup-sam@v2`. -- **Logs:** CloudWatch Logs, 7-day retention. -- **Cost guard:** AWS Budgets $1/mo alert. - -## Phases - -| # | Phase | Status | Effort | Key deliverable | -|---|-------|--------|--------|-----------------| -| 01 | [AWS bootstrap + IAM OIDC + SAM skeleton](phase-01-aws-bootstrap.md) | pending (manual) | 3h | AWS account, OIDC trust, empty SAM stack deployable | -| 02 | [Lambda runtime (Go ZIP + LWA + Function URL)](phase-02-lambda-runtime.md) | code-done; awaits first deploy | 4h | `/` and `/webhook` served from Lambda; secret-token check passes | -| 03 | [DynamoDB KV provider](phase-03-dynamodb-kv.md) | code-done; integration tests skip without DDB Local | 4h | `dynamodb_kv.go` + `dynamodb_provider.go` sibling to Firestore impl, parity tests pass | -| 04 | [EventBridge cron wiring](phase-04-eventbridge-cron.md) | pending (blocked on cron handlers, see 260510-0234-pre-deploy-wrapup) | 3h | Scheduler → `/cron/{name}` with token, two crons firing on schedule | -| 05 | [GitHub Actions deploy (OIDC + SAM)](phase-05-gha-deploy.md) | done | 3h | `deploy.yml` runs on push to `main`, builds + sam deploys idempotently | -| 06 | [Observability + budget alert](phase-06-observability.md) | partial (budget shipped; metric filter in 260510-0234) | 2h | Logs retention set, $1 budget alert, cold-start P95 captured | -| 07 | [Cutover + README + retire GCP paths](phase-07-cutover.md) | pending (deploy + migration gated) | 3h | Webhook flipped to Function URL after green CF→AWS migration report; README rewritten; GCP code paths kept but unwired by default | - -## Dependency graph -``` -01 ──► 02 ──► 03 ──► 04 ──► 05 ──► 06 ──► 07 - └──► 04 ─────►┘ -``` - -## Free-tier budget at peak -| Resource | Cap | Expected | Headroom | -|---|---|---|---| -| Lambda req | 1M/mo | ~30k/mo | 97% | -| Lambda compute | 400k GB-s | <5k | 99% | -| DynamoDB req | 200M/mo | <100k | 99.9% | -| DynamoDB storage | 25 GiB | <50 MiB | 99.8% | -| EventBridge invocations | 14M/mo | ~60 (2 crons × ~30 days) | 99.9% | -| Parameter Store accesses | unlimited (Standard) | <100/cold-start × ~30 starts | n/a | -| Egress | 100 GB/mo | <50 MiB | 99.95% | -| CloudWatch Logs ingest | 5 GB/mo | <500 MiB | 90% | - -## Abort criteria -- **Cold-start P95 > 1.5s** sustained: investigate ARM64→x86_64 swap or pre-warm with provisioned concurrency (kills free tier; only if user-facing latency unacceptable). -- **DynamoDB throttle** under normal load: switch to provisioned mode (still free under 25 RCU/WCU). -- **Function URL auth-bypass risk** discovered: switch to API Gateway HTTP API (12-month free, then $1/M). - -## Rollback -Until Phase 07 webhook flip, the GCP runtime path remains intact. Per-phase rollback documented in each phase. Phase 07 now depends on `plans/260515-2250-cf-data-to-aws-migration/` producing a green parity report before the webhook moves or any Cloudflare data source is deleted. - -## Open questions -1. Direct Lambda invoke for cron vs HTTP loopback via Function URL — final call deferred to Phase 04 implementation. -2. Whether to delete Firestore impl after parity confirmed, or keep as offline test backend permanently. -3. Single SAM stack vs split (data + compute) — start single, split if iteration speed suffers. diff --git a/plans/260510-0234-pre-deploy-wrapup/phase-01-cosmetics.md b/plans/260510-0234-pre-deploy-wrapup/phase-01-cosmetics.md deleted file mode 100644 index 99ecc6f..0000000 --- a/plans/260510-0234-pre-deploy-wrapup/phase-01-cosmetics.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -phase: 1 -title: "Cosmetics: README + plan status sync" -status: pending -priority: P3 -effort: "30m" -dependencies: [] ---- - -# Phase 01: Cosmetics — README + plan status sync - -## Overview -README still says "Cloud Run + Firestore"; AWS-port phases 02/03/05 say "pending" though their code already shipped. Sync both before any new clone or visitor reads stale docs. - -## Requirements -- **Functional:** README describes AWS as default deploy, with link to `docs/deploy-aws.md`. AWS-port plan's phase-status table reflects shipped code. GCP plan's tagline note about supersession remains. -- **Non-functional:** No code changes. Markdown-only. - -## Architecture -N/A — documentation update. - -## Related Code Files -- Modify: `README.md` -- Modify: `plans/260510-0114-aws-port/plan.md` (status column) -- Reference (no edit): `plans/260508-2222-go-port-cloud-run/plan.md` (already annotated) - -## Implementation Steps -1. Rewrite `README.md`: - - Tagline: "Plug-n-play Telegram bot framework in Go. Default deploy: AWS Lambda + DynamoDB + EventBridge (free tier). Cloud Run path retained as alt." - - Status table: collapse to one row per work-phase; mark current state honestly. - - "Run locally" section: keep in-memory KV path; add note about `make dynamodb-local` for DynamoDB integration testing; keep `make firestore-emulator` line. - - "Deploy" section: link `docs/deploy-aws.md` as canonical; mention Dockerfile for non-Lambda hosts. - - "Test" section: add `make test-dynamodb` line. -2. Update `plans/260510-0114-aws-port/plan.md` phases table: - - Phase 01 → still pending (manual user steps) - - Phase 02 → "code-done; awaiting first deploy" - - Phase 03 → "code-done; integration tests skip without DynamoDB Local" - - Phase 04 → still pending (this plan unblocks it) - - Phase 05 → "done" - - Phase 06 → "partial" — budget alert in template; metric filter deferred to this plan's Phase 02 - - Phase 07 → still pending (deploy-gated) -3. Smoke-render the README locally (`grip` or just open in editor) — confirm headings, links, and code blocks render cleanly. - -## Success Criteria -- [ ] README's intro line names AWS as default -- [ ] README's status table accurate (no "pending" rows that are actually done) -- [ ] AWS-port plan.md phase statuses reflect shipped code -- [ ] All link targets resolve (no 404 in `docs/deploy-aws.md`, `aws/README.md`, both plan files) -- [ ] No broken markdown rendering - -## Risk Assessment -- **Drift between README and plan.md** if updated separately later — Mitigation: this phase is the single place both get touched together; future drift caught in Phase 06 cutover. -- **Stale Cloud Run instructions misleading new contributors** — Mitigation: prefix the alt-path section with "Alternative: Cloud Run (deferred)" so the canonical path is unambiguous. - -## Open questions -1. Move Cloud Run instructions into `docs/deploy-gcp-cloud-run.md` instead of inlining? Cleaner README but adds a file. Default: inline a short note + link to old plan. -2. Add CI badge to README? Skip for v1 — no public bot, no marketing pressure. diff --git a/plans/260510-0234-pre-deploy-wrapup/phase-02-metric-filter.md b/plans/260510-0234-pre-deploy-wrapup/phase-02-metric-filter.md deleted file mode 100644 index d54d855..0000000 --- a/plans/260510-0234-pre-deploy-wrapup/phase-02-metric-filter.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -phase: 2 -title: "Cold-start metric filter" -status: pending -priority: P3 -effort: "30m" -dependencies: [] ---- - -# Phase 02: Cold-start metric filter - -## Overview -Add a CloudWatch Logs metric filter that extracts Lambda's `Init Duration` from the auto-emitted `REPORT` line so the AWS-port plan's "P95 < 1.5s" abort criterion is measurable from day one. - -## Requirements -- **Functional:** A custom metric `miti99bot/ColdStartInitDuration` exists; samples appear in CloudWatch Metrics within 5 minutes of cold-start. -- **Non-functional:** Stays inside CloudWatch's always-free 10 custom metrics. No additional ingest cost (filter operates on existing log stream). - -## Architecture -Lambda emits a synthetic `REPORT` line at the end of every invocation. On a cold start, that line includes `Init Duration: `. A `AWS::Logs::MetricFilter` parses the line and emits a custom metric value into `miti99bot/ColdStartInitDuration` namespace. - -``` -Lambda invocation → CloudWatch log stream → - filter pattern matches "REPORT ... Init Duration: " → - publish metric (Namespace: miti99bot, Name: ColdStartInitDuration, Value: ) -``` - -## Related Code Files -- Modify: `template.yaml` — add `ColdStartMetricFilter` resource - -## Implementation Steps -1. Append to `template.yaml` after `BotFunctionLogGroup`: - ```yaml - ColdStartMetricFilter: - Type: AWS::Logs::MetricFilter - Properties: - LogGroupName: !Ref BotFunctionLogGroup - FilterPattern: '[report="REPORT", reqid_label="RequestId:", reqid, dur_label="Duration:", dur, dur_unit="ms", bill_label="Billed", bill_dur_label, bill_dur, bill_unit, mem_label, mem_size_label, mem_size, mem_unit, max_label="Max", max_used_label="Memory", max_used_label2="Used:", max_used, max_used_unit, init_label="Init", init_dur_label="Duration:", init_dur, init_unit="ms"]' - MetricTransformations: - - MetricName: ColdStartInitDuration - MetricNamespace: miti99bot - MetricValue: $init_dur - Unit: Milliseconds - ``` -2. Validate locally: `make sam-validate` (offline lint). Should pass. -3. After AWS-port Phase 01 manual deploy completes, verify: - - `aws logs describe-metric-filters --log-group-name /aws/lambda/miti99bot-aws-port-bot` - - Trigger a cold start (`aws lambda update-function-configuration --environment ... ` flip). - - `aws cloudwatch get-metric-statistics --namespace miti99bot --metric-name ColdStartInitDuration --statistics Average,Maximum --start-time ... --end-time ... --period 300` - -## Success Criteria -- [ ] `template.yaml` has `ColdStartMetricFilter` resource -- [ ] `sam validate` passes -- [ ] Post-deploy: metric filter visible in AWS console, samples flow within 5 min of a cold start - -## Risk Assessment -- **Filter pattern brittleness** — Lambda's REPORT format is stable but unofficial. Mitigation: pattern uses positional + label parsing, tolerant of whitespace; if AWS changes format, filter just stops matching (no error, just zero data). -- **Warm-only invocations don't include `Init Duration`** — pattern won't match those, which is correct; we only want cold-start samples. - -## Open questions -1. Capture `Duration:` (warm + cold) too as a separate metric? YAGNI — request latency is already in CloudWatch's built-in `Duration` metric for the function. -2. Per-region dashboard? Skip — solo dev uses Insights queries. diff --git a/plans/260510-0234-pre-deploy-wrapup/phase-03-lolschedule-cron.md b/plans/260510-0234-pre-deploy-wrapup/phase-03-lolschedule-cron.md deleted file mode 100644 index d9a7526..0000000 --- a/plans/260510-0234-pre-deploy-wrapup/phase-03-lolschedule-cron.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -phase: 3 -title: "lolschedule daily-push cron handler" -status: pending -priority: P2 -effort: "3h" -dependencies: [] ---- - -# Phase 03: lolschedule daily-push cron handler - -## Overview -Implement the deferred `lolschedule_daily_push` cron — fans out today's match schedule to subscribers at 08:00 ICT. Requires extending `modules.Deps` to expose `*bot.Bot` (current blocker noted in `internal/modules/lolschedule/lolschedule.go:8-12`). - -## Requirements -- **Functional:** - - `lolschedule.Module.Crons()` returns one entry: name `daily_push`, schedule `0 1 * * *` (UTC = 08:00 ICT). - - Handler reads subscribers, fetches today's matches via existing `api_client.go`, sends formatted message to each chat via `*bot.Bot`. - - Failed sends per-chat are logged, do not abort the batch (one bad chat doesn't take down the whole push). -- **Non-functional:** - - Handler completes within Lambda's 30s default timeout for typical subscriber counts (<100). Past that, paginate or move to async. - - Rate-limit-aware: respect Telegram's 30 messages/sec global cap. For low subscriber counts, no batching needed. - -## Architecture -**Deps extension (the real work):** -```go -// internal/modules/module.go -type Deps struct { - KV storage.KVStore - Embedder ai.Embedder - Chatter ai.Chatter - Env map[string]string - Bot *bot.Bot // NEW — nil-safe; modules check before use -} -``` - -`*bot.Bot` is already constructed in `cmd/server/main.go` before `modules.Build`. Wire it into `BuildOptions` (typed, like `Embedder`/`Chatter`) and have `modules.Build` thread it into each module's `Deps`. Modules that don't need it ignore it — same pattern as Gemini. - -**Cron handler:** -```go -// internal/modules/lolschedule/cron.go (new file) -func (m *Module) dailyPush(ctx context.Context) error { - if m.deps.Bot == nil { - return errors.New("lolschedule: daily push requires bot reference") - } - subs, err := listSubscribers(ctx, m.kv) - if err != nil { return err } - matches, err := m.api.TodayMatches(ctx) // existing api_client method - if err != nil { return err } - msg := formatMatches(matches) // existing format.go helper - - var sent, failed int - for _, chatID := range subs { - if _, err := m.deps.Bot.SendMessage(ctx, &bot.SendMessageParams{ChatID: chatID, Text: msg}); err != nil { - log.Warn("lolschedule push failed", "chat", chatID, "err", err) - failed++ - continue - } - sent++ - } - log.Info("lolschedule daily push complete", "sent", sent, "failed", failed) - return nil -} -``` - -## Related Code Files -- Modify: `internal/modules/module.go` — add `Bot *bot.Bot` to `Deps` and `BuildOptions` -- Modify: `internal/modules/registry.go` (or wherever `modules.Build` constructs Deps) — thread `Bot` through -- Modify: `cmd/server/main.go` — pass `b` (already constructed) into `modules.BuildOptions{Bot: b}` -- Create: `internal/modules/lolschedule/cron.go` — `dailyPush` handler + helper -- Modify: `internal/modules/lolschedule/lolschedule.go` — implement `Crons()` returning the registration; remove the deferred-cron comment block at line 8-12 -- Create: `internal/modules/lolschedule/cron_test.go` — table tests for handler with mock bot + KV - -## Implementation Steps -1. **Deps extension:** - - Add `Bot *bot.Bot` field to `Deps` and `BuildOptions` (in `internal/modules/module.go`) - - Update `modules.Build` to copy `BuildOptions.Bot` into each constructed `Deps` - - Update `cmd/server/main.go` to pass `Bot: b` in the options literal - - Run `go vet ./...` + `go build ./...` — should be clean (additive change) -2. **lolschedule cron registration:** - - Add `Crons() []modules.Cron` to `Module`, returning `{{Name: "daily_push", Schedule: "0 1 * * *", Handler: m.dailyPush}}` (verify the exact struct shape from `modules.Cron` definition) - - Remove the deferred-cron comment block in `lolschedule.go` -3. **Handler implementation:** - - Write `cron.go` per architecture above - - Reuse existing `api_client.TodayMatches` and `format.go` helpers (read these to confirm signatures; adjust if needed) -4. **Tests:** - - Mock `*bot.Bot` via interface or wrapper; assert SendMessage called once per subscriber - - Cover: happy path (3 subs, all succeed); partial failure (1 of 3 fails, batch continues); empty subscribers (no-op, no error); API failure (returns error) -5. **Wire dispatch:** confirm `internal/modules/cron_dispatcher.go` (or wherever crons are surfaced to `internal/server/router.go`) picks up the new registration without further wiring. Check by hitting `/cron/lolschedule_daily_push` locally with the right secret token; should call the handler. -6. **Local smoke:** `go test ./internal/modules/lolschedule/...` green; manual `curl` against running server with at least one subscriber. - -## Success Criteria -- [ ] `modules.Deps.Bot` exposed; nil-safe (modules without it work unchanged) -- [ ] `lolschedule.Module.Crons()` returns one entry -- [ ] `dailyPush` handler implemented per architecture -- [ ] Cron-handler unit tests pass (happy path + partial failure + empty subs) -- [ ] `go vet`, `go build`, full `go test` green -- [ ] Manual `/cron/lolschedule_daily_push` invocation works locally and triggers fan-out - -## Risk Assessment -- **Deps extension breaks every module** if not nil-safe — Mitigation: zero-value `*bot.Bot` is `nil`, all current modules ignore the field, additive change. Add a registry test that builds a module without Bot to confirm. -- **Telegram global rate limit** (30 msg/sec) on large subscriber counts — Mitigation: add a 50ms sleep between sends if subs > 30; below that, send hot. Document threshold in cron handler. -- **Handler exceeds Lambda 30s timeout** at very large sub counts — Mitigation: estimate at 100 subs × 100ms each = 10s, comfortably under. If breached, raise function timeout to 60s in `template.yaml` (still free). -- **Long-poll bot client used elsewhere** vs cron-time short-lived calls — current `bot.Bot` instance is shared; SendMessage is goroutine-safe per the lib's design. Confirm in upstream go-telegram/bot docs if uncertain. - -## Open questions -1. Do we want per-subscriber timezone awareness, or push to all at the global 08:00 ICT? Original miti99bot pushes globally — match for parity. -2. Failure handling: store failed-chat IDs for retry on next push, or just log? Default: log only; transient Telegram failures resolve naturally. -3. Should daily push respect a "no matches today" outcome with a quiet skip vs. sending an empty message? Quiet skip; matches parity with original. diff --git a/plans/260510-0234-pre-deploy-wrapup/phase-04-trading-module.md b/plans/260510-0234-pre-deploy-wrapup/phase-04-trading-module.md deleted file mode 100644 index 8238887..0000000 --- a/plans/260510-0234-pre-deploy-wrapup/phase-04-trading-module.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -phase: 4 -title: "Trading module port (VN stocks paper trading)" -status: pending -priority: P2 -effort: "6h" -dependencies: [] ---- - -# Phase 04: Trading module port (VN stocks paper trading) - -## Overview -Port the `trading` module from the original miti99bot to Go. Paper-trading on Vietnam-listed stocks: per-user portfolio + buy/sell commands + daily price refresh cron. Largest remaining cloud-agnostic chunk; carries from `plans/260508-2222-go-port-cloud-run/` Phase 08 unchanged in scope. - -## Requirements -- **Functional:** - - Commands (parity with original): `/buy `, `/sell `, `/portfolio`, `/price `, `/leaderboard` - - Daily price refresh cron at market-close (Vietnam: 15:00 ICT = UTC 08:00) — fetch latest closes for tracked tickers, store snapshot, recompute portfolio P&L - - Per-user paper-money starting balance, persistent ledger of trades - - Leaderboard: top-N users by total portfolio value -- **Non-functional:** - - Stays inside Firestore + DynamoDB free tiers (per-user state in 1-3 KV keys, leaderboard a single derived doc) - - Daily price API call counts: <50/day (well inside any reasonable free tier on the data source) - -## Architecture -**Module shape** mirrors existing modules (e.g. `wordle`): -``` -internal/modules/trading/ - trading.go Module struct + factory + Commands() + Crons() - api_client.go HTTP client to VN-stocks data source - api_client_test.go - portfolio.go core domain: Portfolio struct, buy/sell, mark-to-market - portfolio_test.go - handlers.go command handlers (/buy, /sell, /portfolio, /price, /leaderboard) - handlers_test.go - cron.go daily price refresh handler - cron_test.go - format.go message formatting helpers - format_test.go -``` - -**KV layout** (per-module partition, no cross-keys): -- `user::portfolio` → JSON Portfolio (cash, positions, trade history) -- `prices:` → JSON {price, timestamp} -- `leaderboard` → JSON sorted list (recomputed by cron) -- `tickers` → JSON array (set of tracked tickers across all users; cron iterates this) - -**Concurrency:** Buy/sell mutate the same user portfolio; reuse existing `internal/keylock` (already in repo from wordle work) keyed by `user::portfolio`. - -**Cron:** registered with name `daily_refresh`, schedule `0 8 * * *` (UTC = 15:00 ICT, market close). Iterates `tickers`, fetches each price, updates `prices:*`, recomputes leaderboard. - -## Related Code Files -- Create: all files under `internal/modules/trading/` (per architecture above) -- Modify: `cmd/server/main.go` — add `"trading": trading.New` to factories map -- Reference: `internal/modules/wordle/` as the closest existing template (commands + state + per-user mutex) -- Reference: `internal/modules/lolschedule/api_client.go` as the closest HTTP-fetching template - -## Implementation Steps -1. **Locate original miti99bot trading source** — review the JS implementation in https://github.com/tiennm99/miti99bot to nail down exact command shape, message formats, leaderboard rules, and the data source URL. -2. **Verify data source** — confirm the API used by original miti99bot is still free + accessible. If not, evaluate alternatives (TCBS public API, VPS public API, etc.). Document the choice in `api_client.go` header. -3. **Stub `api_client.go`** with the chosen endpoint + request shape; unit-test with golden HTTP fixtures (no network calls in tests). -4. **Domain (`portfolio.go`):** pure Go, no I/O — Portfolio struct, Buy/Sell methods returning new state + delta, mark-to-market against a price map. Heavily unit-tested (this is the easiest part to get wrong silently). -5. **Handlers (`handlers.go`):** parse args, load portfolio from KV under `keylock`, call domain method, persist, format reply. Mirror error paths from original (insufficient funds, unknown ticker, etc.). -6. **Cron (`cron.go`):** fetch tickers list, iterate, fetch each price, update KV, recompute leaderboard. Returns aggregate counts in log. -7. **Wire in `cmd/server/main.go`:** add factory line; bump `MODULES` env default in `template.yaml` to include `trading`. -8. **Tests:** ≥80% coverage on `portfolio.go` (domain), happy paths on handlers + cron with mock `api_client`. Match the bar set by `wordle`. -9. **Smoke locally:** `MODULES=trading go run ./cmd/server`; exercise `/buy`, `/sell`, `/portfolio`; manually trigger `/cron/trading_daily_refresh`. - -## Success Criteria -- [ ] All five commands implemented at parity with original miti99bot -- [ ] Daily refresh cron registered (Phase 05 wires it to AWS Scheduler) -- [ ] Portfolio domain has ≥80% test coverage -- [ ] No flaky tests; no network calls in unit tests (HTTP fixtures only) -- [ ] `go vet`, `go test ./internal/modules/trading/...`, full `go build` green -- [ ] Local smoke against in-memory KV exercises buy/sell/portfolio/leaderboard end-to-end -- [ ] `template.yaml` MODULES default updated to include `trading` - -## Risk Assessment -- **Original API source no longer free** — Mitigation: pre-flight check in step 2; if blocked, document fallback (web scrape with caching, or paid tier acceptance, or feature gate the module) and re-scope this phase. -- **Time zone bugs in market close** — Mitigation: store all timestamps as UTC, format for display only; unit-test the cron's UTC→ICT translation explicitly. -- **Leaderboard recomputation cost** at scale — Mitigation: full recompute is O(users); under 1k users this is single-digit ms. Past that, switch to incremental updates triggered on each trade. -- **Schema drift between paper-money currency and real ticker prices** (VND vs USD vs cents) — Mitigation: portfolio stores integer minor units (VND đồng, no fractional); document this loudly in `portfolio.go` header. -- **Concurrent buy/sell on same user** — Mitigation: `keylock` per `user::portfolio` already proven in wordle. - -## Open questions -1. Starting balance default — match original (likely 100M VND)? Confirm in step 1. -2. Allow short selling? Original likely doesn't; default to "long only, can't sell what you don't own." -3. Price ticks during market hours: refresh on `/price ` command, or only on cron? Cron-only is simpler and free-tier-friendlier; confirm against original behavior. -4. Should the cron also run on weekends (Vietnam market closed)? Skip Sat/Sun in handler; emit a no-op log line. -5. Multi-user leaderboard privacy — show user IDs or display names? Match original behavior; default to display name with fallback to "user-". diff --git a/plans/260510-0234-pre-deploy-wrapup/phase-05-eventbridge-schedules.md b/plans/260510-0234-pre-deploy-wrapup/phase-05-eventbridge-schedules.md deleted file mode 100644 index e32768b..0000000 --- a/plans/260510-0234-pre-deploy-wrapup/phase-05-eventbridge-schedules.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -phase: 5 -title: "Wire EventBridge schedules to live cron handlers" -status: deferred -priority: P3 -effort: "30m" -dependencies: [3, 4] ---- - -> **Status update 2026-05-10:** Deferred to first-deploy decision. Two issues surfaced during Phase 03/04 implementation: -> 1. **Trading module has no cron** in upstream — only one schedule needed (lolschedule_daily_push), not two -> 2. **Lambda Web Adapter only handles HTTP-shape events** — direct Scheduler→Lambda invokes bypass LWA, requiring either an event-shape detector in `main.go` or the HTTPS-target universal-invoke pattern (`arn:aws:scheduler:::http-invoke`, added in 2024) -> -> The HTTPS-target syntax in `AWS::Scheduler::Schedule` needs validation against the deploy-region SAM transform; doing this offline without `sam validate` access risks committing infra that won't deploy. Decision deferred to deploy-time. Once user runs Phase 01 of AWS-port plan and has SAM available, add a single schedule for `lolschedule_daily_push` per the prose below — pick HTTPS or direct invoke based on what `sam validate` accepts. - -# Phase 05: Wire EventBridge schedules to live cron handlers - -## Overview -With Phases 03 + 04 landed, two cron routes exist (`/cron/lolschedule_daily_push`, `/cron/trading_daily_refresh`). This phase adds concrete `AWS::Scheduler::Schedule` resources to `template.yaml` so AWS Scheduler invokes them on schedule via the existing `SchedulerExecutionRole` + `CronDLQ` already provisioned by AWS-port Phase 01. - -## Requirements -- **Functional:** Two schedules deploy via SAM. Each fires at the correct cron expression with `X-Cron-Token` header sourced from Parameter Store. Failures land in `CronDLQ`. -- **Non-functional:** Stays inside EventBridge Scheduler free tier (14M invocations/mo; we use ~60). Token rotation = update SSM param + redeploy (acceptable trade-off). - -## Architecture -``` -EventBridge Scheduler (rule: 0 1 * * ? *) ─HTTPS POST─► /cron/lolschedule_daily_push - + Headers: X-Cron-Token: {{from SSM}} - + Retry: max 2, max-age 600s - + DLQ: CronDLQ on permanent failure - -EventBridge Scheduler (rule: 0 8 * * ? *) ─HTTPS POST─► /cron/trading_daily_refresh - (same auth + retry + DLQ shape) -``` - -**HTTPS target syntax:** EventBridge Scheduler uses `arn:aws:scheduler:::http-invoke` with `HttpParameters` carrying headers. SAM's `AWS::Scheduler::Schedule` resource passes through to this; no SAM transform magic needed. - -## Related Code Files -- Modify: `template.yaml` — append `LolscheduleDailyPushSchedule` + `TradingDailyRefreshSchedule` resources -- Reference (no edit): existing `SchedulerExecutionRole` + `CronDLQ` in `template.yaml` -- Reference: `aws/README.md` (SSM parameter setup for `/miti99bot/prod/cron-shared-secret`) - -## Implementation Steps -1. Confirm AWS SDK / CloudFormation supports `aws.HttpInvoke` target via `AWS::Scheduler::Schedule` for the deploy region (`ap-southeast-1`). Check via `aws cloudformation describe-type --type RESOURCE --type-name AWS::Scheduler::Schedule` if uncertain. -2. Append to `template.yaml`: - ```yaml - LolscheduleDailyPushSchedule: - Type: AWS::Scheduler::Schedule - Properties: - Name: !Sub "${AWS::StackName}-lolschedule-daily-push" - ScheduleExpression: "cron(0 1 * * ? *)" # 01:00 UTC = 08:00 ICT - FlexibleTimeWindow: { Mode: OFF } - State: ENABLED - Target: - Arn: !GetAtt BotFunction.Arn # Lambda direct? Or HTTPS? Decide per step 1 - RoleArn: !GetAtt SchedulerExecutionRole.Arn - RetryPolicy: { MaximumRetryAttempts: 2, MaximumEventAgeInSeconds: 600 } - DeadLetterConfig: { Arn: !GetAtt CronDLQ.Arn } - Input: '{"name":"lolschedule_daily_push"}' - # IF using HTTPS invoke (preferred for route preservation): - # Replace `Arn: !GetAtt BotFunction.Arn` with the universal target - # `Arn: arn:aws:scheduler:::http-invoke` and add HttpParameters. - - TradingDailyRefreshSchedule: - Type: AWS::Scheduler::Schedule - Properties: - Name: !Sub "${AWS::StackName}-trading-daily-refresh" - ScheduleExpression: "cron(0 8 * * ? *)" # 08:00 UTC = 15:00 ICT (market close) - FlexibleTimeWindow: { Mode: OFF } - State: ENABLED - Target: - # Same shape as above - RoleArn: !GetAtt SchedulerExecutionRole.Arn - RetryPolicy: { MaximumRetryAttempts: 2, MaximumEventAgeInSeconds: 600 } - DeadLetterConfig: { Arn: !GetAtt CronDLQ.Arn } - Input: '{"name":"trading_daily_refresh"}' - ``` -3. **Decide direct-invoke vs HTTPS** at implementation time: - - **HTTPS (preferred):** preserves `/cron/{name}` route; works with existing dispatcher; same shape as local-dev `curl` smoke. Need `HttpParameters` block with `X-Cron-Token` header. - - **Direct Lambda invoke:** simpler IAM, lower latency, bypasses HTTP layer. Requires a Lambda event-shape branch in `cmd/server/main.go` to detect Scheduler events vs Function URL events. - - Default: HTTPS for KISS; switch only if HTTPS proves flaky. -4. Validate locally: `make sam-validate` should pass. -5. After AWS-port Phase 01 deploy: - - Console → EventBridge Scheduler → "Run now" each rule. Confirm 200 from Lambda. - - Check CloudWatch log group for the cron handler executing. - - Send a synthetic invocation that fails (wrong token) — confirm DLQ receives the failed message. -6. Watch first scheduled fire from the AWS console (use a temporary `rate(2 minutes)` to verify, then revert). - -## Success Criteria -- [ ] Two schedules in `template.yaml` -- [ ] `sam validate` passes -- [ ] Post-deploy: Manual "run now" returns 200 and triggers handler -- [ ] DLQ receives failed invocations (synthetic test) -- [ ] First scheduled fire happens at the correct UTC time - -## Risk Assessment -- **`AWS::Scheduler::Schedule` HTTPS-target syntax** still evolving — mitigated by step 1 confirmation and ability to fall back to direct invoke. -- **Token mismatch between SSM and Lambda env** — both resolve at deploy time from the same parameter; no drift unless one is rotated independently. -- **Cron firing before Lambda is deployed** during stack creation — CloudFormation orders dependencies; Schedules `DependsOn: BotFunction` if needed (probably auto from Arn ref). -- **Time-zone confusion** — cron expressions use UTC; verified in comments next to each expression. - -## Open questions -1. Direct invoke vs HTTPS — final decision lives here, not Phase 04 of AWS-port plan. -2. Add a third schedule for a manual "ad-hoc" endpoint (e.g. for testing without console)? YAGNI — `aws scheduler invoke-now` works. -3. Schedule `State: ENABLED` vs `DISABLED` initially? ENABLED — first deploy implicitly trusts the cron handlers; if either causes prod issues, disable via console immediately. diff --git a/plans/260510-0234-pre-deploy-wrapup/plan.md b/plans/260510-0234-pre-deploy-wrapup/plan.md deleted file mode 100644 index 3ed3106..0000000 --- a/plans/260510-0234-pre-deploy-wrapup/plan.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -title: "Pre-deploy wrap-up: cron handlers + trading + cosmetics" -description: "Cloud-agnostic Go work + small SAM additions to land before Phase 01 AWS bootstrap. Outputs feed directly into AWS-port plan's Phase 04 + 06 + 07 verification." -status: in-progress -priority: P2 -effort: 8h -branch: main -tags: [aws, modules, lolschedule, trading, observability, readme] -created: 2026-05-10 -blockedBy: [] -blocks: [260510-0114-aws-port] ---- - -# Plan: Pre-deploy wrap-up - -Five focused phases that finish all **non-deploy** remaining work. Designed to land *before* the user runs Phase 01 of the AWS-port plan (manual AWS account + first `sam deploy`). After this plan ships, AWS-port Phases 04, 06, 07 become genuinely meaningful (real cron handlers, real metric data, accurate README at cutover). - -## Why these five - -From the punch-list: -1. **Cosmetics** — README still GCP-flavored; AWS-port plan statuses say "pending" for code that already shipped -2. **Metric filter** — small additive SAM change; trivial to land now -3. **lolschedule daily-push cron** — was deferred from GCP plan; without it Phase 04 schedules nothing real -4. **Trading module** — biggest pending Go chunk; cloud-agnostic; from old GCP Phase 08 -5. **EventBridge schedules** — wires the new cron handlers to AWS Scheduler - -## Phases - -| # | Phase | Status | Effort | Key deliverable | -|---|-------|--------|--------|-----------------| -| 01 | [Cosmetics: README + plan status sync](phase-01-cosmetics.md) | done | 30m | README rewritten for AWS default; AWS-port phases marked code-done | -| 02 | [Cold-start metric filter](phase-02-metric-filter.md) | done | 30m | `AWS::Logs::MetricFilter` for `Init Duration` in `template.yaml` | -| 03 | [lolschedule daily-push cron](phase-03-lolschedule-cron.md) | done | 3h | `Crons()` registered; Deps exposes bot for fan-out; daily push at 08:00 ICT | -| 04 | [Trading module port](phase-04-trading-module.md) | done (scope-trimmed: no daily refresh cron, no leaderboard — neither in upstream) | 4h | VN-stocks paper trading: topup/buy/sell/stats/convert; KBS price source | -| 05 | [Wire EventBridge schedules](phase-05-eventbridge-schedules.md) | **deferred to first-deploy decision** | 30m | `AWS::Scheduler::Schedule` resource for lolschedule cron — needs HTTPS-vs-direct-invoke call validated against live SAM CLI | - -## Dependency graph -``` -01 ──┐ (README + status — independent) -02 ──┤ (metric filter — independent) -03 ──┐ - ├──► 05 (schedules need real handlers from 03 + 04) -04 ──┘ -``` - -01 and 02 can ship in any order, including parallel. 05 blocks on 03 + 04 having registered crons. - -## Relation to other plans -- Builds on: `plans/260510-0114-aws-port/` (the offline artifacts already shipped) -- Carried-over from `plans/260508-2222-go-port-cloud-run/` Phase 08 (trading) — that phase is fulfilled by this plan's Phase 04 -- After this lands, AWS-port Phase 04 (EventBridge) and Phase 06 (metric capture) become verifiable on first deploy - -## Out of scope (explicit non-goals) -- AWS account creation / IAM OIDC / first `sam deploy` (= AWS-port Phase 01, user-manual) -- Telegram webhook flip (= AWS-port Phase 07, deploy-gated) -- 7-day soak observations (= AWS-port Phase 07) -- Provisioned concurrency, DynamoDB TTL, X-Ray dashboard customization (YAGNI for v1) - -## Abort criteria -- Phase 03 hits architectural friction extending `modules.Deps` to expose `*bot.Bot` cleanly → split into a smaller Deps refactor PR first, defer cron handler. -- Phase 04 trading API source unavailable / paywalled → stub the data layer, mark module disabled by default. - -## Open questions -1. Daily-push timezone: ICT 08:00 = UTC 01:00 — confirm cron expression matches in Phase 03. -2. Trading data source: original miti99bot uses VN stocks API — confirm it's still free + accessible in Phase 04. -3. Should Phase 01 (README) wait until trading module is done, so the README can advertise it? Tradeoff: ship docs sooner vs. ship complete picture. Default: ship now, update README when trading lands. diff --git a/plans/260510-0234-pre-deploy-wrapup/reports/code-reviewer-260510-0244-cron-and-trading.md b/plans/260510-0234-pre-deploy-wrapup/reports/code-reviewer-260510-0244-cron-and-trading.md deleted file mode 100644 index becf364..0000000 --- a/plans/260510-0234-pre-deploy-wrapup/reports/code-reviewer-260510-0244-cron-and-trading.md +++ /dev/null @@ -1,189 +0,0 @@ -# Code review — lolschedule cron + trading module + framework changes - -Date: 2026-05-10 -Reviewer: code-reviewer (staff) -Scope: 1× framework change, 1× new cron, 1× new module (~30 files) -Verification: build clean, `go vet` clean, `go test -race` 24/24 green; **but `gofmt -l` reports 1 file dirty** (see B1). - ---- - -## Critical (blocks merge) - -### C1. `gofmt -l` fails on `internal/modules/trading/handlers.go` -Line 22-28 of `state` struct has misaligned field tags. `gofmt -d` proposes: -``` -- kv storage.KVStore -- prices *PriceClient -- locks keylock.Map -- nowFn func() time.Time -+ kv storage.KVStore -+ prices *PriceClient -+ locks keylock.Map -+ nowFn func() time.Time - commingSoonMessage string -``` -golangci-lint v2.12 enforces `gofmt`; CI will fail. Run `gofmt -w internal/modules/trading/handlers.go`. - -### C2. Sell-rollback can silently lose user shares -`internal/modules/trading/handlers.go:188-189`: -```go -p.AddAsset(symbol, qty) -_ = SavePortfolio(ctx, s.kv, userID, p) -``` -The deduction-rollback Save's error is dropped with `_ =`. If KBS is down (already true at this codepath) AND the rollback write also fails (transient DynamoDB throttle, ctx deadline near expiry), the user's prior `DeductAsset` is in-memory only, never reverted, and on the next Load they will see the previous (post-deduct) state. Net result: shares deleted, no VND credited. - -Fix: log + replace user-facing message when rollback fails so an op is alerted, e.g.: -```go -if err := SavePortfolio(ctx, s.kv, userID, p); err != nil { - log.Error("trading_sell_rollback_failed", "user", userID, "symbol", symbol, "qty", qty, "err", err) - return chathelper.Reply(ctx, b, chatID, "Sell failed and rollback errored — contact support before retrying.") -} -``` -Or — better — fetch the price BEFORE acquiring the lock (parallel to handleBuy's structure) so no rollback path is needed. handleSell currently fetches *under* the lock too, which also blocks the user's mutex on a 10-second HTTP call (see H1). - ---- - -## High - -### H1. handleSell holds the per-user lock across a 10s HTTP call -`handlers.go:174-185`: lock acquired → Load → DeductAsset → **FetchPrice (10s timeout)** → AddCurrency → Save. Concurrent operations on the same user serialise behind a transient HTTP call to KBS. handleBuy correctly fetches the price *before* the lock. Rewrite handleSell to mirror handleBuy's order: validate, FetchPrice, then acquire lock, then Load → check holdings → Deduct + AddCurrency → Save. Eliminates the lock-held HTTP call AND removes the rollback hazard from C2. - -### H2. lolschedule cron will exceed the 60s server timeout above ~1100 subscribers -`internal/server/timeouts.go:9`: `defaultCronTimeout = 60 * time.Second`. -`internal/modules/lolschedule/cron.go:31`: `telegramRateLimitDelay = 50 * time.Millisecond` when subs > 30. -At N subscribers, throttled inter-send delay alone is `(N-1) * 50ms`. SendMessage HTTP latency adds ~50–200ms each. Effective ceiling ≈ 600–800 subscribers before the cron context cancels mid-batch. The handler does check `ctx.Done()` (good — returns ctx.Err) but the run is then logged as a failure with no resume state; the next day's run starts from chat[0] again so early subscribers are over-served and tail subscribers are starved (not fair). - -Mitigations (pick one for v1; defer the rest): -- a. Cap N: refuse new subscribers above e.g. 800 (warn user). -- b. Shard schedule: emit one cron per 500-sub group with offset times. Requires Phase 05 EventBridge work. -- c. Async fan-out: cron enqueues N SQS messages, each consumer SendMessages → done in parallel. Best long-term but needs new IaC. - -For v1 with realistic JS-source subscriber counts (<100), this is probably fine; flag for monitoring after deploy. - -### H3. handleSell: silent rollback save error swallows user data loss -Already covered in C2 — also high-severity from the data-integrity angle. - -### H4. No `From.ID == 0` defense -The user's task description says "we explicitly refuse but verify" — code does NOT verify. `senderInfo` (handlers.go:50) only refuses `nil` From, not `From.ID == 0`. If Telegram (or a malicious local-dev fixture) ever produces a User with ID=0, all such users would key into `user:0` and share a portfolio. Telegram's spec says IDs are positive, so this is defense-in-depth, but trivial to add: -```go -if msg == nil || msg.From == nil || msg.From.ID == 0 { - return 0, 0, false -} -``` - ---- - -## Medium - -### M1. `Currency` is `map[string]float64` for VND — should be int64 -VND has no sub-unit; the smallest legal denomination is 1 VND. `float64` arithmetic on `cost := float64(qty) * price` and `Meta.Invested += amount` accumulates IEEE-754 drift. After a few hundred buys at non-round prices (24,500 × 137 = 3,356,500 — exact, ok; but 18,750 × 31 = 581,250 — also exact, but compounded sums of non-power-of-two integers eventually drift). At Vietnamese stock-trade volumes this is unlikely to materialise as user-visible cents, but flagging because: -- (a) JSON decode `float64` of a saved 24,500,000 then `× 137` could round-trip-shift if KV ever stores e.g. "1.5e7"; -- (b) `FormatVND` uses `math.Round` which masks drift in the UI but not in the stored ledger. - -Severity is medium (not high) because the upstream JS likely had the same issue and no incident has been reported. Recommend documenting the trade-off in `portfolio.go` or migrating to int64 in v2. - -### M2. No ticker length / alphabet validation -`symbols.go:30-35` accepts any non-empty `args[1]` after upper+trim. There's no length cap, no `[A-Z0-9]` enforcement, no defence against unicode lookalikes. While `url.PathEscape` makes the HTTP call safe and `ErrNoPrice` paths skip cache writes (so no KV pollution from invalid lookups), a user could still spam: -``` -/trade_buy 1 АAA (Cyrillic А, looks like ASCII A) -``` -which generates KBS HTTP calls + a partial DoS amplification through your Lambda. Add a regex check, e.g. `^[A-Z0-9]{1,16}$` after upper+trim, return `ErrUnknownTicker` for misses. Cheap, principled. - -### M3. Field typo: `commingSoonMessage` -`handlers.go:27, 118, 212` — should be `comingSoonMessage` (one m). User-invisible (it's a private field), but CI linters with spell-check rules flag this. Style grep'd consistently (3 occurrences); a single rename works. - -### M4. `chatIDString` in cron_test.go is dead/wrong code -`cron_test.go:35-37`: -```go -func chatIDString(id int64) string { - return time.Unix(id, 0).Format("00") // arbitrary stringification -} -``` -`Format("00")` returns the literal string `"00"` because "00" contains no Go time-format directives. So every chat error message is identical: `"fakeSender: induced failure for chat 00"`. Replace with `strconv.FormatInt(id, 10)` or just inline `fmt.Errorf("fakeSender: induced failure for chat %d", id)`. - -### M5. lolschedule daily-push has no retry / dead-chat unsubscribe -A subscriber who blocks the bot returns 403 from SendMessage. The cron logs `failed++` and never removes them. Over weeks, the failure count grows. Not a correctness issue but an operational drag. Consider, in a future PR, trimming subscribers whose SendMessage returns specific 403/400 error codes. - -### M6. `Phase 05 EventBridge schedule` deferred — daily-push is dead-on-deploy -Per `plan.md:35`, Phase 05 is deferred. Without the `AWS::Scheduler::Schedule`, the registered `lolschedule_daily_push` cron will never fire in production. This is intentional per the plan, but I'm flagging because the README / changelog should not advertise the daily-push feature until Phase 05 ships. Verify the README copy doesn't promise active push. - ---- - -## Low - -### L1. `runDailyPush`: throttle decision is binary on `len(subs) > 30` -At N=31, the cron suddenly serialises with 50ms delays. Telegram's 30/sec global limit is a target rate not a hard ceiling — 30 contiguous sends is fine. The threshold is conservative; not wrong, just unnecessarily slow at N=31..100. - -### L2. `handleStats` allocates a per-call `heldList` slice — tiny GC churn at scale, fine for v1. - -### L3. `prices.go:103` shadows builtin `close` -```go -close := body.DataDay[0].C -if close <= 0 { ... } -``` -`close` is a Go builtin (channel close). Shadowing is legal but lint-noisy. Rename to `c` or `lastClose`. - -### L4. Test `TestRunDailyPush_SendsToAllSubscribers` asserts ordering of `sender.calls` -Subscribers come back from `listSubscribers` in JSON-array order, which is the order they were added — *currently*. If the persistence layer ever switches to a set-like backend, the test breaks. Either lock the contract in `listSubscribers`'s godoc or sort before asserting in the test. - -### L5. README / template.yaml — `trading` enabled by default -`template.yaml:17` adds `trading` to ModulesCSV. Trading is a financial-looking command surface (paper or not). For a personal bot this is fine, but consider whether it should be opt-in via a `--with-trading` deploy flag in case future operators want to disable it without editing the template. v1: leave as-is. - ---- - -## Edge cases / scout findings - -- **handleStats** reads portfolio without keylock; safe because LoadPortfolio JSON-decodes a fresh struct each call (no shared map memory with concurrent buy/sell). **No race.** -- **`defer s.locks.Acquire(key)()` semantics** — verified correct: outer call evaluates immediately (acquires), Unlock is deferred. Both buy and sell hold the lock over the right region. -- **ResolveSymbol cache-write fallback** — `_ = kv.PutJSON(...)` on cache miss + successful KBS lookup is intentional and safe (next call will reresolve). Acceptable. -- **`from.ID` collision** — see H4. -- **KBS HTTP error semantics** — 4xx/5xx → ErrNoPrice (verified by test). Network errors → wrapped. JSON decode errors → wrapped. Negative close → ErrNoPrice. Empty data_day → ErrNoPrice. **All paths covered.** -- **Cron auth** — `subtle.ConstantTimeCompare` used (router.go:68). No constant-time bypass via header probing. Good. -- **Cron name regex** — `^[a-z0-9_]{1,32}$`, blocks log injection. `lolschedule_daily_push` matches. Good. -- **No PII / secret leak** — error messages to users are generic ("Could not load portfolio. Try again later."); KBS upstream URLs not echoed; SendMessage params not logged with chat content; no stack traces propagated. -- **Stats fan-out latency** — sequential per-ticker FetchPrice; for a portfolio of 50 tickers at 100ms KBS latency that's 5s of dead time before the user sees anything. Below the 60s ceiling but bad UX. Probably fine for v1 (typical user holds <10). -- **Integer overflow** — `int64` for `qty` and `Assets` map values; max 9.2e18, never reachable for stock counts. `float64` for VND has 53-bit mantissa (~9e15 = 9 quadrillion VND ≈ $360 billion); not reachable. - ---- - -## Positive observations - -- Lock granularity (per-user) is correct, not over-broad. Distinct users never block each other. -- handleBuy correctly fetches price *before* acquiring lock — minimises lock duration. -- Tests use `httptest.NewServer` everywhere; **no real KBS calls in `go test`**. Hermetic. -- Dependency injection via `messageSender` interface in cron.go is exemplary: enables real-bot test without mocking the full `*bot.Bot` API. -- `BuildOptions` extension pattern: future deps (Bot, Embedder, Chatter) are added without breaking the `Build` signature. Good API stability hygiene. -- Cache write failure on `ResolveSymbol` is correctly non-fatal (one-line comment explains why). -- `senderInfo` correctly refuses channel posts / inline queries to avoid `user:0` collision. -- Defensive nil-map repair in `LoadPortfolio` is correct defence-in-depth. -- Throttle implementation in cron is select-based on `ctx.Done` — cooperative cancellation is wired. -- 24/24 packages green with `-race`; CI integration looks healthy. - ---- - -## Recommended action order - -1. **Fix C1** (gofmt) — 10s, unblocks CI. -2. **Fix C2 + H1 together** by reordering handleSell to fetch price before lock (mirrors handleBuy). One change, two issues resolved. -3. **Add H4** (From.ID == 0 check) — 3 lines. -4. **Add M2** (ticker regex) — 5 lines + 1 test. -5. **Rename M3** (`commingSoonMessage` → `comingSoonMessage`) — global replace. -6. **Fix M4** (chatIDString dead code) — 2-line fix. -7. **Defer rest** (M1 float→int64, M5 dead-chat unsub, L-series) to a follow-up PR. - -After (1)–(6), the change is mergeable. (1)–(3) are mandatory before deploy. - ---- - -## Unresolved questions - -1. Is the upstream JS `trading` module also using float64 for VND? If yes, M1 is parity (acceptable v1) — if no, this is a regression worth fixing now. -2. What's the realistic peak `lolschedule` subscriber count? If <300 ever, H2 is non-blocking; if growth is plausible, the decision in H2 (a/b/c) needs choosing before Phase 05 EventBridge ships. -3. Should the handleSell rollback path also restore `Meta.Invested` symmetry? Currently Buy doesn't touch Invested and Sell doesn't either — Invested only moves on `trade_topup`. This makes "Invested" mean "total deposits", not "cost basis", which deviates from typical brokerage semantics. Confirm intent matches JS source. -4. Is `Phase 05 EventBridge` going to land before public release? If yes, the daily-push code is exercised on first deploy. If no, it's dead-but-tested code. Either is fine — just confirm. - ---- - -**Status:** DONE_WITH_CONCERNS -**Summary:** Code is well-structured, hermetic-tested, race-clean. Two real correctness issues (C1 gofmt blocker, C2 silent rollback save) and one architectural smell (H1 lock-held HTTP call) need fixing before deploy. Trading module is a credible peer of wordle/loldle in shape and discipline; lolschedule cron is testable and correctly authenticated. -**Concerns:** C1 will fail CI. C2 + H1 are data-integrity (low probability, but not negligible at production scale). H4 is defense-in-depth. M-tier are quality-of-life. Phase 05 EventBridge schedule is deferred-by-design — verify README doesn't over-promise active push. diff --git a/plans/260515-2250-cf-data-to-aws-migration/phase-01-source-inventory-and-migration-policy.md b/plans/260515-2250-cf-data-to-aws-migration/phase-01-source-inventory-and-migration-policy.md deleted file mode 100644 index 55bc6dc..0000000 --- a/plans/260515-2250-cf-data-to-aws-migration/phase-01-source-inventory-and-migration-policy.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -phase: 1 -title: "Source inventory and migration policy" -status: completed -priority: P1 -effort: "2-3h" -dependencies: [] -completed: 2026-05-16 ---- - -# Phase 01: Source inventory and migration policy - -## Overview -Identify the exact Cloudflare KV namespaces and D1 tables still carrying production data, then lock a per-key policy: migrate, skip, or archive. The goal is to prevent a noisy "copy everything" migration that drags stale caches, retired modules, or incompatible schemas into DynamoDB. - -## Requirements -- Functional: produce a concrete inventory of live CF data sources, active key prefixes, D1 tables, and the AWS target shape for each kept dataset. -- Non-functional: decisions are explicit, reversible, and tied to current code paths — not guesses from old plans. - -## Architecture -- Inspect current AWS consumers first: `wordle stats:*`, `loldle stats:*` / `config:*`, `twentyq stats:*`, `lolschedule subscribers`, `misc:last_ping`, `trading user:*` portfolios. -- Inspect legacy CF sources second: KV namespace(s), D1 trading tables, and any retired-module prefixes still present. -- Lock default policy: - - **Migrate:** long-lived, user-visible state. - - **Skip:** `game:*`, `matches:*`, `sym:*`, other caches. - - **Archive-only:** retired modules and optional historical trade rows not consumed by current AWS runtime. - -## Related Code Files -- Create: `docs/cf-to-aws-migration-runbook.md` -- Modify: `plans/260510-0114-aws-port/phase-07-cutover.md` -- Read only: `internal/modules/wordle/state.go`, `internal/modules/loldle/state.go`, `internal/modules/twentyq/state.go`, `internal/modules/lolschedule/subscribers.go`, `internal/modules/trading/portfolio.go` - -## Implementation Steps -1. Enumerate current AWS key shapes from live code. -2. Pull a source inventory from Cloudflare KV and D1 using operator credentials. -3. Build a migration matrix: source dataset → target DynamoDB key → action (`migrate|skip|archive`). -4. Lock the exact D1 source tables/columns for `Portfolio.Meta.CreatedAt` and `Portfolio.Meta.Invested`. -5. Mark retired namespaces explicitly so they are not silently reintroduced. -6. Update the AWS cutover phase to say final webhook flip is gated on this migration matrix. - -## Success Criteria -- [x] Every live CF dataset is classified as migrate, skip, or archive. (matrix in `docs/cf-to-aws-migration-runbook.md`) -- [x] Every migrated dataset has an explicit AWS target key shape. -- [x] Trading `meta.createdAt` and `meta.invested` have authoritative source fields. (KV `trading:user:` — flat JSON snapshot, no D1 derivation) -- [x] Retired-module data is excluded by policy. (`doantu`, `loldle-ability`, `loldle-emoji`, `semantle` stats keys skipped; no archive) -- [x] The cutover plan references this migration gate. (`plans/260510-0114-aws-port/phase-07-cutover.md` lines 13, 20, 42, 72) - -## Outcome notes (2026-05-16) -- Live inventory taken via wrangler against prod CF account `miti99` (D1 `miti99bot-db`, KV `f7f190fcb2fa42eb84a05542911334b0`). -- D1: only `trading_trades` exists (11 rows, 1 user). No `users` / `holdings` tables. -- KV: 21 keys total. 9 durable, 7 cache, 5 retired-module stats. `misc:last_ping` does not exist upstream. -- Trading transform branch invalidated — JS Worker already stores final `Portfolio` JSON in KV. Phase 03 rewritten to flat KV copy (effort 4-6h → ~2h). - -## Risk Assessment -Main risk is misclassifying a dataset as disposable when users still care about it. Mitigation: classify by current runtime consumers first, then validate Cloudflare inventory against those exact consumers before any tooling is written. diff --git a/plans/260515-2250-cf-data-to-aws-migration/phase-02-backfill-toolchain-and-safety-rails.md b/plans/260515-2250-cf-data-to-aws-migration/phase-02-backfill-toolchain-and-safety-rails.md deleted file mode 100644 index f59d600..0000000 --- a/plans/260515-2250-cf-data-to-aws-migration/phase-02-backfill-toolchain-and-safety-rails.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -phase: 2 -title: "Backfill toolchain and safety rails" -status: completed -priority: P1 -effort: "3-4h" -dependencies: [1] -completed: 2026-05-16 ---- - -# Phase 02: Backfill toolchain and safety rails - -## Overview -Build operator-run migration tooling in Go that can read legacy Cloudflare data, write DynamoDB records idempotently, and support dry-runs plus checkpoints. This phase is about controlled mechanics, not the actual production import yet. - -## Requirements -- Functional: provide commands for KV export/import, trading import, and parity verification inputs. -- Non-functional: idempotent writes, dry-run mode, resumable progress, zero admin HTTP surface, and no dependency on the running AWS bot process. - -## Architecture -- Keep tooling inside this repo and language stack, but keep it small: - - `cmd/migrate_cf_data/` for inventory + KV import + trading import modes - - `cmd/verify_cf_aws_parity/` for verification only -- Shared logic lives under `internal/migration/` for Cloudflare REST reads, DynamoDB writes, and report formatting. -- D1 source extraction stays simple: operator uses `wrangler d1 execute ... --json --remote` to create local JSON exports; Go import code consumes those files instead of re-implementing remote SQL access. -- Checkpoint/resume is conditional: add it only if Phase 01 proves the keyspace is large enough to justify it. -- No writes happen during `--dry-run`; output is a machine-readable summary plus human-readable progress logs. - -## Related Code Files -- Create: `cmd/migrate_cf_data/main.go` -- Create: `cmd/verify_cf_aws_parity/main.go` -- Create: `internal/migration/cloudflare_kv_client.go` -- Create: `internal/migration/dynamodb_writer.go` -- Create: `internal/migration/report.go` -- Optional create: `internal/migration/checkpoint_store.go` -- Modify: `go.mod` -- Modify: `docs/cf-to-aws-migration-runbook.md` - -## Implementation Steps -1. Define CLI flags and env contract for Cloudflare and AWS credentials. -2. Implement KV list/get readers against Cloudflare REST with pagination support. -3. Implement DynamoDB writers against the live runtime shape: `pk = moduleName`, `sk = caller key`. -4. Add checkpoint files only if Phase 01 proves resume support is worth the extra surface area. -5. Add dry-run and report output before any real import path is allowed. -6. Document the exact operator workflow in the runbook. - -## Success Criteria -- [x] Tooling reads CF KV metadata and values without touching app code paths. (`internal/migration/cloudflare_kv_client.go`) -- [x] ~~Trading import mode accepts local D1 JSON exports.~~ → invalidated by Phase 01. Trading is a flat KV copy; D1 is audit-only via `trading-audit-dump --out=`. -- [x] Every command supports `--dry-run`. (kv-import has --dry-run; inventory and trading-audit-dump are read-only by construction so a dry-run flag is redundant) -- [x] Import path is idempotent or safely merge-based. (`attribute_not_exists(pk)` guard; `--overwrite` is explicit opt-in) -- [x] Checkpoint/resume behavior is intentionally omitted: Phase 01 inventory shows 21 keys total (well below the threshold where resume earns its complexity cost). - -## Outcome notes (2026-05-16) -- Files created: `cmd/migrate_cf_data/main.go`, `internal/migration/policy.go`, `cloudflare_kv_client.go`, `cloudflare_d1_client.go`, `dynamodb_writer.go`, `report.go` + four `*_test.go` files. -- Verify command (`cmd/verify_cf_aws_parity/`) intentionally moved to Phase 04 to remove the cross-phase ownership collision. -- Toolchain smoke-tested against prod CF: `inventory` → 22 keys observed (one cache key drift since Phase 01 inventory); `kv-import --dry-run` → 9 durable keys map to runtime `(pk, sk)` exactly. -- All `go test ./...` pass; `go vet ./...` clean. - -## Risk Assessment -The main risk is embedding too much migration logic into one giant binary. Mitigation: split command entrypoints and keep shared logic in small `internal/migration/` helpers so each command stays reviewable and under the repo's file-size guidance. diff --git a/plans/260515-2250-cf-data-to-aws-migration/phase-03-trading-and-durable-kv-import.md b/plans/260515-2250-cf-data-to-aws-migration/phase-03-trading-and-durable-kv-import.md deleted file mode 100644 index 30c4e72..0000000 --- a/plans/260515-2250-cf-data-to-aws-migration/phase-03-trading-and-durable-kv-import.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -phase: 3 -title: "Durable KV import (trading included)" -status: pending -priority: P1 -effort: "2h" -dependencies: [1, 2] ---- - -# Phase 03: Durable KV import (trading included) - -## Overview -Copy the 9 durable Cloudflare KV records into DynamoDB under the live runtime key shape. Trading is included as a flat KV copy — the JS Worker already snapshots `Portfolio` JSON into `trading:user:*`, so no D1 transform is required (locked in Phase 01). - -## Requirements -- Functional: each migrated KV key lands in DynamoDB at `(pk=moduleName, sk=callerKey)` with the original CF KV value placed in the `value` attribute, byte-for-byte where possible. -- Non-functional: idempotent — rerun must not duplicate or corrupt; skipped/failed records must be reported, not silently dropped. - -## Architecture -- Single import mode: KV → DynamoDB. No D1 transform path. -- Durable key set (locked in Phase 01): - - `wordle:stats:*` - - `loldle:stats:*` - - `loldle:config:*` - - `twentyq:stats:*` - - `lolschedule:subscribers` - - `trading:user:*` -- Skip set: `trading:sym:*` (cache), retired modules (`doantu`, `loldle-ability`, `loldle-emoji`, `semantle` stats keys), `misc:last_ping` (never written upstream). -- Optional sub-task: dump `trading_trades` (D1) to JSONL for cold audit. Not an import input. Operator-elective. - -## Related Code Files -- Modify: `cmd/migrate_cf_data/main.go` — wire up the KV-copy run mode -- Create: `internal/migration/kv_filter.go` — durable/skip allowlist driven by the Phase 01 matrix -- Create: `internal/migration/import_report.go` — counts for imported / skipped / failed -- Create: `internal/migration/trading_audit_dump.go` — optional D1 → JSONL exporter (operator-elective) -- Modify: `docs/cf-to-aws-migration-runbook.md` — append import command + report layout -- Read only: `internal/storage/dynamodb_kv.go`, `internal/modules/trading/portfolio.go` - -## Implementation Steps -1. Wire the KV allowlist filter into the import binary so only Phase 01 durable keys are read. -2. For each durable KV key, read the raw value and write to DynamoDB at the matching `(pk, sk)`. Preserve original bytes. -3. Implement idempotency via conditional `PutItem` or `attribute_not_exists` guard, fall back to overwrite with operator flag. -4. Emit an import report (stdout + JSON file) with per-prefix counts: imported, skipped, failed. -5. Add the optional `--trading-audit-dump ` flag that streams `trading_trades` rows to a local JSONL file. Default off. -6. Update the runbook with the exact command operators run, and the expected report layout. - -## Success Criteria -- [ ] All 9 durable keys land in DynamoDB at the runtime-expected `(pk, sk)`. -- [ ] `trading:user:` round-trips through the Go runtime's `Portfolio` JSON unmarshal without modification (parity check). -- [ ] Re-running the import without flags is a no-op (no duplicates, no corruption). -- [ ] Skipped-by-policy keys (cache, retired modules) are listed in the report, not silently dropped. -- [ ] Optional `trading_trades` audit dump produces JSONL with one row per trade when flag is passed. - -## Risk Assessment -- KV value encoding drift: CF KV may return values as strings while DynamoDB attribute typing prefers `B`/`S`. Mitigation: round-trip a known portfolio record and assert byte parity before bulk import. -- Idempotency: an overwrite-by-default rerun could silently revert post-cutover writes. Mitigation: default to `attribute_not_exists` guard and require explicit `--overwrite` flag. - -## Notes -- Phase 03 used to assume a D1 → Portfolio transform. That was wrong: KV already holds the final shape. Earlier `trading_transform.go` work is dropped. diff --git a/plans/260515-2250-cf-data-to-aws-migration/phase-04-parity-verification-and-rehearsal.md b/plans/260515-2250-cf-data-to-aws-migration/phase-04-parity-verification-and-rehearsal.md deleted file mode 100644 index 34f8f41..0000000 --- a/plans/260515-2250-cf-data-to-aws-migration/phase-04-parity-verification-and-rehearsal.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -phase: 4 -title: "Parity verification and rehearsal" -status: pending -priority: P1 -effort: "2-3h" -dependencies: [2, 3] ---- - -# Phase 04: Parity verification and rehearsal - -## Overview -Prove the imported AWS data matches the Cloudflare source closely enough to trust a real cutover. This phase turns migration from a one-off script run into a repeatable, auditable procedure with a staging-table rehearsal. - -## Requirements -- Functional: verify counts, sample payload parity, and trading portfolio correctness between CF exports and DynamoDB. -- Non-functional: produce a saved report, support reruns, and define rollback steps before the production webhook is moved. - -## Architecture -- Verifier compares source exports against the AWS target table using the same module/key selectors from Phase 01. -- Checks by dataset type: - - KV durable records: count parity + payload/hash comparisons - - trading portfolios: count parity + deep field comparison on currency, assets, and invested metadata -- Rehearsal happens against a staging DynamoDB table only. No destructive rerun path is allowed against the live table. -- Final output is a migration report under `plans/reports/` plus runbook updates. - -## Related Code Files -- Create: `cmd/verify_cf_aws_parity/main.go` -- Create: `internal/migration/parity_checks.go` -- Create: `internal/migration/rollback_scope.go` -- Modify: `docs/cf-to-aws-migration-runbook.md` -- Create during execution: `plans/reports/migration-260515-2250-cf-data-to-aws-parity.md` - -## Implementation Steps -1. Implement count and payload verification per migrated dataset. -2. Add trading-specific deep checks against the current `Portfolio` shape. -3. Save the verifier result as a markdown report under `plans/reports/`. -4. Rehearse import + verify against a staging DynamoDB table. -5. Promote the exact same procedure to the live table only after staging is green. -6. Mark the migration runbook ready only after a green verifier report. - -## Success Criteria -- [ ] Verifier reports pass for all migrated datasets. -- [ ] Trading portfolios match expected balances and holdings on spot checks. -- [ ] A staging-table rehearsal completes successfully without touching the live table. -- [ ] The migration report is saved and linked from the runbook. -- [ ] The cutover checklist now depends on a green parity report. - -## Risk Assessment -The main risks are false confidence from count-only checks and accidental destructive rehearsal against production storage. Mitigation: include dataset-specific deep comparisons, especially for trading portfolios, and require a saved report plus staging-table-only rehearsal before the cutover phase can begin. diff --git a/plans/260515-2250-cf-data-to-aws-migration/phase-05-cutover-integration-and-cloudflare-decommission.md b/plans/260515-2250-cf-data-to-aws-migration/phase-05-cutover-integration-and-cloudflare-decommission.md deleted file mode 100644 index ef0d523..0000000 --- a/plans/260515-2250-cf-data-to-aws-migration/phase-05-cutover-integration-and-cloudflare-decommission.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -phase: 5 -title: "Cutover integration and Cloudflare decommission" -status: pending -priority: P1 -effort: "2-3h" -dependencies: [1, 2, 3, 4] ---- - -# Phase 05: Cutover integration and Cloudflare decommission - -## Overview -Fold the verified migration into the AWS cutover path, then decommission Cloudflare resources only after a successful freeze-window migration and AWS soak. This phase closes the data-consistency gap left by the original AWS port plan. - -## Requirements -- Functional: production cutover moves webhook ownership to AWS without losing durable writes from the old Cloudflare stack. -- Non-functional: rollback is fast only before the first AWS-served write; after that, the plan is forward-fix only unless reverse sync is built later. Cloudflare resources are deleted only after verification, and operator steps are explicit. - -## Architecture -- There is no legacy dual-write path, so final cutover uses a short **write-freeze window**: - 1. disable/pause Cloudflare cron triggers - 2. stop Cloudflare webhook intake so no new writes land there - 3. run final delta export/import + parity verify - 4. point Telegram webhook to AWS - 5. begin AWS soak -- Before the first AWS-served write, rollback is still a webhook restore. -- After the first AWS-served write, rollback is not symmetry; it becomes forward-fix only unless a reverse-sync mechanism exists. -- This keeps migration correctness simple and avoids inventing temporary cross-runtime replication. -- Cloudflare teardown is a separate final step after the AWS soak, not part of the initial webhook flip. - -## Related Code Files -- Modify: `plans/260510-0114-aws-port/plan.md` -- Modify: `plans/260510-0114-aws-port/phase-07-cutover.md` -- Modify: `docs/deploy-aws.md` -- Modify: `docs/cf-to-aws-migration-runbook.md` -- Optional create: `docs/cf-decommission-checklist.md` - -## Implementation Steps -1. Update the AWS cutover phase to depend on a green migration report. -2. Add the freeze-window sequence and pre-flip vs post-flip rollback semantics to the runbook. -3. Define the final delta import and verification command set. -4. Flip the Telegram webhook only after the final delta verify succeeds. -5. Add post-flip smoke checks for a migrated trading account, an existing lolschedule subscriber, and `/mstats` if `last_ping` is kept. -6. Soak on AWS, then remove CF Worker/KV/D1 only when rollback is no longer needed. -7. Archive or document any intentionally skipped legacy datasets before teardown. - -## Success Criteria -- [ ] AWS cutover docs explicitly require a green migration report. -- [ ] Freeze-window steps are documented end-to-end. -- [ ] Pre-flip rollback and post-flip forward-fix semantics are documented explicitly. -- [ ] Final delta import and rollback commands are ready before webhook flip. -- [ ] Cloudflare resources are not deleted during the initial cutover window. -- [ ] After soak, CF teardown is documented and low-risk. - -## Risk Assessment -The biggest risk is write drift between an early backfill and the final webhook flip. Mitigation: use a short freeze window for the last delta import instead of trying to add temporary dual-write behavior to a legacy system that lives outside this repo. diff --git a/plans/260515-2250-cf-data-to-aws-migration/plan.md b/plans/260515-2250-cf-data-to-aws-migration/plan.md deleted file mode 100644 index 947f93f..0000000 --- a/plans/260515-2250-cf-data-to-aws-migration/plan.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: "Migrate Cloudflare data to AWS DynamoDB" -description: "Export durable data from the legacy Cloudflare Worker stack, import it into the live AWS DynamoDB store, verify parity, and gate final cutover on a proven migration runbook." -status: pending -priority: P1 -effort: 1-2d -branch: main -tags: [migration, cloudflare, aws, dynamodb, cutover, data] -created: 2026-05-15 -blockedBy: [] -blocks: [260510-0114-aws-port] ---- - -# Plan: Cloudflare data → AWS DynamoDB - -This plan adds the missing data-migration leg to the in-progress AWS port. AWS runtime + DynamoDB already exist; the gap is getting durable user data out of the legacy Cloudflare KV/D1 stack before final decommission. - -## Why separate plan -- `plans/260510-0114-aws-port/` covers runtime + deploy cutover. -- This plan covers source-data inventory, export/import tooling, parity verification, and rollback. -- `aws-port` should not be considered done until this plan passes. - -## Locked decisions -- Migrate **durable user-visible data only**. -- Skip ephemeral or disposable data: in-flight game state, schedule caches, stale price caches. -- ~~Keep `misc:last_ping`~~ → skip; KV inventory shows it was never written by the JS Worker (`/mstats` resets on AWS). -- No admin HTTP routes. Migration runs as operator-invoked one-shot tooling. -- **Pre-cutover bulk import may target the live table directly** while the Telegram webhook still points to Cloudflare and the AWS bot has served zero writes (amended 2026-05-16). Rationale: until webhook flip, the AWS DynamoDB table is empty and no real user traffic depends on it, so a separate staging table adds setup cost without de-risking anything. **After webhook cutover, any re-import must use a staging table first.** The `attribute_not_exists` idempotency guard is the only safe write path at any time — no wipe-and-rerun flow is allowed against the live table. -- ~~Trading import is a transform~~ → **flat KV copy**. CF KV `trading:user:` already holds the final `Portfolio` JSON shape; no D1 derivation needed. (Phase 01 inventory, 2026-05-16.) -- After the first AWS-served write, rollback is forward-fix only unless a reverse-sync path is built later. -- Retired module namespaces from the old CF stack are skipped (no archive — operator decision 2026-05-16). - -## Source data classified in Phase 01 (closed 2026-05-16) -- **Migrate (9 keys):** `wordle:stats:*` (1), `loldle:stats:*` (4), `loldle:config:*` (1), `twentyq:stats:*` (1), `lolschedule:subscribers` (1), `trading:user:*` (1). -- **Skip (cache + missing + retired):** `trading:sym:*` (7), `misc:last_ping` (0), `doantu:stats:*` (2), `loldle-ability:stats:*` (1), `loldle-emoji:stats:*` (1), `semantle:stats:*` (1). -- **Archive-only (optional, operator-elective):** D1 `trading_trades` (11 rows, 1 user) — audit dump, not import input. - -Full matrix lives in `docs/cf-to-aws-migration-runbook.md`. - -## Related current code -- `cmd/server/main.go:167` — runtime storage backend selection (`dynamodb|firestore|memory`) -- `internal/storage/dynamodb_provider.go:7` — live DynamoDB partitioning (`pk = moduleName`) -- `internal/storage/dynamodb_kv.go:24` — live DynamoDB sort-key contract (`sk = caller key`) -- `internal/modules/wordle/state.go:50` — `game:*` + `stats:*` -- `internal/modules/loldle/state.go:48` — `game:*`, `stats:*`, `config:*` -- `internal/modules/twentyq/state.go:37` — `game:*` + `stats:*` -- `internal/modules/lolschedule/subscribers.go:14` — `subscribers` -- `internal/modules/trading/portfolio.go:39` — current AWS target shape: per-user KV portfolio JSON -- `plans/260508-2222-go-port-cloud-run/phase-12-cutover.md:37` — prior CF→Go cutover notes (trading-only import assumption) -- `plans/260510-0114-aws-port/phase-07-cutover.md:13` — current AWS final cutover phase - -## Phases - -| # | Phase | Status | Effort | Key deliverable | -|---|-------|--------|--------|-----------------| -| 01 | [Source inventory and migration policy](phase-01-source-inventory-and-migration-policy.md) | completed | 2-3h | exact CF namespaces/tables mapped to migrate vs skip vs archive | -| 02 | [Backfill toolchain and safety rails](phase-02-backfill-toolchain-and-safety-rails.md) | completed | 3-4h | operator-run `cmd/migrate_cf_data` binary (inventory, kv-import, trading-audit-dump) + dry-run + idempotency | -| 03 | [Durable KV import (trading included)](phase-03-trading-and-durable-kv-import.md) | pending | 2h | flat KV→DynamoDB copy for 9 durable keys + optional D1 audit dump | -| 04 | [Parity verification and rehearsal](phase-04-parity-verification-and-rehearsal.md) | pending | 2-3h | repeatable verifier, mismatch report, rollback drill | -| 05 | [Cutover integration and Cloudflare decommission](phase-05-cutover-integration-and-cloudflare-decommission.md) | pending | 2-3h | AWS cutover checklist updated; CF teardown gated on verified migration | - -## Key dependencies -- Blocks: `plans/260510-0114-aws-port/phase-07-cutover.md` -- Uses the already-live AWS target from `plans/260510-0114-aws-port/` -- Should finish before deleting CF Worker/KV/D1 resources referenced in `plans/260508-2222-go-port-cloud-run/phase-12-cutover.md` - -## Success bar -- Durable CF data imported into DynamoDB with counts + sampled payload parity. -- Trading balances/holdings and required portfolio metadata match the old system byte-for-byte (flat KV copy). -- Cutover runbook explicitly distinguishes pre-flip rollback from post-flip forward-fix semantics. -- CF resources are not deleted until parity report is green. diff --git a/plans/260516-1409-trongtruonghop-command/phase-01-add-trongtruonghop-command.md b/plans/260516-1409-trongtruonghop-command/phase-01-add-trongtruonghop-command.md deleted file mode 100644 index 35c7741..0000000 --- a/plans/260516-1409-trongtruonghop-command/phase-01-add-trongtruonghop-command.md +++ /dev/null @@ -1,138 +0,0 @@ -# Phase 01 — Add `/trongtruonghop` to misc module - -**Status:** Planned -**Priority:** Low (additive, no migration, no infra change) -**Mode:** fast - -## Context links - -- Module under change: `internal/modules/misc/misc.go` -- Helper API used: `internal/modules/util/chathelper/chathelper.go` (`ArgAfterCommand`, `ReplyHTML`) -- Visibility / validation rules: `internal/modules/module.go`, `internal/modules/validate.go` -- Forum-topic reply-routing fix that mandates `chathelper.Reply*` over raw `SendMessage`: commit 3a12615 - -## Overview - -Stateless command. Two interpolation points (``, `@` × 2) into a fixed Vietnamese template. No KV. No new dependency. No new helper. - -## Key insights - -- `chathelper.ArgAfterCommand` already strips command + `@botname` correctly — covers `/trongtruonghop arg`, `/trongtruonghop@miti99bot arg`, etc. -- `chathelper.ReplyHTML` already forwards `MessageThreadID` (forum-topic safe). Do NOT bypass. -- Telegram's HTML parser accepts `@username` literally (it's not a tag) and resolves the mention server-side. Mixing `@username` with `Name` in the same message is allowed and standard. -- `From.Username` can be empty (account never set one). `From.FirstName` is also optional (deleted accounts). Both can be empty simultaneously — handle. - -## Requirements - -### Functional - -- Command name: `trongtruonghop`, Visibility: `VisibilityPublic`, Description: `"Phát biểu disclaimer cho thành viên hiện tại"` (Vietnamese — keep short, fits `/help`). -- On `/trongtruonghop [text]`: - 1. `arg := strings.TrimSpace(chathelper.ArgAfterCommand(msg.Text))` - 2. If `arg == ""` → `arg = defaultTarget` (= `"VNG"`). - 3. Resolve sender mention from `msg.From` (see algorithm below). - 4. Send single HTML message via `chathelper.ReplyHTML`. -- If `msg == nil` or `msg.From == nil` → return `nil` (silent skip). - -### Sender-mention algorithm - -```go -func senderMention(u *models.User) string { - if u.Username != "" { - return "@" + u.Username // safe verbatim; charset is [A-Za-z0-9_] - } - name := strings.TrimSpace(u.FirstName + " " + u.LastName) - if name == "" { - name = "thành viên" - } - return fmt.Sprintf(`%s`, u.ID, html.EscapeString(name)) -} -``` - -### Template - -Package-level `const`: - -```go -const trongTruongHopTemplate = "Trong trường hợp nhóm này bị điều tra bởi %s, %s khẳng định không liên quan tới nhóm hoặc những cá nhân khác trong nhóm này. %s không rõ tại sao lại có mặt ở đây vào thời điểm này, có lẽ tài khoản đã được thêm bởi một bên thứ ba." -const defaultTarget = "VNG" -``` - -Render: `fmt.Sprintf(trongTruongHopTemplate, html.EscapeString(arg), mention, mention)`. - -## Architecture - -No new types, no state, no new files. Single command added to `New(deps modules.Deps)` in `misc.go`. `deps.KV` not used by this command but `New` already receives it for the other two — no signature change. - -## Related code files - -**Modify:** -- `internal/modules/misc/misc.go` -- `internal/modules/misc/misc_test.go` -- `internal/modules/misc/handlers_test.go` -- `README.md` (misc-row description only) - -**Create:** none. -**Delete:** none. - -## Implementation steps - -1. **misc.go** - - Add imports: `fmt`, `html`, `strings` (only those not already imported). - - Add `const trongTruongHopTemplate` and `const defaultTarget` near the existing `const lastPingKey`. - - Add private function `senderMention(*models.User) string` (algorithm above). - - Add `trongTruongHopCommand() modules.Command` (no `deps` needed — stateless). - - Append it to the `Commands` slice in `New`. - -2. **misc_test.go** - - Extend the `want` map in `TestNew_RegistersExpectedCommands` with `"trongtruonghop": modules.VisibilityPublic`. The existing length check then implicitly verifies registration. - -3. **handlers_test.go** — new test cases (reuse existing `installMisc`): - - `TestTrongTruongHop_DefaultArgUsesVNG`: send `/trongtruonghop` from user 999 with username `boss`. Assert reply contains `"VNG"` and `"@boss"` (occurring twice). - - `TestTrongTruongHop_CustomArg`: send `/trongtruonghop Acme Corp`. Assert reply contains `"Acme Corp"` and not `"VNG"`. - - `TestTrongTruongHop_HTMLEscapesArg`: send `/trongtruonghop