refactor(deploy): remove retired aws support

This commit is contained in:
2026-06-29 00:11:42 +07:00
parent 35be2c0acb
commit 5cabf4e1f4
48 changed files with 142 additions and 2918 deletions
-83
View File
@@ -1,83 +0,0 @@
# migrate-dynamo-to-mongo
One-off CLI that copies every item from the prod DynamoDB KV table
(`miti99bot-data`) into MongoDB Atlas using the exact document schema the live
app writes, then verifies per-module parity. Idempotent and re-runnable.
## What it does
- Full-table `Scan` of DynamoDB (the table is a small KV) → group by `pk`.
- Writes each item through the typed Mongo store (`storage.Typed[bson.M]`) as a
**flattened native document** — the value's JSON fields are hoisted to the
document root alongside `_id`/`version`/`updatedAt`, with no `value` envelope.
This is the exact shape the running bot writes, so the app reads migrated docs
directly. Integers keep int64 fidelity (decoded with `UseNumber`).
- The two non-object values are wrapped into named root fields to match the
module's typed shape: lolschedule `subscribers` (a JSON array) → `{subscribers: [...]}`
and `daily_push:last_date` (a bare date string) → `{date: "..."}`. Any other
non-object value fails loud so a missing wrap rule is obvious (see `encode.go`).
- Writing through the store validates the module/collection name and key and
upserts by `_id`, so a re-run produces no duplicates and bad input fails loud.
| DynamoDB | MongoDB |
|---|---|
| `pk` (module name) | collection name |
| `sk` (user key) | document `_id` |
| `value` (JSON object) | payload fields hoisted to the document root (no `value` field) |
| `value` (array/scalar, lolschedule) | wrapped in a named root field (`subscribers` / `date`) |
| — | `version` = 1, `updatedAt` = migration time (write-only; nothing reads it) |
## Usage
```sh
export MONGO_URL='mongodb+srv://botuser:PASS@cluster0.xxxxx.mongodb.net/?retryWrites=true&w=majority'
export MONGO_DATABASE=miti99bot
export AWS_PROFILE=miti99bot-migrate # READ-ONLY profile (see IAM below)
# 1. Dry run — report per-module counts, write nothing.
go run ./cmd/migrate-dynamo-to-mongo --dry-run
# or: make migrate-dynamo-to-mongo DRY_RUN=1
# 2. Real migration.
go run ./cmd/migrate-dynamo-to-mongo
# or: make migrate-dynamo-to-mongo
# 3. Verify — per-module counts must match; exits non-zero on mismatch.
go run ./cmd/migrate-dynamo-to-mongo --verify
# or: make migrate-verify
```
Flags: `--dynamodb-table` (default `miti99bot-data`), `--dry-run`, `--verify`.
For a local end-to-end test, point at DynamoDB Local + a local Mongo:
```sh
DYNAMODB_LOCAL_URL=http://localhost:8001 \
MONGODB_TEST_URL=mongodb://127.0.0.1:27017 \
MONGO_DATABASE=migrate_test \
go test ./cmd/migrate-dynamo-to-mongo/ -run TestMigrateAndVerify
```
## IAM — least privilege
The runner needs **exactly** `dynamodb:Scan` on the table ARN and nothing else.
Verify uses a Scan tally (not Query), so no `dynamodb:Query` is needed; there
are **no write actions on the source**, enforcing the read-only requirement and
removing the destructive-credential foot-gun.
```json
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "dynamodb:Scan",
"Resource": "arn:aws:dynamodb:ap-southeast-1:225603493174:table/miti99bot-data"
}]
}
```
## Cutover runbook
The full zero-loss cutover (disable EventBridge → `deleteWebhook` → migrate →
verify → start the polling container) lives in
[`docs/deploy-coolify-selfhosted.md`](../../docs/deploy-coolify-selfhosted.md#cutover-runbook).
-109
View File
@@ -1,109 +0,0 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"go.mongodb.org/mongo-driver/v2/bson"
)
// reservedRootFields are owned by the storage layer (see internal/storage); a
// flattened object payload must not collide with them.
var reservedRootFields = map[string]bool{"_id": true, "version": true, "updatedAt": true}
// wrapRule names the root field a non-object DynamoDB value must be wrapped in
// to match the typed Mongo store's named-struct shape. rawString treats the
// DynamoDB value as a bare (non-JSON) string; otherwise it is JSON-decoded.
type wrapRule struct {
field string
rawString bool
}
// wrapRules covers the only KV entries whose value is not a JSON object. They
// mirror the named-struct wrappers the lolschedule module persists:
// - subscribers: a JSON array → {subscribers: [...]}
// - daily_push:last_date: a bare date string → {date: "..."}
//
// Any other non-object value fails loud in payloadForItem so a missing rule is
// obvious rather than silently dropped.
var wrapRules = map[[2]string]wrapRule{
{"lolschedule", "subscribers"}: {field: "subscribers"},
{"lolschedule", "daily_push:last_date"}: {field: "date", rawString: true},
}
// payloadForItem converts one migrated KV row's value into the flattened payload
// map the typed Mongo store stores at the document root. The store adds _id,
// version, and updatedAt; this returns only the payload fields.
func payloadForItem(module, key string, value []byte) (bson.M, error) {
if rule, ok := wrapRules[[2]string{module, key}]; ok {
if rule.rawString {
return bson.M{rule.field: string(value)}, nil
}
decoded, err := decodeJSONNumber(value)
if err != nil {
return nil, fmt.Errorf("%s/%s: decode value: %w", module, key, err)
}
return bson.M{rule.field: decoded}, nil
}
decoded, err := decodeJSONNumber(value)
if err != nil {
return nil, fmt.Errorf("%s/%s: decode value: %w", module, key, err)
}
obj, ok := decoded.(bson.M)
if !ok {
return nil, fmt.Errorf("%s/%s: value is not a JSON object (type %T) and has no wrap rule — add one to wrapRules", module, key, decoded)
}
for k := range obj {
if reservedRootFields[k] {
return nil, fmt.Errorf("%s/%s: payload key %q collides with a reserved root field — add a wrap rule", module, key, k)
}
}
return obj, nil
}
// decodeJSONNumber decodes JSON into a BSON-native value, preserving integral
// numbers as int64 (UseNumber) so migrated numbers keep int64 fidelity, matching
// the live store's value codec.
func decodeJSONNumber(value []byte) (any, error) {
dec := json.NewDecoder(bytes.NewReader(value))
dec.UseNumber()
var v any
if err := dec.Decode(&v); err != nil {
return nil, err
}
if dec.More() {
return nil, fmt.Errorf("trailing data after JSON value")
}
return numberToBSON(v), nil
}
// numberToBSON walks a json.Unmarshal(UseNumber) tree into bson.M/bson.A,
// converting json.Number to int64 when integral, else float64.
func numberToBSON(v any) any {
switch t := v.(type) {
case map[string]any:
m := make(bson.M, len(t))
for k, val := range t {
m[k] = numberToBSON(val)
}
return m
case []any:
a := make(bson.A, len(t))
for i, val := range t {
a[i] = numberToBSON(val)
}
return a
case json.Number:
if i, err := t.Int64(); err == nil {
return i
}
if f, err := t.Float64(); err == nil {
return f
}
return t.String()
default:
return v
}
}
@@ -1,87 +0,0 @@
package main
import (
"testing"
"go.mongodb.org/mongo-driver/v2/bson"
)
func TestPayloadForItem_Object(t *testing.T) {
got, err := payloadForItem("coin", "user:1", []byte(`{"bal":100,"meta":{"createdAt":5}}`))
if err != nil {
t.Fatalf("payloadForItem: %v", err)
}
if got["bal"] != int64(100) {
t.Errorf("bal = %v (%T), want int64(100)", got["bal"], got["bal"])
}
if _, ok := got["value"]; ok {
t.Error("payload must not contain a 'value' envelope field")
}
meta, ok := got["meta"].(bson.M)
if !ok || meta["createdAt"] != int64(5) {
t.Errorf("nested meta = %v, want {createdAt: int64(5)}", got["meta"])
}
}
func TestPayloadForItem_LolscheduleSubscribers(t *testing.T) {
got, err := payloadForItem("lolschedule", "subscribers", []byte(`[{"chatId":1},{"chatId":2}]`))
if err != nil {
t.Fatalf("payloadForItem: %v", err)
}
arr, ok := got["subscribers"].(bson.A)
if !ok || len(arr) != 2 {
t.Fatalf("subscribers = %v (%T), want 2-element bson.A", got["subscribers"], got["subscribers"])
}
}
func TestPayloadForItem_LolscheduleLastPushRawString(t *testing.T) {
// last-push date is stored as a bare (non-JSON) string in DynamoDB.
got, err := payloadForItem("lolschedule", "daily_push:last_date", []byte(`2026-06-28`))
if err != nil {
t.Fatalf("payloadForItem: %v", err)
}
if got["date"] != "2026-06-28" {
t.Errorf("date = %v, want 2026-06-28", got["date"])
}
}
func TestPayloadForItem_UnknownNonObjectFailsLoud(t *testing.T) {
if _, err := payloadForItem("coin", "user:1", []byte(`"a-bare-string"`)); err == nil {
t.Error("non-object value with no wrap rule must fail loud")
}
if _, err := payloadForItem("coin", "user:1", []byte(`[1,2,3]`)); err == nil {
t.Error("array value with no wrap rule must fail loud")
}
}
func TestPayloadForItem_ReservedKeyCollision(t *testing.T) {
if _, err := payloadForItem("coin", "user:1", []byte(`{"version":7}`)); err == nil {
t.Error("payload key colliding with reserved root field must fail loud")
}
}
// TestPayloadForItem_CamelCaseFidelity guards the invariant that lets migration
// work: the migrator preserves the original (camelCase) JSON keys, so a typed
// store struct must declare bson tags matching those names. A struct whose bson
// tags drifted to the driver's lowercased default would read these back empty.
func TestPayloadForItem_CamelCaseFidelity(t *testing.T) {
payload, err := payloadForItem("lolschedule", "events", []byte(`{"startTime":"t","gameWins":3,"blockName":"Week 1"}`))
if err != nil {
t.Fatalf("payloadForItem: %v", err)
}
raw, err := bson.Marshal(payload)
if err != nil {
t.Fatalf("bson.Marshal: %v", err)
}
var got struct {
StartTime string `bson:"startTime"`
GameWins int `bson:"gameWins"`
BlockName string `bson:"blockName"`
}
if err := bson.Unmarshal(raw, &got); err != nil {
t.Fatalf("bson.Unmarshal: %v", err)
}
if got.StartTime != "t" || got.GameWins != 3 || got.BlockName != "Week 1" {
t.Fatalf("camelCase round-trip lost data: %+v", got)
}
}
-225
View File
@@ -1,225 +0,0 @@
// Command migrate-dynamo-to-mongo copies every item from the prod DynamoDB KV
// table into MongoDB Atlas using the same document schema the live app writes,
// then verifies per-module parity. It is idempotent (re-runs overwrite by key,
// never duplicate) and read-only on DynamoDB (Scan only).
//
// Usage:
//
// migrate-dynamo-to-mongo [--dynamodb-table miti99bot-data] [--dry-run] [--verify]
//
// Required env: MONGO_URL, MONGO_DATABASE. DynamoDB credentials come from the
// AWS default chain — use a DEDICATED READ-ONLY profile whose policy grants
// exactly `dynamodb:Scan` on the table ARN (nothing else, no write actions).
// For local testing, set DYNAMODB_LOCAL_URL to point at DynamoDB Local.
//
// - default: scan + write every item through the typed Mongo store as a
// flattened native document (payload fields hoisted to the root, no `value`
// envelope) — the exact shape the live app writes — then report per-module
// counts.
// - --dry-run: scan + report counts, write nothing.
// - --verify: tally DynamoDB per pk via Scan vs Mongo CountDocuments per
// collection; print a table and exit non-zero on any mismatch.
//
// Note: each migrated doc gets a fresh updatedAt and version=1; nothing in the
// app reads updatedAt, and --verify compares counts, so this is intentional and
// harmless. The two non-object values (lolschedule subscribers array and
// last-push date) are wrapped into named root fields (see encode.go).
package main
import (
"context"
"flag"
"fmt"
"os"
"sort"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"go.mongodb.org/mongo-driver/v2/bson"
"go.mongodb.org/mongo-driver/v2/mongo"
"github.com/tiennm99/miti99bot/internal/storage"
)
const defaultTable = "miti99bot-data"
func main() {
table := flag.String("dynamodb-table", defaultTable, "source DynamoDB table name")
dryRun := flag.Bool("dry-run", false, "scan and report counts without writing")
verify := flag.Bool("verify", false, "compare per-module counts DynamoDB vs Mongo and exit non-zero on mismatch")
flag.Parse()
if err := run(*table, *dryRun, *verify); err != nil {
fmt.Fprintln(os.Stderr, "error:", err)
os.Exit(1)
}
}
func run(table string, dryRun, verify bool) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
mongoURL := os.Getenv("MONGO_URL")
mongoDB := os.Getenv("MONGO_DATABASE")
if mongoURL == "" || mongoDB == "" {
return fmt.Errorf("MONGO_URL and MONGO_DATABASE are required")
}
ddb, err := storage.NewDynamoDBClient(ctx, storage.DynamoDBEndpointFromEnv())
if err != nil {
return fmt.Errorf("dynamodb client: %w", err)
}
mclient, err := storage.NewMongoClient(ctx, mongoURL)
if err != nil {
return fmt.Errorf("mongo client: %w", err)
}
defer func() { _ = mclient.Disconnect(context.Background()) }()
mdb, err := storage.NewMongoDatabase(mclient, mongoDB)
if err != nil {
return err
}
if verify {
return runVerify(ctx, ddb, mdb, table)
}
return runMigrate(ctx, ddb, mdb, table, dryRun)
}
// item is one decoded DynamoDB KV row.
type item struct {
pk string // module name → collection
sk string // user key → _id
value []byte // raw value bytes
}
// scanTable reads every row from the table via a full Scan (the KV table is
// small) and decodes pk/sk/value. Scan is the ONLY DynamoDB action used, so the
// runner needs just `dynamodb:Scan` on the table ARN.
func scanTable(ctx context.Context, ddb *dynamodb.Client, table string) ([]item, error) {
var items []item
pager := dynamodb.NewScanPaginator(ddb, &dynamodb.ScanInput{TableName: aws.String(table)})
for pager.HasMorePages() {
page, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("scan %s: %w", table, err)
}
for _, raw := range page.Items {
it, err := decodeItem(raw)
if err != nil {
return nil, err
}
items = append(items, it)
}
}
return items, nil
}
func decodeItem(raw map[string]types.AttributeValue) (item, error) {
pk, ok := raw["pk"].(*types.AttributeValueMemberS)
if !ok {
return item{}, fmt.Errorf("item missing string pk: %v", raw)
}
sk, ok := raw["sk"].(*types.AttributeValueMemberS)
if !ok {
return item{}, fmt.Errorf("item %s missing string sk", pk.Value)
}
val, ok := raw["value"].(*types.AttributeValueMemberS)
if !ok {
return item{}, fmt.Errorf("item %s/%s missing string value", pk.Value, sk.Value)
}
return item{pk: pk.Value, sk: sk.Value, value: []byte(val.Value)}, nil
}
// sortedKeys returns the map keys sorted for stable report output.
func sortedKeys(m map[string]int) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// runMigrate scans the table, then (unless dry-run) writes every item through
// the typed Mongo store as a flattened native document (payload fields hoisted
// to the root, no `value` envelope) — the exact shape the live app writes.
// payloadForItem wraps the two non-object values (lolschedule subscribers array
// and last-push date) into named-struct fields. Writing through
// storage.Typed[bson.M] reuses the store's module/key validation and version
// semantics, so a re-run produces no duplicates (upsert by _id) and an invalid
// module name or key fails loud rather than writing data the app cannot load.
func runMigrate(ctx context.Context, ddb *dynamodb.Client, mdb *mongo.Database, table string, dryRun bool) error {
items, err := scanTable(ctx, ddb, table)
if err != nil {
return err
}
counts := map[string]int{}
provider := storage.NewMongoProvider(mdb)
for _, it := range items {
counts[it.pk]++
if dryRun {
continue
}
payload, err := payloadForItem(it.pk, it.sk, it.value)
if err != nil {
return err
}
if err := storage.Typed[bson.M](provider.Collection(it.pk)).Put(ctx, it.sk, payload); err != nil {
return fmt.Errorf("put %s/%s: %w", it.pk, it.sk, err)
}
}
mode := "MIGRATED"
if dryRun {
mode = "DRY-RUN (no writes)"
}
fmt.Printf("%s — %d items across %d modules\n", mode, len(items), len(counts))
for _, pk := range sortedKeys(counts) {
fmt.Printf(" %-20s %d\n", pk, counts[pk])
}
return nil
}
// runVerify tallies DynamoDB per pk via a Scan (so only dynamodb:Scan is
// needed — never Query) and compares against Mongo CountDocuments per
// collection. Prints a table and returns an error on any mismatch so the
// process exits non-zero.
func runVerify(ctx context.Context, ddb *dynamodb.Client, mdb *mongo.Database, table string) error {
items, err := scanTable(ctx, ddb, table)
if err != nil {
return err
}
ddbCounts := map[string]int{}
for _, it := range items {
ddbCounts[it.pk]++
}
mongoCounts := map[string]int{}
for pk := range ddbCounts {
n, err := mdb.Collection(pk).CountDocuments(ctx, bson.M{})
if err != nil {
return fmt.Errorf("count mongo collection %s: %w", pk, err)
}
mongoCounts[pk] = int(n)
}
fmt.Printf("%-20s %10s %10s %s\n", "MODULE", "DYNAMODB", "MONGO", "STATUS")
mismatch := false
for _, pk := range sortedKeys(ddbCounts) {
status := "OK"
if ddbCounts[pk] != mongoCounts[pk] {
status = "MISMATCH"
mismatch = true
}
fmt.Printf("%-20s %10d %10d %s\n", pk, ddbCounts[pk], mongoCounts[pk], status)
}
if mismatch {
return fmt.Errorf("verification failed: per-module counts differ")
}
fmt.Println("verification OK: all per-module counts match")
return nil
}
-158
View File
@@ -1,158 +0,0 @@
package main
import (
"context"
"os"
"reflect"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"go.mongodb.org/mongo-driver/v2/bson"
"go.mongodb.org/mongo-driver/v2/mongo"
"github.com/tiennm99/miti99bot/internal/storage"
)
func TestDecodeItem(t *testing.T) {
raw := map[string]types.AttributeValue{
"pk": &types.AttributeValueMemberS{Value: "coin"},
"sk": &types.AttributeValueMemberS{Value: "user:1"},
"value": &types.AttributeValueMemberS{Value: `{"x":1}`},
}
it, err := decodeItem(raw)
if err != nil {
t.Fatalf("decodeItem: %v", err)
}
if it.pk != "coin" || it.sk != "user:1" || string(it.value) != `{"x":1}` {
t.Errorf("decoded %+v", it)
}
// Missing value attribute is a hard error.
if _, err := decodeItem(map[string]types.AttributeValue{
"pk": &types.AttributeValueMemberS{Value: "coin"},
"sk": &types.AttributeValueMemberS{Value: "user:1"},
}); err == nil {
t.Error("decodeItem with missing value: want error, got nil")
}
}
func TestSortedKeys(t *testing.T) {
got := sortedKeys(map[string]int{"b": 1, "a": 2, "c": 3})
if !reflect.DeepEqual(got, []string{"a", "b", "c"}) {
t.Errorf("sortedKeys = %v", got)
}
}
// TestMigrateAndVerify is the end-to-end gate: seed DynamoDB Local across two
// modules, migrate into Mongo, assert values round-trip byte-identically, and
// confirm --verify reports matching counts. Skips unless BOTH emulators are
// configured.
func TestMigrateAndVerify(t *testing.T) {
ddbURL := os.Getenv("DYNAMODB_LOCAL_URL")
mongoURL := os.Getenv("MONGODB_TEST_URL")
mongoDB := os.Getenv("MONGO_DATABASE")
if ddbURL == "" || mongoURL == "" || mongoDB == "" {
t.Skip("set DYNAMODB_LOCAL_URL, MONGODB_TEST_URL, MONGO_DATABASE to run the migrator e2e test")
}
t.Setenv("AWS_ACCESS_KEY_ID", "test")
t.Setenv("AWS_SECRET_ACCESS_KEY", "test")
t.Setenv("AWS_REGION", "ap-southeast-1")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
ddb, err := storage.NewDynamoDBClient(ctx, ddbURL)
if err != nil {
t.Fatalf("dynamodb client: %v", err)
}
table := "migrate-test"
createTable(t, ctx, ddb, table)
defer func() {
_, _ = ddb.DeleteTable(ctx, &dynamodb.DeleteTableInput{TableName: aws.String(table)})
}()
seed := []item{
{pk: "coin", sk: "user:1", value: []byte(`{"bal":100}`)},
{pk: "coin", sk: "user:2", value: []byte(`{"bal":200}`)},
{pk: "stock", sk: "user:1", value: []byte(`{"vnd":5000}`)},
}
for _, it := range seed {
putDynamoItem(t, ctx, ddb, table, it)
}
// Migrate.
if err := runMigrate(ctx, ddb, mongoDatabase(t, ctx, mongoURL, mongoDB), table, false); err != nil {
t.Fatalf("runMigrate: %v", err)
}
// Re-run must stay idempotent.
if err := runMigrate(ctx, ddb, mongoDatabase(t, ctx, mongoURL, mongoDB), table, false); err != nil {
t.Fatalf("runMigrate re-run: %v", err)
}
// Verify passes.
if err := runVerify(ctx, ddb, mongoDatabase(t, ctx, mongoURL, mongoDB), table); err != nil {
t.Fatalf("runVerify: %v", err)
}
// Spot-check the migrated value round-trips through the typed store as a
// flattened native doc: bal hoisted to the root, preserved as int64.
db := mongoDatabase(t, ctx, mongoURL, mongoDB)
provider := storage.NewMongoProvider(db)
got, _, err := storage.Typed[bson.M](provider.Collection("coin")).Get(ctx, "user:1")
if err != nil {
t.Fatalf("Get migrated value: %v", err)
}
if got["bal"] != int64(100) {
t.Errorf("migrated value bal = %v (%T), want int64(100)", got["bal"], got["bal"])
}
}
func mongoDatabase(t *testing.T, ctx context.Context, uri, db string) *mongo.Database {
t.Helper()
client, err := storage.NewMongoClient(ctx, uri)
if err != nil {
t.Fatalf("NewMongoClient: %v", err)
}
t.Cleanup(func() { _ = client.Disconnect(context.Background()) })
mdb, err := storage.NewMongoDatabase(client, db)
if err != nil {
t.Fatalf("NewMongoDatabase: %v", err)
}
return mdb
}
func createTable(t *testing.T, ctx context.Context, c *dynamodb.Client, table string) {
t.Helper()
_, err := c.CreateTable(ctx, &dynamodb.CreateTableInput{
TableName: aws.String(table),
BillingMode: types.BillingModePayPerRequest,
AttributeDefinitions: []types.AttributeDefinition{
{AttributeName: aws.String("pk"), AttributeType: types.ScalarAttributeTypeS},
{AttributeName: aws.String("sk"), AttributeType: types.ScalarAttributeTypeS},
},
KeySchema: []types.KeySchemaElement{
{AttributeName: aws.String("pk"), KeyType: types.KeyTypeHash},
{AttributeName: aws.String("sk"), KeyType: types.KeyTypeRange},
},
})
if err != nil {
t.Fatalf("CreateTable: %v", err)
}
}
func putDynamoItem(t *testing.T, ctx context.Context, c *dynamodb.Client, table string, it item) {
t.Helper()
_, err := c.PutItem(ctx, &dynamodb.PutItemInput{
TableName: aws.String(table),
Item: map[string]types.AttributeValue{
"pk": &types.AttributeValueMemberS{Value: it.pk},
"sk": &types.AttributeValueMemberS{Value: it.sk},
"value": &types.AttributeValueMemberS{Value: string(it.value)},
},
})
if err != nil {
t.Fatalf("PutItem: %v", err)
}
}
+41 -118
View File
@@ -12,9 +12,6 @@ import (
"syscall"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/ssm"
"github.com/tiennm99/miti99bot/internal/ai"
"github.com/tiennm99/miti99bot/internal/cron"
"github.com/tiennm99/miti99bot/internal/deploynotify"
@@ -79,18 +76,11 @@ func factories() map[string]modules.Factory {
// container; 10s leaves headroom without hiding a wedged cluster.
const mongodbInitTimeout = 10 * time.Second
// ssmInitTimeout caps cold-start secret resolution. Secrets are fetched once
// at startup from Parameter Store when *_PARAMETER_NAME env vars are set.
const ssmInitTimeout = 5 * time.Second
func main() {
rootCtx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
cfg := loadConfig()
if err := resolveSSMSecrets(rootCtx, &cfg); err != nil {
log.Fatal("ssm secret resolution failed", "err", err)
}
if cfg.TelegramBotToken == "" {
log.Fatal("missing required env", "key", "TELEGRAM_BOT_TOKEN")
}
@@ -144,10 +134,8 @@ func main() {
"commands", len(reg.AllCommands),
"crons", len(reg.Crons()))
// In-process cron scheduler. Replaces EventBridge Scheduler off AWS; runs
// unconditionally so the long-lived container fires module crons (e.g. the
// lolschedule daily push) on their Schedule. Cutover safety comes from
// ordering + the per-date idempotency guard, not from a gate.
// In-process cron scheduler runs unconditionally so the long-lived container
// fires module crons (e.g. the lolschedule daily push) on their Schedule.
stopCron, err := cron.Run(rootCtx, reg)
if err != nil {
log.Fatal("cron scheduler init failed", "err", err)
@@ -158,10 +146,10 @@ func main() {
log.Warn("OWNER_ID unset; all Private + Protected commands will be denied")
}
// Clear any webhook left over from the AWS deployment at startup, before the
// owner DM and before polling. getUpdates (long polling, below) returns HTTP
// 409 while a webhook is set, so a stuck webhook silently breaks the bot.
// Best-effort, one shot: a real failure here is logged, not retried.
// Clear any existing webhook at startup before the owner DM and before
// polling. getUpdates returns HTTP 409 while a webhook is set, so a stuck
// webhook silently breaks the bot. Best-effort, one shot: a real failure
// here is logged, not retried.
if err := telegram.DeleteWebhook(rootCtx, cfg.TelegramBotToken); err != nil {
log.Warn("deleteWebhook failed; getUpdates may 409 if a webhook is set", "err", err)
} else {
@@ -218,8 +206,7 @@ func main() {
//
// The self-host default is mongodb (just set MONGO_URL + MONGO_DATABASE — no
// KV_PROVIDER needed). The memory backend is for tests and local no-database
// runs (MODULES=). DynamoDB is no longer a runtime backend — it survives only
// as the one-off migration source (cmd/migrate-dynamo-to-mongo).
// runs (MODULES=).
//
// Returned closer is always non-nil and safe to call exactly once.
func buildProvider(ctx context.Context, cfg config) (storage.Provider, func(), error) {
@@ -265,33 +252,28 @@ func buildProvider(ctx context.Context, cfg config) (storage.Provider, func(), e
return storage.NewMongoProvider(db), closer, nil
default:
// DynamoDB is no longer a runtime backend — it survives only as the
// one-off migration source (cmd/migrate-dynamo-to-mongo).
return nil, func() {}, fmt.Errorf("unknown KV_PROVIDER %q (want memory|mongodb)", backend)
}
}
type config struct {
Port string
TelegramBotToken string
SourceCommit string // Coolify-injected commit SHA (runtime env) for deploynotify
GeminiAPIKey string
GoldPriceAPIURL string
GoldFXAPIURL string
GoldVNAppAPIURL string
GoldVNAppAPIKey string
CoinBinanceAPIURL string
CoinCoinbaseAPIURL string
CoinCoinGeckoAPIURL string
Modules []string
BotOwnerID int64
AdminUserIDs map[int64]bool
KVProvider string // empty = auto-detect; or "memory"|"mongodb"
MongoURL string // required when KVProvider=mongodb (Atlas SRV connection string; SECRET — never log)
MongoDatabase string // required when KVProvider=mongodb
TelegramBotTokenParam string
GeminiAPIKeyParam string
GoldVNAppAPIKeyParam string
Port string
TelegramBotToken string
SourceCommit string // Coolify-injected commit SHA (runtime env) for deploynotify
GeminiAPIKey string
GoldPriceAPIURL string
GoldFXAPIURL string
GoldVNAppAPIURL string
GoldVNAppAPIKey string
CoinBinanceAPIURL string
CoinCoinbaseAPIURL string
CoinCoinGeckoAPIURL string
Modules []string
BotOwnerID int64
AdminUserIDs map[int64]bool
KVProvider string // empty = auto-detect; or "memory"|"mongodb"
MongoURL string // required when KVProvider=mongodb (Atlas SRV connection string; SECRET — never log)
MongoDatabase string // required when KVProvider=mongodb
}
func loadConfig() config {
@@ -312,85 +294,26 @@ func loadConfig() config {
log.Fatal("invalid PORT", "value", port)
}
return config{
Port: port,
TelegramBotToken: envMap["TELEGRAM_BOT_TOKEN"],
SourceCommit: envMap["SOURCE_COMMIT"],
GeminiAPIKey: envMap["GEMINI_API_KEY"],
GoldPriceAPIURL: envMap["GOLD_PRICE_API_URL"],
GoldFXAPIURL: envMap["GOLD_FX_API_URL"],
GoldVNAppAPIURL: envMap["GOLD_VNAPP_API_URL"],
GoldVNAppAPIKey: envMap["GOLD_VNAPP_API_KEY"],
CoinBinanceAPIURL: envMap["COIN_BINANCE_API_URL"],
CoinCoinbaseAPIURL: envMap["COIN_COINBASE_API_URL"],
CoinCoinGeckoAPIURL: envMap["COIN_COINGECKO_API_URL"],
Modules: splitCSV(envMap["MODULES"]),
BotOwnerID: parseInt64(envMap["OWNER_ID"]),
AdminUserIDs: parseInt64Set(envMap["ADMIN_IDS"]),
KVProvider: envMap["KV_PROVIDER"],
MongoURL: envMap["MONGO_URL"],
MongoDatabase: envMap["MONGO_DATABASE"],
TelegramBotTokenParam: strings.TrimSpace(envMap["TELEGRAM_BOT_TOKEN_PARAMETER_NAME"]),
GeminiAPIKeyParam: strings.TrimSpace(envMap["GEMINI_API_KEY_PARAMETER_NAME"]),
GoldVNAppAPIKeyParam: strings.TrimSpace(envMap["GOLD_VNAPP_API_KEY_PARAMETER_NAME"]),
Port: port,
TelegramBotToken: envMap["TELEGRAM_BOT_TOKEN"],
SourceCommit: envMap["SOURCE_COMMIT"],
GeminiAPIKey: envMap["GEMINI_API_KEY"],
GoldPriceAPIURL: envMap["GOLD_PRICE_API_URL"],
GoldFXAPIURL: envMap["GOLD_FX_API_URL"],
GoldVNAppAPIURL: envMap["GOLD_VNAPP_API_URL"],
GoldVNAppAPIKey: envMap["GOLD_VNAPP_API_KEY"],
CoinBinanceAPIURL: envMap["COIN_BINANCE_API_URL"],
CoinCoinbaseAPIURL: envMap["COIN_COINBASE_API_URL"],
CoinCoinGeckoAPIURL: envMap["COIN_COINGECKO_API_URL"],
Modules: splitCSV(envMap["MODULES"]),
BotOwnerID: parseInt64(envMap["OWNER_ID"]),
AdminUserIDs: parseInt64Set(envMap["ADMIN_IDS"]),
KVProvider: envMap["KV_PROVIDER"],
MongoURL: envMap["MONGO_URL"],
MongoDatabase: envMap["MONGO_DATABASE"],
}
}
func resolveSSMSecrets(ctx context.Context, cfg *config) error {
bindings := []struct {
name string
target *string
}{
{name: cfg.TelegramBotTokenParam, target: &cfg.TelegramBotToken},
{name: cfg.GeminiAPIKeyParam, target: &cfg.GeminiAPIKey},
{name: cfg.GoldVNAppAPIKeyParam, target: &cfg.GoldVNAppAPIKey},
}
targetsByName := map[string][]*string{}
names := make([]string, 0, len(bindings))
for _, b := range bindings {
if b.name == "" || *b.target != "" {
continue
}
if _, ok := targetsByName[b.name]; !ok {
names = append(names, b.name)
}
targetsByName[b.name] = append(targetsByName[b.name], b.target)
}
if len(names) == 0 {
return nil
}
initCtx, cancel := context.WithTimeout(ctx, ssmInitTimeout)
defer cancel()
awsCfg, err := awsconfig.LoadDefaultConfig(initCtx, awsconfig.WithHTTPClient(&http.Client{
Timeout: ssmInitTimeout,
}))
if err != nil {
return fmt.Errorf("load AWS config: %w", err)
}
client := ssm.NewFromConfig(awsCfg)
out, err := client.GetParameters(initCtx, &ssm.GetParametersInput{
Names: names,
WithDecryption: aws.Bool(true),
})
if err != nil {
return fmt.Errorf("get parameters: %w", err)
}
if len(out.InvalidParameters) > 0 {
return fmt.Errorf("missing SSM parameters: %s", strings.Join(out.InvalidParameters, ","))
}
for _, p := range out.Parameters {
name := aws.ToString(p.Name)
value := aws.ToString(p.Value)
for _, target := range targetsByName[name] {
*target = value
}
}
log.Info("loaded secrets from ssm", "count", len(out.Parameters))
return nil
}
func exportOptionalEnv(key, value string) {
if strings.TrimSpace(value) == "" {
return