mirror of
https://github.com/tiennm99/miti99bot.git
synced 2026-08-23 20:26:24 +00:00
refactor(deploy): remove retired aws support
This commit is contained in:
@@ -1,83 +0,0 @@
|
||||
# migrate-dynamo-to-mongo
|
||||
|
||||
One-off CLI that copies every item from the prod DynamoDB KV table
|
||||
(`miti99bot-data`) into MongoDB Atlas using the exact document schema the live
|
||||
app writes, then verifies per-module parity. Idempotent and re-runnable.
|
||||
|
||||
## What it does
|
||||
|
||||
- Full-table `Scan` of DynamoDB (the table is a small KV) → group by `pk`.
|
||||
- Writes each item through the typed Mongo store (`storage.Typed[bson.M]`) as a
|
||||
**flattened native document** — the value's JSON fields are hoisted to the
|
||||
document root alongside `_id`/`version`/`updatedAt`, with no `value` envelope.
|
||||
This is the exact shape the running bot writes, so the app reads migrated docs
|
||||
directly. Integers keep int64 fidelity (decoded with `UseNumber`).
|
||||
- The two non-object values are wrapped into named root fields to match the
|
||||
module's typed shape: lolschedule `subscribers` (a JSON array) → `{subscribers: [...]}`
|
||||
and `daily_push:last_date` (a bare date string) → `{date: "..."}`. Any other
|
||||
non-object value fails loud so a missing wrap rule is obvious (see `encode.go`).
|
||||
- Writing through the store validates the module/collection name and key and
|
||||
upserts by `_id`, so a re-run produces no duplicates and bad input fails loud.
|
||||
|
||||
| DynamoDB | MongoDB |
|
||||
|---|---|
|
||||
| `pk` (module name) | collection name |
|
||||
| `sk` (user key) | document `_id` |
|
||||
| `value` (JSON object) | payload fields hoisted to the document root (no `value` field) |
|
||||
| `value` (array/scalar, lolschedule) | wrapped in a named root field (`subscribers` / `date`) |
|
||||
| — | `version` = 1, `updatedAt` = migration time (write-only; nothing reads it) |
|
||||
|
||||
## Usage
|
||||
|
||||
```sh
|
||||
export MONGO_URL='mongodb+srv://botuser:PASS@cluster0.xxxxx.mongodb.net/?retryWrites=true&w=majority'
|
||||
export MONGO_DATABASE=miti99bot
|
||||
export AWS_PROFILE=miti99bot-migrate # READ-ONLY profile (see IAM below)
|
||||
|
||||
# 1. Dry run — report per-module counts, write nothing.
|
||||
go run ./cmd/migrate-dynamo-to-mongo --dry-run
|
||||
# or: make migrate-dynamo-to-mongo DRY_RUN=1
|
||||
|
||||
# 2. Real migration.
|
||||
go run ./cmd/migrate-dynamo-to-mongo
|
||||
# or: make migrate-dynamo-to-mongo
|
||||
|
||||
# 3. Verify — per-module counts must match; exits non-zero on mismatch.
|
||||
go run ./cmd/migrate-dynamo-to-mongo --verify
|
||||
# or: make migrate-verify
|
||||
```
|
||||
|
||||
Flags: `--dynamodb-table` (default `miti99bot-data`), `--dry-run`, `--verify`.
|
||||
|
||||
For a local end-to-end test, point at DynamoDB Local + a local Mongo:
|
||||
|
||||
```sh
|
||||
DYNAMODB_LOCAL_URL=http://localhost:8001 \
|
||||
MONGODB_TEST_URL=mongodb://127.0.0.1:27017 \
|
||||
MONGO_DATABASE=migrate_test \
|
||||
go test ./cmd/migrate-dynamo-to-mongo/ -run TestMigrateAndVerify
|
||||
```
|
||||
|
||||
## IAM — least privilege
|
||||
|
||||
The runner needs **exactly** `dynamodb:Scan` on the table ARN and nothing else.
|
||||
Verify uses a Scan tally (not Query), so no `dynamodb:Query` is needed; there
|
||||
are **no write actions on the source**, enforcing the read-only requirement and
|
||||
removing the destructive-credential foot-gun.
|
||||
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": "dynamodb:Scan",
|
||||
"Resource": "arn:aws:dynamodb:ap-southeast-1:225603493174:table/miti99bot-data"
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
## Cutover runbook
|
||||
|
||||
The full zero-loss cutover (disable EventBridge → `deleteWebhook` → migrate →
|
||||
verify → start the polling container) lives in
|
||||
[`docs/deploy-coolify-selfhosted.md`](../../docs/deploy-coolify-selfhosted.md#cutover-runbook).
|
||||
@@ -1,109 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
)
|
||||
|
||||
// reservedRootFields are owned by the storage layer (see internal/storage); a
|
||||
// flattened object payload must not collide with them.
|
||||
var reservedRootFields = map[string]bool{"_id": true, "version": true, "updatedAt": true}
|
||||
|
||||
// wrapRule names the root field a non-object DynamoDB value must be wrapped in
|
||||
// to match the typed Mongo store's named-struct shape. rawString treats the
|
||||
// DynamoDB value as a bare (non-JSON) string; otherwise it is JSON-decoded.
|
||||
type wrapRule struct {
|
||||
field string
|
||||
rawString bool
|
||||
}
|
||||
|
||||
// wrapRules covers the only KV entries whose value is not a JSON object. They
|
||||
// mirror the named-struct wrappers the lolschedule module persists:
|
||||
// - subscribers: a JSON array → {subscribers: [...]}
|
||||
// - daily_push:last_date: a bare date string → {date: "..."}
|
||||
//
|
||||
// Any other non-object value fails loud in payloadForItem so a missing rule is
|
||||
// obvious rather than silently dropped.
|
||||
var wrapRules = map[[2]string]wrapRule{
|
||||
{"lolschedule", "subscribers"}: {field: "subscribers"},
|
||||
{"lolschedule", "daily_push:last_date"}: {field: "date", rawString: true},
|
||||
}
|
||||
|
||||
// payloadForItem converts one migrated KV row's value into the flattened payload
|
||||
// map the typed Mongo store stores at the document root. The store adds _id,
|
||||
// version, and updatedAt; this returns only the payload fields.
|
||||
func payloadForItem(module, key string, value []byte) (bson.M, error) {
|
||||
if rule, ok := wrapRules[[2]string{module, key}]; ok {
|
||||
if rule.rawString {
|
||||
return bson.M{rule.field: string(value)}, nil
|
||||
}
|
||||
decoded, err := decodeJSONNumber(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s/%s: decode value: %w", module, key, err)
|
||||
}
|
||||
return bson.M{rule.field: decoded}, nil
|
||||
}
|
||||
|
||||
decoded, err := decodeJSONNumber(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s/%s: decode value: %w", module, key, err)
|
||||
}
|
||||
obj, ok := decoded.(bson.M)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s/%s: value is not a JSON object (type %T) and has no wrap rule — add one to wrapRules", module, key, decoded)
|
||||
}
|
||||
for k := range obj {
|
||||
if reservedRootFields[k] {
|
||||
return nil, fmt.Errorf("%s/%s: payload key %q collides with a reserved root field — add a wrap rule", module, key, k)
|
||||
}
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
// decodeJSONNumber decodes JSON into a BSON-native value, preserving integral
|
||||
// numbers as int64 (UseNumber) so migrated numbers keep int64 fidelity, matching
|
||||
// the live store's value codec.
|
||||
func decodeJSONNumber(value []byte) (any, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(value))
|
||||
dec.UseNumber()
|
||||
var v any
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if dec.More() {
|
||||
return nil, fmt.Errorf("trailing data after JSON value")
|
||||
}
|
||||
return numberToBSON(v), nil
|
||||
}
|
||||
|
||||
// numberToBSON walks a json.Unmarshal(UseNumber) tree into bson.M/bson.A,
|
||||
// converting json.Number to int64 when integral, else float64.
|
||||
func numberToBSON(v any) any {
|
||||
switch t := v.(type) {
|
||||
case map[string]any:
|
||||
m := make(bson.M, len(t))
|
||||
for k, val := range t {
|
||||
m[k] = numberToBSON(val)
|
||||
}
|
||||
return m
|
||||
case []any:
|
||||
a := make(bson.A, len(t))
|
||||
for i, val := range t {
|
||||
a[i] = numberToBSON(val)
|
||||
}
|
||||
return a
|
||||
case json.Number:
|
||||
if i, err := t.Int64(); err == nil {
|
||||
return i
|
||||
}
|
||||
if f, err := t.Float64(); err == nil {
|
||||
return f
|
||||
}
|
||||
return t.String()
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
)
|
||||
|
||||
func TestPayloadForItem_Object(t *testing.T) {
|
||||
got, err := payloadForItem("coin", "user:1", []byte(`{"bal":100,"meta":{"createdAt":5}}`))
|
||||
if err != nil {
|
||||
t.Fatalf("payloadForItem: %v", err)
|
||||
}
|
||||
if got["bal"] != int64(100) {
|
||||
t.Errorf("bal = %v (%T), want int64(100)", got["bal"], got["bal"])
|
||||
}
|
||||
if _, ok := got["value"]; ok {
|
||||
t.Error("payload must not contain a 'value' envelope field")
|
||||
}
|
||||
meta, ok := got["meta"].(bson.M)
|
||||
if !ok || meta["createdAt"] != int64(5) {
|
||||
t.Errorf("nested meta = %v, want {createdAt: int64(5)}", got["meta"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPayloadForItem_LolscheduleSubscribers(t *testing.T) {
|
||||
got, err := payloadForItem("lolschedule", "subscribers", []byte(`[{"chatId":1},{"chatId":2}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("payloadForItem: %v", err)
|
||||
}
|
||||
arr, ok := got["subscribers"].(bson.A)
|
||||
if !ok || len(arr) != 2 {
|
||||
t.Fatalf("subscribers = %v (%T), want 2-element bson.A", got["subscribers"], got["subscribers"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPayloadForItem_LolscheduleLastPushRawString(t *testing.T) {
|
||||
// last-push date is stored as a bare (non-JSON) string in DynamoDB.
|
||||
got, err := payloadForItem("lolschedule", "daily_push:last_date", []byte(`2026-06-28`))
|
||||
if err != nil {
|
||||
t.Fatalf("payloadForItem: %v", err)
|
||||
}
|
||||
if got["date"] != "2026-06-28" {
|
||||
t.Errorf("date = %v, want 2026-06-28", got["date"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPayloadForItem_UnknownNonObjectFailsLoud(t *testing.T) {
|
||||
if _, err := payloadForItem("coin", "user:1", []byte(`"a-bare-string"`)); err == nil {
|
||||
t.Error("non-object value with no wrap rule must fail loud")
|
||||
}
|
||||
if _, err := payloadForItem("coin", "user:1", []byte(`[1,2,3]`)); err == nil {
|
||||
t.Error("array value with no wrap rule must fail loud")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPayloadForItem_ReservedKeyCollision(t *testing.T) {
|
||||
if _, err := payloadForItem("coin", "user:1", []byte(`{"version":7}`)); err == nil {
|
||||
t.Error("payload key colliding with reserved root field must fail loud")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPayloadForItem_CamelCaseFidelity guards the invariant that lets migration
|
||||
// work: the migrator preserves the original (camelCase) JSON keys, so a typed
|
||||
// store struct must declare bson tags matching those names. A struct whose bson
|
||||
// tags drifted to the driver's lowercased default would read these back empty.
|
||||
func TestPayloadForItem_CamelCaseFidelity(t *testing.T) {
|
||||
payload, err := payloadForItem("lolschedule", "events", []byte(`{"startTime":"t","gameWins":3,"blockName":"Week 1"}`))
|
||||
if err != nil {
|
||||
t.Fatalf("payloadForItem: %v", err)
|
||||
}
|
||||
raw, err := bson.Marshal(payload)
|
||||
if err != nil {
|
||||
t.Fatalf("bson.Marshal: %v", err)
|
||||
}
|
||||
var got struct {
|
||||
StartTime string `bson:"startTime"`
|
||||
GameWins int `bson:"gameWins"`
|
||||
BlockName string `bson:"blockName"`
|
||||
}
|
||||
if err := bson.Unmarshal(raw, &got); err != nil {
|
||||
t.Fatalf("bson.Unmarshal: %v", err)
|
||||
}
|
||||
if got.StartTime != "t" || got.GameWins != 3 || got.BlockName != "Week 1" {
|
||||
t.Fatalf("camelCase round-trip lost data: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
// Command migrate-dynamo-to-mongo copies every item from the prod DynamoDB KV
|
||||
// table into MongoDB Atlas using the same document schema the live app writes,
|
||||
// then verifies per-module parity. It is idempotent (re-runs overwrite by key,
|
||||
// never duplicate) and read-only on DynamoDB (Scan only).
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// migrate-dynamo-to-mongo [--dynamodb-table miti99bot-data] [--dry-run] [--verify]
|
||||
//
|
||||
// Required env: MONGO_URL, MONGO_DATABASE. DynamoDB credentials come from the
|
||||
// AWS default chain — use a DEDICATED READ-ONLY profile whose policy grants
|
||||
// exactly `dynamodb:Scan` on the table ARN (nothing else, no write actions).
|
||||
// For local testing, set DYNAMODB_LOCAL_URL to point at DynamoDB Local.
|
||||
//
|
||||
// - default: scan + write every item through the typed Mongo store as a
|
||||
// flattened native document (payload fields hoisted to the root, no `value`
|
||||
// envelope) — the exact shape the live app writes — then report per-module
|
||||
// counts.
|
||||
// - --dry-run: scan + report counts, write nothing.
|
||||
// - --verify: tally DynamoDB per pk via Scan vs Mongo CountDocuments per
|
||||
// collection; print a table and exit non-zero on any mismatch.
|
||||
//
|
||||
// Note: each migrated doc gets a fresh updatedAt and version=1; nothing in the
|
||||
// app reads updatedAt, and --verify compares counts, so this is intentional and
|
||||
// harmless. The two non-object values (lolschedule subscribers array and
|
||||
// last-push date) are wrapped into named root fields (see encode.go).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
|
||||
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||
|
||||
"github.com/tiennm99/miti99bot/internal/storage"
|
||||
)
|
||||
|
||||
const defaultTable = "miti99bot-data"
|
||||
|
||||
func main() {
|
||||
table := flag.String("dynamodb-table", defaultTable, "source DynamoDB table name")
|
||||
dryRun := flag.Bool("dry-run", false, "scan and report counts without writing")
|
||||
verify := flag.Bool("verify", false, "compare per-module counts DynamoDB vs Mongo and exit non-zero on mismatch")
|
||||
flag.Parse()
|
||||
|
||||
if err := run(*table, *dryRun, *verify); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(table string, dryRun, verify bool) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
mongoURL := os.Getenv("MONGO_URL")
|
||||
mongoDB := os.Getenv("MONGO_DATABASE")
|
||||
if mongoURL == "" || mongoDB == "" {
|
||||
return fmt.Errorf("MONGO_URL and MONGO_DATABASE are required")
|
||||
}
|
||||
|
||||
ddb, err := storage.NewDynamoDBClient(ctx, storage.DynamoDBEndpointFromEnv())
|
||||
if err != nil {
|
||||
return fmt.Errorf("dynamodb client: %w", err)
|
||||
}
|
||||
|
||||
mclient, err := storage.NewMongoClient(ctx, mongoURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mongo client: %w", err)
|
||||
}
|
||||
defer func() { _ = mclient.Disconnect(context.Background()) }()
|
||||
mdb, err := storage.NewMongoDatabase(mclient, mongoDB)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if verify {
|
||||
return runVerify(ctx, ddb, mdb, table)
|
||||
}
|
||||
return runMigrate(ctx, ddb, mdb, table, dryRun)
|
||||
}
|
||||
|
||||
// item is one decoded DynamoDB KV row.
|
||||
type item struct {
|
||||
pk string // module name → collection
|
||||
sk string // user key → _id
|
||||
value []byte // raw value bytes
|
||||
}
|
||||
|
||||
// scanTable reads every row from the table via a full Scan (the KV table is
|
||||
// small) and decodes pk/sk/value. Scan is the ONLY DynamoDB action used, so the
|
||||
// runner needs just `dynamodb:Scan` on the table ARN.
|
||||
func scanTable(ctx context.Context, ddb *dynamodb.Client, table string) ([]item, error) {
|
||||
var items []item
|
||||
pager := dynamodb.NewScanPaginator(ddb, &dynamodb.ScanInput{TableName: aws.String(table)})
|
||||
for pager.HasMorePages() {
|
||||
page, err := pager.NextPage(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan %s: %w", table, err)
|
||||
}
|
||||
for _, raw := range page.Items {
|
||||
it, err := decodeItem(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, it)
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func decodeItem(raw map[string]types.AttributeValue) (item, error) {
|
||||
pk, ok := raw["pk"].(*types.AttributeValueMemberS)
|
||||
if !ok {
|
||||
return item{}, fmt.Errorf("item missing string pk: %v", raw)
|
||||
}
|
||||
sk, ok := raw["sk"].(*types.AttributeValueMemberS)
|
||||
if !ok {
|
||||
return item{}, fmt.Errorf("item %s missing string sk", pk.Value)
|
||||
}
|
||||
val, ok := raw["value"].(*types.AttributeValueMemberS)
|
||||
if !ok {
|
||||
return item{}, fmt.Errorf("item %s/%s missing string value", pk.Value, sk.Value)
|
||||
}
|
||||
return item{pk: pk.Value, sk: sk.Value, value: []byte(val.Value)}, nil
|
||||
}
|
||||
|
||||
// sortedKeys returns the map keys sorted for stable report output.
|
||||
func sortedKeys(m map[string]int) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// runMigrate scans the table, then (unless dry-run) writes every item through
|
||||
// the typed Mongo store as a flattened native document (payload fields hoisted
|
||||
// to the root, no `value` envelope) — the exact shape the live app writes.
|
||||
// payloadForItem wraps the two non-object values (lolschedule subscribers array
|
||||
// and last-push date) into named-struct fields. Writing through
|
||||
// storage.Typed[bson.M] reuses the store's module/key validation and version
|
||||
// semantics, so a re-run produces no duplicates (upsert by _id) and an invalid
|
||||
// module name or key fails loud rather than writing data the app cannot load.
|
||||
func runMigrate(ctx context.Context, ddb *dynamodb.Client, mdb *mongo.Database, table string, dryRun bool) error {
|
||||
items, err := scanTable(ctx, ddb, table)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
counts := map[string]int{}
|
||||
provider := storage.NewMongoProvider(mdb)
|
||||
for _, it := range items {
|
||||
counts[it.pk]++
|
||||
if dryRun {
|
||||
continue
|
||||
}
|
||||
payload, err := payloadForItem(it.pk, it.sk, it.value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := storage.Typed[bson.M](provider.Collection(it.pk)).Put(ctx, it.sk, payload); err != nil {
|
||||
return fmt.Errorf("put %s/%s: %w", it.pk, it.sk, err)
|
||||
}
|
||||
}
|
||||
|
||||
mode := "MIGRATED"
|
||||
if dryRun {
|
||||
mode = "DRY-RUN (no writes)"
|
||||
}
|
||||
fmt.Printf("%s — %d items across %d modules\n", mode, len(items), len(counts))
|
||||
for _, pk := range sortedKeys(counts) {
|
||||
fmt.Printf(" %-20s %d\n", pk, counts[pk])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runVerify tallies DynamoDB per pk via a Scan (so only dynamodb:Scan is
|
||||
// needed — never Query) and compares against Mongo CountDocuments per
|
||||
// collection. Prints a table and returns an error on any mismatch so the
|
||||
// process exits non-zero.
|
||||
func runVerify(ctx context.Context, ddb *dynamodb.Client, mdb *mongo.Database, table string) error {
|
||||
items, err := scanTable(ctx, ddb, table)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ddbCounts := map[string]int{}
|
||||
for _, it := range items {
|
||||
ddbCounts[it.pk]++
|
||||
}
|
||||
|
||||
mongoCounts := map[string]int{}
|
||||
for pk := range ddbCounts {
|
||||
n, err := mdb.Collection(pk).CountDocuments(ctx, bson.M{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("count mongo collection %s: %w", pk, err)
|
||||
}
|
||||
mongoCounts[pk] = int(n)
|
||||
}
|
||||
|
||||
fmt.Printf("%-20s %10s %10s %s\n", "MODULE", "DYNAMODB", "MONGO", "STATUS")
|
||||
mismatch := false
|
||||
for _, pk := range sortedKeys(ddbCounts) {
|
||||
status := "OK"
|
||||
if ddbCounts[pk] != mongoCounts[pk] {
|
||||
status = "MISMATCH"
|
||||
mismatch = true
|
||||
}
|
||||
fmt.Printf("%-20s %10d %10d %s\n", pk, ddbCounts[pk], mongoCounts[pk], status)
|
||||
}
|
||||
if mismatch {
|
||||
return fmt.Errorf("verification failed: per-module counts differ")
|
||||
}
|
||||
fmt.Println("verification OK: all per-module counts match")
|
||||
return nil
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
|
||||
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||
|
||||
"github.com/tiennm99/miti99bot/internal/storage"
|
||||
)
|
||||
|
||||
func TestDecodeItem(t *testing.T) {
|
||||
raw := map[string]types.AttributeValue{
|
||||
"pk": &types.AttributeValueMemberS{Value: "coin"},
|
||||
"sk": &types.AttributeValueMemberS{Value: "user:1"},
|
||||
"value": &types.AttributeValueMemberS{Value: `{"x":1}`},
|
||||
}
|
||||
it, err := decodeItem(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeItem: %v", err)
|
||||
}
|
||||
if it.pk != "coin" || it.sk != "user:1" || string(it.value) != `{"x":1}` {
|
||||
t.Errorf("decoded %+v", it)
|
||||
}
|
||||
|
||||
// Missing value attribute is a hard error.
|
||||
if _, err := decodeItem(map[string]types.AttributeValue{
|
||||
"pk": &types.AttributeValueMemberS{Value: "coin"},
|
||||
"sk": &types.AttributeValueMemberS{Value: "user:1"},
|
||||
}); err == nil {
|
||||
t.Error("decodeItem with missing value: want error, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortedKeys(t *testing.T) {
|
||||
got := sortedKeys(map[string]int{"b": 1, "a": 2, "c": 3})
|
||||
if !reflect.DeepEqual(got, []string{"a", "b", "c"}) {
|
||||
t.Errorf("sortedKeys = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMigrateAndVerify is the end-to-end gate: seed DynamoDB Local across two
|
||||
// modules, migrate into Mongo, assert values round-trip byte-identically, and
|
||||
// confirm --verify reports matching counts. Skips unless BOTH emulators are
|
||||
// configured.
|
||||
func TestMigrateAndVerify(t *testing.T) {
|
||||
ddbURL := os.Getenv("DYNAMODB_LOCAL_URL")
|
||||
mongoURL := os.Getenv("MONGODB_TEST_URL")
|
||||
mongoDB := os.Getenv("MONGO_DATABASE")
|
||||
if ddbURL == "" || mongoURL == "" || mongoDB == "" {
|
||||
t.Skip("set DYNAMODB_LOCAL_URL, MONGODB_TEST_URL, MONGO_DATABASE to run the migrator e2e test")
|
||||
}
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "test")
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "test")
|
||||
t.Setenv("AWS_REGION", "ap-southeast-1")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ddb, err := storage.NewDynamoDBClient(ctx, ddbURL)
|
||||
if err != nil {
|
||||
t.Fatalf("dynamodb client: %v", err)
|
||||
}
|
||||
table := "migrate-test"
|
||||
createTable(t, ctx, ddb, table)
|
||||
defer func() {
|
||||
_, _ = ddb.DeleteTable(ctx, &dynamodb.DeleteTableInput{TableName: aws.String(table)})
|
||||
}()
|
||||
|
||||
seed := []item{
|
||||
{pk: "coin", sk: "user:1", value: []byte(`{"bal":100}`)},
|
||||
{pk: "coin", sk: "user:2", value: []byte(`{"bal":200}`)},
|
||||
{pk: "stock", sk: "user:1", value: []byte(`{"vnd":5000}`)},
|
||||
}
|
||||
for _, it := range seed {
|
||||
putDynamoItem(t, ctx, ddb, table, it)
|
||||
}
|
||||
|
||||
// Migrate.
|
||||
if err := runMigrate(ctx, ddb, mongoDatabase(t, ctx, mongoURL, mongoDB), table, false); err != nil {
|
||||
t.Fatalf("runMigrate: %v", err)
|
||||
}
|
||||
// Re-run must stay idempotent.
|
||||
if err := runMigrate(ctx, ddb, mongoDatabase(t, ctx, mongoURL, mongoDB), table, false); err != nil {
|
||||
t.Fatalf("runMigrate re-run: %v", err)
|
||||
}
|
||||
// Verify passes.
|
||||
if err := runVerify(ctx, ddb, mongoDatabase(t, ctx, mongoURL, mongoDB), table); err != nil {
|
||||
t.Fatalf("runVerify: %v", err)
|
||||
}
|
||||
|
||||
// Spot-check the migrated value round-trips through the typed store as a
|
||||
// flattened native doc: bal hoisted to the root, preserved as int64.
|
||||
db := mongoDatabase(t, ctx, mongoURL, mongoDB)
|
||||
provider := storage.NewMongoProvider(db)
|
||||
got, _, err := storage.Typed[bson.M](provider.Collection("coin")).Get(ctx, "user:1")
|
||||
if err != nil {
|
||||
t.Fatalf("Get migrated value: %v", err)
|
||||
}
|
||||
if got["bal"] != int64(100) {
|
||||
t.Errorf("migrated value bal = %v (%T), want int64(100)", got["bal"], got["bal"])
|
||||
}
|
||||
}
|
||||
|
||||
func mongoDatabase(t *testing.T, ctx context.Context, uri, db string) *mongo.Database {
|
||||
t.Helper()
|
||||
client, err := storage.NewMongoClient(ctx, uri)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMongoClient: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = client.Disconnect(context.Background()) })
|
||||
mdb, err := storage.NewMongoDatabase(client, db)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMongoDatabase: %v", err)
|
||||
}
|
||||
return mdb
|
||||
}
|
||||
|
||||
func createTable(t *testing.T, ctx context.Context, c *dynamodb.Client, table string) {
|
||||
t.Helper()
|
||||
_, err := c.CreateTable(ctx, &dynamodb.CreateTableInput{
|
||||
TableName: aws.String(table),
|
||||
BillingMode: types.BillingModePayPerRequest,
|
||||
AttributeDefinitions: []types.AttributeDefinition{
|
||||
{AttributeName: aws.String("pk"), AttributeType: types.ScalarAttributeTypeS},
|
||||
{AttributeName: aws.String("sk"), AttributeType: types.ScalarAttributeTypeS},
|
||||
},
|
||||
KeySchema: []types.KeySchemaElement{
|
||||
{AttributeName: aws.String("pk"), KeyType: types.KeyTypeHash},
|
||||
{AttributeName: aws.String("sk"), KeyType: types.KeyTypeRange},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateTable: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func putDynamoItem(t *testing.T, ctx context.Context, c *dynamodb.Client, table string, it item) {
|
||||
t.Helper()
|
||||
_, err := c.PutItem(ctx, &dynamodb.PutItemInput{
|
||||
TableName: aws.String(table),
|
||||
Item: map[string]types.AttributeValue{
|
||||
"pk": &types.AttributeValueMemberS{Value: it.pk},
|
||||
"sk": &types.AttributeValueMemberS{Value: it.sk},
|
||||
"value": &types.AttributeValueMemberS{Value: string(it.value)},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PutItem: %v", err)
|
||||
}
|
||||
}
|
||||
+41
-118
@@ -12,9 +12,6 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/service/ssm"
|
||||
"github.com/tiennm99/miti99bot/internal/ai"
|
||||
"github.com/tiennm99/miti99bot/internal/cron"
|
||||
"github.com/tiennm99/miti99bot/internal/deploynotify"
|
||||
@@ -79,18 +76,11 @@ func factories() map[string]modules.Factory {
|
||||
// container; 10s leaves headroom without hiding a wedged cluster.
|
||||
const mongodbInitTimeout = 10 * time.Second
|
||||
|
||||
// ssmInitTimeout caps cold-start secret resolution. Secrets are fetched once
|
||||
// at startup from Parameter Store when *_PARAMETER_NAME env vars are set.
|
||||
const ssmInitTimeout = 5 * time.Second
|
||||
|
||||
func main() {
|
||||
rootCtx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
cfg := loadConfig()
|
||||
if err := resolveSSMSecrets(rootCtx, &cfg); err != nil {
|
||||
log.Fatal("ssm secret resolution failed", "err", err)
|
||||
}
|
||||
if cfg.TelegramBotToken == "" {
|
||||
log.Fatal("missing required env", "key", "TELEGRAM_BOT_TOKEN")
|
||||
}
|
||||
@@ -144,10 +134,8 @@ func main() {
|
||||
"commands", len(reg.AllCommands),
|
||||
"crons", len(reg.Crons()))
|
||||
|
||||
// In-process cron scheduler. Replaces EventBridge Scheduler off AWS; runs
|
||||
// unconditionally so the long-lived container fires module crons (e.g. the
|
||||
// lolschedule daily push) on their Schedule. Cutover safety comes from
|
||||
// ordering + the per-date idempotency guard, not from a gate.
|
||||
// In-process cron scheduler runs unconditionally so the long-lived container
|
||||
// fires module crons (e.g. the lolschedule daily push) on their Schedule.
|
||||
stopCron, err := cron.Run(rootCtx, reg)
|
||||
if err != nil {
|
||||
log.Fatal("cron scheduler init failed", "err", err)
|
||||
@@ -158,10 +146,10 @@ func main() {
|
||||
log.Warn("OWNER_ID unset; all Private + Protected commands will be denied")
|
||||
}
|
||||
|
||||
// Clear any webhook left over from the AWS deployment at startup, before the
|
||||
// owner DM and before polling. getUpdates (long polling, below) returns HTTP
|
||||
// 409 while a webhook is set, so a stuck webhook silently breaks the bot.
|
||||
// Best-effort, one shot: a real failure here is logged, not retried.
|
||||
// Clear any existing webhook at startup before the owner DM and before
|
||||
// polling. getUpdates returns HTTP 409 while a webhook is set, so a stuck
|
||||
// webhook silently breaks the bot. Best-effort, one shot: a real failure
|
||||
// here is logged, not retried.
|
||||
if err := telegram.DeleteWebhook(rootCtx, cfg.TelegramBotToken); err != nil {
|
||||
log.Warn("deleteWebhook failed; getUpdates may 409 if a webhook is set", "err", err)
|
||||
} else {
|
||||
@@ -218,8 +206,7 @@ func main() {
|
||||
//
|
||||
// The self-host default is mongodb (just set MONGO_URL + MONGO_DATABASE — no
|
||||
// KV_PROVIDER needed). The memory backend is for tests and local no-database
|
||||
// runs (MODULES=). DynamoDB is no longer a runtime backend — it survives only
|
||||
// as the one-off migration source (cmd/migrate-dynamo-to-mongo).
|
||||
// runs (MODULES=).
|
||||
//
|
||||
// Returned closer is always non-nil and safe to call exactly once.
|
||||
func buildProvider(ctx context.Context, cfg config) (storage.Provider, func(), error) {
|
||||
@@ -265,33 +252,28 @@ func buildProvider(ctx context.Context, cfg config) (storage.Provider, func(), e
|
||||
return storage.NewMongoProvider(db), closer, nil
|
||||
|
||||
default:
|
||||
// DynamoDB is no longer a runtime backend — it survives only as the
|
||||
// one-off migration source (cmd/migrate-dynamo-to-mongo).
|
||||
return nil, func() {}, fmt.Errorf("unknown KV_PROVIDER %q (want memory|mongodb)", backend)
|
||||
}
|
||||
}
|
||||
|
||||
type config struct {
|
||||
Port string
|
||||
TelegramBotToken string
|
||||
SourceCommit string // Coolify-injected commit SHA (runtime env) for deploynotify
|
||||
GeminiAPIKey string
|
||||
GoldPriceAPIURL string
|
||||
GoldFXAPIURL string
|
||||
GoldVNAppAPIURL string
|
||||
GoldVNAppAPIKey string
|
||||
CoinBinanceAPIURL string
|
||||
CoinCoinbaseAPIURL string
|
||||
CoinCoinGeckoAPIURL string
|
||||
Modules []string
|
||||
BotOwnerID int64
|
||||
AdminUserIDs map[int64]bool
|
||||
KVProvider string // empty = auto-detect; or "memory"|"mongodb"
|
||||
MongoURL string // required when KVProvider=mongodb (Atlas SRV connection string; SECRET — never log)
|
||||
MongoDatabase string // required when KVProvider=mongodb
|
||||
TelegramBotTokenParam string
|
||||
GeminiAPIKeyParam string
|
||||
GoldVNAppAPIKeyParam string
|
||||
Port string
|
||||
TelegramBotToken string
|
||||
SourceCommit string // Coolify-injected commit SHA (runtime env) for deploynotify
|
||||
GeminiAPIKey string
|
||||
GoldPriceAPIURL string
|
||||
GoldFXAPIURL string
|
||||
GoldVNAppAPIURL string
|
||||
GoldVNAppAPIKey string
|
||||
CoinBinanceAPIURL string
|
||||
CoinCoinbaseAPIURL string
|
||||
CoinCoinGeckoAPIURL string
|
||||
Modules []string
|
||||
BotOwnerID int64
|
||||
AdminUserIDs map[int64]bool
|
||||
KVProvider string // empty = auto-detect; or "memory"|"mongodb"
|
||||
MongoURL string // required when KVProvider=mongodb (Atlas SRV connection string; SECRET — never log)
|
||||
MongoDatabase string // required when KVProvider=mongodb
|
||||
}
|
||||
|
||||
func loadConfig() config {
|
||||
@@ -312,85 +294,26 @@ func loadConfig() config {
|
||||
log.Fatal("invalid PORT", "value", port)
|
||||
}
|
||||
return config{
|
||||
Port: port,
|
||||
TelegramBotToken: envMap["TELEGRAM_BOT_TOKEN"],
|
||||
SourceCommit: envMap["SOURCE_COMMIT"],
|
||||
GeminiAPIKey: envMap["GEMINI_API_KEY"],
|
||||
GoldPriceAPIURL: envMap["GOLD_PRICE_API_URL"],
|
||||
GoldFXAPIURL: envMap["GOLD_FX_API_URL"],
|
||||
GoldVNAppAPIURL: envMap["GOLD_VNAPP_API_URL"],
|
||||
GoldVNAppAPIKey: envMap["GOLD_VNAPP_API_KEY"],
|
||||
CoinBinanceAPIURL: envMap["COIN_BINANCE_API_URL"],
|
||||
CoinCoinbaseAPIURL: envMap["COIN_COINBASE_API_URL"],
|
||||
CoinCoinGeckoAPIURL: envMap["COIN_COINGECKO_API_URL"],
|
||||
Modules: splitCSV(envMap["MODULES"]),
|
||||
BotOwnerID: parseInt64(envMap["OWNER_ID"]),
|
||||
AdminUserIDs: parseInt64Set(envMap["ADMIN_IDS"]),
|
||||
KVProvider: envMap["KV_PROVIDER"],
|
||||
MongoURL: envMap["MONGO_URL"],
|
||||
MongoDatabase: envMap["MONGO_DATABASE"],
|
||||
TelegramBotTokenParam: strings.TrimSpace(envMap["TELEGRAM_BOT_TOKEN_PARAMETER_NAME"]),
|
||||
GeminiAPIKeyParam: strings.TrimSpace(envMap["GEMINI_API_KEY_PARAMETER_NAME"]),
|
||||
GoldVNAppAPIKeyParam: strings.TrimSpace(envMap["GOLD_VNAPP_API_KEY_PARAMETER_NAME"]),
|
||||
Port: port,
|
||||
TelegramBotToken: envMap["TELEGRAM_BOT_TOKEN"],
|
||||
SourceCommit: envMap["SOURCE_COMMIT"],
|
||||
GeminiAPIKey: envMap["GEMINI_API_KEY"],
|
||||
GoldPriceAPIURL: envMap["GOLD_PRICE_API_URL"],
|
||||
GoldFXAPIURL: envMap["GOLD_FX_API_URL"],
|
||||
GoldVNAppAPIURL: envMap["GOLD_VNAPP_API_URL"],
|
||||
GoldVNAppAPIKey: envMap["GOLD_VNAPP_API_KEY"],
|
||||
CoinBinanceAPIURL: envMap["COIN_BINANCE_API_URL"],
|
||||
CoinCoinbaseAPIURL: envMap["COIN_COINBASE_API_URL"],
|
||||
CoinCoinGeckoAPIURL: envMap["COIN_COINGECKO_API_URL"],
|
||||
Modules: splitCSV(envMap["MODULES"]),
|
||||
BotOwnerID: parseInt64(envMap["OWNER_ID"]),
|
||||
AdminUserIDs: parseInt64Set(envMap["ADMIN_IDS"]),
|
||||
KVProvider: envMap["KV_PROVIDER"],
|
||||
MongoURL: envMap["MONGO_URL"],
|
||||
MongoDatabase: envMap["MONGO_DATABASE"],
|
||||
}
|
||||
}
|
||||
|
||||
func resolveSSMSecrets(ctx context.Context, cfg *config) error {
|
||||
bindings := []struct {
|
||||
name string
|
||||
target *string
|
||||
}{
|
||||
{name: cfg.TelegramBotTokenParam, target: &cfg.TelegramBotToken},
|
||||
{name: cfg.GeminiAPIKeyParam, target: &cfg.GeminiAPIKey},
|
||||
{name: cfg.GoldVNAppAPIKeyParam, target: &cfg.GoldVNAppAPIKey},
|
||||
}
|
||||
|
||||
targetsByName := map[string][]*string{}
|
||||
names := make([]string, 0, len(bindings))
|
||||
for _, b := range bindings {
|
||||
if b.name == "" || *b.target != "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := targetsByName[b.name]; !ok {
|
||||
names = append(names, b.name)
|
||||
}
|
||||
targetsByName[b.name] = append(targetsByName[b.name], b.target)
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
initCtx, cancel := context.WithTimeout(ctx, ssmInitTimeout)
|
||||
defer cancel()
|
||||
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(initCtx, awsconfig.WithHTTPClient(&http.Client{
|
||||
Timeout: ssmInitTimeout,
|
||||
}))
|
||||
if err != nil {
|
||||
return fmt.Errorf("load AWS config: %w", err)
|
||||
}
|
||||
client := ssm.NewFromConfig(awsCfg)
|
||||
out, err := client.GetParameters(initCtx, &ssm.GetParametersInput{
|
||||
Names: names,
|
||||
WithDecryption: aws.Bool(true),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("get parameters: %w", err)
|
||||
}
|
||||
if len(out.InvalidParameters) > 0 {
|
||||
return fmt.Errorf("missing SSM parameters: %s", strings.Join(out.InvalidParameters, ","))
|
||||
}
|
||||
for _, p := range out.Parameters {
|
||||
name := aws.ToString(p.Name)
|
||||
value := aws.ToString(p.Value)
|
||||
for _, target := range targetsByName[name] {
|
||||
*target = value
|
||||
}
|
||||
}
|
||||
log.Info("loaded secrets from ssm", "count", len(out.Parameters))
|
||||
return nil
|
||||
}
|
||||
|
||||
func exportOptionalEnv(key, value string) {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user