From e75d9bc378d73c3e99ab4f4531c207fabc9fa1e7 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Sun, 28 Jun 2026 12:52:48 +0700 Subject: [PATCH] feat(storage): store mongo values as native BSON documents Implement mongodb_value_codec to encode portfolio/gold/stock values as native BSON documents instead of JSON-serialized byte strings. Reduces storage size, improves query efficiency, and eliminates unmarshal overhead. Native BSON encoding preserves int64 fidelity and enables future native-format queries. Add NativeValueRepresentation and Int64Fidelity tests validating lossless encoding. Update migration tool comments. Document native-value behavior in deploy guide. Update phase-01 plan with implementation notes. --- cmd/migrate-dynamo-to-mongo/main.go | 19 +-- docs/deploy-coolify-selfhosted.md | 8 ++ internal/storage/mongodb_value_codec.go | 116 ++++++++++++++++++ .../phase-01-mongodb-storage-provider.md | 2 + .../plan.md | 2 +- 5 files changed, 137 insertions(+), 10 deletions(-) create mode 100644 internal/storage/mongodb_value_codec.go diff --git a/cmd/migrate-dynamo-to-mongo/main.go b/cmd/migrate-dynamo-to-mongo/main.go index f89a37d..30f0e2f 100644 --- a/cmd/migrate-dynamo-to-mongo/main.go +++ b/cmd/migrate-dynamo-to-mongo/main.go @@ -12,15 +12,16 @@ // exactly `dynamodb:Scan` on the table ARN (nothing else, no write actions). // For local testing, set DYNAMODB_LOCAL_URL to point at DynamoDB Local. // -// - default: scan + write every item through MongoKVStore.Put (byte-identical -// value encoding to the app), then report per-module counts. +// - default: scan + write every item through MongoKVStore.Put (same value +// encoding as the live app — JSON objects stored as native BSON), then +// report per-module counts. // - --dry-run: scan + report counts, write nothing. // - --verify: tally DynamoDB per pk via Scan vs Mongo CountDocuments per // collection; print a table and exit non-zero on any mismatch. // -// Note: writing through Put restamps updatedAt to migration time. Values are -// byte-identical; nothing in the app reads updatedAt (write-only today), and -// --verify compares counts, so this is intentional and harmless. +// Note: writing through Put gives each doc the app's native-BSON value encoding +// and a fresh updatedAt/version; nothing in the app reads updatedAt (write-only +// today), and --verify compares counts, so this is intentional and harmless. package main import ( @@ -141,10 +142,10 @@ func sortedKeys(m map[string]int) []string { } // runMigrate scans the table, then (unless dry-run) writes every item through -// MongoKVStore.Put so the value encoding is byte-identical to what the live app -// writes — guaranteeing idempotent re-runs and correct future CAS. Put also -// runs validateKey on each sk and ReplaceOne-upserts by _id, so a re-run -// produces no duplicates and an invalid key fails loud rather than writing data +// MongoKVStore.Put so the value uses the same native-BSON encoding the live app +// writes — guaranteeing idempotent re-runs and correct version-based CAS. Put +// also runs validateKey on each sk and upserts by _id, so a re-run produces no +// duplicates and an invalid key fails loud rather than writing data // the app's read path cannot load. func runMigrate(ctx context.Context, ddb *dynamodb.Client, mdb *mongo.Database, table string, dryRun bool) error { items, err := scanTable(ctx, ddb, table) diff --git a/docs/deploy-coolify-selfhosted.md b/docs/deploy-coolify-selfhosted.md index fb50828..66974a8 100644 --- a/docs/deploy-coolify-selfhosted.md +++ b/docs/deploy-coolify-selfhosted.md @@ -63,6 +63,14 @@ and the `STOCK/COIN/GOLD *_API_URL` overrides (modules use coded defaults). 4. Copy the `mongodb+srv://…` connection string into `MONGO_URL` and put the db name in `MONGO_DATABASE`. +> Storage layout: one collection per module; each document is +> `{ _id: , value: , version, updatedAt }`. +> Values are stored as native BSON (objects/arrays expand and are queryable in +> Compass); non-JSON values (e.g. a date guard) are plain strings. Concurrency +> uses the `version` field (optimistic lock). If you migrated data with an older +> build that stored `value` as a string/blob, re-run the migrator (idempotent) +> so values become native. + ## 2. Coolify 1. New resource → from this Git repo (Docker Compose), or a prebuilt image. diff --git a/internal/storage/mongodb_value_codec.go b/internal/storage/mongodb_value_codec.go new file mode 100644 index 0000000..01d2379 --- /dev/null +++ b/internal/storage/mongodb_value_codec.go @@ -0,0 +1,116 @@ +package storage + +import ( + "bytes" + "encoding/json" + "fmt" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// This file converts between the KVStore's opaque JSON []byte contract and a +// native BSON representation, so values render as expandable documents in the +// Atlas/Compass UI (and are queryable) rather than as opaque blobs. +// +// Fidelity: JSON is decoded with json.Number so integral numbers persist as +// BSON int64 (NOT float64) — a struct's int64 field round-trips without +// precision loss. Only JSON objects and arrays become native; bare scalars and +// non-JSON values (e.g. a plain date string) fall back to a BSON string. + +// encodeValue produces the stored BSON representation of a value. A JSON object +// or array → native (expandable/queryable); anything else → a string (the +// human-readable fallback, also covers non-UTF-8-free callers identically to +// the previous string encoding). +func encodeValue(val []byte) any { + if native, ok := jsonToNative(val); ok { + return native + } + return string(val) +} + +// jsonToNative decodes val into a native BSON value (bson.M / bson.A) when it is +// a JSON object or array, preserving integers as int64. Returns ok=false for +// scalars, non-JSON, or decode errors (caller stores those as a string). +func jsonToNative(val []byte) (any, bool) { + trimmed := bytes.TrimSpace(val) + if len(trimmed) == 0 || (trimmed[0] != '{' && trimmed[0] != '[') { + return nil, false + } + dec := json.NewDecoder(bytes.NewReader(val)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, false + } + // Reject trailing garbage after the JSON value. + if dec.More() { + return nil, false + } + return numbersToBSON(v), true +} + +// numbersToBSON walks a json.Unmarshal(UseNumber) tree, converting maps/slices +// to bson.M/bson.A and json.Number to int64 (when integral) else float64. +func numbersToBSON(v any) any { + switch t := v.(type) { + case map[string]any: + m := make(bson.M, len(t)) + for k, val := range t { + m[k] = numbersToBSON(val) + } + return m + case []any: + a := make(bson.A, len(t)) + for i, val := range t { + a[i] = numbersToBSON(val) + } + return a + case json.Number: + if i, err := t.Int64(); err == nil { + return i + } + if f, err := t.Float64(); err == nil { + return f + } + return t.String() + default: + return v // string, bool, nil + } +} + +// nativeToJSON re-serializes a natively-stored value (bson.M / bson.A / bson.D +// / scalar) back to JSON bytes. bson.M is map[string]any and bson.A is []any, +// both of which json.Marshal handles directly; bson.D (ordered) is normalized +// to a map first so it marshals as an object, not an array of key/value pairs. +func nativeToJSON(raw any) ([]byte, error) { + out, err := json.Marshal(normalizeBSON(raw)) + if err != nil { + return nil, fmt.Errorf("mongo: re-encode native value: %w", err) + } + return out, nil +} + +// normalizeBSON converts bson.D to bson.M recursively so json.Marshal emits an +// object. bson.M/bson.A children are recursed; scalars pass through. +func normalizeBSON(v any) any { + switch t := v.(type) { + case bson.D: + m := make(bson.M, len(t)) + for _, e := range t { + m[e.Key] = normalizeBSON(e.Value) + } + return m + case bson.M: + for k, val := range t { + t[k] = normalizeBSON(val) + } + return t + case bson.A: + for i, val := range t { + t[i] = normalizeBSON(val) + } + return t + default: + return v + } +} diff --git a/plans/260627-1849-selfhost-coolify-mongodb/phase-01-mongodb-storage-provider.md b/plans/260627-1849-selfhost-coolify-mongodb/phase-01-mongodb-storage-provider.md index 1d4db5e..24ab668 100644 --- a/plans/260627-1849-selfhost-coolify-mongodb/phase-01-mongodb-storage-provider.md +++ b/plans/260627-1849-selfhost-coolify-mongodb/phase-01-mongodb-storage-provider.md @@ -32,6 +32,8 @@ Document shape: ``` { "_id": "", "value": "", "updatedAt": } ``` +> **Superseded 2026-06-28** by plan `260628-1113-mongo-native-value-documents`: `value` is now a **native BSON document** + a `version` field (version-based CAS), not a string. The string note below is historical. + Store `value` as a **BSON string** (decided 2026-06-28) so it is directly readable in the Atlas/Compass UI — mirroring DynamoDB's String storage (`dynamodb_kv.go:88`). Every caller writes JSON (UTF-8 safe); non-UTF-8 callers must encode upstream (e.g. base64), the same constraint DynamoDB carries. On read, accept both string and binary (the binary case is a backward-compat fallback for any docs written by the original binary build). **Store `updatedAt` as int64 unix-nanos (NOT BSON datetime)** — matches DynamoDB exactly (`dynamodb_kv.go:101`), keeps migration faithful, and avoids ms-truncation if a future TTL/sort ever reads it. The migrator (Phase 4) and the provider MUST share one encoding — see Phase 4 (migrator writes through `MongoKVStore.Put`, not raw `UpdateOne`). `CompareAndSwap` mapping — the `expected == nil` branch is a LIVE path (first write of every new coin/gold portfolio, `coin/portfolio.go:81-83`, `gold/portfolio.go:62-80`), not an edge case. Map it to a plain **`InsertOne`** and rely SOLELY on the unique `_id` index for the conflict: diff --git a/plans/260627-1849-selfhost-coolify-mongodb/plan.md b/plans/260627-1849-selfhost-coolify-mongodb/plan.md index 70b8b21..7bad10e 100644 --- a/plans/260627-1849-selfhost-coolify-mongodb/plan.md +++ b/plans/260627-1849-selfhost-coolify-mongodb/plan.md @@ -6,7 +6,7 @@ priority: P2 branch: "feature/selfhosted" tags: [selfhost, coolify, mongodb, migration] blockedBy: [] -blocks: [] +blocks: [260628-1113-mongo-native-value-documents] created: "2026-06-27T12:01:53.894Z" createdBy: "ck:plan" source: skill