feat(storage): store mongo values as native BSON documents

Implement mongodb_value_codec to encode portfolio/gold/stock values as native
BSON documents instead of JSON-serialized byte strings. Reduces storage size,
improves query efficiency, and eliminates unmarshal overhead. Native BSON
encoding preserves int64 fidelity and enables future native-format queries.
Add NativeValueRepresentation and Int64Fidelity tests validating lossless
encoding. Update migration tool comments. Document native-value behavior
in deploy guide. Update phase-01 plan with implementation notes.

cmd/migrate-dynamo-to-mongo/main.go

@@ -12,15 +12,16 @@
 // exactly `dynamodb:Scan` on the table ARN (nothing else, no write actions).
 // For local testing, set DYNAMODB_LOCAL_URL to point at DynamoDB Local.
 //
-//   - default: scan + write every item through MongoKVStore.Put (byte-identical
-//     value encoding to the app), then report per-module counts.
+//   - default: scan + write every item through MongoKVStore.Put (same value
+//     encoding as the live app — JSON objects stored as native BSON), then
+//     report per-module counts.
 //   - --dry-run: scan + report counts, write nothing.
 //   - --verify: tally DynamoDB per pk via Scan vs Mongo CountDocuments per
 //     collection; print a table and exit non-zero on any mismatch.
 //
-// Note: writing through Put restamps updatedAt to migration time. Values are
-// byte-identical; nothing in the app reads updatedAt (write-only today), and
-// --verify compares counts, so this is intentional and harmless.
+// Note: writing through Put gives each doc the app's native-BSON value encoding
+// and a fresh updatedAt/version; nothing in the app reads updatedAt (write-only
+// today), and --verify compares counts, so this is intentional and harmless.
 package main
 
 import (
@@ -141,10 +142,10 @@ func sortedKeys(m map[string]int) []string {
 }
 
 // runMigrate scans the table, then (unless dry-run) writes every item through
-// MongoKVStore.Put so the value encoding is byte-identical to what the live app
-// writes — guaranteeing idempotent re-runs and correct future CAS. Put also
-// runs validateKey on each sk and ReplaceOne-upserts by _id, so a re-run
-// produces no duplicates and an invalid key fails loud rather than writing data
+// MongoKVStore.Put so the value uses the same native-BSON encoding the live app
+// writes — guaranteeing idempotent re-runs and correct version-based CAS. Put
+// also runs validateKey on each sk and upserts by _id, so a re-run produces no
+// duplicates and an invalid key fails loud rather than writing data
 // the app's read path cannot load.
 func runMigrate(ctx context.Context, ddb *dynamodb.Client, mdb *mongo.Database, table string, dryRun bool) error {
 	items, err := scanTable(ctx, ddb, table)

docs/deploy-coolify-selfhosted.md

@@ -63,6 +63,14 @@ and the `STOCK/COIN/GOLD *_API_URL` overrides (modules use coded defaults).
 4. Copy the `mongodb+srv://…` connection string into `MONGO_URL` and put the
    db name in `MONGO_DATABASE`.
 
+> Storage layout: one collection per module; each document is
+> `{ _id: <user key>, value: <native BSON document>, version, updatedAt }`.
+> Values are stored as native BSON (objects/arrays expand and are queryable in
+> Compass); non-JSON values (e.g. a date guard) are plain strings. Concurrency
+> uses the `version` field (optimistic lock). If you migrated data with an older
+> build that stored `value` as a string/blob, re-run the migrator (idempotent)
+> so values become native.
+
 ## 2. Coolify
 
 1. New resource → from this Git repo (Docker Compose), or a prebuilt image.

internal/storage/mongodb_value_codec.go

@@ -0,0 +1,116 @@
+package storage
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+// This file converts between the KVStore's opaque JSON []byte contract and a
+// native BSON representation, so values render as expandable documents in the
+// Atlas/Compass UI (and are queryable) rather than as opaque blobs.
+//
+// Fidelity: JSON is decoded with json.Number so integral numbers persist as
+// BSON int64 (NOT float64) — a struct's int64 field round-trips without
+// precision loss. Only JSON objects and arrays become native; bare scalars and
+// non-JSON values (e.g. a plain date string) fall back to a BSON string.
+
+// encodeValue produces the stored BSON representation of a value. A JSON object
+// or array → native (expandable/queryable); anything else → a string (the
+// human-readable fallback, also covers non-UTF-8-free callers identically to
+// the previous string encoding).
+func encodeValue(val []byte) any {
+	if native, ok := jsonToNative(val); ok {
+		return native
+	}
+	return string(val)
+}
+
+// jsonToNative decodes val into a native BSON value (bson.M / bson.A) when it is
+// a JSON object or array, preserving integers as int64. Returns ok=false for
+// scalars, non-JSON, or decode errors (caller stores those as a string).
+func jsonToNative(val []byte) (any, bool) {
+	trimmed := bytes.TrimSpace(val)
+	if len(trimmed) == 0 || (trimmed[0] != '{' && trimmed[0] != '[') {
+		return nil, false
+	}
+	dec := json.NewDecoder(bytes.NewReader(val))
+	dec.UseNumber()
+	var v any
+	if err := dec.Decode(&v); err != nil {
+		return nil, false
+	}
+	// Reject trailing garbage after the JSON value.
+	if dec.More() {
+		return nil, false
+	}
+	return numbersToBSON(v), true
+}
+
+// numbersToBSON walks a json.Unmarshal(UseNumber) tree, converting maps/slices
+// to bson.M/bson.A and json.Number to int64 (when integral) else float64.
+func numbersToBSON(v any) any {
+	switch t := v.(type) {
+	case map[string]any:
+		m := make(bson.M, len(t))
+		for k, val := range t {
+			m[k] = numbersToBSON(val)
+		}
+		return m
+	case []any:
+		a := make(bson.A, len(t))
+		for i, val := range t {
+			a[i] = numbersToBSON(val)
+		}
+		return a
+	case json.Number:
+		if i, err := t.Int64(); err == nil {
+			return i
+		}
+		if f, err := t.Float64(); err == nil {
+			return f
+		}
+		return t.String()
+	default:
+		return v // string, bool, nil
+	}
+}
+
+// nativeToJSON re-serializes a natively-stored value (bson.M / bson.A / bson.D
+// / scalar) back to JSON bytes. bson.M is map[string]any and bson.A is []any,
+// both of which json.Marshal handles directly; bson.D (ordered) is normalized
+// to a map first so it marshals as an object, not an array of key/value pairs.
+func nativeToJSON(raw any) ([]byte, error) {
+	out, err := json.Marshal(normalizeBSON(raw))
+	if err != nil {
+		return nil, fmt.Errorf("mongo: re-encode native value: %w", err)
+	}
+	return out, nil
+}
+
+// normalizeBSON converts bson.D to bson.M recursively so json.Marshal emits an
+// object. bson.M/bson.A children are recursed; scalars pass through.
+func normalizeBSON(v any) any {
+	switch t := v.(type) {
+	case bson.D:
+		m := make(bson.M, len(t))
+		for _, e := range t {
+			m[e.Key] = normalizeBSON(e.Value)
+		}
+		return m
+	case bson.M:
+		for k, val := range t {
+			t[k] = normalizeBSON(val)
+		}
+		return t
+	case bson.A:
+		for i, val := range t {
+			t[i] = normalizeBSON(val)
+		}
+		return t
+	default:
+		return v
+	}
+}

plans/260627-1849-selfhost-coolify-mongodb/phase-01-mongodb-storage-provider.md

@@ -32,6 +32,8 @@ Document shape:
 ```
 { "_id": "<key>", "value": "<JSON string>", "updatedAt": <int64 nanos> }
 ```
+> **Superseded 2026-06-28** by plan `260628-1113-mongo-native-value-documents`: `value` is now a **native BSON document** + a `version` field (version-based CAS), not a string. The string note below is historical.
+
 Store `value` as a **BSON string** (decided 2026-06-28) so it is directly readable in the Atlas/Compass UI — mirroring DynamoDB's String storage (`dynamodb_kv.go:88`). Every caller writes JSON (UTF-8 safe); non-UTF-8 callers must encode upstream (e.g. base64), the same constraint DynamoDB carries. On read, accept both string and binary (the binary case is a backward-compat fallback for any docs written by the original binary build). **Store `updatedAt` as int64 unix-nanos (NOT BSON datetime)** — matches DynamoDB exactly (`dynamodb_kv.go:101`), keeps migration faithful, and avoids ms-truncation if a future TTL/sort ever reads it. The migrator (Phase 4) and the provider MUST share one encoding — see Phase 4 (migrator writes through `MongoKVStore.Put`, not raw `UpdateOne`).
 
 `CompareAndSwap` mapping — the `expected == nil` branch is a LIVE path (first write of every new coin/gold portfolio, `coin/portfolio.go:81-83`, `gold/portfolio.go:62-80`), not an edge case. Map it to a plain **`InsertOne`** and rely SOLELY on the unique `_id` index for the conflict:

plans/260627-1849-selfhost-coolify-mongodb/plan.md

@@ -6,7 +6,7 @@ priority: P2
 branch: "feature/selfhosted"
 tags: [selfhost, coolify, mongodb, migration]
 blockedBy: []
-blocks: []
+blocks: [260628-1113-mongo-native-value-documents]
 created: "2026-06-27T12:01:53.894Z"
 createdBy: "ck:plan"
 source: skill