diff --git a/cmd/server/main.go b/cmd/server/main.go index 16bc12b..a51d123 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -29,7 +29,6 @@ import ( "github.com/tiennm99/miti99bot/internal/modules/wordle" "github.com/tiennm99/miti99bot/internal/server" "github.com/tiennm99/miti99bot/internal/storage" - "github.com/tiennm99/miti99bot/internal/systemstate" "github.com/tiennm99/miti99bot/internal/telegram" ) @@ -95,7 +94,7 @@ func main() { } defer closeProvider() - if err := stats.InitStore(rootCtx, provider.Collection("stats"), provider.Collection(systemstate.CollectionName)); err != nil { + if err := stats.InitStore(rootCtx, provider.Collection("stats")); err != nil { log.Fatal("stats storage init failed", "err", err) } if err := lol.InitStore(rootCtx, provider.Collection(lol.CollectionName)); err != nil { diff --git a/docs/deploy-coolify-selfhosted.md b/docs/deploy-coolify-selfhosted.md index 283b0a5..9b532bd 100644 --- a/docs/deploy-coolify-selfhosted.md +++ b/docs/deploy-coolify-selfhosted.md @@ -99,11 +99,11 @@ Successful GIF replies include the result behind Telegram spoiler formatting. > `updatedAt` is a BSON Date. > > The `stats` collection uses queryable aggregate documents for command/user -> counts and creates indexes on startup. Renamed command rows are merged into the -> current command name. Deleted legacy command rows are retained with -> `deleted: true`; `/stats` queries filter those rows from visible results. A -> historical `system` collection may remain in MongoDB with completed migration -> records and can be reused if a future one-time startup migration is needed. +> counts and creates indexes on startup. Deleted legacy command rows are +> retained with `deleted: true`; `/stats` queries filter those rows from visible +> results. A historical `system` collection may remain in MongoDB with completed +> migration records and can be reused if a future one-time startup migration is +> needed. ## 2. Coolify diff --git a/internal/modules/stats/startup.go b/internal/modules/stats/startup.go index 90137ac..45fc2bd 100644 --- a/internal/modules/stats/startup.go +++ b/internal/modules/stats/startup.go @@ -2,152 +2,29 @@ package stats import ( "context" - "errors" "fmt" - "slices" - "strings" - "time" "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo" "go.mongodb.org/mongo-driver/v2/mongo/options" "github.com/tiennm99/miti99bot/internal/storage" - "github.com/tiennm99/miti99bot/internal/systemstate" ) const ( statsCommandUsersIndexName = "stats_cmd_n_user" statsUserCommandsIndexName = "stats_uid_n_cmd" statsUsernameLookupIndexName = "stats_user_uid" - - renameLolNextWeekStatsKey = "stats:command-rename:lol_nextweek-to-lol_next_week" - oldLolNextWeekCommand = "lol_nextweek" - newLolNextWeekCommand = "lol_next_week" - - renameWheelOfNamesBetaStatsKey = "stats:command-rename:wheelofnamesbeta-to-wheelofnames" - legacyWheelOfNamesBetaCommand = "wheelofnamesbeta" - currentWheelOfNamesCommand = "wheelofnames" ) // InitStore performs stats collection startup maintenance. It is safe to call -// every boot: MongoDB indexes are created idempotently and one-time command -// rename migrations are guarded by the shared system collection. -func InitStore(ctx context.Context, statsColl, systemColl storage.Collection) error { +// every boot: MongoDB indexes are created idempotently. +func InitStore(ctx context.Context, statsColl storage.Collection) error { if mongoColl, ok := storage.MongoCollection(statsColl); ok { if err := ensureUsageIndexes(ctx, mongoColl); err != nil { return err } } - if err := migrateCommandRename(ctx, statsColl, systemColl, oldLolNextWeekCommand, newLolNextWeekCommand, renameLolNextWeekStatsKey); err != nil { - return err - } - if err := migrateCommandRename(ctx, statsColl, systemColl, legacyWheelOfNamesBetaCommand, currentWheelOfNamesCommand, renameWheelOfNamesBetaStatsKey); err != nil { - return err - } - return nil -} - -func migrateCommandRename(ctx context.Context, statsColl, systemColl storage.Collection, oldCmd, newCmd, markerKey string) error { - state := systemstate.New(systemColl) - if rec, ok, err := state.Get(ctx, markerKey); err != nil { - return fmt.Errorf("stats command rename marker %s: %w", markerKey, err) - } else if ok && rec.Status == "complete" { - return nil - } - - docs := storage.Typed[usageEntry](statsColl) - keys, err := docs.List(ctx, oldCmd) - if err != nil { - return fmt.Errorf("stats command rename list %s: %w", oldCmd, err) - } - - var moved int64 - for _, key := range keys { - if key != oldCmd && !strings.HasPrefix(key, oldCmd+":") { - continue - } - entry, _, err := docs.Get(ctx, key) - if err != nil { - return fmt.Errorf("stats command rename get %s: %w", key, err) - } - if entry.Cmd != oldCmd { - continue - } - - targetKey := usageKey(newCmd, entry.UserID) - target, _, err := docs.Get(ctx, targetKey) - if err != nil && !errors.Is(err, storage.ErrNotFound) { - return fmt.Errorf("stats command rename get target %s: %w", targetKey, err) - } - if errors.Is(err, storage.ErrNotFound) { - target = usageEntry{} - } - if slices.Contains(target.MergedFrom, key) { - if err := docs.Delete(ctx, key); err != nil { - return fmt.Errorf("stats command rename delete already-merged old %s: %w", key, err) - } - continue - } - - target.Cmd = newCmd - target.UserID = entry.UserID - if entry.UserID == 0 { - target.Username = "" - } else if entry.Username != "" { - target.Username = entry.Username - } - target.N += entry.N - target.Deleted = false - target.MergedFrom = append(target.MergedFrom, key) - if err := docs.Put(ctx, targetKey, target); err != nil { - return fmt.Errorf("stats command rename put target %s: %w", targetKey, err) - } - if err := docs.Delete(ctx, key); err != nil { - return fmt.Errorf("stats command rename delete old %s: %w", key, err) - } - moved += entry.N - } - - if err := clearCommandMergeMarkers(ctx, docs, newCmd); err != nil { - return err - } - - now := time.Now().UTC().UnixMilli() - if err := state.Put(ctx, markerKey, systemstate.Record{ - Kind: "migration", - Name: markerKey, - Status: "complete", - Count: moved, - CompletedAt: now, - UpdatedAt: now, - }); err != nil { - return fmt.Errorf("stats command rename marker put %s: %w", markerKey, err) - } - return nil -} - -func clearCommandMergeMarkers(ctx context.Context, docs storage.DocStore[usageEntry], cmd string) error { - keys, err := docs.List(ctx, cmd) - if err != nil { - return fmt.Errorf("stats command rename cleanup list %s: %w", cmd, err) - } - for _, key := range keys { - if key != cmd && !strings.HasPrefix(key, cmd+":") { - continue - } - entry, _, err := docs.Get(ctx, key) - if err != nil { - return fmt.Errorf("stats command rename cleanup get %s: %w", key, err) - } - if entry.Cmd != cmd || len(entry.MergedFrom) == 0 { - continue - } - entry.MergedFrom = nil - if err := docs.Put(ctx, key, entry); err != nil { - return fmt.Errorf("stats command rename cleanup put %s: %w", key, err) - } - } return nil } diff --git a/internal/modules/stats/startup_mongo_test.go b/internal/modules/stats/startup_mongo_test.go index 5eb2c82..b46189a 100644 --- a/internal/modules/stats/startup_mongo_test.go +++ b/internal/modules/stats/startup_mongo_test.go @@ -2,29 +2,26 @@ package stats import ( "context" - "errors" "fmt" "os" - "strings" "testing" "time" "github.com/tiennm99/miti99bot/internal/storage" - "github.com/tiennm99/miti99bot/internal/systemstate" ) func TestInitStore_MongoCreatesIndexes(t *testing.T) { - ctx, statsColl, systemColl := setupMongoStatsTest(t) + ctx, statsColl := setupMongoStatsTest(t) rawStatsColl, ok := storage.MongoCollection(statsColl) if !ok { t.Fatal("stats collection is not Mongo-backed") } - if err := InitStore(ctx, statsColl, systemColl); err != nil { + if err := InitStore(ctx, statsColl); err != nil { t.Fatalf("InitStore: %v", err) } - if err := InitStore(ctx, statsColl, systemColl); err != nil { + if err := InitStore(ctx, statsColl); err != nil { t.Fatalf("InitStore second run: %v", err) } @@ -54,64 +51,7 @@ func TestInitStore_MongoCreatesIndexes(t *testing.T) { } } -func TestInitStore_MongoMergesLegacyWheelOfNamesBetaStats(t *testing.T) { - ctx, statsColl, systemColl := setupMongoStatsTest(t) - docs := storage.Typed[usageEntry](statsColl) - - seeds := map[string]usageEntry{ - usageKey(legacyWheelOfNamesBetaCommand, 0): {Cmd: legacyWheelOfNamesBetaCommand, N: 2}, - usageKey(legacyWheelOfNamesBetaCommand, 7): { - Cmd: legacyWheelOfNamesBetaCommand, - UserID: 7, - Username: "alice", - N: 3, - Deleted: true, - }, - usageKey(currentWheelOfNamesCommand, 7): { - Cmd: currentWheelOfNamesCommand, - UserID: 7, - Username: "alice", - N: 5, - }, - } - for key, entry := range seeds { - if err := docs.Put(ctx, key, entry); err != nil { - t.Fatalf("seed %s: %v", key, err) - } - } - - if err := InitStore(ctx, statsColl, systemColl); err != nil { - t.Fatalf("InitStore: %v", err) - } - if err := InitStore(ctx, statsColl, systemColl); err != nil { - t.Fatalf("InitStore second run: %v", err) - } - - anon, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 0)) - if err != nil { - t.Fatalf("merged anonymous wheelofnames stats: %v", err) - } - if anon.N != 2 || anon.Deleted || len(anon.MergedFrom) != 0 { - t.Fatalf("merged anonymous stats = %+v, want visible count 2", anon) - } - alice, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 7)) - if err != nil { - t.Fatalf("merged alice wheelofnames stats: %v", err) - } - if alice.N != 8 || alice.Deleted || len(alice.MergedFrom) != 0 { - t.Fatalf("merged alice stats = %+v, want visible count 8", alice) - } - for _, key := range []string{usageKey(legacyWheelOfNamesBetaCommand, 0), usageKey(legacyWheelOfNamesBetaCommand, 7)} { - if _, _, err := docs.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) { - t.Fatalf("legacy key %s err = %v, want ErrNotFound", key, err) - } - } - if got := renderStats(ctx, newCounter(statsColl), ""); !strings.Contains(got, "/wheelofnames: 10") || strings.Contains(got, "wheelofnamesbeta") { - t.Fatalf("top commands after Mongo migration = %q, want merged visible /wheelofnames count 10 only", got) - } -} - -func setupMongoStatsTest(t *testing.T) (context.Context, storage.Collection, storage.Collection) { +func setupMongoStatsTest(t *testing.T) (context.Context, storage.Collection) { t.Helper() uri := os.Getenv("MONGODB_TEST_URL") @@ -136,5 +76,5 @@ func setupMongoStatsTest(t *testing.T) (context.Context, storage.Collection, sto }) provider := storage.NewMongoProvider(db) - return ctx, provider.Collection("stats"), provider.Collection(systemstate.CollectionName) + return ctx, provider.Collection("stats") } diff --git a/internal/modules/stats/startup_test.go b/internal/modules/stats/startup_test.go deleted file mode 100644 index 39a42e2..0000000 --- a/internal/modules/stats/startup_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package stats - -import ( - "context" - "errors" - "strings" - "testing" - - "github.com/tiennm99/miti99bot/internal/storage" - "github.com/tiennm99/miti99bot/internal/systemstate" -) - -func TestInitStore_RenamesLolNextWeekStatsOnce(t *testing.T) { - ctx := context.Background() - provider := storage.NewMemoryProvider() - statsColl := provider.Collection("stats") - systemColl := provider.Collection(systemstate.CollectionName) - docs := storage.Typed[usageEntry](statsColl) - - seeds := map[string]usageEntry{ - usageKey(oldLolNextWeekCommand, 0): {Cmd: oldLolNextWeekCommand, N: 2}, - usageKey(oldLolNextWeekCommand, 7): { - Cmd: oldLolNextWeekCommand, - UserID: 7, - Username: "alice", - N: 3, - }, - usageKey(newLolNextWeekCommand, 7): { - Cmd: newLolNextWeekCommand, - UserID: 7, - Username: "alice", - N: 5, - }, - } - for key, entry := range seeds { - if err := docs.Put(ctx, key, entry); err != nil { - t.Fatalf("seed %s: %v", key, err) - } - } - - if err := InitStore(ctx, statsColl, systemColl); err != nil { - t.Fatalf("InitStore: %v", err) - } - if err := InitStore(ctx, statsColl, systemColl); err != nil { - t.Fatalf("InitStore second run: %v", err) - } - - anon, _, err := docs.Get(ctx, usageKey(newLolNextWeekCommand, 0)) - if err != nil { - t.Fatalf("new anonymous stats: %v", err) - } - if anon.Cmd != newLolNextWeekCommand || anon.N != 2 || anon.UserID != 0 { - t.Fatalf("new anonymous stats = %+v, want cmd %q count 2", anon, newLolNextWeekCommand) - } - - user, _, err := docs.Get(ctx, usageKey(newLolNextWeekCommand, 7)) - if err != nil { - t.Fatalf("new user stats: %v", err) - } - if user.Cmd != newLolNextWeekCommand || user.UserID != 7 || user.Username != "alice" || user.N != 8 { - t.Fatalf("new user stats = %+v, want merged count 8", user) - } - - for _, key := range []string{usageKey(oldLolNextWeekCommand, 0), usageKey(oldLolNextWeekCommand, 7)} { - if _, _, err := docs.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) { - t.Fatalf("old key %s err = %v, want ErrNotFound", key, err) - } - } - - rec, ok, err := systemstate.New(systemColl).Get(ctx, renameLolNextWeekStatsKey) - if err != nil { - t.Fatalf("migration marker: %v", err) - } - if !ok || rec.Status != "complete" || rec.Count != 5 { - t.Fatalf("migration marker = %+v ok=%v, want complete count 5", rec, ok) - } -} - -func TestInitStore_MergesLegacyWheelOfNamesBetaStatsOnce(t *testing.T) { - ctx := context.Background() - provider := storage.NewMemoryProvider() - statsColl := provider.Collection("stats") - systemColl := provider.Collection(systemstate.CollectionName) - docs := storage.Typed[usageEntry](statsColl) - - seeds := map[string]usageEntry{ - usageKey(legacyWheelOfNamesBetaCommand, 0): {Cmd: legacyWheelOfNamesBetaCommand, N: 2}, - usageKey(legacyWheelOfNamesBetaCommand, 7): { - Cmd: legacyWheelOfNamesBetaCommand, - UserID: 7, - Username: "alice", - N: 3, - }, - usageKey(legacyWheelOfNamesBetaCommand, 8): { - Cmd: legacyWheelOfNamesBetaCommand, - UserID: 8, - Username: "bob", - N: 4, - Deleted: true, - }, - usageKey(currentWheelOfNamesCommand, 7): { - Cmd: currentWheelOfNamesCommand, - UserID: 7, - Username: "alice", - N: 5, - }, - } - for key, entry := range seeds { - if err := docs.Put(ctx, key, entry); err != nil { - t.Fatalf("seed %s: %v", key, err) - } - } - - if err := InitStore(ctx, statsColl, systemColl); err != nil { - t.Fatalf("InitStore: %v", err) - } - if err := InitStore(ctx, statsColl, systemColl); err != nil { - t.Fatalf("InitStore second run: %v", err) - } - - anon, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 0)) - if err != nil { - t.Fatalf("merged anonymous wheelofnames stats: %v", err) - } - if anon.Cmd != currentWheelOfNamesCommand || anon.N != 2 || anon.UserID != 0 || anon.Deleted || len(anon.MergedFrom) != 0 { - t.Fatalf("merged anonymous stats = %+v, want visible count 2", anon) - } - - alice, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 7)) - if err != nil { - t.Fatalf("merged alice wheelofnames stats: %v", err) - } - if alice.Cmd != currentWheelOfNamesCommand || alice.UserID != 7 || alice.Username != "alice" || alice.N != 8 || alice.Deleted || len(alice.MergedFrom) != 0 { - t.Fatalf("merged alice stats = %+v, want visible count 8", alice) - } - - bob, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 8)) - if err != nil { - t.Fatalf("merged bob wheelofnames stats: %v", err) - } - if bob.Cmd != currentWheelOfNamesCommand || bob.UserID != 8 || bob.Username != "bob" || bob.N != 4 || bob.Deleted || len(bob.MergedFrom) != 0 { - t.Fatalf("merged bob stats = %+v, want visible count 4", bob) - } - - renderedTopCommands := renderStats(ctx, newCounter(statsColl), "") - if !strings.Contains(renderedTopCommands, "/wheelofnames: 14") || strings.Contains(renderedTopCommands, "wheelofnamesbeta") { - t.Fatalf("top commands after migration = %q, want merged visible /wheelofnames count 14 only", renderedTopCommands) - } - renderedCommandUsers := renderStats(ctx, newCounter(statsColl), "cmd wheelofnames") - if !strings.Contains(renderedCommandUsers, "@alice: 8") || !strings.Contains(renderedCommandUsers, "@bob: 4") { - t.Fatalf("wheelofnames users after migration = %q, want merged users", renderedCommandUsers) - } - - for _, key := range []string{ - usageKey(legacyWheelOfNamesBetaCommand, 0), - usageKey(legacyWheelOfNamesBetaCommand, 7), - usageKey(legacyWheelOfNamesBetaCommand, 8), - } { - if _, _, err := docs.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) { - t.Fatalf("legacy key %s err = %v, want ErrNotFound", key, err) - } - } - - rec, ok, err := systemstate.New(systemColl).Get(ctx, renameWheelOfNamesBetaStatsKey) - if err != nil { - t.Fatalf("migration marker: %v", err) - } - if !ok || rec.Status != "complete" || rec.Count != 9 { - t.Fatalf("migration marker = %+v ok=%v, want complete count 9", rec, ok) - } -} - -func TestInitStore_RetriesPartiallyMergedWheelOfNamesBetaStats(t *testing.T) { - ctx := context.Background() - provider := storage.NewMemoryProvider() - statsColl := provider.Collection("stats") - systemColl := provider.Collection(systemstate.CollectionName) - docs := storage.Typed[usageEntry](statsColl) - - alreadyMergedKey := usageKey(legacyWheelOfNamesBetaCommand, 7) - seeds := map[string]usageEntry{ - alreadyMergedKey: { - Cmd: legacyWheelOfNamesBetaCommand, - UserID: 7, - Username: "alice", - N: 3, - }, - usageKey(currentWheelOfNamesCommand, 7): { - Cmd: currentWheelOfNamesCommand, - UserID: 7, - Username: "alice", - N: 8, - MergedFrom: []string{alreadyMergedKey}, - }, - usageKey(legacyWheelOfNamesBetaCommand, 9): { - Cmd: legacyWheelOfNamesBetaCommand, - UserID: 9, - Username: "carol", - N: 6, - }, - } - for key, entry := range seeds { - if err := docs.Put(ctx, key, entry); err != nil { - t.Fatalf("seed %s: %v", key, err) - } - } - - if err := InitStore(ctx, statsColl, systemColl); err != nil { - t.Fatalf("InitStore: %v", err) - } - if err := InitStore(ctx, statsColl, systemColl); err != nil { - t.Fatalf("InitStore second run: %v", err) - } - - alice, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 7)) - if err != nil { - t.Fatalf("alice wheelofnames stats: %v", err) - } - if alice.N != 8 || len(alice.MergedFrom) != 0 { - t.Fatalf("alice stats = %+v, want count still 8 with cleanup marker removed", alice) - } - carol, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 9)) - if err != nil { - t.Fatalf("carol wheelofnames stats: %v", err) - } - if carol.N != 6 || carol.Deleted || len(carol.MergedFrom) != 0 { - t.Fatalf("carol stats = %+v, want visible count 6", carol) - } - for _, key := range []string{alreadyMergedKey, usageKey(legacyWheelOfNamesBetaCommand, 9)} { - if _, _, err := docs.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) { - t.Fatalf("legacy key %s err = %v, want ErrNotFound", key, err) - } - } - if got := renderStats(ctx, newCounter(statsColl), "cmd wheelofnames"); !strings.Contains(got, "@alice: 8") || !strings.Contains(got, "@carol: 6") { - t.Fatalf("wheelofnames users after retry = %q, want no duplicate alice count and visible carol count", got) - } -} diff --git a/internal/modules/stats/usage_store.go b/internal/modules/stats/usage_store.go index 4273512..9d81f4d 100644 --- a/internal/modules/stats/usage_store.go +++ b/internal/modules/stats/usage_store.go @@ -26,9 +26,6 @@ type usageEntry struct { Username string `json:"user,omitempty" bson:"user,omitempty"` N int64 `json:"n" bson:"n"` Deleted bool `json:"deleted,omitempty" bson:"deleted,omitempty"` - // MergedFrom is temporary startup-migration bookkeeping. Completed - // migrations clear it so normal stats documents stay compact. - MergedFrom []string `json:"mergedFrom,omitempty" bson:"mergedFrom,omitempty"` } type usageUser struct {