refactor(stats): remove completed startup migrations

This commit is contained in:
2026-07-09 15:44:23 +07:00
parent a59e9a32c3
commit 5600a80ae0
6 changed files with 13 additions and 437 deletions
+1 -2
View File
@@ -29,7 +29,6 @@ import (
"github.com/tiennm99/miti99bot/internal/modules/wordle"
"github.com/tiennm99/miti99bot/internal/server"
"github.com/tiennm99/miti99bot/internal/storage"
"github.com/tiennm99/miti99bot/internal/systemstate"
"github.com/tiennm99/miti99bot/internal/telegram"
)
@@ -95,7 +94,7 @@ func main() {
}
defer closeProvider()
if err := stats.InitStore(rootCtx, provider.Collection("stats"), provider.Collection(systemstate.CollectionName)); err != nil {
if err := stats.InitStore(rootCtx, provider.Collection("stats")); err != nil {
log.Fatal("stats storage init failed", "err", err)
}
if err := lol.InitStore(rootCtx, provider.Collection(lol.CollectionName)); err != nil {
+5 -5
View File
@@ -99,11 +99,11 @@ Successful GIF replies include the result behind Telegram spoiler formatting.
> `updatedAt` is a BSON Date.
>
> The `stats` collection uses queryable aggregate documents for command/user
> counts and creates indexes on startup. Renamed command rows are merged into the
> current command name. Deleted legacy command rows are retained with
> `deleted: true`; `/stats` queries filter those rows from visible results. A
> historical `system` collection may remain in MongoDB with completed migration
> records and can be reused if a future one-time startup migration is needed.
> counts and creates indexes on startup. Deleted legacy command rows are
> retained with `deleted: true`; `/stats` queries filter those rows from visible
> results. A historical `system` collection may remain in MongoDB with completed
> migration records and can be reused if a future one-time startup migration is
> needed.
## 2. Coolify
+2 -125
View File
@@ -2,152 +2,29 @@ package stats
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"time"
"go.mongodb.org/mongo-driver/v2/bson"
"go.mongodb.org/mongo-driver/v2/mongo"
"go.mongodb.org/mongo-driver/v2/mongo/options"
"github.com/tiennm99/miti99bot/internal/storage"
"github.com/tiennm99/miti99bot/internal/systemstate"
)
const (
statsCommandUsersIndexName = "stats_cmd_n_user"
statsUserCommandsIndexName = "stats_uid_n_cmd"
statsUsernameLookupIndexName = "stats_user_uid"
renameLolNextWeekStatsKey = "stats:command-rename:lol_nextweek-to-lol_next_week"
oldLolNextWeekCommand = "lol_nextweek"
newLolNextWeekCommand = "lol_next_week"
renameWheelOfNamesBetaStatsKey = "stats:command-rename:wheelofnamesbeta-to-wheelofnames"
legacyWheelOfNamesBetaCommand = "wheelofnamesbeta"
currentWheelOfNamesCommand = "wheelofnames"
)
// InitStore performs stats collection startup maintenance. It is safe to call
// every boot: MongoDB indexes are created idempotently and one-time command
// rename migrations are guarded by the shared system collection.
func InitStore(ctx context.Context, statsColl, systemColl storage.Collection) error {
// every boot: MongoDB indexes are created idempotently.
func InitStore(ctx context.Context, statsColl storage.Collection) error {
if mongoColl, ok := storage.MongoCollection(statsColl); ok {
if err := ensureUsageIndexes(ctx, mongoColl); err != nil {
return err
}
}
if err := migrateCommandRename(ctx, statsColl, systemColl, oldLolNextWeekCommand, newLolNextWeekCommand, renameLolNextWeekStatsKey); err != nil {
return err
}
if err := migrateCommandRename(ctx, statsColl, systemColl, legacyWheelOfNamesBetaCommand, currentWheelOfNamesCommand, renameWheelOfNamesBetaStatsKey); err != nil {
return err
}
return nil
}
func migrateCommandRename(ctx context.Context, statsColl, systemColl storage.Collection, oldCmd, newCmd, markerKey string) error {
state := systemstate.New(systemColl)
if rec, ok, err := state.Get(ctx, markerKey); err != nil {
return fmt.Errorf("stats command rename marker %s: %w", markerKey, err)
} else if ok && rec.Status == "complete" {
return nil
}
docs := storage.Typed[usageEntry](statsColl)
keys, err := docs.List(ctx, oldCmd)
if err != nil {
return fmt.Errorf("stats command rename list %s: %w", oldCmd, err)
}
var moved int64
for _, key := range keys {
if key != oldCmd && !strings.HasPrefix(key, oldCmd+":") {
continue
}
entry, _, err := docs.Get(ctx, key)
if err != nil {
return fmt.Errorf("stats command rename get %s: %w", key, err)
}
if entry.Cmd != oldCmd {
continue
}
targetKey := usageKey(newCmd, entry.UserID)
target, _, err := docs.Get(ctx, targetKey)
if err != nil && !errors.Is(err, storage.ErrNotFound) {
return fmt.Errorf("stats command rename get target %s: %w", targetKey, err)
}
if errors.Is(err, storage.ErrNotFound) {
target = usageEntry{}
}
if slices.Contains(target.MergedFrom, key) {
if err := docs.Delete(ctx, key); err != nil {
return fmt.Errorf("stats command rename delete already-merged old %s: %w", key, err)
}
continue
}
target.Cmd = newCmd
target.UserID = entry.UserID
if entry.UserID == 0 {
target.Username = ""
} else if entry.Username != "" {
target.Username = entry.Username
}
target.N += entry.N
target.Deleted = false
target.MergedFrom = append(target.MergedFrom, key)
if err := docs.Put(ctx, targetKey, target); err != nil {
return fmt.Errorf("stats command rename put target %s: %w", targetKey, err)
}
if err := docs.Delete(ctx, key); err != nil {
return fmt.Errorf("stats command rename delete old %s: %w", key, err)
}
moved += entry.N
}
if err := clearCommandMergeMarkers(ctx, docs, newCmd); err != nil {
return err
}
now := time.Now().UTC().UnixMilli()
if err := state.Put(ctx, markerKey, systemstate.Record{
Kind: "migration",
Name: markerKey,
Status: "complete",
Count: moved,
CompletedAt: now,
UpdatedAt: now,
}); err != nil {
return fmt.Errorf("stats command rename marker put %s: %w", markerKey, err)
}
return nil
}
func clearCommandMergeMarkers(ctx context.Context, docs storage.DocStore[usageEntry], cmd string) error {
keys, err := docs.List(ctx, cmd)
if err != nil {
return fmt.Errorf("stats command rename cleanup list %s: %w", cmd, err)
}
for _, key := range keys {
if key != cmd && !strings.HasPrefix(key, cmd+":") {
continue
}
entry, _, err := docs.Get(ctx, key)
if err != nil {
return fmt.Errorf("stats command rename cleanup get %s: %w", key, err)
}
if entry.Cmd != cmd || len(entry.MergedFrom) == 0 {
continue
}
entry.MergedFrom = nil
if err := docs.Put(ctx, key, entry); err != nil {
return fmt.Errorf("stats command rename cleanup put %s: %w", key, err)
}
}
return nil
}
+5 -65
View File
@@ -2,29 +2,26 @@ package stats
import (
"context"
"errors"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/tiennm99/miti99bot/internal/storage"
"github.com/tiennm99/miti99bot/internal/systemstate"
)
func TestInitStore_MongoCreatesIndexes(t *testing.T) {
ctx, statsColl, systemColl := setupMongoStatsTest(t)
ctx, statsColl := setupMongoStatsTest(t)
rawStatsColl, ok := storage.MongoCollection(statsColl)
if !ok {
t.Fatal("stats collection is not Mongo-backed")
}
if err := InitStore(ctx, statsColl, systemColl); err != nil {
if err := InitStore(ctx, statsColl); err != nil {
t.Fatalf("InitStore: %v", err)
}
if err := InitStore(ctx, statsColl, systemColl); err != nil {
if err := InitStore(ctx, statsColl); err != nil {
t.Fatalf("InitStore second run: %v", err)
}
@@ -54,64 +51,7 @@ func TestInitStore_MongoCreatesIndexes(t *testing.T) {
}
}
func TestInitStore_MongoMergesLegacyWheelOfNamesBetaStats(t *testing.T) {
ctx, statsColl, systemColl := setupMongoStatsTest(t)
docs := storage.Typed[usageEntry](statsColl)
seeds := map[string]usageEntry{
usageKey(legacyWheelOfNamesBetaCommand, 0): {Cmd: legacyWheelOfNamesBetaCommand, N: 2},
usageKey(legacyWheelOfNamesBetaCommand, 7): {
Cmd: legacyWheelOfNamesBetaCommand,
UserID: 7,
Username: "alice",
N: 3,
Deleted: true,
},
usageKey(currentWheelOfNamesCommand, 7): {
Cmd: currentWheelOfNamesCommand,
UserID: 7,
Username: "alice",
N: 5,
},
}
for key, entry := range seeds {
if err := docs.Put(ctx, key, entry); err != nil {
t.Fatalf("seed %s: %v", key, err)
}
}
if err := InitStore(ctx, statsColl, systemColl); err != nil {
t.Fatalf("InitStore: %v", err)
}
if err := InitStore(ctx, statsColl, systemColl); err != nil {
t.Fatalf("InitStore second run: %v", err)
}
anon, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 0))
if err != nil {
t.Fatalf("merged anonymous wheelofnames stats: %v", err)
}
if anon.N != 2 || anon.Deleted || len(anon.MergedFrom) != 0 {
t.Fatalf("merged anonymous stats = %+v, want visible count 2", anon)
}
alice, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 7))
if err != nil {
t.Fatalf("merged alice wheelofnames stats: %v", err)
}
if alice.N != 8 || alice.Deleted || len(alice.MergedFrom) != 0 {
t.Fatalf("merged alice stats = %+v, want visible count 8", alice)
}
for _, key := range []string{usageKey(legacyWheelOfNamesBetaCommand, 0), usageKey(legacyWheelOfNamesBetaCommand, 7)} {
if _, _, err := docs.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) {
t.Fatalf("legacy key %s err = %v, want ErrNotFound", key, err)
}
}
if got := renderStats(ctx, newCounter(statsColl), ""); !strings.Contains(got, "/wheelofnames: 10") || strings.Contains(got, "wheelofnamesbeta") {
t.Fatalf("top commands after Mongo migration = %q, want merged visible /wheelofnames count 10 only", got)
}
}
func setupMongoStatsTest(t *testing.T) (context.Context, storage.Collection, storage.Collection) {
func setupMongoStatsTest(t *testing.T) (context.Context, storage.Collection) {
t.Helper()
uri := os.Getenv("MONGODB_TEST_URL")
@@ -136,5 +76,5 @@ func setupMongoStatsTest(t *testing.T) (context.Context, storage.Collection, sto
})
provider := storage.NewMongoProvider(db)
return ctx, provider.Collection("stats"), provider.Collection(systemstate.CollectionName)
return ctx, provider.Collection("stats")
}
-237
View File
@@ -1,237 +0,0 @@
package stats
import (
"context"
"errors"
"strings"
"testing"
"github.com/tiennm99/miti99bot/internal/storage"
"github.com/tiennm99/miti99bot/internal/systemstate"
)
func TestInitStore_RenamesLolNextWeekStatsOnce(t *testing.T) {
ctx := context.Background()
provider := storage.NewMemoryProvider()
statsColl := provider.Collection("stats")
systemColl := provider.Collection(systemstate.CollectionName)
docs := storage.Typed[usageEntry](statsColl)
seeds := map[string]usageEntry{
usageKey(oldLolNextWeekCommand, 0): {Cmd: oldLolNextWeekCommand, N: 2},
usageKey(oldLolNextWeekCommand, 7): {
Cmd: oldLolNextWeekCommand,
UserID: 7,
Username: "alice",
N: 3,
},
usageKey(newLolNextWeekCommand, 7): {
Cmd: newLolNextWeekCommand,
UserID: 7,
Username: "alice",
N: 5,
},
}
for key, entry := range seeds {
if err := docs.Put(ctx, key, entry); err != nil {
t.Fatalf("seed %s: %v", key, err)
}
}
if err := InitStore(ctx, statsColl, systemColl); err != nil {
t.Fatalf("InitStore: %v", err)
}
if err := InitStore(ctx, statsColl, systemColl); err != nil {
t.Fatalf("InitStore second run: %v", err)
}
anon, _, err := docs.Get(ctx, usageKey(newLolNextWeekCommand, 0))
if err != nil {
t.Fatalf("new anonymous stats: %v", err)
}
if anon.Cmd != newLolNextWeekCommand || anon.N != 2 || anon.UserID != 0 {
t.Fatalf("new anonymous stats = %+v, want cmd %q count 2", anon, newLolNextWeekCommand)
}
user, _, err := docs.Get(ctx, usageKey(newLolNextWeekCommand, 7))
if err != nil {
t.Fatalf("new user stats: %v", err)
}
if user.Cmd != newLolNextWeekCommand || user.UserID != 7 || user.Username != "alice" || user.N != 8 {
t.Fatalf("new user stats = %+v, want merged count 8", user)
}
for _, key := range []string{usageKey(oldLolNextWeekCommand, 0), usageKey(oldLolNextWeekCommand, 7)} {
if _, _, err := docs.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) {
t.Fatalf("old key %s err = %v, want ErrNotFound", key, err)
}
}
rec, ok, err := systemstate.New(systemColl).Get(ctx, renameLolNextWeekStatsKey)
if err != nil {
t.Fatalf("migration marker: %v", err)
}
if !ok || rec.Status != "complete" || rec.Count != 5 {
t.Fatalf("migration marker = %+v ok=%v, want complete count 5", rec, ok)
}
}
func TestInitStore_MergesLegacyWheelOfNamesBetaStatsOnce(t *testing.T) {
ctx := context.Background()
provider := storage.NewMemoryProvider()
statsColl := provider.Collection("stats")
systemColl := provider.Collection(systemstate.CollectionName)
docs := storage.Typed[usageEntry](statsColl)
seeds := map[string]usageEntry{
usageKey(legacyWheelOfNamesBetaCommand, 0): {Cmd: legacyWheelOfNamesBetaCommand, N: 2},
usageKey(legacyWheelOfNamesBetaCommand, 7): {
Cmd: legacyWheelOfNamesBetaCommand,
UserID: 7,
Username: "alice",
N: 3,
},
usageKey(legacyWheelOfNamesBetaCommand, 8): {
Cmd: legacyWheelOfNamesBetaCommand,
UserID: 8,
Username: "bob",
N: 4,
Deleted: true,
},
usageKey(currentWheelOfNamesCommand, 7): {
Cmd: currentWheelOfNamesCommand,
UserID: 7,
Username: "alice",
N: 5,
},
}
for key, entry := range seeds {
if err := docs.Put(ctx, key, entry); err != nil {
t.Fatalf("seed %s: %v", key, err)
}
}
if err := InitStore(ctx, statsColl, systemColl); err != nil {
t.Fatalf("InitStore: %v", err)
}
if err := InitStore(ctx, statsColl, systemColl); err != nil {
t.Fatalf("InitStore second run: %v", err)
}
anon, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 0))
if err != nil {
t.Fatalf("merged anonymous wheelofnames stats: %v", err)
}
if anon.Cmd != currentWheelOfNamesCommand || anon.N != 2 || anon.UserID != 0 || anon.Deleted || len(anon.MergedFrom) != 0 {
t.Fatalf("merged anonymous stats = %+v, want visible count 2", anon)
}
alice, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 7))
if err != nil {
t.Fatalf("merged alice wheelofnames stats: %v", err)
}
if alice.Cmd != currentWheelOfNamesCommand || alice.UserID != 7 || alice.Username != "alice" || alice.N != 8 || alice.Deleted || len(alice.MergedFrom) != 0 {
t.Fatalf("merged alice stats = %+v, want visible count 8", alice)
}
bob, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 8))
if err != nil {
t.Fatalf("merged bob wheelofnames stats: %v", err)
}
if bob.Cmd != currentWheelOfNamesCommand || bob.UserID != 8 || bob.Username != "bob" || bob.N != 4 || bob.Deleted || len(bob.MergedFrom) != 0 {
t.Fatalf("merged bob stats = %+v, want visible count 4", bob)
}
renderedTopCommands := renderStats(ctx, newCounter(statsColl), "")
if !strings.Contains(renderedTopCommands, "/wheelofnames: 14") || strings.Contains(renderedTopCommands, "wheelofnamesbeta") {
t.Fatalf("top commands after migration = %q, want merged visible /wheelofnames count 14 only", renderedTopCommands)
}
renderedCommandUsers := renderStats(ctx, newCounter(statsColl), "cmd wheelofnames")
if !strings.Contains(renderedCommandUsers, "@alice: 8") || !strings.Contains(renderedCommandUsers, "@bob: 4") {
t.Fatalf("wheelofnames users after migration = %q, want merged users", renderedCommandUsers)
}
for _, key := range []string{
usageKey(legacyWheelOfNamesBetaCommand, 0),
usageKey(legacyWheelOfNamesBetaCommand, 7),
usageKey(legacyWheelOfNamesBetaCommand, 8),
} {
if _, _, err := docs.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) {
t.Fatalf("legacy key %s err = %v, want ErrNotFound", key, err)
}
}
rec, ok, err := systemstate.New(systemColl).Get(ctx, renameWheelOfNamesBetaStatsKey)
if err != nil {
t.Fatalf("migration marker: %v", err)
}
if !ok || rec.Status != "complete" || rec.Count != 9 {
t.Fatalf("migration marker = %+v ok=%v, want complete count 9", rec, ok)
}
}
func TestInitStore_RetriesPartiallyMergedWheelOfNamesBetaStats(t *testing.T) {
ctx := context.Background()
provider := storage.NewMemoryProvider()
statsColl := provider.Collection("stats")
systemColl := provider.Collection(systemstate.CollectionName)
docs := storage.Typed[usageEntry](statsColl)
alreadyMergedKey := usageKey(legacyWheelOfNamesBetaCommand, 7)
seeds := map[string]usageEntry{
alreadyMergedKey: {
Cmd: legacyWheelOfNamesBetaCommand,
UserID: 7,
Username: "alice",
N: 3,
},
usageKey(currentWheelOfNamesCommand, 7): {
Cmd: currentWheelOfNamesCommand,
UserID: 7,
Username: "alice",
N: 8,
MergedFrom: []string{alreadyMergedKey},
},
usageKey(legacyWheelOfNamesBetaCommand, 9): {
Cmd: legacyWheelOfNamesBetaCommand,
UserID: 9,
Username: "carol",
N: 6,
},
}
for key, entry := range seeds {
if err := docs.Put(ctx, key, entry); err != nil {
t.Fatalf("seed %s: %v", key, err)
}
}
if err := InitStore(ctx, statsColl, systemColl); err != nil {
t.Fatalf("InitStore: %v", err)
}
if err := InitStore(ctx, statsColl, systemColl); err != nil {
t.Fatalf("InitStore second run: %v", err)
}
alice, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 7))
if err != nil {
t.Fatalf("alice wheelofnames stats: %v", err)
}
if alice.N != 8 || len(alice.MergedFrom) != 0 {
t.Fatalf("alice stats = %+v, want count still 8 with cleanup marker removed", alice)
}
carol, _, err := docs.Get(ctx, usageKey(currentWheelOfNamesCommand, 9))
if err != nil {
t.Fatalf("carol wheelofnames stats: %v", err)
}
if carol.N != 6 || carol.Deleted || len(carol.MergedFrom) != 0 {
t.Fatalf("carol stats = %+v, want visible count 6", carol)
}
for _, key := range []string{alreadyMergedKey, usageKey(legacyWheelOfNamesBetaCommand, 9)} {
if _, _, err := docs.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) {
t.Fatalf("legacy key %s err = %v, want ErrNotFound", key, err)
}
}
if got := renderStats(ctx, newCounter(statsColl), "cmd wheelofnames"); !strings.Contains(got, "@alice: 8") || !strings.Contains(got, "@carol: 6") {
t.Fatalf("wheelofnames users after retry = %q, want no duplicate alice count and visible carol count", got)
}
}
-3
View File
@@ -26,9 +26,6 @@ type usageEntry struct {
Username string `json:"user,omitempty" bson:"user,omitempty"`
N int64 `json:"n" bson:"n"`
Deleted bool `json:"deleted,omitempty" bson:"deleted,omitempty"`
// MergedFrom is temporary startup-migration bookkeeping. Completed
// migrations clear it so normal stats documents stay compact.
MergedFrom []string `json:"mergedFrom,omitempty" bson:"mergedFrom,omitempty"`
}
type usageUser struct {