diff --git a/docs/deploy-coolify-selfhosted.md b/docs/deploy-coolify-selfhosted.md index bdad904..e281ae2 100644 --- a/docs/deploy-coolify-selfhosted.md +++ b/docs/deploy-coolify-selfhosted.md @@ -76,7 +76,10 @@ overrides are not supported in runtime env; modules use coded defaults. > The `stats` collection uses queryable aggregate documents for command/user > counts and creates indexes on startup. First startup after the schema change > migrates legacy `count:`, `user:`, and `pair:` stats keys into the new shape, -> deletes the legacy keys, and records completion in `system`. +> deletes the legacy keys, and records completion in `system`. Startup also +> migrates renamed command stats to the current command names and marks removed +> command rows with `deleted: true`; `/stats` queries filter those retained +> legacy rows. ## 2. Coolify diff --git a/internal/modules/stats/startup.go b/internal/modules/stats/startup.go index 80daf26..dbd78bf 100644 --- a/internal/modules/stats/startup.go +++ b/internal/modules/stats/startup.go @@ -2,6 +2,7 @@ package stats import ( "context" + "errors" "fmt" "strconv" "strings" @@ -22,8 +23,35 @@ const ( usageMigrationName = "stats-usage-v2" usageMigrationKey = "migration:" + usageMigrationName + + commandHistoryMigrationName = "stats-command-history-v1" + commandHistoryMigrationKey = "migration:" + commandHistoryMigrationName ) +type commandRename struct { + Old string + New string +} + +var commandRenames = []commandRename{ + {Old: "lolschedule", New: "lol"}, + {Old: "lolschedule_week", New: "lol_this_week"}, + {Old: "lolschedule_subscribe", New: "lol_subscribe"}, + {Old: "lolschedule_unsubscribe", New: "lol_unsubscribe"}, + {Old: "wc_week", New: "wc_this_week"}, + {Old: "gold_stats", New: "gold_portfolio"}, + {Old: "coin_stats", New: "coin_portfolio"}, + {Old: "stock_stats", New: "stock_portfolio"}, + {Old: "stock_income_stock", New: "stock_bonus"}, + {Old: "stock_income_vnd", New: "stock_dividend"}, +} + +var deletedCommandNames = []string{ + "lolschedule_today", + "wc_today", + "stock_convert", +} + type legacyCountEntry struct { N int64 `json:"n" bson:"n"` } @@ -42,7 +70,10 @@ func InitStore(ctx context.Context, statsColl, systemColl storage.Collection) er return err } } - return migrateLegacyUsage(ctx, statsColl, systemColl) + if err := migrateLegacyUsage(ctx, statsColl, systemColl); err != nil { + return err + } + return migrateCommandHistory(ctx, statsColl, systemColl) } func ensureUsageIndexes(ctx context.Context, coll *mongo.Collection) error { @@ -130,6 +161,130 @@ func migrateLegacyUsage(ctx context.Context, statsColl, systemColl storage.Colle return nil } +func migrateCommandHistory(ctx context.Context, statsColl, systemColl storage.Collection) error { + sys := systemstate.New(systemColl) + if rec, ok, err := sys.Get(ctx, commandHistoryMigrationKey); err != nil { + return fmt.Errorf("stats command history migration marker get: %w", err) + } else if ok && rec.Status == "done" { + return nil + } + + docs := storage.Typed[usageEntry](statsColl) + changed := int64(0) + for _, rename := range commandRenames { + n, err := migrateCommandRename(ctx, docs, rename) + if err != nil { + return err + } + changed += n + } + for _, cmd := range deletedCommandNames { + n, err := markCommandDeleted(ctx, docs, cmd) + if err != nil { + return err + } + changed += n + } + + now := nowMillis() + if err := sys.Put(ctx, commandHistoryMigrationKey, systemstate.Record{ + Kind: "migration", + Name: commandHistoryMigrationName, + Status: "done", + Count: changed, + CompletedAt: now, + UpdatedAt: now, + }); err != nil { + return fmt.Errorf("stats command history migration marker put: %w", err) + } + return nil +} + +func migrateCommandRename(ctx context.Context, docs storage.DocStore[usageEntry], rename commandRename) (int64, error) { + keys, err := docs.List(ctx, rename.Old) + if err != nil { + return 0, fmt.Errorf("stats command rename list %s: %w", rename.Old, err) + } + changed := int64(0) + for _, key := range keys { + if !usageKeyBelongsToCommand(key, rename.Old) { + continue + } + entry, _, err := docs.Get(ctx, key) + if err != nil { + return changed, fmt.Errorf("stats command rename get %s: %w", key, err) + } + if entry.Cmd != "" && entry.Cmd != rename.Old { + continue + } + entry.Cmd = rename.Old + targetKey := usageKey(rename.New, entry.UserID) + target, _, err := docs.Get(ctx, targetKey) + switch { + case errors.Is(err, storage.ErrNotFound): + target = usageEntry{ + Cmd: rename.New, + UserID: entry.UserID, + Username: entry.Username, + } + case err != nil: + return changed, fmt.Errorf("stats command rename target get %s: %w", targetKey, err) + } + + target.Cmd = rename.New + target.UserID = entry.UserID + if entry.UserID == 0 { + target.Username = "" + } else if entry.Username != "" { + target.Username = entry.Username + } + target.N += entry.N + target.Deleted = false + if err := docs.Put(ctx, targetKey, target); err != nil { + return changed, fmt.Errorf("stats command rename put %s: %w", targetKey, err) + } + if err := docs.Delete(ctx, key); err != nil { + return changed, fmt.Errorf("stats command rename delete %s: %w", key, err) + } + changed++ + } + return changed, nil +} + +func markCommandDeleted(ctx context.Context, docs storage.DocStore[usageEntry], cmd string) (int64, error) { + keys, err := docs.List(ctx, cmd) + if err != nil { + return 0, fmt.Errorf("stats command delete list %s: %w", cmd, err) + } + changed := int64(0) + for _, key := range keys { + if !usageKeyBelongsToCommand(key, cmd) { + continue + } + entry, _, err := docs.Get(ctx, key) + if err != nil { + return changed, fmt.Errorf("stats command delete get %s: %w", key, err) + } + if entry.Cmd != "" && entry.Cmd != cmd { + continue + } + entry.Cmd = cmd + if entry.Deleted { + continue + } + entry.Deleted = true + if err := docs.Put(ctx, key, entry); err != nil { + return changed, fmt.Errorf("stats command delete put %s: %w", key, err) + } + changed++ + } + return changed, nil +} + +func usageKeyBelongsToCommand(key, cmd string) bool { + return key == cmd || strings.HasPrefix(key, cmd+":") +} + func loadLegacyUsernames(ctx context.Context, users storage.DocStore[legacyUserEntry]) (map[int64]string, []string, error) { keys, err := users.List(ctx, legacyUserPrefix) if err != nil { diff --git a/internal/modules/stats/startup_mongo_test.go b/internal/modules/stats/startup_mongo_test.go index f837362..9fe1cd8 100644 --- a/internal/modules/stats/startup_mongo_test.go +++ b/internal/modules/stats/startup_mongo_test.go @@ -45,12 +45,18 @@ func TestInitStore_MongoCreatesIndexesAndMigratesLegacy(t *testing.T) { if err := legacyCounts.Put(ctx, legacyCountPrefix+"ping", legacyCountEntry{N: 2}); err != nil { t.Fatalf("legacy count: %v", err) } + if err := legacyCounts.Put(ctx, legacyCountPrefix+"gold_stats", legacyCountEntry{N: 4}); err != nil { + t.Fatalf("legacy renamed count: %v", err) + } if err := legacyUsers.Put(ctx, legacyUserPrefix+"7", legacyUserEntry{Username: "alice", N: 1}); err != nil { t.Fatalf("legacy user: %v", err) } if err := legacyCounts.Put(ctx, legacyPairPrefix+"ping:7", legacyCountEntry{N: 1}); err != nil { t.Fatalf("legacy pair: %v", err) } + if err := legacyCounts.Put(ctx, legacyPairPrefix+"stock_convert:7", legacyCountEntry{N: 3}); err != nil { + t.Fatalf("legacy deleted pair: %v", err) + } if err := InitStore(ctx, statsColl, systemColl); err != nil { t.Fatalf("InitStore: %v", err) @@ -59,6 +65,8 @@ func TestInitStore_MongoCreatesIndexesAndMigratesLegacy(t *testing.T) { usageDocs := storage.Typed[usageEntry](statsColl) assertUsageEntry(t, usageDocs, usageKey("ping", 7), usageEntry{Cmd: "ping", UserID: 7, Username: "alice", N: 1}) assertUsageEntry(t, usageDocs, usageKey("ping", 0), usageEntry{Cmd: "ping", N: 1}) + assertUsageEntry(t, usageDocs, usageKey("gold_portfolio", 0), usageEntry{Cmd: "gold_portfolio", N: 4}) + assertUsageEntry(t, usageDocs, usageKey("stock_convert", 7), usageEntry{Cmd: "stock_convert", UserID: 7, Username: "alice", N: 3, Deleted: true}) rawStatsColl, ok := storage.MongoCollection(statsColl) if !ok { @@ -89,6 +97,15 @@ func TestInitStore_MongoCreatesIndexesAndMigratesLegacy(t *testing.T) { } } + sys := systemstate.New(systemColl) + rec, ok, err := sys.Get(ctx, commandHistoryMigrationKey) + if err != nil || !ok { + t.Fatalf("command history marker ok=%v err=%v", ok, err) + } + if rec.Status != "done" || rec.Count != 2 { + t.Fatalf("command history marker = %+v, want done count 2", rec) + } + rawDoc := bson.M{} err = rawStatsColl.FindOne(ctx, bson.M{"_id": legacyCountPrefix + "ping"}).Decode(&rawDoc) if err == nil { diff --git a/internal/modules/stats/stats_test.go b/internal/modules/stats/stats_test.go index dc8afe2..084c284 100644 --- a/internal/modules/stats/stats_test.go +++ b/internal/modules/stats/stats_test.go @@ -312,6 +312,110 @@ func TestInitStore_MigratesLegacyStatsOnceAndDeletesOldKeys(t *testing.T) { } } +func TestInitStore_MigratesCommandHistoryAndMarksDeleted(t *testing.T) { + ctx := context.Background() + provider := storage.NewMemoryProvider() + statsColl := provider.Collection("stats") + systemColl := provider.Collection(systemstate.CollectionName) + usageDocs := storage.Typed[usageEntry](statsColl) + + seed := map[string]usageEntry{ + usageKey("gold_stats", 0): {Cmd: "gold_stats", N: 2}, + usageKey("gold_portfolio", 0): {Cmd: "gold_portfolio", N: 5}, + usageKey("stock_income_stock", 7): {Cmd: "stock_income_stock", UserID: 7, Username: "alice", N: 3}, + usageKey("stock_bonus", 7): {Cmd: "stock_bonus", UserID: 7, Username: "alice", N: 4}, + usageKey("lolschedule_week", 8): {Cmd: "lolschedule_week", UserID: 8, Username: "bob", N: 1}, + usageKey("lolschedule_today", 0): {Cmd: "lolschedule_today", N: 9}, + usageKey("stock_convert", 7): {Cmd: "stock_convert", UserID: 7, Username: "alice", N: 2}, + usageKey("stock_convert_extra", 7): {Cmd: "stock_convert_extra", UserID: 7, Username: "alice", N: 99}, + usageKey("lolschedule_weekly", 10): {Cmd: "lolschedule_weekly", UserID: 10, Username: "carol", N: 99}, + usageKey("stock_income_stocked", 11): {Cmd: "stock_income_stocked", UserID: 11, Username: "dan", N: 99}, + } + for key, entry := range seed { + if err := usageDocs.Put(ctx, key, entry); err != nil { + t.Fatalf("seed %s: %v", key, err) + } + } + + if err := InitStore(ctx, statsColl, systemColl); err != nil { + t.Fatalf("InitStore: %v", err) + } + + assertUsageEntry(t, usageDocs, usageKey("gold_portfolio", 0), usageEntry{Cmd: "gold_portfolio", N: 7}) + assertUsageEntry(t, usageDocs, usageKey("stock_bonus", 7), usageEntry{Cmd: "stock_bonus", UserID: 7, Username: "alice", N: 7}) + assertUsageEntry(t, usageDocs, usageKey("lol_this_week", 8), usageEntry{Cmd: "lol_this_week", UserID: 8, Username: "bob", N: 1}) + assertUsageEntry(t, usageDocs, usageKey("lolschedule_today", 0), usageEntry{Cmd: "lolschedule_today", N: 9, Deleted: true}) + assertUsageEntry(t, usageDocs, usageKey("stock_convert", 7), usageEntry{Cmd: "stock_convert", UserID: 7, Username: "alice", N: 2, Deleted: true}) + assertUsageEntry(t, usageDocs, usageKey("stock_convert_extra", 7), usageEntry{Cmd: "stock_convert_extra", UserID: 7, Username: "alice", N: 99}) + assertUsageEntry(t, usageDocs, usageKey("lolschedule_weekly", 10), usageEntry{Cmd: "lolschedule_weekly", UserID: 10, Username: "carol", N: 99}) + assertUsageEntry(t, usageDocs, usageKey("stock_income_stocked", 11), usageEntry{Cmd: "stock_income_stocked", UserID: 11, Username: "dan", N: 99}) + + for _, key := range []string{ + usageKey("gold_stats", 0), + usageKey("stock_income_stock", 7), + usageKey("lolschedule_week", 8), + } { + if _, _, err := usageDocs.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("renamed key %s should be deleted, got err=%v", key, err) + } + } + + sys := systemstate.New(systemColl) + rec, ok, err := sys.Get(ctx, commandHistoryMigrationKey) + if err != nil || !ok { + t.Fatalf("command history marker ok=%v err=%v", ok, err) + } + if rec.Status != "done" || rec.Count != 5 { + t.Fatalf("command history marker = %+v, want done count 5", rec) + } + + if err := InitStore(ctx, statsColl, systemColl); err != nil { + t.Fatalf("InitStore second run: %v", err) + } + assertUsageEntry(t, usageDocs, usageKey("gold_portfolio", 0), usageEntry{Cmd: "gold_portfolio", N: 7}) +} + +func TestStatsViewsFilterDeletedLegacyRows(t *testing.T) { + ctx := context.Background() + provider := storage.NewMemoryProvider() + c := newCounter(provider.Collection("stats")) + docs := storage.Typed[usageEntry](provider.Collection("stats")) + + for key, entry := range map[string]usageEntry{ + usageKey("ping", 1): {Cmd: "ping", UserID: 1, Username: "alice", N: 3}, + usageKey("wordle", 2): {Cmd: "wordle", UserID: 2, Username: "bob", N: 2}, + usageKey("wc_today", 0): {Cmd: "wc_today", N: 100, Deleted: true}, + usageKey("wc_today", 1): {Cmd: "wc_today", UserID: 1, Username: "alice", N: 100, Deleted: true}, + usageKey("stock_convert", 3): {Cmd: "stock_convert", UserID: 3, Username: "ghost", N: 50, Deleted: true}, + } { + if err := docs.Put(ctx, key, entry); err != nil { + t.Fatalf("seed %s: %v", key, err) + } + } + + for name, got := range map[string]string{ + "top commands": renderStats(ctx, c, ""), + "top users": renderStats(ctx, c, "users"), + "commands by user": renderStats(ctx, c, "user alice"), + } { + if strings.Contains(got, "wc_today") || strings.Contains(got, "stock_convert") || strings.Contains(got, "ghost") { + t.Fatalf("%s leaked deleted stats:\n%s", name, got) + } + } + if got := renderStats(ctx, c, "users"); !strings.Contains(got, "@alice: 3") || strings.Contains(got, "@alice: 103") { + t.Fatalf("top users did not filter deleted rows correctly:\n%s", got) + } + if got := renderStats(ctx, c, "user alice"); !strings.Contains(got, "/ping: 3") || strings.Contains(got, "/wc_today") { + t.Fatalf("commands by user did not filter deleted rows correctly:\n%s", got) + } + if got := renderStats(ctx, c, "cmd wc_today"); got != "Command /wc_today has no users yet." { + t.Fatalf("deleted command users reply = %q", got) + } + if got := renderStats(ctx, c, "user ghost"); got != "User @ghost not found." { + t.Fatalf("deleted-only user reply = %q", got) + } +} + func assertUsageEntry(t *testing.T, docs storage.DocStore[usageEntry], key string, want usageEntry) { t.Helper() got, _, err := docs.Get(context.Background(), key) diff --git a/internal/modules/stats/usage_store.go b/internal/modules/stats/usage_store.go index dbf7bf5..896e7c4 100644 --- a/internal/modules/stats/usage_store.go +++ b/internal/modules/stats/usage_store.go @@ -25,6 +25,7 @@ type usageEntry struct { UserID int64 `json:"uid,omitempty" bson:"uid,omitempty"` Username string `json:"user,omitempty" bson:"user,omitempty"` N int64 `json:"n" bson:"n"` + Deleted bool `json:"deleted,omitempty" bson:"deleted,omitempty"` } type usageUser struct { @@ -78,6 +79,7 @@ func (s *docUsageStore) Increment(ctx context.Context, cmd string, user usageUse entry.Cmd = cmd entry.N++ + entry.Deleted = false if hasUser { entry.UserID = user.ID entry.Username = user.Username @@ -121,6 +123,9 @@ func (s *docUsageStore) TopCommands(ctx context.Context, limit int) ([]row, erro } totals := make(map[string]int64) for _, e := range entries { + if e.Deleted { + continue + } totals[e.Cmd] += e.N } rows := make([]row, 0, len(totals)) @@ -145,7 +150,7 @@ func (s *docUsageStore) TopUsers(ctx context.Context, limit int) ([]row, error) } totals := make(map[int64]total) for _, e := range entries { - if e.UserID == 0 || e.Username == "" { + if e.Deleted || e.UserID == 0 || e.Username == "" { continue } t := totals[e.UserID] @@ -174,7 +179,7 @@ func (s *docUsageStore) CommandsByUser(ctx context.Context, username string, lim found bool ) for _, e := range entries { - if e.UserID != 0 && e.Username == username { + if !e.Deleted && e.UserID != 0 && e.Username == username { userID = e.UserID found = true break @@ -186,7 +191,7 @@ func (s *docUsageStore) CommandsByUser(ctx context.Context, username string, lim rows := make([]row, 0) for _, e := range entries { - if e.UserID == userID { + if !e.Deleted && e.UserID == userID { rows = append(rows, row{display: "/" + e.Cmd, n: e.N}) } } @@ -204,7 +209,7 @@ func (s *docUsageStore) UsersByCommand(ctx context.Context, cmd string, limit in } rows := make([]row, 0) for _, e := range entries { - if e.Cmd == cmd && e.UserID != 0 && e.Username != "" { + if !e.Deleted && e.Cmd == cmd && e.UserID != 0 && e.Username != "" { rows = append(rows, row{display: "@" + e.Username, n: e.N}) } } @@ -242,16 +247,19 @@ func (s *mongoUsageStore) Increment(ctx context.Context, cmd string, user usageU } set := bson.M{"cmd": cmd} + unset := bson.M{"deleted": ""} update := bson.M{ "$set": set, "$inc": bson.M{"n": int64(1), "version": int64(1)}, + "$unset": unset, "$currentDate": bson.M{"updatedAt": true}, } if hasUser { set["uid"] = user.ID set["user"] = user.Username } else { - update["$unset"] = bson.M{"uid": "", "user": ""} + unset["uid"] = "" + unset["user"] = "" } key := usageKey(cmd, userID) @@ -280,7 +288,10 @@ func (s *mongoUsageStore) Increment(ctx context.Context, cmd string, user usageU func (s *mongoUsageStore) TopCommands(ctx context.Context, limit int) ([]row, error) { pipeline := mongo.Pipeline{ - bson.D{bsonField("$match", bson.D{bsonField("cmd", bson.D{bsonField("$type", "string")})})}, + bson.D{bsonField("$match", bson.D{ + bsonField("cmd", bson.D{bsonField("$type", "string")}), + bsonField("deleted", bson.D{bsonField("$ne", true)}), + })}, bson.D{bsonField("$group", bson.D{bsonField("_id", "$cmd"), bsonField("n", bson.D{bsonField("$sum", "$n")})})}, bson.D{bsonField("$sort", bson.D{bsonField("n", -1), bsonField("_id", 1)})}, } @@ -314,6 +325,7 @@ func (s *mongoUsageStore) TopUsers(ctx context.Context, limit int) ([]row, error bson.D{bsonField("$match", bson.D{ bsonField("uid", bson.D{bsonField("$gt", int64(0))}), bsonField("user", bson.D{bsonField("$type", "string"), bsonField("$ne", "")}), + bsonField("deleted", bson.D{bsonField("$ne", true)}), })}, bson.D{bsonField("$sort", bson.D{bsonField("updatedAt", -1)})}, bson.D{bsonField("$group", bson.D{ @@ -361,8 +373,9 @@ func (s *mongoUsageStore) CommandsByUser(ctx context.Context, username string, l opts.SetLimit(int64(limit)) } cur, err := s.coll.Find(ctx, bson.M{ - "uid": userID, - "cmd": bson.M{"$type": "string"}, + "uid": userID, + "cmd": bson.M{"$type": "string"}, + "deleted": bson.M{"$ne": true}, }, opts) if err != nil { return nil, true, fmt.Errorf("mongo stats commands by user %s: %w", username, err) @@ -391,9 +404,10 @@ func (s *mongoUsageStore) UsersByCommand(ctx context.Context, cmd string, limit opts.SetLimit(int64(limit)) } cur, err := s.coll.Find(ctx, bson.M{ - "cmd": cmd, - "uid": bson.M{"$gt": int64(0)}, - "user": bson.M{"$type": "string", "$ne": ""}, + "cmd": cmd, + "uid": bson.M{"$gt": int64(0)}, + "user": bson.M{"$type": "string", "$ne": ""}, + "deleted": bson.M{"$ne": true}, }, opts) if err != nil { return nil, fmt.Errorf("mongo stats users by command %s: %w", cmd, err) @@ -420,8 +434,9 @@ func (s *mongoUsageStore) userIDByUsername(ctx context.Context, username string) } err := s.coll.FindOne(ctx, bson.M{ - "uid": bson.M{"$gt": int64(0)}, - "user": username, + "uid": bson.M{"$gt": int64(0)}, + "user": username, + "deleted": bson.M{"$ne": true}, }, options.FindOne(). SetProjection(bson.M{"uid": 1}).