feat(stats): migrate command history

docs/deploy-coolify-selfhosted.md

@@ -76,7 +76,10 @@ overrides are not supported in runtime env; modules use coded defaults.
 > The `stats` collection uses queryable aggregate documents for command/user
 > counts and creates indexes on startup. First startup after the schema change
 > migrates legacy `count:`, `user:`, and `pair:` stats keys into the new shape,
-> deletes the legacy keys, and records completion in `system`.
+> deletes the legacy keys, and records completion in `system`. Startup also
+> migrates renamed command stats to the current command names and marks removed
+> command rows with `deleted: true`; `/stats` queries filter those retained
+> legacy rows.
 
 ## 2. Coolify
 

internal/modules/stats/startup.go

@@ -2,6 +2,7 @@ package stats
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -22,8 +23,35 @@ const (
 
 	usageMigrationName = "stats-usage-v2"
 	usageMigrationKey  = "migration:" + usageMigrationName
+
+	commandHistoryMigrationName = "stats-command-history-v1"
+	commandHistoryMigrationKey  = "migration:" + commandHistoryMigrationName
 )
 
+type commandRename struct {
+	Old string
+	New string
+}
+
+var commandRenames = []commandRename{
+	{Old: "lolschedule", New: "lol"},
+	{Old: "lolschedule_week", New: "lol_this_week"},
+	{Old: "lolschedule_subscribe", New: "lol_subscribe"},
+	{Old: "lolschedule_unsubscribe", New: "lol_unsubscribe"},
+	{Old: "wc_week", New: "wc_this_week"},
+	{Old: "gold_stats", New: "gold_portfolio"},
+	{Old: "coin_stats", New: "coin_portfolio"},
+	{Old: "stock_stats", New: "stock_portfolio"},
+	{Old: "stock_income_stock", New: "stock_bonus"},
+	{Old: "stock_income_vnd", New: "stock_dividend"},
+}
+
+var deletedCommandNames = []string{
+	"lolschedule_today",
+	"wc_today",
+	"stock_convert",
+}
+
 type legacyCountEntry struct {
 	N int64 `json:"n" bson:"n"`
 }
@@ -42,7 +70,10 @@ func InitStore(ctx context.Context, statsColl, systemColl storage.Collection) er
 			return err
 		}
 	}
-	return migrateLegacyUsage(ctx, statsColl, systemColl)
+	if err := migrateLegacyUsage(ctx, statsColl, systemColl); err != nil {
+		return err
+	}
+	return migrateCommandHistory(ctx, statsColl, systemColl)
 }
 
 func ensureUsageIndexes(ctx context.Context, coll *mongo.Collection) error {
@@ -130,6 +161,130 @@ func migrateLegacyUsage(ctx context.Context, statsColl, systemColl storage.Colle
 	return nil
 }
 
+func migrateCommandHistory(ctx context.Context, statsColl, systemColl storage.Collection) error {
+	sys := systemstate.New(systemColl)
+	if rec, ok, err := sys.Get(ctx, commandHistoryMigrationKey); err != nil {
+		return fmt.Errorf("stats command history migration marker get: %w", err)
+	} else if ok && rec.Status == "done" {
+		return nil
+	}
+
+	docs := storage.Typed[usageEntry](statsColl)
+	changed := int64(0)
+	for _, rename := range commandRenames {
+		n, err := migrateCommandRename(ctx, docs, rename)
+		if err != nil {
+			return err
+		}
+		changed += n
+	}
+	for _, cmd := range deletedCommandNames {
+		n, err := markCommandDeleted(ctx, docs, cmd)
+		if err != nil {
+			return err
+		}
+		changed += n
+	}
+
+	now := nowMillis()
+	if err := sys.Put(ctx, commandHistoryMigrationKey, systemstate.Record{
+		Kind:        "migration",
+		Name:        commandHistoryMigrationName,
+		Status:      "done",
+		Count:       changed,
+		CompletedAt: now,
+		UpdatedAt:   now,
+	}); err != nil {
+		return fmt.Errorf("stats command history migration marker put: %w", err)
+	}
+	return nil
+}
+
+func migrateCommandRename(ctx context.Context, docs storage.DocStore[usageEntry], rename commandRename) (int64, error) {
+	keys, err := docs.List(ctx, rename.Old)
+	if err != nil {
+		return 0, fmt.Errorf("stats command rename list %s: %w", rename.Old, err)
+	}
+	changed := int64(0)
+	for _, key := range keys {
+		if !usageKeyBelongsToCommand(key, rename.Old) {
+			continue
+		}
+		entry, _, err := docs.Get(ctx, key)
+		if err != nil {
+			return changed, fmt.Errorf("stats command rename get %s: %w", key, err)
+		}
+		if entry.Cmd != "" && entry.Cmd != rename.Old {
+			continue
+		}
+		entry.Cmd = rename.Old
+		targetKey := usageKey(rename.New, entry.UserID)
+		target, _, err := docs.Get(ctx, targetKey)
+		switch {
+		case errors.Is(err, storage.ErrNotFound):
+			target = usageEntry{
+				Cmd:      rename.New,
+				UserID:   entry.UserID,
+				Username: entry.Username,
+			}
+		case err != nil:
+			return changed, fmt.Errorf("stats command rename target get %s: %w", targetKey, err)
+		}
+
+		target.Cmd = rename.New
+		target.UserID = entry.UserID
+		if entry.UserID == 0 {
+			target.Username = ""
+		} else if entry.Username != "" {
+			target.Username = entry.Username
+		}
+		target.N += entry.N
+		target.Deleted = false
+		if err := docs.Put(ctx, targetKey, target); err != nil {
+			return changed, fmt.Errorf("stats command rename put %s: %w", targetKey, err)
+		}
+		if err := docs.Delete(ctx, key); err != nil {
+			return changed, fmt.Errorf("stats command rename delete %s: %w", key, err)
+		}
+		changed++
+	}
+	return changed, nil
+}
+
+func markCommandDeleted(ctx context.Context, docs storage.DocStore[usageEntry], cmd string) (int64, error) {
+	keys, err := docs.List(ctx, cmd)
+	if err != nil {
+		return 0, fmt.Errorf("stats command delete list %s: %w", cmd, err)
+	}
+	changed := int64(0)
+	for _, key := range keys {
+		if !usageKeyBelongsToCommand(key, cmd) {
+			continue
+		}
+		entry, _, err := docs.Get(ctx, key)
+		if err != nil {
+			return changed, fmt.Errorf("stats command delete get %s: %w", key, err)
+		}
+		if entry.Cmd != "" && entry.Cmd != cmd {
+			continue
+		}
+		entry.Cmd = cmd
+		if entry.Deleted {
+			continue
+		}
+		entry.Deleted = true
+		if err := docs.Put(ctx, key, entry); err != nil {
+			return changed, fmt.Errorf("stats command delete put %s: %w", key, err)
+		}
+		changed++
+	}
+	return changed, nil
+}
+
+func usageKeyBelongsToCommand(key, cmd string) bool {
+	return key == cmd || strings.HasPrefix(key, cmd+":")
+}
+
 func loadLegacyUsernames(ctx context.Context, users storage.DocStore[legacyUserEntry]) (map[int64]string, []string, error) {
 	keys, err := users.List(ctx, legacyUserPrefix)
 	if err != nil {

internal/modules/stats/startup_mongo_test.go

@@ -45,12 +45,18 @@ func TestInitStore_MongoCreatesIndexesAndMigratesLegacy(t *testing.T) {
 	if err := legacyCounts.Put(ctx, legacyCountPrefix+"ping", legacyCountEntry{N: 2}); err != nil {
 		t.Fatalf("legacy count: %v", err)
 	}
+	if err := legacyCounts.Put(ctx, legacyCountPrefix+"gold_stats", legacyCountEntry{N: 4}); err != nil {
+		t.Fatalf("legacy renamed count: %v", err)
+	}
 	if err := legacyUsers.Put(ctx, legacyUserPrefix+"7", legacyUserEntry{Username: "alice", N: 1}); err != nil {
 		t.Fatalf("legacy user: %v", err)
 	}
 	if err := legacyCounts.Put(ctx, legacyPairPrefix+"ping:7", legacyCountEntry{N: 1}); err != nil {
 		t.Fatalf("legacy pair: %v", err)
 	}
+	if err := legacyCounts.Put(ctx, legacyPairPrefix+"stock_convert:7", legacyCountEntry{N: 3}); err != nil {
+		t.Fatalf("legacy deleted pair: %v", err)
+	}
 
 	if err := InitStore(ctx, statsColl, systemColl); err != nil {
 		t.Fatalf("InitStore: %v", err)
@@ -59,6 +65,8 @@ func TestInitStore_MongoCreatesIndexesAndMigratesLegacy(t *testing.T) {
 	usageDocs := storage.Typed[usageEntry](statsColl)
 	assertUsageEntry(t, usageDocs, usageKey("ping", 7), usageEntry{Cmd: "ping", UserID: 7, Username: "alice", N: 1})
 	assertUsageEntry(t, usageDocs, usageKey("ping", 0), usageEntry{Cmd: "ping", N: 1})
+	assertUsageEntry(t, usageDocs, usageKey("gold_portfolio", 0), usageEntry{Cmd: "gold_portfolio", N: 4})
+	assertUsageEntry(t, usageDocs, usageKey("stock_convert", 7), usageEntry{Cmd: "stock_convert", UserID: 7, Username: "alice", N: 3, Deleted: true})
 
 	rawStatsColl, ok := storage.MongoCollection(statsColl)
 	if !ok {
@@ -89,6 +97,15 @@ func TestInitStore_MongoCreatesIndexesAndMigratesLegacy(t *testing.T) {
 		}
 	}
 
+	sys := systemstate.New(systemColl)
+	rec, ok, err := sys.Get(ctx, commandHistoryMigrationKey)
+	if err != nil || !ok {
+		t.Fatalf("command history marker ok=%v err=%v", ok, err)
+	}
+	if rec.Status != "done" || rec.Count != 2 {
+		t.Fatalf("command history marker = %+v, want done count 2", rec)
+	}
+
 	rawDoc := bson.M{}
 	err = rawStatsColl.FindOne(ctx, bson.M{"_id": legacyCountPrefix + "ping"}).Decode(&rawDoc)
 	if err == nil {

internal/modules/stats/stats_test.go

@@ -312,6 +312,110 @@ func TestInitStore_MigratesLegacyStatsOnceAndDeletesOldKeys(t *testing.T) {
 	}
 }
 
+func TestInitStore_MigratesCommandHistoryAndMarksDeleted(t *testing.T) {
+	ctx := context.Background()
+	provider := storage.NewMemoryProvider()
+	statsColl := provider.Collection("stats")
+	systemColl := provider.Collection(systemstate.CollectionName)
+	usageDocs := storage.Typed[usageEntry](statsColl)
+
+	seed := map[string]usageEntry{
+		usageKey("gold_stats", 0):            {Cmd: "gold_stats", N: 2},
+		usageKey("gold_portfolio", 0):        {Cmd: "gold_portfolio", N: 5},
+		usageKey("stock_income_stock", 7):    {Cmd: "stock_income_stock", UserID: 7, Username: "alice", N: 3},
+		usageKey("stock_bonus", 7):           {Cmd: "stock_bonus", UserID: 7, Username: "alice", N: 4},
+		usageKey("lolschedule_week", 8):      {Cmd: "lolschedule_week", UserID: 8, Username: "bob", N: 1},
+		usageKey("lolschedule_today", 0):     {Cmd: "lolschedule_today", N: 9},
+		usageKey("stock_convert", 7):         {Cmd: "stock_convert", UserID: 7, Username: "alice", N: 2},
+		usageKey("stock_convert_extra", 7):   {Cmd: "stock_convert_extra", UserID: 7, Username: "alice", N: 99},
+		usageKey("lolschedule_weekly", 10):   {Cmd: "lolschedule_weekly", UserID: 10, Username: "carol", N: 99},
+		usageKey("stock_income_stocked", 11): {Cmd: "stock_income_stocked", UserID: 11, Username: "dan", N: 99},
+	}
+	for key, entry := range seed {
+		if err := usageDocs.Put(ctx, key, entry); err != nil {
+			t.Fatalf("seed %s: %v", key, err)
+		}
+	}
+
+	if err := InitStore(ctx, statsColl, systemColl); err != nil {
+		t.Fatalf("InitStore: %v", err)
+	}
+
+	assertUsageEntry(t, usageDocs, usageKey("gold_portfolio", 0), usageEntry{Cmd: "gold_portfolio", N: 7})
+	assertUsageEntry(t, usageDocs, usageKey("stock_bonus", 7), usageEntry{Cmd: "stock_bonus", UserID: 7, Username: "alice", N: 7})
+	assertUsageEntry(t, usageDocs, usageKey("lol_this_week", 8), usageEntry{Cmd: "lol_this_week", UserID: 8, Username: "bob", N: 1})
+	assertUsageEntry(t, usageDocs, usageKey("lolschedule_today", 0), usageEntry{Cmd: "lolschedule_today", N: 9, Deleted: true})
+	assertUsageEntry(t, usageDocs, usageKey("stock_convert", 7), usageEntry{Cmd: "stock_convert", UserID: 7, Username: "alice", N: 2, Deleted: true})
+	assertUsageEntry(t, usageDocs, usageKey("stock_convert_extra", 7), usageEntry{Cmd: "stock_convert_extra", UserID: 7, Username: "alice", N: 99})
+	assertUsageEntry(t, usageDocs, usageKey("lolschedule_weekly", 10), usageEntry{Cmd: "lolschedule_weekly", UserID: 10, Username: "carol", N: 99})
+	assertUsageEntry(t, usageDocs, usageKey("stock_income_stocked", 11), usageEntry{Cmd: "stock_income_stocked", UserID: 11, Username: "dan", N: 99})
+
+	for _, key := range []string{
+		usageKey("gold_stats", 0),
+		usageKey("stock_income_stock", 7),
+		usageKey("lolschedule_week", 8),
+	} {
+		if _, _, err := usageDocs.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) {
+			t.Fatalf("renamed key %s should be deleted, got err=%v", key, err)
+		}
+	}
+
+	sys := systemstate.New(systemColl)
+	rec, ok, err := sys.Get(ctx, commandHistoryMigrationKey)
+	if err != nil || !ok {
+		t.Fatalf("command history marker ok=%v err=%v", ok, err)
+	}
+	if rec.Status != "done" || rec.Count != 5 {
+		t.Fatalf("command history marker = %+v, want done count 5", rec)
+	}
+
+	if err := InitStore(ctx, statsColl, systemColl); err != nil {
+		t.Fatalf("InitStore second run: %v", err)
+	}
+	assertUsageEntry(t, usageDocs, usageKey("gold_portfolio", 0), usageEntry{Cmd: "gold_portfolio", N: 7})
+}
+
+func TestStatsViewsFilterDeletedLegacyRows(t *testing.T) {
+	ctx := context.Background()
+	provider := storage.NewMemoryProvider()
+	c := newCounter(provider.Collection("stats"))
+	docs := storage.Typed[usageEntry](provider.Collection("stats"))
+
+	for key, entry := range map[string]usageEntry{
+		usageKey("ping", 1):          {Cmd: "ping", UserID: 1, Username: "alice", N: 3},
+		usageKey("wordle", 2):        {Cmd: "wordle", UserID: 2, Username: "bob", N: 2},
+		usageKey("wc_today", 0):      {Cmd: "wc_today", N: 100, Deleted: true},
+		usageKey("wc_today", 1):      {Cmd: "wc_today", UserID: 1, Username: "alice", N: 100, Deleted: true},
+		usageKey("stock_convert", 3): {Cmd: "stock_convert", UserID: 3, Username: "ghost", N: 50, Deleted: true},
+	} {
+		if err := docs.Put(ctx, key, entry); err != nil {
+			t.Fatalf("seed %s: %v", key, err)
+		}
+	}
+
+	for name, got := range map[string]string{
+		"top commands":     renderStats(ctx, c, ""),
+		"top users":        renderStats(ctx, c, "users"),
+		"commands by user": renderStats(ctx, c, "user alice"),
+	} {
+		if strings.Contains(got, "wc_today") || strings.Contains(got, "stock_convert") || strings.Contains(got, "ghost") {
+			t.Fatalf("%s leaked deleted stats:\n%s", name, got)
+		}
+	}
+	if got := renderStats(ctx, c, "users"); !strings.Contains(got, "@alice: 3") || strings.Contains(got, "@alice: 103") {
+		t.Fatalf("top users did not filter deleted rows correctly:\n%s", got)
+	}
+	if got := renderStats(ctx, c, "user alice"); !strings.Contains(got, "/ping: 3") || strings.Contains(got, "/wc_today") {
+		t.Fatalf("commands by user did not filter deleted rows correctly:\n%s", got)
+	}
+	if got := renderStats(ctx, c, "cmd wc_today"); got != "Command /wc_today has no users yet." {
+		t.Fatalf("deleted command users reply = %q", got)
+	}
+	if got := renderStats(ctx, c, "user ghost"); got != "User @ghost not found." {
+		t.Fatalf("deleted-only user reply = %q", got)
+	}
+}
+
 func assertUsageEntry(t *testing.T, docs storage.DocStore[usageEntry], key string, want usageEntry) {
 	t.Helper()
 	got, _, err := docs.Get(context.Background(), key)

internal/modules/stats/usage_store.go

@@ -25,6 +25,7 @@ type usageEntry struct {
 	UserID   int64  `json:"uid,omitempty" bson:"uid,omitempty"`
 	Username string `json:"user,omitempty" bson:"user,omitempty"`
 	N        int64  `json:"n" bson:"n"`
+	Deleted  bool   `json:"deleted,omitempty" bson:"deleted,omitempty"`
 }
 
 type usageUser struct {
@@ -78,6 +79,7 @@ func (s *docUsageStore) Increment(ctx context.Context, cmd string, user usageUse
 
 	entry.Cmd = cmd
 	entry.N++
+	entry.Deleted = false
 	if hasUser {
 		entry.UserID = user.ID
 		entry.Username = user.Username
@@ -121,6 +123,9 @@ func (s *docUsageStore) TopCommands(ctx context.Context, limit int) ([]row, erro
 	}
 	totals := make(map[string]int64)
 	for _, e := range entries {
+		if e.Deleted {
+			continue
+		}
 		totals[e.Cmd] += e.N
 	}
 	rows := make([]row, 0, len(totals))
@@ -145,7 +150,7 @@ func (s *docUsageStore) TopUsers(ctx context.Context, limit int) ([]row, error)
 	}
 	totals := make(map[int64]total)
 	for _, e := range entries {
-		if e.UserID == 0 || e.Username == "" {
+		if e.Deleted || e.UserID == 0 || e.Username == "" {
 			continue
 		}
 		t := totals[e.UserID]
@@ -174,7 +179,7 @@ func (s *docUsageStore) CommandsByUser(ctx context.Context, username string, lim
 		found  bool
 	)
 	for _, e := range entries {
-		if e.UserID != 0 && e.Username == username {
+		if !e.Deleted && e.UserID != 0 && e.Username == username {
 			userID = e.UserID
 			found = true
 			break
@@ -186,7 +191,7 @@ func (s *docUsageStore) CommandsByUser(ctx context.Context, username string, lim
 
 	rows := make([]row, 0)
 	for _, e := range entries {
-		if e.UserID == userID {
+		if !e.Deleted && e.UserID == userID {
 			rows = append(rows, row{display: "/" + e.Cmd, n: e.N})
 		}
 	}
@@ -204,7 +209,7 @@ func (s *docUsageStore) UsersByCommand(ctx context.Context, cmd string, limit in
 	}
 	rows := make([]row, 0)
 	for _, e := range entries {
-		if e.Cmd == cmd && e.UserID != 0 && e.Username != "" {
+		if !e.Deleted && e.Cmd == cmd && e.UserID != 0 && e.Username != "" {
 			rows = append(rows, row{display: "@" + e.Username, n: e.N})
 		}
 	}
@@ -242,16 +247,19 @@ func (s *mongoUsageStore) Increment(ctx context.Context, cmd string, user usageU
 	}
 
 	set := bson.M{"cmd": cmd}
+	unset := bson.M{"deleted": ""}
 	update := bson.M{
 		"$set":         set,
 		"$inc":         bson.M{"n": int64(1), "version": int64(1)},
+		"$unset":       unset,
 		"$currentDate": bson.M{"updatedAt": true},
 	}
 	if hasUser {
 		set["uid"] = user.ID
 		set["user"] = user.Username
 	} else {
-		update["$unset"] = bson.M{"uid": "", "user": ""}
+		unset["uid"] = ""
+		unset["user"] = ""
 	}
 
 	key := usageKey(cmd, userID)
@@ -280,7 +288,10 @@ func (s *mongoUsageStore) Increment(ctx context.Context, cmd string, user usageU
 
 func (s *mongoUsageStore) TopCommands(ctx context.Context, limit int) ([]row, error) {
 	pipeline := mongo.Pipeline{
-		bson.D{bsonField("$match", bson.D{bsonField("cmd", bson.D{bsonField("$type", "string")})})},
+		bson.D{bsonField("$match", bson.D{
+			bsonField("cmd", bson.D{bsonField("$type", "string")}),
+			bsonField("deleted", bson.D{bsonField("$ne", true)}),
+		})},
 		bson.D{bsonField("$group", bson.D{bsonField("_id", "$cmd"), bsonField("n", bson.D{bsonField("$sum", "$n")})})},
 		bson.D{bsonField("$sort", bson.D{bsonField("n", -1), bsonField("_id", 1)})},
 	}
@@ -314,6 +325,7 @@ func (s *mongoUsageStore) TopUsers(ctx context.Context, limit int) ([]row, error
 		bson.D{bsonField("$match", bson.D{
 			bsonField("uid", bson.D{bsonField("$gt", int64(0))}),
 			bsonField("user", bson.D{bsonField("$type", "string"), bsonField("$ne", "")}),
+			bsonField("deleted", bson.D{bsonField("$ne", true)}),
 		})},
 		bson.D{bsonField("$sort", bson.D{bsonField("updatedAt", -1)})},
 		bson.D{bsonField("$group", bson.D{
@@ -361,8 +373,9 @@ func (s *mongoUsageStore) CommandsByUser(ctx context.Context, username string, l
 		opts.SetLimit(int64(limit))
 	}
 	cur, err := s.coll.Find(ctx, bson.M{
-		"uid": userID,
-		"cmd": bson.M{"$type": "string"},
+		"uid":     userID,
+		"cmd":     bson.M{"$type": "string"},
+		"deleted": bson.M{"$ne": true},
 	}, opts)
 	if err != nil {
 		return nil, true, fmt.Errorf("mongo stats commands by user %s: %w", username, err)
@@ -391,9 +404,10 @@ func (s *mongoUsageStore) UsersByCommand(ctx context.Context, cmd string, limit
 		opts.SetLimit(int64(limit))
 	}
 	cur, err := s.coll.Find(ctx, bson.M{
-		"cmd":  cmd,
-		"uid":  bson.M{"$gt": int64(0)},
-		"user": bson.M{"$type": "string", "$ne": ""},
+		"cmd":     cmd,
+		"uid":     bson.M{"$gt": int64(0)},
+		"user":    bson.M{"$type": "string", "$ne": ""},
+		"deleted": bson.M{"$ne": true},
 	}, opts)
 	if err != nil {
 		return nil, fmt.Errorf("mongo stats users by command %s: %w", cmd, err)
@@ -420,8 +434,9 @@ func (s *mongoUsageStore) userIDByUsername(ctx context.Context, username string)
 	}
 	err := s.coll.FindOne(ctx,
 		bson.M{
-			"uid":  bson.M{"$gt": int64(0)},
-			"user": username,
+			"uid":     bson.M{"$gt": int64(0)},
+			"user":    username,
+			"deleted": bson.M{"$ne": true},
 		},
 		options.FindOne().
 			SetProjection(bson.M{"uid": 1}).