mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-07-25 12:22:44 +00:00
viettranx
eb6723d674
- Add TokenCounter.CountToolSchemas() to measure JSON schema size for all tools - Include tool schemas in OverheadTokens calculation for accurate context usage - Implement dynamic max_tokens: in/25 clamp [1024, 8192] for compaction - Add characterization tests: count_tool_schemas_test.go - Add overhead verification tests: context_stage_overhead_test.go, context_stage_tool_overhead_test.go - Add integration tests: context_stage_integration_test.go - Add compact tests: loop_compact_dynamic_max_test.go, loop_compact_max_tokens_test.go - Add sanitize tests: loop_history_sanitize_max_tokens_test.go - Add integration test: loop_compact_integration_test.go
69 lines
1.9 KiB
Go
69 lines
1.9 KiB
Go
package tokencount
|
|
|
|
import (
|
|
"cmp"
|
|
"encoding/json"
|
|
"slices"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
|
)
|
|
|
|
// FallbackCounter uses rune-count/3 heuristic (matches v2 behavior).
|
|
// Used when tiktoken-go is unavailable or model is unknown.
|
|
type FallbackCounter struct{}
|
|
|
|
func NewFallbackCounter() *FallbackCounter { return &FallbackCounter{} }
|
|
|
|
func (c *FallbackCounter) Count(_ string, text string) int {
|
|
return utf8.RuneCountInString(text) / 3
|
|
}
|
|
|
|
func (c *FallbackCounter) CountMessages(_ string, msgs []providers.Message) int {
|
|
total := 0
|
|
for _, m := range msgs {
|
|
total += utf8.RuneCountInString(m.Content)/3 + PerMessageOverhead
|
|
for _, tc := range m.ToolCalls {
|
|
total += utf8.RuneCountInString(tc.ID)/3 + utf8.RuneCountInString(tc.Name)/3
|
|
for k, v := range tc.Arguments {
|
|
total += utf8.RuneCountInString(k) / 3
|
|
if s, ok := v.(string); ok {
|
|
total += utf8.RuneCountInString(s) / 3
|
|
} else {
|
|
total += 10
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return total
|
|
}
|
|
|
|
// CountToolSchemas returns rune/3 heuristic count for the JSON-serialised tool list.
|
|
// Returns 0 for nil or empty slice.
|
|
func (c *FallbackCounter) CountToolSchemas(_ string, tools []providers.ToolDefinition) int {
|
|
if len(tools) == 0 {
|
|
return 0
|
|
}
|
|
blob, _ := json.Marshal(tools)
|
|
return utf8.RuneCountInString(string(blob)) / 3
|
|
}
|
|
|
|
// ModelContextWindow uses longest-prefix-match to avoid ambiguity
|
|
// (e.g., "gpt-4o" must match before "gpt-4").
|
|
func (c *FallbackCounter) ModelContextWindow(model string) int {
|
|
// Sort prefixes longest-first for correct matching.
|
|
keys := make([]string, 0, len(DefaultRegistry))
|
|
for k := range DefaultRegistry {
|
|
keys = append(keys, k)
|
|
}
|
|
slices.SortFunc(keys, func(a, b string) int { return cmp.Compare(len(b), len(a)) })
|
|
|
|
for _, prefix := range keys {
|
|
if strings.HasPrefix(model, prefix) {
|
|
return DefaultRegistry[prefix].ContextWindow
|
|
}
|
|
}
|
|
return 200_000 // conservative default
|
|
}
|