tiennm99/goclaw
1
0
Fork 0
mirror of https://github.com/tiennm99/goclaw.git synced 2026-08-11 02:22:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into codex/issue-48-google-workspace-cli

Browse Source 
# Conflicts:
#	docs/project-changelog.md
This commit is contained in:
Goon
2026-05-24 18:57:47 +07:00
parent 12fb1baf79 34a2cb24da
commit 3a749ff91c
189 changed files with 8860 additions and 848 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/gateway.go
+17 -4
View File
@@ -46,6 +46,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/skills"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
"github.com/nextlevelbuilder/goclaw/internal/vault"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
@@ -225,6 +226,7 @@ func runGateway() {
}
}
setupMemoryEmbeddings(pgStores, providerRegistry)
usageCapSvc := usagecaps.NewService(pgStores.UsageCaps, pgStores.Providers)
// Resolve background provider for consolidation + vault enrichment.
// Fallback: background.provider → agent.default_provider → first registered provider.
@@ -236,6 +238,7 @@ func runGateway() {
var kgExtractor *kg.Extractor
if pgStores.KnowledgeGraph != nil {
kgExtractor = kg.NewExtractor(bgProvider, bgModel, 0)
kgExtractor.SetUsageCapService(usageCapSvc)
}
cleanupConsolidation := consolidation.Register(consolidation.ConsolidationDeps{
EpisodicStore: pgStores.Episodic,
@@ -247,6 +250,7 @@ func runGateway() {
Registry: providerRegistry,
Extractor: kgExtractor,
AlertDeps: bgalert.AlertDeps{SystemConfigs: pgStores.SystemConfigs, MsgBus: msgBus},
UsageCaps: usageCapSvc,
AgentStore: pgStores.Agents,
})
defer cleanupConsolidation()
@@ -269,6 +273,7 @@ func runGateway() {
MsgBus: msgBus,
TeamStore: pgStores.Teams,
AlertDeps: bgalert.AlertDeps{SystemConfigs: pgStores.SystemConfigs, MsgBus: msgBus},
UsageCaps: usageCapSvc,
})
enrichProgress = ep
enrichWorker = ew
@@ -285,8 +290,14 @@ func runGateway() {
slog.Info("bootstrap: capabilities backfill complete", "agents", count)
}
if readImage, ok := toolsReg.Get("read_image"); ok {
if t, ok := readImage.(*tools.ReadImageTool); ok {
t.SetUsageCapService(usageCapSvc)
}
}
// Subagent system (secureCLI store wired so subagent ExecTools enforce the gate)
subagentMgr := setupSubagents(providerRegistry, cfg, msgBus, toolsReg, workspace, sandboxMgr, pgStores.SecureCLI)
subagentMgr := setupSubagents(providerRegistry, cfg, msgBus, toolsReg, workspace, sandboxMgr, pgStores.SecureCLI, usageCapSvc)
if subagentMgr != nil {
// Wire announce queue for batched subagent result delivery (matching TS debounce pattern).
announceQueue := tools.NewAnnounceQueue(1000, 20, makeDelegateAnnounceCallback(subagentMgr, msgBus))
@@ -339,7 +350,7 @@ func runGateway() {
var mcpPool *mcpbridge.Pool
var mediaStore *media.Store
var postTurn tools.PostTurnProcessor
contextFileInterceptor, mcpPool, mediaStore, postTurn = wireExtras(pgStores, agentRouter, providerRegistry, modelReg, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, redisClient, domainBus)
contextFileInterceptor, mcpPool, mediaStore, postTurn = wireExtras(pgStores, agentRouter, providerRegistry, modelReg, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, redisClient, domainBus, usageCapSvc)
if mcpPool != nil {
defer mcpPool.Stop()
}
@@ -359,6 +370,7 @@ func runGateway() {
workspace: workspace,
dataDir: dataDir,
domainBus: domainBus,
usageCapSvc: usageCapSvc,
audioMgr: audioMgr,
}
@@ -371,7 +383,7 @@ func runGateway() {
httpapi.InitGatewayNoAuthFallbackAllowed(config.GatewayNoAuthFallbackAllowed(cfg.Gateway))
exportTokenStore := httpapi.InitExportTokenStore()
defer exportTokenStore.Stop()
agentsH, skillsH, tracesH, mcpH, channelInstancesH, providersH, builtinToolsH, pendingMessagesH, teamEventsH, secureCLIH, secureCLIGrantH, mcpUserCredsH := wireHTTP(pgStores, cfg.Agents.Defaults.Workspace, dataDir, bundledSkillsDir, msgBus, toolsReg, providerRegistry, modelReg, permPE.IsOwner, gatewayAddr, mcpToolLister)
agentsH, skillsH, tracesH, mcpH, channelInstancesH, providersH, builtinToolsH, pendingMessagesH, teamEventsH, secureCLIH, secureCLIGrantH, mcpUserCredsH := wireHTTP(pgStores, cfg.Agents.Defaults.Workspace, dataDir, bundledSkillsDir, msgBus, toolsReg, providerRegistry, modelReg, permPE.IsOwner, gatewayAddr, mcpToolLister, usageCapSvc, cfg.Skills)
// Wire dependencies for system prompt preview parity.
if agentsH != nil {
@@ -428,7 +440,7 @@ func runGateway() {
// Register all RPC methods
server.SetLogTee(logTee)
server.SetRuntimeLogsHandler(httpapi.NewRuntimeLogsHandler(logTee))
pairingMethods, heartbeatMethods, chatMethods, cfgPermsMethods := registerAllMethods(server, agentRouter, pgStores.Sessions, pgStores.Cron, pgStores.Pairing, cfg, cfgPath, workspace, dataDir, msgBus, execApprovalMgr, pgStores.Agents, pgStores.Skills, pgStores.ConfigSecrets, pgStores.Teams, contextFileInterceptor, logTee, pgStores.Heartbeats, pgStores.ConfigPermissions, pgStores.SystemConfigs, pgStores.Tenants, pgStores.SkillTenantCfgs, audioMgr)
pairingMethods, heartbeatMethods, chatMethods, cfgPermsMethods := registerAllMethods(server, agentRouter, pgStores.Sessions, pgStores.Cron, pgStores.Pairing, cfg, cfgPath, workspace, dataDir, msgBus, execApprovalMgr, pgStores.Agents, pgStores.Skills, pgStores.ConfigSecrets, pgStores.Teams, contextFileInterceptor, logTee, pgStores.Heartbeats, pgStores.ConfigPermissions, pgStores.SystemConfigs, pgStores.Tenants, pgStores.SkillTenantCfgs, audioMgr, usageCapSvc)
// Phase 3: Agent hooks RPC methods (hooks.list/create/update/delete/toggle/test/history).
if hs, ok := pgStores.Hooks.(hooks.HookStore); ok && hs != nil {
@@ -514,6 +526,7 @@ func runGateway() {
instanceLoader = channels.NewInstanceLoader(pgStores.ChannelInstances, pgStores.Agents, channelMgr, msgBus, pgStores.Pairing)
instanceLoader.SetProviderRegistry(providerRegistry)
instanceLoader.SetPendingCompactionConfig(cfg.Channels.PendingCompaction)
instanceLoader.SetUsageCapService(usageCapSvc)
instanceLoader.RegisterFactory(channels.TypeTelegram, telegram.FactoryWithStoresAndAudio(pgStores.Agents, pgStores.ConfigPermissions, pgStores.Teams, pgStores.SubagentTasks, pgStores.PendingMessages, audioMgr))
instanceLoader.RegisterFactory(channels.TypeDiscord, discord.FactoryWithStoresAndAudio(pgStores.Agents, pgStores.ConfigPermissions, pgStores.PendingMessages, audioMgr))
instanceLoader.RegisterFactory(channels.TypeFeishu, feishu.FactoryWithPendingStoreAndAudio(pgStores.PendingMessages, audioMgr))
cmd/gateway_agents.go
+5 -2
View File
@@ -15,6 +15,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
"github.com/nextlevelbuilder/goclaw/internal/tts"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// resolveEmbeddingProvider selects an embedding provider from DB only.
@@ -159,7 +160,7 @@ func buildEmbeddingProvider(
return nil
}
func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus *bus.MessageBus, toolsReg *tools.Registry, workspace string, sandboxMgr sandbox.Manager, secureCLIStore store.SecureCLIStore) *tools.SubagentManager {
func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus *bus.MessageBus, toolsReg *tools.Registry, workspace string, sandboxMgr sandbox.Manager, secureCLIStore store.SecureCLIStore, usageCapSvc *usagecaps.Service) *tools.SubagentManager {
names := providerReg.List(context.Background())
if len(names) == 0 {
return nil
@@ -207,7 +208,9 @@ func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus
return reg
}
return tools.NewSubagentManager(provider, providerReg, agentCfg.Model, msgBus, toolsFactory, subCfg)
manager := tools.NewSubagentManager(provider, providerReg, agentCfg.Model, msgBus, toolsFactory, subCfg)
manager.SetUsageCapService(usageCapSvc)
return manager
}
// buildSubagentToolsRegistry produces a cloned tool registry for a subagent
cmd/gateway_consumer.go
+3 -1
View File
@@ -19,6 +19,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/scheduler"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
@@ -26,7 +27,7 @@ import (
// and routes them through the scheduler/agent loop, then publishes the response back.
// Also handles subagent announcements: routes them through the parent agent's session
// (matching TS subagent-announce.ts pattern) so the agent can reformulate for the user.
func consumeInboundMessages(ctx context.Context, msgBus *bus.MessageBus, agents *agent.Router, cfg *config.Config, sched *scheduler.Scheduler, channelMgr *channels.Manager, teamStore store.TeamStore, quotaChecker *channels.QuotaChecker, sessStore store.SessionStore, agentStore store.AgentStore, contactCollector *store.ContactCollector, postTurn tools.PostTurnProcessor, subagentMgr *tools.SubagentManager) {
func consumeInboundMessages(ctx context.Context, msgBus *bus.MessageBus, agents *agent.Router, cfg *config.Config, sched *scheduler.Scheduler, channelMgr *channels.Manager, teamStore store.TeamStore, quotaChecker *channels.QuotaChecker, sessStore store.SessionStore, agentStore store.AgentStore, contactCollector *store.ContactCollector, postTurn tools.PostTurnProcessor, subagentMgr *tools.SubagentManager, usageCapSvc *usagecaps.Service) {
slog.Info("inbound message consumer started")
// Inbound message deduplication (matching TS src/infra/dedupe.ts + inbound-dedupe.ts).
@@ -58,6 +59,7 @@ func consumeInboundMessages(ctx context.Context, msgBus *bus.MessageBus, agents
QuotaChecker: quotaChecker,
ContactCollector: contactCollector,
SubagentMgr: subagentMgr,
UsageCaps: usageCapSvc,
GetAnnounceMu: getAnnounceMu,
}
cmd/gateway_consumer_deps.go
+2
View File
@@ -10,6 +10,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/scheduler"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// ConsumerDeps bundles shared dependencies for consumer message handlers.
@@ -28,6 +29,7 @@ type ConsumerDeps struct {
ContactCollector *store.ContactCollector
TaskRunSessions sync.Map
SubagentMgr *tools.SubagentManager
UsageCaps *usagecaps.Service
BgWg sync.WaitGroup
GetAnnounceMu func(string) *sync.Mutex
}
cmd/gateway_consumer_normal.go
+5 -1
View File
@@ -333,7 +333,11 @@ func processNormalMessage(
if locale == "" {
locale = "en"
}
intent := agent.ClassifyIntent(ctx, loop.Provider(), loop.Model(), msg.Content)
classifyCtx := ctx
if uid := loop.UUID(); uid != uuid.Nil {
classifyCtx = store.WithAgentID(classifyCtx, uid)
}
intent := agent.ClassifyIntentWithUsageCaps(classifyCtx, deps.UsageCaps, loop.Provider(), loop.Model(), msg.Content)
switch intent {
case agent.IntentStatusQuery:
status := deps.Agents.GetActivity(sessionKey)
cmd/gateway_deps.go
+5 -3
View File
@@ -14,6 +14,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/skills"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
"github.com/nextlevelbuilder/goclaw/internal/vault"
)
@@ -28,13 +29,14 @@ type gatewayDeps struct {
channelMgr *channels.Manager
agentRouter *agent.Router
toolsReg *tools.Registry
skillsLoader *skills.Loader // optional: enables skill creation in evolution approval
skillsLoader *skills.Loader // optional: enables skill creation in evolution approval
permCache *cache.PermissionCache // nil if no tenant store; closed on shutdown to stop sweep goroutines
enrichProgress *vault.EnrichProgress // nil if enrichment worker not registered
enrichWorker *vault.EnrichWorker // nil if enrichment worker not registered; for stop/enqueue
enrichProgress *vault.EnrichProgress // nil if enrichment worker not registered
enrichWorker *vault.EnrichWorker // nil if enrichment worker not registered; for stop/enqueue
workspace string
dataDir string
domainBus eventbus.DomainEventBus
usageCapSvc *usagecaps.Service
audioMgr *audio.Manager // nil if TTS not configured; used by TTSHandler
ttsHandler *httpapi.TTSHandler // nil if TTS not configured; for hot-reload
}
cmd/gateway_hooks.go
+4 -2
View File
@@ -7,12 +7,13 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/edition"
"github.com/nextlevelbuilder/goclaw/internal/hooks"
hookhandlers "github.com/nextlevelbuilder/goclaw/internal/hooks/handlers"
"github.com/nextlevelbuilder/goclaw/internal/hooks/budget"
hookhandlers "github.com/nextlevelbuilder/goclaw/internal/hooks/handlers"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/security"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/store/pg"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// sharedHookHandlers is populated by wireExtras so the gateway.go router
@@ -27,7 +28,7 @@ var sharedHookHandlers map[hooks.HandlerType]hooks.Handler
// Budget wiring (C1 fix): the PromptHandler receives a budget.Store bound
// to pg.NewPGHookBudget so token spend is atomically deducted per tenant.
// When the DB handle is unavailable, budget falls back to nil (Lite desktop).
func buildHookHandlers(stores *store.Stores, providerReg *providers.Registry, hooksCfg config.HooksConfig) map[hooks.HandlerType]hooks.Handler {
func buildHookHandlers(stores *store.Stores, providerReg *providers.Registry, hooksCfg config.HooksConfig, usageCapSvc *usagecaps.Service) map[hooks.HandlerType]hooks.Handler {
encryptKey := os.Getenv("GOCLAW_ENCRYPTION_KEY")
var budgetStore *budget.Store
@@ -38,6 +39,7 @@ func buildHookHandlers(stores *store.Stores, providerReg *providers.Registry, ho
promptHandler := &hookhandlers.PromptHandler{
Resolver: hookhandlers.NewRegistryResolver(providerReg, stores.SystemConfigs),
Budget: budgetStore,
UsageCaps: usageCapSvc,
DefaultModel: "haiku",
}
cmd/gateway_http_handlers.go
+10 -2
View File
@@ -2,14 +2,16 @@ package cmd
import (
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/config"
httpapi "github.com/nextlevelbuilder/goclaw/internal/http"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// wireHTTP creates HTTP handlers (agents + skills + traces + MCP + channel instances + providers + builtin tools + pending messages).
func wireHTTP(stores *store.Stores, defaultWorkspace, dataDir, bundledSkillsDir string, msgBus *bus.MessageBus, toolsReg *tools.Registry, providerReg *providers.Registry, modelReg providers.ModelRegistry, isOwner func(string) bool, gatewayAddr string, mcpToolLister httpapi.MCPToolLister) (*httpapi.AgentsHandler, *httpapi.SkillsHandler, *httpapi.TracesHandler, *httpapi.MCPHandler, *httpapi.ChannelInstancesHandler, *httpapi.ProvidersHandler, *httpapi.BuiltinToolsHandler, *httpapi.PendingMessagesHandler, *httpapi.TeamEventsHandler, *httpapi.SecureCLIHandler, *httpapi.SecureCLIGrantHandler, *httpapi.MCPUserCredentialsHandler) {
func wireHTTP(stores *store.Stores, defaultWorkspace, dataDir, bundledSkillsDir string, msgBus *bus.MessageBus, toolsReg *tools.Registry, providerReg *providers.Registry, modelReg providers.ModelRegistry, isOwner func(string) bool, gatewayAddr string, mcpToolLister httpapi.MCPToolLister, usageCapSvc *usagecaps.Service, skillUploadConfig config.SkillsConfig) (*httpapi.AgentsHandler, *httpapi.SkillsHandler, *httpapi.TracesHandler, *httpapi.MCPHandler, *httpapi.ChannelInstancesHandler, *httpapi.ProvidersHandler, *httpapi.BuiltinToolsHandler, *httpapi.PendingMessagesHandler, *httpapi.TeamEventsHandler, *httpapi.SecureCLIHandler, *httpapi.SecureCLIGrantHandler, *httpapi.MCPUserCredentialsHandler) {
var agentsH *httpapi.AgentsHandler
var skillsH *httpapi.SkillsHandler
var tracesH *httpapi.TracesHandler
@@ -24,7 +26,7 @@ func wireHTTP(stores *store.Stores, defaultWorkspace, dataDir, bundledSkillsDir
if stores != nil && stores.Agents != nil {
var summoner *httpapi.AgentSummoner
if providerReg != nil {
summoner = httpapi.NewAgentSummoner(stores.Agents, providerReg, msgBus)
summoner = httpapi.NewAgentSummoner(stores.Agents, providerReg, msgBus, usageCapSvc)
}
agentsH = httpapi.NewAgentsHandler(stores.Agents, stores.Providers, providerReg, stores.DB, stores.Tracing, defaultWorkspace, msgBus, summoner, isOwner)
agentsH.SetImportStores(stores.Memory, stores.KnowledgeGraph)
@@ -37,6 +39,10 @@ func wireHTTP(stores *store.Stores, defaultWorkspace, dataDir, bundledSkillsDir
if len(dirs) > 0 {
skillsH = httpapi.NewSkillsHandler(manageStore, dirs[0], dataDir, bundledSkillsDir, msgBus, stores.SkillTenantCfgs, stores.Tenants)
skillsH.SetDB(stores.DB)
skillsH.SetUploadLimitConfig(skillUploadConfig)
if stores.SystemConfigs != nil {
skillsH.SetSystemConfigStore(stores.SystemConfigs)
}
}
}
}
@@ -61,6 +67,7 @@ func wireHTTP(stores *store.Stores, defaultWorkspace, dataDir, bundledSkillsDir
if stores != nil && stores.Providers != nil {
providersH = httpapi.NewProvidersHandler(stores.Providers, stores.ConfigSecrets, providerReg, gatewayAddr)
providersH.SetMessageBus(msgBus)
providersH.SetUsageCapService(usageCapSvc)
if modelReg != nil {
providersH.SetModelRegistry(modelReg)
}
@@ -90,6 +97,7 @@ func wireHTTP(stores *store.Stores, defaultWorkspace, dataDir, bundledSkillsDir
if stores != nil && stores.PendingMessages != nil {
pendingMessagesH = httpapi.NewPendingMessagesHandler(stores.PendingMessages, stores.Agents, providerReg)
pendingMessagesH.SetUsageCapService(usageCapSvc)
}
if stores != nil && stores.SecureCLI != nil {
cmd/gateway_http_wiring.go
+6 -1
View File
@@ -137,6 +137,9 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer(
if d.pgStores.Snapshots != nil {
d.server.SetUsageHandler(httpapi.NewUsageHandler(d.pgStores.Snapshots, d.pgStores.DB))
}
if d.pgStores.UsageCaps != nil {
d.server.SetUsageCapsHandler(httpapi.NewUsageCapsHandler(d.pgStores.UsageCaps, d.pgStores.Tenants))
}
// Runtime package management (install/uninstall system/pip/npm/github packages)
// Wire the update registry AFTER initGitHubInstaller so DefaultGitHubInstaller() is set.
@@ -234,7 +237,9 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer(
// Knowledge graph API
if d.pgStores != nil && d.pgStores.KnowledgeGraph != nil {
d.server.SetKnowledgeGraphHandler(httpapi.NewKnowledgeGraphHandler(d.pgStores.KnowledgeGraph, d.providerRegistry))
kgHandler := httpapi.NewKnowledgeGraphHandler(d.pgStores.KnowledgeGraph, d.providerRegistry)
kgHandler.SetUsageCapService(d.usageCapSvc)
d.server.SetKnowledgeGraphHandler(kgHandler)
}
// V3: Evolution metrics + suggestions API
cmd/gateway_lifecycle.go
+1 -1
View File
@@ -140,7 +140,7 @@ func (d *gatewayDeps) runLifecycle(
d.channelMgr.SetContactCollector(contactCollector)
}
go consumeInboundMessages(ctx, d.msgBus, d.agentRouter, d.cfg, deps.sched, d.channelMgr, deps.consumerTeamStore, deps.quotaChecker, d.pgStores.Sessions, d.pgStores.Agents, contactCollector, deps.postTurn, deps.subagentMgr)
go consumeInboundMessages(ctx, d.msgBus, d.agentRouter, d.cfg, deps.sched, d.channelMgr, deps.consumerTeamStore, deps.quotaChecker, d.pgStores.Sessions, d.pgStores.Agents, contactCollector, deps.postTurn, deps.subagentMgr, d.usageCapSvc)
// Webhook callback worker — delivers async webhook_calls rows to receiver callback_url.
// Runs in both editions: Standard (PG, concurrency=4) and Lite (SQLite, concurrency=1).
cmd/gateway_lifecycle_shell_deny_groups.go
+5 -1
View File
@@ -30,6 +30,10 @@ func subscribeShellDenyGroupsReload(msgBus *bus.MessageBus, toolsReg *tools.Regi
return
}
et.SetGlobalShellDenyGroups(updatedCfg.Tools.ShellDenyGroups)
slog.Info("shell deny groups reloaded via pub/sub", "groups", len(updatedCfg.Tools.ShellDenyGroups))
et.SetCommandKeywordAllowlist(updatedCfg.Tools.CommandKeywordAllowlist)
slog.Info("shell deny groups reloaded via pub/sub",
"groups", len(updatedCfg.Tools.ShellDenyGroups),
"command_keyword_allowlist_rules", len(updatedCfg.Tools.CommandKeywordAllowlist),
)
})
}
cmd/gateway_lifecycle_shell_deny_groups_test.go
+31
View File
@@ -35,6 +35,37 @@ func TestShellDenyGroupsConfigReload_UpdatesGlobal(t *testing.T) {
if v, ok := got["package_install"]; !ok || v != true {
t.Fatalf("expected pub/sub to set global package_install=true, got %v", got)
}
rules := execTool.CommandKeywordAllowlistForTest()
if len(rules) != 0 {
t.Fatalf("expected empty command keyword allowlist, got %v", rules)
}
}
func TestShellDenyGroupsConfigReload_UpdatesCommandKeywordAllowlist(t *testing.T) {
msgBus := bus.New()
defer msgBus.Unsubscribe("shell-deny-groups-config-reload")
toolsReg := tools.NewRegistry()
execTool := tools.NewExecTool("/tmp", false)
toolsReg.Register(execTool)
subscribeShellDenyGroupsReload(msgBus, toolsReg)
msgBus.Broadcast(bus.Event{
Name: bus.TopicConfigChanged,
Payload: &config.Config{
Tools: config.ToolsConfig{
CommandKeywordAllowlist: []config.CommandKeywordAllowlistRule{
{ID: "github-content", Command: "gh", Args: []string{"--body"}, Keywords: []string{"secret"}},
},
},
},
})
rules := execTool.CommandKeywordAllowlistForTest()
if len(rules) != 1 || rules[0].ID != "github-content" {
t.Fatalf("expected command keyword allowlist to reload, got %v", rules)
}
}
// TestShellDenyGroupsConfigReload_IgnoresOtherEvents: subscriber must guard
cmd/gateway_managed.go
+17 -6
View File
@@ -31,6 +31,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/store/pg"
"github.com/nextlevelbuilder/goclaw/internal/tools"
"github.com/nextlevelbuilder/goclaw/internal/tracing"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
@@ -58,6 +59,7 @@ func wireExtras(
sandboxMgr sandbox.Manager,
redisClient any, // nil when built without -tags redis or when Redis is unconfigured
domainBus eventbus.DomainEventBus,
usageCapSvc *usagecaps.Service,
) (*tools.ContextFileInterceptor, *mcpbridge.Pool, *media.Store, tools.PostTurnProcessor) {
// 1. Build cache instances (in-memory or Redis depending on build tags)
agentCtxCache, userCtxCache := makeCaches(redisClient)
@@ -82,9 +84,15 @@ func wireExtras(
}
}
// Register media analysis tools (need mediaStore for file access).
toolsReg.Register(tools.NewReadDocumentTool(providerReg, mediaStore))
toolsReg.Register(tools.NewReadAudioTool(providerReg, mediaStore))
toolsReg.Register(tools.NewReadVideoTool(providerReg, mediaStore))
readDocumentTool := tools.NewReadDocumentTool(providerReg, mediaStore)
readDocumentTool.SetUsageCapService(usageCapSvc)
toolsReg.Register(readDocumentTool)
readAudioTool := tools.NewReadAudioTool(providerReg, mediaStore)
readAudioTool.SetUsageCapService(usageCapSvc)
toolsReg.Register(readAudioTool)
readVideoTool := tools.NewReadVideoTool(providerReg, mediaStore)
readVideoTool.SetUsageCapService(usageCapSvc)
toolsReg.Register(readVideoTool)
toolsReg.Register(tools.NewCreateVideoTool(providerReg))
slog.Info("media tools registered", "tools", "read_document,read_audio,read_video,create_video")
}
@@ -177,7 +185,7 @@ func wireExtras(
"disabled_count", n, "edition", edition.Current().Name)
}
handlers := buildHookHandlers(stores, providerReg, appCfg.Hooks)
handlers := buildHookHandlers(stores, providerReg, appCfg.Hooks, usageCapSvc)
stdOpts := hooks.StdDispatcherOpts{
Store: hs,
Audit: hooks.NewAuditWriter(hs, ""),
@@ -201,6 +209,7 @@ func wireExtras(
ToolPolicy: toolPE,
Skills: skillsLoader,
SkillAccessStore: skillAccessStore,
SkillSlashCommands: appCfg.Skills.SlashCommands,
HasMemory: hasMemory,
TraceCollector: traceCollector,
EnsureUserProfile: ensureUserProfile,
@@ -228,6 +237,7 @@ func wireExtras(
MediaStore: mediaStore,
ModelPricing: appCfg.Telemetry.ModelPricing,
TracingStore: stores.Tracing,
UsageCaps: usageCapSvc,
MemoryStore: stores.Memory,
ContactStore: stores.Contacts,
TenantStore: stores.Tenants,
@@ -295,7 +305,7 @@ func wireExtras(
writeMemIntc = tools.NewMemoryInterceptor(stores.Memory, workspace)
// Hook KG extraction on memory writes if KG store is available
if stores.KnowledgeGraph != nil && stores.BuiltinTools != nil {
writeMemIntc.SetKGExtractFunc(buildKGExtractFunc(stores.KnowledgeGraph, stores.BuiltinTools, providerReg))
writeMemIntc.SetKGExtractFunc(buildKGExtractFunc(stores.KnowledgeGraph, stores.BuiltinTools, providerReg, usageCapSvc))
}
}
if readTool, ok := toolsReg.Get("read_file"); ok {
@@ -703,7 +713,7 @@ type kgSettings struct {
// buildKGExtractFunc returns a callback that extracts entities from memory content.
// Settings are read from the builtin_tools table on each invocation (not cached),
// so changes take effect immediately without restart.
func buildKGExtractFunc(kgStore store.KnowledgeGraphStore, bts store.BuiltinToolStore, providerReg *providers.Registry) tools.KGExtractFunc {
func buildKGExtractFunc(kgStore store.KnowledgeGraphStore, bts store.BuiltinToolStore, providerReg *providers.Registry, usageCapSvc *usagecaps.Service) tools.KGExtractFunc {
return func(ctx context.Context, agentID, userID, content string) {
slog.Info("kg extract: triggered", "agent", agentID, "user", userID, "content_len", len(content))
// Read settings from DB on each call so admin changes take effect immediately
@@ -727,6 +737,7 @@ func buildKGExtractFunc(kgStore store.KnowledgeGraphStore, bts store.BuiltinTool
return
}
extractor := kg.NewExtractor(p, settings.ExtractionModel, settings.MinConfidence)
extractor.SetUsageCapService(usageCapSvc)
result, err := extractor.Extract(ctx, content)
if err != nil {
slog.Warn("kg extract: extraction failed", "agent", agentID, "error", err)
cmd/gateway_methods.go
+3 -1
View File
@@ -12,14 +12,16 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/gateway/methods"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
func registerAllMethods(server *gateway.Server, agents *agent.Router, sessStore store.SessionStore, cronStore store.CronStore, pairingStore store.PairingStore, cfg *config.Config, cfgPath, workspace, dataDir string, msgBus *bus.MessageBus, execApprovalMgr *tools.ExecApprovalManager, agentStore store.AgentStore, skillStore store.SkillStore, configSecretsStore store.ConfigSecretsStore, teamStore store.TeamStore, contextFileInterceptor *tools.ContextFileInterceptor, logTee *gateway.LogTee, heartbeatStore store.HeartbeatStore, configPermStore store.ConfigPermissionStore, sysConfigStore store.SystemConfigStore, tenantStore store.TenantStore, skillTenantCfgStore store.SkillTenantConfigStore, audioMgr *audio.Manager) (*methods.PairingMethods, *methods.HeartbeatMethods, *methods.ChatMethods, *methods.ConfigPermissionsMethods) {
func registerAllMethods(server *gateway.Server, agents *agent.Router, sessStore store.SessionStore, cronStore store.CronStore, pairingStore store.PairingStore, cfg *config.Config, cfgPath, workspace, dataDir string, msgBus *bus.MessageBus, execApprovalMgr *tools.ExecApprovalManager, agentStore store.AgentStore, skillStore store.SkillStore, configSecretsStore store.ConfigSecretsStore, teamStore store.TeamStore, contextFileInterceptor *tools.ContextFileInterceptor, logTee *gateway.LogTee, heartbeatStore store.HeartbeatStore, configPermStore store.ConfigPermissionStore, sysConfigStore store.SystemConfigStore, tenantStore store.TenantStore, skillTenantCfgStore store.SkillTenantConfigStore, audioMgr *audio.Manager, usageCapSvc *usagecaps.Service) (*methods.PairingMethods, *methods.HeartbeatMethods, *methods.ChatMethods, *methods.ConfigPermissionsMethods) {
router := server.Router()
// Phase 1: Core methods
chatMethods := methods.NewChatMethods(agents, sessStore, cfg, server.RateLimiter(), msgBus)
chatMethods.SetAudioManager(audioMgr) // Wire TTS auto-apply for WS responses
chatMethods.SetUsageCapService(usageCapSvc)
chatMethods.Register(router)
methods.NewAgentsMethods(agents, cfg, cfgPath, workspace, agentStore, contextFileInterceptor, msgBus).Register(router)
methods.NewSessionsMethods(sessStore, msgBus, cfg).Register(router)
cmd/gateway_setup.go
+2 -2
View File
@@ -14,11 +14,11 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/bootstrap"
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/edition"
mcpbridge "github.com/nextlevelbuilder/goclaw/internal/mcp"
"github.com/nextlevelbuilder/goclaw/internal/permissions"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/sandbox"
"github.com/nextlevelbuilder/goclaw/internal/edition"
"github.com/nextlevelbuilder/goclaw/internal/skills"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/store/pg"
@@ -214,6 +214,7 @@ func setupToolRegistry(
// Apply global shell deny-group toggles before any request can arrive.
// Per-agent overrides via store.WithShellDenyGroups still win per-key.
et.SetGlobalShellDenyGroups(cfg.Tools.ShellDenyGroups)
et.SetCommandKeywordAllowlist(cfg.Tools.CommandKeywordAllowlist)
et.DenyPaths(dataDir, ".goclaw/")
// Allow skills execution: master-tenant skills-store + all tenant-scoped skills-store dirs.
et.AllowPathExemptions(
@@ -603,4 +604,3 @@ func setupSkillsSystem(
return skillsLoader, skillSearchTool, globalSkillsDir, bundledSkillsDir, builtinSkillsDir
}
cmd/gateway_system_config_sync_test.go
+14
View File
@@ -46,3 +46,17 @@ func TestSeedConfigForContextPersistsZeroInboundDebounce(t *testing.T) {
t.Fatalf("gateway.inbound_debounce_ms = %q, want 0", got)
}
}
func TestSeedConfigForContextDoesNotCreateSkillUploadTenantOverride(t *testing.T) {
t.Parallel()
sc := &captureSystemConfigStore{data: map[string]string{}}
cfg := config.Default()
cfg.Skills.MaxUploadSizeMB = 64
seedConfigForContext(store.WithTenantID(context.Background(), store.MasterTenantID), sc, cfg, false)
if _, ok := sc.data[config.SkillMaxUploadSizeSystemConfigKey]; ok {
t.Fatalf("%s should not be seeded; missing key lets SKILL.md frontmatter override global config", config.SkillMaxUploadSizeSystemConfigKey)
}
}
docs/02-providers.md
+23
View File
@@ -54,6 +54,29 @@ Streaming fallback is conservative: backup models are tried only if the stream f
---
## Usage Cap Pricing Enforcement
Standard edition can enforce AI budget caps before billable provider dispatch. API-key providers use OpenRouter `/models` pricing as the catalog source, with optional tenant/provider/model overrides in the dashboard.
Excluded provider classes:
- `chatgpt_oauth`, `claude_cli`, and `bailian` are treated as subscription/non-API pricing in round one.
- local/no-key subprocess providers such as `acp` and `ollama` are skipped unless a future feature explicitly enables pricing for them.
Runtime flow:
1. Resolve the stored provider by name and skip non-billable provider classes.
2. Load matching policies for tenant, agent, provider, provider type, and model.
3. Resolve custom pricing override first, then OpenRouter catalog pricing when a matching policy has a cost ceiling. Native provider model IDs are mapped to OpenRouter prefixes for common providers such as OpenAI, Anthropic, and Gemini.
4. Reserve estimated tokens and cost atomically before each dispatch attempt.
5. Reconcile reserved counters after the provider returns usage or after a failed call.
Token-only policies do not require catalog pricing. Model fallback routes reserve against the actual candidate provider/model before each attempt. Cached input is separated from uncached input for OpenAI-compatible usage accounting. Partial stream failures keep the estimate, or actual provider usage when available, instead of clearing billed output to zero. Internal LLM calls for memory flush, compaction, media reading tools (`read_image`, `read_document`, `read_audio`, `read_video`), and subagents use the same preflight/reconcile path.
The legacy agent-level `budget_monthly_cents` field is treated as a generated monthly agent USD cap. Existing values are backfilled during migration, and later agent budget edits update or remove the generated cap policy.
Supported price units: input, output, cache read, cache write, reasoning, request, image, and web search.
---
## 2. Supported Providers
### Six Core Provider Types
docs/03-tools-system.md
+34 -1
View File
@@ -246,6 +246,32 @@ User-facing parameter schemas for the most commonly configured tools.
```
Available presets: `gh`, `gcloud`, `aws`, `kubectl`, `terraform`.
### Credentialed CLI keyword allowlist
`config.tools.commandKeywordAllowlist` lets operators allow specific product or security vocabulary inside selected credentialed CLI content arguments without disabling `deny_args`.
Example:
```json
{
"tools": {
"commandKeywordAllowlist": [
{
"id": "github-content",
"command": "gh",
"subcommands": ["issue create", "issue edit", "pr create", "pr comment"],
"args": ["--body", "--title"],
"argPositions": [],
"keywords": ["secret", "secrets", "token", "credential"],
"reason": "Allow security vocabulary in GitHub issue and PR prose"
}
]
}
}
```
The rule above allows `gh issue create --body "secret rotation notes"` but still blocks command paths like `gh secret set TOKEN`. `argPositions` are 0-based after the matched subcommand. The scanner evaluates command arguments only; it does not read the contents of files passed through arguments such as `--body-file`.
---
## 6. Interception Layer
@@ -346,6 +372,13 @@ Custom tools are shell-based tools defined at runtime via the HTTP API — no re
| `env` | no | Encrypted environment variables injected at runtime |
| `enabled` | no | Toggle without deleting (default true) |
Credentialed CLI env entries support two API/UI kinds:
- `sensitive` (default): encrypted at rest, masked in normal API responses, replace-only in UI, and flattened only at credential injection time.
- `value`: encrypted at rest but visible to authorized admins in API/UI for non-secret settings such as public URLs, domains, limits, regions, and feature flags.
Legacy env JSON like `{"TOKEN":"..."}` is still accepted and treated as `sensitive`.
**Execution:** Template placeholders are rendered with shell-escaped argument values, then run via `sh -c`. The same deny-pattern check as the `exec` tool applies — no reverse shells, no `curl | sh`, etc.
**Scope:**
@@ -406,7 +439,7 @@ Current adopters: `web_search`, `web_fetch`, `tts`, `create_image`, `read_image`
- Per-key: agent value takes precedence over global value
- Multi-tenant invariant: each tenant's config is isolated
**Live reload:** Changes to `config.tools.shellDenyGroups` propagate via `bus.TopicConfigChanged` pub/sub. Next agent turn automatically applies new toggles.
**Live reload:** Changes to `config.tools.shellDenyGroups` and `config.tools.commandKeywordAllowlist` propagate via `bus.TopicConfigChanged` pub/sub. Next agent turn automatically applies new toggles.
**Deny-group classes** (from `internal/tools/shell_deny_groups.go` — all denied by default):
docs/04-gateway-protocol.md
+1 -1
View File
@@ -442,7 +442,7 @@ All CRUD endpoints require `Authorization: Bearer <token>` and `X-GoClaw-User-Id
| Method | Path | Description |
|--------|------|-------------|
| GET | `/v1/skills` | List skills |
| POST | `/v1/skills/upload` | Upload skill ZIP (max 20 MB) |
| POST | `/v1/skills/upload` | Upload skill ZIP (configurable, default 20 MB, max 500 MB) |
| DELETE | `/v1/skills/{id}` | Delete a skill |
**Traces** (`/v1/traces`):
docs/06-store-data-model.md
+28
View File
@@ -45,6 +45,7 @@ The `Stores` struct is the top-level container holding all PostgreSQL-backed sto
| ContactStore | `PGContactStore` | Channel contacts (auto-collected), cross-channel deduplication, merge |
| ActivityStore | `PGActivityStore` | Audit logs, action tracking, compliance |
| SnapshotStore | `PGSnapshotStore` | Hourly usage snapshots, cost aggregation, time series queries |
| UsageCapStore | `PGUsageCapStore` | OpenRouter pricing catalog, pricing overrides, cap policies, reservations, counters, events |
| SecureCLIStore | `PGSecureCLIStore` | CLI binary configs with encrypted credential injection |
| APIKeyStore | `PGAPIKeyStore` | Gateway API keys, scopes, expiration, revocation |
| HookStore | `PGHookStore` | Lifecycle hook definitions (event, handler type, matcher, config), execution audit log |
@@ -85,6 +86,33 @@ Migration versions:
---
## Usage Cap Storage
Usage cap enforcement is Standard/PostgreSQL-only in round one. The `UsageCapStore` is wired on the PostgreSQL store factory and left nil in SQLite/Lite builds.
Tables:
- `usage_pricing_catalog`: OpenRouter model catalog prices, raw upstream model payload, sync time.
- `usage_pricing_overrides`: tenant/provider/model override prices for custom billing assumptions.
- `usage_cap_policies`: cap definitions scoped by tenant, agent, provider, provider type, model, `window_key`, and `source`.
- `usage_cap_counters`: current window used and reserved token/cost counters.
- `usage_cap_reservations`: preflight reservations keyed by LLM call attempt.
- `usage_cap_events`: allow/block/reconcile/skip audit events.
Reservation updates are atomic: counters are updated only when `used + reserved + estimate` remains below configured token and cost ceilings.
Reservation keys are idempotent per policy, so a retry using the same key does not double-increment reserved counters.
Policy `agent_id` references must belong to the same tenant as the policy. Policy and pricing override `provider_id` references may belong to the same tenant or the master tenant for default provider fallback, but not another non-master tenant.
Catalog and override price fields are nullable decimal strings with non-negative validation in the store layer and database checks.
Pricing resolution checks exact override/catalog model IDs first, then provider-derived OpenRouter aliases for native unprefixed model IDs.
Agent `budget_monthly_cents` values are bridged into `usage_cap_policies` with `source = 'agent_budget_monthly_cents'`, an agent scope, `window_key = 'month'`, and `max_cost_micros = budget_monthly_cents * 10000`. Updating or clearing the agent budget keeps that generated policy in sync; manual cap policies continue to use `source = 'manual'`.
Migration versions:
- PostgreSQL: `000070_usage_caps_pricing`, `000071_usage_cap_policies`, `000072_agent_budget_usage_cap_bridge`.
- SQLite: no schema change; feature is not active in Lite.
---
## 3. Session Caching
The session store uses an in-memory write-behind cache to minimize database I/O during the agent tool loop. All reads and writes happen in memory; data is flushed to the persistent backend only when `Save()` is called at the end of a run.
docs/07-bootstrap-skills-memory.md
+24
View File
@@ -279,6 +279,30 @@ This decision is re-evaluated each time the system prompt is built, so newly hot
---
## 9.5. Explicit Slash Skill Commands
Users can bypass implicit skill matching by starting a prompt with a slash command:
| Pattern | Behavior |
|---------|----------|
| `/<slug> prompt` | Activates the skill by slug and treats `prompt` as the skill input |
| `/use <slug-or-name> prompt` | Activates the skill by slug or display name |
| `/list-skills` | Shows available skills for the current agent context |
| `/help <slug-or-name>` | Shows description and usage guidance for one skill |
Slash detection runs during prompt construction after request context is scoped and before the skills section is built. A matched skill narrows the per-request `SkillFilter` to that skill and injects the full `SKILL.md` instructions into the system prompt for the current turn only. Normal matching remains unchanged for messages that do not start with the configured prefix, path-like strings such as `/home/user/file`, or unresolved commands without suggestions.
Tenant settings live in `system_configs`:
| Key | Default | Behavior |
|-----|---------|----------|
| `skills.slash_commands.enabled` | `true` | Enable slash command detection |
| `skills.slash_commands.suggest_not_found` | `true` | Suggest similar skills for unknown commands |
| `skills.slash_commands.partial_matching` | `false` | Allow unique prefixes such as `/frontend` |
| `skills.slash_commands.prefix` | `/` | Single-character command prefix |
---
## 10. Skills -- BM25 Search
An in-memory BM25 index provides keyword-based skill search. The index is lazily rebuilt whenever the skill version changes.
docs/09-security.md
+3
View File
@@ -221,11 +221,14 @@ AES-256-GCM encryption for secrets stored in PostgreSQL. Key provided via `GOCLA
| LLM provider API keys | `llm_providers` | `api_key` |
| MCP server API keys | `mcp_servers` | `api_key` |
| Custom tool env vars | `custom_tools` | `env` |
| Credentialed CLI env vars | `secure_cli_binaries`, `secure_cli_agent_grants`, `secure_cli_user_credentials` | `encrypted_env` |
**Format**: `"aes-gcm:" + base64(12-byte nonce + ciphertext + GCM tag)`
Backward compatible: values without the `aes-gcm:` prefix are returned as plaintext (for migration from unencrypted data).
Credentialed CLI env entries have a separate visibility kind inside the encrypted JSON blob when `GOCLAW_ENCRYPTION_KEY` is configured. `sensitive` entries are masked in normal API/UI responses and never returned raw except through the explicit audited grant reveal flow. `value` entries use the same at-rest storage path but are returned to authorized admins for operational review.
---
## 4. Rate Limiting -- Gateway + Tool
docs/14-skills-runtime.md
+11 -4
View File
@@ -29,6 +29,11 @@ How skills access Python, Node.js, and system tools inside Docker containers and
└─────────────────────────────────────────────────────────┘
```
Explicit skill activation is handled before runtime execution. When a user starts
their prompt with `/<skill-slug>` or `/use <skill name>`, the gateway resolves
the skill, injects its `SKILL.md` into the current turn, and then normal runtime
rules apply to any scripts or package dependencies that skill uses.
---
## 2. Pre-installed Packages (Option A)
@@ -139,7 +144,7 @@ To install additional packages: pip3 install <pkg> or npm install -g <pkg>
- Run Python/Node scripts via exec tool
- Install packages via `pip3 install` / `npm install -g`
- Access files in `/app/workspace/` including `.media/` subdirectory
- Access files in `/app/workspace/`, including `.uploads/` for current user uploads and `.media/` for legacy media refs
- Read skill files from `.goclaw/skills-store/`
### What Agents CANNOT Do
@@ -156,16 +161,18 @@ To install additional packages: pip3 install <pkg> or npm install -g <pkg>
Uploaded files (from web chat, Telegram, Discord, etc.) are persisted to:
```
/app/workspace/.media/{sessionHash}/{uuid}.{ext}
/app/workspace/.uploads/{safe-original-name}-{8hex}.{ext}
```
Uploads without a usable original filename fall back to `{uuid}.{ext}`. Legacy media refs may still resolve from `.media/{sessionHash}/{uuid}.{ext}`.
The `enrichDocumentPaths()` function injects the full path into `<media:document>` tags:
```
<media:document name="report.pdf" path="/app/workspace/.media/abc123/uuid.pdf">
<media:document name="report.pdf" path="/app/workspace/.uploads/report-a1b2c3d4.pdf">
```
Agents can read these files directly via exec — no copy to `/tmp` needed.
Agents can read these files directly via exec — no copy to `/tmp` needed. For archive uploads such as `.zip`, inspect or extract with commands like `unzip -l "<path>"` or `unzip -q "<path>" -d <output-dir>`.
---
docs/16-skill-publishing.md
+1 -1
View File
@@ -183,7 +183,7 @@ Unlike the HTTP upload handler, the tool does **not** archive the skill on missi
| `..` in relative path | Skip (prevent traversal) |
| Symlinks | Skip (prevent escape) |
| System artifacts | Skip (`.DS_Store`, `__MACOSX`, `Thumbs.db`, etc.) |
| Total dir size > 20 MB | Reject with error |
| Total upload size exceeds configured limit | Reject with error. Default is 20 MB; configurable via `skills.max_upload_size_mb`, `GOCLAW_SKILLS_MAX_UPLOAD_SIZE_MB`, tenant `skills.max_upload_size_mb`, or SKILL.md frontmatter `max_upload_size_mb`; clamped to 1-500 MB. |
---
docs/18-http-api.md
+66 -1
View File
@@ -321,7 +321,7 @@ Use `direct_selection_count` plus the `selected_provider` sequence to verify rea
| Method | Path | Description |
|--------|------|-------------|
| `GET` | `/v1/skills` | List all skills |
| `POST` | `/v1/skills/upload` | Upload ZIP with SKILL.md (20 MB limit) |
| `POST` | `/v1/skills/upload` | Upload ZIP with SKILL.md (configurable 20 MB default, 1-500 MB range) |
| `GET` | `/v1/skills/{id}` | Get skill details |
| `PUT` | `/v1/skills/{id}` | Update skill metadata |
| `DELETE` | `/v1/skills/{id}` | Delete skill (not system skills) |
@@ -331,6 +331,18 @@ Use `direct_selection_count` plus the `selected_provider` sequence to verify rea
### Skill Grants
Skill upload size is enforced per ZIP file. The effective limit resolves in this order:
tenant `system_configs["skills.max_upload_size_mb"]`, then `SKILL.md` frontmatter
`max_upload_size_mb`, then config/env `skills.max_upload_size_mb` /
`GOCLAW_SKILLS_MAX_UPLOAD_SIZE_MB`, then the default 20 MB. Values are clamped
to 1-500 MB.
Skill slash-command behavior is configured through tenant `system_configs`:
`skills.slash_commands.enabled`, `skills.slash_commands.suggest_not_found`,
`skills.slash_commands.partial_matching`, and `skills.slash_commands.prefix`.
The default prefix is `/`; supported prompt forms are `/<slug> prompt`,
`/use <slug-or-name> prompt`, `/list-skills`, and `/help <slug-or-name>`.
| Method | Path | Description |
|--------|------|-------------|
| `POST` | `/v1/skills/{id}/grants/agent` | Grant skill to agent |
@@ -390,6 +402,36 @@ LLM provider management. API keys are encrypted with AES-256-GCM in the database
| `GET` | `/v1/embedding/status` | Check global embedding availability |
| `GET` | `/v1/providers/claude-cli/auth-status` | Check Claude CLI login status |
### Model Pricing
| Method | Path | Description |
|--------|------|-------------|
| `POST` | `/v1/model-pricing/sync-openrouter` | Sync OpenRouter `/models` pricing catalog (master scope) |
| `GET` | `/v1/model-pricing` | Search catalog models by `model` query |
| `GET` | `/v1/model-pricing/overrides` | List tenant pricing overrides, optionally by `provider_id` |
| `PUT` | `/v1/model-pricing/overrides` | Upsert provider/model custom pricing |
| `DELETE` | `/v1/model-pricing/overrides/{id}` | Delete pricing override |
Override body:
```json
{
"provider_id": "0193a5b0-7000-7000-8000-000000000123",
"provider_type": "openrouter",
"model_id": "anthropic/claude-sonnet-4-5",
"pricing": {
"input": "0.000003",
"output": "0.000015",
"cache_read": "0.0000003",
"cache_write": "0.00000375",
"reasoning": "0.000015",
"request": "0",
"image": "0",
"web_search": "0"
}
}
```
**Supported types:** `anthropic_native`, `openai_compat`, `chatgpt_oauth`, `gemini_native`, `dashscope`, `bailian`, `minimax`, `claude_cli`, `acp`
Reconnect response:
@@ -1271,11 +1313,34 @@ Follow response:
| `GET` | `/v1/usage/timeseries` | Time-series usage points |
| `GET` | `/v1/usage/breakdown` | Breakdown by provider/model/channel |
| `GET` | `/v1/usage/summary` | Summary with period comparison |
| `GET` | `/v1/usage-caps/policies` | List usage cap policies |
| `POST` | `/v1/usage-caps/policies` | Create token/cost cap policy |
| `PATCH` | `/v1/usage-caps/policies/{id}` | Update cap policy |
| `DELETE` | `/v1/usage-caps/policies/{id}` | Delete cap policy |
| `GET` | `/v1/usage-caps/utilization` | Current-window used and reserved counters |
| `GET` | `/v1/usage-caps/events` | Recent allow/block/reconcile/skip events |
**Query params:** `from`, `to` (RFC 3339), `agent_id`, `provider`, `model`, `channel`, `group_by`
**Periods:** `24h`, `today`, `7d`, `30d`
Usage cap policy body:
```json
{
"agent_id": "optional-agent-uuid",
"provider_id": "optional-provider-uuid",
"provider_type": "openrouter",
"model_id": "anthropic/claude-sonnet-4-5",
"window": "day",
"max_tokens": 500000,
"max_cost_usd": 25,
"enabled": true
}
```
Policy responses include read-only `source`. `source="agent_budget_monthly_cents"` means the policy is generated from the agent monthly budget field and must be changed there; direct policy updates/deletes return `409 Conflict`.
---
## 24. Activity & Audit
docs/21-agent-evolution-and-skill-management.md
+1 -1
View File
@@ -450,7 +450,7 @@ System skills (`is_system=true`) cannot be modified through any path.
| Symlink detection | `filepath.WalkDir` + `d.Type()&os.ModeSymlink` check |
| Path traversal | `strings.Contains(rel, "..")` rejection |
| Content size limit | 100KB max for SKILL.md content |
| Companion size limit | 20MB max total for companion files (scripts, assets) |
| Companion size limit | Configurable per ZIP upload; default 20MB, clamped to 1-500MB |
| Soft-delete | Files moved to `.trash/`, never hard-deleted |
---
docs/project-changelog.md
+74
View File
@@ -18,6 +18,60 @@ Significant changes, features, and fixes in reverse chronological order.
- Added runtime binary discovery, SecureCLI preset, deny-pattern, and Dockerfile contract coverage for Google Workspace CLI.
### Slash skill commands
**Features**
- Added explicit slash skill activation for `/<slug>`, `/use <slug-or-name>`, `/list-skills`, and `/help <slug-or-name>`.
- Added tenant settings for slash command enablement, similar-skill suggestions, partial matching, and custom prefix.
**Tests**
- Added backend coverage for parser false positives, exact/partial skill resolution, suggestions, help/list commands, and config overlays.
### Configurable skill upload limits
**Features**
- Added configurable skill ZIP upload limits with config/env, SKILL.md frontmatter, and tenant system setting support.
- Added dashboard settings and dynamic upload validation so the Web UI follows the tenant limit instead of hardcoding 20MB.
**Tests**
- Added backend coverage for limit precedence, clamping, oversized rejection, and frontend coverage for parameterized upload validation.
### CLI environment variable visibility
**Features**
- Added `sensitive` and `value` kinds for secure CLI environment variables across binary defaults, agent grant overrides, and user overrides.
- Plain value entries are visible to authorized admins for operational config review, while sensitive entries remain masked and replace-only.
**Fixes**
- Stopped per-user credential reads from returning legacy sensitive env values raw.
- Kept legacy `{"KEY":"value"}` env blobs backward-compatible by treating them as sensitive.
**Tests**
- Added backend regression coverage for env kind parsing, sanitized API responses, runtime flattening, and invalid kind rejection.
- Verified Web UI build after adding env-kind controls and warnings.
### Command keyword allowlist
**Features**
- Added scoped credentialed CLI keyword allowlist config for content arguments and positional arguments, with runtime reload and web config editing.
**Fixes**
- Kept credentialed CLI `deny_args` active for command paths such as `gh secret set` while allowing approved GitHub issue/PR prose to mention security vocabulary.
- Added security audit logging for real allowlisted pass-throughs without logging full argument values.
**Tests**
- Added regression coverage for scoped keyword masking, disabled rules, unsafe positional rules, config reload, and race-safe policy snapshots.
### Browser cookie sync and config UI
**Features**
@@ -39,6 +93,26 @@ Significant changes, features, and fixes in reverse chronological order.
## 2026-05-22
### Usage Cap budget controls
**Features**
- Added Standard/PostgreSQL usage caps for AI budget control by hour, day, week, or month.
- Caps support token and USD cost ceilings at tenant, agent, provider, provider type, and model scopes.
- Added OpenRouter catalog sync plus tenant/provider/model pricing overrides for input, output, cache read/write, reasoning, request, image, and web search units.
- Enforced caps in agent, fallback model, subagent, memory flush, compaction, and media reading tools (`read_image`, `read_document`, `read_audio`, `read_video`) with preflight reservation and post-call reconciliation.
- Added non-negative validation for catalog and override pricing fields.
- Added OpenRouter alias resolution for native model IDs, cached-input accounting normalization, and partial-stream failure reconciliation.
- Bridged legacy `budget_monthly_cents` into generated monthly agent USD cap policies, including migration backfill and save-time sync.
- Added web dashboard controls on Usage and Provider detail pages.
- Added Usage page editing for manual cap policies, including enable/disable, scope clearing, and token/USD limit clearing while keeping generated agent-budget caps read-only.
- Added usage-cap decision metadata to LLM spans, including allow/skip/block reason, policy IDs, estimates, actuals, and reconcile status.
**Tests**
- Added pricing and cap service coverage.
- Verified Go builds, SQLite build compatibility, full Go test suite, integration race suite, and Web UI production build.
### Messaging debounce hardening
**Features**
internal/agent/intent_classify.go
+11 -1
View File
@@ -8,6 +8,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/providers"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// IntentType represents the classified intent of a user message.
@@ -108,6 +109,10 @@ func containsWholeWord(s, kw string) bool {
// Uses keyword fast-path first, then falls back to LLM classification.
// Falls back to IntentNewTask on any error.
func ClassifyIntent(ctx context.Context, provider providers.Provider, model, userMessage string) IntentType {
return ClassifyIntentWithUsageCaps(ctx, nil, provider, model, userMessage)
}
func ClassifyIntentWithUsageCaps(ctx context.Context, usageCaps *usagecaps.Service, provider providers.Provider, model, userMessage string) IntentType {
// Fast-path: keyword matching for obvious patterns (no LLM cost).
if intent, ok := quickClassify(userMessage); ok {
return intent
@@ -116,7 +121,7 @@ func ClassifyIntent(ctx context.Context, provider providers.Provider, model, use
ctx, cancel := context.WithTimeout(ctx, intentClassifyTimeout)
defer cancel()
resp, err := provider.Chat(ctx, providers.ChatRequest{
req := providers.ChatRequest{
Messages: []providers.Message{
{Role: "system", Content: intentSystemPrompt},
{Role: "user", Content: userMessage},
@@ -126,6 +131,11 @@ func ClassifyIntent(ctx context.Context, provider providers.Provider, model, use
providers.OptMaxTokens: 20,
providers.OptTemperature: 0.0,
},
}
resp, err := usageCaps.Chat(ctx, provider, req, usagecaps.ChatOptions{
ModelID: model,
Purpose: "intent-classify",
MaxOutputTokens: 20,
})
if err != nil {
return IntentNewTask
internal/agent/loop_compact.go
+3 -2
View File
@@ -89,14 +89,15 @@ func (l *Loop) compactMessagesInPlace(ctx context.Context, messages []providers.
inTokens := l.estimateSummaryInputTokens(toSummarize)
slog.Info("compact_budget", "agent", l.id, "in_tokens", inTokens, "out_tokens", dynamicSummaryMax(inTokens))
resp, err := l.provider.Chat(sctx, providers.ChatRequest{
chatReq := providers.ChatRequest{
Messages: []providers.Message{{
Role: "user",
Content: compactionSummaryPrompt + sb.String(),
}},
Model: l.model,
Options: map[string]any{"max_tokens": dynamicSummaryMax(inTokens), "temperature": 0.3},
})
}
resp, err := l.callInternalLLMWithUsage(sctx, chatReq, "mid-loop-compaction")
if err != nil {
slog.Warn("mid_loop_compaction_failed", "agent", l.id, "error", err)
return nil
internal/agent/loop_history.go
+2
View File
@@ -117,6 +117,8 @@ func (l *Loop) buildMessages(ctx context.Context, history []providers.Message, s
}
}
userMessage, extraSystemPrompt, skillFilter = l.applySkillSlashCommand(ctx, userMessage, extraSystemPrompt, skillFilter)
// Build tool list, filtering out skill_manage when skill_evolve is off.
// Also applies ChannelAware filtering so channel-specific tools don't
// appear in ## Tooling when the current channel doesn't support them.
internal/agent/loop_history_sanitize.go
+3 -2
View File
@@ -284,11 +284,12 @@ func (l *Loop) maybeSummarize(ctx context.Context, sessionKey string) {
inTokens := l.estimateSummaryInputTokens(toSummarize)
slog.Info("compact_budget", "agent", l.id, "in_tokens", inTokens, "out_tokens", dynamicSummaryMax(inTokens))
resp, err := l.provider.Chat(sctx, providers.ChatRequest{
chatReq := providers.ChatRequest{
Messages: []providers.Message{{Role: "user", Content: prompt.String()}},
Model: l.model,
Options: map[string]any{"max_tokens": dynamicSummaryMax(inTokens), "temperature": 0.3},
})
}
resp, err := l.callInternalLLMWithUsage(sctx, chatReq, "session-summarization")
if err != nil {
slog.Warn("summarization failed", "session", sessionKey, "error", err)
return
internal/agent/loop_input_media.go
+3 -2
View File
@@ -27,10 +27,11 @@ func isTextMime(mime string) bool {
}
// collectRefsByKind gathers MediaRefs of a given kind from message history
// (reverse order) and current-turn refs. Historical first, current last.
// in chronological order, then appends current-turn refs. The last ref is the
// newest document for read_document's omitted media_id fallback.
func collectRefsByKind(messages []providers.Message, currentRefs []providers.MediaRef, kind string) []providers.MediaRef {
var refs []providers.MediaRef
for i := len(messages) - 1; i >= 0; i-- {
for i := range messages {
for _, ref := range messages[i].MediaRefs {
if ref.Kind == kind {
refs = append(refs, ref)
internal/agent/loop_pipeline_callbacks.go
+109 -46
View File
@@ -2,6 +2,7 @@ package agent
import (
"context"
"fmt"
"log/slog"
"strings"
"time"
@@ -15,6 +16,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
"github.com/nextlevelbuilder/goclaw/internal/workspace"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
@@ -293,31 +295,86 @@ func (l *Loop) makeCallLLM(req *RunRequest, emitRun func(AgentEvent)) func(ctx c
opts = append(opts, withProvider(provider.Name()))
}
spanID := l.emitLLMSpanStart(ctx, start, state.Iteration+1, chatReq.Messages, opts...)
var resp *providers.ChatResponse
var err error
if req.Stream {
resp, err = provider.ChatStream(ctx, chatReq, func(chunk providers.StreamChunk) {
if chunk.Thinking != "" {
emitRun(AgentEvent{
Type: protocol.ChatEventThinking,
AgentID: l.id,
RunID: req.RunID,
Payload: map[string]string{"content": chunk.Thinking},
})
}
if chunk.Content != "" {
emitRun(AgentEvent{
Type: protocol.ChatEventChunk,
AgentID: l.id,
RunID: req.RunID,
Payload: map[string]string{"content": chunk.Content},
})
}
})
} else {
resp, err = provider.Chat(ctx, chatReq)
recordUsageCapAttempt := func(reservation *usagecaps.Reservation) {
if reservation != nil {
opts = append(opts, withUsageCapMetadata(reservation.TraceMetadata()))
}
}
emitChunk := func(chunk providers.StreamChunk) {
if chunk.Thinking != "" {
emitRun(AgentEvent{
Type: protocol.ChatEventThinking,
AgentID: l.id,
RunID: req.RunID,
Payload: map[string]string{"content": chunk.Thinking},
})
}
if chunk.Content != "" {
emitRun(AgentEvent{
Type: protocol.ChatEventChunk,
AgentID: l.id,
RunID: req.RunID,
Payload: map[string]string{"content": chunk.Content},
})
}
}
callProvider := func(attempt string, request providers.ChatRequest) (*providers.ChatResponse, error) {
if fallbackProvider, ok := provider.(*providers.ModelFallbackProvider); ok {
before := func(callCtx context.Context, entry providers.FallbackCandidate, actualReq providers.ChatRequest) (providers.FallbackAfterCall, error) {
candidateAttempt := fmt.Sprintf("%s:%s:%s", attempt, entry.ProviderName, actualReq.Model)
reservation, reserveErr := l.reserveLLMUsageFor(callCtx, req, state.Iteration, actualReq, candidateAttempt, entry.ProviderName, actualReq.Model)
if reserveErr != nil {
recordUsageCapAttempt(reservation)
return nil, reserveErr
}
return func(callResp *providers.ChatResponse, callErr error, info providers.FallbackCallInfo) {
if reservation != nil {
if info.Streamed {
reservation.ReconcileStream(callCtx, callResp, callErr, true)
} else {
reservation.Reconcile(callCtx, callResp, callErr)
}
recordUsageCapAttempt(reservation)
}
}, nil
}
if req.Stream {
return fallbackProvider.ChatStreamWithHook(ctx, request, emitChunk, before)
}
return fallbackProvider.ChatWithHook(ctx, request, before)
}
reservation, reserveErr := l.reserveLLMUsage(ctx, req, state, request, attempt)
if reserveErr != nil {
recordUsageCapAttempt(reservation)
return nil, reserveErr
}
var callResp *providers.ChatResponse
var callErr error
if req.Stream {
streamed := false
callResp, callErr = provider.ChatStream(ctx, request, func(chunk providers.StreamChunk) {
if chunk.Content != "" || chunk.Thinking != "" || len(chunk.Images) > 0 {
streamed = true
}
emitChunk(chunk)
})
if reservation != nil {
reservation.ReconcileStream(ctx, callResp, callErr, streamed)
recordUsageCapAttempt(reservation)
}
return callResp, callErr
} else {
callResp, callErr = provider.Chat(ctx, request)
}
if reservation != nil {
reservation.Reconcile(ctx, callResp, callErr)
recordUsageCapAttempt(reservation)
}
return callResp, callErr
}
resp, err := callProvider("initial", chatReq)
slog.Info("debug.llm.first_response",
"has_error", err != nil,
"tool_calls_count", func() int {
@@ -342,28 +399,7 @@ func (l *Loop) makeCallLLM(req *RunRequest, emitRun func(AgentEvent)) func(ctx c
Role: "system",
Content: "MCP task tools are available in this turn. Do not ask for CRM identifier/email first. Call the relevant MCP task tool immediately, then answer with the tool result.",
})
if req.Stream {
resp, err = provider.ChatStream(ctx, retryReq, func(chunk providers.StreamChunk) {
if chunk.Thinking != "" {
emitRun(AgentEvent{
Type: protocol.ChatEventThinking,
AgentID: l.id,
RunID: req.RunID,
Payload: map[string]string{"content": chunk.Thinking},
})
}
if chunk.Content != "" {
emitRun(AgentEvent{
Type: protocol.ChatEventChunk,
AgentID: l.id,
RunID: req.RunID,
Payload: map[string]string{"content": chunk.Content},
})
}
})
} else {
resp, err = provider.Chat(ctx, retryReq)
}
resp, err = callProvider("retry-tool-choice", retryReq)
slog.Info("debug.llm.retry_response",
"has_error", err != nil,
"tool_calls_count", func() int {
@@ -536,3 +572,30 @@ func (l *Loop) makeBootstrapCleanup() func(ctx context.Context, state *pipeline.
return l.bootstrapCleanup(ctx, l.agentUUID, state.Input.UserID)
}
}
func (l *Loop) reserveLLMUsage(ctx context.Context, req *RunRequest, state *pipeline.RunState, chatReq providers.ChatRequest, attempt string) (*usagecaps.Reservation, error) {
if l.usageCaps == nil || state.Provider == nil {
return nil, nil
}
return l.reserveLLMUsageFor(ctx, req, state.Iteration, chatReq, attempt, state.Provider.Name(), state.Model)
}
func (l *Loop) reserveLLMUsageFor(ctx context.Context, req *RunRequest, iteration int, chatReq providers.ChatRequest, attempt, providerName, model string) (*usagecaps.Reservation, error) {
if l.usageCaps == nil {
return nil, nil
}
tenantID := store.TenantIDFromContext(ctx)
if tenantID == uuid.Nil {
tenantID = l.tenantID
}
key := fmt.Sprintf("%s:%s:%d:%s", req.RunID, l.agentUUID.String(), iteration+1, attempt)
return l.usageCaps.Preflight(ctx, usagecaps.Request{
TenantID: tenantID,
AgentID: l.agentUUID,
ProviderName: providerName,
ModelID: model,
ReservationKey: key,
Messages: chatReq.Messages,
MaxOutputTokens: l.maxOutputTokensFromRequest(chatReq),
})
}
internal/agent/loop_tracing.go
+22 -5
View File
@@ -14,6 +14,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
"github.com/nextlevelbuilder/goclaw/internal/tracing"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
func (l *Loop) emit(event AgentEvent) {
@@ -51,20 +52,33 @@ func (l *Loop) IsRunning() bool { return l.activeRuns.Load() > 0 }
type spanOption func(*spanOverrides)
type spanOverrides struct {
model string
provider string
model string
provider string
usageCapAttempts []usagecaps.TraceMetadata
}
func withModel(m string) spanOption { return func(o *spanOverrides) { o.model = m } }
func withProvider(p string) spanOption { return func(o *spanOverrides) { o.provider = p } }
func withUsageCapMetadata(metadata usagecaps.TraceMetadata) spanOption {
return func(o *spanOverrides) {
if !metadata.Empty() {
o.usageCapAttempts = append(o.usageCapAttempts, metadata)
}
}
}
// resolveSpan returns (model, provider) applying any overrides on top of agent defaults.
func (l *Loop) resolveSpan(opts []spanOption) (string, string) {
o := l.resolveSpanOverrides(opts)
return o.model, o.provider
}
func (l *Loop) resolveSpanOverrides(opts []spanOption) spanOverrides {
o := spanOverrides{model: l.model, provider: l.provider.Name()}
for _, fn := range opts {
fn(&o)
}
return o.model, o.provider
return o
}
// ---------------------------------------------------------------------------
@@ -150,6 +164,7 @@ func (l *Loop) emitLLMSpanEnd(ctx context.Context, spanID uuid.UUID, start time.
"status": store.SpanStatusCompleted,
}
var spanMetadata json.RawMessage
spanOpts := l.resolveSpanOverrides(opts)
if callErr != nil {
updates["status"] = store.SpanStatusError
@@ -176,8 +191,7 @@ func (l *Loop) emitLLMSpanEnd(ctx context.Context, spanID uuid.UUID, start time.
}
}
// Calculate cost if pricing config is available.
model, providerName := l.resolveSpan(opts)
if pricing := tracing.LookupPricing(l.modelPricing, providerName, model); pricing != nil {
if pricing := tracing.LookupPricing(l.modelPricing, spanOpts.provider, spanOpts.model); pricing != nil {
cost := tracing.CalculateCost(pricing, resp.Usage)
if cost > 0 {
updates["total_cost"] = cost
@@ -203,6 +217,9 @@ func (l *Loop) emitLLMSpanEnd(ctx context.Context, spanID uuid.UUID, start time.
if decision := providers.ReasoningDecisionFromContext(ctx); decision != nil {
spanMetadata = providers.MergeReasoningMetadata(spanMetadata, *decision)
}
if len(spanOpts.usageCapAttempts) > 0 {
spanMetadata = usagecaps.MergeTraceMetadata(spanMetadata, spanOpts.usageCapAttempts)
}
if len(spanMetadata) > 0 {
updates["metadata"] = spanMetadata
}
internal/agent/loop_types.go
+74 -64
View File
@@ -23,6 +23,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/tokencount"
"github.com/nextlevelbuilder/goclaw/internal/tools"
"github.com/nextlevelbuilder/goclaw/internal/tracing"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// bootstrapAutoCleanupTurns is the number of user messages after which
@@ -80,13 +81,13 @@ type Loop struct {
// agentUUID is the canonical DB primary key. Use for SQL WHERE/JOIN,
// DomainEvent.AgentID, OTel span attributes, and context propagation via
// store.WithAgentID. See docs/agent-identity-conventions.md.
agentUUID uuid.UUID
tenantID uuid.UUID // agent's owning tenant
agentUUID uuid.UUID
tenantID uuid.UUID // agent's owning tenant
// agentOtherConfig is a defensive byte copy of agents.other_config JSONB.
// Copied once at Loop construction; used to build AgentAudioSnapshot at tool dispatch.
agentOtherConfig json.RawMessage
agentType string // "open" or "predefined"
defaultTimezone string // system default timezone for bootstrap pre-fill
agentType string // "open" or "predefined"
defaultTimezone string // system default timezone for bootstrap pre-fill
provider providers.Provider
model string
modelRegistry providers.ModelRegistry // resolves per-model context window at run time (nil = use static contextWindow)
@@ -108,7 +109,7 @@ type Loop struct {
// Memory flush runs if callback != nil; auto-inject runs if AutoInjector != nil.
autoInjector memory.AutoInjector // v3 L0 memory auto-inject (nil = disabled)
eventPub bus.EventPublisher // currently unused by Loop; kept for future use
eventPub bus.EventPublisher // currently unused by Loop; kept for future use
domainBus eventbus.DomainEventBus // V3 domain event bus for consolidation pipeline
sessions store.SessionStore
tools tools.ToolExecutor
@@ -121,11 +122,12 @@ type Loop struct {
summarizeMu sync.Map // sessionKey → *sync.Mutex
// Bootstrap/persona context (loaded at startup, injected into system prompt)
ownerIDs []string
skillsLoader *skills.Loader
skillAllowList []string // nil = all, [] = none, ["x","y"] = filter
hasMemory bool
contextFiles []bootstrap.ContextFile
ownerIDs []string
skillsLoader *skills.Loader
skillAllowList []string // nil = all, [] = none, ["x","y"] = filter
skillSlashCommands config.SkillSlashCommandConfig
hasMemory bool
contextFiles []bootstrap.ContextFile
// Per-user profile + file seeding + dynamic context loading
ensureUserProfile EnsureUserProfileFunc // create/resolve user profile + workspace
@@ -137,11 +139,11 @@ type Loop struct {
userSetups sync.Map // userID → *userSetup (workspace + seeding state, per Loop instance)
// Per-user MCP tools: servers requiring user credentials get connected per-request.
mcpStore store.MCPServerStore // for credential lookup
mcpPool *mcpbridge.Pool // user-keyed connection pool
mcpUserCredSrvs []store.MCPAccessInfo // servers needing per-user creds
mcpUserTools sync.Map // userID → []tools.Tool (cached per-user tools)
mcpGrantChecker mcpbridge.GrantChecker // runtime grant verification (nil = skip)
mcpStore store.MCPServerStore // for credential lookup
mcpPool *mcpbridge.Pool // user-keyed connection pool
mcpUserCredSrvs []store.MCPAccessInfo // servers needing per-user creds
mcpUserTools sync.Map // userID → []tools.Tool (cached per-user tools)
mcpGrantChecker mcpbridge.GrantChecker // runtime grant verification (nil = skip)
// Compaction config (memory flush settings)
compactionCfg *config.CompactionConfig
@@ -184,6 +186,7 @@ type Loop struct {
// Tenant-specific allowed paths beyond workspace (from system_configs['allowed_paths']).
// Filesystem tools (read_file, write_file, edit, list_files) check these at execution time.
tenantAllowedPaths []string
systemConfigs store.SystemConfigStore
// Per-tenant disabled tools (tool name → true means excluded from LLM)
disabledTools map[string]bool
@@ -239,13 +242,14 @@ type Loop struct {
// Budget enforcement: monthly spending limit in cents (0 = unlimited)
budgetMonthlyCents int
tracingStore store.TracingStore
usageCaps *usagecaps.Service
// Memory store for extractive memory fallback (writes directly when LLM flush fails)
memStore store.MemoryStore
// v3 orchestration mode (spawn/delegate/team) — controls tool visibility
orchMode OrchestrationMode
delegateTargets []DelegateTargetEntry // delegation targets for prompt injection
orchMode OrchestrationMode
delegateTargets []DelegateTargetEntry // delegation targets for prompt injection
// v3 evolution metrics store (nil = disabled)
evolutionMetricsStore store.EvolutionMetricsStore
@@ -328,11 +332,12 @@ type LoopConfig struct {
OnEvent func(AgentEvent)
// Bootstrap/persona context
OwnerIDs []string
SkillsLoader *skills.Loader
SkillAllowList []string // nil = all, [] = none, ["x","y"] = filter
HasMemory bool
ContextFiles []bootstrap.ContextFile
OwnerIDs []string
SkillsLoader *skills.Loader
SkillAllowList []string // nil = all, [] = none, ["x","y"] = filter
SkillSlashCommands config.SkillSlashCommandConfig
HasMemory bool
ContextFiles []bootstrap.ContextFile
// Compaction config
CompactionCfg *config.CompactionConfig
@@ -350,11 +355,11 @@ type LoopConfig struct {
// Agent UUID + tenant for context propagation to tools
AgentUUID uuid.UUID
TenantID uuid.UUID // agent's owning tenant — injected into execution context
AgentOtherConfig json.RawMessage // raw other_config JSONB — copied defensively in NewLoop
AgentType string // "open" or "predefined"
DisplayName string // human-readable agent display name (for runtime section)
IsTeamLead bool // agent leads a team (from resolver detection)
TenantID uuid.UUID // agent's owning tenant — injected into execution context
AgentOtherConfig json.RawMessage // raw other_config JSONB — copied defensively in NewLoop
AgentType string // "open" or "predefined"
DisplayName string // human-readable agent display name (for runtime section)
IsTeamLead bool // agent leads a team (from resolver detection)
// Per-user profile + file seeding + dynamic context loading
EnsureUserProfile EnsureUserProfileFunc // preferred: separate profile + workspace
@@ -381,6 +386,7 @@ type LoopConfig struct {
// Tenant-specific allowed paths beyond workspace (from system_configs['allowed_paths']).
TenantAllowedPaths []string
SystemConfigs store.SystemConfigStore
// Per-tenant disabled tools (tool name → true means excluded)
DisabledTools map[string]bool
@@ -430,19 +436,20 @@ type LoopConfig struct {
// Budget enforcement
BudgetMonthlyCents int
TracingStore store.TracingStore
UsageCaps *usagecaps.Service
// Memory store for extractive memory fallback (writes directly when LLM flush fails)
MemoryStore store.MemoryStore
// Per-user MCP tools (servers requiring per-user credentials)
MCPStore store.MCPServerStore // for credential lookup
MCPPool *mcpbridge.Pool // user-keyed connection pool
MCPUserCredSrvs []store.MCPAccessInfo // servers needing per-user creds
MCPGrantChecker mcpbridge.GrantChecker // runtime grant verification (nil = skip)
MCPStore store.MCPServerStore // for credential lookup
MCPPool *mcpbridge.Pool // user-keyed connection pool
MCPUserCredSrvs []store.MCPAccessInfo // servers needing per-user creds
MCPGrantChecker mcpbridge.GrantChecker // runtime grant verification (nil = skip)
// V3 orchestration mode (resolved by resolver, controls tool visibility)
OrchMode OrchestrationMode
DelegateTargets []DelegateTargetEntry // delegation targets for prompt injection
OrchMode OrchestrationMode
DelegateTargets []DelegateTargetEntry // delegation targets for prompt injection
// V3 evolution metrics store for recording tool/retrieval/feedback metrics
EvolutionMetricsStore store.EvolutionMetricsStore
@@ -527,6 +534,7 @@ func NewLoop(cfg LoopConfig) *Loop {
ownerIDs: cfg.OwnerIDs,
skillsLoader: cfg.SkillsLoader,
skillAllowList: cfg.SkillAllowList,
skillSlashCommands: cfg.SkillSlashCommands,
hasMemory: cfg.HasMemory,
contextFiles: cfg.ContextFiles,
defaultTimezone: cfg.DefaultTimezone,
@@ -550,6 +558,7 @@ func NewLoop(cfg LoopConfig) *Loop {
builtinToolSettings: cfg.BuiltinToolSettings,
tenantToolSettings: cfg.TenantToolSettings,
tenantAllowedPaths: cfg.TenantAllowedPaths,
systemConfigs: cfg.SystemConfigs,
disabledTools: cfg.DisabledTools,
reasoningConfig: cfg.ReasoningConfig,
promptMode: cfg.PromptMode,
@@ -568,6 +577,7 @@ func NewLoop(cfg LoopConfig) *Loop {
modelPricing: cfg.ModelPricing,
budgetMonthlyCents: cfg.BudgetMonthlyCents,
tracingStore: cfg.TracingStore,
usageCaps: cfg.UsageCaps,
memStore: cfg.MemoryStore,
mcpStore: cfg.MCPStore,
mcpPool: cfg.MCPPool,
@@ -582,36 +592,36 @@ func NewLoop(cfg LoopConfig) *Loop {
// RunRequest is the input for processing a message through the agent.
type RunRequest struct {
SessionKey string // composite key: agent:{agentId}:{channel}:{peerKind}:{chatId}
Message string // user message
Media []bus.MediaFile // local media files with MIME types
ForwardMedia []bus.MediaFile // media files to forward to output (from delegation results)
Channel string // source channel instance name (e.g. "my-telegram-bot")
ChannelType string // platform type (e.g. "zalo_personal", "telegram") — for system prompt context
BitrixPortalDomain string // bitrix24-only: portal domain (e.g. "tamgiac.bitrix24.com") for entity URL construction
ChatTitle string // group chat display name (e.g. Telegram group title)
ChatID string // source chat ID
PeerKind string // "direct" or "group" (for session key building and tool context)
RunID string // unique run identifier
UserID string // external user ID (TEXT, free-form) for multi-tenant scoping
SenderID string // original individual sender ID (preserved in group chats for permission checks)
SenderName string // display name from channel metadata (for bootstrap auto-contact)
Role string // caller's RBAC role (admin/operator/viewer/owner); bypasses per-user grants for authenticated admins (#915)
Stream bool // whether to stream response chunks
ExtraSystemPrompt string // optional: injected into system prompt (skills, subagent context, etc.)
SkillFilter []string // per-request skill override: nil=use agent default, []=no skills, ["x","y"]=whitelist
HistoryLimit int // max user turns to keep in context (0=unlimited, from channel config)
ToolAllow []string // per-group tool allow list (nil = no restriction, supports "group:xxx")
LocalKey string // composite key with topic/thread suffix for routing (e.g. "-100123:topic:42")
ParentTraceID uuid.UUID // if set, reuse parent trace instead of creating new (announce runs)
ParentRootSpanID uuid.UUID // if set, nest announce agent span under this parent span
LinkedTraceID uuid.UUID // if set, create new trace with parent_trace_id pointing to this (team task runs)
TraceName string // override trace name (default: "chat <agentID>")
TraceTags []string // additional tags for the trace (e.g. "cron")
MaxIterations int // per-request override (0 = use agent default, must be lower)
ModelOverride string // per-request model override (heartbeat uses cheaper model)
ProviderOverride providers.Provider // per-request provider override (heartbeat uses different provider)
LightContext bool // skip loading context files (only inject ExtraSystemPrompt)
SessionKey string // composite key: agent:{agentId}:{channel}:{peerKind}:{chatId}
Message string // user message
Media []bus.MediaFile // local media files with MIME types
ForwardMedia []bus.MediaFile // media files to forward to output (from delegation results)
Channel string // source channel instance name (e.g. "my-telegram-bot")
ChannelType string // platform type (e.g. "zalo_personal", "telegram") — for system prompt context
BitrixPortalDomain string // bitrix24-only: portal domain (e.g. "tamgiac.bitrix24.com") for entity URL construction
ChatTitle string // group chat display name (e.g. Telegram group title)
ChatID string // source chat ID
PeerKind string // "direct" or "group" (for session key building and tool context)
RunID string // unique run identifier
UserID string // external user ID (TEXT, free-form) for multi-tenant scoping
SenderID string // original individual sender ID (preserved in group chats for permission checks)
SenderName string // display name from channel metadata (for bootstrap auto-contact)
Role string // caller's RBAC role (admin/operator/viewer/owner); bypasses per-user grants for authenticated admins (#915)
Stream bool // whether to stream response chunks
ExtraSystemPrompt string // optional: injected into system prompt (skills, subagent context, etc.)
SkillFilter []string // per-request skill override: nil=use agent default, []=no skills, ["x","y"]=whitelist
HistoryLimit int // max user turns to keep in context (0=unlimited, from channel config)
ToolAllow []string // per-group tool allow list (nil = no restriction, supports "group:xxx")
LocalKey string // composite key with topic/thread suffix for routing (e.g. "-100123:topic:42")
ParentTraceID uuid.UUID // if set, reuse parent trace instead of creating new (announce runs)
ParentRootSpanID uuid.UUID // if set, nest announce agent span under this parent span
LinkedTraceID uuid.UUID // if set, create new trace with parent_trace_id pointing to this (team task runs)
TraceName string // override trace name (default: "chat <agentID>")
TraceTags []string // additional tags for the trace (e.g. "cron")
MaxIterations int // per-request override (0 = use agent default, must be lower)
ModelOverride string // per-request model override (heartbeat uses cheaper model)
ProviderOverride providers.Provider // per-request provider override (heartbeat uses different provider)
LightContext bool // skip loading context files (only inject ExtraSystemPrompt)
// Run classification
RunKind string // "delegation", "announce" — empty for user-initiated runs
@@ -646,7 +656,7 @@ type RunRequest struct {
// RunResult is the output of a completed agent run.
type RunResult struct {
Content string `json:"content"`
Thinking string `json:"thinking,omitempty"` // reasoning content from thinking models (Claude, o3, DeepSeek-R1, Kimi)
Thinking string `json:"thinking,omitempty"` // reasoning content from thinking models (Claude, o3, DeepSeek-R1, Kimi)
RunID string `json:"runId"`
Iterations int `json:"iterations"`
Usage *providers.Usage `json:"usage,omitempty"`
internal/agent/media_persist_test.go
+8
View File
@@ -72,6 +72,14 @@ func TestPersistMedia_NamingScheme(t *testing.T) {
wantSlug: "",
wantExtPat: `\.pdf`,
},
{
name: "zip_archive_preserves_archive_extension",
filename: "codex.zip",
content: "zip bytes",
mime: "application/zip",
wantSlug: "codex",
wantExtPat: `\.zip`,
},
}
var loop Loop
internal/agent/media_test.go
+28
View File
@@ -73,6 +73,34 @@ func TestEnrichImageIDs_SkipsAlreadyEnriched(t *testing.T) {
}
}
func TestCollectRefsByKindOrdersOldestToNewestThenCurrent(t *testing.T) {
messages := []providers.Message{
{
Role: "user",
Content: "old",
MediaRefs: []providers.MediaRef{{ID: "old-doc", Kind: "document"}},
},
{
Role: "user",
Content: "latest-history",
MediaRefs: []providers.MediaRef{{ID: "latest-history-doc", Kind: "document"}},
},
}
current := []providers.MediaRef{{ID: "current-doc", Kind: "document"}}
got := collectRefsByKind(messages, current, "document")
want := []string{"old-doc", "latest-history-doc", "current-doc"}
if len(got) != len(want) {
t.Fatalf("got %d refs, want %d", len(got), len(want))
}
for i := range want {
if got[i].ID != want[i] {
t.Fatalf("ref[%d] = %q, want %q", i, got[i].ID, want[i])
}
}
}
// testMediaStore creates a temporary media.Store for tests.
func testMediaStore(t *testing.T) *media.Store {
t.Helper()
internal/agent/memoryflush.go
+3 -2
View File
@@ -186,7 +186,7 @@ func (l *Loop) runMemoryFlush(ctx context.Context, sessionKey string, settings *
// Run LLM iteration loop (max 5 iterations for flush)
maxFlushIter := 5
for range maxFlushIter {
resp, err := l.provider.Chat(flushCtx, providers.ChatRequest{
chatReq := providers.ChatRequest{
Messages: messages,
Tools: toolDefs,
Model: l.model,
@@ -194,7 +194,8 @@ func (l *Loop) runMemoryFlush(ctx context.Context, sessionKey string, settings *
"max_tokens": 4096,
"temperature": 0.3,
},
})
}
resp, err := l.callInternalLLMWithUsage(flushCtx, chatReq, "memory-flush")
if err != nil {
slog.Warn("memory flush: LLM call failed", "error", err)
l.extractiveMemoryFallback(flushCtx, sessionKey, history, "LLM error")
internal/agent/resolver.go
+7 -1
View File
@@ -25,6 +25,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
"github.com/nextlevelbuilder/goclaw/internal/tracing"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// ResolverDeps holds shared dependencies for the agent resolver.
@@ -84,7 +85,8 @@ type ResolverDeps struct {
MCPGrantChecker mcpbridge.GrantChecker
// Skill access store — for per-agent skill visibility filtering
SkillAccessStore store.SkillAccessStore
SkillAccessStore store.SkillAccessStore
SkillSlashCommands config.SkillSlashCommandConfig
// Config permission store for group file writer checks
ConfigPermStore store.ConfigPermissionStore
@@ -97,6 +99,7 @@ type ResolverDeps struct {
// Tracing store for budget enforcement queries
TracingStore store.TracingStore
UsageCaps *usagecaps.Service
// Memory store for extractive memory fallback
MemoryStore store.MemoryStore
@@ -489,6 +492,7 @@ func NewManagedResolver(deps ResolverDeps) ResolverFunc {
AgentToolPolicy: agentToolPolicyForTeam(agentToolPolicyWithWorkspace(agentToolPolicyWithMCP(ag.ParseToolsConfig(), hasMCPTools), hasTeam), isTeamLead),
SkillsLoader: deps.Skills,
SkillAllowList: skillAllowList,
SkillSlashCommands: deps.SkillSlashCommands,
HasMemory: hasMemory,
ContextFiles: contextFiles,
EnsureUserProfile: deps.EnsureUserProfile,
@@ -509,6 +513,7 @@ func NewManagedResolver(deps ResolverDeps) ResolverFunc {
BuiltinToolSettings: builtinSettings,
TenantToolSettings: tenantToolSettings,
TenantAllowedPaths: tenantAllowedPaths,
SystemConfigs: deps.SystemConfigs,
DisabledTools: disabledTools,
ReasoningConfig: store.ResolveEffectiveReasoningConfig(providerReasoningDefaults, ag.ParseReasoningConfig()),
PromptMode: PromptMode(ag.ParsePromptMode()),
@@ -528,6 +533,7 @@ func NewManagedResolver(deps ResolverDeps) ResolverFunc {
ModelPricing: deps.ModelPricing,
BudgetMonthlyCents: derefInt(ag.BudgetMonthlyCents),
TracingStore: deps.TracingStore,
UsageCaps: deps.UsageCaps,
MemoryStore: deps.MemoryStore,
MCPStore: deps.MCPStore,
MCPPool: deps.MCPPool,
internal/agent/skill_slash_command_guidance.go
+65
View File
@@ -0,0 +1,65 @@
package agent
import (
"fmt"
"path/filepath"
"sort"
"strings"
"github.com/nextlevelbuilder/goclaw/internal/skills"
)
func (r skillSlashCommandResult) systemPromptSection() string {
switch r.Kind {
case skillSlashCommandActivate:
return fmt.Sprintf("## Explicit Skill Activation\n\nThe user explicitly requested skill `%s` (%s). Use this skill for the current request and treat the remaining user message as the skill input. Call `use_skill` with name `%s` for observability when available, then follow these instructions:\n\n%s", r.Skill.Slug, r.Skill.Name, r.Skill.Slug, r.SkillContent)
case skillSlashCommandList, skillSlashCommandHelp, skillSlashCommandUnknown:
return r.Guidance
default:
return ""
}
}
func buildSkillSlashListGuidance(all []skills.Info) string {
if len(all) == 0 {
return "## Skill Slash Command\n\nNo skills are currently available."
}
sort.Slice(all, func(i, j int) bool { return all[i].Slug < all[j].Slug })
var lines []string
lines = append(lines, "## Skill Slash Command", "", "Available skills:")
for _, skill := range all {
lines = append(lines, fmt.Sprintf("- `%s` - %s", skill.Slug, skillDisplayDescription(skill)))
}
return strings.Join(lines, "\n")
}
func buildSkillSlashHelpGuidance(skill skills.Info) string {
return fmt.Sprintf("## Skill Slash Command\n\nSkill `%s` (%s)\nDescription: %s\nLocation: %s\nExplain this skill and how to invoke it with the configured slash prefix.", skill.Slug, skill.Name, skillDisplayDescription(skill), filepath.ToSlash(skill.Path))
}
func buildSkillSlashUnknownGuidance(target string, suggestions []skills.Info) string {
var lines []string
lines = append(lines, "## Skill Slash Command", "", fmt.Sprintf("Requested skill `%s` was not found. Suggest these available alternatives:", target))
for _, skill := range suggestions {
lines = append(lines, fmt.Sprintf("- `%s` - %s", skill.Slug, skillDisplayDescription(skill)))
}
return strings.Join(lines, "\n")
}
func skillDisplayDescription(skill skills.Info) string {
if strings.TrimSpace(skill.Description) != "" {
return strings.TrimSpace(skill.Description)
}
return skill.Name
}
func appendExtraPrompt(existing, addition string) string {
addition = strings.TrimSpace(addition)
if addition == "" {
return existing
}
if strings.TrimSpace(existing) == "" {
return addition
}
return existing + "\n\n" + addition
}
internal/agent/skill_slash_command_matching.go
+124
View File
@@ -0,0 +1,124 @@
package agent
import (
"sort"
"strings"
"github.com/nextlevelbuilder/goclaw/internal/skills"
)
type parsedSkillSlashCommand struct {
verb string
target string
rest string
}
func parseSkillSlashCommand(message, prefix string) (parsedSkillSlashCommand, bool) {
message = strings.TrimSpace(message)
if message == "" || !strings.HasPrefix(message, prefix) {
return parsedSkillSlashCommand{}, false
}
after := strings.TrimSpace(strings.TrimPrefix(message, prefix))
if after == "" || looksLikePath(after) {
return parsedSkillSlashCommand{}, false
}
first, rest, _ := strings.Cut(after, " ")
first = strings.TrimSpace(first)
rest = strings.TrimSpace(rest)
switch strings.ToLower(first) {
case "list-skills":
return parsedSkillSlashCommand{verb: "list-skills"}, true
case "help":
if rest == "" {
return parsedSkillSlashCommand{}, false
}
return parsedSkillSlashCommand{verb: "help", target: rest}, true
case "use", "activate":
if rest == "" {
return parsedSkillSlashCommand{}, false
}
return parsedSkillSlashCommand{verb: strings.ToLower(first), rest: rest}, true
default:
return parsedSkillSlashCommand{verb: "direct", target: first, rest: rest}, true
}
}
func looksLikePath(value string) bool {
first, _, _ := strings.Cut(value, " ")
return strings.Contains(first, "/") || strings.Contains(first, "\\") || strings.Contains(first, ".")
}
func firstWord(value string) (string, string) {
first, rest, ok := strings.Cut(strings.TrimSpace(value), " ")
if !ok {
return strings.TrimSpace(value), ""
}
return strings.TrimSpace(first), strings.TrimSpace(rest)
}
func matchSkillCommandTarget(all []skills.Info, raw string, partial bool) (skills.Info, bool, string) {
raw = strings.TrimSpace(raw)
if raw == "" {
return skills.Info{}, false, ""
}
type candidate struct {
info skills.Info
matchText string
remainder string
score int
}
var matches []candidate
lowerRaw := strings.ToLower(raw)
partialTarget, partialRemainder := firstWord(raw)
lowerPartialTarget := strings.ToLower(partialTarget)
for _, skill := range all {
for _, value := range []string{skill.Slug, skill.Name} {
value = strings.TrimSpace(value)
if value == "" {
continue
}
lowerValue := strings.ToLower(value)
if lowerRaw == lowerValue {
matches = append(matches, candidate{info: skill, matchText: value, score: len([]rune(value))})
continue
}
if strings.HasPrefix(lowerRaw, lowerValue+" ") {
remainder := trimMatchedSkillCommandPrefix(raw, value)
matches = append(matches, candidate{info: skill, matchText: value, remainder: remainder, score: len([]rune(value))})
continue
}
if partial && lowerPartialTarget != "" && strings.HasPrefix(lowerValue, lowerPartialTarget) {
matches = append(matches, candidate{info: skill, matchText: value, remainder: partialRemainder, score: len([]rune(partialTarget))})
}
}
}
if len(matches) == 0 {
return skills.Info{}, false, ""
}
sort.Slice(matches, func(i, j int) bool {
return matches[i].score > matches[j].score
})
bestBySlug := make([]candidate, 0, len(matches))
seenSlug := make(map[string]struct{}, len(matches))
for _, match := range matches {
if _, ok := seenSlug[match.info.Slug]; ok {
continue
}
seenSlug[match.info.Slug] = struct{}{}
bestBySlug = append(bestBySlug, match)
}
best := bestBySlug[0]
if len(bestBySlug) > 1 && bestBySlug[0].score == bestBySlug[1].score {
return skills.Info{}, false, ""
}
return best.info, true, best.remainder
}
func trimMatchedSkillCommandPrefix(raw, matched string) string {
rawRunes := []rune(raw)
matchedRunes := []rune(matched)
if len(rawRunes) < len(matchedRunes) {
return ""
}
return strings.TrimSpace(string(rawRunes[len(matchedRunes):]))
}
internal/agent/skill_slash_command_similarity.go
+102
View File
@@ -0,0 +1,102 @@
package agent
import (
"sort"
"strings"
"unicode"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/skills"
)
func unknownSkillSlashResult(all []skills.Info, target string, cfg config.SkillSlashCommandConfig) skillSlashCommandResult {
if !cfg.EffectiveSuggestNotFound() {
return skillSlashCommandResult{Kind: skillSlashCommandNone}
}
suggestions := similarSkills(all, target, 3)
if len(suggestions) == 0 {
return skillSlashCommandResult{Kind: skillSlashCommandNone}
}
return skillSlashCommandResult{
Kind: skillSlashCommandUnknown,
Guidance: buildSkillSlashUnknownGuidance(target, suggestions),
Suggestions: suggestions,
}
}
func similarSkills(all []skills.Info, target string, limit int) []skills.Info {
target = strings.ToLower(strings.TrimSpace(target))
if target == "" {
return nil
}
type scored struct {
info skills.Info
score int
}
var scoredSkills []scored
for _, skill := range all {
best := scoreSimilarSkill(target, skill)
if best <= 3 {
scoredSkills = append(scoredSkills, scored{info: skill, score: best})
}
}
sort.Slice(scoredSkills, func(i, j int) bool {
if scoredSkills[i].score == scoredSkills[j].score {
return scoredSkills[i].info.Slug < scoredSkills[j].info.Slug
}
return scoredSkills[i].score < scoredSkills[j].score
})
if len(scoredSkills) > limit {
scoredSkills = scoredSkills[:limit]
}
out := make([]skills.Info, len(scoredSkills))
for i, scored := range scoredSkills {
out[i] = scored.info
}
return out
}
func scoreSimilarSkill(target string, skill skills.Info) int {
targetRunes := []rune(target)
slug := strings.ToLower(skill.Slug)
best := slashSkillDistance(target, slug)
if slugRunes := []rune(slug); len(slugRunes) >= len(targetRunes) {
best = min(best, slashSkillDistance(target, string(slugRunes[:len(targetRunes)])))
}
name := strings.ToLower(skill.Name)
if name != "" {
best = min(best, slashSkillDistance(target, name))
if nameRunes := []rune(name); len(nameRunes) >= len(targetRunes) {
best = min(best, slashSkillDistance(target, string(nameRunes[:len(targetRunes)])))
}
}
if strings.HasPrefix(slug, target) || strings.Contains(name, target) {
best = 0
}
return best
}
func slashSkillDistance(a, b string) int {
ar := []rune(a)
br := []rune(b)
if len(ar) == 0 {
return len(br)
}
prev := make([]int, len(br)+1)
for j := range prev {
prev[j] = j
}
for i, ca := range ar {
cur := make([]int, len(br)+1)
cur[0] = i + 1
for j, cb := range br {
cost := 0
if unicode.ToLower(ca) != unicode.ToLower(cb) {
cost = 1
}
cur[j+1] = min(min(cur[j]+1, prev[j+1]+1), prev[j]+cost)
}
prev = cur
}
return prev[len(br)]
}
internal/agent/skill_slash_commands.go
+130
View File
@@ -0,0 +1,130 @@
package agent
import (
"context"
"strings"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/skills"
)
type skillSlashCommandKind int
const (
skillSlashCommandNone skillSlashCommandKind = iota
skillSlashCommandActivate
skillSlashCommandList
skillSlashCommandHelp
skillSlashCommandUnknown
)
type skillSlashCommandResult struct {
Kind skillSlashCommandKind
Skill skills.Info
SkillContent string
RemainingPrompt string
Guidance string
Suggestions []skills.Info
}
func (l *Loop) applySkillSlashCommand(ctx context.Context, message, extraPrompt string, skillFilter []string) (string, string, []string) {
result := resolveSkillSlashCommand(ctx, l.skillsLoader, l.resolveSkillSlashCommandConfig(ctx), message)
if result.Kind == skillSlashCommandNone {
return message, extraPrompt, skillFilter
}
extraPrompt = appendExtraPrompt(extraPrompt, result.systemPromptSection())
switch result.Kind {
case skillSlashCommandActivate:
if result.RemainingPrompt == "" {
message = "Use the activated skill to help with the user's request."
} else {
message = result.RemainingPrompt
}
skillFilter = []string{result.Skill.Slug}
case skillSlashCommandList:
message = "List the available skills shown in the system instructions."
case skillSlashCommandHelp:
message = "Explain the requested skill and how it should be used."
case skillSlashCommandUnknown:
message = "Explain that the requested skill was not found and suggest available alternatives."
}
return message, extraPrompt, skillFilter
}
func (l *Loop) resolveSkillSlashCommandConfig(ctx context.Context) config.SkillSlashCommandConfig {
cfg := l.skillSlashCommands
if l.systemConfigs == nil {
return cfg
}
if raw, err := l.systemConfigs.Get(ctx, config.SkillSlashCommandsEnabledSystemConfigKey); err == nil && strings.TrimSpace(raw) != "" {
v := parseSkillSlashBool(raw)
cfg.Enabled = &v
}
if raw, err := l.systemConfigs.Get(ctx, config.SkillSlashSuggestNotFoundSystemConfigKey); err == nil && strings.TrimSpace(raw) != "" {
v := parseSkillSlashBool(raw)
cfg.SuggestNotFound = &v
}
if raw, err := l.systemConfigs.Get(ctx, config.SkillSlashPartialMatchingSystemConfigKey); err == nil && strings.TrimSpace(raw) != "" {
cfg.PartialMatching = parseSkillSlashBool(raw)
}
if raw, err := l.systemConfigs.Get(ctx, config.SkillSlashCommandPrefixSystemConfigKey); err == nil && strings.TrimSpace(raw) != "" {
cfg.Prefix = raw
}
return cfg
}
func resolveSkillSlashCommand(ctx context.Context, loader *skills.Loader, cfg config.SkillSlashCommandConfig, message string) skillSlashCommandResult {
if loader == nil || !cfg.EffectiveEnabled() {
return skillSlashCommandResult{Kind: skillSlashCommandNone}
}
parsed, ok := parseSkillSlashCommand(message, cfg.EffectivePrefix())
if !ok {
return skillSlashCommandResult{Kind: skillSlashCommandNone}
}
all := loader.ListSkills(ctx)
switch parsed.verb {
case "list-skills":
return skillSlashCommandResult{Kind: skillSlashCommandList, Guidance: buildSkillSlashListGuidance(all)}
case "help":
skill, matched, _ := matchSkillCommandTarget(all, parsed.target, cfg.EffectivePartialMatching())
if !matched {
return unknownSkillSlashResult(all, parsed.target, cfg)
}
return skillSlashCommandResult{Kind: skillSlashCommandHelp, Skill: skill, Guidance: buildSkillSlashHelpGuidance(skill)}
case "use", "activate":
return resolveSkillActivation(ctx, loader, all, parsed.rest, cfg)
default:
return resolveSkillActivation(ctx, loader, all, parsed.target+" "+parsed.rest, cfg)
}
}
func resolveSkillActivation(ctx context.Context, loader *skills.Loader, all []skills.Info, raw string, cfg config.SkillSlashCommandConfig) skillSlashCommandResult {
skill, matched, remainder := matchSkillCommandTarget(all, raw, cfg.EffectivePartialMatching())
if !matched {
fields := strings.Fields(raw)
target := strings.TrimSpace(raw)
if len(fields) > 0 {
target = fields[0]
}
return unknownSkillSlashResult(all, target, cfg)
}
content, ok := loader.LoadSkill(ctx, skill.Slug)
if !ok {
return unknownSkillSlashResult(all, skill.Slug, cfg)
}
return skillSlashCommandResult{
Kind: skillSlashCommandActivate,
Skill: skill,
SkillContent: content,
RemainingPrompt: strings.TrimSpace(remainder),
}
}
func parseSkillSlashBool(raw string) bool {
switch strings.ToLower(strings.TrimSpace(raw)) {
case "1", "true", "yes", "on":
return true
default:
return false
}
}
internal/agent/skill_slash_commands_test.go
+138
View File
@@ -0,0 +1,138 @@
package agent
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/skills"
)
func TestResolveSkillSlashCommandExactSlug(t *testing.T) {
loader := newSlashTestLoader(t)
result := resolveSkillSlashCommand(context.Background(), loader, config.SkillSlashCommandConfig{Enabled: boolPtr(true), Prefix: "/"}, "/frontend-design build a landing page")
if result.Kind != skillSlashCommandActivate {
t.Fatalf("kind = %v, want activate", result.Kind)
}
if result.Skill.Slug != "frontend-design" {
t.Fatalf("slug = %q, want frontend-design", result.Skill.Slug)
}
if result.RemainingPrompt != "build a landing page" {
t.Fatalf("remaining = %q", result.RemainingPrompt)
}
if !strings.Contains(result.SkillContent, "Use responsive components.") {
t.Fatal("expected loaded SKILL.md content")
}
}
func TestResolveSkillSlashCommandExactNameUseSyntax(t *testing.T) {
loader := newSlashTestLoader(t)
result := resolveSkillSlashCommand(context.Background(), loader, config.SkillSlashCommandConfig{Enabled: boolPtr(true), Prefix: "/"}, "/use Frontend Design build a landing page")
if result.Kind != skillSlashCommandActivate {
t.Fatalf("kind = %v, want activate", result.Kind)
}
if result.Skill.Slug != "frontend-design" {
t.Fatalf("slug = %q, want frontend-design", result.Skill.Slug)
}
if result.RemainingPrompt != "build a landing page" {
t.Fatalf("remaining = %q", result.RemainingPrompt)
}
}
func TestResolveSkillSlashCommandPartialMatchRequiresUniqueEnabled(t *testing.T) {
loader := newSlashTestLoader(t)
disabled := resolveSkillSlashCommand(context.Background(), loader, config.SkillSlashCommandConfig{Enabled: boolPtr(true), Prefix: "/"}, "/front build")
if disabled.Kind != skillSlashCommandUnknown {
t.Fatalf("disabled partial kind = %v, want unknown", disabled.Kind)
}
enabled := resolveSkillSlashCommand(context.Background(), loader, config.SkillSlashCommandConfig{Enabled: boolPtr(true), Prefix: "/", PartialMatching: true}, "/front build")
if enabled.Kind != skillSlashCommandActivate {
t.Fatalf("enabled partial kind = %v, want activate", enabled.Kind)
}
if enabled.Skill.Slug != "frontend-design" {
t.Fatalf("slug = %q, want frontend-design", enabled.Skill.Slug)
}
}
func TestResolveSkillSlashCommandFalsePositives(t *testing.T) {
loader := newSlashTestLoader(t)
for _, msg := range []string{"/home/user/project", "/etc/config.yaml", "https://example.com/path", "regular prompt"} {
result := resolveSkillSlashCommand(context.Background(), loader, config.SkillSlashCommandConfig{Enabled: boolPtr(true), Prefix: "/"}, msg)
if result.Kind != skillSlashCommandNone {
t.Fatalf("%q kind = %v, want none", msg, result.Kind)
}
}
}
func TestResolveSkillSlashCommandListAndHelp(t *testing.T) {
loader := newSlashTestLoader(t)
cfg := config.SkillSlashCommandConfig{Enabled: boolPtr(true), Prefix: "/"}
list := resolveSkillSlashCommand(context.Background(), loader, cfg, "/list-skills")
if list.Kind != skillSlashCommandList {
t.Fatalf("list kind = %v, want list", list.Kind)
}
if !strings.Contains(list.Guidance, "frontend-design") || !strings.Contains(list.Guidance, "git-helper") {
t.Fatalf("list guidance missing skills: %s", list.Guidance)
}
help := resolveSkillSlashCommand(context.Background(), loader, cfg, "/help frontend-design")
if help.Kind != skillSlashCommandHelp {
t.Fatalf("help kind = %v, want help", help.Kind)
}
if help.Skill.Slug != "frontend-design" || !strings.Contains(help.Guidance, "Frontend Design") {
t.Fatalf("unexpected help result: %#v", help)
}
helpByName := resolveSkillSlashCommand(context.Background(), loader, cfg, "/help Frontend Design")
if helpByName.Kind != skillSlashCommandHelp {
t.Fatalf("help by name kind = %v, want help", helpByName.Kind)
}
if helpByName.Skill.Slug != "frontend-design" {
t.Fatalf("help by name slug = %q, want frontend-design", helpByName.Skill.Slug)
}
}
func TestResolveSkillSlashCommandSuggestsUnknown(t *testing.T) {
loader := newSlashTestLoader(t)
result := resolveSkillSlashCommand(context.Background(), loader, config.SkillSlashCommandConfig{Enabled: boolPtr(true), Prefix: "/", SuggestNotFound: boolPtr(true)}, "/fronted build")
if result.Kind != skillSlashCommandUnknown {
t.Fatalf("kind = %v, want unknown", result.Kind)
}
if len(result.Suggestions) == 0 || result.Suggestions[0].Slug != "frontend-design" {
t.Fatalf("suggestions = %#v", result.Suggestions)
}
}
func boolPtr(v bool) *bool {
return &v
}
func newSlashTestLoader(t *testing.T) *skills.Loader {
t.Helper()
t.Setenv("GOCLAW_DISABLE_PERSONAL_SKILLS", "1")
root := t.TempDir()
writeSkill(t, root, "frontend-design", "Frontend Design", "Create polished UI layouts.", "Use responsive components.")
writeSkill(t, root, "git-helper", "Git Helper", "Handle git workflows.", "Use clean commits.")
return skills.NewLoader("", root, "")
}
func writeSkill(t *testing.T, root, slug, name, description, body string) {
t.Helper()
dir := filepath.Join(root, slug)
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatal(err)
}
content := "---\nname: " + name + "\ndescription: " + description + "\n---\n\n" + body + "\n"
if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte(content), 0644); err != nil {
t.Fatal(err)
}
}
internal/agent/title_generate.go
+13 -3
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/nextlevelbuilder/goclaw/internal/providers"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
const titleGenerateTimeout = 120 * time.Second
@@ -16,10 +17,14 @@ const titleSystemPrompt = `Generate a short title (max 15 words) for this conver
// GenerateTitle uses a lightweight LLM call to create a short conversation title
// from the user's first message. Returns empty string on error.
func GenerateTitle(ctx context.Context, provider providers.Provider, model, userMessage string) string {
return GenerateTitleWithUsageCaps(ctx, nil, provider, model, userMessage)
}
func GenerateTitleWithUsageCaps(ctx context.Context, usageCaps *usagecaps.Service, provider providers.Provider, model, userMessage string) string {
ctx, cancel := context.WithTimeout(ctx, titleGenerateTimeout)
defer cancel()
resp, err := provider.Chat(ctx, providers.ChatRequest{
req := providers.ChatRequest{
Messages: []providers.Message{
{Role: "system", Content: titleSystemPrompt},
{Role: "user", Content: userMessage},
@@ -29,13 +34,18 @@ func GenerateTitle(ctx context.Context, provider providers.Provider, model, user
// Larger budget: thinking-capable models (Gemini 2.5/3, GPT-5 reasoning)
// can consume output tokens on reasoning traces. 256 leaves room for a
// 15-word title even when the provider allocates some budget to thinking.
providers.OptMaxTokens: 256,
providers.OptTemperature: 0.3,
providers.OptMaxTokens: 256,
providers.OptTemperature: 0.3,
// Disable extended thinking for title generation — it's a trivial task
// that doesn't benefit from reasoning and defaults (esp. Gemini's "high")
// otherwise eat the entire max_tokens budget, truncating the title to 1 word.
providers.OptThinkingLevel: "off",
},
}
resp, err := usageCaps.Chat(ctx, provider, req, usagecaps.ChatOptions{
ModelID: model,
Purpose: "session-title",
MaxOutputTokens: 256,
})
if err != nil {
slog.Warn("title generation failed", "error", err)
internal/agent/usage_caps_runtime.go
+88
View File
@@ -0,0 +1,88 @@
package agent
import (
"context"
"fmt"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
func (l *Loop) reserveInternalLLMUsage(ctx context.Context, chatReq providers.ChatRequest, purpose string) (*usagecaps.Reservation, error) {
if l.usageCaps == nil || l.provider == nil {
return nil, nil
}
return l.reserveInternalLLMUsageFor(ctx, chatReq, purpose, l.provider.Name(), chatReq.Model)
}
func (l *Loop) reserveInternalLLMUsageFor(ctx context.Context, chatReq providers.ChatRequest, purpose, providerName, model string) (*usagecaps.Reservation, error) {
if l.usageCaps == nil {
return nil, nil
}
if model == "" {
model = chatReq.Model
}
if model == "" {
model = l.model
}
tenantID := store.TenantIDFromContext(ctx)
if tenantID == uuid.Nil {
tenantID = l.tenantID
}
return l.usageCaps.Preflight(ctx, usagecaps.Request{
TenantID: tenantID,
AgentID: l.agentUUID,
ProviderName: providerName,
ModelID: model,
ReservationKey: fmt.Sprintf("%s:%s:%s", purpose, l.agentUUID.String(), uuid.NewString()),
Messages: chatReq.Messages,
MaxOutputTokens: l.maxOutputTokensFromRequest(chatReq),
})
}
func (l *Loop) callInternalLLMWithUsage(ctx context.Context, chatReq providers.ChatRequest, purpose string) (*providers.ChatResponse, error) {
if fallbackProvider, ok := l.provider.(*providers.ModelFallbackProvider); ok {
before := func(callCtx context.Context, entry providers.FallbackCandidate, actualReq providers.ChatRequest) (providers.FallbackAfterCall, error) {
candidatePurpose := fmt.Sprintf("%s:%s:%s", purpose, entry.ProviderName, actualReq.Model)
reservation, err := l.reserveInternalLLMUsageFor(callCtx, actualReq, candidatePurpose, entry.ProviderName, actualReq.Model)
if err != nil {
return nil, err
}
return func(resp *providers.ChatResponse, callErr error, _ providers.FallbackCallInfo) {
if reservation != nil {
reservation.Reconcile(callCtx, resp, callErr)
}
}, nil
}
return fallbackProvider.ChatWithHook(ctx, chatReq, before)
}
reservation, reserveErr := l.reserveInternalLLMUsage(ctx, chatReq, purpose)
if reserveErr != nil {
return nil, reserveErr
}
resp, err := l.provider.Chat(ctx, chatReq)
if reservation != nil {
reservation.Reconcile(ctx, resp, err)
}
return resp, err
}
func (l *Loop) maxOutputTokensFromRequest(chatReq providers.ChatRequest) int {
maxTokens := l.effectiveMaxTokens()
if chatReq.Options == nil {
return maxTokens
}
if v, ok := chatReq.Options[providers.OptMaxTokens]; ok {
switch n := v.(type) {
case int:
maxTokens = n
case int64:
maxTokens = int(n)
case float64:
maxTokens = int(n)
}
}
return maxTokens
}
internal/channels/history.go
+5 -3
View File
@@ -36,9 +36,11 @@ const (
// Stored in RAM only (not persisted to DB) — used by channels that defer media
// download until the bot is actually mentioned (e.g. Telegram).
type MediaRef struct {
Type string // "image", "video", "audio", "voice", "document", "animation"
FileID string // platform-specific file ID for lazy download
FileSize int64 // file size in bytes (0 if unknown) — used to skip large files
Type string // "image", "video", "audio", "voice", "document", "animation"
FileID string // platform-specific file ID for lazy download
FileSize int64 // file size in bytes (0 if unknown) — used to skip large files
FileName string // original filename, if the platform exposes one
ContentType string // MIME type, if the platform exposes one
}
// HistoryEntry represents a single tracked group message.
internal/channels/history_compaction.go
+10 -3
View File
@@ -11,6 +11,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// CompactionConfig configures LLM-based history compaction.
@@ -20,6 +21,7 @@ type CompactionConfig struct {
MaxTokens int // max output tokens for summarization (default 4096)
Provider providers.Provider // LLM provider for summarization
Model string // model to use for summarization
UsageCaps *usagecaps.Service
}
// MaybeCompact checks if compaction is needed for a history key and triggers it in background.
@@ -55,7 +57,7 @@ func (ph *PendingHistory) MaybeCompact(historyKey string, currentCount int, cfg
// CompactGroup performs LLM-based compaction on a pending message group.
// Reused by both auto-compact (channel) and HTTP compact endpoint.
// Returns the number of entries remaining after compaction.
func CompactGroup(ctx context.Context, s store.PendingMessageStore, channelName, historyKey string, provider providers.Provider, model string, keepRecent, maxTokens int) (int, error) {
func CompactGroup(ctx context.Context, s store.PendingMessageStore, channelName, historyKey string, provider providers.Provider, model string, keepRecent, maxTokens int, usageCaps *usagecaps.Service) (int, error) {
if keepRecent <= 0 {
keepRecent = 40
}
@@ -94,13 +96,18 @@ func CompactGroup(ctx context.Context, s store.PendingMessageStore, channelName,
fmt.Fprintf(&sb, "%s%s: %s\n", prefix, ts, e.Body)
}
resp, err := provider.Chat(ctx, providers.ChatRequest{
req := providers.ChatRequest{
Messages: []providers.Message{{
Role: "user",
Content: "Summarize these group chat messages concisely, preserving key topics, decisions, names, and important context:\n\n" + sb.String(),
}},
Model: model,
Options: map[string]any{"max_tokens": maxTokens, "temperature": 0.3},
}
resp, err := usageCaps.Chat(ctx, provider, req, usagecaps.ChatOptions{
ModelID: model,
Purpose: "pending-history-compaction",
MaxOutputTokens: maxTokens,
})
if err != nil {
return 0, fmt.Errorf("llm summarize: %w", err)
@@ -185,7 +192,7 @@ func (ph *PendingHistory) runCompaction(historyKey string, cfg *CompactionConfig
return
}
_, err = CompactGroup(ctx, ph.store, ph.channelName, historyKey, cfg.Provider, cfg.Model, cfg.KeepRecent, cfg.MaxTokens)
_, err = CompactGroup(ctx, ph.store, ph.channelName, historyKey, cfg.Provider, cfg.Model, cfg.KeepRecent, cfg.MaxTokens, cfg.UsageCaps)
if err != nil {
slog.Warn("compaction.failed", "channel", ph.channelName, "key", historyKey, "error", err)
return
internal/channels/instance_loader.go
+9 -2
View File
@@ -16,6 +16,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/providerresolve"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// reloadStartTimeout bounds how long Reload() will wait for a single channel's
@@ -39,6 +40,7 @@ type InstanceLoader struct {
agentStore store.AgentStore
providerReg *providers.Registry
pendingCompactCfg *config.PendingCompactionConfig
usageCaps *usagecaps.Service
factories map[string]ChannelFactory
manager *Manager
msgBus *bus.MessageBus
@@ -78,6 +80,10 @@ func (l *InstanceLoader) SetPendingCompactionConfig(cfg *config.PendingCompactio
l.pendingCompactCfg = cfg
}
func (l *InstanceLoader) SetUsageCapService(s *usagecaps.Service) {
l.usageCaps = s
}
// RegisterFactory registers a factory for a channel type (e.g., "telegram", "discord").
func (l *InstanceLoader) RegisterFactory(channelType string, factory ChannelFactory) {
l.factories[channelType] = factory
@@ -306,8 +312,9 @@ func (l *InstanceLoader) loadInstance(ctx context.Context, inst store.ChannelIns
if p != nil && model != "" {
cc := &CompactionConfig{
Provider: p,
Model: model,
Provider: p,
Model: model,
UsageCaps: l.usageCaps,
}
if l.pendingCompactCfg != nil {
cc.Threshold = l.pendingCompactCfg.Threshold
internal/channels/telegram/handlers.go
+6 -12
View File
@@ -513,14 +513,8 @@ func (c *Channel) handleMessage(ctx context.Context, update telego.Update) {
// Resolve deferred media from history entries (lazy download).
if histRefs := c.GroupHistory().CollectMediaRefs(localKey); len(histRefs) > 0 {
histMedia, histErrors := c.resolveMediaRefs(ctx, histRefs)
for _, m := range histMedia {
mediaFiles = append(mediaFiles, bus.MediaFile{
Path: m.FilePath,
MimeType: m.ContentType,
Filename: m.FileName,
})
}
if len(histMedia) > 0 {
mediaFiles = prependMediaInfoFiles(mediaFiles, histMedia)
histTags := buildMediaTags(histMedia)
annotated = "[Media from recent group messages — only analyze if user asks about them]\n" + histTags + "\n[/Media]\n\n" + annotated
}
@@ -577,12 +571,12 @@ func (c *Channel) handleMessage(ctx context.Context, update telego.Update) {
// user sees typing indicator → first content appears directly.
metadata := map[string]string{
"message_id": fmt.Sprintf("%d", message.MessageID),
"user_id": fmt.Sprintf("%d", user.ID),
"message_id": fmt.Sprintf("%d", message.MessageID),
"user_id": fmt.Sprintf("%d", user.ID),
tools.MetaUsername: user.Username,
"first_name": user.FirstName,
"is_group": fmt.Sprintf("%t", isGroup),
"local_key": localKey,
"first_name": user.FirstName,
"is_group": fmt.Sprintf("%t", isGroup),
"local_key": localKey,
}
if message.Chat.Title != "" {
metadata[tools.MetaChatTitle] = message.Chat.Title
internal/channels/telegram/media.go
+59 -8
View File
@@ -14,6 +14,7 @@ import (
"github.com/mymmrac/telego"
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/channels"
"github.com/nextlevelbuilder/goclaw/internal/channels/media"
"github.com/nextlevelbuilder/goclaw/internal/tools"
@@ -359,6 +360,24 @@ func buildMediaTags(mediaList []MediaInfo) string {
return media.BuildMediaTags(mediaList)
}
func prependMediaInfoFiles(current []bus.MediaFile, history []MediaInfo) []bus.MediaFile {
if len(history) == 0 {
return current
}
ordered := make([]bus.MediaFile, 0, len(history)+len(current))
for _, m := range history {
if m.FilePath == "" {
continue
}
ordered = append(ordered, bus.MediaFile{
Path: m.FilePath,
MimeType: m.ContentType,
Filename: m.FileName,
})
}
return append(ordered, current...)
}
// extractDocumentContent delegates to the shared media package.
func extractDocumentContent(filePath, fileName string) (string, error) {
return media.ExtractDocumentContent(filePath, fileName)
@@ -412,22 +431,51 @@ func extractMediaRefs(msg *telego.Message) []channels.MediaRef {
refs = append(refs, channels.MediaRef{Type: "image", FileID: photo.FileID, FileSize: int64(photo.FileSize)})
}
if msg.Video != nil {
refs = append(refs, channels.MediaRef{Type: "video", FileID: msg.Video.FileID, FileSize: int64(msg.Video.FileSize)})
refs = append(refs, channels.MediaRef{
Type: "video",
FileID: msg.Video.FileID,
FileSize: int64(msg.Video.FileSize),
FileName: msg.Video.FileName,
ContentType: msg.Video.MimeType,
})
}
if msg.VideoNote != nil {
refs = append(refs, channels.MediaRef{Type: "video", FileID: msg.VideoNote.FileID, FileSize: int64(msg.VideoNote.FileSize)})
}
if msg.Animation != nil {
refs = append(refs, channels.MediaRef{Type: "animation", FileID: msg.Animation.FileID, FileSize: int64(msg.Animation.FileSize)})
refs = append(refs, channels.MediaRef{
Type: "animation",
FileID: msg.Animation.FileID,
FileSize: int64(msg.Animation.FileSize),
FileName: msg.Animation.FileName,
ContentType: msg.Animation.MimeType,
})
}
if msg.Audio != nil {
refs = append(refs, channels.MediaRef{Type: "audio", FileID: msg.Audio.FileID, FileSize: int64(msg.Audio.FileSize)})
refs = append(refs, channels.MediaRef{
Type: "audio",
FileID: msg.Audio.FileID,
FileSize: int64(msg.Audio.FileSize),
FileName: msg.Audio.FileName,
ContentType: msg.Audio.MimeType,
})
}
if msg.Voice != nil {
refs = append(refs, channels.MediaRef{Type: "voice", FileID: msg.Voice.FileID, FileSize: int64(msg.Voice.FileSize)})
refs = append(refs, channels.MediaRef{
Type: "voice",
FileID: msg.Voice.FileID,
FileSize: int64(msg.Voice.FileSize),
ContentType: msg.Voice.MimeType,
})
}
if msg.Document != nil {
refs = append(refs, channels.MediaRef{Type: "document", FileID: msg.Document.FileID, FileSize: int64(msg.Document.FileSize)})
refs = append(refs, channels.MediaRef{
Type: "document",
FileID: msg.Document.FileID,
FileSize: int64(msg.Document.FileSize),
FileName: msg.Document.FileName,
ContentType: msg.Document.MimeType,
})
}
return refs
}
@@ -492,9 +540,12 @@ func (c *Channel) resolveMediaRefs(ctx context.Context, refs []channels.MediaRef
continue
}
results = append(results, MediaInfo{
Type: ref.Type,
FilePath: filePath,
FileID: ref.FileID,
Type: ref.Type,
FilePath: filePath,
FileID: ref.FileID,
ContentType: ref.ContentType,
FileName: ref.FileName,
FileSize: ref.FileSize,
})
}
return results, errs
internal/channels/telegram/media_test.go
+54
View File
@@ -3,6 +3,11 @@ package telegram
import (
"strings"
"testing"
"github.com/mymmrac/telego"
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/channels"
)
// --- buildMediaTags tests ---
@@ -148,3 +153,52 @@ func TestBuildMediaTags_UnknownType(t *testing.T) {
t.Errorf("expected empty string for unknown type, got: %q", got)
}
}
func TestExtractMediaRefsPreservesDocumentMetadata(t *testing.T) {
msg := &telego.Message{
Document: &telego.Document{
FileID: "telegram-file-id",
FileName: "codex.zip",
MimeType: "application/zip",
FileSize: 1234,
},
}
got := extractMediaRefs(msg)
want := []channels.MediaRef{{
Type: "document",
FileID: "telegram-file-id",
FileSize: 1234,
FileName: "codex.zip",
ContentType: "application/zip",
}}
if len(got) != len(want) {
t.Fatalf("got %d refs, want %d", len(got), len(want))
}
if got[0] != want[0] {
t.Fatalf("ref = %+v, want %+v", got[0], want[0])
}
}
func TestPrependMediaInfoFilesKeepsHistoryBeforeCurrent(t *testing.T) {
current := []bus.MediaFile{{
Path: "/workspace/.uploads/current.pdf",
MimeType: "application/pdf",
Filename: "current.pdf",
}}
history := []MediaInfo{{
Type: "document",
FilePath: "/workspace/.uploads/codex.zip",
ContentType: "application/zip",
FileName: "codex.zip",
}}
got := prependMediaInfoFiles(current, history)
if len(got) != 2 {
t.Fatalf("got %d files, want 2", len(got))
}
if got[0].Path != history[0].FilePath || got[1].Path != current[0].Path {
t.Fatalf("order = [%q, %q], want history before current", got[0].Path, got[1].Path)
}
}
internal/config/config.go
+79 -1
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"log/slog"
"strings"
"sync"
"time"
@@ -47,6 +48,7 @@ type Config struct {
Providers ProvidersConfig `json:"providers"`
Gateway GatewayConfig `json:"gateway"`
Tools ToolsConfig `json:"tools"`
Skills SkillsConfig `json:"skills"`
Sessions SessionsConfig `json:"sessions"`
Database DatabaseConfig `json:"database"`
Tts TtsConfig `json:"tts"`
@@ -129,7 +131,82 @@ type DatabaseConfig struct {
// SkillsConfig configures the skills storage system.
type SkillsConfig struct {
StorageDir string `json:"storage_dir,omitempty"` // directory for skill content (default: dataDir/skills-store/)
StorageDir string `json:"storage_dir,omitempty"` // directory for skill content (default: dataDir/skills-store/)
MaxUploadSizeMB int `json:"max_upload_size_mb,omitempty"` // per-file ZIP upload limit
SlashCommands SkillSlashCommandConfig `json:"slash_commands,omitempty"`
}
// SkillSlashCommandConfig controls explicit slash-command skill activation.
type SkillSlashCommandConfig struct {
Enabled *bool `json:"enabled,omitempty"`
SuggestNotFound *bool `json:"suggest_not_found,omitempty"`
PartialMatching bool `json:"partial_matching,omitempty"`
Prefix string `json:"prefix,omitempty"`
}
const (
DefaultSkillMaxUploadSizeMB = 20
MinSkillMaxUploadSizeMB = 1
MaxSkillMaxUploadSizeMB = 500
DefaultSkillSlashCommandPrefix = "/"
SkillMaxUploadSizeSystemConfigKey = "skills.max_upload_size_mb"
SkillSlashCommandsEnabledSystemConfigKey = "skills.slash_commands.enabled"
SkillSlashSuggestNotFoundSystemConfigKey = "skills.slash_commands.suggest_not_found"
SkillSlashPartialMatchingSystemConfigKey = "skills.slash_commands.partial_matching"
SkillSlashCommandPrefixSystemConfigKey = "skills.slash_commands.prefix"
)
func ClampSkillMaxUploadSizeMB(value int) int {
if value == 0 {
return DefaultSkillMaxUploadSizeMB
}
if value < MinSkillMaxUploadSizeMB {
return MinSkillMaxUploadSizeMB
}
if value > MaxSkillMaxUploadSizeMB {
return MaxSkillMaxUploadSizeMB
}
return value
}
func (c SkillsConfig) EffectiveMaxUploadSizeMB() int {
return ClampSkillMaxUploadSizeMB(c.MaxUploadSizeMB)
}
func (c SkillsConfig) EffectiveMaxUploadSizeBytes() int64 {
return int64(c.EffectiveMaxUploadSizeMB()) << 20
}
func (c SkillSlashCommandConfig) EffectiveEnabled() bool {
if c.Enabled == nil {
return true
}
return *c.Enabled
}
func (c SkillSlashCommandConfig) EffectiveSuggestNotFound() bool {
if c.SuggestNotFound == nil {
return true
}
return *c.SuggestNotFound
}
func (c SkillSlashCommandConfig) EffectivePartialMatching() bool {
return c.PartialMatching
}
func (c SkillSlashCommandConfig) EffectivePrefix() string {
prefix := strings.TrimSpace(c.Prefix)
if prefix == "" {
return DefaultSkillSlashCommandPrefix
}
runes := []rune(prefix)
if len(runes) != 1 || runes[0] == ' ' || runes[0] == '\t' || runes[0] == '\n' || runes[0] == '\r' {
return DefaultSkillSlashCommandPrefix
}
return prefix
}
// AgentBinding maps a channel/peer pattern to a specific agent.
@@ -479,6 +556,7 @@ func (c *Config) ReplaceFrom(src *Config) {
c.Providers = src.Providers
c.Gateway = src.Gateway
c.Tools = src.Tools
c.Skills = src.Skills
c.Sessions = src.Sessions
c.Database = src.Database
c.Tts = src.Tts
internal/config/config_channels.go
+89 -75
View File
@@ -24,25 +24,25 @@ type ChannelsConfig struct {
}
type TelegramConfig struct {
Enabled bool `json:"enabled"`
Token string `json:"token"`
Proxy string `json:"proxy,omitempty"`
APIServer string `json:"api_server,omitempty"` // custom Telegram Bot API server URL (e.g. "http://localhost:8081")
AllowFrom FlexibleStringSlice `json:"allow_from"`
DMPolicy string `json:"dm_policy,omitempty"` // "pairing" (default), "allowlist", "open", "disabled"
GroupPolicy string `json:"group_policy,omitempty"` // "open" (default), "allowlist", "disabled"
RequireMention *bool `json:"require_mention,omitempty"` // require @bot mention in groups (default true)
MentionMode string `json:"mention_mode,omitempty"` // "strict" (default) = only respond when mentioned; "yield" = respond unless another bot is mentioned
HistoryLimit int `json:"history_limit,omitempty"` // max pending group messages for context (default 50, 0=disabled)
DMStream *bool `json:"dm_stream,omitempty"` // enable streaming for DMs (default false) — edits placeholder progressively
GroupStream *bool `json:"group_stream,omitempty"` // enable streaming for groups (default false) — sends new message, edits progressively
DraftTransport *bool `json:"draft_transport,omitempty"` // use sendMessageDraft for DM streaming (default true) — stealth preview, no notifications per edit
ReasoningStream *bool `json:"reasoning_stream,omitempty"` // show reasoning as separate message when provider emits thinking events (default true)
ReactionLevel string `json:"reaction_level,omitempty"` // "off" (default), "minimal", "full" — status emoji reactions
MediaMaxBytes int64 `json:"media_max_bytes,omitempty"` // max media download size in bytes (default 20MB)
LinkPreview *bool `json:"link_preview,omitempty"` // enable URL previews in messages (default true)
BlockReply *bool `json:"block_reply,omitempty"` // override gateway block_reply (nil = inherit)
ForceIPv4 bool `json:"force_ipv4,omitempty"` // force IPv4 for all Telegram API requests (use when IPv6 routing is broken)
Enabled bool `json:"enabled"`
Token string `json:"token"`
Proxy string `json:"proxy,omitempty"`
APIServer string `json:"api_server,omitempty"` // custom Telegram Bot API server URL (e.g. "http://localhost:8081")
AllowFrom FlexibleStringSlice `json:"allow_from"`
DMPolicy string `json:"dm_policy,omitempty"` // "pairing" (default), "allowlist", "open", "disabled"
GroupPolicy string `json:"group_policy,omitempty"` // "open" (default), "allowlist", "disabled"
RequireMention *bool `json:"require_mention,omitempty"` // require @bot mention in groups (default true)
MentionMode string `json:"mention_mode,omitempty"` // "strict" (default) = only respond when mentioned; "yield" = respond unless another bot is mentioned
HistoryLimit int `json:"history_limit,omitempty"` // max pending group messages for context (default 50, 0=disabled)
DMStream *bool `json:"dm_stream,omitempty"` // enable streaming for DMs (default false) — edits placeholder progressively
GroupStream *bool `json:"group_stream,omitempty"` // enable streaming for groups (default false) — sends new message, edits progressively
DraftTransport *bool `json:"draft_transport,omitempty"` // use sendMessageDraft for DM streaming (default true) — stealth preview, no notifications per edit
ReasoningStream *bool `json:"reasoning_stream,omitempty"` // show reasoning as separate message when provider emits thinking events (default true)
ReactionLevel string `json:"reaction_level,omitempty"` // "off" (default), "minimal", "full" — status emoji reactions
MediaMaxBytes int64 `json:"media_max_bytes,omitempty"` // max media download size in bytes (default 20MB)
LinkPreview *bool `json:"link_preview,omitempty"` // enable URL previews in messages (default true)
BlockReply *bool `json:"block_reply,omitempty"` // override gateway block_reply (nil = inherit)
ForceIPv4 bool `json:"force_ipv4,omitempty"` // force IPv4 for all Telegram API requests (use when IPv6 routing is broken)
// Optional STT (Speech-to-Text) pipeline for voice/audio inbound messages.
// When stt_proxy_url is set, audio/voice messages are transcribed before being forwarded to the agent.
@@ -133,7 +133,7 @@ type SlackConfig struct {
type WhatsAppConfig struct {
Enabled bool `json:"enabled"`
AuthDir string `json:"auth_dir,omitempty"` // optional: SQLite auth dir override (desktop)
AuthDir string `json:"auth_dir,omitempty"` // optional: SQLite auth dir override (desktop)
AllowFrom FlexibleStringSlice `json:"allow_from"`
DMPolicy string `json:"dm_policy,omitempty"` // "pairing" (default for DB instances), "open", "allowlist", "disabled"
GroupPolicy string `json:"group_policy,omitempty"` // "pairing" (default for DB instances), "open" (default for config), "allowlist", "disabled"
@@ -196,25 +196,25 @@ type FeishuConfig struct {
// ProvidersConfig maps provider name to its config.
type ProvidersConfig struct {
Anthropic ProviderConfig `json:"anthropic"`
OpenAI ProviderConfig `json:"openai"`
OpenRouter ProviderConfig `json:"openrouter"`
Groq ProviderConfig `json:"groq"`
Gemini ProviderConfig `json:"gemini"`
DeepSeek ProviderConfig `json:"deepseek"`
Mistral ProviderConfig `json:"mistral"`
XAI ProviderConfig `json:"xai"`
MiniMax ProviderConfig `json:"minimax"`
Cohere ProviderConfig `json:"cohere"`
Perplexity ProviderConfig `json:"perplexity"`
DashScope ProviderConfig `json:"dashscope"`
Bailian ProviderConfig `json:"bailian"`
Zai ProviderConfig `json:"zai"`
ZaiCoding ProviderConfig `json:"zai_coding"`
Ollama OllamaConfig `json:"ollama"` // local Ollama instance (no API key needed)
OllamaCloud ProviderConfig `json:"ollama_cloud"` // Ollama Cloud (API key required)
ClaudeCLI ClaudeCLIConfig `json:"claude_cli"`
ACP ACPConfig `json:"acp"`
Anthropic ProviderConfig `json:"anthropic"`
OpenAI ProviderConfig `json:"openai"`
OpenRouter ProviderConfig `json:"openrouter"`
Groq ProviderConfig `json:"groq"`
Gemini ProviderConfig `json:"gemini"`
DeepSeek ProviderConfig `json:"deepseek"`
Mistral ProviderConfig `json:"mistral"`
XAI ProviderConfig `json:"xai"`
MiniMax ProviderConfig `json:"minimax"`
Cohere ProviderConfig `json:"cohere"`
Perplexity ProviderConfig `json:"perplexity"`
DashScope ProviderConfig `json:"dashscope"`
Bailian ProviderConfig `json:"bailian"`
Zai ProviderConfig `json:"zai"`
ZaiCoding ProviderConfig `json:"zai_coding"`
Ollama OllamaConfig `json:"ollama"` // local Ollama instance (no API key needed)
OllamaCloud ProviderConfig `json:"ollama_cloud"` // Ollama Cloud (API key required)
ClaudeCLI ClaudeCLIConfig `json:"claude_cli"`
ACP ACPConfig `json:"acp"`
Novita ProviderConfig `json:"novita"` // Novita AI (OpenAI-compatible endpoint)
BytePlus ProviderConfig `json:"byteplus"` // BytePlus ModelArk (Seed 2.0)
BytePlusCoding ProviderConfig `json:"byteplus_coding"` // BytePlus ModelArk Coding Plan
@@ -362,16 +362,16 @@ type QuotaConfig struct {
// GatewayConfig controls the gateway server.
type GatewayConfig struct {
Host string `json:"host"`
Port int `json:"port"`
Token string `json:"token,omitempty"` // bearer token for WS/HTTP auth
OwnerIDs []string `json:"owner_ids,omitempty"` // sender IDs considered "owner"
AllowedOrigins []string `json:"allowed_origins,omitempty"` // WebSocket CORS whitelist (empty = allow all)
MaxMessageChars int `json:"max_message_chars,omitempty"` // max user message characters (default 32000)
RateLimitRPM int `json:"rate_limit_rpm,omitempty"` // rate limit: requests per minute per user (default 20, 0 = disabled)
InjectionAction string `json:"injection_action,omitempty"` // prompt injection action: "log", "warn" (default), "block", "off"
InboundDebounceMs int `json:"inbound_debounce_ms,omitempty"` // merge rapid channel/Web Chat messages from same sender/session (0 = no wait)
Quota *QuotaConfig `json:"quota,omitempty"` // per-user/group request quotas
Host string `json:"host"`
Port int `json:"port"`
Token string `json:"token,omitempty"` // bearer token for WS/HTTP auth
OwnerIDs []string `json:"owner_ids,omitempty"` // sender IDs considered "owner"
AllowedOrigins []string `json:"allowed_origins,omitempty"` // WebSocket CORS whitelist (empty = allow all)
MaxMessageChars int `json:"max_message_chars,omitempty"` // max user message characters (default 32000)
RateLimitRPM int `json:"rate_limit_rpm,omitempty"` // rate limit: requests per minute per user (default 20, 0 = disabled)
InjectionAction string `json:"injection_action,omitempty"` // prompt injection action: "log", "warn" (default), "block", "off"
InboundDebounceMs int `json:"inbound_debounce_ms,omitempty"` // merge rapid channel/Web Chat messages from same sender/session (0 = no wait)
Quota *QuotaConfig `json:"quota,omitempty"` // per-user/group request quotas
BlockReply *bool `json:"block_reply,omitempty"` // deliver intermediate text during tool iterations (default false)
ToolStatus *bool `json:"tool_status,omitempty"` // show tool name in streaming preview during tool execution (default true)
TaskRecoveryIntervalSec int `json:"task_recovery_interval_sec,omitempty"` // team task recovery ticker interval in seconds (default 300 = 5min)
@@ -381,18 +381,32 @@ type GatewayConfig struct {
// ToolsConfig controls tool availability, policy, and web search.
type ToolsConfig struct {
Profile string `json:"profile,omitempty"` // global profile: "minimal", "coding", "messaging", "full"
Allow []string `json:"allow,omitempty"` // global allow list (tool names or "group:xxx")
Deny []string `json:"deny,omitempty"` // global deny list
AlsoAllow []string `json:"alsoAllow,omitempty"` // additive: adds without removing existing
ByProvider map[string]*ToolPolicySpec `json:"byProvider,omitempty"` // per-provider overrides
ShellDenyGroups map[string]bool `json:"shellDenyGroups,omitempty"` // global shell deny-group toggles (group name -> denied); per-agent overrides win per-key
ExecApproval ExecApprovalCfg `json:"execApproval"` // exec command approval settings
WebFetch WebFetchPolicyConfig `json:"web_fetch"` // domain policy for URL fetching
Browser BrowserToolConfig `json:"browser"`
RateLimitPerHour int `json:"rate_limit_per_hour,omitempty"` // max tool executions per hour per session (0 = disabled)
ScrubCredentials *bool `json:"scrub_credentials,omitempty"` // auto-redact API keys/tokens in tool output (default true)
McpServers map[string]*MCPServerConfig `json:"mcp_servers,omitempty"` // external MCP server connections
Profile string `json:"profile,omitempty"` // global profile: "minimal", "coding", "messaging", "full"
Allow []string `json:"allow,omitempty"` // global allow list (tool names or "group:xxx")
Deny []string `json:"deny,omitempty"` // global deny list
AlsoAllow []string `json:"alsoAllow,omitempty"` // additive: adds without removing existing
ByProvider map[string]*ToolPolicySpec `json:"byProvider,omitempty"` // per-provider overrides
ShellDenyGroups map[string]bool `json:"shellDenyGroups,omitempty"` // global shell deny-group toggles (group name -> denied); per-agent overrides win per-key
CommandKeywordAllowlist []CommandKeywordAllowlistRule `json:"commandKeywordAllowlist,omitempty"` // scoped bypass for credentialed CLI content keywords
ExecApproval ExecApprovalCfg `json:"execApproval"` // exec command approval settings
WebFetch WebFetchPolicyConfig `json:"web_fetch"` // domain policy for URL fetching
Browser BrowserToolConfig `json:"browser"`
RateLimitPerHour int `json:"rate_limit_per_hour,omitempty"` // max tool executions per hour per session (0 = disabled)
ScrubCredentials *bool `json:"scrub_credentials,omitempty"` // auto-redact API keys/tokens in tool output (default true)
McpServers map[string]*MCPServerConfig `json:"mcp_servers,omitempty"` // external MCP server connections
}
// CommandKeywordAllowlistRule scopes product/security vocabulary that may appear
// in credentialed CLI content args without disabling command-path deny patterns.
type CommandKeywordAllowlistRule struct {
ID string `json:"id,omitempty"`
Command string `json:"command"`
Subcommands []string `json:"subcommands,omitempty"`
Args []string `json:"args,omitempty"`
ArgPositions []int `json:"argPositions,omitempty"` // 0-based after matched subcommand
Keywords []string `json:"keywords,omitempty"`
Reason string `json:"reason,omitempty"`
Enabled *bool `json:"enabled,omitempty"` // nil means enabled
}
// MCPServerConfig configures a single external MCP server connection.
@@ -429,24 +443,24 @@ type WebFetchPolicyConfig struct {
// BrowserToolConfig controls the browser automation tool.
type BrowserToolConfig struct {
Enabled bool `json:"enabled"` // enable the browser tool (default false)
Headless bool `json:"headless,omitempty"` // run Chrome in headless mode (ignored when RemoteURL is set)
RemoteURL string `json:"remote_url,omitempty"` // CDP endpoint for remote Chrome sidecar, e.g. "ws://chrome:9222"
ActionTimeoutMs int `json:"action_timeout_ms,omitempty"` // per-action timeout in ms (default 30000)
IdleTimeoutMs int `json:"idle_timeout_ms,omitempty"` // idle page auto-close in ms (default 600000, 0=disabled)
MaxPages int `json:"max_pages,omitempty"` // max open pages per tenant (default 5)
CookieSyncEnabled bool `json:"cookie_sync_enabled"` // apply selected synced cookies to scoped browser sessions
Enabled bool `json:"enabled"` // enable the browser tool (default false)
Headless bool `json:"headless,omitempty"` // run Chrome in headless mode (ignored when RemoteURL is set)
RemoteURL string `json:"remote_url,omitempty"` // CDP endpoint for remote Chrome sidecar, e.g. "ws://chrome:9222"
ActionTimeoutMs int `json:"action_timeout_ms,omitempty"` // per-action timeout in ms (default 30000)
IdleTimeoutMs int `json:"idle_timeout_ms,omitempty"` // idle page auto-close in ms (default 600000, 0=disabled)
MaxPages int `json:"max_pages,omitempty"` // max open pages per tenant (default 5)
CookieSyncEnabled bool `json:"cookie_sync_enabled"` // apply selected synced cookies to scoped browser sessions
}
// ToolPolicySpec defines a tool policy at any level (global, per-agent, per-provider).
type ToolPolicySpec struct {
Profile string `json:"profile,omitempty"`
Allow []string `json:"allow,omitempty"`
Deny []string `json:"deny,omitempty"`
AlsoAllow []string `json:"alsoAllow,omitempty"`
ByProvider map[string]*ToolPolicySpec `json:"byProvider,omitempty"`
Wait *WaitToolPolicy `json:"wait,omitempty"`
ToolCallPrefix string `json:"toolCallPrefix,omitempty"` // prefix to strip from model's tool call names before registry lookup
Profile string `json:"profile,omitempty"`
Allow []string `json:"allow,omitempty"`
Deny []string `json:"deny,omitempty"`
AlsoAllow []string `json:"alsoAllow,omitempty"`
ByProvider map[string]*ToolPolicySpec `json:"byProvider,omitempty"`
Wait *WaitToolPolicy `json:"wait,omitempty"`
ToolCallPrefix string `json:"toolCallPrefix,omitempty"` // prefix to strip from model's tool call names before registry lookup
}
// WaitToolPolicy configures per-agent safety bounds for the wait tool.
internal/config/config_load.go
+32
View File
@@ -63,6 +63,15 @@ func isLoopbackGatewayHost(host string) bool {
return err == nil && addr.IsLoopback()
}
func parseEnvBool(value string) bool {
switch strings.ToLower(strings.TrimSpace(value)) {
case "1", "true", "yes", "on":
return true
default:
return false
}
}
// Default returns a Config with sensible defaults.
func Default() *Config {
return &Config{
@@ -107,6 +116,9 @@ func Default() *Config {
},
RateLimitPerHour: 150,
},
Skills: SkillsConfig{
MaxUploadSizeMB: DefaultSkillMaxUploadSizeMB,
},
Sessions: SessionsConfig{},
}
}
@@ -234,6 +246,26 @@ func (c *Config) applyEnvOverrides() {
c.Gateway.Port = port
}
}
if v := os.Getenv("GOCLAW_SKILLS_MAX_UPLOAD_SIZE_MB"); v != "" {
if mb, err := strconv.Atoi(v); err == nil {
c.Skills.MaxUploadSizeMB = ClampSkillMaxUploadSizeMB(mb)
}
}
envBoolPtr := func(key string, dst **bool) {
if v := os.Getenv(key); v != "" {
b := parseEnvBool(v)
*dst = &b
}
}
envBool := func(key string, dst *bool) {
if v := os.Getenv(key); v != "" {
*dst = parseEnvBool(v)
}
}
envBoolPtr("GOCLAW_SKILLS_SLASH_COMMANDS_ENABLED", &c.Skills.SlashCommands.Enabled)
envBoolPtr("GOCLAW_SKILLS_SLASH_COMMANDS_SUGGEST_NOT_FOUND", &c.Skills.SlashCommands.SuggestNotFound)
envBool("GOCLAW_SKILLS_SLASH_COMMANDS_PARTIAL_MATCHING", &c.Skills.SlashCommands.PartialMatching)
envStr("GOCLAW_SKILLS_SLASH_COMMANDS_PREFIX", &c.Skills.SlashCommands.Prefix)
// Database
envStr("GOCLAW_POSTGRES_DSN", &c.Database.PostgresDSN)
internal/config/config_load_test.go
+133
View File
@@ -24,6 +24,21 @@ func TestDefault_SensibleDefaults(t *testing.T) {
if cfg.Agents.Defaults.MaxToolIterations != DefaultMaxIterations {
t.Fatalf("default max iterations: got %d", cfg.Agents.Defaults.MaxToolIterations)
}
if cfg.Skills.EffectiveMaxUploadSizeMB() != DefaultSkillMaxUploadSizeMB {
t.Fatalf("default skill upload max: got %d, want %d", cfg.Skills.EffectiveMaxUploadSizeMB(), DefaultSkillMaxUploadSizeMB)
}
if !cfg.Skills.SlashCommands.EffectiveEnabled() {
t.Fatal("slash commands should default enabled")
}
if !cfg.Skills.SlashCommands.EffectiveSuggestNotFound() {
t.Fatal("slash command suggestions should default enabled")
}
if cfg.Skills.SlashCommands.EffectivePartialMatching() {
t.Fatal("slash command partial matching should default disabled")
}
if cfg.Skills.SlashCommands.EffectivePrefix() != "/" {
t.Fatalf("slash command prefix = %q, want /", cfg.Skills.SlashCommands.EffectivePrefix())
}
}
@@ -103,6 +118,124 @@ func TestLoad_EnvVarOverrides(t *testing.T) {
}
}
func TestLoad_SkillsMaxUploadSizeFromFileAndEnv(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json5")
os.WriteFile(cfgPath, []byte(`{"skills":{"max_upload_size_mb":64}}`), 0644)
cfg, err := Load(cfgPath)
if err != nil {
t.Fatalf("load error: %v", err)
}
if cfg.Skills.EffectiveMaxUploadSizeMB() != 64 {
t.Fatalf("file skill upload max: got %d, want 64", cfg.Skills.EffectiveMaxUploadSizeMB())
}
t.Setenv("GOCLAW_SKILLS_MAX_UPLOAD_SIZE_MB", "128")
cfg, err = Load(cfgPath)
if err != nil {
t.Fatalf("load with env error: %v", err)
}
if cfg.Skills.EffectiveMaxUploadSizeMB() != 128 {
t.Fatalf("env skill upload max: got %d, want 128", cfg.Skills.EffectiveMaxUploadSizeMB())
}
}
func TestLoad_SkillSlashCommandsFromFileEnvAndSystemConfig(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json5")
os.WriteFile(cfgPath, []byte(`{
"skills": {
"slash_commands": {
"enabled": false,
"suggest_not_found": false,
"partial_matching": true,
"prefix": "!"
}
}
}`), 0644)
cfg, err := Load(cfgPath)
if err != nil {
t.Fatalf("load error: %v", err)
}
if cfg.Skills.SlashCommands.EffectiveEnabled() {
t.Fatal("file enabled override should be false")
}
if cfg.Skills.SlashCommands.EffectiveSuggestNotFound() {
t.Fatal("file suggestion override should be false")
}
if !cfg.Skills.SlashCommands.EffectivePartialMatching() {
t.Fatal("file partial matching override should be true")
}
if cfg.Skills.SlashCommands.EffectivePrefix() != "!" {
t.Fatalf("file prefix = %q, want !", cfg.Skills.SlashCommands.EffectivePrefix())
}
t.Setenv("GOCLAW_SKILLS_SLASH_COMMANDS_ENABLED", "true")
t.Setenv("GOCLAW_SKILLS_SLASH_COMMANDS_SUGGEST_NOT_FOUND", "true")
t.Setenv("GOCLAW_SKILLS_SLASH_COMMANDS_PARTIAL_MATCHING", "false")
t.Setenv("GOCLAW_SKILLS_SLASH_COMMANDS_PREFIX", "#")
cfg, err = Load(cfgPath)
if err != nil {
t.Fatalf("load with env error: %v", err)
}
if !cfg.Skills.SlashCommands.EffectiveEnabled() {
t.Fatal("env enabled override should be true")
}
if !cfg.Skills.SlashCommands.EffectiveSuggestNotFound() {
t.Fatal("env suggestion override should be true")
}
if cfg.Skills.SlashCommands.EffectivePartialMatching() {
t.Fatal("env partial matching override should be false")
}
if cfg.Skills.SlashCommands.EffectivePrefix() != "#" {
t.Fatalf("env prefix = %q, want #", cfg.Skills.SlashCommands.EffectivePrefix())
}
cfg.ApplySystemConfigs(map[string]string{
"skills.slash_commands.enabled": "false",
"skills.slash_commands.suggest_not_found": "false",
"skills.slash_commands.partial_matching": "true",
"skills.slash_commands.prefix": "%",
})
if cfg.Skills.SlashCommands.EffectiveEnabled() {
t.Fatal("system enabled override should be false")
}
if cfg.Skills.SlashCommands.EffectiveSuggestNotFound() {
t.Fatal("system suggestion override should be false")
}
if !cfg.Skills.SlashCommands.EffectivePartialMatching() {
t.Fatal("system partial matching override should be true")
}
if cfg.Skills.SlashCommands.EffectivePrefix() != "%" {
t.Fatalf("system prefix = %q, want %%", cfg.Skills.SlashCommands.EffectivePrefix())
}
}
func TestSkillsMaxUploadSizeClampAndSystemConfigOverlay(t *testing.T) {
cfg := Default()
cfg.Skills.MaxUploadSizeMB = 0
if got := cfg.Skills.EffectiveMaxUploadSizeMB(); got != DefaultSkillMaxUploadSizeMB {
t.Fatalf("zero upload max: got %d, want %d", got, DefaultSkillMaxUploadSizeMB)
}
cfg.Skills.MaxUploadSizeMB = -10
if got := cfg.Skills.EffectiveMaxUploadSizeMB(); got != MinSkillMaxUploadSizeMB {
t.Fatalf("negative upload max: got %d, want %d", got, MinSkillMaxUploadSizeMB)
}
cfg.Skills.MaxUploadSizeMB = 999
if got := cfg.Skills.EffectiveMaxUploadSizeMB(); got != MaxSkillMaxUploadSizeMB {
t.Fatalf("high upload max: got %d, want %d", got, MaxSkillMaxUploadSizeMB)
}
cfg.ApplySystemConfigs(map[string]string{"skills.max_upload_size_mb": "77"})
if got := cfg.Skills.EffectiveMaxUploadSizeMB(); got != 77 {
t.Fatalf("system config upload max: got %d, want 77", got)
}
}
func TestLoad_EnvVarOverrides_InvalidPort(t *testing.T) {
t.Setenv("GOCLAW_PORT", "not-a-number")
internal/config/config_system.go
+8
View File
@@ -74,6 +74,14 @@ func (c *Config) ApplySystemConfigs(configs map[string]string) {
integer("tools.browser.max_pages", &c.Tools.Browser.MaxPages)
boolValue("tools.browser.cookie_sync_enabled", &c.Tools.Browser.CookieSyncEnabled)
// Skills
integer(SkillMaxUploadSizeSystemConfigKey, &c.Skills.MaxUploadSizeMB)
c.Skills.MaxUploadSizeMB = ClampSkillMaxUploadSizeMB(c.Skills.MaxUploadSizeMB)
boolean(SkillSlashCommandsEnabledSystemConfigKey, &c.Skills.SlashCommands.Enabled)
boolean(SkillSlashSuggestNotFoundSystemConfigKey, &c.Skills.SlashCommands.SuggestNotFound)
boolValue(SkillSlashPartialMatchingSystemConfigKey, &c.Skills.SlashCommands.PartialMatching)
str(SkillSlashCommandPrefixSystemConfigKey, &c.Skills.SlashCommands.Prefix)
// TTS
str("tts.provider", &c.Tts.Provider)
str("tts.auto", &c.Tts.Auto)
internal/consolidation/dreaming_worker.go
+14 -2
View File
@@ -11,9 +11,10 @@ import (
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/bgalert"
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/providerresolve"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
const (
@@ -31,6 +32,7 @@ type dreamingWorker struct {
systemConfigs store.SystemConfigStore // per-tenant provider config
registry *providers.Registry // provider resolution
alertDeps bgalert.AlertDeps
usageCaps *usagecaps.Service
// threshold/debounce are the global defaults. Per-agent overrides come
// from resolveConfig which reads the agent's MemoryConfig.Dreaming JSONB.
@@ -95,6 +97,11 @@ func (w *dreamingWorker) Handle(ctx context.Context, event eventbus.DomainEvent)
ctx = store.WithTenantID(ctx, tid)
}
}
if event.AgentID != "" {
if aid, err := uuid.Parse(event.AgentID); err == nil {
ctx = store.WithAgentID(ctx, aid)
}
}
agentID := event.AgentID
userID := event.UserID
@@ -205,7 +212,7 @@ func (w *dreamingWorker) synthesize(ctx context.Context, provider providers.Prov
}
body := strings.Join(summaries, "\n---\n")
resp, err := provider.Chat(ctx, providers.ChatRequest{
req := providers.ChatRequest{
Messages: []providers.Message{
{Role: "system", Content: dreamingSystemPrompt},
{Role: "user", Content: "Session summaries:\n---\n" + body + "\n---"},
@@ -214,6 +221,11 @@ func (w *dreamingWorker) synthesize(ctx context.Context, provider providers.Prov
Options: map[string]any{
providers.OptMaxTokens: dreamingMaxTokens,
},
}
resp, err := w.usageCaps.Chat(ctx, provider, req, usagecaps.ChatOptions{
ModelID: model,
Purpose: "dreaming-synthesis",
MaxOutputTokens: dreamingMaxTokens,
})
if err != nil {
return "", fmt.Errorf("dreaming chat: %w", err)
internal/consolidation/episodic_worker.go
+13 -5
View File
@@ -10,19 +10,21 @@ import (
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/bgalert"
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/providerresolve"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// episodicWorker handles session.completed events → creates episodic summaries.
type episodicWorker struct {
store store.EpisodicStore
sessions store.SessionCoreStore // for reading session messages during summarization
systemConfigs store.SystemConfigStore // per-tenant provider config
registry *providers.Registry // provider resolution
sessions store.SessionCoreStore // for reading session messages during summarization
systemConfigs store.SystemConfigStore // per-tenant provider config
registry *providers.Registry // provider resolution
eventBus eventbus.DomainEventBus
alertDeps bgalert.AlertDeps
usageCaps *usagecaps.Service
}
// resolveProvider delegates to shared background provider resolution.
@@ -53,6 +55,7 @@ func (w *episodicWorker) Handle(ctx context.Context, event eventbus.DomainEvent)
if err != nil {
return fmt.Errorf("episodic: invalid agent_id %q: %w", event.AgentID, err)
}
ctx = store.WithAgentID(ctx, agentUUID)
// Build source_id for idempotency
sourceID := fmt.Sprintf("%s:%d", payload.SessionKey, payload.CompactionCount)
@@ -168,13 +171,18 @@ func (w *episodicWorker) summarizeFromMessages(ctx context.Context, provider pro
sctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
resp, err := provider.Chat(sctx, providers.ChatRequest{
req := providers.ChatRequest{
Messages: []providers.Message{
{Role: "system", Content: summarizationPrompt},
{Role: "user", Content: sb.String()},
},
Model: model,
Options: map[string]any{"max_tokens": 1024, "temperature": 0.3},
}
resp, err := w.usageCaps.Chat(sctx, provider, req, usagecaps.ChatOptions{
ModelID: model,
Purpose: "episodic-summary",
MaxOutputTokens: 1024,
})
if err != nil {
return "", err
internal/consolidation/workers.go
+4
View File
@@ -13,6 +13,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// ConsolidationDeps bundles all dependencies for the consolidation pipeline.
@@ -26,6 +27,7 @@ type ConsolidationDeps struct {
Registry *providers.Registry // provider resolution
Extractor EntityExtractor
AlertDeps bgalert.AlertDeps // for reporting non-retryable LLM errors
UsageCaps *usagecaps.Service
// AgentStore is optional: when present, the dreaming worker reads
// per-agent overrides from MemoryConfig.Dreaming. If nil, the worker
// uses its built-in defaults for every agent.
@@ -42,6 +44,7 @@ func Register(deps ConsolidationDeps) func() {
registry: deps.Registry,
eventBus: deps.EventBus,
alertDeps: deps.AlertDeps,
usageCaps: deps.UsageCaps,
}
semantic := &semanticWorker{
kgStore: deps.KGStore,
@@ -59,6 +62,7 @@ func Register(deps ConsolidationDeps) func() {
systemConfigs: deps.SystemConfigs,
registry: deps.Registry,
alertDeps: deps.AlertDeps,
usageCaps: deps.UsageCaps,
threshold: dreamingDefaultThreshold,
debounce: dreamingDefaultDebounce,
resolveConfig: newAgentStoreResolver(deps.AgentStore),
internal/gateway/methods/agents_update.go
+7 -1
View File
@@ -60,6 +60,10 @@ func (m *AgentsMethods) handleUpdate(ctx context.Context, client *gateway.Client
if req.Params != nil {
json.Unmarshal(req.Params, &params)
}
var rawParams map[string]json.RawMessage
if req.Params != nil {
_ = json.Unmarshal(req.Params, &rawParams)
}
if params.AgentID == "" {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "agentId")))
@@ -104,7 +108,9 @@ func (m *AgentsMethods) handleUpdate(ctx context.Context, client *gateway.Client
if params.IsDefault != nil {
updates["is_default"] = *params.IsDefault
}
if params.BudgetCents != nil {
if rawBudget, ok := rawParams["budget_monthly_cents"]; ok && strings.TrimSpace(string(rawBudget)) == "null" {
updates["budget_monthly_cents"] = nil
} else if params.BudgetCents != nil {
updates["budget_monthly_cents"] = *params.BudgetCents
}
// Per-agent JSONB config overrides
internal/gateway/methods/chat.go
+10 -1
View File
@@ -20,6 +20,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/sessions"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
@@ -32,6 +33,7 @@ type ChatMethods struct {
eventBus bus.EventPublisher
postTurn tools.PostTurnProcessor
audioMgr *audio.Manager // for TTS auto-apply on WS responses (nil = disabled)
usageCaps *usagecaps.Service
debouncer *chatDebouncer
}
@@ -46,6 +48,10 @@ func (m *ChatMethods) SetAudioManager(mgr *audio.Manager) {
m.audioMgr = mgr
}
func (m *ChatMethods) SetUsageCapService(s *usagecaps.Service) {
m.usageCaps = s
}
// SetPostTurnProcessor sets the post-turn processor for team task dispatch.
func (m *ChatMethods) SetPostTurnProcessor(pt tools.PostTurnProcessor) {
m.postTurn = pt
@@ -352,7 +358,10 @@ func (m *ChatMethods) dispatchChatSends(requests []chatSendRequest) {
// Use runCtxBase (WithoutCancel + tenant-aware) so title save uses correct tenant.
titleCtx := runCtxBase
go func() {
title := agent.GenerateTitle(titleCtx, agentProvider, agentModel, userMsg)
if uid := loop.UUID(); uid != uuid.Nil {
titleCtx = store.WithAgentID(titleCtx, uid)
}
title := agent.GenerateTitleWithUsageCaps(titleCtx, m.usageCaps, agentProvider, agentModel, userMsg)
if title == "" {
return
}
internal/gateway/methods/config.go
+40 -1
View File
@@ -22,7 +22,7 @@ type ConfigMethods struct {
cfgPath string
secretsStore store.ConfigSecretsStore
syncFn func(ctx context.Context, cfg *config.Config) // nil-safe; syncs non-secret settings to system_configs
eventBus bus.EventPublisher // nil-safe; broadcasts config change events
eventBus bus.EventPublisher // nil-safe; broadcasts config change events
}
func NewConfigMethods(cfg *config.Config, cfgPath string, secretsStore store.ConfigSecretsStore, eventBus bus.EventPublisher) *ConfigMethods {
@@ -269,6 +269,45 @@ func (m *ConfigMethods) handleSchema(_ context.Context, client *gateway.Client,
"type": "object",
"description": "Tool configuration (browser, exec, web search)",
},
"skills": map[string]any{
"type": "object",
"description": "Skill storage and upload settings",
"properties": map[string]any{
"max_upload_size_mb": map[string]any{
"type": "integer",
"minimum": config.MinSkillMaxUploadSizeMB,
"maximum": config.MaxSkillMaxUploadSizeMB,
"default": config.DefaultSkillMaxUploadSizeMB,
"description": "Maximum skill ZIP upload size in MB",
},
"slash_commands": map[string]any{
"type": "object",
"description": "Explicit slash command skill activation settings",
"properties": map[string]any{
"enabled": map[string]any{
"type": "boolean",
"default": true,
"description": "Enable slash command detection in user prompts",
},
"suggest_not_found": map[string]any{
"type": "boolean",
"default": true,
"description": "Suggest similar skills when a requested skill is not found",
},
"partial_matching": map[string]any{
"type": "boolean",
"default": false,
"description": "Allow unique skill slug/name prefixes",
},
"prefix": map[string]any{
"type": "string",
"default": config.DefaultSkillSlashCommandPrefix,
"description": "Single-character slash command prefix",
},
},
},
},
},
"sessions": map[string]any{
"type": "object",
"description": "Session storage configuration",
internal/gateway/server.go
+5
View File
@@ -639,6 +639,11 @@ func (s *Server) SetSystemConfigsHandler(h *httpapi.SystemConfigsHandler) {
// SetUsageHandler sets the usage analytics handler.
func (s *Server) SetUsageHandler(h *httpapi.UsageHandler) { s.handlers = append(s.handlers, h) }
// SetUsageCapsHandler sets usage cap and model pricing handlers.
func (s *Server) SetUsageCapsHandler(h *httpapi.UsageCapsHandler) {
s.handlers = append(s.handlers, h)
}
// SetBackupHandler sets the system backup handler.
func (s *Server) SetBackupHandler(h *httpapi.BackupHandler) { s.handlers = append(s.handlers, h) }
internal/hooks/handlers/prompt.go
+33 -1
View File
@@ -17,6 +17,8 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/hooks"
"github.com/nextlevelbuilder/goclaw/internal/hooks/budget"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// ── Public surface ──────────────────────────────────────────────────────────
@@ -46,6 +48,9 @@ type PromptHandler struct {
// budget checks are skipped (Lite edition behavior).
Budget *budget.Store
// UsageCaps enforces provider/model/tenant cost caps before the hook LLM call.
UsageCaps *usagecaps.Service
// DefaultModel is used when a hook config does not specify one.
// Recommended: "haiku" for cheap evaluation.
DefaultModel string
@@ -155,7 +160,17 @@ func (h *PromptHandler) Execute(ctx context.Context, cfg hooks.HookConfig, ev ho
req := h.buildChatRequest(cfg, ev, resolvedModel)
// 6. Call provider.
resp, err := provider.Chat(ctx, req)
callCtx := ctx
if ev.TenantID != uuid.Nil {
callCtx = store.WithTenantID(callCtx, ev.TenantID)
}
resp, err := h.UsageCaps.Chat(callCtx, provider, req, usagecaps.ChatOptions{
TenantID: ev.TenantID,
ProviderName: provider.Name(),
ModelID: resolvedModel,
Purpose: "hook-prompt",
MaxOutputTokens: promptMaxTokens(req),
})
if err != nil {
// Fail-closed on transport/provider error for blocking events.
if ev.HookEvent.IsBlocking() {
@@ -300,6 +315,23 @@ func (h *PromptHandler) buildChatRequest(cfg hooks.HookConfig, ev hooks.Event, m
}
}
func promptMaxTokens(req providers.ChatRequest) int {
if req.Options == nil {
return 512
}
if v, ok := req.Options[providers.OptMaxTokens]; ok {
switch n := v.(type) {
case int:
return n
case int64:
return int(n)
case float64:
return int(n)
}
}
return 512
}
// sanitizeToolInput canonicalizes a tool_input map into stable JSON with
// sorted keys, stripping only structural noise. Actual injection-attack
// detection is delegated to the evaluator LLM (which has the anti-injection
internal/http/knowledge_graph.go
+9 -1
View File
@@ -7,12 +7,14 @@ import (
kg "github.com/nextlevelbuilder/goclaw/internal/knowledgegraph"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// KnowledgeGraphHandler handles KG entity/relation management endpoints.
type KnowledgeGraphHandler struct {
store store.KnowledgeGraphStore
providerReg *providers.Registry
usageCaps *usagecaps.Service
}
// NewKnowledgeGraphHandler creates a handler for KG management endpoints.
@@ -20,6 +22,10 @@ func NewKnowledgeGraphHandler(s store.KnowledgeGraphStore, providerReg *provider
return &KnowledgeGraphHandler{store: s, providerReg: providerReg}
}
func (h *KnowledgeGraphHandler) SetUsageCapService(s *usagecaps.Service) {
h.usageCaps = s
}
// NewExtractor creates an Extractor from the given provider name and model.
func (h *KnowledgeGraphHandler) NewExtractor(ctx context.Context, providerName, model string, minConfidence float64) *kg.Extractor {
if h.providerReg == nil || providerName == "" || model == "" {
@@ -29,7 +35,9 @@ func (h *KnowledgeGraphHandler) NewExtractor(ctx context.Context, providerName,
if err != nil {
return nil
}
return kg.NewExtractor(p, model, minConfidence)
extractor := kg.NewExtractor(p, model, minConfidence)
extractor.SetUsageCapService(h.usageCaps)
return extractor
}
// RegisterRoutes registers all KG routes on the given mux.
internal/http/knowledge_graph_handlers.go
+11 -6
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"strconv"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/store"
)
@@ -156,6 +157,10 @@ func (h *KnowledgeGraphHandler) handleTraverse(w http.ResponseWriter, r *http.Re
func (h *KnowledgeGraphHandler) handleExtract(w http.ResponseWriter, r *http.Request) {
locale := extractLocale(r)
agentID := r.PathValue("agentID")
callCtx := r.Context()
if parsedAgentID, err := uuid.Parse(agentID); err == nil {
callCtx = store.WithAgentID(callCtx, parsedAgentID)
}
var body struct {
Text string `json:"text"`
@@ -176,13 +181,13 @@ func (h *KnowledgeGraphHandler) handleExtract(w http.ResponseWriter, r *http.Req
return
}
extractor := h.NewExtractor(r.Context(), body.Provider, body.Model, body.MinConf)
extractor := h.NewExtractor(callCtx, body.Provider, body.Model, body.MinConf)
if extractor == nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidProviderOrModel)})
return
}
result, err := extractor.Extract(r.Context(), body.Text)
result, err := extractor.Extract(callCtx, body.Text)
if err != nil {
slog.Warn("kg.extract failed", "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
@@ -213,10 +218,10 @@ func (h *KnowledgeGraphHandler) handleExtract(w http.ResponseWriter, r *http.Req
}
writeJSON(w, http.StatusOK, map[string]any{
"entities": len(result.Entities),
"relations": len(result.Relations),
"dedup_merged": dedupMerged,
"dedup_flagged": dedupFlagged,
"entities": len(result.Entities),
"relations": len(result.Relations),
"dedup_merged": dedupMerged,
"dedup_flagged": dedupFlagged,
})
}
internal/http/pending_messages.go
+7 -1
View File
@@ -11,6 +11,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/providerresolve"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// PendingMessagesHandler handles pending message HTTP endpoints.
@@ -22,6 +23,7 @@ type PendingMessagesHandler struct {
maxTokens int // max output tokens for LLM summarization (0 = use default)
cfgProvider string // config-level provider override (empty = resolve from agent)
cfgModel string // config-level model override (empty = resolve from agent)
usageCaps *usagecaps.Service
}
func NewPendingMessagesHandler(s store.PendingMessageStore, agentStore store.AgentStore, providerReg *providers.Registry) *PendingMessagesHandler {
@@ -40,6 +42,10 @@ func (h *PendingMessagesHandler) SetProviderModel(provider, model string) {
h.cfgModel = model
}
func (h *PendingMessagesHandler) SetUsageCapService(s *usagecaps.Service) {
h.usageCaps = s
}
func (h *PendingMessagesHandler) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("GET /v1/pending-messages", h.authMiddleware(h.handleListGroups))
mux.HandleFunc("GET /v1/pending-messages/messages", h.authMiddleware(h.handleListMessages))
@@ -148,7 +154,7 @@ func (h *PendingMessagesHandler) handleCompact(w http.ResponseWriter, r *http.Re
go func() {
ctx, cancel := context.WithTimeout(store.WithTenantID(context.Background(), tenantID), 180*time.Second)
defer cancel()
remaining, err := channels.CompactGroup(ctx, h.store, req.ChannelName, req.HistoryKey, provider, model, keepRecent, h.maxTokens)
remaining, err := channels.CompactGroup(ctx, h.store, req.ChannelName, req.HistoryKey, provider, model, keepRecent, h.maxTokens, h.usageCaps)
if err != nil {
slog.Warn("compact.failed", "channel", req.ChannelName, "key", req.HistoryKey, "error", err)
} else {
internal/http/provider_verify.go
+10 -1
View File
@@ -16,6 +16,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// HandleVerifyProviderForTest invokes the verify handler directly without auth
@@ -126,8 +127,9 @@ func (h *ProvidersHandler) handleVerifyProvider(w http.ResponseWriter, r *http.R
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
ctx = store.WithTenantID(ctx, p.TenantID)
_, err = provider.Chat(ctx, providers.ChatRequest{
reqChat := providers.ChatRequest{
Messages: []providers.Message{
{Role: "user", Content: "hi"},
},
@@ -136,6 +138,13 @@ func (h *ProvidersHandler) handleVerifyProvider(w http.ResponseWriter, r *http.R
// Use a small but safe value — reasoning models need headroom beyond 1 token.
"max_tokens": 50,
},
}
_, err = h.usageCaps.Chat(ctx, provider, reqChat, usagecaps.ChatOptions{
TenantID: p.TenantID,
ProviderName: p.Name,
ModelID: req.Model,
Purpose: "provider-verify",
MaxOutputTokens: 50,
})
if err != nil {
writeJSON(w, http.StatusOK, map[string]any{"valid": false, "error": friendlyVerifyError(err)})
internal/http/providers.go
+6
View File
@@ -22,6 +22,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/permissions"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
@@ -39,6 +40,7 @@ type ProvidersHandler struct {
tracingStore store.TracingStore // optional: for provider-scoped pool activity
agents store.AgentCRUDStore // optional: for provider pool activity agent lookup
modelReg providers.ModelRegistry // optional: forward-compat model resolver for Anthropic
usageCaps *usagecaps.Service
}
// NewProvidersHandler creates a handler for provider management endpoints.
@@ -85,6 +87,10 @@ func (h *ProvidersHandler) SetModelRegistry(r providers.ModelRegistry) {
h.modelReg = r
}
func (h *ProvidersHandler) SetUsageCapService(s *usagecaps.Service) {
h.usageCaps = s
}
// resolveAPIBase returns the provider's api_base, falling back to config/env if empty.
// For Ollama/OllamaCloud providers, applies a safety-net normalization: if the stored
// value is missing the /v1 suffix (pre-existing record before write-time normalization),
internal/http/secure_cli.go
+43 -99
View File
@@ -7,7 +7,6 @@ import (
"net/http"
"os/exec"
"regexp"
"sort"
"strings"
"github.com/google/uuid"
@@ -70,68 +69,13 @@ func (h *SecureCLIHandler) emitCacheInvalidate(key string) {
// envKeysFromDecryptedJSON returns sorted env variable names from plaintext env JSON (decrypted blob).
func envKeysFromDecryptedJSON(env []byte) []string {
empty := []string{}
if len(env) == 0 {
return empty
}
var m map[string]any
if err := json.Unmarshal(env, &m); err != nil {
return empty
}
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
return store.SecureCLIEnvKeys(env)
}
// mergeSecureCLIEnv merges incoming env from the UI with existing stored env.
// Incoming defines the full set of keys shown in the form: keys omitted were removed.
// Empty string means "keep existing value" for that key when it already exists.
func mergeSecureCLIEnv(existingJSON []byte, incoming map[string]any) (map[string]string, error) {
existing := map[string]string{}
if len(existingJSON) > 0 {
if err := json.Unmarshal(existingJSON, &existing); err != nil {
return nil, fmt.Errorf("parse existing env: %w", err)
}
}
out := make(map[string]string)
for k, v := range incoming {
if k == "" {
continue
}
sv, err := envValueAsString(v)
if err != nil {
return nil, fmt.Errorf("invalid environment variable value")
}
if sv != "" {
out[k] = sv
continue
}
if ev, ok := existing[k]; ok {
out[k] = ev
}
}
return out, nil
}
func envValueAsString(v any) (string, error) {
switch t := v.(type) {
case string:
return t, nil
case float64:
return fmt.Sprint(t), nil
case bool:
if t {
return "true", nil
}
return "false", nil
case nil:
return "", nil
default:
return "", fmt.Errorf("value must be a string")
}
func populateBinaryEnvResponse(b *store.SecureCLIBinary) {
b.EnvKeys = store.SecureCLIEnvKeys(b.EncryptedEnv)
b.Env = store.SanitizeSecureCLIEnvJSON(b.EncryptedEnv)
b.EncryptedEnv = nil
}
func (h *SecureCLIHandler) handleList(w http.ResponseWriter, r *http.Request) {
@@ -142,27 +86,26 @@ func (h *SecureCLIHandler) handleList(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgFailedToList, "CLI credentials")})
return
}
// Never send env values; only variable names for editing.
// Sensitive env values stay masked; value-kind entries are returned for editing.
for i := range result {
result[i].EnvKeys = envKeysFromDecryptedJSON(result[i].EncryptedEnv)
result[i].EncryptedEnv = nil
populateBinaryEnvResponse(&result[i])
}
writeJSON(w, http.StatusOK, map[string]any{"items": result})
}
// secureCLICreateRequest supports both preset-based and custom creation.
type secureCLICreateRequest struct {
Preset string `json:"preset,omitempty"` // auto-fill from preset
BinaryName string `json:"binary_name"`
BinaryPath *string `json:"binary_path,omitempty"`
Description string `json:"description"`
Env map[string]string `json:"env"` // plaintext env vars (encrypted by store)
DenyArgs json.RawMessage `json:"deny_args,omitempty"`
DenyVerbose json.RawMessage `json:"deny_verbose,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
Tips string `json:"tips,omitempty"`
IsGlobal *bool `json:"is_global,omitempty"`
Enabled bool `json:"enabled"`
Preset string `json:"preset,omitempty"` // auto-fill from preset
BinaryName string `json:"binary_name"`
BinaryPath *string `json:"binary_path,omitempty"`
Description string `json:"description"`
Env json.RawMessage `json:"env"` // plaintext env vars or env entry objects (encrypted by store)
DenyArgs json.RawMessage `json:"deny_args,omitempty"`
DenyVerbose json.RawMessage `json:"deny_verbose,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
Tips string `json:"tips,omitempty"`
IsGlobal *bool `json:"is_global,omitempty"`
Enabled bool `json:"enabled"`
}
func (h *SecureCLIHandler) handleCreate(w http.ResponseWriter, r *http.Request) {
@@ -209,10 +152,14 @@ func (h *SecureCLIHandler) handleCreate(w http.ResponseWriter, r *http.Request)
return
}
// Serialize env as JSON bytes (store layer encrypts)
envJSON, err := json.Marshal(req.Env)
envEntries, err := store.ParseSecureCLIEnv(req.Env)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid env"})
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return
}
envJSON, err := store.SerializeSecureCLIEnv(envEntries)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return
}
@@ -240,7 +187,7 @@ func (h *SecureCLIHandler) handleCreate(w http.ResponseWriter, r *http.Request)
}
tools.ResetCredentialScrubValues() // clear stale scrub values
b.EncryptedEnv = nil // don't return credentials
populateBinaryEnvResponse(b)
emitAudit(h.msgBus, r, "secure_cli.created", "secure_cli", b.ID.String())
h.emitCacheInvalidate(b.ID.String())
writeJSON(w, http.StatusCreated, b)
@@ -260,8 +207,7 @@ func (h *SecureCLIHandler) handleGet(w http.ResponseWriter, r *http.Request) {
return
}
b.EnvKeys = envKeysFromDecryptedJSON(b.EncryptedEnv)
b.EncryptedEnv = nil // don't expose credential values
populateBinaryEnvResponse(b)
writeJSON(w, http.StatusOK, b)
}
@@ -293,25 +239,23 @@ func (h *SecureCLIHandler) handleUpdate(w http.ResponseWriter, r *http.Request)
// If env is updated, merge with stored env so empty values mean "keep existing secret".
if envVal, ok := updates["env"]; ok {
if envMap, isMap := envVal.(map[string]any); isMap {
cur, err := h.store.Get(r.Context(), id)
if err != nil {
writeJSON(w, http.StatusNotFound, map[string]string{"error": i18n.T(locale, i18n.MsgNotFound, "credential", id.String())})
return
}
merged, err := mergeSecureCLIEnv(cur.EncryptedEnv, envMap)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
return
}
envJSON, err := json.Marshal(merged)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid env"})
return
}
updates["encrypted_env"] = string(envJSON)
delete(updates, "env")
envJSON, err := json.Marshal(envVal)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return
}
cur, err := h.store.Get(r.Context(), id)
if err != nil {
writeJSON(w, http.StatusNotFound, map[string]string{"error": i18n.T(locale, i18n.MsgNotFound, "credential", id.String())})
return
}
merged, err := store.MergeSecureCLIEnv(cur.EncryptedEnv, envJSON)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return
}
updates["encrypted_env"] = string(merged)
delete(updates, "env")
}
if err := h.store.Update(r.Context(), id, updates); err != nil {
internal/http/secure_cli_agent_grants.go
+38 -38
View File
@@ -68,7 +68,7 @@ func (h *SecureCLIGrantHandler) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("GET /v1/cli-credentials/{id}/agent-grants/{grantId}", auth(h.handleGet))
mux.HandleFunc("PUT /v1/cli-credentials/{id}/agent-grants/{grantId}", auth(h.handleUpdate))
mux.HandleFunc("DELETE /v1/cli-credentials/{id}/agent-grants/{grantId}", auth(h.handleDelete))
// POST (not GET) to prevent caching and satisfy CSRF semantics per Red Team C1.
// POST keeps revealed secret material out of URL/history and avoids query caching.
mux.HandleFunc("POST /v1/cli-credentials/{id}/agent-grants/{grantId}/env:reveal", auth(h.handleRevealEnv))
}
@@ -76,45 +76,41 @@ func (h *SecureCLIGrantHandler) RegisterRoutes(mux *http.ServeMux) {
// EnvVars is optional; plaintext values are encrypted by the store layer.
// Clients MUST NOT send encrypted_env — that field is never accepted from the wire.
type grantCreateRequest struct {
AgentID uuid.UUID `json:"agent_id"`
EnvVars map[string]string `json:"env_vars,omitempty"`
DenyArgs *json.RawMessage `json:"deny_args,omitempty"`
DenyVerbose *json.RawMessage `json:"deny_verbose,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
Tips *string `json:"tips,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
AgentID uuid.UUID `json:"agent_id"`
EnvVars json.RawMessage `json:"env_vars,omitempty"`
DenyArgs *json.RawMessage `json:"deny_args,omitempty"`
DenyVerbose *json.RawMessage `json:"deny_verbose,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
Tips *string `json:"tips,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
}
// populateGrantEnvFields sets EnvKeys (sorted) and EnvSet from the grant's decrypted env bytes.
// Plaintext values are never exposed — only key names.
// populateGrantEnvFields sets sorted key names, env presence, and sanitized entries.
func populateGrantEnvFields(g *store.SecureCLIAgentGrant) {
if len(g.EncryptedEnv) == 0 {
g.EnvKeys = []string{}
g.Env = nil
g.EnvSet = false
return
}
var m map[string]any
if err := json.Unmarshal(g.EncryptedEnv, &m); err != nil {
g.EnvKeys = []string{}
g.EnvSet = false
return
}
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
keys := store.SecureCLIEnvKeys(g.EncryptedEnv)
g.EnvKeys = keys
g.Env = store.SanitizeSecureCLIEnvJSON(g.EncryptedEnv)
g.EnvSet = len(keys) > 0
}
// validateAndSerializeEnvVars validates env keys/values via denylist and returns serialized JSON.
// Returns (nil, 400 error response written) on denial, (jsonBytes, nil) on success.
// Never logs env values or keys in error paths.
func validateAndSerializeEnvVars(w http.ResponseWriter, locale string, envVars map[string]string) ([]byte, bool) {
if len(envVars) == 0 {
b, _ := json.Marshal(envVars)
return b, true
func validateAndSerializeEnvVars(w http.ResponseWriter, locale string, raw json.RawMessage) ([]byte, bool) {
envEntries, err := store.ParseSecureCLIEnv(raw)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return nil, false
}
envVars := make(map[string]string, len(envEntries))
for key, entry := range envEntries {
envVars[key] = entry.Value
}
denied, valErr := crypto.ValidateGrantEnvVars(envVars)
if valErr != nil {
@@ -129,7 +125,7 @@ func validateAndSerializeEnvVars(w http.ResponseWriter, locale string, envVars m
})
return nil, false
}
b, err := json.Marshal(envVars)
b, err := store.SerializeSecureCLIEnv(envEntries)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, "serialization failed")})
return nil, false
@@ -249,7 +245,7 @@ func (h *SecureCLIGrantHandler) handleCreate(w http.ResponseWriter, r *http.Requ
envJSON, ok := validateAndSerializeEnvVars(w, locale, req.EnvVars)
if !ok {
// Grant was created but env validation failed; clean it up to avoid orphan row.
// Finding #13: log rollback-delete failures for ops visibility.
// Log rollback-delete failures so operators can clean up orphan rows.
if delErr := h.grants.Delete(r.Context(), g.ID); delErr != nil {
slog.Error("secure_cli_grants.create.rollback_delete",
"grant_id", g.ID,
@@ -261,7 +257,7 @@ func (h *SecureCLIGrantHandler) handleCreate(w http.ResponseWriter, r *http.Requ
}
if err := h.grants.UpdateGrantEnv(r.Context(), g.ID, envJSON); err != nil {
slog.Error("secure_cli_grants.create.set_env", "grant_id", g.ID, "error", err)
// Finding #13: log rollback-delete failures for ops visibility.
// Log rollback-delete failures so operators can clean up orphan rows.
if delErr := h.grants.Delete(r.Context(), g.ID); delErr != nil {
slog.Error("secure_cli_grants.create.rollback_delete",
"grant_id", g.ID,
@@ -323,7 +319,7 @@ func (h *SecureCLIGrantHandler) handleUpdate(w http.ResponseWriter, r *http.Requ
}
if allowedScalar[k] {
var decoded any
// Finding #3: return 400 on Unmarshal failure — silent discard means admin
// Return 400 on unmarshal failure; silent discard means admin
// thinks they applied a change (e.g. enabled: "false") but the grant is unchanged.
if err := json.Unmarshal(v, &decoded); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{
@@ -335,7 +331,7 @@ func (h *SecureCLIGrantHandler) handleUpdate(w http.ResponseWriter, r *http.Requ
}
}
// 3-state env_vars semantics: absent=skip, null=clear, {...}=replace.
// Finding #15: {} (empty map) is treated as clear — same as null.
// Empty map is treated as clear, same as null.
// TS type: absent | null | Record<string,string> — see ui/web/src/types/cli-credential.ts.
var envJSON []byte
envPresent := false
@@ -343,18 +339,22 @@ func (h *SecureCLIGrantHandler) handleUpdate(w http.ResponseWriter, r *http.Requ
envPresent = true
var envPtr *map[string]string
if string(envRaw) != "null" {
var m map[string]string
if err := json.Unmarshal(envRaw, &m); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, "env_vars must be a string map")})
envEntries, err := store.ParseSecureCLIEnv(envRaw)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, "env_vars must be a string map or env entries")})
return
}
m := make(map[string]string, len(envEntries))
for key, entry := range envEntries {
m[key] = entry.Value
}
envPtr = &m
}
// envPtr == nil → clear; envPtr != nil → replace.
// Note: envPtr pointing to an empty map ({}) is treated as clear (same as null) —
// envJSON stays nil and UpdateGrantEnv(nil) removes the override.
if envPtr != nil && len(*envPtr) > 0 {
j, ok := validateAndSerializeEnvVars(w, locale, *envPtr)
j, ok := validateAndSerializeEnvVars(w, locale, envRaw)
if !ok {
return
}
@@ -430,8 +430,8 @@ func (h *SecureCLIGrantHandler) handleRevealEnv(w http.ResponseWriter, r *http.R
locale := store.LocaleFromContext(ctx)
// Rate limit: 10 reveals/min per authenticated caller (context UserID).
// Finding #2: require non-empty UserID from authenticated context.
// If UserID is empty, the auth middleware failed to populate it — reject rather
// Require non-empty UserID from authenticated context.
// If UserID is empty, auth middleware failed to populate it — reject rather
// than fall back to a spoofable header or IP address.
callerID := store.UserIDFromContext(ctx)
if callerID == "" {
@@ -476,8 +476,8 @@ func (h *SecureCLIGrantHandler) handleRevealEnv(w http.ResponseWriter, r *http.R
writeJSON(w, http.StatusOK, map[string]any{"env_vars": map[string]string{}})
return
}
var envVars map[string]string
if err := json.Unmarshal(g.EncryptedEnv, &envVars); err != nil {
envVars, err := store.FlattenSecureCLIEnv(g.EncryptedEnv)
if err != nil {
slog.Error("secure_cli_grants.reveal.parse", "grant_id", g.ID, "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "parse grant env")})
return
internal/http/secure_cli_agent_grants_test.go
+50 -2
View File
@@ -3,6 +3,7 @@ package http
import (
"context"
"database/sql"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
@@ -59,8 +60,16 @@ func (s *fakeSecureCLIGrantStore) Delete(context.Context, uuid.UUID) error {
return nil
}
func (s *fakeSecureCLIGrantStore) ListByBinary(context.Context, uuid.UUID) ([]store.SecureCLIAgentGrant, error) {
return nil, nil
func (s *fakeSecureCLIGrantStore) ListByBinary(_ context.Context, binaryID uuid.UUID) ([]store.SecureCLIAgentGrant, error) {
grants := make([]store.SecureCLIAgentGrant, 0, len(s.grants))
for _, grant := range s.grants {
if grant == nil || grant.BinaryID != binaryID {
continue
}
cp := *grant
grants = append(grants, cp)
}
return grants, nil
}
func (s *fakeSecureCLIGrantStore) ListByAgent(context.Context, uuid.UUID) ([]store.SecureCLIAgentGrant, error) {
@@ -202,3 +211,42 @@ func TestSecureCLIGrantUpdateRejectsInvalidEnvVarsBeforeScalarUpdate(t *testing.
t.Fatal("invalid env_vars request must not persist scalar grant updates")
}
}
func TestSecureCLIGrantGetSanitizesMixedEnv(t *testing.T) {
binaryID := uuid.New()
grantID := uuid.New()
fake := &fakeSecureCLIGrantStore{
grants: map[uuid.UUID]*store.SecureCLIAgentGrant{
grantID: {
BaseModel: store.BaseModel{ID: grantID},
BinaryID: binaryID,
AgentID: uuid.New(),
Enabled: true,
EncryptedEnv: []byte(`{"TOKEN":"secret-token","PUBLIC_BASE_URL":{"kind":"value","value":"https://goclaw.sh"}}`),
},
},
}
h := NewSecureCLIGrantHandler(fake, nil, nil)
rr, req := requestWithGrantPath(http.MethodGet, nil, binaryID, grantID)
h.handleGet(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String())
}
if strings.Contains(rr.Body.String(), "secret-token") {
t.Fatalf("sensitive grant env leaked in response: %s", rr.Body.String())
}
var got struct {
Env map[string]store.SecureCLIEnvResponseEntry `json:"env"`
}
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if !got.Env["TOKEN"].Masked || got.Env["TOKEN"].Value != nil {
t.Fatalf("TOKEN not masked: %#v", got.Env["TOKEN"])
}
if got.Env["PUBLIC_BASE_URL"].Value == nil || *got.Env["PUBLIC_BASE_URL"].Value != "https://goclaw.sh" {
t.Fatalf("PUBLIC_BASE_URL not returned: %#v", got.Env["PUBLIC_BASE_URL"])
}
}
internal/http/secure_cli_env_response_test.go
+129
View File
@@ -0,0 +1,129 @@
package http
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/store"
)
type fakeSecureCLIStore struct {
binary *store.SecureCLIBinary
user *store.SecureCLIUserCredential
}
func (s *fakeSecureCLIStore) Create(context.Context, *store.SecureCLIBinary) error { return nil }
func (s *fakeSecureCLIStore) Get(context.Context, uuid.UUID) (*store.SecureCLIBinary, error) {
cp := *s.binary
return &cp, nil
}
func (s *fakeSecureCLIStore) Update(context.Context, uuid.UUID, map[string]any) error { return nil }
func (s *fakeSecureCLIStore) Delete(context.Context, uuid.UUID) error { return nil }
func (s *fakeSecureCLIStore) List(context.Context) ([]store.SecureCLIBinary, error) {
cp := *s.binary
return []store.SecureCLIBinary{cp}, nil
}
func (s *fakeSecureCLIStore) LookupByBinary(context.Context, string, *uuid.UUID, string) (*store.SecureCLIBinary, error) {
return nil, nil
}
func (s *fakeSecureCLIStore) ListEnabled(context.Context) ([]store.SecureCLIBinary, error) {
return nil, nil
}
func (s *fakeSecureCLIStore) ListForAgent(context.Context, uuid.UUID) ([]store.SecureCLIBinary, error) {
return nil, nil
}
func (s *fakeSecureCLIStore) IsRegisteredBinary(context.Context, string) (bool, error) {
return false, nil
}
func (s *fakeSecureCLIStore) GetUserCredentials(context.Context, uuid.UUID, string) (*store.SecureCLIUserCredential, error) {
cp := *s.user
return &cp, nil
}
func (s *fakeSecureCLIStore) SetUserCredentials(context.Context, uuid.UUID, string, []byte) error {
return nil
}
func (s *fakeSecureCLIStore) DeleteUserCredentials(context.Context, uuid.UUID, string) error {
return nil
}
func (s *fakeSecureCLIStore) ListUserCredentials(context.Context, uuid.UUID) ([]store.SecureCLIUserCredential, error) {
cp := *s.user
return []store.SecureCLIUserCredential{cp}, nil
}
func TestSecureCLIGetSanitizesMixedEnv(t *testing.T) {
id := uuid.New()
h := NewSecureCLIHandler(&fakeSecureCLIStore{
binary: &store.SecureCLIBinary{
BaseModel: store.BaseModel{ID: id},
BinaryName: "gh",
EncryptedEnv: []byte(`{"TOKEN":"secret-token","PUBLIC_BASE_URL":{"kind":"value","value":"https://goclaw.sh"}}`),
},
}, nil)
req := httptest.NewRequest(http.MethodGet, "/v1/cli-credentials/"+id.String(), nil)
req.SetPathValue("id", id.String())
rec := httptest.NewRecorder()
h.handleGet(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "secret-token") {
t.Fatalf("sensitive env leaked in response: %s", rec.Body.String())
}
var got struct {
Env map[string]store.SecureCLIEnvResponseEntry `json:"env"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if !got.Env["TOKEN"].Masked || got.Env["TOKEN"].Value != nil {
t.Fatalf("TOKEN not masked: %#v", got.Env["TOKEN"])
}
if got.Env["PUBLIC_BASE_URL"].Value == nil || *got.Env["PUBLIC_BASE_URL"].Value != "https://goclaw.sh" {
t.Fatalf("value env not returned: %#v", got.Env["PUBLIC_BASE_URL"])
}
}
func TestSecureCLIUserCredentialsGetDoesNotReturnLegacySensitiveRaw(t *testing.T) {
binaryID := uuid.New()
h := NewSecureCLIHandler(&fakeSecureCLIStore{
user: &store.SecureCLIUserCredential{
ID: uuid.New(),
BinaryID: binaryID,
UserID: "user-1",
EncryptedEnv: []byte(`{"TOKEN":"secret-token","REGION":{"kind":"value","value":"asia-southeast1"}}`),
},
}, nil)
req := httptest.NewRequest(http.MethodGet, "/v1/cli-credentials/"+binaryID.String()+"/user-credentials/user-1", nil)
req.SetPathValue("id", binaryID.String())
req.SetPathValue("userId", "user-1")
rec := httptest.NewRecorder()
h.handleGetUserCredentials(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "secret-token") {
t.Fatalf("legacy sensitive env leaked in response: %s", rec.Body.String())
}
var got struct {
Env map[string]store.SecureCLIEnvResponseEntry `json:"env"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if !got.Env["TOKEN"].Masked || got.Env["TOKEN"].Value != nil {
t.Fatalf("TOKEN not masked: %#v", got.Env["TOKEN"])
}
if got.Env["REGION"].Value == nil || *got.Env["REGION"].Value != "asia-southeast1" {
t.Fatalf("REGION not returned: %#v", got.Env["REGION"])
}
}
internal/http/secure_cli_user_credentials.go
+24 -18
View File
@@ -25,13 +25,14 @@ func (h *SecureCLIHandler) handleListUserCredentials(w http.ResponseWriter, r *h
}
// Return without env values for listing (names only + timestamps)
type entry struct {
ID uuid.UUID `json:"id"`
BinaryID uuid.UUID `json:"binary_id"`
UserID string `json:"user_id"`
HasEnv bool `json:"has_env"`
EnvKeys []string `json:"env_keys,omitempty"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
ID uuid.UUID `json:"id"`
BinaryID uuid.UUID `json:"binary_id"`
UserID string `json:"user_id"`
HasEnv bool `json:"has_env"`
EnvKeys []string `json:"env_keys,omitempty"`
Env map[string]store.SecureCLIEnvResponseEntry `json:"env,omitempty"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
entries := make([]entry, 0, len(creds))
for _, c := range creds {
@@ -42,6 +43,7 @@ func (h *SecureCLIHandler) handleListUserCredentials(w http.ResponseWriter, r *h
UserID: c.UserID,
HasEnv: len(c.EncryptedEnv) > 0,
EnvKeys: envKeys,
Env: store.SanitizeSecureCLIEnvJSON(c.EncryptedEnv),
CreatedAt: c.CreatedAt,
UpdatedAt: c.UpdatedAt,
})
@@ -73,14 +75,9 @@ func (h *SecureCLIHandler) handleGetUserCredentials(w http.ResponseWriter, r *ht
return
}
// Return decrypted env as JSON object (admin-only endpoint)
var envObj any
if len(cred.EncryptedEnv) > 0 {
_ = json.Unmarshal(cred.EncryptedEnv, &envObj)
}
writeJSON(w, http.StatusOK, map[string]any{
"user_id": cred.UserID,
"env": envObj,
"env": store.SanitizeSecureCLIEnvJSON(cred.EncryptedEnv),
})
}
@@ -108,14 +105,23 @@ func (h *SecureCLIHandler) handleSetUserCredentials(w http.ResponseWriter, r *ht
return
}
// Validate env is a JSON object
var envCheck map[string]string
if err := json.Unmarshal(body.Env, &envCheck); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "env must be a JSON object with string values"})
existing, err := h.store.GetUserCredentials(r.Context(), binaryID, userID)
if err != nil {
locale := store.LocaleFromContext(r.Context())
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, err.Error())})
return
}
var existingEnv []byte
if existing != nil {
existingEnv = existing.EncryptedEnv
}
envJSON, err := store.MergeSecureCLIEnv(existingEnv, body.Env)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return
}
if err := h.store.SetUserCredentials(r.Context(), binaryID, userID, body.Env); err != nil {
if err := h.store.SetUserCredentials(r.Context(), binaryID, userID, envJSON); err != nil {
locale := store.LocaleFromContext(r.Context())
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, err.Error())})
return
internal/http/skills.go
+3 -3
View File
@@ -21,8 +21,6 @@ import (
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
const maxSkillUploadSize = 20 << 20 // 20 MB
var (
aggregateInstallDeps = skills.AggregateMissingDeps
installManagedDeps = skills.InstallDeps
@@ -40,11 +38,13 @@ type SkillsHandler struct {
tenantStore store.TenantStore
db *sql.DB // for export/import direct queries
uploadLocks sync.Map // per-slug mutex; bounded by validated slug set, entries are tiny (*sync.Mutex)
uploadLimitCfg config.SkillsConfig
systemConfigs store.SystemConfigStore
}
// NewSkillsHandler creates a handler for skill management endpoints.
func NewSkillsHandler(skills store.SkillManageStore, baseDir, dataDir, bundledDir string, msgBus *bus.MessageBus, tenantCfgStore store.SkillTenantConfigStore, tenantStore store.TenantStore) *SkillsHandler {
return &SkillsHandler{skills: skills, baseDir: baseDir, dataDir: dataDir, bundledDir: bundledDir, msgBus: msgBus, tenantCfgStore: tenantCfgStore, tenantStore: tenantStore}
return &SkillsHandler{skills: skills, baseDir: baseDir, dataDir: dataDir, bundledDir: bundledDir, msgBus: msgBus, tenantCfgStore: tenantCfgStore, tenantStore: tenantStore, uploadLimitCfg: config.SkillsConfig{MaxUploadSizeMB: config.DefaultSkillMaxUploadSizeMB}}
}
// tenantSkillsDir returns the skills-store directory scoped to the requesting tenant.
internal/http/skills_grants.go
-15
View File
@@ -1,8 +1,6 @@
package http
import (
"archive/zip"
"io"
"log/slog"
"net/http"
@@ -225,16 +223,3 @@ func (h *SkillsHandler) handleRevokeUser(w http.ResponseWriter, r *http.Request)
}
// --- Helpers ---
func readZipFile(f *zip.File) (string, error) {
rc, err := f.Open()
if err != nil {
return "", err
}
defer rc.Close()
data, err := io.ReadAll(rc)
if err != nil {
return "", err
}
return string(data), nil
}
internal/http/skills_upload.go
+14 -8
View File
@@ -5,6 +5,7 @@ import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
@@ -43,7 +44,8 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
return
}
r.Body = http.MaxBytesReader(w, r.Body, maxSkillUploadSize)
preParseLimitMB := h.resolvePreParseUploadLimitMB(r.Context())
r.Body = http.MaxBytesReader(w, r.Body, skillUploadBodyBytes(preParseLimitMB))
file, header, err := r.FormFile("file")
if err != nil {
@@ -70,6 +72,11 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
size, err := io.Copy(tmp, file)
if err != nil {
tmp.Close()
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidRequest, skillUploadTooLargeMessage(maxBytesErr.Limit, preParseLimitMB))})
return
}
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "failed to save upload")})
return
}
@@ -141,6 +148,11 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
}
name, description, slug, frontmatter := skills.ParseSkillFrontmatter(skillContent)
uploadLimitMB := h.resolveSkillUploadLimitMB(r.Context(), frontmatter)
if size > skillUploadLimitBytes(uploadLimitMB) {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidRequest, skillUploadTooLargeMessage(size, uploadLimitMB))})
return
}
if name == "" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgRequired, "name in SKILL.md frontmatter")})
return
@@ -239,17 +251,11 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "failed to create skill file directory")})
return
}
data, err := readZipFile(f)
if err != nil {
if err := copyZipFileToPath(f, destPath); err != nil {
os.RemoveAll(destDir)
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidRequest, "failed to read ZIP entry")})
return
}
if err := os.WriteFile(destPath, []byte(data), 0644); err != nil {
os.RemoveAll(destDir)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "failed to write skill files")})
return
}
if cleanName == "SKILL.md" {
wroteSkillMD = true
}
internal/http/skills_upload_limits.go
+88
View File
@@ -0,0 +1,88 @@
package http
import (
"context"
"fmt"
"strconv"
"strings"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/store"
)
const (
skillUploadMaxSizeConfigKey = config.SkillMaxUploadSizeSystemConfigKey
skillUploadMultipartOverheadBytes = int64(1 << 20)
)
func (h *SkillsHandler) SetUploadLimitConfig(cfg config.SkillsConfig) {
h.uploadLimitCfg = cfg
}
func (h *SkillsHandler) SetSystemConfigStore(s store.SystemConfigStore) {
h.systemConfigs = s
}
func (h *SkillsHandler) resolveSkillUploadLimitMB(ctx context.Context, frontmatter map[string]string) int {
if mb, ok := h.resolveTenantSkillUploadLimitMB(ctx); ok {
return mb
}
if mb, ok := parseSkillUploadLimitMB(frontmatter["max_upload_size_mb"]); ok {
return config.ClampSkillMaxUploadSizeMB(mb)
}
return h.uploadLimitCfg.EffectiveMaxUploadSizeMB()
}
func (h *SkillsHandler) resolvePreParseUploadLimitMB(ctx context.Context) int {
if mb, ok := h.resolveTenantSkillUploadLimitMB(ctx); ok {
return mb
}
return config.MaxSkillMaxUploadSizeMB
}
func (h *SkillsHandler) resolveTenantSkillUploadLimitMB(ctx context.Context) (int, bool) {
if h.systemConfigs == nil {
return 0, false
}
raw, err := h.systemConfigs.Get(ctx, skillUploadMaxSizeConfigKey)
if err != nil {
return 0, false
}
mb, ok := parseSkillUploadLimitMB(raw)
if !ok {
return 0, false
}
return config.ClampSkillMaxUploadSizeMB(mb), true
}
func parseSkillUploadLimitMB(raw string) (int, bool) {
raw = strings.TrimSpace(raw)
if raw == "" {
return 0, false
}
mb, err := strconv.Atoi(raw)
if err != nil {
return 0, false
}
return mb, true
}
func skillUploadLimitBytes(limitMB int) int64 {
return int64(config.ClampSkillMaxUploadSizeMB(limitMB)) << 20
}
func skillUploadBodyBytes(limitMB int) int64 {
return skillUploadLimitBytes(limitMB) + skillUploadMultipartOverheadBytes
}
func skillUploadTooLargeMessage(size int64, limitMB int) string {
return fmt.Sprintf("skill ZIP size %s exceeds %d MB limit", formatUploadBytes(size), limitMB)
}
func formatUploadBytes(size int64) string {
const mb = int64(1 << 20)
if size >= mb {
return fmt.Sprintf("%.1f MB", float64(size)/float64(mb))
}
return fmt.Sprintf("%d bytes", size)
}
internal/http/skills_upload_test.go
+126
View File
@@ -12,11 +12,13 @@ import (
"net/http/httptest"
"path/filepath"
"reflect"
"strings"
"testing"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/skills"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
@@ -670,6 +672,65 @@ func TestHandleUpload_ReturnsGrantErrors(t *testing.T) {
}
}
func TestResolveSkillUploadLimitMBPrecedenceAndClamp(t *testing.T) {
ctx := store.WithTenantID(context.Background(), store.MasterTenantID)
handler, _, _, _ := newTestUploadHandler(t)
handler.SetUploadLimitConfig(config.SkillsConfig{MaxUploadSizeMB: 30})
if got := handler.resolveSkillUploadLimitMB(ctx, nil); got != 30 {
t.Fatalf("global limit = %d, want 30", got)
}
if got := handler.resolveSkillUploadLimitMB(ctx, map[string]string{"max_upload_size_mb": "100"}); got != 100 {
t.Fatalf("frontmatter limit = %d, want 100", got)
}
handler.SetSystemConfigStore(&skillUploadSystemConfigStore{data: map[string]string{skillUploadMaxSizeConfigKey: "40"}})
if got := handler.resolveSkillUploadLimitMB(ctx, map[string]string{"max_upload_size_mb": "100"}); got != 40 {
t.Fatalf("tenant limit = %d, want 40", got)
}
handler.SetSystemConfigStore(&skillUploadSystemConfigStore{data: map[string]string{skillUploadMaxSizeConfigKey: "999"}})
if got := handler.resolveSkillUploadLimitMB(ctx, nil); got != config.MaxSkillMaxUploadSizeMB {
t.Fatalf("clamped tenant limit = %d, want %d", got, config.MaxSkillMaxUploadSizeMB)
}
}
func TestUploadRejectsZipAboveConfiguredLimit(t *testing.T) {
handler, _, ctx, _ := newTestUploadHandler(t)
handler.SetUploadLimitConfig(config.SkillsConfig{MaxUploadSizeMB: 1})
req := newZipUploadRequestWithBinary(t, ctx, map[string][]byte{
"SKILL.md": []byte(skillMarkdown("Large Skill", "large-skill")),
"large.bin": bytes.Repeat([]byte("x"), (1<<20)+1),
})
w := httptest.NewRecorder()
handler.handleUpload(w, req)
if w.Code != http.StatusBadRequest {
t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
}
if !strings.Contains(w.Body.String(), "exceeds") || !strings.Contains(w.Body.String(), "1 MB") {
t.Fatalf("body = %s, want configured upload limit error", w.Body.String())
}
}
func TestUploadAllowsFrontmatterLimitAboveGlobalDefault(t *testing.T) {
handler, _, ctx, _ := newTestUploadHandler(t)
handler.SetUploadLimitConfig(config.SkillsConfig{MaxUploadSizeMB: 1})
skillMD := "---\nname: Video Skill\nslug: video-skill\nmax_upload_size_mb: 2\n---\nSkill body\n"
req := newZipUploadRequestWithBinary(t, ctx, map[string][]byte{
"SKILL.md": []byte(skillMD),
"large.bin": bytes.Repeat([]byte("x"), (1<<20)+(128<<10)),
})
w := httptest.NewRecorder()
handler.handleUpload(w, req)
if w.Code != http.StatusCreated {
t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
}
}
func newTestUploadHandler(t *testing.T) (*SkillsHandler, *skillManageStoreStub, context.Context, string) {
t.Helper()
@@ -738,6 +799,43 @@ func newZipUploadRequestWithManagers(t *testing.T, ctx context.Context, files ma
return req.WithContext(ctx)
}
func newZipUploadRequestWithBinary(t *testing.T, ctx context.Context, files map[string][]byte) *http.Request {
t.Helper()
var zipBuf bytes.Buffer
zw := zip.NewWriter(&zipBuf)
for name, content := range files {
header := &zip.FileHeader{Name: name, Method: zip.Store}
w, err := zw.CreateHeader(header)
if err != nil {
t.Fatalf("zip create %s: %v", name, err)
}
if _, err := w.Write(content); err != nil {
t.Fatalf("zip write %s: %v", name, err)
}
}
if err := zw.Close(); err != nil {
t.Fatalf("zip close: %v", err)
}
var body bytes.Buffer
mw := multipart.NewWriter(&body)
part, err := mw.CreateFormFile("file", "skill.zip")
if err != nil {
t.Fatalf("multipart file: %v", err)
}
if _, err := part.Write(zipBuf.Bytes()); err != nil {
t.Fatalf("multipart write: %v", err)
}
if err := mw.Close(); err != nil {
t.Fatalf("multipart close: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/v1/skills/upload", &body)
req.Header.Set("Content-Type", mw.FormDataContentType())
return req.WithContext(ctx)
}
func newUploadManagerIDsFormRequest(t *testing.T, raw string) *http.Request {
t.Helper()
@@ -770,6 +868,34 @@ type skillManageStoreStub struct {
lastUpdates map[uuid.UUID]map[string]any
}
type skillUploadSystemConfigStore struct {
data map[string]string
}
func (s *skillUploadSystemConfigStore) Get(_ context.Context, key string) (string, error) {
if v, ok := s.data[key]; ok {
return v, nil
}
return "", errors.New("not found")
}
func (s *skillUploadSystemConfigStore) Set(_ context.Context, key, value string) error {
if s.data == nil {
s.data = map[string]string{}
}
s.data[key] = value
return nil
}
func (s *skillUploadSystemConfigStore) Delete(_ context.Context, key string) error {
delete(s.data, key)
return nil
}
func (s *skillUploadSystemConfigStore) List(_ context.Context) (map[string]string, error) {
return s.data, nil
}
type skillGrantCall struct {
SkillID uuid.UUID
AgentID uuid.UUID
internal/http/skills_zip_helpers.go
+46
View File
@@ -0,0 +1,46 @@
package http
import (
"archive/zip"
"errors"
"io"
"os"
)
const maxSkillMarkdownBytes = 100 << 10
var errSkillMarkdownTooLarge = errors.New("SKILL.md exceeds 100 KB limit")
func readZipFile(f *zip.File) (string, error) {
rc, err := f.Open()
if err != nil {
return "", err
}
defer rc.Close()
data, err := io.ReadAll(io.LimitReader(rc, maxSkillMarkdownBytes+1))
if err != nil {
return "", err
}
if len(data) > maxSkillMarkdownBytes {
return "", errSkillMarkdownTooLarge
}
return string(data), nil
}
func copyZipFileToPath(f *zip.File, destPath string) error {
rc, err := f.Open()
if err != nil {
return err
}
defer rc.Close()
dst, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, rc)
return err
}
internal/http/summoner.go
+5 -1
View File
@@ -12,6 +12,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// Summoning event type constants.
@@ -48,14 +49,16 @@ type AgentSummoner struct {
agents store.AgentStore
providerReg *providers.Registry
msgBus *bus.MessageBus
usageCaps *usagecaps.Service
}
// NewAgentSummoner creates a summoner backed by the given stores and provider registry.
func NewAgentSummoner(agents store.AgentStore, providerReg *providers.Registry, msgBus *bus.MessageBus) *AgentSummoner {
func NewAgentSummoner(agents store.AgentStore, providerReg *providers.Registry, msgBus *bus.MessageBus, usageCaps *usagecaps.Service) *AgentSummoner {
return &AgentSummoner{
agents: agents,
providerReg: providerReg,
msgBus: msgBus,
usageCaps: usageCaps,
}
}
@@ -73,6 +76,7 @@ const singleCallTimeout = 300 * time.Second
func (s *AgentSummoner) SummonAgent(agentID uuid.UUID, tenantID uuid.UUID, providerName, model, description string) {
ctx, cancel := context.WithTimeout(store.WithTenantID(context.Background(), tenantID), 600*time.Second)
defer cancel()
ctx = store.WithAgentID(ctx, agentID)
s.ensureBackfillFiles(ctx, agentID)
s.emitEvent(agentID, tenantID, SummonEventStarted, "", "")
internal/http/summoner_regenerate.go
+9 -1
View File
@@ -14,6 +14,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
@@ -23,6 +24,7 @@ import (
func (s *AgentSummoner) RegenerateAgent(agentID uuid.UUID, tenantID uuid.UUID, providerName, model, editPrompt string) {
ctx, cancel := context.WithTimeout(store.WithTenantID(context.Background(), tenantID), 300*time.Second)
defer cancel()
ctx = store.WithAgentID(ctx, agentID)
s.ensureBackfillFiles(ctx, agentID)
@@ -121,7 +123,7 @@ func (s *AgentSummoner) generateFiles(ctx context.Context, providerName, model,
slog.Info("summoning: calling LLM", "provider", providerName, "model", model, "prompt_len", len(prompt))
resp, err := provider.Chat(ctx, providers.ChatRequest{
req := providers.ChatRequest{
Messages: []providers.Message{
{Role: "system", Content: "You are a file generator. Output ONLY the requested XML-tagged files. No extra commentary."},
{Role: "user", Content: prompt},
@@ -133,6 +135,12 @@ func (s *AgentSummoner) generateFiles(ctx context.Context, providerName, model,
providers.OptSessionKey: summonSessionKey,
providers.OptDisableTools: true,
},
}
resp, err := s.usageCaps.Chat(ctx, provider, req, usagecaps.ChatOptions{
ProviderName: providerName,
ModelID: model,
Purpose: "agent-summoner",
MaxOutputTokens: 8192,
})
if err != nil {
return nil, fmt.Errorf("%s: %w", providerName, err)
internal/http/usage_caps.go
+397
View File
@@ -0,0 +1,397 @@
package http
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"strconv"
"strings"
"time"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/permissions"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/usage/pricing"
)
type UsageCapsHandler struct {
store store.UsageCapStore
tenants store.TenantStore
client *http.Client
}
func NewUsageCapsHandler(s store.UsageCapStore, tenants store.TenantStore) *UsageCapsHandler {
return &UsageCapsHandler{store: s, tenants: tenants, client: &http.Client{Timeout: 30 * time.Second}}
}
func (h *UsageCapsHandler) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("GET /v1/usage-caps/policies", h.auth(h.handleListPolicies))
mux.HandleFunc("POST /v1/usage-caps/policies", h.writeAuth(h.handleCreatePolicy))
mux.HandleFunc("PATCH /v1/usage-caps/policies/{id}", h.writeAuth(h.handleUpdatePolicy))
mux.HandleFunc("DELETE /v1/usage-caps/policies/{id}", h.writeAuth(h.handleDeletePolicy))
mux.HandleFunc("GET /v1/usage-caps/utilization", h.auth(h.handleUtilization))
mux.HandleFunc("GET /v1/usage-caps/events", h.auth(h.handleEvents))
mux.HandleFunc("POST /v1/model-pricing/sync-openrouter", h.masterAuth(h.handleSyncOpenRouter))
mux.HandleFunc("GET /v1/model-pricing", h.auth(h.handleListPricing))
mux.HandleFunc("PUT /v1/model-pricing/overrides", h.writeAuth(h.handlePutOverride))
mux.HandleFunc("GET /v1/model-pricing/overrides", h.auth(h.handleListOverrides))
mux.HandleFunc("DELETE /v1/model-pricing/overrides/{id}", h.writeAuth(h.handleDeleteOverride))
}
func (h *UsageCapsHandler) auth(next http.HandlerFunc) http.HandlerFunc {
return requireAuth(permissions.RoleAdmin, next)
}
func (h *UsageCapsHandler) writeAuth(next http.HandlerFunc) http.HandlerFunc {
return h.auth(func(w http.ResponseWriter, r *http.Request) {
if !requireTenantAdmin(w, r, h.tenants) {
return
}
next(w, r)
})
}
func (h *UsageCapsHandler) masterAuth(next http.HandlerFunc) http.HandlerFunc {
return h.auth(func(w http.ResponseWriter, r *http.Request) {
if !requireMasterScope(w, r) {
return
}
next(w, r)
})
}
func writeUsageCapError(w http.ResponseWriter, r *http.Request, status int, key string, args ...any) {
writeJSON(w, status, map[string]string{"error": i18n.T(store.LocaleFromContext(r.Context()), key, args...)})
}
func (h *UsageCapsHandler) handleListPolicies(w http.ResponseWriter, r *http.Request) {
scope := store.UsageCapScope{TenantID: tenantIDOrMaster(r)}
policies, err := h.store.ListUsageCapPolicies(r.Context(), scope, true)
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsListPoliciesFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"policies": policies})
}
func (h *UsageCapsHandler) handleCreatePolicy(w http.ResponseWriter, r *http.Request) {
var body policyBody
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidJSON)
return
}
p, err := body.toPolicy(tenantIDOrMaster(r))
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidRequest, err.Error())
return
}
if err := h.store.CreateUsageCapPolicy(r.Context(), &p); err != nil {
slog.Warn("usage_caps.create_policy_failed", "error", err)
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsageCapPolicyValidationFailed)
return
}
writeJSON(w, http.StatusCreated, p)
}
func (h *UsageCapsHandler) handleUpdatePolicy(w http.ResponseWriter, r *http.Request) {
id, err := uuid.Parse(r.PathValue("id"))
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "policy")
return
}
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidJSON)
return
}
patch, err := policyPatchFromBody(bodyBytes)
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidRequest, err.Error())
return
}
p, err := h.store.UpdateUsageCapPolicy(r.Context(), tenantIDOrMaster(r), id, patch)
if err != nil {
if errors.Is(err, store.ErrUsageCapPolicyManaged) {
writeUsageCapError(w, r, http.StatusConflict, i18n.MsgUsageCapPolicyManaged)
return
}
slog.Warn("usage_caps.update_policy_failed", "error", err)
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsageCapPolicyValidationFailed)
return
}
writeJSON(w, http.StatusOK, p)
}
func (h *UsageCapsHandler) handleDeletePolicy(w http.ResponseWriter, r *http.Request) {
id, err := uuid.Parse(r.PathValue("id"))
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "policy")
return
}
if err := h.store.DeleteUsageCapPolicy(r.Context(), tenantIDOrMaster(r), id); err != nil {
if errors.Is(err, store.ErrUsageCapPolicyManaged) {
writeUsageCapError(w, r, http.StatusConflict, i18n.MsgUsageCapPolicyManaged)
return
}
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsDeletePolicyFailed)
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *UsageCapsHandler) handleUtilization(w http.ResponseWriter, r *http.Request) {
rows, err := h.store.ListUsageCapUtilization(r.Context(), tenantIDOrMaster(r))
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsUtilizationFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"rows": rows})
}
func (h *UsageCapsHandler) handleEvents(w http.ResponseWriter, r *http.Request) {
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
events, err := h.store.ListUsageCapEvents(r.Context(), tenantIDOrMaster(r), limit)
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsEventsFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"events": events})
}
func (h *UsageCapsHandler) handleSyncOpenRouter(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
defer cancel()
entries, err := pricing.FetchOpenRouterCatalog(ctx, h.client)
if err != nil {
slog.Warn("usage_pricing.openrouter_sync", "error", err)
writeUsageCapError(w, r, http.StatusBadGateway, i18n.MsgUsagePricingSyncOpenRouterFailed, err.Error())
return
}
count, err := h.store.UpsertPricingCatalog(r.Context(), entries)
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingStoreCatalogFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"count": count})
}
func (h *UsageCapsHandler) handleListPricing(w http.ResponseWriter, r *http.Request) {
rows, err := h.store.ListPricingCatalog(r.Context(), store.UsagePricingQuery{
ModelID: r.URL.Query().Get("model"),
Limit: queryInt(r, "limit", 100),
})
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingListFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"models": rows})
}
func (h *UsageCapsHandler) handlePutOverride(w http.ResponseWriter, r *http.Request) {
var body overrideBody
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidJSON)
return
}
providerID, err := uuid.Parse(body.ProviderID)
if err != nil || body.ModelID == "" {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsagePricingProviderModelRequired)
return
}
o := &store.UsagePricingOverride{
TenantID: tenantIDOrMaster(r), ProviderID: providerID,
ProviderType: body.ProviderType, ModelID: body.ModelID,
Pricing: body.Pricing, Enabled: body.Enabled == nil || *body.Enabled,
}
if err := h.store.PutPricingOverride(r.Context(), o); err != nil {
slog.Warn("usage_pricing.put_override_failed", "error", err)
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsagePricingOverrideValidationFailed)
return
}
writeJSON(w, http.StatusOK, o)
}
func (h *UsageCapsHandler) handleListOverrides(w http.ResponseWriter, r *http.Request) {
var providerID uuid.UUID
if raw := r.URL.Query().Get("provider_id"); raw != "" {
var err error
providerID, err = uuid.Parse(raw)
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "provider")
return
}
}
rows, err := h.store.ListPricingOverrides(r.Context(), store.UsagePricingQuery{TenantID: tenantIDOrMaster(r), ProviderID: providerID})
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingListOverridesFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"overrides": rows})
}
func (h *UsageCapsHandler) handleDeleteOverride(w http.ResponseWriter, r *http.Request) {
id, err := uuid.Parse(r.PathValue("id"))
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "override")
return
}
if err := h.store.DeletePricingOverride(r.Context(), tenantIDOrMaster(r), id); err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingDeleteOverrideFailed)
return
}
w.WriteHeader(http.StatusNoContent)
}
type policyBody struct {
AgentID *string `json:"agent_id"`
ProviderID *string `json:"provider_id"`
ProviderType *string `json:"provider_type"`
ModelID *string `json:"model_id"`
Window string `json:"window"`
MaxTokens *int64 `json:"max_tokens"`
MaxCostMicros *int64 `json:"max_cost_micros"`
MaxCostUSD *float64 `json:"max_cost_usd"`
Enabled *bool `json:"enabled"`
Priority *int `json:"priority"`
}
type overrideBody struct {
ProviderID string `json:"provider_id"`
ProviderType string `json:"provider_type"`
ModelID string `json:"model_id"`
Pricing store.UsagePricingFields `json:"pricing"`
Enabled *bool `json:"enabled"`
}
func (b policyBody) toPolicy(tenantID uuid.UUID) (store.UsageCapPolicy, error) {
p := store.UsageCapPolicy{TenantID: tenantID, Window: b.Window, Enabled: true, Priority: 100}
if b.Enabled != nil {
p.Enabled = *b.Enabled
}
if b.Priority != nil {
p.Priority = *b.Priority
}
if p.Window == "" {
p.Window = store.UsageCapWindowDay
}
var err error
p.AgentID, err = parseOptionalUUIDStrict("agent_id", b.AgentID)
if err != nil {
return p, err
}
p.ProviderID, err = parseOptionalUUIDStrict("provider_id", b.ProviderID)
if err != nil {
return p, err
}
if b.ProviderType != nil {
p.ProviderType = *b.ProviderType
}
if b.ModelID != nil {
p.ModelID = *b.ModelID
}
p.MaxTokens = b.MaxTokens
p.MaxCostMicros = maxCostMicros(b.MaxCostMicros, b.MaxCostUSD)
return p, nil
}
func (b policyBody) toPatch() (store.UsageCapPolicyPatch, error) {
var patch store.UsageCapPolicyPatch
if b.AgentID != nil {
v, err := parseOptionalUUIDStrict("agent_id", b.AgentID)
if err != nil {
return patch, err
}
patch.AgentID = &v
}
if b.ProviderID != nil {
v, err := parseOptionalUUIDStrict("provider_id", b.ProviderID)
if err != nil {
return patch, err
}
patch.ProviderID = &v
}
patch.ProviderType = b.ProviderType
patch.ModelID = b.ModelID
if b.Window != "" {
patch.Window = &b.Window
}
if b.MaxTokens != nil {
patch.MaxTokens = &b.MaxTokens
}
if b.MaxCostMicros != nil || b.MaxCostUSD != nil {
v := maxCostMicros(b.MaxCostMicros, b.MaxCostUSD)
patch.MaxCostMicros = &v
}
patch.Enabled = b.Enabled
patch.Priority = b.Priority
return patch, nil
}
func policyPatchFromBody(bodyBytes []byte) (store.UsageCapPolicyPatch, error) {
var body policyBody
if err := json.Unmarshal(bodyBytes, &body); err != nil {
return store.UsageCapPolicyPatch{}, err
}
patch, err := body.toPatch()
if err != nil {
return patch, err
}
var raw map[string]json.RawMessage
if err := json.Unmarshal(bodyBytes, &raw); err == nil {
if isJSONNull(raw["max_tokens"]) {
var v *int64
patch.MaxTokens = &v
}
if isJSONNull(raw["max_cost_micros"]) || isJSONNull(raw["max_cost_usd"]) {
var v *int64
patch.MaxCostMicros = &v
}
}
return patch, nil
}
func isJSONNull(raw json.RawMessage) bool {
return strings.TrimSpace(string(raw)) == "null"
}
func tenantIDOrMaster(r *http.Request) uuid.UUID {
if tid := store.TenantIDFromContext(r.Context()); tid != uuid.Nil {
return tid
}
return store.MasterTenantID
}
func queryInt(r *http.Request, key string, fallback int) int {
n, err := strconv.Atoi(r.URL.Query().Get(key))
if err != nil || n <= 0 {
return fallback
}
return n
}
func parseOptionalUUIDStrict(field string, raw *string) (*uuid.UUID, error) {
if raw == nil || *raw == "" {
return nil, nil
}
id, err := uuid.Parse(*raw)
if err != nil {
return nil, fmt.Errorf("invalid %s", field)
}
return &id, nil
}
func maxCostMicros(micros *int64, usd *float64) *int64 {
if micros != nil {
return micros
}
if usd == nil {
return nil
}
v := int64(*usd * 1_000_000)
return &v
}
internal/http/usage_caps_policy_patch_test.go
+35
View File
@@ -0,0 +1,35 @@
package http
import "testing"
func TestPolicyPatchFromBodyClearsTokenAndCostLimits(t *testing.T) {
patch, err := policyPatchFromBody([]byte(`{"max_tokens":null,"max_cost_usd":null}`))
if err != nil {
t.Fatalf("policyPatchFromBody: %v", err)
}
if patch.MaxTokens == nil {
t.Fatal("MaxTokens patch missing")
}
if *patch.MaxTokens != nil {
t.Fatalf("MaxTokens = %v, want nil clear", **patch.MaxTokens)
}
if patch.MaxCostMicros == nil {
t.Fatal("MaxCostMicros patch missing")
}
if *patch.MaxCostMicros != nil {
t.Fatalf("MaxCostMicros = %v, want nil clear", **patch.MaxCostMicros)
}
}
func TestPolicyPatchFromBodyPreservesProvidedCostUSD(t *testing.T) {
patch, err := policyPatchFromBody([]byte(`{"max_cost_usd":12.5}`))
if err != nil {
t.Fatalf("policyPatchFromBody: %v", err)
}
if patch.MaxCostMicros == nil || *patch.MaxCostMicros == nil {
t.Fatal("MaxCostMicros patch missing")
}
if got := **patch.MaxCostMicros; got != 12_500_000 {
t.Fatalf("MaxCostMicros = %d, want 12500000", got)
}
}
internal/http/usage_caps_test.go
+17
View File
@@ -0,0 +1,17 @@
package http
import (
"testing"
"github.com/google/uuid"
)
func TestUsageCapPolicyBodyRejectsInvalidUUID(t *testing.T) {
invalid := "not-a-uuid"
if _, err := (policyBody{AgentID: &invalid}).toPolicy(uuid.New()); err == nil {
t.Fatal("toPolicy accepted invalid agent_id")
}
if _, err := (policyBody{ProviderID: &invalid}).toPatch(); err == nil {
t.Fatal("toPatch accepted invalid provider_id")
}
}
internal/i18n/catalog_en.go
+19 -4
View File
@@ -93,6 +93,21 @@ func init() {
// Provider
MsgProviderReqFailed: "%s: request failed: %s",
// Usage caps / pricing
MsgUsageCapsListPoliciesFailed: "failed to list usage cap policies",
MsgUsageCapPolicyValidationFailed: "usage cap policy validation failed",
MsgUsageCapPolicyManaged: "managed usage cap policies cannot be modified",
MsgUsageCapsDeletePolicyFailed: "failed to delete usage cap policy",
MsgUsageCapsUtilizationFailed: "failed to load usage cap utilization",
MsgUsageCapsEventsFailed: "failed to load usage cap events",
MsgUsagePricingSyncOpenRouterFailed: "failed to sync OpenRouter pricing: %s",
MsgUsagePricingStoreCatalogFailed: "failed to store pricing catalog",
MsgUsagePricingListFailed: "failed to list model pricing",
MsgUsagePricingProviderModelRequired: "provider_id and model_id are required",
MsgUsagePricingOverrideValidationFailed: "pricing override validation failed",
MsgUsagePricingListOverridesFailed: "failed to list pricing overrides",
MsgUsagePricingDeleteOverrideFailed: "failed to delete pricing override",
// Unknown method
MsgUnknownMethod: "unknown method: %s",
@@ -206,10 +221,10 @@ func init() {
MsgTenantScopeRequired: "tenant scope is required for this operation",
// TTS / Voices
MsgTtsUnknownModel: "unknown tts model: %s",
MsgVoicesListFailed: "failed to list voices: %s",
MsgTtsGeminiInvalidVoice: "invalid Gemini voice: %s",
MsgTtsGeminiSpeakerLimit: "Gemini TTS supports at most 2 speakers",
MsgTtsUnknownModel: "unknown tts model: %s",
MsgVoicesListFailed: "failed to list voices: %s",
MsgTtsGeminiInvalidVoice: "invalid Gemini voice: %s",
MsgTtsGeminiSpeakerLimit: "Gemini TTS supports at most 2 speakers",
MsgTtsGeminiInvalidModel: "invalid Gemini TTS model: %s",
MsgTtsGeminiTextOnly: "Gemini refused to generate audio. Try simpler text without translation or commentary.",
MsgTtsParamOutOfRange: "TTS param %q value %v is out of range [%v, %v]",
internal/i18n/catalog_vi.go
+19 -4
View File
@@ -93,6 +93,21 @@ func init() {
// Provider
MsgProviderReqFailed: "%s: yêu cầu thất bại: %s",
// Usage caps / pricing
MsgUsageCapsListPoliciesFailed: "không thể liệt kê chính sách usage cap",
MsgUsageCapPolicyValidationFailed: "xác thực chính sách usage cap thất bại",
MsgUsageCapPolicyManaged: "không thể chỉnh sửa chính sách usage cap do hệ thống quản lý",
MsgUsageCapsDeletePolicyFailed: "không thể xóa chính sách usage cap",
MsgUsageCapsUtilizationFailed: "không thể tải mức sử dụng usage cap",
MsgUsageCapsEventsFailed: "không thể tải sự kiện usage cap",
MsgUsagePricingSyncOpenRouterFailed: "không thể đồng bộ giá OpenRouter: %s",
MsgUsagePricingStoreCatalogFailed: "không thể lưu catalog giá",
MsgUsagePricingListFailed: "không thể liệt kê giá model",
MsgUsagePricingProviderModelRequired: "provider_id và model_id là bắt buộc",
MsgUsagePricingOverrideValidationFailed: "xác thực override giá thất bại",
MsgUsagePricingListOverridesFailed: "không thể liệt kê override giá",
MsgUsagePricingDeleteOverrideFailed: "không thể xóa override giá",
// Unknown method
MsgUnknownMethod: "phương thức không xác định: %s",
@@ -206,10 +221,10 @@ func init() {
MsgTenantScopeRequired: "cần xác định tenant để thực hiện thao tác này",
// TTS / Giọng đọc
MsgTtsUnknownModel: "model tts không hỗ trợ: %s",
MsgVoicesListFailed: "không tải được danh sách giọng đọc: %s",
MsgTtsGeminiInvalidVoice: "giọng đọc Gemini không hợp lệ: %s",
MsgTtsGeminiSpeakerLimit: "Gemini TTS hỗ trợ tối đa 2 người nói",
MsgTtsUnknownModel: "model tts không hỗ trợ: %s",
MsgVoicesListFailed: "không tải được danh sách giọng đọc: %s",
MsgTtsGeminiInvalidVoice: "giọng đọc Gemini không hợp lệ: %s",
MsgTtsGeminiSpeakerLimit: "Gemini TTS hỗ trợ tối đa 2 người nói",
MsgTtsGeminiInvalidModel: "mô hình Gemini TTS không hợp lệ: %s",
MsgTtsGeminiTextOnly: "Gemini từ chối tạo âm thanh. Vui lòng thử văn bản đơn giản hơn, không dịch hay bình luận.",
MsgTtsParamOutOfRange: "tham số TTS %q có giá trị %v nằm ngoài phạm vi [%v, %v]",
internal/i18n/catalog_zh.go
+19 -4
View File
@@ -93,6 +93,21 @@ func init() {
// Provider
MsgProviderReqFailed: "%s：请求失败：%s",
// Usage caps / pricing
MsgUsageCapsListPoliciesFailed: "无法列出 usage cap 策略",
MsgUsageCapPolicyValidationFailed: "usage cap 策略验证失败",
MsgUsageCapPolicyManaged: "无法修改系统托管的 usage cap 策略",
MsgUsageCapsDeletePolicyFailed: "无法删除 usage cap 策略",
MsgUsageCapsUtilizationFailed: "无法加载 usage cap 使用量",
MsgUsageCapsEventsFailed: "无法加载 usage cap 事件",
MsgUsagePricingSyncOpenRouterFailed: "无法同步 OpenRouter 价格：%s",
MsgUsagePricingStoreCatalogFailed: "无法保存价格目录",
MsgUsagePricingListFailed: "无法列出模型价格",
MsgUsagePricingProviderModelRequired: "provider_id 和 model_id 是必填项",
MsgUsagePricingOverrideValidationFailed: "价格覆盖验证失败",
MsgUsagePricingListOverridesFailed: "无法列出价格覆盖",
MsgUsagePricingDeleteOverrideFailed: "无法删除价格覆盖",
// Unknown method
MsgUnknownMethod: "未知方法：%s",
@@ -206,10 +221,10 @@ func init() {
MsgTenantScopeRequired: "此操作需要指定租户范围",
// TTS / 声音
MsgTtsUnknownModel: "未知的 tts 模型：%s",
MsgVoicesListFailed: "获取声音列表失败：%s",
MsgTtsGeminiInvalidVoice: "无效的 Gemini 声音：%s",
MsgTtsGeminiSpeakerLimit: "Gemini TTS 最多支持 2 位发言人",
MsgTtsUnknownModel: "未知的 tts 模型：%s",
MsgVoicesListFailed: "获取声音列表失败：%s",
MsgTtsGeminiInvalidVoice: "无效的 Gemini 声音：%s",
MsgTtsGeminiSpeakerLimit: "Gemini TTS 最多支持 2 位发言人",
MsgTtsGeminiInvalidModel: "无效的 Gemini TTS 模型：%s",
MsgTtsGeminiTextOnly: "Gemini 拒绝生成音频。请尝试更简单的文本，不要翻译或添加评论。",
MsgTtsParamOutOfRange: "TTS 参数 %q 的值 %v 超出范围 [%v, %v]",
internal/i18n/keys.go
+63 -48
View File
@@ -94,6 +94,21 @@ const (
// --- Provider ---
MsgProviderReqFailed = "error.provider_request_failed" // "%s: request failed: %s"
// --- Usage caps / pricing ---
MsgUsageCapsListPoliciesFailed = "usage_caps.list_policies_failed"
MsgUsageCapPolicyValidationFailed = "usage_caps.policy_validation_failed"
MsgUsageCapPolicyManaged = "usage_caps.policy_managed"
MsgUsageCapsDeletePolicyFailed = "usage_caps.delete_policy_failed"
MsgUsageCapsUtilizationFailed = "usage_caps.utilization_failed"
MsgUsageCapsEventsFailed = "usage_caps.events_failed"
MsgUsagePricingSyncOpenRouterFailed = "usage_pricing.sync_openrouter_failed"
MsgUsagePricingStoreCatalogFailed = "usage_pricing.store_catalog_failed"
MsgUsagePricingListFailed = "usage_pricing.list_failed"
MsgUsagePricingProviderModelRequired = "usage_pricing.provider_model_required"
MsgUsagePricingOverrideValidationFailed = "usage_pricing.override_validation_failed"
MsgUsagePricingListOverridesFailed = "usage_pricing.list_overrides_failed"
MsgUsagePricingDeleteOverrideFailed = "usage_pricing.delete_override_failed"
// --- Unknown method ---
MsgUnknownMethod = "error.unknown_method" // "unknown method: %s"
@@ -123,14 +138,14 @@ const (
MsgInvalidVisibility = "error.invalid_visibility" // "invalid visibility %q: must be one of private, public"
// --- Package updates (Phase 4+5) ---
MsgPackageNotInstalled = "packages.update.not_installed" // "Package {name} is not installed"
MsgPackageUpdateLocked = "packages.update.locked" // "Package {name} is being updated by another request"
MsgPackageNotInstalled = "packages.update.not_installed" // "Package {name} is not installed"
MsgPackageUpdateLocked = "packages.update.locked" // "Package {name} is being updated by another request"
MsgReleaseNotFound = "packages.update.release_not_found" // "Release {tag} not found for {repo}"
MsgAssetNotFound = "packages.update.asset_not_found" // "No compatible asset for {os}/{arch}"
MsgAssetNotFound = "packages.update.asset_not_found" // "No compatible asset for {os}/{arch}"
MsgChecksumMismatch = "packages.update.checksum_mismatch" // "Checksum mismatch for {name}"
MsgUpdateSwapFailed = "packages.update.swap_failed" // "Failed to install {name}; previous version restored"
MsgUpdateManifestDesync = "packages.update.manifest_desync" // "Binary updated but manifest save failed — manual recovery required for {name}"
MsgUpdateCacheStale = "packages.update.cache_stale" // "Updates cache stale; run refresh before applying an update"
MsgUpdateSwapFailed = "packages.update.swap_failed" // "Failed to install {name}; previous version restored"
MsgUpdateManifestDesync = "packages.update.manifest_desync" // "Binary updated but manifest save failed — manual recovery required for {name}"
MsgUpdateCacheStale = "packages.update.cache_stale" // "Updates cache stale; run refresh before applying an update"
// Package update source labels
MsgPackagesUpdatesSourceGithub = "packages.updates.source.github" // "GitHub"
@@ -240,15 +255,15 @@ const (
MsgInvalidRole = "error.invalid_role" // "invalid role: allowed values are owner, admin, operator, member, viewer"
// --- TTS / Voices ---
MsgTtsUnknownModel = "error.tts_unknown_model" // "unknown tts model: %s"
MsgVoicesListFailed = "error.voices_list_failed" // "failed to list voices: %s"
MsgTtsGeminiInvalidVoice = "error.tts_gemini_invalid_voice" // "invalid Gemini voice: %s"
MsgTtsGeminiSpeakerLimit = "error.tts_gemini_speaker_limit" // "Gemini TTS supports at most 2 speakers"
MsgTtsGeminiInvalidModel = "error.tts_gemini_invalid_model" // "invalid Gemini TTS model: %s"
MsgTtsGeminiTextOnly = "error.tts_gemini_text_only" // "Gemini refused to generate audio; try simpler text without translation or commentary"
MsgTtsParamOutOfRange = "error.tts_param_out_of_range" // "TTS param %q value %v is out of range [%v, %v]"
MsgTtsParamUnknownKey = "error.tts_param_unknown_key" // "TTS param %q is not supported by this provider"
MsgTtsMiniMaxVoicesFailed = "error.tts_minimax_voices_failed" // "failed to fetch MiniMax voices: %s"
MsgTtsUnknownModel = "error.tts_unknown_model" // "unknown tts model: %s"
MsgVoicesListFailed = "error.voices_list_failed" // "failed to list voices: %s"
MsgTtsGeminiInvalidVoice = "error.tts_gemini_invalid_voice" // "invalid Gemini voice: %s"
MsgTtsGeminiSpeakerLimit = "error.tts_gemini_speaker_limit" // "Gemini TTS supports at most 2 speakers"
MsgTtsGeminiInvalidModel = "error.tts_gemini_invalid_model" // "invalid Gemini TTS model: %s"
MsgTtsGeminiTextOnly = "error.tts_gemini_text_only" // "Gemini refused to generate audio; try simpler text without translation or commentary"
MsgTtsParamOutOfRange = "error.tts_param_out_of_range" // "TTS param %q value %v is out of range [%v, %v]"
MsgTtsParamUnknownKey = "error.tts_param_unknown_key" // "TTS param %q is not supported by this provider"
MsgTtsMiniMaxVoicesFailed = "error.tts_minimax_voices_failed" // "failed to fetch MiniMax voices: %s"
// --- STT ---
MsgSTTAllProvidersFailed = "error.stt_all_providers_failed" // "All STT providers failed"
@@ -264,50 +279,50 @@ const (
MsgTenantScopeRequired = "error.tenant_scope_required" // "tenant scope is required for this operation"
// --- Webhooks ---
MsgWebhookAuthFailed = "webhook.auth_failed" // "webhook authentication failed"
MsgWebhookHMACInvalid = "webhook.hmac_invalid" // "HMAC signature is invalid"
MsgWebhookHMACTimestampSkew = "webhook.hmac_timestamp_skew" // "request timestamp outside acceptable window"
MsgWebhookBearerRequiredHMAC = "webhook.bearer_required_hmac" // "this webhook requires HMAC authentication"
MsgWebhookRevoked = "webhook.revoked" // "webhook has been revoked"
MsgWebhookKindMismatch = "webhook.kind_mismatch" // "request kind does not match webhook configuration"
MsgWebhookRateLimited = "webhook.rate_limited" // "webhook rate limit exceeded"
MsgWebhookBodyTooLarge = "webhook.body_too_large" // "request body exceeds size limit"
MsgWebhookIdempotencyConflict = "webhook.idempotency_conflict" // "idempotency key conflict: request body mismatch"
MsgWebhookTenantMismatch = "webhook.tenant_mismatch" // "webhook tenant mismatch"
MsgWebhookAgentNotFound = "webhook.agent_not_found" // "webhook agent not found"
MsgWebhookChannelNotFound = "webhook.channel_not_found" // "webhook channel not found"
MsgWebhookMediaSSRFBlocked = "webhook.media_ssrf_blocked" // "media URL blocked by SSRF policy"
MsgWebhookMediaTooLarge = "webhook.media_too_large" // "media file exceeds size limit"
MsgWebhookMediaMIMEDenied = "webhook.media_mime_denied" // "media MIME type is not allowed"
MsgWebhookCallbackURLInvalid = "webhook.callback_url_invalid" // "callback URL is invalid or blocked"
MsgWebhookLLMTimeout = "webhook.llm_timeout" // "LLM processing timed out"
MsgWebhookLaneSaturated = "webhook.lane_saturated" // "webhook processing lane is at capacity"
MsgWebhookAuthFailed = "webhook.auth_failed" // "webhook authentication failed"
MsgWebhookHMACInvalid = "webhook.hmac_invalid" // "HMAC signature is invalid"
MsgWebhookHMACTimestampSkew = "webhook.hmac_timestamp_skew" // "request timestamp outside acceptable window"
MsgWebhookBearerRequiredHMAC = "webhook.bearer_required_hmac" // "this webhook requires HMAC authentication"
MsgWebhookRevoked = "webhook.revoked" // "webhook has been revoked"
MsgWebhookKindMismatch = "webhook.kind_mismatch" // "request kind does not match webhook configuration"
MsgWebhookRateLimited = "webhook.rate_limited" // "webhook rate limit exceeded"
MsgWebhookBodyTooLarge = "webhook.body_too_large" // "request body exceeds size limit"
MsgWebhookIdempotencyConflict = "webhook.idempotency_conflict" // "idempotency key conflict: request body mismatch"
MsgWebhookTenantMismatch = "webhook.tenant_mismatch" // "webhook tenant mismatch"
MsgWebhookAgentNotFound = "webhook.agent_not_found" // "webhook agent not found"
MsgWebhookChannelNotFound = "webhook.channel_not_found" // "webhook channel not found"
MsgWebhookMediaSSRFBlocked = "webhook.media_ssrf_blocked" // "media URL blocked by SSRF policy"
MsgWebhookMediaTooLarge = "webhook.media_too_large" // "media file exceeds size limit"
MsgWebhookMediaMIMEDenied = "webhook.media_mime_denied" // "media MIME type is not allowed"
MsgWebhookCallbackURLInvalid = "webhook.callback_url_invalid" // "callback URL is invalid or blocked"
MsgWebhookLLMTimeout = "webhook.llm_timeout" // "LLM processing timed out"
MsgWebhookLaneSaturated = "webhook.lane_saturated" // "webhook processing lane is at capacity"
MsgWebhookLocalhostOnlyViolation = "webhook.localhost_only_violation" // "this webhook is restricted to localhost callers"
MsgWebhookMediaChannelUnsupported = "webhook.media_channel_unsupported" // "channel does not support media attachments"
MsgWebhookIPDenied = "webhook.ip_denied" // "request origin is not in the IP allowlist"
MsgWebhookEncryptionUnavailable = "webhook.encryption_unavailable" // "webhook encryption key not configured; set GOCLAW_ENCRYPTION_KEY to enable webhooks"
// --- Workstation permissions ---
MsgWorkstationCmdDenied = "error.workstation_cmd_denied" // "command denied by workstation policy: %s"
MsgWorkstationEnvDenied = "error.workstation_env_denied" // "env var denied by policy: %s"
MsgWorkstationInputInvalid = "error.workstation_input_invalid" // "command contains invalid characters: %s"
MsgWorkstationRateLimit = "error.workstation_rate_limit" // "workstation rate limit exceeded"
MsgWorkstationPermNotFound = "error.workstation_perm_not_found" // "permission entry not found: %s"
MsgWorkstationCmdDenied = "error.workstation_cmd_denied" // "command denied by workstation policy: %s"
MsgWorkstationEnvDenied = "error.workstation_env_denied" // "env var denied by policy: %s"
MsgWorkstationInputInvalid = "error.workstation_input_invalid" // "command contains invalid characters: %s"
MsgWorkstationRateLimit = "error.workstation_rate_limit" // "workstation rate limit exceeded"
MsgWorkstationPermNotFound = "error.workstation_perm_not_found" // "permission entry not found: %s"
// --- Workstation activity (Phase 7) ---
MsgWorkstationActivityTitle = "ui.workstations.activity.title" // "Recent Activity"
MsgWorkstationActionExec = "ui.workstations.activity.action_exec" // "Exec"
MsgWorkstationActionDeny = "ui.workstations.activity.action_deny" // "Denied"
MsgWorkstationActivityTitle = "ui.workstations.activity.title" // "Recent Activity"
MsgWorkstationActionExec = "ui.workstations.activity.action_exec" // "Exec"
MsgWorkstationActionDeny = "ui.workstations.activity.action_deny" // "Denied"
// --- Workstation ---
MsgWorkstationNotFound = "error.workstation_not_found" // "workstation not found: %s"
MsgWorkstationKeyExists = "error.workstation_key_exists" // "workstation key already in use: %s"
MsgInvalidBackend = "error.invalid_backend" // "invalid backend type: %s (must be ssh|docker)"
MsgWorkstationInactive = "error.workstation_inactive" // "workstation is inactive: %s"
MsgInvalidMetadataShape = "error.invalid_metadata_shape" // "invalid metadata for %s backend: %s"
MsgWorkstationRequired = "error.workstation_required" // "no workstation bound to agent; pass workstation_id"
MsgWorkstationNotFound = "error.workstation_not_found" // "workstation not found: %s"
MsgWorkstationKeyExists = "error.workstation_key_exists" // "workstation key already in use: %s"
MsgInvalidBackend = "error.invalid_backend" // "invalid backend type: %s (must be ssh|docker)"
MsgWorkstationInactive = "error.workstation_inactive" // "workstation is inactive: %s"
MsgInvalidMetadataShape = "error.invalid_metadata_shape" // "invalid metadata for %s backend: %s"
MsgWorkstationRequired = "error.workstation_required" // "no workstation bound to agent; pass workstation_id"
MsgWorkstationAccessDenied = "error.workstation_access_denied" // "agent %s not authorized for workstation %s"
MsgBackendNotReady = "error.backend_not_ready" // "workstation backend not ready: %s"
MsgBackendNotReady = "error.backend_not_ready" // "workstation backend not ready: %s"
// --- Hooks ---
MsgHookInvalidMatcher = "hook.invalid_matcher" // "invalid matcher regex: %s"
internal/knowledgegraph/extractor.go
+17 -3
View File
@@ -9,6 +9,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// ExtractionResult holds entities and relations extracted from text.
@@ -22,6 +23,7 @@ type Extractor struct {
provider providers.Provider
model string
minConfidence float64
usageCaps *usagecaps.Service
}
// NewExtractor creates a new Extractor with the given provider, model, and confidence threshold.
@@ -32,6 +34,11 @@ func NewExtractor(provider providers.Provider, model string, minConfidence float
return &Extractor{provider: provider, model: model, minConfidence: minConfidence}
}
// SetUsageCapService enables cost enforcement for LLM extraction calls.
func (e *Extractor) SetUsageCapService(s *usagecaps.Service) {
e.usageCaps = s
}
const maxChunkChars = 12000
// Extract calls the LLM to extract entities and relations from text.
@@ -72,7 +79,11 @@ func (e *Extractor) extractChunk(ctx context.Context, text string) (*ExtractionR
},
}
resp, err := e.provider.Chat(ctx, req)
resp, err := e.usageCaps.Chat(ctx, e.provider, req, usagecaps.ChatOptions{
ModelID: e.model,
Purpose: "knowledge-graph-extract",
MaxOutputTokens: 8192,
})
if err != nil {
return nil, fmt.Errorf("kg extraction LLM call: %w", err)
}
@@ -85,7 +96,11 @@ func (e *Extractor) extractChunk(ctx context.Context, text string) (*ExtractionR
text = text[:retryMaxChars] + "\n\n[...truncated]"
}
req.Messages[1].Content = text
resp, err = e.provider.Chat(ctx, req)
resp, err = e.usageCaps.Chat(ctx, e.provider, req, usagecaps.ChatOptions{
ModelID: e.model,
Purpose: "knowledge-graph-extract-retry",
MaxOutputTokens: 8192,
})
if err != nil {
return nil, fmt.Errorf("kg extraction LLM retry: %w", err)
}
@@ -286,4 +301,3 @@ func stripCodeBlock(s string) string {
}
return strings.TrimSpace(s)
}
internal/media/store.go
+6
View File
@@ -109,6 +109,12 @@ func ExtFromMime(mime string) string {
return ".wav"
case strings.HasPrefix(mime, "application/pdf"):
return ".pdf"
case strings.HasPrefix(mime, "application/zip"), strings.HasPrefix(mime, "application/x-zip-compressed"):
return ".zip"
case strings.HasPrefix(mime, "application/x-tar"):
return ".tar"
case strings.HasPrefix(mime, "application/gzip"), strings.HasPrefix(mime, "application/x-gzip"):
return ".gz"
case mime == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
return ".docx"
case mime == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
internal/media/store_test.go
+23
View File
@@ -0,0 +1,23 @@
package media
import "testing"
func TestExtFromMimeArchiveTypes(t *testing.T) {
tests := []struct {
mime string
want string
}{
{mime: "application/zip", want: ".zip"},
{mime: "application/x-zip-compressed", want: ".zip"},
{mime: "application/x-tar", want: ".tar"},
{mime: "application/gzip", want: ".gz"},
}
for _, tt := range tests {
t.Run(tt.mime, func(t *testing.T) {
if got := ExtFromMime(tt.mime); got != tt.want {
t.Fatalf("ExtFromMime(%q) = %q, want %q", tt.mime, got, tt.want)
}
})
}
}
internal/providers/anthropic_stream.go
+1 -1
View File
@@ -150,7 +150,7 @@ func (p *AnthropicProvider) ChatStream(ctx context.Context, req ChatRequest, onC
}
if err := sse.Err(); err != nil {
return nil, fmt.Errorf("anthropic stream read error: %w", err)
return result, fmt.Errorf("anthropic stream read error: %w", err)
}
// Parse accumulated tool call JSON arguments
internal/providers/codex.go
+1 -1
View File
@@ -168,7 +168,7 @@ func (p *CodexProvider) ChatStream(ctx context.Context, req ChatRequest, onChunk
}
if err := sse.Err(); err != nil {
return nil, fmt.Errorf("%s: stream read error: %w", p.name, err)
return result, fmt.Errorf("%s: stream read error: %w", p.name, err)
}
// Assemble generated images from image accumulator into ChatResponse.
internal/providers/model_fallback.go
+48
View File
@@ -21,6 +21,13 @@ type ModelFallbackProvider struct {
maxAttempts int
}
type FallbackCallInfo struct {
Streamed bool
}
type FallbackAfterCall func(*ChatResponse, error, FallbackCallInfo)
type FallbackBeforeCall func(ctx context.Context, entry FallbackCandidate, req ChatRequest) (after FallbackAfterCall, err error)
func NewModelFallbackProvider(primary FallbackCandidate, fallbacks []FallbackCandidate, maxAttempts int, cooldownEnabled bool) *ModelFallbackProvider {
var tracker *CooldownTracker
if cooldownEnabled {
@@ -64,6 +71,22 @@ func (p *ModelFallbackProvider) Chat(ctx context.Context, req ChatRequest) (*Cha
})
}
func (p *ModelFallbackProvider) ChatWithHook(ctx context.Context, req ChatRequest, before FallbackBeforeCall) (*ChatResponse, error) {
return p.runOrdered(ctx, req, func(ctx context.Context, entry FallbackCandidate, req ChatRequest) (*ChatResponse, error) {
nextReq := req
nextReq.Model = entry.Model
after, err := before(ctx, entry, nextReq)
if err != nil {
return nil, err
}
resp, err := entry.Provider.Chat(ctx, nextReq)
if after != nil {
after(resp, err, FallbackCallInfo{})
}
return resp, err
})
}
func (p *ModelFallbackProvider) ChatStream(ctx context.Context, req ChatRequest, onChunk func(StreamChunk)) (*ChatResponse, error) {
return p.runOrdered(ctx, req, func(ctx context.Context, entry FallbackCandidate, req ChatRequest) (*ChatResponse, error) {
nextReq := req
@@ -82,6 +105,31 @@ func (p *ModelFallbackProvider) ChatStream(ctx context.Context, req ChatRequest,
})
}
func (p *ModelFallbackProvider) ChatStreamWithHook(ctx context.Context, req ChatRequest, onChunk func(StreamChunk), before FallbackBeforeCall) (*ChatResponse, error) {
return p.runOrdered(ctx, req, func(ctx context.Context, entry FallbackCandidate, req ChatRequest) (*ChatResponse, error) {
nextReq := req
nextReq.Model = entry.Model
after, err := before(ctx, entry, nextReq)
if err != nil {
return nil, err
}
streamed := false
resp, err := entry.Provider.ChatStream(ctx, nextReq, func(chunk StreamChunk) {
if chunk.Content != "" || chunk.Thinking != "" || len(chunk.Images) > 0 {
streamed = true
}
onChunk(chunk)
})
if after != nil {
after(resp, err, FallbackCallInfo{Streamed: streamed})
}
if streamed && err != nil {
return nil, noFallbackAfterStreamError{err: err}
}
return resp, err
})
}
func (p *ModelFallbackProvider) runOrdered(
ctx context.Context,
req ChatRequest,
internal/providers/model_fallback_test.go
+27
View File
@@ -94,6 +94,33 @@ func TestModelFallbackProviderDoesNotFallbackAfterStreamChunk(t *testing.T) {
}
}
func TestModelFallbackProviderChatStreamWithHookReportsStreamedChunks(t *testing.T) {
streamErr := &HTTPError{Status: 429, Body: "rate limited"}
primary := &testFallbackProvider{
name: "primary",
model: "primary-model",
streamErr: streamErr,
}
provider := NewModelFallbackProvider(FallbackCandidate{
ProviderName: "primary",
Provider: primary,
Model: "primary-model",
}, nil, 1, false)
var streamed bool
_, err := provider.ChatStreamWithHook(context.Background(), ChatRequest{}, func(StreamChunk) {}, func(context.Context, FallbackCandidate, ChatRequest) (FallbackAfterCall, error) {
return func(_ *ChatResponse, _ error, info FallbackCallInfo) {
streamed = info.Streamed
}, nil
})
if err == nil {
t.Fatal("ChatStreamWithHook() error = nil, want stream error")
}
if !streamed {
t.Fatal("FallbackCallInfo.Streamed = false, want true after partial stream")
}
}
func TestModelFallbackProviderFallsBackToSameModelOnDifferentProvider(t *testing.T) {
primary := &testFallbackProvider{
name: "primary",
internal/providers/openai_chat.go
+7 -1
View File
@@ -108,13 +108,19 @@ func (p *OpenAIProvider) ChatStream(ctx context.Context, req ChatRequest, onChun
PromptTokens: chunk.Usage.PromptTokens,
CompletionTokens: chunk.Usage.CompletionTokens,
TotalTokens: chunk.Usage.TotalTokens,
RequestCount: 1,
}
if chunk.Usage.PromptTokensDetails != nil {
result.Usage.CacheReadTokens = chunk.Usage.PromptTokensDetails.CachedTokens
result.Usage.CacheCreationTokens = chunk.Usage.PromptTokensDetails.CacheWriteTokens
result.Usage.PromptTokensIncludeCachedSegments = true
}
if chunk.Usage.CompletionTokensDetails != nil && chunk.Usage.CompletionTokensDetails.ReasoningTokens > 0 {
result.Usage.ThinkingTokens = chunk.Usage.CompletionTokensDetails.ReasoningTokens
}
if chunk.Usage.ServerToolUse != nil {
result.Usage.WebSearchCount = chunk.Usage.ServerToolUse.WebSearchRequests
}
}
if len(chunk.Choices) == 0 {
@@ -181,7 +187,7 @@ func (p *OpenAIProvider) ChatStream(ctx context.Context, req ChatRequest, onChun
// Check for scanner errors (timeout, connection reset, etc.)
if err := sse.Err(); err != nil {
return nil, fmt.Errorf("%s: stream read error: %w", p.name, err)
return result, fmt.Errorf("%s: stream read error: %w", p.name, err)
}
// Parse accumulated tool call arguments
internal/providers/openai_http.go
+6
View File
@@ -125,13 +125,19 @@ func (p *OpenAIProvider) parseResponse(resp *openAIResponse) *ChatResponse {
PromptTokens: resp.Usage.PromptTokens,
CompletionTokens: resp.Usage.CompletionTokens,
TotalTokens: resp.Usage.TotalTokens,
RequestCount: 1,
}
if resp.Usage.PromptTokensDetails != nil {
result.Usage.CacheReadTokens = resp.Usage.PromptTokensDetails.CachedTokens
result.Usage.CacheCreationTokens = resp.Usage.PromptTokensDetails.CacheWriteTokens
result.Usage.PromptTokensIncludeCachedSegments = true
}
if resp.Usage.CompletionTokensDetails != nil && resp.Usage.CompletionTokensDetails.ReasoningTokens > 0 {
result.Usage.ThinkingTokens = resp.Usage.CompletionTokensDetails.ReasoningTokens
}
if resp.Usage.ServerToolUse != nil {
result.Usage.WebSearchCount = resp.Usage.ServerToolUse.WebSearchRequests
}
}
return result
internal/providers/openai_types.go
+8 -1
View File
@@ -49,18 +49,25 @@ type openAIUsage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
Cost float64 `json:"cost,omitempty"`
PromptTokensDetails *openAIPromptDetails `json:"prompt_tokens_details,omitempty"`
CompletionTokensDetails *openAICompletionDetails `json:"completion_tokens_details,omitempty"`
ServerToolUse *openAIServerToolUse `json:"server_tool_use,omitempty"`
}
type openAIPromptDetails struct {
CachedTokens int `json:"cached_tokens"`
CachedTokens int `json:"cached_tokens"`
CacheWriteTokens int `json:"cache_write_tokens,omitempty"`
}
type openAICompletionDetails struct {
ReasoningTokens int `json:"reasoning_tokens"`
}
type openAIServerToolUse struct {
WebSearchRequests int `json:"web_search_requests,omitempty"`
}
// Streaming types
type openAIStreamChunk struct {

Some files were not shown because too many files have changed in this diff Show More