package cmd import ( "context" "fmt" "log/slog" "os" "path/filepath" "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/bootstrap" "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/config" mcpbridge "github.com/nextlevelbuilder/goclaw/internal/mcp" "github.com/nextlevelbuilder/goclaw/internal/permissions" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/sandbox" "github.com/nextlevelbuilder/goclaw/internal/skills" "github.com/nextlevelbuilder/goclaw/internal/store" "github.com/nextlevelbuilder/goclaw/internal/store/pg" "github.com/nextlevelbuilder/goclaw/internal/tools" "github.com/nextlevelbuilder/goclaw/internal/tracing" "github.com/nextlevelbuilder/goclaw/internal/tts" "github.com/nextlevelbuilder/goclaw/pkg/browser" "github.com/nextlevelbuilder/goclaw/pkg/protocol" ) // setupToolRegistry creates the tool registry and registers all tools. // Returns the registry, exec approval manager, MCP manager, sandbox manager, // browser manager (caller must defer Close), web fetch tool, TTS tool, // permission policy engine, tool policy engine, data directory, and resolved agent defaults. func setupToolRegistry( cfg *config.Config, workspace string, providerRegistry *providers.Registry, ) ( toolsReg *tools.Registry, execApprovalMgr *tools.ExecApprovalManager, mcpMgr *mcpbridge.Manager, sandboxMgr sandbox.Manager, browserMgr *browser.Manager, webFetchTool *tools.WebFetchTool, ttsTool *tools.TtsTool, permPE *permissions.PolicyEngine, toolPE *tools.PolicyEngine, dataDir string, agentCfg config.AgentDefaults, ) { // Create tool registry with all tools toolsReg = tools.NewRegistry() agentCfg = cfg.ResolveAgent("default") // Sandbox manager (optional — routes tools through Docker containers) if sbCfg := cfg.Agents.Defaults.Sandbox; sbCfg != nil && sbCfg.Mode != "" && sbCfg.Mode != "off" { if err := sandbox.CheckDockerAvailable(context.Background()); err != nil { slog.Warn("sandbox disabled: Docker not available", "configured_mode", sbCfg.Mode, "error", err, ) } else { resolved := sbCfg.ToSandboxConfig() sandboxMgr = sandbox.NewDockerManager(resolved) slog.Info("sandbox enabled", "mode", string(resolved.Mode), "image", resolved.Image, "scope", string(resolved.Scope)) } } // Register file tools + exec tool (with sandbox routing via FsBridge if enabled) if sandboxMgr != nil { toolsReg.Register(tools.NewSandboxedReadFileTool(workspace, agentCfg.RestrictToWorkspace, sandboxMgr)) toolsReg.Register(tools.NewSandboxedWriteFileTool(workspace, agentCfg.RestrictToWorkspace, sandboxMgr)) toolsReg.Register(tools.NewSandboxedListFilesTool(workspace, agentCfg.RestrictToWorkspace, sandboxMgr)) toolsReg.Register(tools.NewSandboxedEditTool(workspace, agentCfg.RestrictToWorkspace, sandboxMgr)) toolsReg.Register(tools.NewSandboxedExecTool(workspace, agentCfg.RestrictToWorkspace, sandboxMgr)) } else { toolsReg.Register(tools.NewReadFileTool(workspace, agentCfg.RestrictToWorkspace)) toolsReg.Register(tools.NewWriteFileTool(workspace, agentCfg.RestrictToWorkspace)) toolsReg.Register(tools.NewListFilesTool(workspace, agentCfg.RestrictToWorkspace)) toolsReg.Register(tools.NewEditTool(workspace, agentCfg.RestrictToWorkspace)) toolsReg.Register(tools.NewExecTool(workspace, agentCfg.RestrictToWorkspace)) } // Memory tools — PG-backed; always registered (PG memory is always available) toolsReg.Register(tools.NewMemorySearchTool()) toolsReg.Register(tools.NewMemoryGetTool()) toolsReg.Register(tools.NewKnowledgeGraphSearchTool()) slog.Info("memory + knowledge graph tools registered (PG-backed)") // Browser automation tool if cfg.Tools.Browser.Enabled { var opts []browser.Option if cfg.Tools.Browser.RemoteURL != "" { opts = append(opts, browser.WithRemoteURL(cfg.Tools.Browser.RemoteURL)) slog.Info("browser tool enabled", "remote", cfg.Tools.Browser.RemoteURL) } else { opts = append(opts, browser.WithHeadless(cfg.Tools.Browser.Headless)) slog.Info("browser tool enabled", "headless", cfg.Tools.Browser.Headless) } browserMgr = browser.New(opts...) toolsReg.Register(browser.NewBrowserTool(browserMgr)) } // Web tools (web_search + web_fetch) webSearchTool := tools.NewWebSearchTool(tools.WebSearchConfig{ BraveEnabled: cfg.Tools.Web.Brave.Enabled, BraveAPIKey: cfg.Tools.Web.Brave.APIKey, DDGEnabled: cfg.Tools.Web.DuckDuckGo.Enabled, }) if webSearchTool != nil { toolsReg.Register(webSearchTool) slog.Info("web_search tool enabled") } webFetchTool = tools.NewWebFetchTool(tools.WebFetchConfig{ Policy: cfg.Tools.WebFetch.Policy, AllowedDomains: cfg.Tools.WebFetch.AllowedDomains, BlockedDomains: cfg.Tools.WebFetch.BlockedDomains, }) toolsReg.Register(webFetchTool) slog.Info("web_fetch tool enabled", "policy", cfg.Tools.WebFetch.Policy, "blocked", len(cfg.Tools.WebFetch.BlockedDomains)) // Vision fallback tool (for non-vision providers like MiniMax) toolsReg.Register(tools.NewReadImageTool(providerRegistry)) toolsReg.Register(tools.NewCreateImageTool(providerRegistry)) // Audio generation tool (MiniMax music + ElevenLabs sound effects) toolsReg.Register(tools.NewCreateAudioTool(providerRegistry, cfg.Tts.ElevenLabs.APIKey, cfg.Tts.ElevenLabs.BaseURL)) // TTS (text-to-speech) system — always create TtsTool so config reload can populate it later ttsMgr := setupTTS(cfg) if ttsMgr == nil { ttsMgr = tts.NewManager(tts.ManagerConfig{}) } ttsTool = tools.NewTtsTool(ttsMgr) toolsReg.Register(ttsTool) if ttsMgr.HasProviders() { slog.Info("tts enabled", "provider", ttsMgr.PrimaryProvider(), "auto", string(ttsMgr.AutoMode())) } // Tool rate limiting (per session, sliding window) if cfg.Tools.RateLimitPerHour > 0 { toolsReg.SetRateLimiter(tools.NewToolRateLimiter(cfg.Tools.RateLimitPerHour)) slog.Info("tool rate limiting enabled", "per_hour", cfg.Tools.RateLimitPerHour) } // Credential scrubbing (enabled by default, can be disabled via config) if cfg.Tools.ScrubCredentials != nil && !*cfg.Tools.ScrubCredentials { toolsReg.SetScrubbing(false) slog.Info("credential scrubbing disabled") } // MCP servers (config-based: shared across all agents) if len(cfg.Tools.McpServers) > 0 { mcpMgr = mcpbridge.NewManager(toolsReg, mcpbridge.WithConfigs(cfg.Tools.McpServers)) if err := mcpMgr.Start(context.Background()); err != nil { slog.Warn("mcp.startup_errors", "error", err) } slog.Info("MCP servers initialized", "configured", len(cfg.Tools.McpServers), "tools", len(mcpMgr.ToolNames())) } // Exec approval system — always active (deny patterns + safe bins + configurable ask mode) { approvalCfg := tools.DefaultExecApprovalConfig() // Override from user config (backward compat: explicit values take precedence) if eaCfg := cfg.Tools.ExecApproval; eaCfg.Security != "" { approvalCfg.Security = tools.ExecSecurity(eaCfg.Security) } if eaCfg := cfg.Tools.ExecApproval; eaCfg.Ask != "" { approvalCfg.Ask = tools.ExecAskMode(eaCfg.Ask) } if len(cfg.Tools.ExecApproval.Allowlist) > 0 { approvalCfg.Allowlist = cfg.Tools.ExecApproval.Allowlist } execApprovalMgr = tools.NewExecApprovalManager(approvalCfg) // Wire approval to exec tools in the registry if execTool, ok := toolsReg.Get("exec"); ok { if aa, ok := execTool.(tools.ApprovalAware); ok { aa.SetApprovalManager(execApprovalMgr, "default") } } slog.Info("exec approval enabled", "security", string(approvalCfg.Security), "ask", string(approvalCfg.Ask)) } // --- Enforcement: Policy engines --- // Permission policy engine (role-based RPC access control) permPE = permissions.NewPolicyEngine(cfg.Gateway.OwnerIDs) // Tool policy engine (7-step tool filtering pipeline) toolPE = tools.NewPolicyEngine(&cfg.Tools) // Data directory for Phase 2 services dataDir = cfg.ResolvedDataDir() os.MkdirAll(dataDir, 0755) // Block exec from accessing sensitive directories (data dir, .goclaw, config file). // Prevents `cp /app/data/config.json workspace/` and similar exfiltration. // Exception: .goclaw/skills-store/ is allowed (skills may contain executable scripts). if execTool, ok := toolsReg.Get("exec"); ok { if et, ok := execTool.(*tools.ExecTool); ok { et.DenyPaths(dataDir, ".goclaw/") et.AllowPathExemptions(".goclaw/skills-store/") if cfgPath := os.Getenv("GOCLAW_CONFIG"); cfgPath != "" { et.DenyPaths(cfgPath) } } } // Block filesystem tools from accessing internal system files within the workspace. // Shared-workspace agents have workspace = dataDir root, exposing config.json, // memory.db, .media/, delegate/ etc. via list_files/read_file. // Non-shared agents are already isolated by resolvePath boundary check, but // deny paths add defense-in-depth. internalDenyPaths := []string{ "config.json", "memory.db", "memory.db-wal", "memory.db-shm", "memory/", ".media/", "delegate/", } if rf, ok := toolsReg.Get("read_file"); ok { if t, ok := rf.(*tools.ReadFileTool); ok { t.DenyPaths(internalDenyPaths...) } } if wf, ok := toolsReg.Get("write_file"); ok { if t, ok := wf.(*tools.WriteFileTool); ok { t.DenyPaths(internalDenyPaths...) } } if lf, ok := toolsReg.Get("list_files"); ok { if t, ok := lf.(*tools.ListFilesTool); ok { t.DenyPaths(internalDenyPaths...) } } if ed, ok := toolsReg.Get("edit"); ok { if t, ok := ed.(*tools.EditTool); ok { t.DenyPaths(internalDenyPaths...) } } return } // setupStoresAndTracing creates PG stores, tracing collector, snapshot worker, and wires cron config. // Exits the process on unrecoverable errors (missing DSN, schema mismatch, store creation failure). func setupStoresAndTracing( cfg *config.Config, dataDir string, msgBus *bus.MessageBus, ) (*store.Stores, *tracing.Collector, *tracing.SnapshotWorker) { // --- Store creation (Postgres) --- if cfg.Database.PostgresDSN == "" { slog.Error("GOCLAW_POSTGRES_DSN is required. Set it in your environment or .env.local file.") os.Exit(1) } var traceCollector *tracing.Collector // Schema compatibility check: ensure DB schema matches this binary. if err := checkSchemaOrAutoUpgrade(cfg.Database.PostgresDSN); err != nil { slog.Error("schema compatibility check failed", "error", err) os.Exit(1) } storeCfg := store.StoreConfig{ PostgresDSN: cfg.Database.PostgresDSN, EncryptionKey: os.Getenv("GOCLAW_ENCRYPTION_KEY"), SkillsStorageDir: filepath.Join(dataDir, "skills-store"), } pgStores, pgErr := pg.NewPGStores(storeCfg) if pgErr != nil { slog.Error("failed to create PG stores", "error", pgErr) os.Exit(1) } if pgStores.Tracing != nil { traceCollector = tracing.NewCollector(pgStores.Tracing) traceCollector.OnFlush = func(traceIDs []uuid.UUID) { ids := make([]string, len(traceIDs)) for i, id := range traceIDs { ids[i] = id.String() } msgBus.Broadcast(bus.Event{ Name: protocol.EventTraceUpdated, Payload: map[string]any{"trace_ids": ids}, }) } traceCollector.Start() slog.Info("LLM tracing enabled") } // Start snapshot worker for hourly usage aggregation var snapshotWorker *tracing.SnapshotWorker if pgStores.Snapshots != nil { snapshotWorker = tracing.NewSnapshotWorker(pgStores.DB, pgStores.Snapshots) snapshotWorker.Start() // Backfill historical data in background go func() { count, err := snapshotWorker.Backfill(context.Background()) if err != nil { slog.Warn("snapshot backfill failed", "error", err) } else if count > 0 { slog.Info("snapshot backfill complete", "hours", count) } }() } // Wire cron config from config.json cronRetryCfg := cfg.Cron.ToRetryConfig() // Apply retry config via type assertion on the concrete cron store. pgStores.Cron.SetOnJob(nil) // ensure initialized; actual handler set below _ = cronRetryCfg // config available; pg cron store reads it internally if cfg.Cron.DefaultTimezone != "" { pgStores.Cron.SetDefaultTimezone(cfg.Cron.DefaultTimezone) } // Load secrets from config_secrets table before env overrides. // Precedence: config.json → DB secrets → env vars (highest). if pgStores.ConfigSecrets != nil { if secrets, err := pgStores.ConfigSecrets.GetAll(context.Background()); err == nil && len(secrets) > 0 { cfg.ApplyDBSecrets(secrets) cfg.ApplyEnvOverrides() slog.Info("config secrets loaded from DB", "count", len(secrets)) } } return pgStores, traceCollector, snapshotWorker } // setupMemoryEmbeddings wires embedding provider to PGMemoryStore and triggers backfill. // Per-agent DB config takes priority over config file defaults. func setupMemoryEmbeddings( cfg *config.Config, pgStores *store.Stores, providerRegistry *providers.Registry, ) { // Wire embedding provider to PGMemoryStore so IndexDocument generates vectors. // Per-agent DB config takes priority over config file defaults. if pgStores.Memory != nil { memCfg := cfg.Agents.Defaults.Memory if pgStores.Agents != nil { if defaultAgent, agErr := pgStores.Agents.GetByKey(context.Background(), "default"); agErr == nil { if agentMemCfg := defaultAgent.ParseMemoryConfig(); agentMemCfg != nil { memCfg = agentMemCfg slog.Debug("using per-agent memory config from DB", "agent", defaultAgent.AgentKey) } } } if embProvider := resolveEmbeddingProvider(cfg, memCfg, providerRegistry); embProvider != nil { pgStores.Memory.SetEmbeddingProvider(embProvider) slog.Info("memory embeddings enabled", "provider", embProvider.Name(), "model", embProvider.Model()) // Backfill embeddings for existing chunks that were stored without vectors. type backfiller interface { BackfillEmbeddings(ctx context.Context) (int, error) } if bf, ok := pgStores.Memory.(backfiller); ok { go func() { bgCtx := context.Background() count, err := bf.BackfillEmbeddings(bgCtx) if err != nil { slog.Warn("memory embeddings backfill failed", "error", err) } else if count > 0 { slog.Info("memory embeddings backfill complete", "chunks_updated", count) } }() } } else { slog.Warn("memory embeddings disabled (no API key), chunks stored without vectors") } } } // loadBootstrapFiles loads bootstrap files for the default agent's system prompt from DB. // Seeds if empty; falls back to filesystem as last resort. func loadBootstrapFiles( pgStores *store.Stores, workspace string, agentCfg config.AgentDefaults, ) []bootstrap.ContextFile { // Load bootstrap files for default agent's system prompt from DB. // Seeds if empty; falls back to filesystem as last resort. var contextFiles []bootstrap.ContextFile if pgStores.Agents != nil { bgCtx := context.Background() defaultAgent, agErr := pgStores.Agents.GetByKey(bgCtx, "default") if agErr == nil { dbFiles := bootstrap.LoadFromStore(bgCtx, pgStores.Agents, defaultAgent.ID) if len(dbFiles) > 0 { contextFiles = dbFiles slog.Info("bootstrap loaded from store", "count", len(dbFiles)) } else { // DB empty → seed templates, then load if _, seedErr := bootstrap.SeedToStore(bgCtx, pgStores.Agents, defaultAgent.ID, defaultAgent.AgentType); seedErr != nil { slog.Warn("failed to seed bootstrap to store", "error", seedErr) } else { contextFiles = bootstrap.LoadFromStore(bgCtx, pgStores.Agents, defaultAgent.ID) slog.Info("bootstrap seeded and loaded from store", "count", len(contextFiles)) } } } } if len(contextFiles) == 0 { // DB fallback: load from workspace filesystem rawFiles := bootstrap.LoadWorkspaceFiles(workspace) truncCfg := bootstrap.TruncateConfig{ MaxCharsPerFile: agentCfg.BootstrapMaxChars, TotalMaxChars: agentCfg.BootstrapTotalMaxChars, } if truncCfg.MaxCharsPerFile <= 0 { truncCfg.MaxCharsPerFile = bootstrap.DefaultMaxCharsPerFile } if truncCfg.TotalMaxChars <= 0 { truncCfg.TotalMaxChars = bootstrap.DefaultTotalMaxChars } contextFiles = bootstrap.BuildContextFiles(rawFiles, truncCfg) slog.Info("bootstrap loaded from filesystem", "count", len(contextFiles)) } // Debug: log bootstrap file loading results { var loadedNames []string for _, cf := range contextFiles { loadedNames = append(loadedNames, fmt.Sprintf("%s(%d)", cf.Path, len(cf.Content))) } slog.Info("bootstrap context files", "count", len(contextFiles), "files", loadedNames) } return contextFiles } // setupSkillsSystem creates the skills loader, registers skill tools, wires skills-store, // seeds bundled skills, and enables embedding-based skill search. func setupSkillsSystem( cfg *config.Config, workspace string, dataDir string, pgStores *store.Stores, toolsReg *tools.Registry, providerRegistry *providers.Registry, msgBus *bus.MessageBus, ) (*skills.Loader, *tools.SkillSearchTool, string) { // Skills loader + search tool // Global skills live under ~/.goclaw/skills/ (user-managed), not data/skills/. globalSkillsDir := os.Getenv("GOCLAW_SKILLS_DIR") if globalSkillsDir == "" { globalSkillsDir = filepath.Join(dataDir, "skills") } // Bundled skills: shipped with the Docker image at /app/bundled-skills/. // Lowest priority — managed (skills-store) and user-uploaded skills override these. builtinSkillsDir := os.Getenv("GOCLAW_BUILTIN_SKILLS_DIR") if builtinSkillsDir == "" { builtinSkillsDir = "/app/bundled-skills" } skillsLoader := skills.NewLoader(workspace, globalSkillsDir, builtinSkillsDir) skillSearchTool := tools.NewSkillSearchTool(skillsLoader) toolsReg.Register(skillSearchTool) toolsReg.Register(tools.NewUseSkillTool()) slog.Info("skill_search tool registered", "skills", len(skillsLoader.ListSkills())) // Wire skills-store directory into filesystem loader so agents // can discover uploaded skills in their system prompt and BM25 search index. if pgStores.Skills != nil { storeDirs := pgStores.Skills.Dirs() if len(storeDirs) > 0 { skillsLoader.SetManagedDir(storeDirs[0]) slog.Info("skills-store directory wired into loader", "dir", storeDirs[0]) // Seed system/bundled skills into DB bundledSkillsDir := os.Getenv("GOCLAW_BUNDLED_SKILLS_DIR") if bundledSkillsDir == "" { // Check common locations: Docker default, then local dev for _, candidate := range []string{"bundled-skills", "/app/bundled-skills", "skills"} { if info, err := os.Stat(candidate); err == nil && info.IsDir() { bundledSkillsDir = candidate break } } } if bundledSkillsDir != "" { if pgSkills, ok := pgStores.Skills.(*pg.PGSkillStore); ok { seeder := skills.NewSeeder(bundledSkillsDir, storeDirs[0], pgSkills) seeded, skipped, seededSkills, err := seeder.Seed(context.Background()) if err != nil { slog.Warn("system skills seed failed", "error", err) } else { if seeded > 0 { slog.Info("system skills seeded", "seeded", seeded, "skipped", skipped) } // Check dependencies asynchronously — does not block startup. // Emits WS events per-skill so UI updates in realtime. if len(seededSkills) > 0 { seeder.CheckDepsAsync(seededSkills, msgBus) } } } } } } // Publish skill tool — lets agents register created skills in the database if pgStores.Skills != nil { if pgSkills, ok := pgStores.Skills.(*pg.PGSkillStore); ok { storeDirs := pgStores.Skills.Dirs() if len(storeDirs) > 0 { toolsReg.Register(tools.NewPublishSkillTool(pgSkills, storeDirs[0], skillsLoader)) slog.Info("publish_skill tool registered") toolsReg.Register(tools.NewSkillManageTool(pgSkills, storeDirs[0], skillsLoader)) slog.Info("skill_manage tool registered") } } } // Wire embedding-based skill search + per-agent access filtering if pgStores.Skills != nil { if sas, ok := pgStores.Skills.(store.SkillAccessStore); ok { skillSearchTool.SetSkillAccessStore(sas) } if pgSkills, ok := pgStores.Skills.(*pg.PGSkillStore); ok { memCfg := cfg.Agents.Defaults.Memory if embProvider := resolveEmbeddingProvider(cfg, memCfg, providerRegistry); embProvider != nil { pgSkills.SetEmbeddingProvider(embProvider) skillSearchTool.SetEmbeddingSearcher(pgSkills, embProvider) slog.Info("skill embeddings enabled", "provider", embProvider.Name()) // Backfill embeddings for existing skills go func() { count, err := pgSkills.BackfillSkillEmbeddings(context.Background()) if err != nil { slog.Warn("skill embeddings backfill failed", "error", err) } else if count > 0 { slog.Info("skill embeddings backfill complete", "skills_updated", count) } }() } } } return skillsLoader, skillSearchTool, globalSkillsDir }