From dabb1eaa11fb2d452352bb5b6077e9ba0c83983c Mon Sep 17 00:00:00 2001 From: viettranx Date: Fri, 10 Apr 2026 13:28:44 +0700 Subject: [PATCH] feat(vault): workspace rescan endpoint with symlink-safe walker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add POST /v1/agents/{agentID}/vault/rescan to backfill vault_documents from filesystem. Walks agent workspace, registers missing/changed files, publishes enrichment events for async summarize + embed + link classify. Backend: - SafeWalkWorkspace: symlink-safe walker with exclusion patterns, resource limits (5K files, 500MB, 50MB/file), context deadline - RescanWorkspace: hash-based dedup, path-based scope detection (teams/{id}/ → team scope), idempotent via UpsertDocument ON CONFLICT - Per-agent mutex (409 Conflict for concurrent rescans) - Tenant-scoped workspace resolution (config.TenantWorkspace) - Enrichment worker: semaphore-based parallel summarize (max 3 concurrent) - Media files without summary skip LLM summarize, embed title+path only - DRY: export InferTitle/InferDocType from vault pkg, remove from interceptor - Auto-register text uploads in vault via onTextUploaded callback Frontend: - Rescan button in vault page header (FolderSync icon) - Toast with result counts, 409 warning, error handling - i18n strings for en/vi/zh Security: skip all symlinks, boundary checks, per-file size limit, tenant isolation via server-side workspace resolution. --- cmd/gateway.go | 3 + cmd/gateway_deps.go | 2 + cmd/gateway_http_wiring.go | 2 +- cmd/gateway_managed.go | 10 +- cmd/gateway_vault_wiring.go | 8 +- internal/agent/loop_input_media.go | 28 +++ internal/agent/loop_types.go | 8 + internal/agent/resolver.go | 4 + internal/http/vault_handlers.go | 65 ++++- internal/tools/vault_interceptor.go | 41 +--- internal/vault/enrich_worker.go | 115 ++++++--- internal/vault/rescan.go | 161 ++++++++++++ internal/vault/rescan_test.go | 80 ++++++ internal/vault/safe_walk.go | 225 +++++++++++++++++ internal/vault/safe_walk_test.go | 232 ++++++++++++++++++ ui/web/src/i18n/locales/en/vault.json | 8 + ui/web/src/i18n/locales/vi/vault.json | 8 + ui/web/src/i18n/locales/zh/vault.json | 8 + .../pages/vault/hooks/use-vault-mutations.ts | 49 ++++ ui/web/src/pages/vault/hooks/use-vault.ts | 1 + ui/web/src/pages/vault/vault-page.tsx | 18 +- 21 files changed, 992 insertions(+), 84 deletions(-) create mode 100644 internal/vault/rescan.go create mode 100644 internal/vault/rescan_test.go create mode 100644 internal/vault/safe_walk.go create mode 100644 internal/vault/safe_walk_test.go diff --git a/cmd/gateway.go b/cmd/gateway.go index 501d2080..b78f955c 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -202,6 +202,8 @@ func runGateway() { Provider: consolidationProvider, Model: consolidationProvider.DefaultModel(), Extractor: kgExtractor, + // Per-agent dreaming overrides (MemoryConfig.Dreaming JSONB). + AgentStore: pgStores.Agents, }) defer cleanupConsolidation() slog.Info("consolidation pipeline registered") @@ -301,6 +303,7 @@ func runGateway() { skillsLoader: skillsLoader, workspace: workspace, dataDir: dataDir, + domainBus: domainBus, } gatewayAddr := loopbackAddr(cfg.Gateway.Host, cfg.Gateway.Port) diff --git a/cmd/gateway_deps.go b/cmd/gateway_deps.go index 17811be4..4573b240 100644 --- a/cmd/gateway_deps.go +++ b/cmd/gateway_deps.go @@ -6,6 +6,7 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/cache" "github.com/nextlevelbuilder/goclaw/internal/channels" "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/eventbus" "github.com/nextlevelbuilder/goclaw/internal/gateway" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/skills" @@ -28,4 +29,5 @@ type gatewayDeps struct { permCache *cache.PermissionCache // nil if no tenant store; closed on shutdown to stop sweep goroutines workspace string dataDir string + domainBus eventbus.DomainEventBus } diff --git a/cmd/gateway_http_wiring.go b/cmd/gateway_http_wiring.go index 5b88d421..1bf0653a 100644 --- a/cmd/gateway_http_wiring.go +++ b/cmd/gateway_http_wiring.go @@ -175,7 +175,7 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer( // V3: Knowledge Vault document API if d.pgStores != nil && d.pgStores.Vault != nil { - d.server.SetVaultHandler(httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams)) + d.server.SetVaultHandler(httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams, d.workspace, d.domainBus)) } // V3: Episodic memory summaries API diff --git a/cmd/gateway_managed.go b/cmd/gateway_managed.go index 444ca785..081e1cbe 100644 --- a/cmd/gateway_managed.go +++ b/cmd/gateway_managed.go @@ -141,6 +141,9 @@ func wireExtras( autoInjector = memorypkg.NewAutoInjector(stores.Episodic, stores.EvolutionMetrics) } + // vaultIntc is set later by wireVault but captured by closure in OnTextUploaded. + var vaultIntc *tools.VaultInterceptor + resolver := agent.NewManagedResolver(agent.ResolverDeps{ AgentStore: stores.Agents, ProviderStore: stores.Providers, @@ -187,6 +190,11 @@ func wireExtras( AutoInjector: autoInjector, EvolutionMetricsStore: stores.EvolutionMetrics, DomainBus: domainBus, + OnTextUploaded: func(ctx context.Context, path, content string) { + if vaultIntc != nil { + vaultIntc.AfterWrite(ctx, path, content) + } + }, OnEvent: func(event agent.AgentEvent) { // Sign /v1/files/ and /v1/media/ URLs in content before delivery. // Sessions store clean paths; signing happens only at delivery time. @@ -342,7 +350,7 @@ func wireExtras( } // Wire vault tools and interceptors (conditional on vault store availability) - wireVault(stores, toolsReg, workspace, domainBus) + vaultIntc = wireVault(stores, toolsReg, workspace, domainBus) // Wire delegate tool for inter-agent delegation via agent_links. if stores.AgentLinks != nil && stores.Agents != nil { diff --git a/cmd/gateway_vault_wiring.go b/cmd/gateway_vault_wiring.go index ce63b342..8b51f486 100644 --- a/cmd/gateway_vault_wiring.go +++ b/cmd/gateway_vault_wiring.go @@ -12,9 +12,12 @@ import ( // wireVault wires Knowledge Vault tools and interceptors into the tool registry. // All wiring is skipped if stores.Vault is nil. // Pattern mirrors wireExtras KG wiring: register tools, set stores, set interceptors. -func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, bus eventbus.DomainEventBus) { +// wireVault wires Knowledge Vault tools and interceptors into the tool registry. +// Returns the shared VaultInterceptor for use by other subsystems (e.g. agent upload hook). +// Returns nil if stores.Vault is nil. +func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, bus eventbus.DomainEventBus) *tools.VaultInterceptor { if stores.Vault == nil { - return + return nil } // Register vault tools — these are always available when vault store is present. @@ -71,4 +74,5 @@ func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, } slog.Info("vault tools registered", "tools", "vault_search,create_image,create_video,create_audio,tts,edit") + return vaultIntc } diff --git a/internal/agent/loop_input_media.go b/internal/agent/loop_input_media.go index 75b5d56c..3797fc9a 100644 --- a/internal/agent/loop_input_media.go +++ b/internal/agent/loop_input_media.go @@ -3,12 +3,28 @@ package agent import ( "context" "log/slog" + "os" + "strings" "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/tools" ) +// isTextMime returns true for MIME types representing human-readable text content. +// Covers text/* family plus common application/* types that are text-based. +func isTextMime(mime string) bool { + if strings.HasPrefix(mime, "text/") { + return true + } + switch mime { + case "application/json", "application/xml", "application/yaml", + "application/x-yaml", "application/javascript": + return true + } + return false +} + // collectRefsByKind gathers MediaRefs of a given kind from message history // (reverse order) and current-turn refs. Historical first, current last. func collectRefsByKind(messages []providers.Message, currentRefs []providers.MediaRef, kind string) []providers.MediaRef { @@ -47,6 +63,18 @@ func (l *Loop) enrichInputMedia(ctx context.Context, req *RunRequest, messages [ var mediaRefs []providers.MediaRef if len(req.Media) > 0 { mediaRefs = l.persistMedia(req.SessionKey, req.Media, tools.ToolWorkspaceFromCtx(ctx)) + + // Register persisted text uploads in vault (async, non-blocking). + if l.onTextUploaded != nil { + for _, ref := range mediaRefs { + if ref.Path != "" && isTextMime(ref.MimeType) { + if content, err := os.ReadFile(ref.Path); err == nil { + go l.onTextUploaded(context.WithoutCancel(ctx), ref.Path, string(content)) + } + } + } + } + // Load current-turn images from persisted refs (Path is always set for new uploads). var imageFiles []bus.MediaFile for _, ref := range mediaRefs { diff --git a/internal/agent/loop_types.go b/internal/agent/loop_types.go index 88f5f200..acef5aeb 100644 --- a/internal/agent/loop_types.go +++ b/internal/agent/loop_types.go @@ -189,6 +189,10 @@ type Loop struct { // Secure CLI store for credentialed exec context injection secureCLIStore store.SecureCLIStore + // Vault hook: called when a text file is persisted from user upload. + // Enables vault registration without agent package importing vault. + onTextUploaded func(ctx context.Context, path, content string) + // Persistent media storage for cross-turn image/document access mediaStore *media.Store @@ -347,6 +351,9 @@ type LoopConfig struct { // Secure CLI store for credentialed exec context injection SecureCLIStore store.SecureCLIStore + // Vault hook: called asynchronously when a text file is persisted from user upload. + OnTextUploaded func(ctx context.Context, path, content string) + // Persistent media storage for cross-turn image/document access MediaStore *media.Store @@ -471,6 +478,7 @@ func NewLoop(cfg LoopConfig) *Loop { configPermStore: cfg.ConfigPermStore, teamStore: cfg.TeamStore, secureCLIStore: cfg.SecureCLIStore, + onTextUploaded: cfg.OnTextUploaded, mediaStore: cfg.MediaStore, modelPricing: cfg.ModelPricing, budgetMonthlyCents: cfg.BudgetMonthlyCents, diff --git a/internal/agent/resolver.go b/internal/agent/resolver.go index 195ab9a0..fbe49d57 100644 --- a/internal/agent/resolver.go +++ b/internal/agent/resolver.go @@ -117,6 +117,9 @@ type ResolverDeps struct { // V3 domain event bus for consolidation pipeline (nil = disabled) DomainBus eventbus.DomainEventBus + + // Vault hook: called when a text file is uploaded by user (nil = no vault registration) + OnTextUploaded func(ctx context.Context, path, content string) } // NewManagedResolver creates a ResolverFunc that builds Loops from DB agent data. @@ -461,6 +464,7 @@ func NewManagedResolver(deps ResolverDeps) ResolverFunc { ConfigPermStore: deps.ConfigPermStore, TeamStore: deps.TeamStore, SecureCLIStore: deps.SecureCLIStore, + OnTextUploaded: deps.OnTextUploaded, MediaStore: deps.MediaStore, ModelPricing: deps.ModelPricing, BudgetMonthlyCents: derefInt(ag.BudgetMonthlyCents), diff --git a/internal/http/vault_handlers.go b/internal/http/vault_handlers.go index b5fe2a7c..bcdef4ad 100644 --- a/internal/http/vault_handlers.go +++ b/internal/http/vault_handlers.go @@ -4,11 +4,17 @@ import ( "context" "log/slog" "net/http" + "path/filepath" "strconv" "strings" + "sync" + "time" "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/eventbus" "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/vault" ) // vaultDocListResponse wraps the document list with total count for pagination. @@ -21,10 +27,13 @@ type vaultDocListResponse struct { type VaultHandler struct { store store.VaultStore teamAccess store.TeamAccessStore // nil = skip team membership validation (e.g. lite edition) + workspace string + eventBus eventbus.DomainEventBus + rescanMu sync.Map // key: agentID → struct{}, per-agent concurrency guard } -func NewVaultHandler(s store.VaultStore, ta store.TeamAccessStore) *VaultHandler { - return &VaultHandler{store: s, teamAccess: ta} +func NewVaultHandler(s store.VaultStore, ta store.TeamAccessStore, workspace string, bus eventbus.DomainEventBus) *VaultHandler { + return &VaultHandler{store: s, teamAccess: ta, workspace: workspace, eventBus: bus} } // validateTeamMembership checks that the requesting user belongs to the given team. @@ -91,6 +100,7 @@ func (h *VaultHandler) RegisterRoutes(mux *http.ServeMux) { mux.HandleFunc("POST /v1/agents/{agentID}/vault/documents", h.auth(h.handleCreateDocument)) mux.HandleFunc("PUT /v1/agents/{agentID}/vault/documents/{docID}", h.auth(h.handleUpdateDocument)) mux.HandleFunc("DELETE /v1/agents/{agentID}/vault/documents/{docID}", h.auth(h.handleDeleteDocument)) + mux.HandleFunc("POST /v1/agents/{agentID}/vault/rescan", h.auth(h.handleRescan)) mux.HandleFunc("POST /v1/agents/{agentID}/vault/search", h.auth(h.handleSearch)) mux.HandleFunc("GET /v1/agents/{agentID}/vault/documents/{docID}/links", h.auth(h.handleGetLinks)) mux.HandleFunc("POST /v1/agents/{agentID}/vault/links", h.auth(h.handleCreateLink)) @@ -554,6 +564,57 @@ func (h *VaultHandler) handleDeleteLink(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNoContent) } +// handleRescan walks agent workspace and registers missing/changed files in vault. +func (h *VaultHandler) handleRescan(w http.ResponseWriter, r *http.Request) { + agentID := r.PathValue("agentID") + if _, err := uuid.Parse(agentID); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid agent_id"}) + return + } + + // Per-agent concurrency guard: only one rescan at a time. + if _, loaded := h.rescanMu.LoadOrStore(agentID, struct{}{}); loaded { + writeJSON(w, http.StatusConflict, map[string]string{"error": "rescan already in progress"}) + return + } + defer h.rescanMu.Delete(agentID) + + // Resolve workspace path for this agent. + wsPath := h.resolveAgentWorkspace(r.Context(), agentID) + if wsPath == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "workspace not available"}) + return + } + + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Minute) + defer cancel() + + result, err := vault.RescanWorkspace(ctx, vault.RescanParams{ + TenantID: store.TenantIDFromContext(r.Context()).String(), + AgentID: agentID, + Workspace: wsPath, + }, h.store, h.eventBus) + if err != nil { + slog.Warn("vault.rescan failed", "agent", agentID, "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + + writeJSON(w, http.StatusOK, result) +} + +// resolveAgentWorkspace returns the filesystem path to an agent's workspace. +// Scopes by tenant to prevent cross-tenant file access. +func (h *VaultHandler) resolveAgentWorkspace(ctx context.Context, agentID string) string { + if h.workspace == "" { + return "" + } + tenantID := store.TenantIDFromContext(ctx) + slug := store.TenantSlugFromContext(ctx) + ws := config.TenantWorkspace(h.workspace, tenantID, slug) + return filepath.Join(ws, agentID) +} + var allowedDocTypes = map[string]bool{"context": true, "memory": true, "note": true, "skill": true, "episodic": true, "media": true} var allowedScopes = map[string]bool{"personal": true, "team": true, "shared": true} diff --git a/internal/tools/vault_interceptor.go b/internal/tools/vault_interceptor.go index fc0746c2..246d8b02 100644 --- a/internal/tools/vault_interceptor.go +++ b/internal/tools/vault_interceptor.go @@ -56,8 +56,8 @@ func (v *VaultInterceptor) AfterWrite(ctx context.Context, resolvedPath, content } hash := vault.ContentHash([]byte(content)) - title := inferVaultTitle(relPath) - docType := inferVaultDocType(relPath) + title := vault.InferTitle(relPath) + docType := vault.InferDocType(relPath) scope, teamID := inferScopeFromContext(ctx) doc := &store.VaultDocument{ @@ -123,7 +123,7 @@ func (v *VaultInterceptor) AfterWriteMedia(ctx context.Context, resolvedPath, su return } - title := inferVaultTitle(relPath) + title := vault.InferTitle(relPath) scope, teamID := inferScopeFromContext(ctx) doc := &store.VaultDocument{ @@ -199,38 +199,3 @@ func (v *VaultInterceptor) BeforeRead(ctx context.Context, resolvedPath string) } } -// inferVaultTitle extracts a human-readable title from a file path. -func inferVaultTitle(relPath string) string { - base := filepath.Base(relPath) - ext := filepath.Ext(base) - return strings.TrimSuffix(base, ext) -} - -// inferVaultDocType guesses doc_type from path conventions. -// Media extension check runs first — doc_type describes content format, not location. -func inferVaultDocType(relPath string) string { - lower := strings.ToLower(relPath) - ext := strings.ToLower(filepath.Ext(relPath)) - - // Media types (images, video, audio) - switch ext { - case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp", - ".mp4", ".webm", ".mov", ".avi", ".mkv", - ".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a": - return "media" - } - - // Path-based inference - switch { - case strings.HasPrefix(lower, "memory/"): - return "memory" - case strings.Contains(lower, "soul.md") || strings.Contains(lower, "identity.md") || strings.Contains(lower, "agents.md"): - return "context" - case strings.HasPrefix(lower, "skills/") || strings.HasSuffix(lower, "skill.md"): - return "skill" - case strings.HasPrefix(lower, "episodic/"): - return "episodic" - default: - return "note" - } -} diff --git a/internal/vault/enrich_worker.go b/internal/vault/enrich_worker.go index 6fcbe269..25c567cd 100644 --- a/internal/vault/enrich_worker.go +++ b/internal/vault/enrich_worker.go @@ -12,6 +12,7 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/eventbus" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/store" + "golang.org/x/sync/semaphore" ) const ( @@ -20,6 +21,7 @@ const ( enrichLLMTimeout = 5 * time.Minute enrichSimilarityLimit = 10 enrichSimilarityMin = 0.7 + enrichMaxConcurrent = 3 // max concurrent LLM summarize calls ) // EnrichWorkerDeps bundles dependencies for the vault enrichment worker. @@ -38,6 +40,7 @@ func RegisterEnrichWorker(deps EnrichWorkerDeps) func() { provider: deps.Provider, model: deps.Model, dedup: make(map[string]string), + sem: semaphore.NewWeighted(enrichMaxConcurrent), } return deps.EventBus.Subscribe(eventbus.EventVaultDocUpserted, w.Handle) } @@ -53,6 +56,7 @@ type enrichWorker struct { // Bounded dedup: docID → content_hash. Prevents re-processing unchanged files. dedupMu sync.Mutex dedup map[string]string + sem *semaphore.Weighted // limits concurrent LLM summarize calls } // Handle is the EventBus handler for vault.doc_upserted events. @@ -96,10 +100,15 @@ func (w *enrichWorker) processBatch(ctx context.Context, key string) { continue } - var results []enriched + // Phase 1 — Summarize (parallel, bounded by semaphore). + var ( + mu sync.Mutex + results []enriched + wg sync.WaitGroup + ) for _, item := range items { - // Dedup check. + // Pre-check dedup before spawning goroutine (cheap). w.dedupMu.Lock() if prev, exists := w.dedup[item.DocID]; exists && prev == item.ContentHash { w.dedupMu.Unlock() @@ -107,44 +116,22 @@ func (w *enrichWorker) processBatch(ctx context.Context, key string) { } w.dedupMu.Unlock() - // Check if doc already has a summary (e.g., media with caption). - existing, err := w.vault.GetDocumentByID(ctx, item.TenantID, item.DocID) - if err != nil { - slog.Warn("vault.enrich: get_doc", "doc", item.DocID, "err", err) - continue - } - if existing != nil && existing.Summary != "" { - // Already has summary — skip LLM, still embed+link. - results = append(results, enriched{payload: item, summary: existing.Summary}) - continue - } + wg.Add(1) + go func(it eventbus.VaultDocUpsertedPayload) { + defer wg.Done() + if err := w.sem.Acquire(ctx, 1); err != nil { + return // context cancelled + } + defer w.sem.Release(1) - // Read file content from disk. - fullPath := filepath.Join(item.Workspace, item.Path) - content, err := os.ReadFile(fullPath) - if err != nil { - slog.Warn("vault.enrich: read_file", "path", item.Path, "err", err) - continue // file deleted or moved — don't record in dedup - } - - // UTF-8 safe truncation. - runes := []rune(string(content)) - if len(runes) > enrichContentMaxRunes { - runes = runes[:enrichContentMaxRunes] - } - text := string(runes) - - // LLM summarize. - sctx, cancel := context.WithTimeout(ctx, enrichLLMTimeout) - summary, err := w.summarize(sctx, item.Path, text) - cancel() - if err != nil { - slog.Warn("vault.enrich: summarize", "path", item.Path, "err", err) - continue // don't record in dedup — allow retry on next write - } - - results = append(results, enriched{payload: item, summary: summary}) + if r, ok := w.summarizeItem(ctx, it); ok { + mu.Lock() + results = append(results, r) + mu.Unlock() + } + }(item) } + wg.Wait() // Phase 2 — Embed: update summary + embed for all results. // Do NOT record dedup here (moved to Phase 4 after classify). @@ -177,6 +164,58 @@ func (w *enrichWorker) processBatch(ctx context.Context, key string) { } } +// summarizeItem handles dedup check, file read, and LLM summarize for one item. +// Returns (enriched, true) on success, (zero, false) on skip/error. +func (w *enrichWorker) summarizeItem(ctx context.Context, item eventbus.VaultDocUpsertedPayload) (enriched, bool) { + // Dedup re-check (another goroutine may have processed same docID). + w.dedupMu.Lock() + if prev, exists := w.dedup[item.DocID]; exists && prev == item.ContentHash { + w.dedupMu.Unlock() + return enriched{}, false + } + w.dedupMu.Unlock() + + // Check if doc already has a summary (e.g., media with caption). + existing, err := w.vault.GetDocumentByID(ctx, item.TenantID, item.DocID) + if err != nil { + slog.Warn("vault.enrich: get_doc", "doc", item.DocID, "err", err) + return enriched{}, false + } + if existing != nil && existing.Summary != "" { + return enriched{payload: item, summary: existing.Summary}, true + } + + // Media files without summary: skip LLM summarize (binary content is not text). + // Still proceed to embed+link using title+path only. + if existing != nil && existing.DocType == "media" { + return enriched{payload: item, summary: ""}, true + } + + // Read file content from disk. + fullPath := filepath.Join(item.Workspace, item.Path) + content, err := os.ReadFile(fullPath) + if err != nil { + slog.Warn("vault.enrich: read_file", "path", item.Path, "err", err) + return enriched{}, false + } + + // UTF-8 safe truncation. + runes := []rune(string(content)) + if len(runes) > enrichContentMaxRunes { + runes = runes[:enrichContentMaxRunes] + } + + sctx, cancel := context.WithTimeout(ctx, enrichLLMTimeout) + summary, err := w.summarize(sctx, item.Path, string(runes)) + cancel() + if err != nil { + slog.Warn("vault.enrich: summarize", "path", item.Path, "err", err) + return enriched{}, false + } + + return enriched{payload: item, summary: summary}, true +} + const vaultSummarizePrompt = `Summarize this document in 2-3 sentences. Focus on: - Main topic and purpose - Key concepts, entities, or decisions diff --git a/internal/vault/rescan.go b/internal/vault/rescan.go new file mode 100644 index 00000000..713cc481 --- /dev/null +++ b/internal/vault/rescan.go @@ -0,0 +1,161 @@ +package vault + +import ( + "context" + "log/slog" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/eventbus" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// RescanParams holds input for workspace rescan. +type RescanParams struct { + TenantID string + AgentID string + Workspace string // absolute path to agent's workspace root +} + +// RescanResult holds the outcome of a workspace rescan. +type RescanResult struct { + Scanned int `json:"scanned"` + New int `json:"new"` + Updated int `json:"updated"` + Unchanged int `json:"unchanged"` + Skipped int `json:"skipped"` + Errors int `json:"errors"` + Truncated bool `json:"truncated"` +} + +// RescanWorkspace walks the agent workspace and registers missing or changed +// files in vault_documents. Publishes EventVaultDocUpserted for each new or +// updated file so the enrichment worker can process them asynchronously. +func RescanWorkspace(ctx context.Context, params RescanParams, vs store.VaultStore, bus eventbus.DomainEventBus) (*RescanResult, error) { + entries, walkStats, err := SafeWalkWorkspace(ctx, params.Workspace, DefaultWalkOptions()) + if err != nil { + return nil, err + } + + result := &RescanResult{ + Scanned: walkStats.Eligible, + Skipped: walkStats.SkippedExcluded + walkStats.SkippedSymlinks + walkStats.SkippedTooLarge, + Truncated: walkStats.Truncated, + } + + for _, entry := range entries { + hash, hashErr := ContentHashFile(entry.AbsPath) + if hashErr != nil { + result.Errors++ + continue + } + + // Check if document already exists with same hash. + existing, _ := vs.GetDocument(ctx, params.TenantID, params.AgentID, entry.RelPath) + if existing != nil && existing.ContentHash == hash { + result.Unchanged++ + continue + } + + scope, teamID := inferScopeFromPath(entry.RelPath) + doc := &store.VaultDocument{ + TenantID: params.TenantID, + AgentID: params.AgentID, + TeamID: teamID, + Scope: scope, + Path: entry.RelPath, + Title: InferTitle(entry.RelPath), + DocType: InferDocType(entry.RelPath), + ContentHash: hash, + } + + if err := vs.UpsertDocument(ctx, doc); err != nil { + slog.Warn("vault.rescan: upsert", "path", entry.RelPath, "err", err) + result.Errors++ + continue + } + + if existing != nil { + result.Updated++ + } else { + result.New++ + } + + // Publish enrichment event. + if bus != nil { + bus.Publish(eventbus.DomainEvent{ + ID: uuid.Must(uuid.NewV7()).String(), + Type: eventbus.EventVaultDocUpserted, + SourceID: doc.ID + ":" + hash, + TenantID: params.TenantID, + AgentID: params.AgentID, + Timestamp: time.Now(), + Payload: eventbus.VaultDocUpsertedPayload{ + DocID: doc.ID, + TenantID: params.TenantID, + AgentID: params.AgentID, + Path: entry.RelPath, + ContentHash: hash, + Workspace: params.Workspace, + }, + }) + } + } + + slog.Info("vault.rescan", "agent", params.AgentID, + "scanned", result.Scanned, "new", result.New, + "updated", result.Updated, "unchanged", result.Unchanged, + "errors", result.Errors, "truncated", result.Truncated) + + return result, nil +} + +// inferScopeFromPath detects scope and team from workspace-relative path. +// Paths starting with "teams/{id}/" are team-scoped; everything else is personal. +func inferScopeFromPath(relPath string) (scope string, teamID *string) { + if !strings.HasPrefix(relPath, "teams/") { + return "personal", nil + } + rest := relPath[len("teams/"):] + id, _, hasSlash := strings.Cut(rest, "/") + if !hasSlash || id == "" { + return "personal", nil + } + return "team", &id +} + +// InferDocType guesses doc_type from path conventions. +// Exported so both rescan and vault interceptor share the same logic. +func InferDocType(relPath string) string { + lower := strings.ToLower(relPath) + ext := strings.ToLower(filepath.Ext(relPath)) + + switch ext { + case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp", + ".mp4", ".webm", ".mov", ".avi", ".mkv", + ".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a": + return "media" + } + + switch { + case strings.HasPrefix(lower, "memory/"): + return "memory" + case strings.Contains(lower, "soul.md") || strings.Contains(lower, "identity.md") || strings.Contains(lower, "agents.md"): + return "context" + case strings.HasPrefix(lower, "skills/") || strings.HasSuffix(lower, "skill.md"): + return "skill" + case strings.HasPrefix(lower, "episodic/"): + return "episodic" + default: + return "note" + } +} + +// InferTitle extracts a human-readable title from a file path. +// Exported so both rescan and vault interceptor share the same logic. +func InferTitle(relPath string) string { + base := filepath.Base(relPath) + return strings.TrimSuffix(base, filepath.Ext(base)) +} diff --git a/internal/vault/rescan_test.go b/internal/vault/rescan_test.go new file mode 100644 index 00000000..0c2ebbe4 --- /dev/null +++ b/internal/vault/rescan_test.go @@ -0,0 +1,80 @@ +package vault + +import "testing" + +func TestInferScopeFromPath(t *testing.T) { + tests := []struct { + path string + scope string + hasTeam bool + teamID string + }{ + {"notes/doc.md", "personal", false, ""}, + {"web-fetch/page.txt", "personal", false, ""}, + {"report.md", "personal", false, ""}, + {"teams/abc-def-123/report.md", "team", true, "abc-def-123"}, + {"teams/abc-def-123/deep/nested.md", "team", true, "abc-def-123"}, + {"teams/", "personal", false, ""}, // malformed, no team ID + {"teams", "personal", false, ""}, // no slash + {"teamsfoo/bar.md", "personal", false, ""}, // not teams/ prefix + {"telegram/123/teams/x/y.md", "personal", false, ""}, // teams not at root + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + scope, teamID := inferScopeFromPath(tt.path) + if scope != tt.scope { + t.Errorf("scope = %q, want %q", scope, tt.scope) + } + if tt.hasTeam && (teamID == nil || *teamID != tt.teamID) { + t.Errorf("teamID = %v, want %q", teamID, tt.teamID) + } + if !tt.hasTeam && teamID != nil { + t.Errorf("teamID = %v, want nil", teamID) + } + }) + } +} + +func TestInferVaultDocType(t *testing.T) { + tests := []struct { + path string + docType string + }{ + {"screenshot.png", "media"}, + {"photo.jpg", "media"}, + {"video.mp4", "media"}, + {"audio.mp3", "media"}, + {"notes/meeting.md", "note"}, + {"report.txt", "note"}, + {"web-fetch/page.html", "note"}, + {"skills/my-skill/SKILL.md", "skill"}, + {"deep/soul.md", "context"}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + got := InferDocType(tt.path) + if got != tt.docType { + t.Errorf("InferDocType(%q) = %q, want %q", tt.path, got, tt.docType) + } + }) + } +} + +func TestInferTitle(t *testing.T) { + tests := []struct { + path string + title string + }{ + {"report.md", "report"}, + {"notes/meeting-notes.txt", "meeting-notes"}, + {"deep/nested/file.png", "file"}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + got := InferTitle(tt.path) + if got != tt.title { + t.Errorf("InferTitle(%q) = %q, want %q", tt.path, got, tt.title) + } + }) + } +} diff --git a/internal/vault/safe_walk.go b/internal/vault/safe_walk.go new file mode 100644 index 00000000..0fe813ba --- /dev/null +++ b/internal/vault/safe_walk.go @@ -0,0 +1,225 @@ +package vault + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +// WalkEntry represents an eligible file found during workspace walk. +type WalkEntry struct { + RelPath string // workspace-relative, forward-slash separated + AbsPath string // absolute filesystem path + Size int64 // file size in bytes + ModTime time.Time // last modification time +} + +// WalkStats holds metrics from a workspace walk. +type WalkStats struct { + TotalWalked int + Eligible int + SkippedSymlinks int + SkippedExcluded int + SkippedTooLarge int + Truncated bool +} + +// WalkOptions configures resource limits for workspace walking. +type WalkOptions struct { + MaxFiles int // max eligible files to return (0 = unlimited) + MaxTotalBytes int64 // max cumulative file size in bytes (0 = unlimited) + MaxFileBytes int64 // skip individual files larger than this (0 = unlimited) +} + +// DefaultWalkOptions returns safe defaults for production use. +func DefaultWalkOptions() WalkOptions { + return WalkOptions{ + MaxFiles: 5000, + MaxTotalBytes: 500 * 1024 * 1024, // 500MB + MaxFileBytes: 50 * 1024 * 1024, // 50MB per file + } +} + +// SafeWalkWorkspace walks root directory collecting eligible files. +// Symlinks are skipped unconditionally. Excluded paths are filtered. +// Resource limits (file count, total size, context deadline) are enforced. +func SafeWalkWorkspace(ctx context.Context, root string, opts WalkOptions) ([]WalkEntry, WalkStats, error) { + root = filepath.Clean(root) + var ( + entries []WalkEntry + stats WalkStats + totalBytes int64 + walkCount int + ) + + // Early context check before starting walk. + if ctx.Err() != nil { + return nil, stats, ctx.Err() + } + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil // skip unreadable entries + } + + // Check context periodically. + walkCount++ + if walkCount%100 == 0 { + if ctx.Err() != nil { + return ctx.Err() + } + } + + // Skip ALL symlinks unconditionally (both files and dirs). + if d.Type()&os.ModeSymlink != 0 { + stats.SkippedSymlinks++ + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + + // Compute workspace-relative path (forward-slash). + relPath, relErr := filepath.Rel(root, path) + if relErr != nil || relPath == "." { + return nil + } + relPath = filepath.ToSlash(relPath) + + // Boundary check: path must stay inside root. + if strings.HasPrefix(relPath, "..") { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + + // Directory-level exclusions: skip entire subtree. + if d.IsDir() { + if isExcludedDir(relPath) { + stats.SkippedExcluded++ + return filepath.SkipDir + } + return nil + } + + stats.TotalWalked++ + + // File-level exclusions. + if isExcludedPath(relPath) { + stats.SkippedExcluded++ + return nil + } + + // File info for size check. + info, infoErr := d.Info() + if infoErr != nil { + return nil // skip unreadable + } + + // Per-file size limit. + if opts.MaxFileBytes > 0 && info.Size() > opts.MaxFileBytes { + stats.SkippedTooLarge++ + return nil + } + + // Total size limit. + if opts.MaxTotalBytes > 0 && totalBytes+info.Size() > opts.MaxTotalBytes { + stats.Truncated = true + return filepath.SkipAll + } + + // File count limit. + if opts.MaxFiles > 0 && len(entries) >= opts.MaxFiles { + stats.Truncated = true + return filepath.SkipAll + } + + totalBytes += info.Size() + stats.Eligible++ + entries = append(entries, WalkEntry{ + RelPath: relPath, + AbsPath: path, + Size: info.Size(), + ModTime: info.ModTime(), + }) + return nil + }) + + // Context cancellation is expected, not an error for partial results. + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return entries, stats, err + } + return entries, stats, err +} + +// isExcludedDir returns true if an entire directory subtree should be skipped. +func isExcludedDir(relPath string) bool { + first, _, _ := strings.Cut(relPath, "/") + + // Skip memory/ subtree entirely. + if first == "memory" { + return true + } + + // Skip hidden dirs (. prefix) EXCEPT .uploads (user content). + if strings.HasPrefix(first, ".") && first != ".uploads" { + return true + } + + return false +} + +// contextFiles are root-level bootstrap files managed by ContextFileInterceptor. +var contextFiles = map[string]bool{ + "SOUL.md": true, + "IDENTITY.md": true, + "USER.md": true, + "BOOTSTRAP.md": true, + "AGENTS.md": true, + "TOOLS.md": true, + "CAPABILITIES.md": true, + "MEMORY.md": true, + "AGENTS_CORE.md": true, + "AGENTS_TASK.md": true, +} + +// isExcludedPath returns true if a file should be excluded from vault registration. +// Defense-in-depth: also checks parent directory exclusions for callers that bypass dir walk. +func isExcludedPath(relPath string) bool { + // Check first path segment for excluded directories. + first, _, _ := strings.Cut(relPath, "/") + + // memory/ prefix. + if first == "memory" { + return true + } + + // Hidden dirs (. prefix) except .uploads. + if strings.HasPrefix(first, ".") && first != ".uploads" { + return true + } + + base := filepath.Base(relPath) + + // SQLite database files. + if strings.HasSuffix(base, ".db") || strings.HasSuffix(base, ".db-wal") || strings.HasSuffix(base, ".db-shm") { + return true + } + + // Root-level context files only (not nested). + if !strings.Contains(relPath, "/") && contextFiles[base] { + return true + } + + // Hidden files at root level. + if strings.HasPrefix(base, ".") && !strings.Contains(relPath, "/") { + return true + } + + return false +} diff --git a/internal/vault/safe_walk_test.go b/internal/vault/safe_walk_test.go new file mode 100644 index 00000000..e36895d4 --- /dev/null +++ b/internal/vault/safe_walk_test.go @@ -0,0 +1,232 @@ +package vault + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestSafeWalkWorkspace_BasicFiles(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "notes/meeting.md", "hello") + writeFile(t, dir, "report.txt", "world") + writeFile(t, dir, "images/screenshot.png", "png-data") + + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, DefaultWalkOptions()) + if err != nil { + t.Fatal(err) + } + if len(entries) != 3 { + t.Fatalf("got %d entries, want 3", len(entries)) + } + if stats.Eligible != 3 { + t.Errorf("stats.Eligible = %d, want 3", stats.Eligible) + } + if stats.Truncated { + t.Error("unexpected truncation") + } +} + +func TestSafeWalkWorkspace_SkipsSymlinks(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks require elevated privileges on Windows") + } + dir := t.TempDir() + writeFile(t, dir, "real.txt", "data") + // Symlink file + os.Symlink(filepath.Join(dir, "real.txt"), filepath.Join(dir, "link.txt")) + // Symlink dir (pointing outside workspace) + os.Symlink("/tmp", filepath.Join(dir, "escape")) + + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, DefaultWalkOptions()) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("got %d entries, want 1 (only real.txt)", len(entries)) + } + if entries[0].RelPath != "real.txt" { + t.Errorf("entry path = %q, want real.txt", entries[0].RelPath) + } + if stats.SkippedSymlinks != 2 { + t.Errorf("stats.SkippedSymlinks = %d, want 2", stats.SkippedSymlinks) + } +} + +func TestSafeWalkWorkspace_ExcludedPaths(t *testing.T) { + dir := t.TempDir() + // Should be excluded + writeFile(t, dir, "memory/session.json", "x") + writeFile(t, dir, ".hidden/file.txt", "x") + writeFile(t, dir, ".wrangler/config.json", "x") + writeFile(t, dir, ".media/thumb.png", "x") + writeFile(t, dir, "SOUL.md", "x") + writeFile(t, dir, "IDENTITY.md", "x") + writeFile(t, dir, "USER.md", "x") + writeFile(t, dir, "BOOTSTRAP.md", "x") + writeFile(t, dir, "AGENTS.md", "x") + writeFile(t, dir, "TOOLS.md", "x") + writeFile(t, dir, "CAPABILITIES.md", "x") + writeFile(t, dir, "MEMORY.md", "x") + writeFile(t, dir, "data.db", "x") + writeFile(t, dir, "data.db-wal", "x") + writeFile(t, dir, "data.db-shm", "x") + // Should NOT be excluded + writeFile(t, dir, "notes/meeting.md", "x") + writeFile(t, dir, "web-fetch/page.txt", "x") + writeFile(t, dir, ".uploads/photo.jpg", "x") + writeFile(t, dir, "soul-notes.md", "x") + writeFile(t, dir, "deep/SOUL.md", "x") // not root level + writeFile(t, dir, "teams/abc-123/doc.md", "x") + + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, DefaultWalkOptions()) + if err != nil { + t.Fatal(err) + } + if len(entries) != 6 { + names := make([]string, len(entries)) + for i, e := range entries { + names[i] = e.RelPath + } + t.Fatalf("got %d entries %v, want 6 non-excluded", len(entries), names) + } + if stats.SkippedExcluded == 0 { + t.Error("expected some excluded files") + } +} + +func TestSafeWalkWorkspace_MaxFileLimit(t *testing.T) { + dir := t.TempDir() + for i := 0; i < 20; i++ { + writeFile(t, dir, filepath.Join("files", string(rune('a'+i))+".txt"), "data") + } + + opts := DefaultWalkOptions() + opts.MaxFiles = 10 + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, opts) + if err != nil { + t.Fatal(err) + } + if len(entries) > 10 { + t.Fatalf("got %d entries, want <=10", len(entries)) + } + if !stats.Truncated { + t.Error("expected truncated=true") + } +} + +func TestSafeWalkWorkspace_MaxTotalBytes(t *testing.T) { + dir := t.TempDir() + // Create files that exceed total byte limit. + bigContent := make([]byte, 1024) // 1KB each + for i := 0; i < 10; i++ { + writeFile(t, dir, filepath.Join("data", string(rune('a'+i))+".bin"), string(bigContent)) + } + + opts := DefaultWalkOptions() + opts.MaxTotalBytes = 5 * 1024 // 5KB limit + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, opts) + if err != nil { + t.Fatal(err) + } + if len(entries) >= 10 { + t.Fatalf("got %d entries, expected fewer than 10 due to size limit", len(entries)) + } + if !stats.Truncated { + t.Error("expected truncated=true") + } + _ = stats +} + +func TestSafeWalkWorkspace_ContextCancel(t *testing.T) { + dir := t.TempDir() + for i := 0; i < 50; i++ { + writeFile(t, dir, filepath.Join("files", string(rune('a'+i/26))+"_"+string(rune('a'+i%26))+".txt"), "data") + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + _, _, err := SafeWalkWorkspace(ctx, dir, DefaultWalkOptions()) + if err == nil { + t.Error("expected error from cancelled context") + } +} + +func TestSafeWalkWorkspace_PerFileSizeSkip(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "small.txt", "ok") + // Create a file larger than MaxFileBytes + bigContent := make([]byte, 100*1024) // 100KB + writeFile(t, dir, "huge.bin", string(bigContent)) + + opts := DefaultWalkOptions() + opts.MaxFileBytes = 50 * 1024 // 50KB per-file limit + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, opts) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("got %d entries, want 1 (only small.txt)", len(entries)) + } + if stats.SkippedTooLarge != 1 { + t.Errorf("stats.SkippedTooLarge = %d, want 1", stats.SkippedTooLarge) + } +} + +func TestIsExcludedPath(t *testing.T) { + tests := []struct { + path string + excluded bool + }{ + {"memory/session-123.json", true}, + {"memory/deep/nested.md", true}, + {".hidden/file.txt", true}, + {".wrangler/config.json", true}, + {".media/thumb.png", true}, + {"SOUL.md", true}, + {"IDENTITY.md", true}, + {"USER.md", true}, + {"BOOTSTRAP.md", true}, + {"AGENTS.md", true}, + {"TOOLS.md", true}, + {"CAPABILITIES.md", true}, + {"MEMORY.md", true}, + {"AGENTS_CORE.md", true}, + {"AGENTS_TASK.md", true}, + {"data.db", true}, + {"data.db-wal", true}, + {"data.db-shm", true}, + // NOT excluded: + {"notes/meeting.md", false}, + {"web-fetch/page.txt", false}, + {"images/screenshot.png", false}, + {"teams/abc-123/doc.md", false}, + {"soul-notes.md", false}, + {"deep/SOUL.md", false}, // not root-level context file + {".uploads/photo.jpg", false}, + {"report.pdf", false}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + got := isExcludedPath(tt.path) + if got != tt.excluded { + t.Errorf("isExcludedPath(%q) = %v, want %v", tt.path, got, tt.excluded) + } + }) + } +} + +// writeFile creates a file with the given relative path and content inside dir. +func writeFile(t *testing.T, dir, relPath, content string) { + t.Helper() + abs := filepath.Join(dir, relPath) + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(abs, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/ui/web/src/i18n/locales/en/vault.json b/ui/web/src/i18n/locales/en/vault.json index 9e549892..8d6c4599 100644 --- a/ui/web/src/i18n/locales/en/vault.json +++ b/ui/web/src/i18n/locales/en/vault.json @@ -64,6 +64,14 @@ "skill": "Skill", "episodic": "Episodic" }, + "rescanTooltip": "Rescan workspace", + "rescanNew": "{{count}} new", + "rescanUpdated": "{{count}} updated", + "rescanUnchanged": "{{count}} unchanged", + "rescanTruncated": "scan truncated (limit reached)", + "rescanNoFiles": "No new files found", + "rescanBusy": "Rescan already in progress", + "rescanError": "Rescan failed", "toast": { "docCreated": "Document created", "docCreateFailed": "Failed to create document", diff --git a/ui/web/src/i18n/locales/vi/vault.json b/ui/web/src/i18n/locales/vi/vault.json index 14dab426..a839d2a7 100644 --- a/ui/web/src/i18n/locales/vi/vault.json +++ b/ui/web/src/i18n/locales/vi/vault.json @@ -64,6 +64,14 @@ "skill": "Kỹ năng", "episodic": "Giai thoại" }, + "rescanTooltip": "Quét lại workspace", + "rescanNew": "{{count}} mới", + "rescanUpdated": "{{count}} cập nhật", + "rescanUnchanged": "{{count}} không đổi", + "rescanTruncated": "quét bị cắt (đạt giới hạn)", + "rescanNoFiles": "Không tìm thấy file mới", + "rescanBusy": "Đang quét, vui lòng đợi", + "rescanError": "Quét thất bại", "toast": { "docCreated": "Đã tạo tài liệu", "docCreateFailed": "Tạo tài liệu thất bại", diff --git a/ui/web/src/i18n/locales/zh/vault.json b/ui/web/src/i18n/locales/zh/vault.json index 094d74f3..dabf0ba5 100644 --- a/ui/web/src/i18n/locales/zh/vault.json +++ b/ui/web/src/i18n/locales/zh/vault.json @@ -64,6 +64,14 @@ "skill": "技能", "episodic": "情景记忆" }, + "rescanTooltip": "重新扫描工作区", + "rescanNew": "{{count}} 个新文件", + "rescanUpdated": "{{count}} 个已更新", + "rescanUnchanged": "{{count}} 个未变", + "rescanTruncated": "扫描已截断（达到限制）", + "rescanNoFiles": "未发现新文件", + "rescanBusy": "扫描正在进行中", + "rescanError": "扫描失败", "toast": { "docCreated": "文档已创建", "docCreateFailed": "创建文档失败", diff --git a/ui/web/src/pages/vault/hooks/use-vault-mutations.ts b/ui/web/src/pages/vault/hooks/use-vault-mutations.ts index 9f00687b..c5561ab6 100644 --- a/ui/web/src/pages/vault/hooks/use-vault-mutations.ts +++ b/ui/web/src/pages/vault/hooks/use-vault-mutations.ts @@ -1,10 +1,21 @@ import { useCallback } from "react"; +import { useState } from "react"; import { useQueryClient } from "@tanstack/react-query"; import { useHttp } from "@/hooks/use-ws"; import { toast } from "@/stores/use-toast-store"; import i18n from "@/i18n"; import type { VaultDocument, VaultLink } from "@/types/vault"; +interface RescanResult { + scanned: number; + new: number; + updated: number; + unchanged: number; + skipped: number; + errors: number; + truncated: boolean; +} + const VAULT_KEY = "vault"; /** Create a new vault document. */ @@ -96,6 +107,44 @@ export function useCreateLink(agentId: string) { return { create }; } +/** Rescan workspace to sync vault documents from filesystem. */ +export function useRescanWorkspace(agentId: string) { + const http = useHttp(); + const queryClient = useQueryClient(); + const [isPending, setIsPending] = useState(false); + + const rescan = useCallback(async () => { + if (!agentId) return; + setIsPending(true); + try { + const result = await http.post(`/v1/agents/${agentId}/vault/rescan`, {}); + await queryClient.invalidateQueries({ queryKey: [VAULT_KEY] }); + + const parts: string[] = []; + if (result.new > 0) parts.push(i18n.t("vault:rescanNew", { count: result.new })); + if (result.updated > 0) parts.push(i18n.t("vault:rescanUpdated", { count: result.updated })); + if (result.unchanged > 0) parts.push(i18n.t("vault:rescanUnchanged", { count: result.unchanged })); + + const title = parts.length > 0 ? parts.join(", ") : i18n.t("vault:rescanNoFiles"); + const desc = result.truncated ? i18n.t("vault:rescanTruncated") : undefined; + toast.success(title, desc); + return result; + } catch (err) { + const status = (err as { status?: number })?.status; + if (status === 409) { + toast.warning(i18n.t("vault:rescanBusy")); + } else { + toast.error(i18n.t("vault:rescanError"), err instanceof Error ? err.message : ""); + } + throw err; + } finally { + setIsPending(false); + } + }, [http, agentId, queryClient]); + + return { rescan, isPending }; +} + /** Delete a vault link. */ export function useDeleteLink(agentId: string, linkId: string) { const http = useHttp(); diff --git a/ui/web/src/pages/vault/hooks/use-vault.ts b/ui/web/src/pages/vault/hooks/use-vault.ts index 154a2819..9e202c36 100644 --- a/ui/web/src/pages/vault/hooks/use-vault.ts +++ b/ui/web/src/pages/vault/hooks/use-vault.ts @@ -226,4 +226,5 @@ export { useDeleteDocument, useCreateLink, useDeleteLink, + useRescanWorkspace, } from "./use-vault-mutations"; diff --git a/ui/web/src/pages/vault/vault-page.tsx b/ui/web/src/pages/vault/vault-page.tsx index d0e8a3ca..21b8e7ad 100644 --- a/ui/web/src/pages/vault/vault-page.tsx +++ b/ui/web/src/pages/vault/vault-page.tsx @@ -1,12 +1,12 @@ import { useState, useEffect, useMemo, lazy, Suspense } from "react"; import { useTranslation } from "react-i18next"; -import { Search, FileArchive, Plus, PanelLeftOpen } from "lucide-react"; +import { Search, FileArchive, Plus, PanelLeftOpen, FolderSync } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { useAgents } from "@/pages/agents/hooks/use-agents"; import { useTeams } from "@/pages/teams/hooks/use-teams"; import { useIsMobile } from "@/hooks/use-media-query"; -import { useVaultDocuments, useVaultGraphData } from "./hooks/use-vault"; +import { useVaultDocuments, useVaultGraphData, useRescanWorkspace } from "./hooks/use-vault"; import { VaultDocumentSidebar } from "./vault-document-sidebar"; import { VaultSearchDialog } from "./vault-search-dialog"; import { VaultCreateDialog } from "./vault-create-dialog"; @@ -38,6 +38,8 @@ export function VaultPage() { const [sidebarOpen, setSidebarOpen] = useState(false); const [page, setPage] = useState(0); + const { rescan, isPending: rescanPending } = useRescanWorkspace(selectedAgent); + const { documents, total, loading } = useVaultDocuments(selectedAgent, { teamId: selectedTeam || undefined, limit: PAGE_SIZE, @@ -131,6 +133,18 @@ export function VaultPage() { + + + + + + + + {!selectedAgent ? t("selectAgentFirst", "Select an agent first") : t("rescanTooltip", "Rescan workspace")} + +