mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-09-10 02:17:23 +00:00
feat(vault): workspace rescan endpoint with symlink-safe walker
Add POST /v1/agents/{agentID}/vault/rescan to backfill vault_documents
from filesystem. Walks agent workspace, registers missing/changed files,
publishes enrichment events for async summarize + embed + link classify.
Backend:
- SafeWalkWorkspace: symlink-safe walker with exclusion patterns,
resource limits (5K files, 500MB, 50MB/file), context deadline
- RescanWorkspace: hash-based dedup, path-based scope detection
(teams/{id}/ → team scope), idempotent via UpsertDocument ON CONFLICT
- Per-agent mutex (409 Conflict for concurrent rescans)
- Tenant-scoped workspace resolution (config.TenantWorkspace)
- Enrichment worker: semaphore-based parallel summarize (max 3 concurrent)
- Media files without summary skip LLM summarize, embed title+path only
- DRY: export InferTitle/InferDocType from vault pkg, remove from interceptor
- Auto-register text uploads in vault via onTextUploaded callback
Frontend:
- Rescan button in vault page header (FolderSync icon)
- Toast with result counts, 409 warning, error handling
- i18n strings for en/vi/zh
Security: skip all symlinks, boundary checks, per-file size limit,
tenant isolation via server-side workspace resolution.
This commit is contained in:
@@ -202,6 +202,8 @@ func runGateway() {
|
||||
Provider: consolidationProvider,
|
||||
Model: consolidationProvider.DefaultModel(),
|
||||
Extractor: kgExtractor,
|
||||
// Per-agent dreaming overrides (MemoryConfig.Dreaming JSONB).
|
||||
AgentStore: pgStores.Agents,
|
||||
})
|
||||
defer cleanupConsolidation()
|
||||
slog.Info("consolidation pipeline registered")
|
||||
@@ -301,6 +303,7 @@ func runGateway() {
|
||||
skillsLoader: skillsLoader,
|
||||
workspace: workspace,
|
||||
dataDir: dataDir,
|
||||
domainBus: domainBus,
|
||||
}
|
||||
|
||||
gatewayAddr := loopbackAddr(cfg.Gateway.Host, cfg.Gateway.Port)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/cache"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/channels"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/gateway"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/skills"
|
||||
@@ -28,4 +29,5 @@ type gatewayDeps struct {
|
||||
permCache *cache.PermissionCache // nil if no tenant store; closed on shutdown to stop sweep goroutines
|
||||
workspace string
|
||||
dataDir string
|
||||
domainBus eventbus.DomainEventBus
|
||||
}
|
||||
|
||||
@@ -175,7 +175,7 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer(
|
||||
|
||||
// V3: Knowledge Vault document API
|
||||
if d.pgStores != nil && d.pgStores.Vault != nil {
|
||||
d.server.SetVaultHandler(httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams))
|
||||
d.server.SetVaultHandler(httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams, d.workspace, d.domainBus))
|
||||
}
|
||||
|
||||
// V3: Episodic memory summaries API
|
||||
|
||||
@@ -141,6 +141,9 @@ func wireExtras(
|
||||
autoInjector = memorypkg.NewAutoInjector(stores.Episodic, stores.EvolutionMetrics)
|
||||
}
|
||||
|
||||
// vaultIntc is set later by wireVault but captured by closure in OnTextUploaded.
|
||||
var vaultIntc *tools.VaultInterceptor
|
||||
|
||||
resolver := agent.NewManagedResolver(agent.ResolverDeps{
|
||||
AgentStore: stores.Agents,
|
||||
ProviderStore: stores.Providers,
|
||||
@@ -187,6 +190,11 @@ func wireExtras(
|
||||
AutoInjector: autoInjector,
|
||||
EvolutionMetricsStore: stores.EvolutionMetrics,
|
||||
DomainBus: domainBus,
|
||||
OnTextUploaded: func(ctx context.Context, path, content string) {
|
||||
if vaultIntc != nil {
|
||||
vaultIntc.AfterWrite(ctx, path, content)
|
||||
}
|
||||
},
|
||||
OnEvent: func(event agent.AgentEvent) {
|
||||
// Sign /v1/files/ and /v1/media/ URLs in content before delivery.
|
||||
// Sessions store clean paths; signing happens only at delivery time.
|
||||
@@ -342,7 +350,7 @@ func wireExtras(
|
||||
}
|
||||
|
||||
// Wire vault tools and interceptors (conditional on vault store availability)
|
||||
wireVault(stores, toolsReg, workspace, domainBus)
|
||||
vaultIntc = wireVault(stores, toolsReg, workspace, domainBus)
|
||||
|
||||
// Wire delegate tool for inter-agent delegation via agent_links.
|
||||
if stores.AgentLinks != nil && stores.Agents != nil {
|
||||
|
||||
@@ -12,9 +12,12 @@ import (
|
||||
// wireVault wires Knowledge Vault tools and interceptors into the tool registry.
|
||||
// All wiring is skipped if stores.Vault is nil.
|
||||
// Pattern mirrors wireExtras KG wiring: register tools, set stores, set interceptors.
|
||||
func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, bus eventbus.DomainEventBus) {
|
||||
// wireVault wires Knowledge Vault tools and interceptors into the tool registry.
|
||||
// Returns the shared VaultInterceptor for use by other subsystems (e.g. agent upload hook).
|
||||
// Returns nil if stores.Vault is nil.
|
||||
func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, bus eventbus.DomainEventBus) *tools.VaultInterceptor {
|
||||
if stores.Vault == nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
// Register vault tools — these are always available when vault store is present.
|
||||
@@ -71,4 +74,5 @@ func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string,
|
||||
}
|
||||
|
||||
slog.Info("vault tools registered", "tools", "vault_search,create_image,create_video,create_audio,tts,edit")
|
||||
return vaultIntc
|
||||
}
|
||||
|
||||
@@ -3,12 +3,28 @@ package agent
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tools"
|
||||
)
|
||||
|
||||
// isTextMime returns true for MIME types representing human-readable text content.
|
||||
// Covers text/* family plus common application/* types that are text-based.
|
||||
func isTextMime(mime string) bool {
|
||||
if strings.HasPrefix(mime, "text/") {
|
||||
return true
|
||||
}
|
||||
switch mime {
|
||||
case "application/json", "application/xml", "application/yaml",
|
||||
"application/x-yaml", "application/javascript":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// collectRefsByKind gathers MediaRefs of a given kind from message history
|
||||
// (reverse order) and current-turn refs. Historical first, current last.
|
||||
func collectRefsByKind(messages []providers.Message, currentRefs []providers.MediaRef, kind string) []providers.MediaRef {
|
||||
@@ -47,6 +63,18 @@ func (l *Loop) enrichInputMedia(ctx context.Context, req *RunRequest, messages [
|
||||
var mediaRefs []providers.MediaRef
|
||||
if len(req.Media) > 0 {
|
||||
mediaRefs = l.persistMedia(req.SessionKey, req.Media, tools.ToolWorkspaceFromCtx(ctx))
|
||||
|
||||
// Register persisted text uploads in vault (async, non-blocking).
|
||||
if l.onTextUploaded != nil {
|
||||
for _, ref := range mediaRefs {
|
||||
if ref.Path != "" && isTextMime(ref.MimeType) {
|
||||
if content, err := os.ReadFile(ref.Path); err == nil {
|
||||
go l.onTextUploaded(context.WithoutCancel(ctx), ref.Path, string(content))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Load current-turn images from persisted refs (Path is always set for new uploads).
|
||||
var imageFiles []bus.MediaFile
|
||||
for _, ref := range mediaRefs {
|
||||
|
||||
@@ -189,6 +189,10 @@ type Loop struct {
|
||||
// Secure CLI store for credentialed exec context injection
|
||||
secureCLIStore store.SecureCLIStore
|
||||
|
||||
// Vault hook: called when a text file is persisted from user upload.
|
||||
// Enables vault registration without agent package importing vault.
|
||||
onTextUploaded func(ctx context.Context, path, content string)
|
||||
|
||||
// Persistent media storage for cross-turn image/document access
|
||||
mediaStore *media.Store
|
||||
|
||||
@@ -347,6 +351,9 @@ type LoopConfig struct {
|
||||
// Secure CLI store for credentialed exec context injection
|
||||
SecureCLIStore store.SecureCLIStore
|
||||
|
||||
// Vault hook: called asynchronously when a text file is persisted from user upload.
|
||||
OnTextUploaded func(ctx context.Context, path, content string)
|
||||
|
||||
// Persistent media storage for cross-turn image/document access
|
||||
MediaStore *media.Store
|
||||
|
||||
@@ -471,6 +478,7 @@ func NewLoop(cfg LoopConfig) *Loop {
|
||||
configPermStore: cfg.ConfigPermStore,
|
||||
teamStore: cfg.TeamStore,
|
||||
secureCLIStore: cfg.SecureCLIStore,
|
||||
onTextUploaded: cfg.OnTextUploaded,
|
||||
mediaStore: cfg.MediaStore,
|
||||
modelPricing: cfg.ModelPricing,
|
||||
budgetMonthlyCents: cfg.BudgetMonthlyCents,
|
||||
|
||||
@@ -117,6 +117,9 @@ type ResolverDeps struct {
|
||||
|
||||
// V3 domain event bus for consolidation pipeline (nil = disabled)
|
||||
DomainBus eventbus.DomainEventBus
|
||||
|
||||
// Vault hook: called when a text file is uploaded by user (nil = no vault registration)
|
||||
OnTextUploaded func(ctx context.Context, path, content string)
|
||||
}
|
||||
|
||||
// NewManagedResolver creates a ResolverFunc that builds Loops from DB agent data.
|
||||
@@ -461,6 +464,7 @@ func NewManagedResolver(deps ResolverDeps) ResolverFunc {
|
||||
ConfigPermStore: deps.ConfigPermStore,
|
||||
TeamStore: deps.TeamStore,
|
||||
SecureCLIStore: deps.SecureCLIStore,
|
||||
OnTextUploaded: deps.OnTextUploaded,
|
||||
MediaStore: deps.MediaStore,
|
||||
ModelPricing: deps.ModelPricing,
|
||||
BudgetMonthlyCents: derefInt(ag.BudgetMonthlyCents),
|
||||
|
||||
@@ -4,11 +4,17 @@ import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/vault"
|
||||
)
|
||||
|
||||
// vaultDocListResponse wraps the document list with total count for pagination.
|
||||
@@ -21,10 +27,13 @@ type vaultDocListResponse struct {
|
||||
type VaultHandler struct {
|
||||
store store.VaultStore
|
||||
teamAccess store.TeamAccessStore // nil = skip team membership validation (e.g. lite edition)
|
||||
workspace string
|
||||
eventBus eventbus.DomainEventBus
|
||||
rescanMu sync.Map // key: agentID → struct{}, per-agent concurrency guard
|
||||
}
|
||||
|
||||
func NewVaultHandler(s store.VaultStore, ta store.TeamAccessStore) *VaultHandler {
|
||||
return &VaultHandler{store: s, teamAccess: ta}
|
||||
func NewVaultHandler(s store.VaultStore, ta store.TeamAccessStore, workspace string, bus eventbus.DomainEventBus) *VaultHandler {
|
||||
return &VaultHandler{store: s, teamAccess: ta, workspace: workspace, eventBus: bus}
|
||||
}
|
||||
|
||||
// validateTeamMembership checks that the requesting user belongs to the given team.
|
||||
@@ -91,6 +100,7 @@ func (h *VaultHandler) RegisterRoutes(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /v1/agents/{agentID}/vault/documents", h.auth(h.handleCreateDocument))
|
||||
mux.HandleFunc("PUT /v1/agents/{agentID}/vault/documents/{docID}", h.auth(h.handleUpdateDocument))
|
||||
mux.HandleFunc("DELETE /v1/agents/{agentID}/vault/documents/{docID}", h.auth(h.handleDeleteDocument))
|
||||
mux.HandleFunc("POST /v1/agents/{agentID}/vault/rescan", h.auth(h.handleRescan))
|
||||
mux.HandleFunc("POST /v1/agents/{agentID}/vault/search", h.auth(h.handleSearch))
|
||||
mux.HandleFunc("GET /v1/agents/{agentID}/vault/documents/{docID}/links", h.auth(h.handleGetLinks))
|
||||
mux.HandleFunc("POST /v1/agents/{agentID}/vault/links", h.auth(h.handleCreateLink))
|
||||
@@ -554,6 +564,57 @@ func (h *VaultHandler) handleDeleteLink(w http.ResponseWriter, r *http.Request)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleRescan walks agent workspace and registers missing/changed files in vault.
|
||||
func (h *VaultHandler) handleRescan(w http.ResponseWriter, r *http.Request) {
|
||||
agentID := r.PathValue("agentID")
|
||||
if _, err := uuid.Parse(agentID); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid agent_id"})
|
||||
return
|
||||
}
|
||||
|
||||
// Per-agent concurrency guard: only one rescan at a time.
|
||||
if _, loaded := h.rescanMu.LoadOrStore(agentID, struct{}{}); loaded {
|
||||
writeJSON(w, http.StatusConflict, map[string]string{"error": "rescan already in progress"})
|
||||
return
|
||||
}
|
||||
defer h.rescanMu.Delete(agentID)
|
||||
|
||||
// Resolve workspace path for this agent.
|
||||
wsPath := h.resolveAgentWorkspace(r.Context(), agentID)
|
||||
if wsPath == "" {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "workspace not available"})
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
result, err := vault.RescanWorkspace(ctx, vault.RescanParams{
|
||||
TenantID: store.TenantIDFromContext(r.Context()).String(),
|
||||
AgentID: agentID,
|
||||
Workspace: wsPath,
|
||||
}, h.store, h.eventBus)
|
||||
if err != nil {
|
||||
slog.Warn("vault.rescan failed", "agent", agentID, "error", err)
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, result)
|
||||
}
|
||||
|
||||
// resolveAgentWorkspace returns the filesystem path to an agent's workspace.
|
||||
// Scopes by tenant to prevent cross-tenant file access.
|
||||
func (h *VaultHandler) resolveAgentWorkspace(ctx context.Context, agentID string) string {
|
||||
if h.workspace == "" {
|
||||
return ""
|
||||
}
|
||||
tenantID := store.TenantIDFromContext(ctx)
|
||||
slug := store.TenantSlugFromContext(ctx)
|
||||
ws := config.TenantWorkspace(h.workspace, tenantID, slug)
|
||||
return filepath.Join(ws, agentID)
|
||||
}
|
||||
|
||||
var allowedDocTypes = map[string]bool{"context": true, "memory": true, "note": true, "skill": true, "episodic": true, "media": true}
|
||||
var allowedScopes = map[string]bool{"personal": true, "team": true, "shared": true}
|
||||
|
||||
|
||||
@@ -56,8 +56,8 @@ func (v *VaultInterceptor) AfterWrite(ctx context.Context, resolvedPath, content
|
||||
}
|
||||
|
||||
hash := vault.ContentHash([]byte(content))
|
||||
title := inferVaultTitle(relPath)
|
||||
docType := inferVaultDocType(relPath)
|
||||
title := vault.InferTitle(relPath)
|
||||
docType := vault.InferDocType(relPath)
|
||||
scope, teamID := inferScopeFromContext(ctx)
|
||||
|
||||
doc := &store.VaultDocument{
|
||||
@@ -123,7 +123,7 @@ func (v *VaultInterceptor) AfterWriteMedia(ctx context.Context, resolvedPath, su
|
||||
return
|
||||
}
|
||||
|
||||
title := inferVaultTitle(relPath)
|
||||
title := vault.InferTitle(relPath)
|
||||
scope, teamID := inferScopeFromContext(ctx)
|
||||
|
||||
doc := &store.VaultDocument{
|
||||
@@ -199,38 +199,3 @@ func (v *VaultInterceptor) BeforeRead(ctx context.Context, resolvedPath string)
|
||||
}
|
||||
}
|
||||
|
||||
// inferVaultTitle extracts a human-readable title from a file path.
|
||||
func inferVaultTitle(relPath string) string {
|
||||
base := filepath.Base(relPath)
|
||||
ext := filepath.Ext(base)
|
||||
return strings.TrimSuffix(base, ext)
|
||||
}
|
||||
|
||||
// inferVaultDocType guesses doc_type from path conventions.
|
||||
// Media extension check runs first — doc_type describes content format, not location.
|
||||
func inferVaultDocType(relPath string) string {
|
||||
lower := strings.ToLower(relPath)
|
||||
ext := strings.ToLower(filepath.Ext(relPath))
|
||||
|
||||
// Media types (images, video, audio)
|
||||
switch ext {
|
||||
case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp",
|
||||
".mp4", ".webm", ".mov", ".avi", ".mkv",
|
||||
".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a":
|
||||
return "media"
|
||||
}
|
||||
|
||||
// Path-based inference
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "memory/"):
|
||||
return "memory"
|
||||
case strings.Contains(lower, "soul.md") || strings.Contains(lower, "identity.md") || strings.Contains(lower, "agents.md"):
|
||||
return "context"
|
||||
case strings.HasPrefix(lower, "skills/") || strings.HasSuffix(lower, "skill.md"):
|
||||
return "skill"
|
||||
case strings.HasPrefix(lower, "episodic/"):
|
||||
return "episodic"
|
||||
default:
|
||||
return "note"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -20,6 +21,7 @@ const (
|
||||
enrichLLMTimeout = 5 * time.Minute
|
||||
enrichSimilarityLimit = 10
|
||||
enrichSimilarityMin = 0.7
|
||||
enrichMaxConcurrent = 3 // max concurrent LLM summarize calls
|
||||
)
|
||||
|
||||
// EnrichWorkerDeps bundles dependencies for the vault enrichment worker.
|
||||
@@ -38,6 +40,7 @@ func RegisterEnrichWorker(deps EnrichWorkerDeps) func() {
|
||||
provider: deps.Provider,
|
||||
model: deps.Model,
|
||||
dedup: make(map[string]string),
|
||||
sem: semaphore.NewWeighted(enrichMaxConcurrent),
|
||||
}
|
||||
return deps.EventBus.Subscribe(eventbus.EventVaultDocUpserted, w.Handle)
|
||||
}
|
||||
@@ -53,6 +56,7 @@ type enrichWorker struct {
|
||||
// Bounded dedup: docID → content_hash. Prevents re-processing unchanged files.
|
||||
dedupMu sync.Mutex
|
||||
dedup map[string]string
|
||||
sem *semaphore.Weighted // limits concurrent LLM summarize calls
|
||||
}
|
||||
|
||||
// Handle is the EventBus handler for vault.doc_upserted events.
|
||||
@@ -96,10 +100,15 @@ func (w *enrichWorker) processBatch(ctx context.Context, key string) {
|
||||
continue
|
||||
}
|
||||
|
||||
var results []enriched
|
||||
// Phase 1 — Summarize (parallel, bounded by semaphore).
|
||||
var (
|
||||
mu sync.Mutex
|
||||
results []enriched
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
|
||||
for _, item := range items {
|
||||
// Dedup check.
|
||||
// Pre-check dedup before spawning goroutine (cheap).
|
||||
w.dedupMu.Lock()
|
||||
if prev, exists := w.dedup[item.DocID]; exists && prev == item.ContentHash {
|
||||
w.dedupMu.Unlock()
|
||||
@@ -107,44 +116,22 @@ func (w *enrichWorker) processBatch(ctx context.Context, key string) {
|
||||
}
|
||||
w.dedupMu.Unlock()
|
||||
|
||||
// Check if doc already has a summary (e.g., media with caption).
|
||||
existing, err := w.vault.GetDocumentByID(ctx, item.TenantID, item.DocID)
|
||||
if err != nil {
|
||||
slog.Warn("vault.enrich: get_doc", "doc", item.DocID, "err", err)
|
||||
continue
|
||||
}
|
||||
if existing != nil && existing.Summary != "" {
|
||||
// Already has summary — skip LLM, still embed+link.
|
||||
results = append(results, enriched{payload: item, summary: existing.Summary})
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(it eventbus.VaultDocUpsertedPayload) {
|
||||
defer wg.Done()
|
||||
if err := w.sem.Acquire(ctx, 1); err != nil {
|
||||
return // context cancelled
|
||||
}
|
||||
defer w.sem.Release(1)
|
||||
|
||||
// Read file content from disk.
|
||||
fullPath := filepath.Join(item.Workspace, item.Path)
|
||||
content, err := os.ReadFile(fullPath)
|
||||
if err != nil {
|
||||
slog.Warn("vault.enrich: read_file", "path", item.Path, "err", err)
|
||||
continue // file deleted or moved — don't record in dedup
|
||||
}
|
||||
|
||||
// UTF-8 safe truncation.
|
||||
runes := []rune(string(content))
|
||||
if len(runes) > enrichContentMaxRunes {
|
||||
runes = runes[:enrichContentMaxRunes]
|
||||
}
|
||||
text := string(runes)
|
||||
|
||||
// LLM summarize.
|
||||
sctx, cancel := context.WithTimeout(ctx, enrichLLMTimeout)
|
||||
summary, err := w.summarize(sctx, item.Path, text)
|
||||
cancel()
|
||||
if err != nil {
|
||||
slog.Warn("vault.enrich: summarize", "path", item.Path, "err", err)
|
||||
continue // don't record in dedup — allow retry on next write
|
||||
}
|
||||
|
||||
results = append(results, enriched{payload: item, summary: summary})
|
||||
if r, ok := w.summarizeItem(ctx, it); ok {
|
||||
mu.Lock()
|
||||
results = append(results, r)
|
||||
mu.Unlock()
|
||||
}
|
||||
}(item)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Phase 2 — Embed: update summary + embed for all results.
|
||||
// Do NOT record dedup here (moved to Phase 4 after classify).
|
||||
@@ -177,6 +164,58 @@ func (w *enrichWorker) processBatch(ctx context.Context, key string) {
|
||||
}
|
||||
}
|
||||
|
||||
// summarizeItem handles dedup check, file read, and LLM summarize for one item.
|
||||
// Returns (enriched, true) on success, (zero, false) on skip/error.
|
||||
func (w *enrichWorker) summarizeItem(ctx context.Context, item eventbus.VaultDocUpsertedPayload) (enriched, bool) {
|
||||
// Dedup re-check (another goroutine may have processed same docID).
|
||||
w.dedupMu.Lock()
|
||||
if prev, exists := w.dedup[item.DocID]; exists && prev == item.ContentHash {
|
||||
w.dedupMu.Unlock()
|
||||
return enriched{}, false
|
||||
}
|
||||
w.dedupMu.Unlock()
|
||||
|
||||
// Check if doc already has a summary (e.g., media with caption).
|
||||
existing, err := w.vault.GetDocumentByID(ctx, item.TenantID, item.DocID)
|
||||
if err != nil {
|
||||
slog.Warn("vault.enrich: get_doc", "doc", item.DocID, "err", err)
|
||||
return enriched{}, false
|
||||
}
|
||||
if existing != nil && existing.Summary != "" {
|
||||
return enriched{payload: item, summary: existing.Summary}, true
|
||||
}
|
||||
|
||||
// Media files without summary: skip LLM summarize (binary content is not text).
|
||||
// Still proceed to embed+link using title+path only.
|
||||
if existing != nil && existing.DocType == "media" {
|
||||
return enriched{payload: item, summary: ""}, true
|
||||
}
|
||||
|
||||
// Read file content from disk.
|
||||
fullPath := filepath.Join(item.Workspace, item.Path)
|
||||
content, err := os.ReadFile(fullPath)
|
||||
if err != nil {
|
||||
slog.Warn("vault.enrich: read_file", "path", item.Path, "err", err)
|
||||
return enriched{}, false
|
||||
}
|
||||
|
||||
// UTF-8 safe truncation.
|
||||
runes := []rune(string(content))
|
||||
if len(runes) > enrichContentMaxRunes {
|
||||
runes = runes[:enrichContentMaxRunes]
|
||||
}
|
||||
|
||||
sctx, cancel := context.WithTimeout(ctx, enrichLLMTimeout)
|
||||
summary, err := w.summarize(sctx, item.Path, string(runes))
|
||||
cancel()
|
||||
if err != nil {
|
||||
slog.Warn("vault.enrich: summarize", "path", item.Path, "err", err)
|
||||
return enriched{}, false
|
||||
}
|
||||
|
||||
return enriched{payload: item, summary: summary}, true
|
||||
}
|
||||
|
||||
const vaultSummarizePrompt = `Summarize this document in 2-3 sentences. Focus on:
|
||||
- Main topic and purpose
|
||||
- Key concepts, entities, or decisions
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package vault
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
// RescanParams holds input for workspace rescan.
|
||||
type RescanParams struct {
|
||||
TenantID string
|
||||
AgentID string
|
||||
Workspace string // absolute path to agent's workspace root
|
||||
}
|
||||
|
||||
// RescanResult holds the outcome of a workspace rescan.
|
||||
type RescanResult struct {
|
||||
Scanned int `json:"scanned"`
|
||||
New int `json:"new"`
|
||||
Updated int `json:"updated"`
|
||||
Unchanged int `json:"unchanged"`
|
||||
Skipped int `json:"skipped"`
|
||||
Errors int `json:"errors"`
|
||||
Truncated bool `json:"truncated"`
|
||||
}
|
||||
|
||||
// RescanWorkspace walks the agent workspace and registers missing or changed
|
||||
// files in vault_documents. Publishes EventVaultDocUpserted for each new or
|
||||
// updated file so the enrichment worker can process them asynchronously.
|
||||
func RescanWorkspace(ctx context.Context, params RescanParams, vs store.VaultStore, bus eventbus.DomainEventBus) (*RescanResult, error) {
|
||||
entries, walkStats, err := SafeWalkWorkspace(ctx, params.Workspace, DefaultWalkOptions())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := &RescanResult{
|
||||
Scanned: walkStats.Eligible,
|
||||
Skipped: walkStats.SkippedExcluded + walkStats.SkippedSymlinks + walkStats.SkippedTooLarge,
|
||||
Truncated: walkStats.Truncated,
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
hash, hashErr := ContentHashFile(entry.AbsPath)
|
||||
if hashErr != nil {
|
||||
result.Errors++
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if document already exists with same hash.
|
||||
existing, _ := vs.GetDocument(ctx, params.TenantID, params.AgentID, entry.RelPath)
|
||||
if existing != nil && existing.ContentHash == hash {
|
||||
result.Unchanged++
|
||||
continue
|
||||
}
|
||||
|
||||
scope, teamID := inferScopeFromPath(entry.RelPath)
|
||||
doc := &store.VaultDocument{
|
||||
TenantID: params.TenantID,
|
||||
AgentID: params.AgentID,
|
||||
TeamID: teamID,
|
||||
Scope: scope,
|
||||
Path: entry.RelPath,
|
||||
Title: InferTitle(entry.RelPath),
|
||||
DocType: InferDocType(entry.RelPath),
|
||||
ContentHash: hash,
|
||||
}
|
||||
|
||||
if err := vs.UpsertDocument(ctx, doc); err != nil {
|
||||
slog.Warn("vault.rescan: upsert", "path", entry.RelPath, "err", err)
|
||||
result.Errors++
|
||||
continue
|
||||
}
|
||||
|
||||
if existing != nil {
|
||||
result.Updated++
|
||||
} else {
|
||||
result.New++
|
||||
}
|
||||
|
||||
// Publish enrichment event.
|
||||
if bus != nil {
|
||||
bus.Publish(eventbus.DomainEvent{
|
||||
ID: uuid.Must(uuid.NewV7()).String(),
|
||||
Type: eventbus.EventVaultDocUpserted,
|
||||
SourceID: doc.ID + ":" + hash,
|
||||
TenantID: params.TenantID,
|
||||
AgentID: params.AgentID,
|
||||
Timestamp: time.Now(),
|
||||
Payload: eventbus.VaultDocUpsertedPayload{
|
||||
DocID: doc.ID,
|
||||
TenantID: params.TenantID,
|
||||
AgentID: params.AgentID,
|
||||
Path: entry.RelPath,
|
||||
ContentHash: hash,
|
||||
Workspace: params.Workspace,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
slog.Info("vault.rescan", "agent", params.AgentID,
|
||||
"scanned", result.Scanned, "new", result.New,
|
||||
"updated", result.Updated, "unchanged", result.Unchanged,
|
||||
"errors", result.Errors, "truncated", result.Truncated)
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// inferScopeFromPath detects scope and team from workspace-relative path.
|
||||
// Paths starting with "teams/{id}/" are team-scoped; everything else is personal.
|
||||
func inferScopeFromPath(relPath string) (scope string, teamID *string) {
|
||||
if !strings.HasPrefix(relPath, "teams/") {
|
||||
return "personal", nil
|
||||
}
|
||||
rest := relPath[len("teams/"):]
|
||||
id, _, hasSlash := strings.Cut(rest, "/")
|
||||
if !hasSlash || id == "" {
|
||||
return "personal", nil
|
||||
}
|
||||
return "team", &id
|
||||
}
|
||||
|
||||
// InferDocType guesses doc_type from path conventions.
|
||||
// Exported so both rescan and vault interceptor share the same logic.
|
||||
func InferDocType(relPath string) string {
|
||||
lower := strings.ToLower(relPath)
|
||||
ext := strings.ToLower(filepath.Ext(relPath))
|
||||
|
||||
switch ext {
|
||||
case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp",
|
||||
".mp4", ".webm", ".mov", ".avi", ".mkv",
|
||||
".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a":
|
||||
return "media"
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "memory/"):
|
||||
return "memory"
|
||||
case strings.Contains(lower, "soul.md") || strings.Contains(lower, "identity.md") || strings.Contains(lower, "agents.md"):
|
||||
return "context"
|
||||
case strings.HasPrefix(lower, "skills/") || strings.HasSuffix(lower, "skill.md"):
|
||||
return "skill"
|
||||
case strings.HasPrefix(lower, "episodic/"):
|
||||
return "episodic"
|
||||
default:
|
||||
return "note"
|
||||
}
|
||||
}
|
||||
|
||||
// InferTitle extracts a human-readable title from a file path.
|
||||
// Exported so both rescan and vault interceptor share the same logic.
|
||||
func InferTitle(relPath string) string {
|
||||
base := filepath.Base(relPath)
|
||||
return strings.TrimSuffix(base, filepath.Ext(base))
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package vault
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestInferScopeFromPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
path string
|
||||
scope string
|
||||
hasTeam bool
|
||||
teamID string
|
||||
}{
|
||||
{"notes/doc.md", "personal", false, ""},
|
||||
{"web-fetch/page.txt", "personal", false, ""},
|
||||
{"report.md", "personal", false, ""},
|
||||
{"teams/abc-def-123/report.md", "team", true, "abc-def-123"},
|
||||
{"teams/abc-def-123/deep/nested.md", "team", true, "abc-def-123"},
|
||||
{"teams/", "personal", false, ""}, // malformed, no team ID
|
||||
{"teams", "personal", false, ""}, // no slash
|
||||
{"teamsfoo/bar.md", "personal", false, ""}, // not teams/ prefix
|
||||
{"telegram/123/teams/x/y.md", "personal", false, ""}, // teams not at root
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
scope, teamID := inferScopeFromPath(tt.path)
|
||||
if scope != tt.scope {
|
||||
t.Errorf("scope = %q, want %q", scope, tt.scope)
|
||||
}
|
||||
if tt.hasTeam && (teamID == nil || *teamID != tt.teamID) {
|
||||
t.Errorf("teamID = %v, want %q", teamID, tt.teamID)
|
||||
}
|
||||
if !tt.hasTeam && teamID != nil {
|
||||
t.Errorf("teamID = %v, want nil", teamID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInferVaultDocType(t *testing.T) {
|
||||
tests := []struct {
|
||||
path string
|
||||
docType string
|
||||
}{
|
||||
{"screenshot.png", "media"},
|
||||
{"photo.jpg", "media"},
|
||||
{"video.mp4", "media"},
|
||||
{"audio.mp3", "media"},
|
||||
{"notes/meeting.md", "note"},
|
||||
{"report.txt", "note"},
|
||||
{"web-fetch/page.html", "note"},
|
||||
{"skills/my-skill/SKILL.md", "skill"},
|
||||
{"deep/soul.md", "context"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
got := InferDocType(tt.path)
|
||||
if got != tt.docType {
|
||||
t.Errorf("InferDocType(%q) = %q, want %q", tt.path, got, tt.docType)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInferTitle(t *testing.T) {
|
||||
tests := []struct {
|
||||
path string
|
||||
title string
|
||||
}{
|
||||
{"report.md", "report"},
|
||||
{"notes/meeting-notes.txt", "meeting-notes"},
|
||||
{"deep/nested/file.png", "file"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
got := InferTitle(tt.path)
|
||||
if got != tt.title {
|
||||
t.Errorf("InferTitle(%q) = %q, want %q", tt.path, got, tt.title)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package vault
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WalkEntry represents an eligible file found during workspace walk.
|
||||
type WalkEntry struct {
|
||||
RelPath string // workspace-relative, forward-slash separated
|
||||
AbsPath string // absolute filesystem path
|
||||
Size int64 // file size in bytes
|
||||
ModTime time.Time // last modification time
|
||||
}
|
||||
|
||||
// WalkStats holds metrics from a workspace walk.
|
||||
type WalkStats struct {
|
||||
TotalWalked int
|
||||
Eligible int
|
||||
SkippedSymlinks int
|
||||
SkippedExcluded int
|
||||
SkippedTooLarge int
|
||||
Truncated bool
|
||||
}
|
||||
|
||||
// WalkOptions configures resource limits for workspace walking.
|
||||
type WalkOptions struct {
|
||||
MaxFiles int // max eligible files to return (0 = unlimited)
|
||||
MaxTotalBytes int64 // max cumulative file size in bytes (0 = unlimited)
|
||||
MaxFileBytes int64 // skip individual files larger than this (0 = unlimited)
|
||||
}
|
||||
|
||||
// DefaultWalkOptions returns safe defaults for production use.
|
||||
func DefaultWalkOptions() WalkOptions {
|
||||
return WalkOptions{
|
||||
MaxFiles: 5000,
|
||||
MaxTotalBytes: 500 * 1024 * 1024, // 500MB
|
||||
MaxFileBytes: 50 * 1024 * 1024, // 50MB per file
|
||||
}
|
||||
}
|
||||
|
||||
// SafeWalkWorkspace walks root directory collecting eligible files.
|
||||
// Symlinks are skipped unconditionally. Excluded paths are filtered.
|
||||
// Resource limits (file count, total size, context deadline) are enforced.
|
||||
func SafeWalkWorkspace(ctx context.Context, root string, opts WalkOptions) ([]WalkEntry, WalkStats, error) {
|
||||
root = filepath.Clean(root)
|
||||
var (
|
||||
entries []WalkEntry
|
||||
stats WalkStats
|
||||
totalBytes int64
|
||||
walkCount int
|
||||
)
|
||||
|
||||
// Early context check before starting walk.
|
||||
if ctx.Err() != nil {
|
||||
return nil, stats, ctx.Err()
|
||||
}
|
||||
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil // skip unreadable entries
|
||||
}
|
||||
|
||||
// Check context periodically.
|
||||
walkCount++
|
||||
if walkCount%100 == 0 {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Skip ALL symlinks unconditionally (both files and dirs).
|
||||
if d.Type()&os.ModeSymlink != 0 {
|
||||
stats.SkippedSymlinks++
|
||||
if d.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Compute workspace-relative path (forward-slash).
|
||||
relPath, relErr := filepath.Rel(root, path)
|
||||
if relErr != nil || relPath == "." {
|
||||
return nil
|
||||
}
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
// Boundary check: path must stay inside root.
|
||||
if strings.HasPrefix(relPath, "..") {
|
||||
if d.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Directory-level exclusions: skip entire subtree.
|
||||
if d.IsDir() {
|
||||
if isExcludedDir(relPath) {
|
||||
stats.SkippedExcluded++
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
stats.TotalWalked++
|
||||
|
||||
// File-level exclusions.
|
||||
if isExcludedPath(relPath) {
|
||||
stats.SkippedExcluded++
|
||||
return nil
|
||||
}
|
||||
|
||||
// File info for size check.
|
||||
info, infoErr := d.Info()
|
||||
if infoErr != nil {
|
||||
return nil // skip unreadable
|
||||
}
|
||||
|
||||
// Per-file size limit.
|
||||
if opts.MaxFileBytes > 0 && info.Size() > opts.MaxFileBytes {
|
||||
stats.SkippedTooLarge++
|
||||
return nil
|
||||
}
|
||||
|
||||
// Total size limit.
|
||||
if opts.MaxTotalBytes > 0 && totalBytes+info.Size() > opts.MaxTotalBytes {
|
||||
stats.Truncated = true
|
||||
return filepath.SkipAll
|
||||
}
|
||||
|
||||
// File count limit.
|
||||
if opts.MaxFiles > 0 && len(entries) >= opts.MaxFiles {
|
||||
stats.Truncated = true
|
||||
return filepath.SkipAll
|
||||
}
|
||||
|
||||
totalBytes += info.Size()
|
||||
stats.Eligible++
|
||||
entries = append(entries, WalkEntry{
|
||||
RelPath: relPath,
|
||||
AbsPath: path,
|
||||
Size: info.Size(),
|
||||
ModTime: info.ModTime(),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
|
||||
// Context cancellation is expected, not an error for partial results.
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return entries, stats, err
|
||||
}
|
||||
return entries, stats, err
|
||||
}
|
||||
|
||||
// isExcludedDir returns true if an entire directory subtree should be skipped.
|
||||
func isExcludedDir(relPath string) bool {
|
||||
first, _, _ := strings.Cut(relPath, "/")
|
||||
|
||||
// Skip memory/ subtree entirely.
|
||||
if first == "memory" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Skip hidden dirs (. prefix) EXCEPT .uploads (user content).
|
||||
if strings.HasPrefix(first, ".") && first != ".uploads" {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// contextFiles are root-level bootstrap files managed by ContextFileInterceptor.
|
||||
var contextFiles = map[string]bool{
|
||||
"SOUL.md": true,
|
||||
"IDENTITY.md": true,
|
||||
"USER.md": true,
|
||||
"BOOTSTRAP.md": true,
|
||||
"AGENTS.md": true,
|
||||
"TOOLS.md": true,
|
||||
"CAPABILITIES.md": true,
|
||||
"MEMORY.md": true,
|
||||
"AGENTS_CORE.md": true,
|
||||
"AGENTS_TASK.md": true,
|
||||
}
|
||||
|
||||
// isExcludedPath returns true if a file should be excluded from vault registration.
|
||||
// Defense-in-depth: also checks parent directory exclusions for callers that bypass dir walk.
|
||||
func isExcludedPath(relPath string) bool {
|
||||
// Check first path segment for excluded directories.
|
||||
first, _, _ := strings.Cut(relPath, "/")
|
||||
|
||||
// memory/ prefix.
|
||||
if first == "memory" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Hidden dirs (. prefix) except .uploads.
|
||||
if strings.HasPrefix(first, ".") && first != ".uploads" {
|
||||
return true
|
||||
}
|
||||
|
||||
base := filepath.Base(relPath)
|
||||
|
||||
// SQLite database files.
|
||||
if strings.HasSuffix(base, ".db") || strings.HasSuffix(base, ".db-wal") || strings.HasSuffix(base, ".db-shm") {
|
||||
return true
|
||||
}
|
||||
|
||||
// Root-level context files only (not nested).
|
||||
if !strings.Contains(relPath, "/") && contextFiles[base] {
|
||||
return true
|
||||
}
|
||||
|
||||
// Hidden files at root level.
|
||||
if strings.HasPrefix(base, ".") && !strings.Contains(relPath, "/") {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
package vault
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSafeWalkWorkspace_BasicFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "notes/meeting.md", "hello")
|
||||
writeFile(t, dir, "report.txt", "world")
|
||||
writeFile(t, dir, "images/screenshot.png", "png-data")
|
||||
|
||||
entries, stats, err := SafeWalkWorkspace(context.Background(), dir, DefaultWalkOptions())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 3 {
|
||||
t.Fatalf("got %d entries, want 3", len(entries))
|
||||
}
|
||||
if stats.Eligible != 3 {
|
||||
t.Errorf("stats.Eligible = %d, want 3", stats.Eligible)
|
||||
}
|
||||
if stats.Truncated {
|
||||
t.Error("unexpected truncation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeWalkWorkspace_SkipsSymlinks(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("symlinks require elevated privileges on Windows")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "real.txt", "data")
|
||||
// Symlink file
|
||||
os.Symlink(filepath.Join(dir, "real.txt"), filepath.Join(dir, "link.txt"))
|
||||
// Symlink dir (pointing outside workspace)
|
||||
os.Symlink("/tmp", filepath.Join(dir, "escape"))
|
||||
|
||||
entries, stats, err := SafeWalkWorkspace(context.Background(), dir, DefaultWalkOptions())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("got %d entries, want 1 (only real.txt)", len(entries))
|
||||
}
|
||||
if entries[0].RelPath != "real.txt" {
|
||||
t.Errorf("entry path = %q, want real.txt", entries[0].RelPath)
|
||||
}
|
||||
if stats.SkippedSymlinks != 2 {
|
||||
t.Errorf("stats.SkippedSymlinks = %d, want 2", stats.SkippedSymlinks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeWalkWorkspace_ExcludedPaths(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// Should be excluded
|
||||
writeFile(t, dir, "memory/session.json", "x")
|
||||
writeFile(t, dir, ".hidden/file.txt", "x")
|
||||
writeFile(t, dir, ".wrangler/config.json", "x")
|
||||
writeFile(t, dir, ".media/thumb.png", "x")
|
||||
writeFile(t, dir, "SOUL.md", "x")
|
||||
writeFile(t, dir, "IDENTITY.md", "x")
|
||||
writeFile(t, dir, "USER.md", "x")
|
||||
writeFile(t, dir, "BOOTSTRAP.md", "x")
|
||||
writeFile(t, dir, "AGENTS.md", "x")
|
||||
writeFile(t, dir, "TOOLS.md", "x")
|
||||
writeFile(t, dir, "CAPABILITIES.md", "x")
|
||||
writeFile(t, dir, "MEMORY.md", "x")
|
||||
writeFile(t, dir, "data.db", "x")
|
||||
writeFile(t, dir, "data.db-wal", "x")
|
||||
writeFile(t, dir, "data.db-shm", "x")
|
||||
// Should NOT be excluded
|
||||
writeFile(t, dir, "notes/meeting.md", "x")
|
||||
writeFile(t, dir, "web-fetch/page.txt", "x")
|
||||
writeFile(t, dir, ".uploads/photo.jpg", "x")
|
||||
writeFile(t, dir, "soul-notes.md", "x")
|
||||
writeFile(t, dir, "deep/SOUL.md", "x") // not root level
|
||||
writeFile(t, dir, "teams/abc-123/doc.md", "x")
|
||||
|
||||
entries, stats, err := SafeWalkWorkspace(context.Background(), dir, DefaultWalkOptions())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 6 {
|
||||
names := make([]string, len(entries))
|
||||
for i, e := range entries {
|
||||
names[i] = e.RelPath
|
||||
}
|
||||
t.Fatalf("got %d entries %v, want 6 non-excluded", len(entries), names)
|
||||
}
|
||||
if stats.SkippedExcluded == 0 {
|
||||
t.Error("expected some excluded files")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeWalkWorkspace_MaxFileLimit(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for i := 0; i < 20; i++ {
|
||||
writeFile(t, dir, filepath.Join("files", string(rune('a'+i))+".txt"), "data")
|
||||
}
|
||||
|
||||
opts := DefaultWalkOptions()
|
||||
opts.MaxFiles = 10
|
||||
entries, stats, err := SafeWalkWorkspace(context.Background(), dir, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) > 10 {
|
||||
t.Fatalf("got %d entries, want <=10", len(entries))
|
||||
}
|
||||
if !stats.Truncated {
|
||||
t.Error("expected truncated=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeWalkWorkspace_MaxTotalBytes(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// Create files that exceed total byte limit.
|
||||
bigContent := make([]byte, 1024) // 1KB each
|
||||
for i := 0; i < 10; i++ {
|
||||
writeFile(t, dir, filepath.Join("data", string(rune('a'+i))+".bin"), string(bigContent))
|
||||
}
|
||||
|
||||
opts := DefaultWalkOptions()
|
||||
opts.MaxTotalBytes = 5 * 1024 // 5KB limit
|
||||
entries, stats, err := SafeWalkWorkspace(context.Background(), dir, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) >= 10 {
|
||||
t.Fatalf("got %d entries, expected fewer than 10 due to size limit", len(entries))
|
||||
}
|
||||
if !stats.Truncated {
|
||||
t.Error("expected truncated=true")
|
||||
}
|
||||
_ = stats
|
||||
}
|
||||
|
||||
func TestSafeWalkWorkspace_ContextCancel(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for i := 0; i < 50; i++ {
|
||||
writeFile(t, dir, filepath.Join("files", string(rune('a'+i/26))+"_"+string(rune('a'+i%26))+".txt"), "data")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel immediately
|
||||
|
||||
_, _, err := SafeWalkWorkspace(ctx, dir, DefaultWalkOptions())
|
||||
if err == nil {
|
||||
t.Error("expected error from cancelled context")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeWalkWorkspace_PerFileSizeSkip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "small.txt", "ok")
|
||||
// Create a file larger than MaxFileBytes
|
||||
bigContent := make([]byte, 100*1024) // 100KB
|
||||
writeFile(t, dir, "huge.bin", string(bigContent))
|
||||
|
||||
opts := DefaultWalkOptions()
|
||||
opts.MaxFileBytes = 50 * 1024 // 50KB per-file limit
|
||||
entries, stats, err := SafeWalkWorkspace(context.Background(), dir, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("got %d entries, want 1 (only small.txt)", len(entries))
|
||||
}
|
||||
if stats.SkippedTooLarge != 1 {
|
||||
t.Errorf("stats.SkippedTooLarge = %d, want 1", stats.SkippedTooLarge)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsExcludedPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
path string
|
||||
excluded bool
|
||||
}{
|
||||
{"memory/session-123.json", true},
|
||||
{"memory/deep/nested.md", true},
|
||||
{".hidden/file.txt", true},
|
||||
{".wrangler/config.json", true},
|
||||
{".media/thumb.png", true},
|
||||
{"SOUL.md", true},
|
||||
{"IDENTITY.md", true},
|
||||
{"USER.md", true},
|
||||
{"BOOTSTRAP.md", true},
|
||||
{"AGENTS.md", true},
|
||||
{"TOOLS.md", true},
|
||||
{"CAPABILITIES.md", true},
|
||||
{"MEMORY.md", true},
|
||||
{"AGENTS_CORE.md", true},
|
||||
{"AGENTS_TASK.md", true},
|
||||
{"data.db", true},
|
||||
{"data.db-wal", true},
|
||||
{"data.db-shm", true},
|
||||
// NOT excluded:
|
||||
{"notes/meeting.md", false},
|
||||
{"web-fetch/page.txt", false},
|
||||
{"images/screenshot.png", false},
|
||||
{"teams/abc-123/doc.md", false},
|
||||
{"soul-notes.md", false},
|
||||
{"deep/SOUL.md", false}, // not root-level context file
|
||||
{".uploads/photo.jpg", false},
|
||||
{"report.pdf", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
got := isExcludedPath(tt.path)
|
||||
if got != tt.excluded {
|
||||
t.Errorf("isExcludedPath(%q) = %v, want %v", tt.path, got, tt.excluded)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// writeFile creates a file with the given relative path and content inside dir.
|
||||
func writeFile(t *testing.T, dir, relPath, content string) {
|
||||
t.Helper()
|
||||
abs := filepath.Join(dir, relPath)
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(abs, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,14 @@
|
||||
"skill": "Skill",
|
||||
"episodic": "Episodic"
|
||||
},
|
||||
"rescanTooltip": "Rescan workspace",
|
||||
"rescanNew": "{{count}} new",
|
||||
"rescanUpdated": "{{count}} updated",
|
||||
"rescanUnchanged": "{{count}} unchanged",
|
||||
"rescanTruncated": "scan truncated (limit reached)",
|
||||
"rescanNoFiles": "No new files found",
|
||||
"rescanBusy": "Rescan already in progress",
|
||||
"rescanError": "Rescan failed",
|
||||
"toast": {
|
||||
"docCreated": "Document created",
|
||||
"docCreateFailed": "Failed to create document",
|
||||
|
||||
@@ -64,6 +64,14 @@
|
||||
"skill": "Kỹ năng",
|
||||
"episodic": "Giai thoại"
|
||||
},
|
||||
"rescanTooltip": "Quét lại workspace",
|
||||
"rescanNew": "{{count}} mới",
|
||||
"rescanUpdated": "{{count}} cập nhật",
|
||||
"rescanUnchanged": "{{count}} không đổi",
|
||||
"rescanTruncated": "quét bị cắt (đạt giới hạn)",
|
||||
"rescanNoFiles": "Không tìm thấy file mới",
|
||||
"rescanBusy": "Đang quét, vui lòng đợi",
|
||||
"rescanError": "Quét thất bại",
|
||||
"toast": {
|
||||
"docCreated": "Đã tạo tài liệu",
|
||||
"docCreateFailed": "Tạo tài liệu thất bại",
|
||||
|
||||
@@ -64,6 +64,14 @@
|
||||
"skill": "技能",
|
||||
"episodic": "情景记忆"
|
||||
},
|
||||
"rescanTooltip": "重新扫描工作区",
|
||||
"rescanNew": "{{count}} 个新文件",
|
||||
"rescanUpdated": "{{count}} 个已更新",
|
||||
"rescanUnchanged": "{{count}} 个未变",
|
||||
"rescanTruncated": "扫描已截断(达到限制)",
|
||||
"rescanNoFiles": "未发现新文件",
|
||||
"rescanBusy": "扫描正在进行中",
|
||||
"rescanError": "扫描失败",
|
||||
"toast": {
|
||||
"docCreated": "文档已创建",
|
||||
"docCreateFailed": "创建文档失败",
|
||||
|
||||
@@ -1,10 +1,21 @@
|
||||
import { useCallback } from "react";
|
||||
import { useState } from "react";
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
import { useHttp } from "@/hooks/use-ws";
|
||||
import { toast } from "@/stores/use-toast-store";
|
||||
import i18n from "@/i18n";
|
||||
import type { VaultDocument, VaultLink } from "@/types/vault";
|
||||
|
||||
interface RescanResult {
|
||||
scanned: number;
|
||||
new: number;
|
||||
updated: number;
|
||||
unchanged: number;
|
||||
skipped: number;
|
||||
errors: number;
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
const VAULT_KEY = "vault";
|
||||
|
||||
/** Create a new vault document. */
|
||||
@@ -96,6 +107,44 @@ export function useCreateLink(agentId: string) {
|
||||
return { create };
|
||||
}
|
||||
|
||||
/** Rescan workspace to sync vault documents from filesystem. */
|
||||
export function useRescanWorkspace(agentId: string) {
|
||||
const http = useHttp();
|
||||
const queryClient = useQueryClient();
|
||||
const [isPending, setIsPending] = useState(false);
|
||||
|
||||
const rescan = useCallback(async () => {
|
||||
if (!agentId) return;
|
||||
setIsPending(true);
|
||||
try {
|
||||
const result = await http.post<RescanResult>(`/v1/agents/${agentId}/vault/rescan`, {});
|
||||
await queryClient.invalidateQueries({ queryKey: [VAULT_KEY] });
|
||||
|
||||
const parts: string[] = [];
|
||||
if (result.new > 0) parts.push(i18n.t("vault:rescanNew", { count: result.new }));
|
||||
if (result.updated > 0) parts.push(i18n.t("vault:rescanUpdated", { count: result.updated }));
|
||||
if (result.unchanged > 0) parts.push(i18n.t("vault:rescanUnchanged", { count: result.unchanged }));
|
||||
|
||||
const title = parts.length > 0 ? parts.join(", ") : i18n.t("vault:rescanNoFiles");
|
||||
const desc = result.truncated ? i18n.t("vault:rescanTruncated") : undefined;
|
||||
toast.success(title, desc);
|
||||
return result;
|
||||
} catch (err) {
|
||||
const status = (err as { status?: number })?.status;
|
||||
if (status === 409) {
|
||||
toast.warning(i18n.t("vault:rescanBusy"));
|
||||
} else {
|
||||
toast.error(i18n.t("vault:rescanError"), err instanceof Error ? err.message : "");
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
setIsPending(false);
|
||||
}
|
||||
}, [http, agentId, queryClient]);
|
||||
|
||||
return { rescan, isPending };
|
||||
}
|
||||
|
||||
/** Delete a vault link. */
|
||||
export function useDeleteLink(agentId: string, linkId: string) {
|
||||
const http = useHttp();
|
||||
|
||||
@@ -226,4 +226,5 @@ export {
|
||||
useDeleteDocument,
|
||||
useCreateLink,
|
||||
useDeleteLink,
|
||||
useRescanWorkspace,
|
||||
} from "./use-vault-mutations";
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { useState, useEffect, useMemo, lazy, Suspense } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Search, FileArchive, Plus, PanelLeftOpen } from "lucide-react";
|
||||
import { Search, FileArchive, Plus, PanelLeftOpen, FolderSync } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { useAgents } from "@/pages/agents/hooks/use-agents";
|
||||
import { useTeams } from "@/pages/teams/hooks/use-teams";
|
||||
import { useIsMobile } from "@/hooks/use-media-query";
|
||||
import { useVaultDocuments, useVaultGraphData } from "./hooks/use-vault";
|
||||
import { useVaultDocuments, useVaultGraphData, useRescanWorkspace } from "./hooks/use-vault";
|
||||
import { VaultDocumentSidebar } from "./vault-document-sidebar";
|
||||
import { VaultSearchDialog } from "./vault-search-dialog";
|
||||
import { VaultCreateDialog } from "./vault-create-dialog";
|
||||
@@ -38,6 +38,8 @@ export function VaultPage() {
|
||||
const [sidebarOpen, setSidebarOpen] = useState(false);
|
||||
const [page, setPage] = useState(0);
|
||||
|
||||
const { rescan, isPending: rescanPending } = useRescanWorkspace(selectedAgent);
|
||||
|
||||
const { documents, total, loading } = useVaultDocuments(selectedAgent, {
|
||||
teamId: selectedTeam || undefined,
|
||||
limit: PAGE_SIZE,
|
||||
@@ -131,6 +133,18 @@ export function VaultPage() {
|
||||
<Button size="sm" variant="outline" onClick={() => setSearchOpen(true)} disabled={!selectedAgent}>
|
||||
<Search className="h-3.5 w-3.5" />
|
||||
</Button>
|
||||
<TooltipProvider>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<span>
|
||||
<Button size="sm" variant="outline" onClick={() => rescan()} disabled={!selectedAgent || rescanPending}>
|
||||
<FolderSync className={`h-3.5 w-3.5${rescanPending ? " animate-spin" : ""}`} />
|
||||
</Button>
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{!selectedAgent ? t("selectAgentFirst", "Select an agent first") : t("rescanTooltip", "Rescan workspace")}</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
<TooltipProvider>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
|
||||
Reference in New Issue
Block a user