diff --git a/internal/tools/web_fetch.go b/internal/tools/web_fetch.go index ab43c001..c32bfdd2 100644 --- a/internal/tools/web_fetch.go +++ b/internal/tools/web_fetch.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/hex" + "encoding/json" "fmt" "io" "log/slog" @@ -79,6 +80,49 @@ func (t *WebFetchTool) UpdatePolicy(policy string, allowed, blocked []string) { slog.Info("web_fetch policy updated", "policy", policy, "allowed", len(allowed), "blocked", len(blocked)) } +// webFetchPolicy holds the resolved domain policy for a single request. +type webFetchPolicy struct { + mode string // "allow_all" | "allowlist" + allowedDomains []string + blockedDomains []string +} + +// webFetchPolicyOverride is the tenant settings shape for web_fetch +// (stored in builtin_tool_tenant_configs.settings). +type webFetchPolicyOverride struct { + Policy string `json:"policy,omitempty"` + AllowedDomains []string `json:"allowed_domains,omitempty"` + BlockedDomains []string `json:"blocked_domains,omitempty"` +} + +// resolvePolicy returns the effective domain policy for this request. +// Checks tenant override via BuiltinToolSettingsFromCtx first; falls back +// to the tool's default policy when no override is present. +func (t *WebFetchTool) resolvePolicy(ctx context.Context) webFetchPolicy { + if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil { + if raw, ok := settings["web_fetch"]; ok && len(raw) > 0 { + var override webFetchPolicyOverride + if err := json.Unmarshal(raw, &override); err != nil { + slog.Warn("web_fetch: failed to parse tenant override, using defaults", "error", err) + } else if override.Policy != "" { + return webFetchPolicy{ + mode: override.Policy, + allowedDomains: override.AllowedDomains, + blockedDomains: override.BlockedDomains, + } + } + } + } + // Fall back to tool defaults + t.mu.RLock() + defer t.mu.RUnlock() + return webFetchPolicy{ + mode: t.policy, + allowedDomains: t.allowedDomains, + blockedDomains: t.blockedDomains, + } +} + // matchDomainList checks if a hostname matches any pattern in the list. // Supports exact match ("github.com") and wildcard prefix ("*.example.com"). func matchDomainList(hostname string, patterns []string) bool { @@ -99,22 +143,6 @@ func matchDomainList(hostname string, patterns []string) bool { return false } -// isDomainAllowed checks if a hostname matches the allowlist. -func (t *WebFetchTool) isDomainAllowed(hostname string) bool { - t.mu.RLock() - domains := t.allowedDomains - t.mu.RUnlock() - return matchDomainList(hostname, domains) -} - -// isDomainBlocked checks if a hostname matches the blocklist. -func (t *WebFetchTool) isDomainBlocked(hostname string) bool { - t.mu.RLock() - domains := t.blockedDomains - t.mu.RUnlock() - return matchDomainList(hostname, domains) -} - func (t *WebFetchTool) Name() string { return "web_fetch" } func (t *WebFetchTool) Description() string { @@ -167,18 +195,17 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result return ErrorResult(fmt.Sprintf("SSRF protection: %v", err)) } + // Resolve domain policy (tenant override via ctx, or tool defaults) + pol := t.resolvePolicy(ctx) hostname := parsed.Hostname() // Domain blocklist check (always enforced regardless of policy) - if t.isDomainBlocked(hostname) { + if matchDomainList(hostname, pol.blockedDomains) { return ErrorResult(fmt.Sprintf("domain %q is blocked by policy", hostname)) } // Domain allowlist check - t.mu.RLock() - policy := t.policy - t.mu.RUnlock() - if policy == "allowlist" && !t.isDomainAllowed(hostname) { + if pol.mode == "allowlist" && !matchDomainList(hostname, pol.allowedDomains) { return ErrorResult(fmt.Sprintf("domain %q is not in the allowed domains list", hostname)) } @@ -212,7 +239,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result } // Fetch - result, err := t.doFetch(ctx, rawURL, extractMode, maxChars, policy) + result, err := t.doFetch(ctx, rawURL, extractMode, maxChars, pol) if err != nil { errMsg := truncateStr(err.Error(), defaultErrorMaxChars) return ErrorResult(fmt.Sprintf("fetch failed: %s", errMsg)) @@ -223,7 +250,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result return NewResult(wrapped) } -func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) { +func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, maxChars int, pol webFetchPolicy) (string, error) { // For markdown mode, use the extractor chain (Defuddle → InProcess waterfall) // resolved from builtin_tools settings stored in context. // InProcessExtractor delegates to fetchRawContent (same path as doDirectFetch), @@ -240,7 +267,7 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, } // Text mode or no chain available — use direct HTTP fetch. - return t.doDirectFetch(ctx, rawURL, extractMode, maxChars, policy) + return t.doDirectFetch(ctx, rawURL, extractMode, maxChars, pol) } // fetchRawResult holds the output from fetchRawContent. @@ -254,7 +281,7 @@ type fetchRawResult struct { // fetchRawContent performs HTTP GET with full security checks (SSRF, domain policy on // redirects) and routes content by type. Returns raw extracted content without formatting. // Used by both doDirectFetch (text mode) and InProcessExtractor (chain fallback). -func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (fetchRawResult, error) { +func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode string, maxChars int, pol webFetchPolicy) (fetchRawResult, error) { req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil) if err != nil { return fetchRawResult{}, fmt.Errorf("create request: %w", err) @@ -280,10 +307,10 @@ func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode return fmt.Errorf("redirect SSRF protection: %w", err) } redirectHost := req.URL.Hostname() - if t.isDomainBlocked(redirectHost) { + if matchDomainList(redirectHost, pol.blockedDomains) { return fmt.Errorf("redirect to %q blocked: domain is in blocklist", redirectHost) } - if policy == "allowlist" && !t.isDomainAllowed(redirectHost) { + if pol.mode == "allowlist" && !matchDomainList(redirectHost, pol.allowedDomains) { return fmt.Errorf("redirect to %q blocked: domain not in allowlist", redirectHost) } return nil @@ -348,8 +375,8 @@ func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode // doDirectFetch wraps fetchRawContent with full HTTP metadata formatting. // Used for text mode extraction and as ultimate fallback. -func (t *WebFetchTool) doDirectFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) { - raw, err := t.fetchRawContent(ctx, rawURL, extractMode, maxChars, policy) +func (t *WebFetchTool) doDirectFetch(ctx context.Context, rawURL, extractMode string, maxChars int, pol webFetchPolicy) (string, error) { + raw, err := t.fetchRawContent(ctx, rawURL, extractMode, maxChars, pol) if err != nil { return "", err } diff --git a/internal/tools/web_fetch_extractor_inprocess.go b/internal/tools/web_fetch_extractor_inprocess.go index 77c24687..cafa957e 100644 --- a/internal/tools/web_fetch_extractor_inprocess.go +++ b/internal/tools/web_fetch_extractor_inprocess.go @@ -16,11 +16,8 @@ func (e *InProcessExtractor) Name() string { return "html-to-markdown" } // Extract fetches the URL via the tool's fetchRawContent (full security checks) // and returns the raw extracted markdown content. func (e *InProcessExtractor) Extract(ctx context.Context, rawURL string) (string, error) { - e.tool.mu.RLock() - policy := e.tool.policy - e.tool.mu.RUnlock() - - raw, err := e.tool.fetchRawContent(ctx, rawURL, "markdown", defaultFetchMaxChars, policy) + pol := e.tool.resolvePolicy(ctx) + raw, err := e.tool.fetchRawContent(ctx, rawURL, "markdown", defaultFetchMaxChars, pol) if err != nil { return "", err } diff --git a/internal/tools/web_fetch_policy_test.go b/internal/tools/web_fetch_policy_test.go new file mode 100644 index 00000000..0a28b4da --- /dev/null +++ b/internal/tools/web_fetch_policy_test.go @@ -0,0 +1,128 @@ +package tools + +import ( + "context" + "encoding/json" + "testing" +) + +// ctxWithWebFetchSettings returns a ctx with builtin tool settings containing +// the given web_fetch policy override. +func ctxWithWebFetchSettings(t *testing.T, override webFetchPolicyOverride) context.Context { + t.Helper() + raw, err := json.Marshal(override) + if err != nil { + t.Fatal(err) + } + settings := BuiltinToolSettings{"web_fetch": raw} + return WithBuiltinToolSettings(context.Background(), settings) +} + +func TestResolvePolicy_NoOverride_ReturnsDefaults(t *testing.T) { + tool := NewWebFetchTool(WebFetchConfig{ + Policy: "allow_all", + AllowedDomains: []string{"default.com"}, + BlockedDomains: []string{"blocked.com"}, + }) + + pol := tool.resolvePolicy(context.Background()) + if pol.mode != "allow_all" { + t.Errorf("mode = %q, want allow_all", pol.mode) + } + if len(pol.blockedDomains) != 1 || pol.blockedDomains[0] != "blocked.com" { + t.Errorf("blockedDomains = %v, want [blocked.com]", pol.blockedDomains) + } +} + +func TestResolvePolicy_TenantAllowlist(t *testing.T) { + tool := NewWebFetchTool(WebFetchConfig{ + Policy: "allow_all", + AllowedDomains: []string{"default.com"}, + }) + + ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{ + Policy: "allowlist", + AllowedDomains: []string{"tenant-a.com", "*.api.tenant-a.com"}, + }) + + pol := tool.resolvePolicy(ctx) + if pol.mode != "allowlist" { + t.Errorf("mode = %q, want allowlist", pol.mode) + } + if len(pol.allowedDomains) != 2 { + t.Errorf("allowedDomains = %v, want 2 entries", pol.allowedDomains) + } +} + +func TestResolvePolicy_TenantBlocklist(t *testing.T) { + tool := NewWebFetchTool(WebFetchConfig{ + Policy: "allow_all", + BlockedDomains: []string{"global-blocked.com"}, + }) + + ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{ + Policy: "allow_all", + BlockedDomains: []string{"tenant-blocked.com"}, + }) + + pol := tool.resolvePolicy(ctx) + // Tenant override replaces global blocked list + if len(pol.blockedDomains) != 1 || pol.blockedDomains[0] != "tenant-blocked.com" { + t.Errorf("blockedDomains = %v, want [tenant-blocked.com]", pol.blockedDomains) + } +} + +func TestResolvePolicy_MalformedJSON_FallsBack(t *testing.T) { + tool := NewWebFetchTool(WebFetchConfig{ + Policy: "allow_all", + }) + + settings := BuiltinToolSettings{ + "web_fetch": []byte(`{not valid json}`), + } + ctx := WithBuiltinToolSettings(context.Background(), settings) + + pol := tool.resolvePolicy(ctx) + if pol.mode != "allow_all" { + t.Errorf("mode = %q, want allow_all (fallback)", pol.mode) + } +} + +func TestResolvePolicy_EmptyOverride_FallsBack(t *testing.T) { + tool := NewWebFetchTool(WebFetchConfig{ + Policy: "allowlist", + AllowedDomains: []string{"default.com"}, + }) + + // Override exists but has empty policy — should fall back to defaults + ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{}) + + pol := tool.resolvePolicy(ctx) + if pol.mode != "allowlist" { + t.Errorf("mode = %q, want allowlist (default)", pol.mode) + } + if len(pol.allowedDomains) != 1 || pol.allowedDomains[0] != "default.com" { + t.Errorf("allowedDomains = %v, want [default.com]", pol.allowedDomains) + } +} + +func TestResolvePolicy_TenantOverridesGlobal(t *testing.T) { + tool := NewWebFetchTool(WebFetchConfig{ + Policy: "allowlist", + AllowedDomains: []string{"global.com"}, + BlockedDomains: []string{"global-blocked.com"}, + }) + + ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{ + Policy: "allow_all", + BlockedDomains: []string{"tenant-evil.com"}, + }) + + pol := tool.resolvePolicy(ctx) + if pol.mode != "allow_all" { + t.Errorf("mode = %q, want allow_all (tenant override)", pol.mode) + } + if len(pol.blockedDomains) != 1 || pol.blockedDomains[0] != "tenant-evil.com" { + t.Errorf("blockedDomains = %v, want [tenant-evil.com]", pol.blockedDomains) + } +}