mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-07-12 13:04:57 +00:00
feat(tools): tenant-aware web_fetch domain policy resolution
web_fetch now reads per-tenant policy overrides from BuiltinToolSettingsFromCtx before falling back to the tool's default policy. Same per-call resolution pattern as web_search. - Add resolvePolicy(ctx) with webFetchPolicyOverride struct - Refactor Execute + doFetch + fetchRawContent to use webFetchPolicy - InProcessExtractor also resolves policy from ctx - Remove isDomainAllowed/isDomainBlocked (replaced by matchDomainList) - 6 new unit tests for tenant policy override scenarios
This commit is contained in:
+56
-29
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
@@ -79,6 +80,49 @@ func (t *WebFetchTool) UpdatePolicy(policy string, allowed, blocked []string) {
|
||||
slog.Info("web_fetch policy updated", "policy", policy, "allowed", len(allowed), "blocked", len(blocked))
|
||||
}
|
||||
|
||||
// webFetchPolicy holds the resolved domain policy for a single request.
|
||||
type webFetchPolicy struct {
|
||||
mode string // "allow_all" | "allowlist"
|
||||
allowedDomains []string
|
||||
blockedDomains []string
|
||||
}
|
||||
|
||||
// webFetchPolicyOverride is the tenant settings shape for web_fetch
|
||||
// (stored in builtin_tool_tenant_configs.settings).
|
||||
type webFetchPolicyOverride struct {
|
||||
Policy string `json:"policy,omitempty"`
|
||||
AllowedDomains []string `json:"allowed_domains,omitempty"`
|
||||
BlockedDomains []string `json:"blocked_domains,omitempty"`
|
||||
}
|
||||
|
||||
// resolvePolicy returns the effective domain policy for this request.
|
||||
// Checks tenant override via BuiltinToolSettingsFromCtx first; falls back
|
||||
// to the tool's default policy when no override is present.
|
||||
func (t *WebFetchTool) resolvePolicy(ctx context.Context) webFetchPolicy {
|
||||
if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil {
|
||||
if raw, ok := settings["web_fetch"]; ok && len(raw) > 0 {
|
||||
var override webFetchPolicyOverride
|
||||
if err := json.Unmarshal(raw, &override); err != nil {
|
||||
slog.Warn("web_fetch: failed to parse tenant override, using defaults", "error", err)
|
||||
} else if override.Policy != "" {
|
||||
return webFetchPolicy{
|
||||
mode: override.Policy,
|
||||
allowedDomains: override.AllowedDomains,
|
||||
blockedDomains: override.BlockedDomains,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fall back to tool defaults
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
return webFetchPolicy{
|
||||
mode: t.policy,
|
||||
allowedDomains: t.allowedDomains,
|
||||
blockedDomains: t.blockedDomains,
|
||||
}
|
||||
}
|
||||
|
||||
// matchDomainList checks if a hostname matches any pattern in the list.
|
||||
// Supports exact match ("github.com") and wildcard prefix ("*.example.com").
|
||||
func matchDomainList(hostname string, patterns []string) bool {
|
||||
@@ -99,22 +143,6 @@ func matchDomainList(hostname string, patterns []string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// isDomainAllowed checks if a hostname matches the allowlist.
|
||||
func (t *WebFetchTool) isDomainAllowed(hostname string) bool {
|
||||
t.mu.RLock()
|
||||
domains := t.allowedDomains
|
||||
t.mu.RUnlock()
|
||||
return matchDomainList(hostname, domains)
|
||||
}
|
||||
|
||||
// isDomainBlocked checks if a hostname matches the blocklist.
|
||||
func (t *WebFetchTool) isDomainBlocked(hostname string) bool {
|
||||
t.mu.RLock()
|
||||
domains := t.blockedDomains
|
||||
t.mu.RUnlock()
|
||||
return matchDomainList(hostname, domains)
|
||||
}
|
||||
|
||||
func (t *WebFetchTool) Name() string { return "web_fetch" }
|
||||
|
||||
func (t *WebFetchTool) Description() string {
|
||||
@@ -167,18 +195,17 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result
|
||||
return ErrorResult(fmt.Sprintf("SSRF protection: %v", err))
|
||||
}
|
||||
|
||||
// Resolve domain policy (tenant override via ctx, or tool defaults)
|
||||
pol := t.resolvePolicy(ctx)
|
||||
hostname := parsed.Hostname()
|
||||
|
||||
// Domain blocklist check (always enforced regardless of policy)
|
||||
if t.isDomainBlocked(hostname) {
|
||||
if matchDomainList(hostname, pol.blockedDomains) {
|
||||
return ErrorResult(fmt.Sprintf("domain %q is blocked by policy", hostname))
|
||||
}
|
||||
|
||||
// Domain allowlist check
|
||||
t.mu.RLock()
|
||||
policy := t.policy
|
||||
t.mu.RUnlock()
|
||||
if policy == "allowlist" && !t.isDomainAllowed(hostname) {
|
||||
if pol.mode == "allowlist" && !matchDomainList(hostname, pol.allowedDomains) {
|
||||
return ErrorResult(fmt.Sprintf("domain %q is not in the allowed domains list", hostname))
|
||||
}
|
||||
|
||||
@@ -212,7 +239,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result
|
||||
}
|
||||
|
||||
// Fetch
|
||||
result, err := t.doFetch(ctx, rawURL, extractMode, maxChars, policy)
|
||||
result, err := t.doFetch(ctx, rawURL, extractMode, maxChars, pol)
|
||||
if err != nil {
|
||||
errMsg := truncateStr(err.Error(), defaultErrorMaxChars)
|
||||
return ErrorResult(fmt.Sprintf("fetch failed: %s", errMsg))
|
||||
@@ -223,7 +250,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result
|
||||
return NewResult(wrapped)
|
||||
}
|
||||
|
||||
func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) {
|
||||
func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, maxChars int, pol webFetchPolicy) (string, error) {
|
||||
// For markdown mode, use the extractor chain (Defuddle → InProcess waterfall)
|
||||
// resolved from builtin_tools settings stored in context.
|
||||
// InProcessExtractor delegates to fetchRawContent (same path as doDirectFetch),
|
||||
@@ -240,7 +267,7 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
|
||||
}
|
||||
|
||||
// Text mode or no chain available — use direct HTTP fetch.
|
||||
return t.doDirectFetch(ctx, rawURL, extractMode, maxChars, policy)
|
||||
return t.doDirectFetch(ctx, rawURL, extractMode, maxChars, pol)
|
||||
}
|
||||
|
||||
// fetchRawResult holds the output from fetchRawContent.
|
||||
@@ -254,7 +281,7 @@ type fetchRawResult struct {
|
||||
// fetchRawContent performs HTTP GET with full security checks (SSRF, domain policy on
|
||||
// redirects) and routes content by type. Returns raw extracted content without formatting.
|
||||
// Used by both doDirectFetch (text mode) and InProcessExtractor (chain fallback).
|
||||
func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (fetchRawResult, error) {
|
||||
func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode string, maxChars int, pol webFetchPolicy) (fetchRawResult, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
|
||||
if err != nil {
|
||||
return fetchRawResult{}, fmt.Errorf("create request: %w", err)
|
||||
@@ -280,10 +307,10 @@ func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode
|
||||
return fmt.Errorf("redirect SSRF protection: %w", err)
|
||||
}
|
||||
redirectHost := req.URL.Hostname()
|
||||
if t.isDomainBlocked(redirectHost) {
|
||||
if matchDomainList(redirectHost, pol.blockedDomains) {
|
||||
return fmt.Errorf("redirect to %q blocked: domain is in blocklist", redirectHost)
|
||||
}
|
||||
if policy == "allowlist" && !t.isDomainAllowed(redirectHost) {
|
||||
if pol.mode == "allowlist" && !matchDomainList(redirectHost, pol.allowedDomains) {
|
||||
return fmt.Errorf("redirect to %q blocked: domain not in allowlist", redirectHost)
|
||||
}
|
||||
return nil
|
||||
@@ -348,8 +375,8 @@ func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode
|
||||
|
||||
// doDirectFetch wraps fetchRawContent with full HTTP metadata formatting.
|
||||
// Used for text mode extraction and as ultimate fallback.
|
||||
func (t *WebFetchTool) doDirectFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) {
|
||||
raw, err := t.fetchRawContent(ctx, rawURL, extractMode, maxChars, policy)
|
||||
func (t *WebFetchTool) doDirectFetch(ctx context.Context, rawURL, extractMode string, maxChars int, pol webFetchPolicy) (string, error) {
|
||||
raw, err := t.fetchRawContent(ctx, rawURL, extractMode, maxChars, pol)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -16,11 +16,8 @@ func (e *InProcessExtractor) Name() string { return "html-to-markdown" }
|
||||
// Extract fetches the URL via the tool's fetchRawContent (full security checks)
|
||||
// and returns the raw extracted markdown content.
|
||||
func (e *InProcessExtractor) Extract(ctx context.Context, rawURL string) (string, error) {
|
||||
e.tool.mu.RLock()
|
||||
policy := e.tool.policy
|
||||
e.tool.mu.RUnlock()
|
||||
|
||||
raw, err := e.tool.fetchRawContent(ctx, rawURL, "markdown", defaultFetchMaxChars, policy)
|
||||
pol := e.tool.resolvePolicy(ctx)
|
||||
raw, err := e.tool.fetchRawContent(ctx, rawURL, "markdown", defaultFetchMaxChars, pol)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ctxWithWebFetchSettings returns a ctx with builtin tool settings containing
|
||||
// the given web_fetch policy override.
|
||||
func ctxWithWebFetchSettings(t *testing.T, override webFetchPolicyOverride) context.Context {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(override)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := BuiltinToolSettings{"web_fetch": raw}
|
||||
return WithBuiltinToolSettings(context.Background(), settings)
|
||||
}
|
||||
|
||||
func TestResolvePolicy_NoOverride_ReturnsDefaults(t *testing.T) {
|
||||
tool := NewWebFetchTool(WebFetchConfig{
|
||||
Policy: "allow_all",
|
||||
AllowedDomains: []string{"default.com"},
|
||||
BlockedDomains: []string{"blocked.com"},
|
||||
})
|
||||
|
||||
pol := tool.resolvePolicy(context.Background())
|
||||
if pol.mode != "allow_all" {
|
||||
t.Errorf("mode = %q, want allow_all", pol.mode)
|
||||
}
|
||||
if len(pol.blockedDomains) != 1 || pol.blockedDomains[0] != "blocked.com" {
|
||||
t.Errorf("blockedDomains = %v, want [blocked.com]", pol.blockedDomains)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePolicy_TenantAllowlist(t *testing.T) {
|
||||
tool := NewWebFetchTool(WebFetchConfig{
|
||||
Policy: "allow_all",
|
||||
AllowedDomains: []string{"default.com"},
|
||||
})
|
||||
|
||||
ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{
|
||||
Policy: "allowlist",
|
||||
AllowedDomains: []string{"tenant-a.com", "*.api.tenant-a.com"},
|
||||
})
|
||||
|
||||
pol := tool.resolvePolicy(ctx)
|
||||
if pol.mode != "allowlist" {
|
||||
t.Errorf("mode = %q, want allowlist", pol.mode)
|
||||
}
|
||||
if len(pol.allowedDomains) != 2 {
|
||||
t.Errorf("allowedDomains = %v, want 2 entries", pol.allowedDomains)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePolicy_TenantBlocklist(t *testing.T) {
|
||||
tool := NewWebFetchTool(WebFetchConfig{
|
||||
Policy: "allow_all",
|
||||
BlockedDomains: []string{"global-blocked.com"},
|
||||
})
|
||||
|
||||
ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{
|
||||
Policy: "allow_all",
|
||||
BlockedDomains: []string{"tenant-blocked.com"},
|
||||
})
|
||||
|
||||
pol := tool.resolvePolicy(ctx)
|
||||
// Tenant override replaces global blocked list
|
||||
if len(pol.blockedDomains) != 1 || pol.blockedDomains[0] != "tenant-blocked.com" {
|
||||
t.Errorf("blockedDomains = %v, want [tenant-blocked.com]", pol.blockedDomains)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePolicy_MalformedJSON_FallsBack(t *testing.T) {
|
||||
tool := NewWebFetchTool(WebFetchConfig{
|
||||
Policy: "allow_all",
|
||||
})
|
||||
|
||||
settings := BuiltinToolSettings{
|
||||
"web_fetch": []byte(`{not valid json}`),
|
||||
}
|
||||
ctx := WithBuiltinToolSettings(context.Background(), settings)
|
||||
|
||||
pol := tool.resolvePolicy(ctx)
|
||||
if pol.mode != "allow_all" {
|
||||
t.Errorf("mode = %q, want allow_all (fallback)", pol.mode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePolicy_EmptyOverride_FallsBack(t *testing.T) {
|
||||
tool := NewWebFetchTool(WebFetchConfig{
|
||||
Policy: "allowlist",
|
||||
AllowedDomains: []string{"default.com"},
|
||||
})
|
||||
|
||||
// Override exists but has empty policy — should fall back to defaults
|
||||
ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{})
|
||||
|
||||
pol := tool.resolvePolicy(ctx)
|
||||
if pol.mode != "allowlist" {
|
||||
t.Errorf("mode = %q, want allowlist (default)", pol.mode)
|
||||
}
|
||||
if len(pol.allowedDomains) != 1 || pol.allowedDomains[0] != "default.com" {
|
||||
t.Errorf("allowedDomains = %v, want [default.com]", pol.allowedDomains)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePolicy_TenantOverridesGlobal(t *testing.T) {
|
||||
tool := NewWebFetchTool(WebFetchConfig{
|
||||
Policy: "allowlist",
|
||||
AllowedDomains: []string{"global.com"},
|
||||
BlockedDomains: []string{"global-blocked.com"},
|
||||
})
|
||||
|
||||
ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{
|
||||
Policy: "allow_all",
|
||||
BlockedDomains: []string{"tenant-evil.com"},
|
||||
})
|
||||
|
||||
pol := tool.resolvePolicy(ctx)
|
||||
if pol.mode != "allow_all" {
|
||||
t.Errorf("mode = %q, want allow_all (tenant override)", pol.mode)
|
||||
}
|
||||
if len(pol.blockedDomains) != 1 || pol.blockedDomains[0] != "tenant-evil.com" {
|
||||
t.Errorf("blockedDomains = %v, want [tenant-evil.com]", pol.blockedDomains)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user