mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-07-12 15:04:53 +00:00
viettranx
def1712f3b
web_fetch now reads per-tenant policy overrides from BuiltinToolSettingsFromCtx before falling back to the tool's default policy. Same per-call resolution pattern as web_search. - Add resolvePolicy(ctx) with webFetchPolicyOverride struct - Refactor Execute + doFetch + fetchRawContent to use webFetchPolicy - InProcessExtractor also resolves policy from ctx - Remove isDomainAllowed/isDomainBlocked (replaced by matchDomainList) - 6 new unit tests for tenant policy override scenarios
26 lines
814 B
Go
26 lines
814 B
Go
package tools
|
|
|
|
import (
|
|
"context"
|
|
)
|
|
|
|
// InProcessExtractor delegates to WebFetchTool.fetchRawContent for HTML→markdown
|
|
// extraction with full security checks (SSRF, domain policy on redirects).
|
|
// This is the fallback when external extractors (Defuddle) are unavailable.
|
|
type InProcessExtractor struct {
|
|
tool *WebFetchTool
|
|
}
|
|
|
|
func (e *InProcessExtractor) Name() string { return "html-to-markdown" }
|
|
|
|
// Extract fetches the URL via the tool's fetchRawContent (full security checks)
|
|
// and returns the raw extracted markdown content.
|
|
func (e *InProcessExtractor) Extract(ctx context.Context, rawURL string) (string, error) {
|
|
pol := e.tool.resolvePolicy(ctx)
|
|
raw, err := e.tool.fetchRawContent(ctx, rawURL, "markdown", defaultFetchMaxChars, pol)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return raw.content, nil
|
|
}
|