goclaw/internal/agent/sanitize.go

// Package agent — response sanitization pipeline.
//
// Matching TS sanitization chain:
//
//	extractAssistantText() → per-block:
//	  1. stripMinimaxToolCallXml()        → Go: stripGarbledToolXML()
//	  2. stripDowngradedToolCallText()     → Go: stripDowngradedToolCallText()
//	  3. stripThinkingTagsFromText()       → Go: stripThinkingTags()
//	  then:
//	  4. sanitizeUserFacingText()          → Go: sanitizeUserFacingText()
//	     - stripFinalTagsFromText()        → Go: stripFinalTags()
//	     - collapseConsecutiveDuplicateBlocks()
//
// Additional Go-specific:
//  5. stripEchoedSystemMessages()       → strip hallucinated [System Message] blocks
//  6. stripGarbledToolXML()             → strip garbled XML from models like DeepSeek
package agent

import (
	"log/slog"
	"regexp"
	"strings"

	"github.com/nextlevelbuilder/goclaw/internal/store"
)

// SanitizeAssistantContent applies the full sanitization pipeline to assistant
// response text before saving to session and sending to user.
// Matching TS extractAssistantText() + sanitizeUserFacingText().
func SanitizeAssistantContent(content string) string {
	if content == "" {
		return content
	}

	original := content

	// 1. Strip garbled tool-call XML (DeepSeek, GLM, Minimax)
	content = stripGarbledToolXML(content)
	if content == "" {
		return ""
	}

	// 2. Strip downgraded tool call text ([Tool Call: ...], [Tool Result ...])
	content = stripDowngradedToolCallText(content)

	// 3. Strip thinking/reasoning tags (<think>, <thinking>, <thought>, <antThinking>)
	content = stripThinkingTags(content)

	// 4. Strip <final> tags (keep content inside)
	content = stripFinalTags(content)

	// 5. Strip echoed [System Message] blocks
	content = stripEchoedSystemMessages(content)

	// 6. Collapse consecutive duplicate blocks
	content = collapseConsecutiveDuplicateBlocks(content)

	// 7. Strip MEDIA: paths from LLM output (media delivered separately)
	content = stripMediaPaths(content)

	// 8. Strip leading blank lines (preserve indentation)
	content = stripLeadingBlankLines(content)

	content = strings.TrimSpace(content)

	if content != original {
		slog.Debug("sanitized assistant content",
			"original_len", len(original),
			"cleaned_len", len(content),
		)
	}

	return content
}

// --- 1. Garbled tool-call XML ---

// garbledToolXMLPattern matches XML-like tool call artifacts that some models
// (DeepSeek, GLM, etc.) emit as text content instead of proper tool calls.
var garbledToolXMLPattern = regexp.MustCompile(
	`(?s)</?(?:function_calls?|functioninvoke|invoke|invfunction_calls|tool_call|tool_use|parameter|minimax:tool_call)[^>]*>`,
)

// fullToolCallBlockPattern matches a COMPLETE Anthropic-style tool-call block
// (`<function_calls>...</function_calls>`) that a model emitted as text instead
// of invoking it natively. Unlike garbledToolXMLPattern — which removes only the
// tags — this captures the whole block, tags plus the inner <parameter> text, so
// a stray block is removed cleanly rather than leaving the argument values
// orphaned in the user-facing reply.
var fullToolCallBlockPattern = regexp.MustCompile(
	`(?is)<function_calls?>.*?</function_calls?>`,
)

// invokeNamePattern extracts the tool name from an `<invoke name="...">` tag so a
// dropped text-encoded tool call can be logged with the tool it tried to call.
var invokeNamePattern = regexp.MustCompile(`(?i)<invoke\s+name="([^"]+)"`)

// bareInvokeBlockPattern matches a complete `<invoke name="...">...</invoke>`
// block emitted as text WITHOUT the surrounding <function_calls> wrapper. Some
// models — and the claude-cli proxy under a degraded session — drop the wrapper
// and emit just the invoke block, which fullToolCallBlockPattern misses; the
// tag-only strip would then leak the inner <parameter> text. Applied after the
// wrapped form so wrapper-nested invokes are already gone.
var bareInvokeBlockPattern = regexp.MustCompile(
	`(?is)<invoke\s+name="[^"]*".*?</invoke>`,
)

var garbledToolXMLIndicators = []string{
	"invfunction_calls",
	"functioninvoke",
	"<invoke name=",
	"<parameter name=",
	"</parameter",
	"<function_call",
	"<tool_call",
	"<tool_use",
	"<minimax:tool_call",
}

func stripGarbledToolXML(content string) string {
	hasIndicator := false
	lower := strings.ToLower(content)
	for _, ind := range garbledToolXMLIndicators {
		if strings.Contains(lower, strings.ToLower(ind)) {
			hasIndicator = true
			break
		}
	}
	if !hasIndicator {
		return content
	}

	original := content

	// A COMPLETE tool-call block is not "garble" — it is a tool call the model
	// wrote as TEXT instead of invoking it natively. This shows up with the
	// claude-cli thin-proxy provider, where tool execution lives inside the CLI:
	// when the model emits the call as text the tool never runs, and the tag-only
	// strip below would leave the inner <parameter> values mangled into the reply.
	// Remove whole blocks — <function_calls>...</function_calls> wrappers first,
	// then any bare <invoke>...</invoke> left without a wrapper — and log the
	// attempted tool name(s) at WARN so this otherwise-silent no-op is diagnosable.
	var droppedTools []string
	var droppedBlocks int
	for _, re := range []*regexp.Regexp{fullToolCallBlockPattern, bareInvokeBlockPattern} {
		blocks := re.FindAllString(content, -1)
		if len(blocks) == 0 {
			continue
		}
		droppedBlocks += len(blocks)
		for _, b := range blocks {
			for _, m := range invokeNamePattern.FindAllStringSubmatch(b, -1) {
				droppedTools = append(droppedTools, m[1])
			}
		}
		content = re.ReplaceAllString(content, "")
	}
	if droppedBlocks > 0 {
		slog.Warn("dropped text-encoded tool call from response",
			"tools", droppedTools,
			"blocks", droppedBlocks,
			"hint", "model wrote a tool call as text instead of invoking it; the tool did not run",
		)
	}

	// Strip any remaining stray tags (partial DeepSeek/GLM/Minimax artifacts).
	cleaned := strings.TrimSpace(garbledToolXMLPattern.ReplaceAllString(content, ""))

	if cleaned == "" {
		slog.Warn("stripped entire response as garbled tool XML", "original_len", len(original))
		return ""
	}

	if cleaned != original {
		slog.Warn("stripped garbled tool call XML from response",
			"original_len", len(original),
			"remaining_len", len(cleaned),
		)
	}
	return cleaned
}

// --- 2. Downgraded tool call text ---

// stripDowngradedToolCallText removes [Tool Call: ...], [Tool Result ...],
// and [Historical context: ...] blocks that some models emit as text.
// Matching TS stripDowngradedToolCallText().
// Uses line-by-line scanning (Go regexp doesn't support lookahead).
func stripDowngradedToolCallText(content string) string {
	if !strings.Contains(content, "[Tool Call:") &&
		!strings.Contains(content, "[Tool Result") &&
		!strings.Contains(content, "[Historical context:") {
		return content
	}

	lines := strings.Split(content, "\n")
	var result []string
	skipping := false

	for _, line := range lines {
		trimmed := strings.TrimSpace(line)

		// Start skipping on these markers
		if strings.HasPrefix(trimmed, "[Tool Call:") ||
			strings.HasPrefix(trimmed, "[Tool Result") ||
			strings.HasPrefix(trimmed, "[Historical context:") {
			skipping = true
			continue
		}

		// Stop skipping on non-indented, non-empty line that isn't part of the block
		if skipping {
			// Arguments JSON and tool output are typically indented or empty
			if trimmed == "" || strings.HasPrefix(trimmed, "Arguments:") ||
				strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "}") {
				continue
			}
			// Non-tool-block line → stop skipping
			skipping = false
		}

		result = append(result, line)
	}

	return strings.TrimSpace(strings.Join(result, "\n"))
}

// --- 3. Thinking/reasoning tags ---

// Matches TS stripThinkingTagsFromText() with strict mode.
// Strips: <redacted_thinking>...</redacted_thinking>, <think>...</think>,
//
//	<thinking>...</thinking>, <thought>...</thought>,
//	<antThinking>...</antThinking>
//
// Go regexp doesn't support backreferences, so we use separate patterns.
var thinkingTagPatterns = []*regexp.Regexp{
	regexp.MustCompile(`(?is)<redacted_thinking\b[^>]*>.*?</redacted_thinking\s*>`),
	regexp.MustCompile(`(?is)<thinking\b[^>]*>.*?</thinking\s*>`),
	regexp.MustCompile(`(?is)<think\b[^>]*>.*?</think\s*>`),
	regexp.MustCompile(`(?is)<thought\b[^>]*>.*?</thought\s*>`),
	regexp.MustCompile(`(?is)<antThinking\b[^>]*>.*?</antThinking\s*>`),
	regexp.MustCompile(`(?is)<antthinking\b[^>]*>.*?</antthinking\s*>`),
}

func stripThinkingTags(content string) string {
	lower := strings.ToLower(content)
	if !strings.Contains(lower, "<think") && !strings.Contains(lower, "<thought") &&
		!strings.Contains(lower, "<antthinking") && !strings.Contains(lower, "<redacted_thinking") {
		return content
	}
	result := content
	for _, pat := range thinkingTagPatterns {
		result = pat.ReplaceAllString(result, "")
	}
	return strings.TrimSpace(result)
}

// --- 4. <final> tags ---

// Matches TS stripFinalTagsFromText(). Removes <final> and </final> tags
// but keeps the content inside.
var finalTagPattern = regexp.MustCompile(`(?i)<\s*/?\s*final\s*>`)

func stripFinalTags(content string) string {
	if !strings.Contains(strings.ToLower(content), "final") {
		return content
	}
	return finalTagPattern.ReplaceAllString(content, "")
}

// --- 5. Echoed [System Message] ---

// stripEchoedSystemMessages removes "[System Message] ..." blocks that LLMs
// hallucinate/echo in their response text.
// Uses line-based scanning (Go regexp doesn't support lookahead).
func stripEchoedSystemMessages(content string) string {
	if !strings.Contains(content, "[System Message]") {
		return content
	}

	lines := strings.Split(content, "\n")
	var result []string
	skipping := false

	for _, line := range lines {
		if strings.HasPrefix(strings.TrimSpace(line), "[System Message]") {
			skipping = true
			continue
		}
		if skipping {
			// Empty line ends the system message block
			if strings.TrimSpace(line) == "" {
				skipping = false
				continue
			}
			// Still part of the system message block (Stats:, reply instructions, etc.)
			continue
		}
		result = append(result, line)
	}

	cleaned := strings.TrimSpace(strings.Join(result, "\n"))

	if cleaned != strings.TrimSpace(content) {
		slog.Warn("stripped echoed [System Message] from assistant response",
			"original_len", len(content),
			"cleaned_len", len(cleaned),
		)
	}

	return cleaned
}

// --- 6. Collapse consecutive duplicate blocks ---

// collapseConsecutiveDuplicateBlocks removes repeated paragraph blocks.
// Matching TS collapseConsecutiveDuplicateBlocks().
func collapseConsecutiveDuplicateBlocks(content string) string {
	blocks := strings.Split(content, "\n\n")
	if len(blocks) <= 1 {
		return content
	}

	var result []string
	for i, block := range blocks {
		trimmed := strings.TrimSpace(block)
		if trimmed == "" {
			continue
		}
		if i > 0 && len(result) > 0 && trimmed == strings.TrimSpace(result[len(result)-1]) {
			continue // skip duplicate
		}
		result = append(result, block)
	}

	collapsed := strings.Join(result, "\n\n")
	if collapsed != content {
		slog.Debug("collapsed duplicate blocks",
			"original_blocks", len(blocks),
			"result_blocks", len(result),
		)
	}
	return collapsed
}

// --- 7. Strip MEDIA: paths ---

// mediaPathPattern matches "MEDIA:" followed by a path (absolute or relative).
var mediaPathPattern = regexp.MustCompile(`MEDIA:\S+`)

// stripMediaPaths removes lines containing MEDIA:/path references from LLM output.
// These are tool result artifacts that should not appear in user-facing text
// (media files are delivered separately via OutboundMessage.Media).
func stripMediaPaths(content string) string {
	if !strings.Contains(content, "MEDIA:") {
		return content
	}
	lines := strings.Split(content, "\n")
	var result []string
	for _, line := range lines {
		trimmed := strings.TrimSpace(line)
		if strings.HasPrefix(trimmed, "[[audio_as_voice]]") {
			continue
		}
		// Strip any line containing a MEDIA: path reference, regardless of wrapping format.
		// LLMs echo these in many forms: bare "MEDIA:/path", markdown "![alt](MEDIA:relative/path)",
		// JSON '{"image":"MEDIA:/path"}', etc. Match MEDIA: followed by any non-space path char.
		if mediaPathPattern.MatchString(trimmed) {
			continue
		}
		result = append(result, line)
	}
	return strings.TrimSpace(strings.Join(result, "\n"))
}

// --- 8. Strip leading blank lines ---

var leadingBlankLinesPattern = regexp.MustCompile(`^(?:[ \t]*\r?\n)+`)

func stripLeadingBlankLines(content string) string {
	return leadingBlankLinesPattern.ReplaceAllString(content, "")
}

// --- 9. Config leak detection (predefined agents) ---

// configLeakFileNames are internal file names that should not appear in user-facing output
// when a predefined agent describes its procedures or configuration.
var configLeakFileNames = []string{
	"SOUL.md", "IDENTITY.md", "AGENTS.md", "BOOTSTRAP.md",
	"internal_config", "system prompt",
}

// Patterns to strip markdown code from content before config leak detection.
// Mentions inside code blocks/inline code are typically architecture docs, not leaks.
var fencedCodeBlockPattern = regexp.MustCompile("(?s)```[^`]*```")
var inlineCodePattern = regexp.MustCompile("`[^`\n]+`")

// stripMarkdownCode removes fenced code blocks and inline code from text.
func stripMarkdownCode(s string) string {
	s = fencedCodeBlockPattern.ReplaceAllString(s, "")
	s = inlineCodePattern.ReplaceAllString(s, "")
	return s
}

// StripConfigLeak detects when a predefined agent dumps its internal configuration
// (e.g. referencing SOUL.md, AGENTS.md, IDENTITY.md) and replaces the entire
// response with a friendly decline.
//
// Only active for predefined agents. Single-gate detection:
// 3+ distinct internal file names mentioned in plain text → replace entire response.
// Mentions inside markdown code blocks and inline code are excluded from counting,
// as they typically appear in architecture explanations rather than actual leaks.
func StripConfigLeak(content, agentType string) string {
	if agentType != store.AgentTypePredefined || content == "" {
		return content
	}

	// Count hits only in plain text (outside code blocks/inline code)
	plain := stripMarkdownCode(content)

	hits := 0
	for _, name := range configLeakFileNames {
		if strings.Contains(plain, name) {
			hits++
		}
	}
	if hits < 3 {
		return content
	}

	slog.Warn("security.config_leak_stripped",
		"file_hits", hits,
		"original_len", len(content),
	)

	return "🔒 Security check not passed."
}

// --- NO_REPLY detection ---

// IsSilentReply checks if the text contains a standalone NO_REPLY token.
//
// Divergent from TS isSilentReplyText() (exact-match only) — we match broadly so
// both prefix forms (`NO_REPLY because offline`) and terminal sentinel forms
// (`not for me. NO_REPLY`) suppress delivery. The token must not be glued to
// another word (`NO_REPLYING` and `XNO_REPLY` are not silent). Case-insensitive.
func IsSilentReply(text string) bool {
	trimmed := strings.TrimSpace(text)
	if trimmed == "" {
		return false
	}
	// Strip decorative wrappers from both ends (quotes, markdown emphasis, punctuation).
	stripped := strings.Trim(trimmed, "_ \t\n\r.,:;!?\"'`*~#>-()[]{}")
	const token = "NO_REPLY"
	return containsStandaloneNoReplyToken(stripped, token)
}

func containsStandaloneNoReplyToken(text, token string) bool {
	if len(text) < len(token) {
		return false
	}
	for i := 0; i+len(token) <= len(text); i++ {
		if !strings.EqualFold(text[i:i+len(token)], token) {
			continue
		}
		beforeOK := i == 0 || !isAlphaNumByte(text[i-1])
		after := i + len(token)
		afterOK := after == len(text) || !isAlphaNumByte(text[after])
		if beforeOK && afterOK {
			return true
		}
	}
	return false
}

func isAlphaNumByte(b byte) bool {
	return (b >= 'a' && b <= 'z') ||
		(b >= 'A' && b <= 'Z') ||
		(b >= '0' && b <= '9')
}

func isAlphaNum(r rune) bool {
	return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')
}

func isWordChar(r rune) bool {
	return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_'
}

// --- Message Directives ([[name:value]]) ---

// messageDirectivePattern matches structured routing tags: [[word]] or [[word:value]].
// Single-line only (no (?s) dotall). Does NOT match arbitrary [[...]] content.
var messageDirectivePattern = regexp.MustCompile(`\[\[\w+(?::[^\]\n]+)?\]\]`)

// StripMessageDirectives removes internal [[...]] routing tags from user-facing text,
// preserving [[tts...]] tags needed by the TTS auto-apply pipeline.
func StripMessageDirectives(content string) string {
	if !strings.Contains(content, "[[") {
		return content
	}
	result := messageDirectivePattern.ReplaceAllStringFunc(content, func(match string) string {
		inner := match[2 : len(match)-2] // strip [[ and ]]
		if strings.HasPrefix(inner, "tts") {
			return match // preserve for TTS AutoTagged mode
		}
		return ""
	})
	return strings.TrimSpace(result)
}