DocsGPT/docsgpt/agents/workflows/workflow_engine.py

import json
import logging
import re
import time
import uuid
from datetime import datetime, timezone
from typing import Any, Dict, Generator, List, Optional, TYPE_CHECKING

from docsgpt import tracing
from docsgpt.agents.workflows.cel_evaluator import CelEvaluationError, evaluate_cel
from docsgpt.agents.workflows.node_agent import WorkflowNodeAgentFactory
from docsgpt.agents.workflows.schemas import (
    AgentNodeConfig,
    AgentType,
    CodeNodeConfig,
    ConditionNodeConfig,
    ExecutionStatus,
    NodeExecutionLog,
    NodeType,
    WorkflowGraph,
    WorkflowNode,
)
from docsgpt.core.json_schema_utils import (
    JsonSchemaValidationError,
    normalize_json_schema_payload,
)
from docsgpt.error import sanitize_api_error
from docsgpt.templates.namespaces import NamespaceManager
from docsgpt.templates.template_engine import TemplateEngine, TemplateRenderError

try:
    import jsonschema
except ImportError:  # pragma: no cover - optional dependency in some deployments.
    jsonschema = None

if TYPE_CHECKING:
    from docsgpt.agents.base import BaseAgent
logger = logging.getLogger(__name__)

StateValue = Any
WorkflowState = Dict[str, StateValue]
TEMPLATE_RESERVED_NAMESPACES = {"agent", "artifacts", "system", "source", "tools", "passthrough"}

# Run ids become a sandbox-session / kernel-workspace path component; the gateway
# only accepts [A-Za-z0-9_-]+, so any disallowed character is stripped before binding.
_SESSION_ID_RE = re.compile(r"[^A-Za-z0-9_-]+")

# State keys never staged into a code node's ``state.json``. ``chat_history`` is
# the caller's conversation and must not be reachable by owner-authored,
# egress-open sandbox code (see ``_json_safe_state``).
_CODE_STATE_EXCLUDED_KEYS = frozenset({"chat_history"})

# Sentinel id stamped on the synthetic text attachment that flags documents a
# node skipped past the per-node blocking-extract cap (callers/tests match on it).
_EXTRACT_TRUNCATION_ID = "workflow-extract-truncated"


class WorkflowEngine:
    MAX_EXECUTION_STEPS = 50

    def __init__(
        self,
        graph: WorkflowGraph,
        agent: "BaseAgent",
        workflow_run_id: Optional[str] = None,
    ):
        """Bind the engine to a graph + agent; mint a run id for run-scoped sandbox/artifacts."""
        self.graph = graph
        self.agent = agent
        # The run id scopes the code-node sandbox session and every produced
        # artifact's parent; mint one when the caller has not supplied a
        # pre-created ``workflow_runs`` id so the engine is self-contained.
        self.workflow_run_id: str = workflow_run_id or str(uuid.uuid4())
        # Per-node tool-call summary collected by the node executors and folded
        # into the step log; reset before each node runs.
        self._last_node_tool_calls: List[Dict[str, Any]] = []
        # False when no ``workflow_runs`` row backs ``workflow_run_id`` (unsaved /
        # embedded draft), so run-scoped artifacts must NOT be persisted as orphans.
        # Defaults True; WorkflowAgent flips it off on the draft path.
        self.run_persisted: bool = True
        # Transient, in-memory ``artifact_id -> text`` for input documents whose text
        # was already extracted when the attachment was stored, so a node reuses it
        # instead of re-parsing (re-OCRing) the same bytes. Deliberately NOT part of
        # ``state``: state is snapshotted into the run row, and the text is large.
        # Populated once per run by ``WorkflowAgent._bridge_attachments``; lifetime is
        # this engine instance, i.e. exactly the run.
        self.preextracted_text: Dict[str, str] = {}
        self.state: WorkflowState = {}
        self.execution_log: List[Dict[str, Any]] = []
        self._condition_result: Optional[str] = None
        self._template_engine = TemplateEngine()
        self._namespace_manager = NamespaceManager()

    def execute(
        self, initial_inputs: WorkflowState, query: str
    ) -> Generator[Dict[str, str], None, None]:
        """Run the workflow graph, closing the run-scoped sandbox session once when the run ends."""
        try:
            yield from self._run_graph(initial_inputs, query)
        finally:
            # The sandbox session is keyed by the run id and shared by every code
            # node and agent-node tool in this run, so it is torn down exactly once
            # here rather than per node. peek_manager() never builds the manager, so
            # a run that never opened a session closes nothing.
            from docsgpt.sandbox.sandbox_creator import SandboxCreator

            mgr = SandboxCreator.peek_manager()
            if mgr is not None:
                try:
                    mgr.close(self._session_id())
                except Exception:
                    logger.exception("Workflow run failed to close its sandbox session")

    def _run_graph(
        self, initial_inputs: WorkflowState, query: str
    ) -> Generator[Dict[str, str], None, None]:
        self._initialize_state(initial_inputs, query)
        # A workflow run's Logs row finds its trace by this id; a nested
        # engine (a workflow inside a workflow) keeps the outermost run's.
        tracing.bind_if_unset(workflow_run_id=self.workflow_run_id)

        # Surface the run id up front so the client can list this run's
        # artifacts (GET /api/artifacts?workflow_run_id=) once it has been
        # persisted; the same id parents every artifact produced by code nodes.
        yield {"type": "workflow_run", "workflow_run_id": self.workflow_run_id}

        start_node = self.graph.get_start_node()
        if not start_node:
            yield {"type": "error", "error": "No start node found in workflow."}
            return
        current_node_id: Optional[str] = start_node.id
        steps = 0
        # Snapshots are stored as per-node DELTAS: full state copied into
        # every step grows the run row O(n^2) and repeats every upstream
        # output verbatim. Point-in-time state = merge of deltas up to a step.
        # The empty baseline attributes the pre-loop initialization (query,
        # input documents) to the first step so steps alone reconstruct state.
        pre_state: Dict[str, Any] = {}

        while current_node_id and steps < self.MAX_EXECUTION_STEPS:
            node = self.graph.get_node_by_id(current_node_id)
            if not node:
                yield {"type": "error", "error": f"Node {current_node_id} not found."}
                break
            log_entry = self._create_log_entry(node)
            self._last_node_tool_calls = []
            step_span = tracing.start_span(
                tracing.KIND_STEP,
                f"workflow_step {node.title or node.type.value}",
                attributes={
                    "docsgpt.workflow.node_id": node.id,
                    "docsgpt.workflow.node_type": node.type.value,
                    "docsgpt.workflow.node_title": node.title or None,
                    "docsgpt.workflow_run_id": self.workflow_run_id,
                },
            )

            yield {
                "type": "workflow_step",
                "node_id": node.id,
                "node_type": node.type.value,
                "node_title": node.title,
                "status": "running",
            }

            try:
                with step_span:
                    yield from self._execute_node(node)
                log_entry["status"] = ExecutionStatus.COMPLETED.value
                self._finalize_log_entry(log_entry, pre_state)

                output_key = f"node_{node.id}_output"
                node_output = self.state.get(output_key)

                yield {
                    "type": "workflow_step",
                    "node_id": node.id,
                    "node_type": node.type.value,
                    "node_title": node.title,
                    "status": "completed",
                    "state_delta": log_entry["state_delta"],
                    "output": node_output,
                }
            except Exception as e:
                logger.error(f"Error executing node {node.id}: {e}", exc_info=True)
                log_entry["status"] = ExecutionStatus.FAILED.value
                log_entry["error"] = str(e)
                self._finalize_log_entry(log_entry, pre_state)
                self.execution_log.append(log_entry)

                # A misconfigured expression is the builder's own typo, and the
                # message says exactly how to fix it. Routing it through
                # sanitize_api_error would match the ``{`` in ``{{query}}`` and
                # replace the whole thing with "An error occurred … try again
                # later" — advice that sends the user round the same loop.
                # Everything else stays sanitized: node internals and upstream
                # provider errors are not for end users.
                is_config_error = isinstance(e, CelEvaluationError)
                user_friendly_error = (
                    f"{node.title or node.type.value}: {e}"
                    if is_config_error
                    else sanitize_api_error(e)
                )
                yield {
                    "type": "workflow_step",
                    "node_id": node.id,
                    "node_type": node.type.value,
                    "node_title": node.title,
                    "status": "failed",
                    "state_delta": log_entry["state_delta"],
                    "error": user_friendly_error,
                }
                yield {
                    "type": "error",
                    "error": user_friendly_error,
                    "user_facing": is_config_error,
                }
                break
            self.execution_log.append(log_entry)
            pre_state = dict(self.state)

            if node.type == NodeType.END:
                break
            current_node_id = self._get_next_node_id(current_node_id)
            if current_node_id is None and node.type != NodeType.END:
                logger.warning(
                    f"Branch ended at node '{node.title}' ({node.id}) without reaching an end node"
                )
            steps += 1
        if steps >= self.MAX_EXECUTION_STEPS:
            logger.warning(
                f"Workflow reached max steps limit ({self.MAX_EXECUTION_STEPS})"
            )

    def _initialize_state(self, initial_inputs: WorkflowState, query: str) -> None:
        self.state.update(initial_inputs)
        self.state["query"] = query
        self.state["chat_history"] = str(self.agent.chat_history)

    def _create_log_entry(self, node: WorkflowNode) -> Dict[str, Any]:
        return {
            "node_id": node.id,
            "node_type": node.type.value,
            "started_at": datetime.now(timezone.utc),
            "completed_at": None,
            "status": ExecutionStatus.RUNNING.value,
            "error": None,
            "state_delta": {},
        }

    def _finalize_log_entry(self, log_entry: Dict[str, Any], pre_state: Dict[str, Any]) -> None:
        """Stamp completion time, duration, the node's state delta, and its tool-call summary."""
        completed_at = datetime.now(timezone.utc)
        log_entry["completed_at"] = completed_at
        log_entry["duration_ms"] = int((completed_at - log_entry["started_at"]).total_seconds() * 1000)
        log_entry["state_delta"] = self._state_delta(pre_state)
        if self._last_node_tool_calls:
            log_entry["tool_calls"] = list(self._last_node_tool_calls)

    def _state_delta(self, previous: Dict[str, Any]) -> Dict[str, Any]:
        """Keys this node added or changed. Deleted keys are not tracked; nothing deletes state today."""
        return {
            key: value
            for key, value in self.state.items()
            if key not in previous or previous[key] != value
        }

    @staticmethod
    def _summarize_tool_calls(node_agent: Any) -> List[Dict[str, Any]]:
        """Compact per-node tool-call summary for the run record (no arguments/results)."""
        executor = getattr(node_agent, "tool_executor", None)
        calls = getattr(executor, "tool_calls", None) or []
        return [
            {
                "tool_name": call.get("tool_name"),
                "action_name": call.get("action_name"),
                "status": call.get("status", "completed"),
            }
            for call in calls
        ]

    def _get_next_node_id(self, current_node_id: str) -> Optional[str]:
        node = self.graph.get_node_by_id(current_node_id)
        edges = self.graph.get_outgoing_edges(current_node_id)
        if not edges:
            return None

        if node and node.type == NodeType.CONDITION and self._condition_result:
            target_handle = self._condition_result
            self._condition_result = None
            for edge in edges:
                if edge.source_handle == target_handle:
                    return edge.target_id
            return None

        return edges[0].target_id

    def _execute_node(
        self, node: WorkflowNode
    ) -> Generator[Dict[str, str], None, None]:
        logger.info(f"Executing node {node.id} ({node.type.value})")

        node_handlers = {
            NodeType.START: self._execute_start_node,
            NodeType.NOTE: self._execute_note_node,
            NodeType.AGENT: self._execute_agent_node,
            NodeType.CODE: self._execute_code_node,
            NodeType.STATE: self._execute_state_node,
            NodeType.CONDITION: self._execute_condition_node,
            NodeType.END: self._execute_end_node,
        }

        handler = node_handlers.get(node.type)
        if handler:
            yield from handler(node)

    def _execute_start_node(
        self, node: WorkflowNode
    ) -> Generator[Dict[str, str], None, None]:
        yield from ()

    def _execute_note_node(
        self, node: WorkflowNode
    ) -> Generator[Dict[str, str], None, None]:
        yield from ()

    def _execute_agent_node(
        self, node: WorkflowNode
    ) -> Generator[Dict[str, str], None, None]:
        from docsgpt.core.model_utils import (
            get_api_key_for_provider,
            get_model_capabilities,
            resolve_dispatch_provider,
        )

        from docsgpt.api.answer.services.prompt_renderer import (
            prompt_embeds_documents as _prompt_embeds_documents,
        )

        node_config = AgentNodeConfig(**node.config.get("config", node.config))

        node_docs = (
            self._retrieve_node_sources(node_config) if node_config.sources else []
        )

        if node_config.prompt_template:
            formatted_prompt = self._format_template(node_config.prompt_template)
        else:
            formatted_prompt = self.state.get("query", "")
        node_json_schema = self._normalize_node_json_schema(
            node_config.json_schema, node.title
        )
        node_model_id = node_config.model_id or self.agent.model_id
        # Inherit BYOM scope from parent agent so owner-stored BYOM
        # resolves on shared workflows.
        node_user_id = getattr(self.agent, "model_user_id", None) or (
            self.agent.decoded_token.get("sub")
            if isinstance(self.agent.decoded_token, dict)
            else None
        )
        # ``node_config.llm_name`` is whatever the builder stored, which for
        # catalogs with a ``display_provider`` is a presentation label
        # ("foundry", "azure_foundry", "cloudflare") that no LLM class is
        # registered under. Trusting it verbatim fails the node before any LLM
        # call and hands the user a blank answer, so resolve it to a real
        # dispatch provider first — this also repairs nodes already saved with
        # a label, without a migration.
        node_llm_name = resolve_dispatch_provider(
            node_config.llm_name,
            node_model_id,
            user_id=node_user_id,
            fallback=self.agent.llm_name,
        )
        # Resolve the key from the normalized name: get_api_key_for_provider
        # silently returns settings.API_KEY for names it does not know, so a
        # label here would leak the deployment key to the wrong endpoint.
        node_api_key = get_api_key_for_provider(node_llm_name) or self.agent.api_key

        # Structured output gates on the model's registry capability flags;
        # fetch them only when a node json_schema needs the check.
        if node_json_schema and node_model_id:
            model_capabilities = get_model_capabilities(node_model_id, user_id=node_user_id)
            if model_capabilities and not model_capabilities.get(
                "supports_structured_output", False
            ):
                raise ValueError(
                    f'Model "{node_model_id}" does not support structured output for node "{node.title}"'
                )

        node_prompt = node_config.system_prompt
        doc_manifest = self._node_document_manifest(node_config)
        if doc_manifest:
            node_prompt = f"{node_prompt}\n\n{doc_manifest}" if node_prompt else doc_manifest

        # No ``agent_config`` here, deliberately: per-agent guardrails are not
        # wired through workflows yet, so a node runs the instance floor (which
        # ``resolve_config(None)`` still applies) but none of the parent
        # agent's own controls. Passing it would need an aggregate output guard
        # in WorkflowAgent to be meaningful.
        factory_kwargs = {
            "agent_type": node_config.agent_type,
            "endpoint": self.agent.endpoint,
            "llm_name": node_llm_name,
            "model_id": node_model_id,
            "model_user_id": getattr(self.agent, "model_user_id", None),
            "api_key": node_api_key,
            "tool_ids": node_config.tools,
            "tool_owner": self._workflow_owner_id(),
            "tool_principals": self._node_tool_principals(node_config.tools),
            "prompt": node_prompt,
            "chat_history": self.agent.chat_history,
            "decoded_token": self.agent.decoded_token,
            # Attributes the node's token usage to the workflow agent.
            "agent_id": getattr(self.agent, "agent_id", None),
            "json_schema": node_json_schema,
            "retrieved_docs": node_docs,
            # A template that interpolates the documents itself already carries
            # them; suppress the user-turn block so they are not sent twice.
            "prompt_embeds_documents": _prompt_embeds_documents(
                node_config.prompt_template
            ),
            "sources_were_searched": bool(node_config.sources),
        }

        # Agentic/research agents need retriever_config for on-demand search
        if node_config.agent_type in (AgentType.AGENTIC, AgentType.RESEARCH):
            factory_kwargs["retriever_config"] = {
                "source": (
                    {"active_docs": self._authorized_node_sources(node_config.sources)}
                    if node_config.sources
                    else {}
                ),
                "retriever_name": node_config.retriever or "classic",
                "chunks": int(node_config.chunks) if node_config.chunks else 6,
                "model_id": node_model_id,
                "llm_name": node_llm_name,
                "api_key": node_api_key,
                "decoded_token": self.agent.decoded_token,
            }

        node_agent = WorkflowNodeAgentFactory.create(**factory_kwargs)
        # Attribute this node's tool calls to the run's message and, crucially,
        # namespace their durability journal keys by it: node executors are built
        # without a message_id, so providers that reuse deterministic call ids
        # ("functions.create_artifact:0") would otherwise collide across runs on
        # the tool_call_attempts primary key and drop the later journal rows.
        node_executor = getattr(node_agent, "tool_executor", None)
        if node_executor is not None:
            run_executor = getattr(self.agent, "tool_executor", None)
            node_executor.message_id = getattr(run_executor, "message_id", None)
            self._inherit_caller_policy(node_executor, run_executor)
        # Run-scope the node agent's tools so artifact_generator / code_executor
        # address artifacts by this workflow run: a short ref (A1) created by one
        # node resolves for edit_artifact in a later node within the same run. Only
        # when a workflow_runs row backs the run -- otherwise an artifact parented
        # to this run id would be an orphan (403 on get/download); left unset, the
        # run-scoped tools persist under a conversation parent or cleanly error.
        if self.run_persisted and getattr(node_agent, "tool_executor", None) is not None:
            node_agent.tool_executor.workflow_run_id = self.workflow_run_id

        # Decide native-eligibility from the SAME supported-types list the provider
        # handler filters on at send time, so a mime is never sent native-but-empty
        # and then silently dropped. Read post-construction: BaseAgent consumes
        # ``self.attachments`` at gen time, so assigning here takes effect.
        node_attachments = self._materialize_node_attachments(
            node_config, node.title, self._agent_supported_attachment_types(node_agent)
        )
        if node_attachments:
            node_agent.attachments = node_attachments

        full_response_parts: List[str] = []
        structured_response_parts: List[str] = []
        has_structured_response = False
        first_chunk = True
        for event in node_agent.gen(formatted_prompt):
            # A tool that pauses for approval makes the LLM handler yield
            # ``tool_calls_pending`` and end. An ephemeral node agent has no resume path,
            # so silently continuing would leave the node with empty output (or a confusing
            # "Structured output was expected" when it has a json_schema). Fail visibly.
            if event.get("type") == "tool_calls_pending":
                raise ValueError(
                    f'Node "{node.title}" uses a tool that requires approval, which is not '
                    "supported inside a workflow. Disable require_approval for tools used in "
                    "workflow nodes."
                )
            if "answer" in event:
                chunk = str(event["answer"])
                full_response_parts.append(chunk)
                if event.get("structured"):
                    has_structured_response = True
                    structured_response_parts.append(chunk)
                if node_config.stream_to_user:
                    if first_chunk and hasattr(self, "_has_streamed"):
                        yield {"answer": "\n\n"}
                        first_chunk = False
                    yield event

        if node_config.stream_to_user:
            self._has_streamed = True

        self._last_node_tool_calls = self._summarize_tool_calls(node_agent)
        full_response = "".join(full_response_parts).strip()
        output_value: Any = full_response
        if has_structured_response:
            structured_response = "".join(structured_response_parts).strip()
            response_to_parse = structured_response or full_response
            parsed_success, parsed_structured = self._parse_structured_output(
                response_to_parse
            )
            output_value = parsed_structured if parsed_success else response_to_parse
            if node_json_schema:
                self._validate_structured_output(node_json_schema, output_value)
        elif node_json_schema:
            parsed_success, parsed_structured = self._parse_structured_output(
                full_response
            )
            if not parsed_success:
                raise ValueError(
                    "Structured output was expected but response was not valid JSON"
                )
            output_value = parsed_structured
            self._validate_structured_output(node_json_schema, output_value)

        default_output_key = f"node_{node.id}_output"
        self.state[default_output_key] = output_value

        if node_config.output_variable:
            self.state[node_config.output_variable] = output_value

    def _execute_code_node(
        self, node: WorkflowNode
    ) -> Generator[Dict[str, str], None, None]:
        """Run code in the run-scoped sandbox, persist produced files, and write an artifact reference."""
        from docsgpt.sandbox.artifacts_capture import capture_artifacts, snapshot_signatures
        from docsgpt.sandbox.sandbox_creator import SandboxCreator

        config = CodeNodeConfig(**node.config.get("config", node.config))
        code = config.code or ""
        if not code.strip():
            raise ValueError(f'Code node "{node.title}" has no code to execute.')
        # Code nodes are NEVER Jinja-rendered: state is untrusted (document-derived)
        # so interpolating it into the program would be code injection. Prior state is
        # passed as DATA via ``state.json`` (read below), never templated into code.

        user_id = self._resolve_user_id()
        if not user_id:
            raise ValueError(f'Code node "{node.title}" requires an authenticated user.')

        node_json_schema = self._normalize_node_json_schema(config.json_schema, node.title)
        session_id = self._session_id()
        timeout = self._resolve_code_timeout(config.timeout)

        manager = SandboxCreator.get_manager()
        # The session is keyed by the run id and shared by every code node and every
        # agent-node tool in this run, so it is NOT closed here: closing per node
        # would cold-drop later nodes' interpreter/filesystem state. The run session
        # is torn down once in ``execute``'s finally when the whole run ends.
        manager.open(session_id)
        loaded = self._materialize_code_inputs(manager, session_id, config.inputs, user_id)
        # Stage prior state as DATA the node code reads with
        # ``json.load(open("state.json"))`` -- e.g. ``state["decision"]``. The
        # file lands at the workspace root, which is the kernel cwd, so a
        # relative open resolves it. State is never templated into the program.
        state_json = json.dumps(self._json_safe_state(), default=str).encode("utf-8")
        manager.put_file(session_id, "state.json", state_json)
        pre_signatures = snapshot_signatures(manager, session_id)
        result = manager.exec(session_id, code, timeout=timeout)
        # Code nodes execute the sandbox directly (no tool_executor); record the
        # run as a synthetic tool call so the step log shows it like agent nodes.
        self._last_node_tool_calls = [
            {
                "tool_name": "code_executor",
                "action_name": "run_code",
                "status": "completed" if result.ok else "error",
            }
        ]
        if result.runtime_invalidated:
            # A hard timeout destroyed the workspace runtime, so there is
            # nothing reachable to capture and the timeout below stays primary.
            artifacts = []
        elif self.run_persisted:
            artifacts = capture_artifacts(
                manager,
                session_id,
                pre_signatures,
                user_id=user_id,
                workflow_run_id=self.workflow_run_id,
                produced_by={"node_id": node.id, "node_type": NodeType.CODE.value},
            )
        else:
            # No workflow_runs row backs this run (unsaved/embedded draft): an
            # artifact parented to this run id would be an unreachable orphan
            # (403 on get/download). Skip persistence; the sandbox still ran and
            # ``_build_code_output`` handles the empty-artifacts case.
            artifacts = []

        if not result.ok:
            error = (
                f"{result.error_name}: {result.error_value}"
                if result.error_name
                else (result.error_value or "execution error")
            )
            raise ValueError(f'Code node "{node.title}" failed: {error}')

        # The primary produced file becomes the node's pass-by-reference output;
        # it is JSON primitives only ({artifact_id, version, mime_type, filename})
        # so it survives the workflow_runs state-snapshot serialization. Bytes
        # never enter state. A structured decision (optional json_schema) is
        # parsed from stdout and validated through the existing jsonschema path.
        output_value: Any = self._build_code_output(node, result, artifacts, loaded, node_json_schema)

        default_output_key = f"node_{node.id}_output"
        self.state[default_output_key] = output_value
        if config.output_variable:
            self.state[config.output_variable] = output_value
        yield from ()

    def _build_code_output(
        self,
        node: WorkflowNode,
        result: Any,
        artifacts: List[Dict[str, Any]],
        inputs_loaded: List[str],
        node_json_schema: Optional[Dict[str, Any]],
    ) -> Any:
        """Shape a code node's pass-by-reference output (artifact ref and/or validated decision)."""
        if node_json_schema is not None:
            parsed_success, decision = self._parse_structured_output(result.stdout or "")
            if not parsed_success:
                raise ValueError(
                    f'Code node "{node.title}" must print JSON matching its schema, '
                    "but stdout was not valid JSON"
                )
            self._validate_structured_output(node_json_schema, decision)
            if artifacts:
                # Carry the produced artifact reference alongside the decision so
                # downstream nodes can branch on both.
                if isinstance(decision, dict) and "artifacts" not in decision:
                    decision = {**decision, "artifacts": artifacts}
            return decision
        if artifacts:
            return artifacts[0]
        return {"artifacts": [], "status": "ok"}

    def _materialize_code_inputs(
        self, manager: Any, session_id: str, inputs: List[str], user_id: str
    ) -> List[str]:
        """Stage referenced input artifacts (run-scoped, never cross-tenant) into the workspace."""
        from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
        from docsgpt.core.settings import settings
        from docsgpt.sandbox.artifacts_capture import unique_input_path
        from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
        from docsgpt.storage.db.session import db_readonly
        from docsgpt.storage.storage_creator import StorageCreator
        from docsgpt.utils import safe_filename

        loaded: List[str] = []
        raw_ids = self._resolve_input_artifact_ids(inputs)
        if not raw_ids:
            return loaded
        max_bytes = int(settings.SANDBOX_MAX_INPUT_BYTES or 0)
        storage = StorageCreator.get_storage()
        # Two inputs whose current versions share a filename would clobber each other at the
        # same ``inputs/{name}`` path; track used paths and disambiguate deterministically.
        used_paths: set = set()
        for raw in raw_ids:
            with db_readonly() as conn:
                repo = ArtifactsRepository(conn)
                # A short ref (A1/A2/...) resolves to an id within this run only;
                # the resolved id is re-checked through the run-scoped gate so a ref
                # can never reach another tenant.
                artifact_id = resolve_artifact_id(repo, raw, workflow_run_id=self.workflow_run_id)
                artifact = (
                    repo.get_artifact_in_parent(artifact_id, workflow_run_id=self.workflow_run_id)
                    if artifact_id is not None
                    else None
                )
                if artifact is None:
                    raise ValueError(f"input artifact {raw} not found in this run.")
                version = repo.get_version(artifact_id, artifact["current_version"])
            if not version or not version.get("storage_path"):
                raise ValueError(f"input artifact {artifact_id} has no stored content.")
            declared_size = version.get("size")
            if max_bytes and isinstance(declared_size, (int, float)) and declared_size > max_bytes:
                raise ValueError(
                    f"input artifact {artifact_id} exceeds the {max_bytes}-byte sandbox input limit."
                )
            filename = safe_filename(version.get("filename") or artifact_id)
            file_obj = storage.get_file(version["storage_path"])
            try:
                data = file_obj.read(max_bytes + 1) if max_bytes else file_obj.read()
            finally:
                close = getattr(file_obj, "close", None)
                if callable(close):
                    close()
            if max_bytes and len(data) > max_bytes:
                raise ValueError(
                    f"input artifact {artifact_id} exceeds the {max_bytes}-byte sandbox input limit."
                )
            rel_path = unique_input_path(f"inputs/{filename}", used_paths)
            manager.put_file(session_id, rel_path, data)
            loaded.append(rel_path)
        return loaded

    def _node_document_manifest(self, node_config: AgentNodeConfig) -> str:
        """One-line-per-document manifest of the node's selected input documents.

        Appended to the node's system prompt so the model knows the concrete
        refs (``A1``) and filenames it can pass to document/code tools —
        without it, models guess placeholder names ("attached_file") or paste
        file contents inline. Scoped to THIS node's ``input_documents``
        selection, so it never widens per-node document access. Best-effort:
        a resolution failure drops the manifest, never the node.
        """
        from docsgpt.agents.tools.artifact_ref import make_ref, resolve_artifact_id
        from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
        from docsgpt.storage.db.session import db_readonly

        try:
            raw_ids = self._resolve_input_artifact_ids(node_config.input_documents)
            if not raw_ids:
                return ""
            lines: List[str] = []
            with db_readonly() as conn:
                repo = ArtifactsRepository(conn)
                for raw in raw_ids:
                    artifact_id = resolve_artifact_id(repo, raw, workflow_run_id=self.workflow_run_id)
                    artifact = (
                        repo.get_artifact_in_parent(artifact_id, workflow_run_id=self.workflow_run_id)
                        if artifact_id is not None
                        else None
                    )
                    if artifact is None:
                        continue
                    version = repo.get_version(artifact_id, artifact["current_version"]) or {}
                    ref_seq = (artifact.get("metadata") or {}).get("ref_seq")
                    # Legacy artifacts predate ref_seq; the full id works in every tool.
                    handle = make_ref(int(ref_seq)) if ref_seq else str(artifact_id)
                    filename = version.get("filename") or artifact.get("title") or str(artifact_id)
                    mime = version.get("mime_type") or "application/octet-stream"
                    lines.append(f"- {handle}: {filename} ({mime})")
        except Exception:
            logger.exception("Node document manifest failed; continuing without it")
            return ""
        if not lines:
            return ""
        return (
            "## Input documents for this node\n"
            "These files are staged for this node. Pass a ref (e.g. A1) or the "
            "filename to a document or code tool to read the file:\n" + "\n".join(lines)
        )

    def _materialize_node_attachments(
        self,
        node_config: AgentNodeConfig,
        node_title: str,
        supported_types: List[str],
    ) -> List[Dict[str, Any]]:
        """Resolve a node's selected documents to native/extracted attachment dicts for its LLM."""
        from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
        from docsgpt.core.settings import settings
        from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
        from docsgpt.storage.db.session import db_readonly

        raw_ids = self._resolve_input_artifact_ids(node_config.input_documents)
        if not raw_ids:
            return []

        supported = set(supported_types)
        supports_images = any(t.startswith("image/") for t in supported)
        max_files = int(settings.WORKFLOW_NODE_NATIVE_MAX_FILES)
        extract_max = int(settings.WORKFLOW_NODE_EXTRACT_MAX_FILES)
        # One wall clock for every blocking parse this node issues. The cap
        # above bounds how MANY parses run; this bounds how LONG they take in
        # total, so N documents cannot serialize N size-scaled windows.
        parse_deadline = time.monotonic() + float(
            settings.WORKFLOW_NODE_EXTRACT_BUDGET_SECONDS
        )
        max_bytes = int(settings.SANDBOX_MAX_INPUT_BYTES)

        # One read-only connection for the whole batch; the resolved-version
        # rows are collected, then storage reads happen outside the DB context.
        resolved: List[tuple] = []
        with db_readonly() as conn:
            repo = ArtifactsRepository(conn)
            for raw in raw_ids:
                # A short ref (A1/...) resolves to an id within this run only; the
                # resolved id is re-checked through the run-scoped gate so a forged
                # or cross-run ref can never reach another tenant's bytes.
                artifact_id = resolve_artifact_id(repo, raw, workflow_run_id=self.workflow_run_id)
                artifact = (
                    repo.get_artifact_in_parent(artifact_id, workflow_run_id=self.workflow_run_id)
                    if artifact_id is not None
                    else None
                )
                if artifact is None:
                    raise ValueError(
                        f'Document "{raw}" for node "{node_title}" was not found in this run.'
                    )
                version = repo.get_version(artifact_id, artifact["current_version"])
                if not version or not version.get("storage_path"):
                    raise ValueError(f"input document {artifact_id} has no stored content.")
                resolved.append((str(artifact_id), artifact, version))

        attachments: List[Dict[str, Any]] = []
        native_count = 0
        extract_count = 0
        dropped_for_cap = 0
        for artifact_id, artifact, version in resolved:
            storage_path = version["storage_path"]
            mime_type = version.get("mime_type") or "application/octet-stream"
            filename = version.get("filename") or artifact.get("title") or artifact_id
            size = version.get("size")
            if isinstance(size, int) and size > max_bytes:
                logger.warning(
                    "Workflow node %s: document %s (%d bytes) exceeds the %d-byte cap; skipping",
                    node_title, artifact_id, size, max_bytes,
                )
                continue

            native_ok = self._is_native_mime(mime_type, supported, supports_images)
            policy = node_config.file_passing
            if policy == "native" and not native_ok:
                raise ValueError(
                    f'Model "{node_config.model_id or self.agent.model_id}" cannot read '
                    f'"{mime_type}" natively for node "{node_title}".'
                )
            go_native = policy == "native" or (policy == "auto" and native_ok)
            if go_native and native_count >= max_files:
                logger.warning(
                    "Workflow node %s: native file cap (%d) reached; extracting %s instead",
                    node_title, max_files, filename,
                )
                go_native = False

            if go_native:
                # No bytes copied: the provider reads them from storage via ``path``.
                attachments.append({"id": artifact_id, "mime_type": mime_type, "path": storage_path})
                native_count += 1
            else:
                # Inline-text mimes are read directly (cheap), and a document whose
                # text was already extracted when its attachment was stored is reused
                # as-is (no parse at all); every other mime routes through the parsing
                # worker -- a blocking, size-scaled per-document call. Cap how many
                # documents a single node sends down THAT path so a node referencing
                # many non-native documents (e.g. the ``*`` token) can't serialize
                # dozens of parses. Inline text and reused text are not capped: they
                # cost no blocking parse, so charging them would starve the budget.
                needs_parse = (
                    artifact_id not in self.preextracted_text
                    and not self._is_inline_text_mime(mime_type)
                )
                if needs_parse:
                    # Count (and gate on) the parse ATTEMPT, not the success: a
                    # timed-out/failed parse is the full-window worst case we must
                    # bound, so it has to consume cap budget too. Otherwise a degraded
                    # parsing backend (every parse fails) never advances the count
                    # and the node keeps issuing blocking parses without limit.
                    if extract_count >= extract_max:
                        dropped_for_cap += 1
                        continue
                    extract_count += 1
                content = self._extract_attachment_text(
                    artifact_id, storage_path, mime_type, filename, max_bytes,
                    size=size, deadline=parse_deadline,
                )
                if content is None:
                    logger.warning(
                        "Workflow node %s: could not extract text from %s; skipping",
                        node_title, filename,
                    )
                    continue
                # A non-native mime routes this through ``_append_unsupported_attachments``,
                # which inlines ``content`` as text in the system prompt.
                attachments.append(
                    {"id": artifact_id, "mime_type": "text/plain", "content": content}
                )
        if dropped_for_cap:
            logger.warning(
                "Workflow node %s: blocking-extract cap (%d) reached; %d document(s) omitted",
                node_title, extract_max, dropped_for_cap,
            )
            attachments.append(
                self._extract_truncation_note(node_title, extract_max, dropped_for_cap)
            )
        return attachments

    @staticmethod
    def _extract_truncation_note(node_title: str, cap: int, dropped: int) -> Dict[str, Any]:
        """Build a non-fatal text attachment flagging documents skipped past the per-node extract cap."""
        content = (
            f'[Notice] Only the first {cap} document(s) for node "{node_title}" were extracted '
            f"to text; {dropped} additional document(s) were omitted to bound execution time. "
            "Reference fewer documents, or use a model that reads them natively."
        )
        return {"id": _EXTRACT_TRUNCATION_ID, "mime_type": "text/plain", "content": content}

    @staticmethod
    def _agent_supported_attachment_types(node_agent: "BaseAgent") -> List[str]:
        """Return the provider's authoritative supported attachment mime types (handler's source)."""
        llm = getattr(node_agent, "llm", None)
        getter = getattr(llm, "get_supported_attachment_types", None)
        if not callable(getter):
            return []
        types = getter()
        return list(types) if isinstance(types, (list, tuple, set)) else []

    @staticmethod
    def _is_native_mime(mime_type: str, supported_types: set, supports_images: bool) -> bool:
        """A mime is native if the model accepts it, or it is a PDF a vision model renders to images."""
        if mime_type in supported_types:
            return True
        return mime_type == "application/pdf" and supports_images

    def _extract_attachment_text(
        self,
        artifact_id: str,
        storage_path: str,
        mime_type: str,
        filename: str,
        max_bytes: int,
        size: Optional[int] = None,
        deadline: Optional[float] = None,
    ) -> Optional[str]:
        """Get an attachment's text: reuse upload-time extraction, else inline text mimes, else parse.

        Args:
            deadline: ``time.monotonic()`` value past which no further blocking
                parse may run, shared across the node's documents.
        """
        from docsgpt.parser.document_reader import truncate_text_head_tail
        from docsgpt.storage.storage_creator import StorageCreator

        # An uploaded chat attachment was already parsed (and possibly OCR'd) when it
        # was stored, so re-parsing it here would repeat the dominant cost of the run
        # for every node that references it. Bounded with the same head+tail window the
        # inline-text path uses.
        preextracted = self.preextracted_text.get(artifact_id)
        if preextracted:
            logger.info(
                "Workflow node: reusing upload-time extraction for %s (%s); skipping re-parse",
                filename,
                artifact_id,
            )
            return truncate_text_head_tail(preextracted)
        if self._is_inline_text_mime(mime_type):
            try:
                data = StorageCreator.get_storage().get_file(storage_path).read()
            except Exception:
                logger.exception("Workflow node: failed to read document bytes for extraction")
                return None
            # Defensive size gate: a NULL/missing version size skips the pre-read cap,
            # so re-check the actual bytes before inlining.
            if len(data) > max_bytes:
                logger.warning(
                    "Workflow node: document at %s (%d bytes) exceeds the %d-byte cap; skipping",
                    storage_path, len(data), max_bytes,
                )
                return None
            try:
                text = data.decode("utf-8", errors="replace")
            except Exception:
                return None
            # Bound the inlined text to a head+tail window so a large-but-under-cap
            # text file can't blow the context (the parse branch is already bounded).
            return truncate_text_head_tail(text)
        # Non-text mimes parse via the dedicated parsing queue (works on any backend,
        # no sandbox): the worker re-resolves the artifact run-scoped and reads its bytes.
        return self._parse_document_text(artifact_id, size=size, deadline=deadline)

    def _parse_document_text(
        self,
        artifact_id: str,
        size: Optional[int] = None,
        deadline: Optional[float] = None,
    ) -> Optional[str]:
        """Enqueue ``parse_document`` for this run and await the size-scaled markdown; None on failure.

        Args:
            artifact_id: Run-scoped artifact to parse.
            size: Stored byte size, used to scale the per-document window.
            deadline: ``time.monotonic()`` value the await must not outlive,
                shared with the node's other documents.
        """
        from celery.exceptions import TimeoutError as CeleryTimeoutError

        from docsgpt.api.user.tasks import (
            parse_document,
            parse_task_time_limits,
            parse_timeout_for_size,
        )
        from docsgpt.core.settings import settings

        user_id = self._resolve_user_id()
        if not user_id:
            return None
        options = {"output": "markdown", "include_tables": False, "persist": False}
        queue = settings.DOCUMENT_PARSE_QUEUE
        # OCR cost scales with pages, so the window grows with the document's size
        # (floored at DOCUMENT_PARSE_TIMEOUT); the task's per-call time limits are
        # raised to match, else the worker would self-terminate mid-parse.
        timeout = parse_timeout_for_size(size)
        if deadline is not None:
            timeout = min(timeout, deadline - time.monotonic())
            if timeout <= 0:
                logger.warning(
                    "Workflow node: parse budget exhausted; skipping %s", artifact_id
                )
                return None
        try:
            async_result = parse_document.apply_async(
                args=[artifact_id, {"workflow_run_id": self.workflow_run_id}, user_id, options],
                queue=queue,
                **parse_task_time_limits(timeout),
            )
            # A workflow can run inside a Celery worker (scheduled runs / webhooks).
            # In a prefork worker ``task_join_will_block()`` is process-wide, so the
            # default ``disable_sync_subtasks=True`` makes ``get()`` raise RuntimeError
            # ("Never call result.get() within a task!"). The dedicated parsing queue +
            # separate workers already avoid the real self-deadlock, so opt out
            # explicitly (mirrors docsgpt/agents/tools/read_document.py).
            result = async_result.get(timeout=timeout, disable_sync_subtasks=False)
        except (CeleryTimeoutError, TimeoutError):
            logger.warning("Workflow node: document parse timed out for %s", artifact_id)
            return None
        except Exception:
            logger.exception("Workflow node: document parse failed")
            return None
        if isinstance(result, dict) and result.get("status") == "ok":
            content = result.get("content")
            return content if isinstance(content, str) else None
        return None

    @staticmethod
    def _is_inline_text_mime(mime_type: str) -> bool:
        """Already-text formats are inlined directly (no Docling round-trip)."""
        if mime_type.startswith("text/"):
            return True
        return mime_type in ("application/json", "application/xml")

    def _resolve_input_artifact_ids(self, inputs: List[str]) -> List[str]:
        """Resolve node ``inputs`` (refs/ids, ``*`` token, or state vars holding a ref or a list of refs)."""
        resolved: List[str] = []
        for raw in inputs or []:
            # ``*`` / ``input_documents`` expands to every run input document.
            if isinstance(raw, str) and raw.strip() in ("*", "input_documents"):
                resolved.extend(self._input_document_ids())
                continue
            ref = self.state.get(raw) if isinstance(raw, str) else None
            if isinstance(ref, dict) and ref.get("artifact_id"):
                resolved.append(str(ref["artifact_id"]))
            elif isinstance(ref, list):
                # A state var holding a list of ref dicts (e.g. input_documents).
                for item in ref:
                    if isinstance(item, dict) and item.get("artifact_id"):
                        resolved.append(str(item["artifact_id"]))
            elif isinstance(raw, str) and raw.strip():
                resolved.append(raw.strip())
        # Dedup preserving order so ``["*", "A1"]`` / duplicate refs don't attach twice.
        return list(dict.fromkeys(resolved))

    def _input_document_ids(self) -> List[str]:
        """Return the artifact ids of every ref in ``state['input_documents']``."""
        docs = self.state.get("input_documents")
        if not isinstance(docs, list):
            return []
        return [str(d["artifact_id"]) for d in docs if isinstance(d, dict) and d.get("artifact_id")]

    def _session_id(self) -> str:
        """Sanitize the run id into the sandbox-gateway charset for the session key."""
        return _SESSION_ID_RE.sub("-", str(self.workflow_run_id)) or str(uuid.uuid4())

    def _json_safe_state(self) -> Dict[str, Any]:
        """Project ``self.state`` to a JSON-safe dict (the code node reads it from state.json).

        ``chat_history`` is excluded: it is the *caller's* full conversation, and a
        code node (authored by the workflow owner, who may differ from the runner
        in a shared agent) has no legitimate need for it. Since sandbox egress is
        open by design, staging it would let owner-authored code exfiltrate the
        runner's conversation. Node outputs and ``query`` are still exposed.
        """
        projection: Dict[str, Any] = {}
        for key, value in self.state.items():
            if not isinstance(key, str):
                continue
            normalized_key = key.strip()
            if not normalized_key or normalized_key in _CODE_STATE_EXCLUDED_KEYS:
                continue
            projection[normalized_key] = value
        return projection

    def _resolve_user_id(self) -> Optional[str]:
        """Resolve the run's owner for artifact ownership/quota accounting."""
        user_id = getattr(self.agent, "user", None)
        if user_id:
            return user_id
        token = getattr(self.agent, "decoded_token", None)
        if isinstance(token, dict):
            return token.get("sub")
        return None

    def _resolve_code_timeout(self, requested: Optional[int]) -> float:
        """Return the stricter of the node's requested timeout and the sandbox cap."""
        from docsgpt.core.settings import settings

        cap = float(settings.SANDBOX_EXEC_TIMEOUT)
        if requested is None:
            return cap
        try:
            parsed = int(requested)
        except (TypeError, ValueError):
            return cap
        return float(min(parsed, cap)) if parsed > 0 else cap

    def _execute_state_node(
        self, node: WorkflowNode
    ) -> Generator[Dict[str, str], None, None]:
        config = node.config.get("config", node.config)
        for op in config.get("operations", []):
            expression = op.get("expression", "")
            target_variable = op.get("target_variable", "")
            if expression and target_variable:
                self.state[target_variable] = evaluate_cel(expression, self.state)
        yield from ()

    def _execute_condition_node(
        self, node: WorkflowNode
    ) -> Generator[Dict[str, str], None, None]:
        config = ConditionNodeConfig(**node.config.get("config", node.config))
        matched_handle = None

        for case in config.cases:
            if not case.expression.strip():
                continue
            try:
                if evaluate_cel(case.expression, self.state):
                    matched_handle = case.source_handle
                    break
            except CelEvaluationError:
                continue

        self._condition_result = matched_handle or "else"
        yield from ()

    def _execute_end_node(
        self, node: WorkflowNode
    ) -> Generator[Dict[str, str], None, None]:
        config = node.config.get("config", node.config)
        output_template = str(config.get("output_template", ""))
        if output_template:
            formatted_output = self._format_template(output_template)
            # A prior streaming node's text otherwise runs straight into the
            # end-node output ("...enterprise growth.Sales analysis complete");
            # insert the same paragraph break streamed nodes use between each
            # other so the segments stay readable.
            if getattr(self, "_has_streamed", False):
                yield {"answer": "\n\n"}
            yield {"answer": formatted_output}
            self._has_streamed = True

    def _parse_structured_output(self, raw_response: str) -> tuple[bool, Optional[Any]]:
        normalized_response = raw_response.strip()
        if not normalized_response:
            return False, None

        try:
            return True, json.loads(normalized_response)
        except json.JSONDecodeError:
            pass

        # Some models wrap structured output in a ```json ... ``` fence or add
        # prose around it; recover the JSON object/array before giving up so a
        # well-formed-but-fenced response still validates.
        candidate = self._strip_json_fence(normalized_response)
        if candidate is not None:
            try:
                return True, json.loads(candidate)
            except json.JSONDecodeError:
                pass

        logger.warning(
            "Workflow agent returned structured output that was not valid JSON"
        )
        return False, None

    @staticmethod
    def _strip_json_fence(text: str) -> Optional[str]:
        """Extract the JSON payload from a fenced/prose-wrapped response, or None."""
        fence = re.search(r"```(?:json)?\s*(.*?)\s*```", text, re.DOTALL)
        if fence:
            return fence.group(1).strip()
        # Fall back to the outermost {...} or [...] span, choosing whichever
        # bracket opens first. Fixing the order to "{" before "[" would slice a
        # top-level array (``[{...},{...}]``) from its first "{" to its last "}",
        # dropping the array framing (invalid JSON) or extracting an inner object
        # that parses as silently-wrong structured data.
        best: Optional[str] = None
        best_start = -1
        for open_ch, close_ch in (("{", "}"), ("[", "]")):
            start = text.find(open_ch)
            end = text.rfind(close_ch)
            if start != -1 and end > start and (best_start == -1 or start < best_start):
                best_start = start
                best = text[start : end + 1]
        return best

    def _normalize_node_json_schema(
        self, schema: Optional[Dict[str, Any]], node_title: str
    ) -> Optional[Dict[str, Any]]:
        if schema is None:
            return None
        try:
            return normalize_json_schema_payload(schema)
        except JsonSchemaValidationError as exc:
            raise ValueError(
                f'Invalid JSON schema for node "{node_title}": {exc}'
            ) from exc

    def _validate_structured_output(self, schema: Dict[str, Any], output_value: Any) -> None:
        if jsonschema is None:
            logger.warning(
                "jsonschema package is not available, skipping structured output validation"
            )
            return

        try:
            normalized_schema = normalize_json_schema_payload(schema)
        except JsonSchemaValidationError as exc:
            raise ValueError(f"Invalid JSON schema: {exc}") from exc

        try:
            jsonschema.validate(instance=output_value, schema=normalized_schema)
        except jsonschema.exceptions.ValidationError as exc:
            raise ValueError(f"Structured output did not match schema: {exc.message}") from exc
        except jsonschema.exceptions.SchemaError as exc:
            raise ValueError(f"Invalid JSON schema: {exc.message}") from exc

    def _format_template(self, template: str) -> str:
        context = self._build_template_context()
        try:
            return self._template_engine.render(template, context)
        except TemplateRenderError as e:
            logger.warning(
                "Workflow template rendering failed, using raw template: %s", str(e)
            )
            return template

    def _build_template_context(self) -> Dict[str, Any]:
        docs, docs_together = self._get_source_template_data()
        passthrough_data = (
            self.state.get("passthrough")
            if isinstance(self.state.get("passthrough"), dict)
            else None
        )
        tools_data = (
            self.state.get("tools") if isinstance(self.state.get("tools"), dict) else None
        )

        context = self._namespace_manager.build_context(
            user_id=getattr(self.agent, "user", None),
            request_id=getattr(self.agent, "request_id", None),
            passthrough_data=passthrough_data,
            docs=docs,
            docs_together=docs_together,
            tools_data=tools_data,
            artifacts_data=self._collect_artifact_refs(),
            artifact_parent={"workflow_run_id": self.workflow_run_id},
            # Node templates gate tool-specific sections on this; an unresolved
            # set would fail open and advertise tools the node does not have.
            enabled_tools=set(),
        )

        agent_context: Dict[str, Any] = {}
        for key, value in self.state.items():
            if not isinstance(key, str):
                continue
            normalized_key = key.strip()
            if not normalized_key:
                continue
            agent_context[normalized_key] = value

        context["agent"] = agent_context

        # Keep legacy top-level variables working while namespaced variables are adopted.
        for key, value in agent_context.items():
            if key in TEMPLATE_RESERVED_NAMESPACES:
                context[f"agent_{key}"] = value
                continue
            if key not in context:
                context[key] = value

        return context

    def _collect_artifact_refs(self) -> Dict[str, Any]:
        """Collect state variables that hold artifact references, keyed by their state name."""
        refs: Dict[str, Any] = {}
        for key, value in self.state.items():
            if not isinstance(key, str):
                continue
            if isinstance(value, dict) and value.get("artifact_id"):
                refs[key] = value
        return refs

    def _get_source_template_data(self) -> tuple[Optional[List[Dict[str, Any]]], Optional[str]]:
        docs = getattr(self.agent, "retrieved_docs", None)
        if not isinstance(docs, list) or len(docs) == 0:
            return None, None

        docs_together_parts: List[str] = []
        for doc in docs:
            if not isinstance(doc, dict):
                continue
            text = doc.get("text")
            if not isinstance(text, str):
                continue

            filename = doc.get("filename") or doc.get("title") or doc.get("source")
            if isinstance(filename, str) and filename.strip():
                docs_together_parts.append(f"{filename}\n{text}")
            else:
                docs_together_parts.append(text)

        docs_together = "\n\n".join(docs_together_parts) if docs_together_parts else None
        return docs, docs_together

    @staticmethod
    def _inherit_caller_policy(node_executor: Any, run_executor: Any) -> None:
        """Give a node's executor the caller rules of the run that started it.

        A node's tools run for the same caller as the workflow agent: a
        scheduled run still can't pause, and an API-key or public-link caller
        still can't write on the owner's account unless it is allowlisted.

        Args:
            node_executor: The node agent's ``ToolExecutor``.
            run_executor: The workflow agent's ``ToolExecutor``, if any.
        """
        if run_executor is None:
            return
        for attr in ("headless", "external_caller", "public_link_caller"):
            setattr(node_executor, attr, bool(getattr(run_executor, attr, False)))
        for attr in ("tool_allowlist", "api_write_allowlist"):
            setattr(node_executor, attr, set(getattr(run_executor, attr, None) or ()))

    def _workflow_owner_id(self) -> Optional[str]:
        """The workflow's owner, whom node tools and sources run as.

        Returns:
            The owner's user id, or None when the run has none.
        """
        resolve_owner = getattr(self.agent, "_resolve_owner_id", None)
        return (resolve_owner() if callable(resolve_owner) else None) or self._resolve_user_id()

    def _node_tool_principals(self, tool_ids) -> Dict[str, str]:
        """Node tool id -> the editor to resolve it as, for sponsored tools.

        Only tools with a live sponsor on the workflow appear; the executor
        still tries the owner first.

        Args:
            tool_ids: The node's configured tool ids.

        Returns:
            dict: ``tool_id -> sponsor`` user id.
        """
        workflow_row = getattr(self.agent, "workflow_row", None)
        if not tool_ids or not workflow_row or not workflow_row.get("resource_sponsors"):
            return {}
        from docsgpt.api.user.resource_access import active_sponsor
        from docsgpt.storage.db.session import db_readonly

        principals: Dict[str, str] = {}
        try:
            with db_readonly() as conn:
                for tid in tool_ids:
                    sponsor = active_sponsor(conn, "workflow", workflow_row, "tool", str(tid))
                    if sponsor:
                        principals[str(tid)] = sponsor
        except Exception:
            logger.exception("Workflow node tool sponsor lookup failed; using the owner only.")
            return {}
        return principals

    def _authorized_node_sources(self, sources) -> list:
        """Filter a node's configured source ids to those its owner may read.

        ``AgentNodeConfig.sources`` is written verbatim from client JSON when a
        workflow is saved and nothing validated it, so a node could name any
        tenant's source id and the retriever — which filters only on
        ``source_id`` — handed the documents back. Gate on the workflow owner
        (not the runner): a shared workflow legitimately reads its owner's
        sources, exactly like a shared agent does. A source the owner can't
        read still passes while the editor who attached it (its sponsor)
        qualifies.

        Args:
            sources: Source ids from the stored node config.

        Returns:
            list: The subset the owner may read.
        """
        if not sources:
            return []
        ids = sources if isinstance(sources, list) else [sources]
        owner = self._workflow_owner_id()
        if not owner:
            logger.warning("Workflow node sources dropped: no owner to authorize.")
            return []

        from docsgpt.api.user.resource_access import active_sponsor
        from docsgpt.api.user.team_sharing import can_access
        from docsgpt.storage.db.session import db_readonly

        workflow_row = getattr(self.agent, "workflow_row", None)
        allowed = []
        try:
            with db_readonly() as conn:
                for sid in ids:
                    if sid and (
                        can_access(conn, "source", str(sid), owner)
                        or active_sponsor(conn, "workflow", workflow_row, "source", str(sid))
                    ):
                        allowed.append(sid)
                    else:
                        logger.warning(
                            "Workflow node source %s dropped: %s has no access.",
                            sid, owner,
                        )
        except Exception:
            logger.exception("Workflow node source authorization failed; dropping all.")
            return []
        return allowed

    def _retrieve_node_sources(self, node_config: AgentNodeConfig) -> list:
        """Retrieve this node's source documents.

        Args:
            node_config: The node's resolved configuration.

        Returns:
            list: Retrieved documents, empty when there was nothing to fetch.
        """
        from docsgpt.retriever.retriever_creator import RetrieverCreator

        query = self.state.get("query", "")
        if not query:
            return []

        try:
            retriever = RetrieverCreator.create_retriever(
                node_config.retriever or "classic",
                source={"active_docs": self._authorized_node_sources(node_config.sources)},
                chat_history=[],
                prompt="",
                chunks=int(node_config.chunks) if node_config.chunks else 6,
                decoded_token=self.agent.decoded_token,
            )
            docs = retriever.search(query)
            if docs:
                # The parent copy still backs ``{{ source.* }}`` template
                # resolution; the return value is what reaches the node agent.
                self.agent.retrieved_docs = docs
            return docs or []
        except Exception:
            logger.exception("Failed to retrieve docs for workflow node")
            return []

    def get_execution_summary(self) -> List[NodeExecutionLog]:
        return [
            NodeExecutionLog(
                node_id=log["node_id"],
                node_type=log["node_type"],
                status=ExecutionStatus(log["status"]),
                started_at=log["started_at"],
                completed_at=log.get("completed_at"),
                duration_ms=log.get("duration_ms"),
                error=log.get("error"),
                state_delta=log.get("state_delta", {}),
                tool_calls=log.get("tool_calls", []),
            )
            for log in self.execution_log
        ]