mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-03 20:12:55 +00:00
Alex
59e9523666
The "Tool approval needed" toast (and several sibling surfaces) could linger after the state they represent was already gone. User-scoped SSE events (tool.approval.required, schedule.autopaused, attachment.queued, …) are durable and replayed on reconnect, but no terminal path emitted a matching clearing event and the reconciler only wrote operator-facing stack_logs — so a failed/expired message replayed its approval prompt with nothing to act on, and the toast trusted event presence over the actual message state. Backend — emit a user-facing event on every terminal path: - reconciler deletes pending_tool_state and publishes tool.approval.cleared when a stuck message is failed; cleanup_pending_tool_state does the same for TTL-reaped rows - reconciler now publishes source.ingest.failed (stalled ingest), schedule.run.failed (timeout/pending) and schedule.completed (once) - schedules PATCH-resume / DELETE publish schedule.resumed / .cancelled so a stale schedule.autopaused can't outvote them on replay - store_attachment gains the on_poison terminal-event hook the ingest tasks already have; mcp_oauth_task gains a soft/hard time limit so a hung flow self-reports mcp.oauth.failed - tighten the reconciler exemption to the (conversation_id, user_id) composite key Frontend — stop trusting event presence over truth: - notificationsSlice.resolveToolApproval evicts the matching tool.approval.required and persists its (stable) id dismissed so the backlog replay stays suppressed - ToolApprovalToast drops approval events older than the resumable TTL window as a backstop for a lost clearing event - schedulesSlice handles schedule.resumed / .cancelled / .completed - upload dismissals now outlive the SSE backlog retention window Tests cover the new clearing events at the reconciler, slice and dispatch layers.
953 lines
32 KiB
Python
953 lines
32 KiB
Python
"""Tests for the reconciler beat task.
|
|
|
|
Seeds stuck rows of each kind (Q1 messages, Q2 proposed tool calls, Q3
|
|
executed tool calls) and verifies that ``run_reconciliation`` either
|
|
retries (Q1 within 3 attempts) or escalates them to terminal status
|
|
with both a structured logger error and a stack_logs row.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from contextlib import contextmanager
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
from sqlalchemy import text
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers — seed rows in shapes that mirror the live writers (T3 reserve_message
|
|
# for stuck messages, T8 record_proposed for tool_call_attempts).
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _create_conv(conn, user_id: str = "u-1") -> dict:
|
|
from application.storage.db.repositories.conversations import (
|
|
ConversationsRepository,
|
|
)
|
|
|
|
return ConversationsRepository(conn).create(user_id, "rec test")
|
|
|
|
|
|
def _seed_pending_message(
|
|
conn,
|
|
*,
|
|
user_id: str = "u-1",
|
|
age_minutes: int = 6,
|
|
status: str = "pending",
|
|
) -> dict:
|
|
"""Insert a conversation_messages row in ``status`` with stale ``timestamp``."""
|
|
conv = _create_conv(conn, user_id=user_id)
|
|
row = conn.execute(
|
|
text(
|
|
"""
|
|
INSERT INTO conversation_messages (
|
|
conversation_id, position, prompt, response, status,
|
|
user_id, timestamp
|
|
)
|
|
VALUES (
|
|
CAST(:conv_id AS uuid), 0, :prompt, :resp, :status,
|
|
:user_id,
|
|
clock_timestamp() - make_interval(mins => :age)
|
|
)
|
|
RETURNING id
|
|
"""
|
|
),
|
|
{
|
|
"conv_id": conv["id"],
|
|
"prompt": "hello",
|
|
"resp": "",
|
|
"status": status,
|
|
"user_id": user_id,
|
|
"age": age_minutes,
|
|
},
|
|
).fetchone()
|
|
return {"id": str(row[0]), "conversation_id": conv["id"], "user_id": user_id}
|
|
|
|
|
|
def _seed_resuming_state(conn, conv_id: str, user_id: str, *, secs_ago: int) -> None:
|
|
"""Insert a pending_tool_state row in ``resuming`` with ``resumed_at`` backdated."""
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
INSERT INTO pending_tool_state (
|
|
conversation_id, user_id, messages, pending_tool_calls,
|
|
tools_dict, tool_schemas, agent_config,
|
|
created_at, expires_at, status, resumed_at
|
|
)
|
|
VALUES (
|
|
CAST(:conv_id AS uuid), :user_id,
|
|
'[]'::jsonb, '[]'::jsonb, '{}'::jsonb, '[]'::jsonb, '{}'::jsonb,
|
|
clock_timestamp(),
|
|
clock_timestamp() + interval '30 minutes',
|
|
'resuming',
|
|
clock_timestamp() - make_interval(secs => :secs_ago)
|
|
)
|
|
"""
|
|
),
|
|
{"conv_id": conv_id, "user_id": user_id, "secs_ago": secs_ago},
|
|
)
|
|
|
|
|
|
def _seed_pending_state(
|
|
conn, conv_id: str, user_id: str, *, expires_in_minutes: int = 30,
|
|
) -> None:
|
|
"""Insert a paused ``pending_tool_state`` row (status='pending').
|
|
|
|
A negative ``expires_in_minutes`` simulates a TTL-expired row that
|
|
the cleanup janitor hasn't yet reaped.
|
|
"""
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
INSERT INTO pending_tool_state (
|
|
conversation_id, user_id, messages, pending_tool_calls,
|
|
tools_dict, tool_schemas, agent_config,
|
|
created_at, expires_at, status, resumed_at
|
|
)
|
|
VALUES (
|
|
CAST(:conv_id AS uuid), :user_id,
|
|
'[]'::jsonb, '[]'::jsonb, '{}'::jsonb, '[]'::jsonb, '{}'::jsonb,
|
|
clock_timestamp(),
|
|
clock_timestamp() + make_interval(mins => :exp),
|
|
'pending',
|
|
NULL
|
|
)
|
|
"""
|
|
),
|
|
{"conv_id": conv_id, "user_id": user_id, "exp": expires_in_minutes},
|
|
)
|
|
|
|
|
|
def _seed_tool_call(
|
|
conn,
|
|
*,
|
|
call_id: str,
|
|
status: str,
|
|
tool_name: str = "notes",
|
|
age_minutes: int = 16,
|
|
tool_id: str | None = None,
|
|
arguments: dict | None = None,
|
|
action_name: str = "view",
|
|
) -> None:
|
|
"""Insert a tool_call_attempts row, then backdate ``attempted_at``/``updated_at``."""
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
INSERT INTO tool_call_attempts (
|
|
call_id, tool_id, tool_name, action_name, arguments, status
|
|
)
|
|
VALUES (
|
|
:call_id, CAST(:tool_id AS uuid), :tool_name, :action_name,
|
|
CAST(:arguments AS jsonb), :status
|
|
)
|
|
"""
|
|
),
|
|
{
|
|
"call_id": call_id,
|
|
"tool_id": tool_id,
|
|
"tool_name": tool_name,
|
|
"action_name": action_name,
|
|
"arguments": json.dumps(arguments or {}),
|
|
"status": status,
|
|
},
|
|
)
|
|
# The set_updated_at BEFORE-UPDATE trigger would clobber updated_at;
|
|
# disable it briefly so the backdate lands.
|
|
conn.execute(text("ALTER TABLE tool_call_attempts DISABLE TRIGGER USER"))
|
|
try:
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
UPDATE tool_call_attempts
|
|
SET attempted_at = clock_timestamp() - make_interval(mins => :age),
|
|
updated_at = clock_timestamp() - make_interval(mins => :age)
|
|
WHERE call_id = :call_id
|
|
"""
|
|
),
|
|
{"call_id": call_id, "age": age_minutes},
|
|
)
|
|
finally:
|
|
conn.execute(text("ALTER TABLE tool_call_attempts ENABLE TRIGGER USER"))
|
|
|
|
|
|
def _stack_logs_count(conn, query: str | None = None) -> int:
|
|
if query is None:
|
|
result = conn.execute(text("SELECT count(*) FROM stack_logs"))
|
|
else:
|
|
result = conn.execute(
|
|
text("SELECT count(*) FROM stack_logs WHERE query = :q"),
|
|
{"q": query},
|
|
)
|
|
return int(result.scalar() or 0)
|
|
|
|
|
|
@contextmanager
|
|
def _route_engine_to(pg_conn):
|
|
"""Patch ``get_engine`` so the run_reconciliation reuses the test conn."""
|
|
|
|
@contextmanager
|
|
def _fake_begin():
|
|
# The repository writes happen on this connection; rolling back
|
|
# at the end of the test cleans them up.
|
|
yield pg_conn
|
|
|
|
fake_engine = MagicMock()
|
|
fake_engine.begin = _fake_begin
|
|
|
|
with patch(
|
|
"application.api.user.reconciliation.get_engine",
|
|
return_value=fake_engine,
|
|
):
|
|
yield
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Q1 — stuck messages
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestStuckMessages:
|
|
@pytest.mark.unit
|
|
def test_first_two_attempts_increment_only(self, pg_conn):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
msg = _seed_pending_message(pg_conn)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
r1 = run_reconciliation()
|
|
r2 = run_reconciliation()
|
|
|
|
assert r1["messages_failed"] == 0
|
|
assert r2["messages_failed"] == 0
|
|
|
|
row = pg_conn.execute(
|
|
text(
|
|
"SELECT status, message_metadata "
|
|
"FROM conversation_messages WHERE id = CAST(:id AS uuid)"
|
|
),
|
|
{"id": msg["id"]},
|
|
).fetchone()
|
|
assert row[0] == "pending"
|
|
assert row[1]["reconcile_attempts"] == 2
|
|
|
|
@pytest.mark.unit
|
|
def test_third_attempt_marks_failed_and_emits_alert(self, pg_conn, caplog):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
msg = _seed_pending_message(pg_conn)
|
|
before_logs = _stack_logs_count(pg_conn, "reconciler_message_failed")
|
|
|
|
with _route_engine_to(pg_conn), caplog.at_level(
|
|
logging.ERROR, logger="application.api.user.reconciliation",
|
|
):
|
|
run_reconciliation()
|
|
run_reconciliation()
|
|
r3 = run_reconciliation()
|
|
|
|
assert r3["messages_failed"] == 1
|
|
|
|
row = pg_conn.execute(
|
|
text(
|
|
"SELECT status, message_metadata "
|
|
"FROM conversation_messages WHERE id = CAST(:id AS uuid)"
|
|
),
|
|
{"id": msg["id"]},
|
|
).fetchone()
|
|
assert row[0] == "failed"
|
|
assert row[1]["reconcile_attempts"] == 3
|
|
assert "reconciler:" in (row[1].get("error") or "")
|
|
|
|
# Structured alert + stack_logs row both surface the failure.
|
|
assert any(
|
|
"reconciler alert" in rec.getMessage()
|
|
and rec.levelname == "ERROR"
|
|
and getattr(rec, "alert", None) == "reconciler_message_failed"
|
|
for rec in caplog.records
|
|
)
|
|
assert (
|
|
_stack_logs_count(pg_conn, "reconciler_message_failed")
|
|
== before_logs + 1
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_streaming_status_also_eligible(self, pg_conn):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
msg = _seed_pending_message(pg_conn, status="streaming")
|
|
with _route_engine_to(pg_conn):
|
|
run_reconciliation()
|
|
run_reconciliation()
|
|
run_reconciliation()
|
|
|
|
row = pg_conn.execute(
|
|
text(
|
|
"SELECT status FROM conversation_messages "
|
|
"WHERE id = CAST(:id AS uuid)"
|
|
),
|
|
{"id": msg["id"]},
|
|
).fetchone()
|
|
assert row[0] == "failed"
|
|
|
|
@pytest.mark.unit
|
|
def test_skipped_when_active_resuming_state(self, pg_conn):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
msg = _seed_pending_message(pg_conn)
|
|
# Active resume started 60 seconds ago — within 10-min grace.
|
|
_seed_resuming_state(pg_conn, msg["conversation_id"], msg["user_id"], secs_ago=60)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
r = run_reconciliation()
|
|
|
|
assert r["messages_failed"] == 0
|
|
row = pg_conn.execute(
|
|
text(
|
|
"SELECT status, message_metadata "
|
|
"FROM conversation_messages WHERE id = CAST(:id AS uuid)"
|
|
),
|
|
{"id": msg["id"]},
|
|
).fetchone()
|
|
assert row[0] == "pending"
|
|
# No attempts recorded: row was skipped, not just not-yet-failed.
|
|
assert "reconcile_attempts" not in row[1]
|
|
|
|
@pytest.mark.unit
|
|
def test_stale_resuming_does_not_skip(self, pg_conn):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
msg = _seed_pending_message(pg_conn)
|
|
# 11 minutes ago — past the 10-minute grace window.
|
|
_seed_resuming_state(
|
|
pg_conn, msg["conversation_id"], msg["user_id"], secs_ago=11 * 60,
|
|
)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
run_reconciliation()
|
|
|
|
row = pg_conn.execute(
|
|
text(
|
|
"SELECT message_metadata FROM conversation_messages "
|
|
"WHERE id = CAST(:id AS uuid)"
|
|
),
|
|
{"id": msg["id"]},
|
|
).fetchone()
|
|
# Stale resuming shouldn't shield: attempt counter must have moved.
|
|
assert row[0].get("reconcile_attempts") == 1
|
|
|
|
@pytest.mark.unit
|
|
def test_fresh_message_left_alone(self, pg_conn):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
# 1 minute old — well under the 5-minute threshold.
|
|
msg = _seed_pending_message(pg_conn, age_minutes=1)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
r = run_reconciliation()
|
|
|
|
assert r["messages_failed"] == 0
|
|
row = pg_conn.execute(
|
|
text(
|
|
"SELECT status, message_metadata FROM conversation_messages "
|
|
"WHERE id = CAST(:id AS uuid)"
|
|
),
|
|
{"id": msg["id"]},
|
|
).fetchone()
|
|
assert row[0] == "pending"
|
|
assert "reconcile_attempts" not in row[1]
|
|
|
|
@pytest.mark.unit
|
|
def test_skipped_when_paused_pending_state_active(self, pg_conn):
|
|
"""A paused conversation (PT.status='pending') must not get failed
|
|
while the user is still considering tool approval. The PT row's
|
|
own ``expires_at`` TTL is the abandonment signal.
|
|
"""
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
msg = _seed_pending_message(pg_conn)
|
|
_seed_pending_state(
|
|
pg_conn, msg["conversation_id"], msg["user_id"],
|
|
expires_in_minutes=30,
|
|
)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
r1 = run_reconciliation()
|
|
r2 = run_reconciliation()
|
|
r3 = run_reconciliation()
|
|
|
|
assert r1["messages_failed"] == 0
|
|
assert r2["messages_failed"] == 0
|
|
assert r3["messages_failed"] == 0
|
|
row = pg_conn.execute(
|
|
text(
|
|
"SELECT status, message_metadata FROM conversation_messages "
|
|
"WHERE id = CAST(:id AS uuid)"
|
|
),
|
|
{"id": msg["id"]},
|
|
).fetchone()
|
|
assert row[0] == "pending"
|
|
# No attempts recorded: the row was excluded from the sweep, not
|
|
# just held under the 3-attempt threshold.
|
|
assert "reconcile_attempts" not in row[1]
|
|
|
|
@pytest.mark.unit
|
|
def test_expired_pending_state_does_not_skip(self, pg_conn):
|
|
"""When the PT row's TTL has expired but the cleanup janitor
|
|
hasn't reaped it yet, the message becomes eligible immediately
|
|
— we don't wait an extra ~60s for janitor cadence to align.
|
|
"""
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
msg = _seed_pending_message(pg_conn)
|
|
_seed_pending_state(
|
|
pg_conn, msg["conversation_id"], msg["user_id"],
|
|
expires_in_minutes=-1,
|
|
)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
run_reconciliation()
|
|
|
|
row = pg_conn.execute(
|
|
text(
|
|
"SELECT message_metadata FROM conversation_messages "
|
|
"WHERE id = CAST(:id AS uuid)"
|
|
),
|
|
{"id": msg["id"]},
|
|
).fetchone()
|
|
assert row[0].get("reconcile_attempts") == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Q2 — proposed tool calls (side effect unknown)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestStuckProposedToolCalls:
|
|
@pytest.mark.unit
|
|
def test_marks_proposed_failed_with_alert(self, pg_conn, caplog):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
_seed_tool_call(pg_conn, call_id="cp-1", status="proposed", age_minutes=6)
|
|
before = _stack_logs_count(pg_conn, "reconciler_tool_call_failed_proposed")
|
|
|
|
with _route_engine_to(pg_conn), caplog.at_level(
|
|
logging.ERROR, logger="application.api.user.reconciliation",
|
|
):
|
|
r = run_reconciliation()
|
|
|
|
assert r["tool_calls_failed"] == 1
|
|
row = pg_conn.execute(
|
|
text("SELECT status, error FROM tool_call_attempts WHERE call_id = :id"),
|
|
{"id": "cp-1"},
|
|
).fetchone()
|
|
assert row[0] == "failed"
|
|
assert "side effect status unknown" in (row[1] or "")
|
|
|
|
assert any(
|
|
getattr(rec, "alert", None) == "reconciler_tool_call_failed_proposed"
|
|
for rec in caplog.records
|
|
)
|
|
assert (
|
|
_stack_logs_count(pg_conn, "reconciler_tool_call_failed_proposed")
|
|
== before + 1
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_fresh_proposed_left_alone(self, pg_conn):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
_seed_tool_call(pg_conn, call_id="cp-2", status="proposed", age_minutes=2)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
r = run_reconciliation()
|
|
|
|
assert r["tool_calls_failed"] == 0
|
|
row = pg_conn.execute(
|
|
text("SELECT status FROM tool_call_attempts WHERE call_id = :id"),
|
|
{"id": "cp-2"},
|
|
).fetchone()
|
|
assert row[0] == "proposed"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Q3 — executed tool calls (escalate to failed; manual cleanup expected)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestStuckExecutedToolCalls:
|
|
@pytest.mark.unit
|
|
def test_executed_past_ttl_marked_failed_with_alert(self, pg_conn, caplog):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
_seed_tool_call(
|
|
pg_conn, call_id="ce-1", status="executed",
|
|
tool_name="notes", age_minutes=16,
|
|
)
|
|
before = _stack_logs_count(pg_conn, "reconciler_tool_call_failed_executed")
|
|
|
|
with _route_engine_to(pg_conn), caplog.at_level(
|
|
logging.ERROR, logger="application.api.user.reconciliation",
|
|
):
|
|
r = run_reconciliation()
|
|
|
|
assert r["tool_calls_failed"] == 1
|
|
row = pg_conn.execute(
|
|
text("SELECT status, error FROM tool_call_attempts WHERE call_id = :id"),
|
|
{"id": "ce-1"},
|
|
).fetchone()
|
|
assert row[0] == "failed"
|
|
assert "executed-not-confirmed" in (row[1] or "")
|
|
assert "manual cleanup" in (row[1] or "")
|
|
assert any(
|
|
getattr(rec, "alert", None) == "reconciler_tool_call_failed_executed"
|
|
for rec in caplog.records
|
|
)
|
|
assert (
|
|
_stack_logs_count(pg_conn, "reconciler_tool_call_failed_executed")
|
|
== before + 1
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_fresh_executed_left_alone(self, pg_conn):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
_seed_tool_call(
|
|
pg_conn, call_id="ce-2", status="executed",
|
|
tool_name="notes", age_minutes=5, # under 15-min threshold
|
|
)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
r = run_reconciliation()
|
|
|
|
assert r["tool_calls_failed"] == 0
|
|
row = pg_conn.execute(
|
|
text("SELECT status FROM tool_call_attempts WHERE call_id = :id"),
|
|
{"id": "ce-2"},
|
|
).fetchone()
|
|
assert row[0] == "executed"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Q4 — stalled ingest checkpoints (escalate to terminal 'stalled' + alert)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _seed_ingest_progress(
|
|
conn,
|
|
*,
|
|
source_id: str,
|
|
embedded: int,
|
|
total: int,
|
|
age_minutes: int = 31,
|
|
status: str = "active",
|
|
) -> str:
|
|
"""Insert an ingest_chunk_progress row with a backdated last_updated."""
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
INSERT INTO ingest_chunk_progress (
|
|
source_id, total_chunks, embedded_chunks, last_index,
|
|
last_updated, status
|
|
)
|
|
VALUES (
|
|
CAST(:sid AS uuid), :total, :embedded, :embedded - 1,
|
|
clock_timestamp() - make_interval(mins => :age),
|
|
:status
|
|
)
|
|
"""
|
|
),
|
|
{
|
|
"sid": source_id,
|
|
"total": total,
|
|
"embedded": embedded,
|
|
"age": age_minutes,
|
|
"status": status,
|
|
},
|
|
)
|
|
return source_id
|
|
|
|
|
|
def _ingest_status(conn, source_id: str) -> str | None:
|
|
"""Return the ``status`` of an ingest_chunk_progress row, or None."""
|
|
row = conn.execute(
|
|
text(
|
|
"SELECT status FROM ingest_chunk_progress "
|
|
"WHERE source_id = CAST(:sid AS uuid)"
|
|
),
|
|
{"sid": source_id},
|
|
).fetchone()
|
|
return row[0] if row is not None else None
|
|
|
|
|
|
def _seed_source(
|
|
conn, *, source_id: str, user_id: str = "u-1", name: str = "My Doc.pdf",
|
|
) -> str:
|
|
"""Insert a minimal ``sources`` row so the ingest sweep can resolve its owner."""
|
|
conn.execute(
|
|
text(
|
|
"INSERT INTO sources (id, user_id, name, type) "
|
|
"VALUES (CAST(:id AS uuid), :user_id, :name, 'file')"
|
|
),
|
|
{"id": source_id, "user_id": user_id, "name": name},
|
|
)
|
|
return source_id
|
|
|
|
|
|
def _capture_published(pg_conn):
|
|
"""Patch ``publish_user_event`` and collect ``(user, type, payload, scope)``.
|
|
|
|
Returns a ``(context_manager, captured_list)`` pair. The reconciler
|
|
imports the publisher lazily inside ``_publish_events``, so patching the
|
|
function on its home module is what intercepts the call.
|
|
"""
|
|
captured: list = []
|
|
|
|
def _fake(user_id, event_type, payload, *, scope=None):
|
|
captured.append((user_id, event_type, payload, scope))
|
|
return "1-0"
|
|
|
|
return patch(
|
|
"application.events.publisher.publish_user_event", _fake,
|
|
), captured
|
|
|
|
|
|
class TestStalledIngests:
|
|
@pytest.mark.unit
|
|
def test_stalled_ingest_escalated_with_alert(self, pg_conn, caplog):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
sid = "1a000000-0000-0000-0000-0000000000a1"
|
|
_seed_ingest_progress(pg_conn, source_id=sid, embedded=9, total=907)
|
|
before = _stack_logs_count(pg_conn, "reconciler_ingest_stalled")
|
|
|
|
with _route_engine_to(pg_conn), caplog.at_level(
|
|
logging.ERROR, logger="application.api.user.reconciliation",
|
|
):
|
|
r = run_reconciliation()
|
|
|
|
assert r["ingests_stalled"] == 1
|
|
# Escalated to a terminal status so the next tick skips it.
|
|
assert _ingest_status(pg_conn, sid) == "stalled"
|
|
# Structured alert + stack_logs row both surface the failure.
|
|
assert any(
|
|
getattr(rec, "alert", None) == "reconciler_ingest_stalled"
|
|
and rec.levelname == "ERROR"
|
|
for rec in caplog.records
|
|
)
|
|
assert (
|
|
_stack_logs_count(pg_conn, "reconciler_ingest_stalled")
|
|
== before + 1
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_stalled_ingest_alerts_once_not_every_tick(self, pg_conn):
|
|
"""The escalate-to-'stalled' write ends the re-alert loop: a
|
|
second tick neither re-counts nor re-logs the same dead ingest.
|
|
"""
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
sid = "1a000000-0000-0000-0000-0000000000a2"
|
|
_seed_ingest_progress(pg_conn, source_id=sid, embedded=1, total=95)
|
|
before = _stack_logs_count(pg_conn, "reconciler_ingest_stalled")
|
|
|
|
with _route_engine_to(pg_conn):
|
|
r1 = run_reconciliation()
|
|
r2 = run_reconciliation()
|
|
|
|
assert r1["ingests_stalled"] == 1
|
|
assert r2["ingests_stalled"] == 0
|
|
# Only the first tick wrote an alert row.
|
|
assert (
|
|
_stack_logs_count(pg_conn, "reconciler_ingest_stalled")
|
|
== before + 1
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_fresh_ingest_left_alone(self, pg_conn):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
sid = "1a000000-0000-0000-0000-0000000000a3"
|
|
# 2 minutes old — well under the 30-minute staleness threshold.
|
|
_seed_ingest_progress(
|
|
pg_conn, source_id=sid, embedded=3, total=20, age_minutes=2,
|
|
)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
r = run_reconciliation()
|
|
|
|
assert r["ingests_stalled"] == 0
|
|
assert _ingest_status(pg_conn, sid) == "active"
|
|
|
|
@pytest.mark.unit
|
|
def test_completed_ingest_left_alone(self, pg_conn):
|
|
"""A stale checkpoint that finished embedding (embedded == total)
|
|
is not a stall and must not be flagged.
|
|
"""
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
sid = "1a000000-0000-0000-0000-0000000000a4"
|
|
_seed_ingest_progress(pg_conn, source_id=sid, embedded=50, total=50)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
r = run_reconciliation()
|
|
|
|
assert r["ingests_stalled"] == 0
|
|
assert _ingest_status(pg_conn, sid) == "active"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Q5 — stuck idempotency pending rows (lease expired + attempts exhausted)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _seed_stuck_idempotency_row(
|
|
conn,
|
|
*,
|
|
key: str,
|
|
attempt_count: int,
|
|
lease_secs_ago: int,
|
|
) -> None:
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
INSERT INTO task_dedup (
|
|
idempotency_key, task_name, task_id, status,
|
|
attempt_count, lease_owner_id, lease_expires_at,
|
|
created_at
|
|
) VALUES (
|
|
:key, 'ingest', :tid, 'pending', :attempts,
|
|
:owner,
|
|
clock_timestamp() - make_interval(secs => :secs),
|
|
clock_timestamp() - make_interval(mins => 5)
|
|
)
|
|
"""
|
|
),
|
|
{
|
|
"key": key,
|
|
"tid": f"task-{key}",
|
|
"attempts": int(attempt_count),
|
|
"owner": f"owner-{key}",
|
|
"secs": int(lease_secs_ago),
|
|
},
|
|
)
|
|
|
|
|
|
class TestStuckIdempotencyPending:
|
|
@pytest.mark.unit
|
|
def test_promotes_to_failed_with_alert(self, pg_conn, caplog):
|
|
"""A pending row whose lease expired and whose attempt counter
|
|
already hit the poison-loop threshold gets escalated to failed
|
|
so a same-key retry can re-claim instead of waiting 24 h.
|
|
"""
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
_seed_stuck_idempotency_row(
|
|
pg_conn, key="abandoned", attempt_count=5, lease_secs_ago=120,
|
|
)
|
|
before = _stack_logs_count(
|
|
pg_conn, "reconciler_idempotency_pending_failed",
|
|
)
|
|
|
|
with _route_engine_to(pg_conn), caplog.at_level(
|
|
logging.ERROR, logger="application.api.user.reconciliation",
|
|
):
|
|
r = run_reconciliation()
|
|
|
|
assert r["idempotency_pending_failed"] == 1
|
|
row = pg_conn.execute(
|
|
text(
|
|
"SELECT status, result_json FROM task_dedup "
|
|
"WHERE idempotency_key = :k"
|
|
),
|
|
{"k": "abandoned"},
|
|
).fetchone()
|
|
assert row[0] == "failed"
|
|
assert row[1]["reconciled"] is True
|
|
assert "lease expired" in row[1]["error"]
|
|
|
|
assert any(
|
|
getattr(rec, "alert", None) == "reconciler_idempotency_pending_failed"
|
|
for rec in caplog.records
|
|
)
|
|
assert (
|
|
_stack_logs_count(
|
|
pg_conn, "reconciler_idempotency_pending_failed",
|
|
)
|
|
== before + 1
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_skips_under_attempt_threshold(self, pg_conn):
|
|
"""Attempt count below the threshold means the wrapper might
|
|
still re-claim cleanly — leave the row alone.
|
|
"""
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
_seed_stuck_idempotency_row(
|
|
pg_conn, key="recoverable", attempt_count=2, lease_secs_ago=120,
|
|
)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
r = run_reconciliation()
|
|
|
|
assert r["idempotency_pending_failed"] == 0
|
|
row = pg_conn.execute(
|
|
text("SELECT status FROM task_dedup WHERE idempotency_key = :k"),
|
|
{"k": "recoverable"},
|
|
).fetchone()
|
|
assert row[0] == "pending"
|
|
|
|
@pytest.mark.unit
|
|
def test_skips_within_lease_grace(self, pg_conn):
|
|
"""A lease that just expired (10 s ago) might be in the
|
|
heartbeat-tick window; the 60 s grace keeps the sweep quiet.
|
|
"""
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
|
|
_seed_stuck_idempotency_row(
|
|
pg_conn, key="just-expired", attempt_count=5, lease_secs_ago=10,
|
|
)
|
|
|
|
with _route_engine_to(pg_conn):
|
|
r = run_reconciliation()
|
|
|
|
assert r["idempotency_pending_failed"] == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Clearing / terminal user-facing events (revoke stale UI surfaces)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestApprovalClearedEvents:
|
|
@pytest.mark.unit
|
|
def test_message_failed_clears_pending_approval(self, pg_conn):
|
|
"""A reconciled-to-failed message deletes its resumable state and
|
|
publishes ``tool.approval.cleared`` so the approval toast doesn't
|
|
linger after reconnect.
|
|
"""
|
|
from application.api.user import reconciliation as recon
|
|
|
|
msg = _seed_pending_message(pg_conn)
|
|
# Expired PT row: doesn't shield the message (past TTL) but is the
|
|
# resumable state the failure path must delete + revoke.
|
|
_seed_pending_state(
|
|
pg_conn, msg["conversation_id"], msg["user_id"],
|
|
expires_in_minutes=-1,
|
|
)
|
|
|
|
ctx, published = _capture_published(pg_conn)
|
|
with _route_engine_to(pg_conn), ctx:
|
|
recon.run_reconciliation()
|
|
recon.run_reconciliation()
|
|
recon.run_reconciliation()
|
|
|
|
status = pg_conn.execute(
|
|
text(
|
|
"SELECT status FROM conversation_messages "
|
|
"WHERE id = CAST(:id AS uuid)"
|
|
),
|
|
{"id": msg["id"]},
|
|
).scalar()
|
|
assert status == "failed"
|
|
|
|
# Resumable state is gone.
|
|
pt_count = pg_conn.execute(
|
|
text(
|
|
"SELECT count(*) FROM pending_tool_state "
|
|
"WHERE conversation_id = CAST(:c AS uuid)"
|
|
),
|
|
{"c": msg["conversation_id"]},
|
|
).scalar()
|
|
assert pt_count == 0
|
|
|
|
cleared = [p for p in published if p[1] == "tool.approval.cleared"]
|
|
assert len(cleared) == 1
|
|
user_id, _, payload, scope = cleared[0]
|
|
assert user_id == msg["user_id"]
|
|
assert payload["conversation_id"] == msg["conversation_id"]
|
|
assert payload["message_id"] == msg["id"]
|
|
assert payload["reason"] == "failed"
|
|
assert scope == {"kind": "conversation", "id": msg["conversation_id"]}
|
|
|
|
@pytest.mark.unit
|
|
def test_message_failed_without_approval_emits_no_clear(self, pg_conn):
|
|
"""A plain stuck message (no resumable state) must not emit a
|
|
spurious clearing event.
|
|
"""
|
|
from application.api.user import reconciliation as recon
|
|
|
|
_seed_pending_message(pg_conn)
|
|
|
|
ctx, published = _capture_published(pg_conn)
|
|
with _route_engine_to(pg_conn), ctx:
|
|
recon.run_reconciliation()
|
|
recon.run_reconciliation()
|
|
recon.run_reconciliation()
|
|
|
|
assert not any(p[1] == "tool.approval.cleared" for p in published)
|
|
|
|
|
|
class TestStalledIngestEvent:
|
|
@pytest.mark.unit
|
|
def test_stalled_ingest_emits_source_failed_event(self, pg_conn):
|
|
from application.api.user import reconciliation as recon
|
|
|
|
sid = "1a000000-0000-0000-0000-0000000000b1"
|
|
_seed_source(pg_conn, source_id=sid, user_id="u-ingest", name="report.pdf")
|
|
_seed_ingest_progress(pg_conn, source_id=sid, embedded=2, total=50)
|
|
|
|
ctx, published = _capture_published(pg_conn)
|
|
with _route_engine_to(pg_conn), ctx:
|
|
r = recon.run_reconciliation()
|
|
|
|
assert r["ingests_stalled"] == 1
|
|
failed = [p for p in published if p[1] == "source.ingest.failed"]
|
|
assert len(failed) == 1
|
|
user_id, _, payload, scope = failed[0]
|
|
assert user_id == "u-ingest"
|
|
assert payload["source_id"] == sid
|
|
assert payload["filename"] == "report.pdf"
|
|
assert scope == {"kind": "source", "id": sid}
|
|
|
|
@pytest.mark.unit
|
|
def test_orphan_source_stalls_without_event(self, pg_conn):
|
|
"""An ingest row with no matching ``sources`` row (deleted source)
|
|
still escalates to 'stalled' but emits no user event.
|
|
"""
|
|
from application.api.user import reconciliation as recon
|
|
|
|
sid = "1a000000-0000-0000-0000-0000000000b2"
|
|
_seed_ingest_progress(pg_conn, source_id=sid, embedded=1, total=20)
|
|
|
|
ctx, published = _capture_published(pg_conn)
|
|
with _route_engine_to(pg_conn), ctx:
|
|
r = recon.run_reconciliation()
|
|
|
|
assert r["ingests_stalled"] == 1
|
|
assert _ingest_status(pg_conn, sid) == "stalled"
|
|
assert not any(p[1] == "source.ingest.failed" for p in published)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Skip path
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPostgresUriMissing:
|
|
@pytest.mark.unit
|
|
def test_returns_skip_dict(self, monkeypatch):
|
|
from application.api.user.reconciliation import run_reconciliation
|
|
from application.core.settings import settings
|
|
|
|
monkeypatch.setattr(settings, "POSTGRES_URI", None, raising=False)
|
|
|
|
result = run_reconciliation()
|
|
assert result == {
|
|
"messages_failed": 0,
|
|
"tool_calls_failed": 0,
|
|
"skipped": "POSTGRES_URI not set",
|
|
}
|