mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-05 18:14:21 +00:00
Alex
04d358ab69
In store mode every user turn chained onto the previous response, so the provider's stored transcript grew without bound (measured: 889k prompt tokens for a 37k-token saved history) while every local guard, the compression pipeline included, measured the saved history. Each chained tool round also re-sent the system message, which the server appends rather than dedupes, and a saved compression point was applied exactly once, in the turn that made it. Chaining is now bounded. A turn starts from the saved history when the previous turn's reported prompt reached the chain budget (default: the model's context window), when the conversation was compressed after that turn was produced, or when OPENAI_RESPONSES_CHAIN_ACROSS_TURNS is off. Chained rounds omit an unchanged system head (hash carried in the persisted Responses state). truncation="auto" is available behind a setting as a backstop against a chain that outgrows the model's window. Compression: a saved point is applied at every turn start; the threshold counts the summary plus the queries after the point instead of the raw history; re-compression summarises only the tail on top of the last point; the mid-execution path marks itself persisted and resets the provider chain so the rebuilt messages are the context; an empty summary is rejected; the visible "[Context Compression Summary]" rows are no longer replayed as history; appending the same point twice is a no-op. Cache hints: a per-user prompt_cache_key and an optional prompt_cache_retention on Responses API calls. Measured on Azure with the same client shape as production (stateless OpenAI client, server-side tools, PDF part): tokens billed on the sixth turn fell from 58k to 35k, tool rounds add tens of tokens instead of ~2.8k, the turn after a compression reused the saved summary in under two seconds instead of re-summarising, and the round after a mid-execution compression started from the compressed context instead of the full stored transcript.
112 lines
4.0 KiB
Python
112 lines
4.0 KiB
Python
"""Compression threshold checking logic."""
|
|
|
|
import logging
|
|
from typing import Any, Dict
|
|
|
|
from application.core.model_utils import get_token_limit
|
|
from application.core.settings import settings
|
|
from application.api.answer.services.compression.token_counter import TokenCounter
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class CompressionThresholdChecker:
|
|
"""Determines if compression is needed based on token thresholds."""
|
|
|
|
def __init__(self, threshold_percentage: float = None):
|
|
"""
|
|
Initialize threshold checker.
|
|
|
|
Args:
|
|
threshold_percentage: Percentage of context to use as threshold
|
|
(defaults to settings.COMPRESSION_THRESHOLD_PERCENTAGE)
|
|
"""
|
|
self.threshold_percentage = (
|
|
threshold_percentage or settings.COMPRESSION_THRESHOLD_PERCENTAGE
|
|
)
|
|
|
|
def should_compress(
|
|
self,
|
|
conversation: Dict[str, Any],
|
|
model_id: str,
|
|
current_query_tokens: int = 500,
|
|
user_id: str | None = None,
|
|
) -> bool:
|
|
"""
|
|
Determine if compression is needed.
|
|
|
|
Args:
|
|
conversation: Full conversation document
|
|
model_id: Target model for this request
|
|
current_query_tokens: Estimated tokens for current query
|
|
user_id: Owner — needed so per-user BYOM custom-model UUIDs
|
|
resolve when looking up the context window.
|
|
|
|
Returns:
|
|
True if tokens >= threshold% of context window
|
|
"""
|
|
try:
|
|
# What the next turn will replay: summary + queries after the
|
|
# last compression point, or the raw history when never compressed.
|
|
total_tokens = TokenCounter.count_effective_conversation_tokens(conversation)
|
|
total_tokens += current_query_tokens
|
|
|
|
# Get context window limit for model
|
|
context_limit = get_token_limit(model_id, user_id=user_id)
|
|
|
|
# Calculate threshold
|
|
threshold = int(context_limit * self.threshold_percentage)
|
|
|
|
compression_needed = total_tokens >= threshold
|
|
percentage_used = (total_tokens / context_limit) * 100
|
|
|
|
if compression_needed:
|
|
logger.warning(
|
|
f"COMPRESSION TRIGGERED: {total_tokens} tokens / {context_limit} limit "
|
|
f"({percentage_used:.1f}% used, threshold: {self.threshold_percentage * 100:.0f}%)"
|
|
)
|
|
else:
|
|
logger.info(
|
|
f"Compression check: {total_tokens}/{context_limit} tokens "
|
|
f"({percentage_used:.1f}% used, threshold: {self.threshold_percentage * 100:.0f}%) - No compression needed"
|
|
)
|
|
|
|
return compression_needed
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error checking compression need: {str(e)}", exc_info=True)
|
|
return False
|
|
|
|
def check_message_tokens(
|
|
self, messages: list, model_id: str, user_id: str | None = None
|
|
) -> bool:
|
|
"""
|
|
Check if message list exceeds threshold.
|
|
|
|
Args:
|
|
messages: List of message dicts
|
|
model_id: Target model
|
|
user_id: Owner — needed so per-user BYOM custom-model UUIDs
|
|
resolve when looking up the context window.
|
|
|
|
Returns:
|
|
True if at or above threshold
|
|
"""
|
|
try:
|
|
current_tokens = TokenCounter.count_message_tokens(messages)
|
|
context_limit = get_token_limit(model_id, user_id=user_id)
|
|
threshold = int(context_limit * self.threshold_percentage)
|
|
|
|
if current_tokens >= threshold:
|
|
logger.warning(
|
|
f"Message context limit approaching: {current_tokens}/{context_limit} tokens "
|
|
f"({(current_tokens/context_limit)*100:.1f}%)"
|
|
)
|
|
return True
|
|
|
|
return False
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error checking message tokens: {str(e)}", exc_info=True)
|
|
return False
|