mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-05 12:13:55 +00:00
Alex
37d93cbd86
Replace the sandbox Docling extractor with read_document, backed by the in-process backend parser (the same one ingestion uses) and offloaded to a dedicated 'parsing' Celery queue so it can run on GPU-capable workers with predictable RAM. The tool resolves the input ref under the run-scoped gate, enqueues the parse, and awaits it with a timeout (degrading to an error rather than hanging); the worker independently re-resolves the artifact through the same gate and never trusts a raw path. Untrusted files get the upload path's safeguards (extension whitelist, size cap, sanitized temp file, cleanup). Options: output (markdown/text/structured/chunks), ocr, pages, engine, max_chars, include_tables, persist, json_schema. The workflow native-file 'extract' fallback now uses the same worker path, so document parsing no longer needs the sandbox and works on every backend. Also fixes the branch's periodic-task test (the sandbox reaper made it 12) and points the dev and e2e Celery workers at the parsing queue.
74 lines
1.9 KiB
YAML
74 lines
1.9 KiB
YAML
services:
|
|
frontend:
|
|
build: ../frontend
|
|
environment:
|
|
- VITE_API_HOST=http://localhost:7091
|
|
- VITE_API_STREAMING=$VITE_API_STREAMING
|
|
ports:
|
|
- "5173:5173"
|
|
depends_on:
|
|
- backend
|
|
|
|
backend:
|
|
build: ../application
|
|
env_file:
|
|
- ../.env
|
|
environment:
|
|
# Override URLs to use docker service names
|
|
- CELERY_BROKER_URL=redis://redis:6379/0
|
|
- CELERY_RESULT_BACKEND=redis://redis:6379/1
|
|
- POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
|
|
ports:
|
|
- "7091:7091"
|
|
volumes:
|
|
- ../application/indexes:/app/application/indexes
|
|
- ../application/inputs:/app/application/inputs
|
|
- ../application/vectors:/app/application/vectors
|
|
depends_on:
|
|
redis:
|
|
condition: service_started
|
|
postgres:
|
|
condition: service_healthy
|
|
|
|
worker:
|
|
build: ../application
|
|
# `parsing` queue carries read_document/parse_document; required for its await to resolve.
|
|
command: celery -A application.app.celery worker -l INFO -Q docsgpt,parsing
|
|
env_file:
|
|
- ../.env
|
|
environment:
|
|
# Override URLs to use docker service names
|
|
- CELERY_BROKER_URL=redis://redis:6379/0
|
|
- CELERY_RESULT_BACKEND=redis://redis:6379/1
|
|
- API_URL=http://backend:7091
|
|
- POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
|
|
depends_on:
|
|
redis:
|
|
condition: service_started
|
|
postgres:
|
|
condition: service_healthy
|
|
|
|
redis:
|
|
image: redis:6-alpine
|
|
ports:
|
|
- 6379:6379
|
|
|
|
postgres:
|
|
image: postgres:16-alpine
|
|
environment:
|
|
- POSTGRES_USER=docsgpt
|
|
- POSTGRES_PASSWORD=docsgpt
|
|
- POSTGRES_DB=docsgpt
|
|
ports:
|
|
- "5432:5432"
|
|
volumes:
|
|
- postgres_data:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U docsgpt -d docsgpt"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 10
|
|
|
|
volumes:
|
|
postgres_data:
|