mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-04 16:13:23 +00:00
Alex
3947c66cda
Conflicts, and how each was taken: - application/core/settings.py — ours. The renamed OCR_ENABLED / OCR_ATTACHMENTS_ENABLED / OCR_MIN_CHARS_PER_PAGE accept main's DOCLING_OCR_* spellings as AliasChoices, so nothing is dropped. - application/Dockerfile — both. Main's install layers plus the INSTALL_DOCLING build arg. - application/parser/file/constants.py — both imports. - deployment/docker-compose.yaml — both. The INSTALL_DOCLING / INSTALL_TESSERACT build args on backend and worker, and main's -Q docsgpt,parsing,embeddings, which query embedding needs. - tests/conftest.py — theirs. Both sides fixed the same pytest-postgresql 9.0.0 autocommit= breakage; main's spelling is the one already on main. - application/requirements.txt — the comments claimed different reasons torch is in core. Main's is the true one now: it removed sentence-transformers, so docling is torch's only remaining consumer. Two things the merge broke without conflicting: - onnxruntime. This branch moved it out of core into the docling extra; main meanwhile made it the runtime local embeddings execute on (fastembed). Git took the deletion, leaving fastembed with no pinned runtime in a repo that pins everything. Restored to core, and no longer pinned twice from the extra. - The frontend copy of ATTACHMENT_PARSER_EXTENSIONS. The backend list is derived and picked up the anydoc suffixes; the hand-kept frontend mirror did not, so the composer would refuse files the API accepts. tests/parser/file/test_constants.py is what caught it. ruff, pytest (9897 passed), frontend build and docs build all pass. The image build is unverified: no Docker daemon on this machine.
103 lines
3.4 KiB
Python
103 lines
3.4 KiB
Python
"""Shared file-extension constants for parsing and ingestion flows."""
|
|
|
|
import os
|
|
|
|
from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
|
|
from application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
|
|
|
|
|
|
SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS = (
|
|
".rst",
|
|
".md",
|
|
".pdf",
|
|
".txt",
|
|
".docx",
|
|
".csv",
|
|
".epub",
|
|
".html",
|
|
".mdx",
|
|
".json",
|
|
".xlsx",
|
|
".pptx",
|
|
# Read by the HTML parsers on every engine.
|
|
".xhtml",
|
|
# Read by the anydoc engine (legacy/macro Office, OpenDocument, RTF).
|
|
# Parseable regardless of DOC_PARSER_ENGINE: anydoc is a core dependency,
|
|
# and both parser maps route these suffixes to it.
|
|
*ANYDOC_GAINED_SUFFIXES,
|
|
)
|
|
|
|
SUPPORTED_SOURCE_IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg")
|
|
|
|
SUPPORTED_SOURCE_EXTENSIONS = (
|
|
*SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS,
|
|
*SUPPORTED_SOURCE_IMAGE_EXTENSIONS,
|
|
*SUPPORTED_AUDIO_EXTENSIONS,
|
|
)
|
|
|
|
# Suffixes the attachment path has a dedicated parser for — exactly the keys
|
|
# of ``get_default_file_extractor()``. Kept as a literal (importing ``bulk``
|
|
# here would drag docling into the API process), with
|
|
# ``tests/parser/file/test_constants.py`` asserting the two agree.
|
|
#
|
|
# This is *not* the whole attachment allow-list: a suffix with no parser is
|
|
# read by ``SimpleDirectoryReader``'s plain-text fallthrough, which is right
|
|
# for a .py or a .log and catastrophic for a video — the reader "extracts"
|
|
# megabytes of binary garbage, truncates it, and stores it with
|
|
# ``extraction.status == "ok"``. So unparsed suffixes are admitted on
|
|
# content instead (``upload_limits.enforce_parseable_attachment``): text
|
|
# passes, binary is refused. Zip is deliberately absent — source ingestion
|
|
# extracts archives, the attachment path does not, and a zip fails the
|
|
# content check like any other binary.
|
|
#
|
|
# Mirrored in ``frontend/src/constants/fileUpload.ts``; update both together.
|
|
ATTACHMENT_PARSER_EXTENSIONS = frozenset(
|
|
{
|
|
*SUPPORTED_SOURCE_EXTENSIONS,
|
|
".xhtml",
|
|
".adoc",
|
|
".asciidoc",
|
|
".tiff",
|
|
".tif",
|
|
".bmp",
|
|
".webp",
|
|
".vtt",
|
|
".xml",
|
|
}
|
|
# .txt has no parser of its own — it *is* the plain-text fallthrough. It
|
|
# must be sniffed like any other unparsed suffix, or renaming a video to
|
|
# notes.txt walks straight back into the bug this gate exists for.
|
|
- {".txt"}
|
|
)
|
|
|
|
|
|
def attachment_extension(filename: str | None) -> str:
|
|
"""Return the lower-cased extension of ``filename`` including the dot, or ``""``.
|
|
|
|
Args:
|
|
filename: A bare filename or path; ``None`` and empty strings yield ``""``.
|
|
|
|
Returns:
|
|
The last suffix in lower case (``".pdf"``), or ``""`` when there is none.
|
|
"""
|
|
if not filename:
|
|
return ""
|
|
return os.path.splitext(os.path.basename(str(filename)))[1].lower()
|
|
|
|
|
|
def has_attachment_parser(filename: str | None) -> bool:
|
|
"""Return whether an attachment's suffix has a dedicated parser.
|
|
|
|
A False result does not mean the file is refused: it means nothing but the
|
|
plain-text fallthrough will read it, so it has to earn its place on
|
|
content. The decision is by extension only, never by the mime type a
|
|
browser reports (mobile pickers ignore ``accept`` and lie).
|
|
|
|
Args:
|
|
filename: The upload's filename.
|
|
|
|
Returns:
|
|
True when the suffix is in ``ATTACHMENT_PARSER_EXTENSIONS``.
|
|
"""
|
|
return attachment_extension(filename) in ATTACHMENT_PARSER_EXTENSIONS
|