mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-04 22:13:08 +00:00
Alex
574f96341e
The backend import package is now docsgpt, the name it will carry on PyPI; application was far too generic to install into anyone's site-packages. git mv plus a mechanical rewrite of every import, dotted string and path reference: 734 Python files, the compose files, Dockerfile, workflows, docs, setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage config, .gitignore. Behaviour is unchanged. Kept for one release: - A top-level application package whose meta-path finder resolves application.x.y to the already-imported docsgpt.x.y object, so old imports and entry points (celery -A application.app.celery, uvicorn application.asgi:asgi_app) keep working with a FutureWarning. - Celery registers every application.* task name as an alias of its docsgpt.* task on start-up, so messages queued by the previous release still run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries the previous release wrote are left unread instead of firing twice. The backend image builds from the repository root (docker build -f docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore allow-lists docsgpt/ and application/ and keeps caches, local data, .env files, the sample index files and the Dockerfile out. Compose and the image workflows point at the new context.
103 lines
3.4 KiB
Python
103 lines
3.4 KiB
Python
"""Shared file-extension constants for parsing and ingestion flows."""
|
|
|
|
import os
|
|
|
|
from docsgpt.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
|
|
from docsgpt.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
|
|
|
|
|
|
SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS = (
|
|
".rst",
|
|
".md",
|
|
".pdf",
|
|
".txt",
|
|
".docx",
|
|
".csv",
|
|
".epub",
|
|
".html",
|
|
".mdx",
|
|
".json",
|
|
".xlsx",
|
|
".pptx",
|
|
# Read by the HTML parsers on every engine.
|
|
".xhtml",
|
|
# Read by the anydoc engine (legacy/macro Office, OpenDocument, RTF).
|
|
# Parseable regardless of DOC_PARSER_ENGINE: anydoc is a core dependency,
|
|
# and both parser maps route these suffixes to it.
|
|
*ANYDOC_GAINED_SUFFIXES,
|
|
)
|
|
|
|
SUPPORTED_SOURCE_IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg")
|
|
|
|
SUPPORTED_SOURCE_EXTENSIONS = (
|
|
*SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS,
|
|
*SUPPORTED_SOURCE_IMAGE_EXTENSIONS,
|
|
*SUPPORTED_AUDIO_EXTENSIONS,
|
|
)
|
|
|
|
# Suffixes the attachment path has a dedicated parser for — exactly the keys
|
|
# of ``get_default_file_extractor()``. Kept as a literal (importing ``bulk``
|
|
# here would drag docling into the API process), with
|
|
# ``tests/parser/file/test_constants.py`` asserting the two agree.
|
|
#
|
|
# This is *not* the whole attachment allow-list: a suffix with no parser is
|
|
# read by ``SimpleDirectoryReader``'s plain-text fallthrough, which is right
|
|
# for a .py or a .log and catastrophic for a video — the reader "extracts"
|
|
# megabytes of binary garbage, truncates it, and stores it with
|
|
# ``extraction.status == "ok"``. So unparsed suffixes are admitted on
|
|
# content instead (``upload_limits.enforce_parseable_attachment``): text
|
|
# passes, binary is refused. Zip is deliberately absent — source ingestion
|
|
# extracts archives, the attachment path does not, and a zip fails the
|
|
# content check like any other binary.
|
|
#
|
|
# Mirrored in ``frontend/src/constants/fileUpload.ts``; update both together.
|
|
ATTACHMENT_PARSER_EXTENSIONS = frozenset(
|
|
{
|
|
*SUPPORTED_SOURCE_EXTENSIONS,
|
|
".xhtml",
|
|
".adoc",
|
|
".asciidoc",
|
|
".tiff",
|
|
".tif",
|
|
".bmp",
|
|
".webp",
|
|
".vtt",
|
|
".xml",
|
|
}
|
|
# .txt has no parser of its own — it *is* the plain-text fallthrough. It
|
|
# must be sniffed like any other unparsed suffix, or renaming a video to
|
|
# notes.txt walks straight back into the bug this gate exists for.
|
|
- {".txt"}
|
|
)
|
|
|
|
|
|
def attachment_extension(filename: str | None) -> str:
|
|
"""Return the lower-cased extension of ``filename`` including the dot, or ``""``.
|
|
|
|
Args:
|
|
filename: A bare filename or path; ``None`` and empty strings yield ``""``.
|
|
|
|
Returns:
|
|
The last suffix in lower case (``".pdf"``), or ``""`` when there is none.
|
|
"""
|
|
if not filename:
|
|
return ""
|
|
return os.path.splitext(os.path.basename(str(filename)))[1].lower()
|
|
|
|
|
|
def has_attachment_parser(filename: str | None) -> bool:
|
|
"""Return whether an attachment's suffix has a dedicated parser.
|
|
|
|
A False result does not mean the file is refused: it means nothing but the
|
|
plain-text fallthrough will read it, so it has to earn its place on
|
|
content. The decision is by extension only, never by the mime type a
|
|
browser reports (mobile pickers ignore ``accept`` and lie).
|
|
|
|
Args:
|
|
filename: The upload's filename.
|
|
|
|
Returns:
|
|
True when the suffix is in ``ATTACHMENT_PARSER_EXTENSIONS``.
|
|
"""
|
|
return attachment_extension(filename) in ATTACHMENT_PARSER_EXTENSIONS
|