mirror of
https://github.com/langgenius/dify.git
synced 2026-09-05 08:48:10 +08:00
feat(knowledge_fs): expand supported document upload formats
This commit is contained in:
parent
46ba3f8b60
commit
d5fd45bb41
@ -55,7 +55,7 @@ class FileService:
|
||||
mimetype: str,
|
||||
user: Account | EndUser,
|
||||
tenant_id: str | None = None,
|
||||
source: Literal["datasets"] | None = None,
|
||||
source: Literal["datasets", "knowledge_fs"] | None = None,
|
||||
source_url: str = "",
|
||||
default_file_size_limit: int | None = None,
|
||||
) -> UploadFile:
|
||||
|
||||
@ -32,6 +32,33 @@ from services.knowledge_fs.product_dto import (
|
||||
)
|
||||
|
||||
STAGED_UPLOAD_TTL = timedelta(hours=24)
|
||||
_KNOWLEDGE_FS_DOCUMENT_MIME_TYPES = {
|
||||
"csv": "text/csv",
|
||||
"doc": "application/msword",
|
||||
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"eml": "message/rfc822",
|
||||
"epub": "application/epub+zip",
|
||||
"htm": "text/html",
|
||||
"html": "text/html",
|
||||
"json": "application/json",
|
||||
"jsonl": "application/x-ndjson",
|
||||
"markdown": "text/markdown",
|
||||
"md": "text/markdown",
|
||||
"mdx": "text/mdx",
|
||||
"msg": "application/vnd.ms-outlook",
|
||||
"odt": "application/vnd.oasis.opendocument.text",
|
||||
"pdf": "application/pdf",
|
||||
"ppt": "application/vnd.ms-powerpoint",
|
||||
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"properties": "text/x-java-properties",
|
||||
"rtf": "application/rtf",
|
||||
"text": "text/plain",
|
||||
"txt": "text/plain",
|
||||
"vtt": "text/vtt",
|
||||
"xls": "application/vnd.ms-excel",
|
||||
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"xml": "application/xml",
|
||||
}
|
||||
|
||||
|
||||
class KnowledgeFSStagedUploadError(ValueError):
|
||||
@ -78,7 +105,12 @@ class KnowledgeFSStagedUploadService:
|
||||
) -> KnowledgeFSStagedUploadResponse:
|
||||
if not body:
|
||||
raise KnowledgeFSStagedUploadInvalidError("KnowledgeFS staged upload is empty")
|
||||
normalized_content_type = content_type.strip() or "application/octet-stream"
|
||||
_, separator, extension = file_name.strip().lower().rpartition(".")
|
||||
if not separator or extension not in _KNOWLEDGE_FS_DOCUMENT_MIME_TYPES:
|
||||
raise KnowledgeFSStagedUploadInvalidError("KnowledgeFS staged upload is invalid")
|
||||
# Browser/OS MIME declarations are inconsistent and can route a binary document through a
|
||||
# text parser. The admitted extension is the product contract, so persist its canonical MIME.
|
||||
normalized_content_type = _KNOWLEDGE_FS_DOCUMENT_MIME_TYPES[extension]
|
||||
checksum = b64encode(sha256(body).digest()).decode()
|
||||
try:
|
||||
upload_file = FileService(self._session_maker).upload_file(
|
||||
@ -87,7 +119,7 @@ class KnowledgeFSStagedUploadService:
|
||||
mimetype=normalized_content_type,
|
||||
user=account,
|
||||
tenant_id=tenant_id,
|
||||
source="datasets",
|
||||
source="knowledge_fs",
|
||||
default_file_size_limit=file_size_limit_mb,
|
||||
)
|
||||
except FileTooLargeError as exc:
|
||||
|
||||
@ -265,10 +265,10 @@ def test_stage_persists_workspace_owned_upload(
|
||||
file_service.upload_file.assert_called_once_with(
|
||||
filename="guide.pdf",
|
||||
content=_BODY,
|
||||
mimetype="application/octet-stream",
|
||||
mimetype="application/pdf",
|
||||
user=account,
|
||||
tenant_id=_TENANT_ID,
|
||||
source="datasets",
|
||||
source="knowledge_fs",
|
||||
default_file_size_limit=15,
|
||||
)
|
||||
with sqlite_session_factory() as session:
|
||||
@ -277,14 +277,73 @@ def test_stage_persists_workspace_owned_upload(
|
||||
assert persisted.checksum_sha256_base64 == b64encode(sha256(_BODY).digest()).decode()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("file_name", "content_type", "expected_content_type"),
|
||||
[
|
||||
("report.pdf", "text/plain", "application/pdf"),
|
||||
("formatted.rtf", "text/rtf", "application/rtf"),
|
||||
("message.msg", "application/x-msg", "application/vnd.ms-outlook"),
|
||||
],
|
||||
)
|
||||
def test_stage_canonicalizes_content_type_from_the_supported_extension(
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
file_name: str,
|
||||
content_type: str,
|
||||
expected_content_type: str,
|
||||
) -> None:
|
||||
upload_file = _upload_file()
|
||||
upload_file.name = file_name
|
||||
upload_file.extension = file_name.rsplit(".", 1)[1]
|
||||
upload_file.mime_type = expected_content_type
|
||||
with sqlite_session_factory.begin() as session:
|
||||
session.add(upload_file)
|
||||
file_service = MagicMock()
|
||||
file_service.upload_file.return_value = upload_file
|
||||
monkeypatch.setattr(staged_upload_module, "FileService", lambda _: file_service)
|
||||
service = KnowledgeFSStagedUploadService(
|
||||
sqlite_session_factory,
|
||||
facade=cast(KnowledgeFSDataFacade, MagicMock()),
|
||||
)
|
||||
account = cast(Account, SimpleNamespace(id=_ACCOUNT_ID))
|
||||
|
||||
response = service.stage(
|
||||
tenant_id=_TENANT_ID,
|
||||
account=account,
|
||||
file_name=file_name,
|
||||
content_type=content_type,
|
||||
body=_BODY,
|
||||
file_size_limit_mb=15,
|
||||
)
|
||||
|
||||
assert response.content_type == expected_content_type
|
||||
file_service.upload_file.assert_called_once_with(
|
||||
filename=file_name,
|
||||
content=_BODY,
|
||||
mimetype=expected_content_type,
|
||||
user=account,
|
||||
tenant_id=_TENANT_ID,
|
||||
source="knowledge_fs",
|
||||
default_file_size_limit=15,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("file_name", "content_type", "body"),
|
||||
[
|
||||
("notes.txt", "text/plain", b"KnowledgeFS notes"),
|
||||
("guide.md", "text/markdown", b"# KnowledgeFS guide"),
|
||||
("README.markdown", "text/markdown", b"# KnowledgeFS guide"),
|
||||
("component.mdx", "text/mdx", b"# KnowledgeFS component"),
|
||||
("captions.vtt", "text/vtt", b"WEBVTT\n\n00:00.000 --> 00:01.000\nKnowledgeFS"),
|
||||
("application.properties", "text/x-java-properties", b"knowledge.fs=enabled"),
|
||||
("feed.xml", "application/xml", b"<knowledge>KnowledgeFS</knowledge>"),
|
||||
("manual.odt", "application/vnd.oasis.opendocument.text", b"odt"),
|
||||
("message.eml", "message/rfc822", b"Subject: KnowledgeFS\n\nBody"),
|
||||
("message.msg", "application/vnd.ms-outlook", b"msg"),
|
||||
],
|
||||
)
|
||||
def test_stage_accepts_supported_text_files_with_the_real_file_service(
|
||||
def test_stage_accepts_knowledge_fs_document_formats_with_the_real_file_service(
|
||||
sqlite_session_factory: sessionmaker[Session],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
file_name: str,
|
||||
@ -322,6 +381,34 @@ def test_stage_accepts_supported_text_files_with_the_real_file_service(
|
||||
assert persisted.upload_file_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize("file_name", ["malware.exe", "md"])
|
||||
def test_stage_rejects_an_unsupported_filename_with_the_real_file_service(
|
||||
sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, file_name: str
|
||||
) -> None:
|
||||
backend = FakeStorage()
|
||||
monkeypatch.setattr(file_service_module, "storage", backend)
|
||||
monkeypatch.setattr(staged_upload_module, "storage", backend)
|
||||
monkeypatch.setattr(file_service_module.file_helpers, "get_signed_file_url", lambda **_: "signed")
|
||||
account = Account(name="KnowledgeFS tester", email="knowledge-fs@example.com")
|
||||
account.id = _ACCOUNT_ID
|
||||
service = KnowledgeFSStagedUploadService(
|
||||
sqlite_session_factory,
|
||||
facade=cast(KnowledgeFSDataFacade, MagicMock()),
|
||||
)
|
||||
|
||||
with pytest.raises(KnowledgeFSStagedUploadInvalidError, match="invalid"):
|
||||
service.stage(
|
||||
tenant_id=_TENANT_ID,
|
||||
account=account,
|
||||
file_name=file_name,
|
||||
content_type="application/octet-stream",
|
||||
body=b"not executable content",
|
||||
file_size_limit_mb=15,
|
||||
)
|
||||
|
||||
assert backend.objects == {}
|
||||
|
||||
|
||||
def test_stage_rejects_empty_and_maps_file_service_errors(
|
||||
sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
|
||||
@ -0,0 +1,73 @@
|
||||
# Expanded document upload formats
|
||||
|
||||
## What changed
|
||||
|
||||
- Added upload admission, MIME validation, and octet-stream inference for the legacy-compatible
|
||||
`.markdown`, `.mdx`, `.vtt`, `.properties`, `.xml`, `.odt`, `.eml`, and `.msg` formats.
|
||||
- Routed VTT and Java properties files through the bounded native text parser. Markdown aliases and
|
||||
XML continue to use the existing native Markdown and structured-data parsers; ODT, EML, and MSG
|
||||
use the existing Unstructured parser boundary.
|
||||
- Kept the Dify New RAG file picker and local upload policy in sync with the KnowledgeFS service
|
||||
allowlist.
|
||||
- Gave the Dify KnowledgeFS staging service the same explicit extension allowlist and a dedicated
|
||||
`knowledge_fs` upload source, so staging no longer inherits the legacy knowledge-base `ETL_TYPE`
|
||||
whitelist. Unsupported extensions are still rejected before storage writes.
|
||||
- Canonicalized staged-upload MIME types from the admitted extension, and changed direct
|
||||
KnowledgeFS admission from two independent allowlists to an extension-to-MIME contract. Common
|
||||
aliases such as `text/rtf` and JSONL declared as `application/json` remain accepted, while
|
||||
unrelated pairs such as PDF plus `text/plain` are rejected.
|
||||
- Prioritized complex binary extensions in parser routing so an inaccurate browser MIME declaration
|
||||
cannot send PDF, Office, EPUB, RTF, ODT, EML, or MSG content through the native text parser.
|
||||
- Preserved visible text inside MDX JSX blocks instead of silently dropping Marked's block HTML
|
||||
tokens. MDX now carries its own `native-mdx@1` parser version so the behavior does not invalidate
|
||||
existing plain-Markdown artifact hashes.
|
||||
- Updated upload guidance in every supported locale to describe the supported format groups and
|
||||
disclose the complex-document parser dependency.
|
||||
- Added behavior tests for declared MIME types, octet-stream inference, native lightweight-text
|
||||
routing, and the browser file-picker contract. The new tests were observed failing before the
|
||||
implementation and passing afterward.
|
||||
|
||||
## Why
|
||||
|
||||
The new knowledge base rejected several formats already accepted by the legacy knowledge base even
|
||||
though its parser stack could process them. Expanding the allowlists and using the lightest existing
|
||||
parser restores compatibility without adding a new parser, storage path, or network dependency.
|
||||
|
||||
## Verification
|
||||
|
||||
- `pnpm --filter @knowledge/api exec vitest run src/document-upload-utils.test.ts` — passed (21 tests).
|
||||
- `pnpm --filter @knowledge/parsers exec vitest run src/parser.test.ts` — passed (55 tests).
|
||||
- `pnpm --filter @knowledge/parsers test:coverage` — passed with 95.69% statements/lines,
|
||||
90.02% branches, and 97.52% functions.
|
||||
- `pnpm --filter @knowledge/api-app exec vitest run src/parser-options.test.ts` — passed (5 tests).
|
||||
- `vp test run --project unit features/new-rag/__tests__/documents-page.spec.tsx` — passed (203 tests).
|
||||
- KnowledgeFS typechecks — passed; the full Turbo test pipeline passed (22 tasks), including the API
|
||||
suite with 4,640 tests passed and 3 skipped.
|
||||
- Targeted KnowledgeFS Biome check for the five changed TypeScript files — passed.
|
||||
- Targeted Dify `vp check` for the two changed Web files — passed.
|
||||
- All 24 localized `dataset.json` files parsed successfully and contain the updated upload-format
|
||||
guidance. The repository-wide dataset i18n alignment check remains blocked by pre-existing
|
||||
missing KnowledgeFS quality-evaluation, task-failure, and related keys outside this change.
|
||||
- Dify KnowledgeFS staged-upload service test — passed (42 tests), including real `FileService`
|
||||
coverage for canonical MIME persistence, expanded formats, and rejection of unsupported or
|
||||
extensionless filenames.
|
||||
- Targeted Ruff format and lint checks for the three changed Python files — passed.
|
||||
- Targeted Pyrefly checks for the changed Python service files — passed.
|
||||
- Targeted Mypy was attempted but the installed Mypy 1.20.2 failed internally while reading its own
|
||||
`typeshed/stdlib/zipimport.pyi`, before reporting project diagnostics.
|
||||
- KnowledgeFS `pnpm build` — passed; the existing Next.js multiple-lockfile and ESLint-plugin warnings
|
||||
remain unchanged.
|
||||
- KnowledgeFS `pnpm lint` — attempted but remains blocked by pre-existing formatting/lint failures in
|
||||
unrelated Admin, test setup, and generated contract files. No unrelated files were modified; the
|
||||
targeted Biome check above covers every KnowledgeFS source and test file changed here.
|
||||
|
||||
## Risks and follow-up
|
||||
|
||||
- ODT, EML, and MSG parsing still requires a configured and capable Unstructured service, matching
|
||||
other complex document types such as DOC and PPT. The upload guidance now calls out this
|
||||
dependency; upload admission remains independent, while downstream parser failures continue to
|
||||
use the existing failed-document lifecycle.
|
||||
- The added allowlist entries are fixed-size `Set` members. Admission remains constant-time and does
|
||||
not change upload byte limits, buffering, database access, or object-storage behavior.
|
||||
- MDX JSX tags and attributes remain syntax rather than searchable text; visible child text is
|
||||
retained, while `script`, `style`, and `noscript` contents remain excluded.
|
||||
@ -67,7 +67,7 @@ describe("createApiDocumentParser", () => {
|
||||
expect(requestedUrl).toBe("https://unstructured.example.test/general/v0/general");
|
||||
expect(artifact).toMatchObject({
|
||||
metadata: {
|
||||
routeReason: "unsupported-file-type",
|
||||
routeReason: "complex-file-type",
|
||||
routedParser: "unstructured",
|
||||
},
|
||||
parser: "unstructured",
|
||||
|
||||
@ -280,6 +280,7 @@ describe("document upload utilities", () => {
|
||||
"application/x-ndjson",
|
||||
"application/jsonl",
|
||||
"application/ndjson",
|
||||
"application/json",
|
||||
"application/octet-stream",
|
||||
]) {
|
||||
const result = await readBulkDocumentUploadWithAdmission(
|
||||
@ -302,6 +303,50 @@ describe("document upload utilities", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
["README.markdown", "text/markdown"],
|
||||
["component.mdx", "text/mdx"],
|
||||
["captions.vtt", "text/vtt"],
|
||||
["application.properties", "text/x-java-properties"],
|
||||
["formatted.rtf", "text/rtf"],
|
||||
["feed.xml", "application/xml"],
|
||||
["manual.odt", "application/vnd.oasis.opendocument.text"],
|
||||
["message.eml", "message/rfc822"],
|
||||
["message.msg", "application/vnd.ms-outlook"],
|
||||
])("accepts legacy-compatible document upload %s", async (filename, declaredMimeType) => {
|
||||
for (const type of [declaredMimeType, "application/octet-stream"]) {
|
||||
const result = await readBulkDocumentUploadWithAdmission(
|
||||
{
|
||||
parseBody: async () => ({
|
||||
files: [new File(["content"], filename, { type })],
|
||||
}),
|
||||
},
|
||||
{
|
||||
maxAcceptedBytesByQuota: null,
|
||||
maxBulkUploadBytes: 100,
|
||||
maxBulkUploadFiles: 20,
|
||||
maxUploadBytes: 100,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.accepted).toHaveLength(1);
|
||||
expect(result.accepted[0]?.filename).toBe(filename);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a supported extension paired with an unrelated MIME type", async () => {
|
||||
await expect(
|
||||
readDocumentUpload(
|
||||
{
|
||||
parseBody: async () => ({
|
||||
file: new File(["%PDF-1.7"], "report.pdf", { type: "text/plain" }),
|
||||
}),
|
||||
},
|
||||
100,
|
||||
),
|
||||
).rejects.toThrow(DocumentUploadValidationError);
|
||||
});
|
||||
|
||||
it("reports quota, aggregate-byte, and count exclusions without discarding earlier files", async () => {
|
||||
const files = [
|
||||
new File(["aa"], "a.txt", { type: "text/plain" }),
|
||||
|
||||
@ -20,25 +20,42 @@ export interface BulkDocumentRevisionTarget {
|
||||
readonly expectedDocumentRowVersion: number;
|
||||
}
|
||||
|
||||
export const SUPPORTED_DOCUMENT_UPLOAD_MIME_TYPES = new Set([
|
||||
"application/jsonl",
|
||||
"application/ndjson",
|
||||
"application/epub+zip",
|
||||
"application/json",
|
||||
"application/msword",
|
||||
"application/pdf",
|
||||
"application/rtf",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/x-ndjson",
|
||||
"text/csv",
|
||||
"text/html",
|
||||
"text/markdown",
|
||||
"text/plain",
|
||||
]);
|
||||
const DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION = {
|
||||
csv: ["text/csv", "application/vnd.ms-excel"],
|
||||
doc: ["application/msword"],
|
||||
docx: ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"],
|
||||
eml: ["message/rfc822"],
|
||||
epub: ["application/epub+zip"],
|
||||
htm: ["text/html"],
|
||||
html: ["text/html"],
|
||||
json: ["application/json"],
|
||||
jsonl: ["application/x-ndjson", "application/jsonl", "application/ndjson", "application/json"],
|
||||
markdown: ["text/markdown", "text/x-markdown", "text/plain"],
|
||||
md: ["text/markdown", "text/x-markdown", "text/plain"],
|
||||
mdx: ["text/mdx", "text/markdown", "text/plain"],
|
||||
msg: ["application/vnd.ms-outlook", "application/x-msg"],
|
||||
odt: ["application/vnd.oasis.opendocument.text"],
|
||||
pdf: ["application/pdf"],
|
||||
ppt: ["application/vnd.ms-powerpoint", "application/mspowerpoint", "application/x-mspowerpoint"],
|
||||
pptx: ["application/vnd.openxmlformats-officedocument.presentationml.presentation"],
|
||||
properties: ["text/x-java-properties", "text/plain"],
|
||||
rtf: ["application/rtf", "text/rtf", "application/x-rtf"],
|
||||
text: ["text/plain"],
|
||||
txt: ["text/plain"],
|
||||
vtt: ["text/vtt", "text/plain"],
|
||||
xls: [
|
||||
"application/vnd.ms-excel",
|
||||
"application/excel",
|
||||
"application/x-excel",
|
||||
"application/x-msexcel",
|
||||
],
|
||||
xlsx: ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"],
|
||||
xml: ["application/xml", "text/xml"],
|
||||
} as const satisfies Readonly<Record<string, readonly string[]>>;
|
||||
|
||||
export const SUPPORTED_DOCUMENT_UPLOAD_MIME_TYPES = new Set(
|
||||
Object.values(DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION).flat(),
|
||||
);
|
||||
|
||||
export const DEFAULT_DOCUMENT_UPLOAD_MAX_BYTES = 15 * 1024 * 1024;
|
||||
export const DEFAULT_BULK_DOCUMENT_UPLOAD_MAX_BYTES = 50 * 1024 * 1024;
|
||||
@ -48,25 +65,9 @@ export const HARD_BULK_DOCUMENT_UPLOAD_MAX_FILES = 25;
|
||||
export const HARD_BULK_DOCUMENT_UPLOAD_MAX_BYTES =
|
||||
HARD_DOCUMENT_UPLOAD_MAX_BYTES * HARD_BULK_DOCUMENT_UPLOAD_MAX_FILES;
|
||||
|
||||
export const SUPPORTED_DOCUMENT_UPLOAD_EXTENSIONS = new Set([
|
||||
"csv",
|
||||
"doc",
|
||||
"docx",
|
||||
"epub",
|
||||
"htm",
|
||||
"html",
|
||||
"json",
|
||||
"jsonl",
|
||||
"md",
|
||||
"pdf",
|
||||
"ppt",
|
||||
"pptx",
|
||||
"rtf",
|
||||
"text",
|
||||
"txt",
|
||||
"xls",
|
||||
"xlsx",
|
||||
]);
|
||||
export const SUPPORTED_DOCUMENT_UPLOAD_EXTENSIONS = new Set(
|
||||
Object.keys(DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION),
|
||||
);
|
||||
|
||||
export type DocumentUploadExclusionReason =
|
||||
| "batch_byte_limit_exceeded"
|
||||
@ -539,27 +540,11 @@ function isJsonObject(value: unknown): value is Record<string, unknown> {
|
||||
export function normalizeDocumentMimeType(file: File): string {
|
||||
const declared = file.type.trim().toLocaleLowerCase();
|
||||
const extension = documentExtension(file.name);
|
||||
const inferred = (
|
||||
{
|
||||
csv: "text/csv",
|
||||
doc: "application/msword",
|
||||
docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
epub: "application/epub+zip",
|
||||
html: "text/html",
|
||||
htm: "text/html",
|
||||
json: "application/json",
|
||||
jsonl: "application/x-ndjson",
|
||||
md: "text/markdown",
|
||||
pdf: "application/pdf",
|
||||
ppt: "application/vnd.ms-powerpoint",
|
||||
pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
rtf: "application/rtf",
|
||||
text: "text/plain",
|
||||
txt: "text/plain",
|
||||
xls: "application/vnd.ms-excel",
|
||||
xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
} as Readonly<Record<string, string>>
|
||||
)[extension ?? ""];
|
||||
const inferred = extension
|
||||
? (DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION as Readonly<Record<string, readonly string[]>>)[
|
||||
extension
|
||||
]?.[0]
|
||||
: undefined;
|
||||
return !declared || declared === "application/octet-stream"
|
||||
? (inferred ?? "application/octet-stream")
|
||||
: declared;
|
||||
@ -567,11 +552,11 @@ export function normalizeDocumentMimeType(file: File): string {
|
||||
|
||||
function isSupportedDocumentUpload(file: File, mimeType: string): boolean {
|
||||
const extension = documentExtension(file.name);
|
||||
return (
|
||||
SUPPORTED_DOCUMENT_UPLOAD_MIME_TYPES.has(mimeType) &&
|
||||
extension !== undefined &&
|
||||
SUPPORTED_DOCUMENT_UPLOAD_EXTENSIONS.has(extension)
|
||||
);
|
||||
if (extension === undefined) return false;
|
||||
const allowedMimeTypes = (
|
||||
DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION as Readonly<Record<string, readonly string[]>>
|
||||
)[extension];
|
||||
return allowedMimeTypes?.includes(mimeType) === true;
|
||||
}
|
||||
|
||||
function documentExtension(filename: string): string | undefined {
|
||||
|
||||
@ -191,6 +191,20 @@ const defaultMaxRows = 20_000;
|
||||
const defaultRetryDelayMs = 100;
|
||||
const defaultNow = () => new Date().toISOString();
|
||||
const defaultGenerateId = () => crypto.randomUUID();
|
||||
const unstructuredDocumentExtensions = new Set([
|
||||
"doc",
|
||||
"docx",
|
||||
"eml",
|
||||
"epub",
|
||||
"msg",
|
||||
"odt",
|
||||
"pdf",
|
||||
"ppt",
|
||||
"pptx",
|
||||
"rtf",
|
||||
"xls",
|
||||
"xlsx",
|
||||
]);
|
||||
|
||||
const UnstructuredElementSchema = z.object({
|
||||
element_id: z.string().min(1).max(512).optional(),
|
||||
@ -209,11 +223,12 @@ export function createNativeMarkdownParser(options: NativeParserOptions = {}): P
|
||||
return {
|
||||
kind: "native-markdown",
|
||||
parse: async (input) => {
|
||||
const parserVersion = options.parserVersion ?? "native-markdown@1";
|
||||
const isMdx = isMdxInput(input);
|
||||
const parserVersion = options.parserVersion ?? (isMdx ? "native-mdx@1" : "native-markdown@1");
|
||||
assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes);
|
||||
const text = decodeUtf8(input.body);
|
||||
const tokens = marked.lexer(text, { gfm: true });
|
||||
const elements = markdownTokensToElements(tokens);
|
||||
const elements = markdownTokensToElements(tokens, { preserveHtmlText: isMdx });
|
||||
|
||||
return createParseArtifact({
|
||||
elements,
|
||||
@ -237,7 +252,7 @@ export function createNativeHtmlParser(options: NativeParserOptions = {}): Parse
|
||||
lowerCaseAttributeNames: true,
|
||||
lowerCaseTags: true,
|
||||
});
|
||||
const nodes = (document.children ?? []) as HtmlNode[];
|
||||
const nodes = document.children as HtmlNode[];
|
||||
const elements = htmlNodesToElements(nodes);
|
||||
const documentTitle = htmlDocumentTitle(nodes);
|
||||
|
||||
@ -742,6 +757,10 @@ function selectParser(
|
||||
return { parser: unstructured, reason: "unsupported-native-language" };
|
||||
}
|
||||
|
||||
if (unstructuredDocumentExtensions.has(filename.split(".").at(-1) ?? "")) {
|
||||
return { parser: unstructured, reason: "complex-file-type" };
|
||||
}
|
||||
|
||||
const structuredFormat = structuredDataFormat(input);
|
||||
|
||||
if (structuredFormat && input.body.byteLength > maxNativeInputBytes) {
|
||||
@ -754,10 +773,15 @@ function selectParser(
|
||||
|
||||
const nativeParser =
|
||||
mimeType === "text/markdown" ||
|
||||
mimeType === "text/mdx" ||
|
||||
mimeType === "text/plain" ||
|
||||
mimeType === "text/vtt" ||
|
||||
mimeType === "text/x-java-properties" ||
|
||||
filename.endsWith(".md") ||
|
||||
filename.endsWith(".markdown") ||
|
||||
filename.endsWith(".mdx")
|
||||
filename.endsWith(".mdx") ||
|
||||
filename.endsWith(".properties") ||
|
||||
filename.endsWith(".vtt")
|
||||
? markdown
|
||||
: mimeType === "text/html" ||
|
||||
mimeType === "application/xhtml+xml" ||
|
||||
@ -841,23 +865,22 @@ function structuredDataFormat({
|
||||
return "csv";
|
||||
}
|
||||
|
||||
if (
|
||||
normalizedMime === "application/json" ||
|
||||
normalizedMime === "text/json" ||
|
||||
normalizedFilename.endsWith(".json")
|
||||
) {
|
||||
if (normalizedFilename.endsWith(".jsonl") || normalizedFilename.endsWith(".ndjson")) {
|
||||
return "jsonl";
|
||||
}
|
||||
|
||||
if (normalizedFilename.endsWith(".json")) {
|
||||
return "json";
|
||||
}
|
||||
|
||||
if (
|
||||
normalizedMime === "application/x-ndjson" ||
|
||||
normalizedMime === "application/jsonl" ||
|
||||
normalizedFilename.endsWith(".jsonl") ||
|
||||
normalizedFilename.endsWith(".ndjson")
|
||||
) {
|
||||
if (normalizedMime === "application/x-ndjson" || normalizedMime === "application/jsonl") {
|
||||
return "jsonl";
|
||||
}
|
||||
|
||||
if (normalizedMime === "application/json" || normalizedMime === "text/json") {
|
||||
return "json";
|
||||
}
|
||||
|
||||
if (
|
||||
normalizedMime === "application/yaml" ||
|
||||
normalizedMime === "text/yaml" ||
|
||||
@ -1024,7 +1047,10 @@ function uniqueStrings(values: readonly string[]): string[] {
|
||||
return [...new Set(values)];
|
||||
}
|
||||
|
||||
function markdownTokensToElements(tokens: readonly Token[]): ParseElementInput[] {
|
||||
function markdownTokensToElements(
|
||||
tokens: readonly Token[],
|
||||
{ preserveHtmlText }: { readonly preserveHtmlText: boolean },
|
||||
): ParseElementInput[] {
|
||||
const elements: ParseElementInput[] = [];
|
||||
const sectionPath: string[] = [];
|
||||
|
||||
@ -1078,6 +1104,12 @@ function markdownTokensToElements(tokens: readonly Token[]): ParseElementInput[]
|
||||
continue;
|
||||
}
|
||||
|
||||
if (token.type === "html" && preserveHtmlText) {
|
||||
const html = token as Tokens.HTML;
|
||||
pushTextElement(elements, "paragraph", markdownHtmlBlockText(html.text), sectionPath);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (token.type === "list") {
|
||||
const list = token as Tokens.List;
|
||||
pushTextElement(
|
||||
@ -1106,6 +1138,38 @@ function markdownTokensToElements(tokens: readonly Token[]): ParseElementInput[]
|
||||
return elements;
|
||||
}
|
||||
|
||||
function isMdxInput({
|
||||
filename,
|
||||
mimeType,
|
||||
}: Pick<ParseDocumentInput, "filename" | "mimeType">): boolean {
|
||||
return (
|
||||
mimeType.trim().toLowerCase() === "text/mdx" || filename.trim().toLowerCase().endsWith(".mdx")
|
||||
);
|
||||
}
|
||||
|
||||
function markdownHtmlBlockText(source: string): string {
|
||||
const document = parseDocument(source, {
|
||||
lowerCaseAttributeNames: true,
|
||||
lowerCaseTags: true,
|
||||
});
|
||||
const nodes = document.children as HtmlNode[];
|
||||
|
||||
return nodes.map(searchableMarkdownHtmlText).join("\n");
|
||||
}
|
||||
|
||||
function searchableMarkdownHtmlText(node: HtmlNode): string {
|
||||
const name = node.name?.toLowerCase();
|
||||
if (name && ["script", "style", "noscript"].includes(name)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (!node.children?.length) {
|
||||
return htmlText(node);
|
||||
}
|
||||
|
||||
return node.children.map(searchableMarkdownHtmlText).join("\n");
|
||||
}
|
||||
|
||||
function htmlNodesToElements(nodes: readonly HtmlNode[]): ParseElementInput[] {
|
||||
const elements: ParseElementInput[] = [];
|
||||
const sectionPath: string[] = [];
|
||||
|
||||
@ -87,6 +87,10 @@ describe("parser adapters", () => {
|
||||
"const answer = 42;",
|
||||
"```",
|
||||
"",
|
||||
"```",
|
||||
"plain code block",
|
||||
"```",
|
||||
"",
|
||||
"| A | B |",
|
||||
"| - | - |",
|
||||
"| 1 | 2 |",
|
||||
@ -143,12 +147,69 @@ describe("parser adapters", () => {
|
||||
id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c45:element-5",
|
||||
metadata: {},
|
||||
sectionPath: ["Overview"],
|
||||
text: "plain code block",
|
||||
type: "code",
|
||||
},
|
||||
{
|
||||
id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c45:element-6",
|
||||
metadata: {},
|
||||
sectionPath: ["Overview"],
|
||||
text: "A | B\n1 | 2",
|
||||
type: "table",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it.each(["text/mdx", "text/plain"])(
|
||||
"preserves searchable text inside MDX JSX blocks declared as %s",
|
||||
async (mimeType) => {
|
||||
const parser = createNativeMarkdownParser({
|
||||
generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c95",
|
||||
now: () => createdAt,
|
||||
});
|
||||
|
||||
const artifact = await parser.parse(
|
||||
createParseInput({
|
||||
body: [
|
||||
"# Overview",
|
||||
"",
|
||||
'<Callout title="Important">',
|
||||
"MDX keeps this searchable.",
|
||||
"<strong>Nested detail</strong>",
|
||||
"<script>ignored()</script>",
|
||||
"</Callout>",
|
||||
].join("\n"),
|
||||
filename: "guide.mdx",
|
||||
mimeType,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(artifact.elements.map((element) => element.text)).toEqual([
|
||||
"Overview",
|
||||
"MDX keeps this searchable.\nNested detail",
|
||||
]);
|
||||
expect(artifact.metadata.parserVersion).toBe("native-mdx@1");
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps plain Markdown raw HTML behavior and parser version unchanged", async () => {
|
||||
const parser = createNativeMarkdownParser({
|
||||
generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c96",
|
||||
now: () => createdAt,
|
||||
});
|
||||
|
||||
const artifact = await parser.parse(
|
||||
createParseInput({
|
||||
body: ["<Callout>", "Plain Markdown keeps its existing behavior.", "</Callout>"].join("\n"),
|
||||
filename: "guide.md",
|
||||
mimeType: "text/markdown",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(artifact.elements).toEqual([]);
|
||||
expect(artifact.metadata.parserVersion).toBe("native-markdown@1");
|
||||
});
|
||||
|
||||
it("normalizes Markdown image references into image parse elements", async () => {
|
||||
const parser = createNativeMarkdownParser({
|
||||
generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2d45",
|
||||
@ -541,8 +602,15 @@ describe("parser adapters", () => {
|
||||
mimeType: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
version: 1,
|
||||
});
|
||||
await router.parse({
|
||||
body: textBytes("%PDF-1.7"),
|
||||
documentAssetId,
|
||||
filename: "report.pdf",
|
||||
mimeType: "text/plain",
|
||||
version: 1,
|
||||
});
|
||||
|
||||
expect(selected).toEqual(["markdown", "html", "unstructured"]);
|
||||
expect(selected).toEqual(["markdown", "html", "unstructured", "unstructured"]);
|
||||
});
|
||||
|
||||
it("routes by file size, OCR need, layout complexity, and language hints", async () => {
|
||||
@ -756,6 +824,31 @@ describe("parser adapters", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the JSONL extension when the declared MIME type is application/json", async () => {
|
||||
const parser = createNativeStructuredDataParser({
|
||||
generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c5d",
|
||||
now: () => createdAt,
|
||||
});
|
||||
|
||||
await expect(
|
||||
parser.parse(
|
||||
createParseInput({
|
||||
body: '{"name":"Ada"}\n{"name":"Lin"}',
|
||||
filename: "records.jsonl",
|
||||
mimeType: "application/json",
|
||||
}),
|
||||
),
|
||||
).resolves.toMatchObject({
|
||||
elements: [
|
||||
{
|
||||
metadata: { columns: ["name"], format: "jsonl", rowCount: 2 },
|
||||
text: "name\nAda\nLin",
|
||||
type: "table",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("routes structured data formats to the native structured parser", async () => {
|
||||
const selected: string[] = [];
|
||||
const structured = createNativeStructuredDataParser({
|
||||
@ -824,6 +917,36 @@ describe("parser adapters", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["captions.vtt", "text/vtt"],
|
||||
["application.properties", "text/x-java-properties"],
|
||||
])("routes lightweight text format %s to the native text parser", async (filename, mimeType) => {
|
||||
const router = createParserRouter({
|
||||
html: createNativeHtmlParser(),
|
||||
markdown: createNativeMarkdownParser(),
|
||||
structured: createNativeStructuredDataParser(),
|
||||
unstructured: {
|
||||
kind: "unstructured",
|
||||
parse: async () => {
|
||||
throw new Error("lightweight text should not require Unstructured");
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
router.parse(
|
||||
createParseInput({
|
||||
body: "first line\nsecond line",
|
||||
filename,
|
||||
mimeType,
|
||||
}),
|
||||
),
|
||||
).resolves.toMatchObject({
|
||||
metadata: { routeReason: "native-file-type", routedParser: "native-markdown" },
|
||||
parser: "native-markdown",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects invalid or unbounded structured data inputs", async () => {
|
||||
await expect(
|
||||
createNativeStructuredDataParser({ maxRows: 1 }).parse(
|
||||
|
||||
@ -2106,7 +2106,7 @@ describe('DocumentsPage', () => {
|
||||
expect(input).toHaveAttribute('tabindex', '-1')
|
||||
expect(input).toHaveAttribute(
|
||||
'accept',
|
||||
'.csv,.doc,.docx,.epub,.htm,.html,.json,.jsonl,.md,.pdf,.ppt,.pptx,.rtf,.text,.txt,.xls,.xlsx',
|
||||
'.csv,.doc,.docx,.eml,.epub,.htm,.html,.json,.jsonl,.markdown,.md,.mdx,.msg,.odt,.pdf,.ppt,.pptx,.properties,.rtf,.text,.txt,.vtt,.xls,.xlsx,.xml',
|
||||
)
|
||||
|
||||
await user.upload(input, new File(['one'], 'one.md', { type: 'text/markdown' }))
|
||||
|
||||
@ -4,20 +4,28 @@ const DOCUMENT_UPLOAD_EXTENSIONS = [
|
||||
'csv',
|
||||
'doc',
|
||||
'docx',
|
||||
'eml',
|
||||
'epub',
|
||||
'htm',
|
||||
'html',
|
||||
'json',
|
||||
'jsonl',
|
||||
'markdown',
|
||||
'md',
|
||||
'mdx',
|
||||
'msg',
|
||||
'odt',
|
||||
'pdf',
|
||||
'ppt',
|
||||
'pptx',
|
||||
'properties',
|
||||
'rtf',
|
||||
'text',
|
||||
'txt',
|
||||
'vtt',
|
||||
'xls',
|
||||
'xlsx',
|
||||
'xml',
|
||||
] as const
|
||||
|
||||
const documentUploadExtensionSet = new Set<string>(DOCUMENT_UPLOAD_EXTENSIONS)
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "تم تجاوز حصة مساحة العمل",
|
||||
"newKnowledge.documentUploadExclusion.target": "لم يعد هدف المستند صالحًا",
|
||||
"newKnowledge.documentUploadFailed": "لم نتمكن من تحميل هذه المستندات. حاول ثانية.",
|
||||
"newKnowledge.documentUploadFormats": "يدعم TXT وMarkdown وPDF وHTML وXLSX وCSV وJSONL · حتى 15 ميغابايت لكل ملف",
|
||||
"newKnowledge.documentUploadFormats": "يدعم النصوص وMarkdown وHTML وPDF وOffice وEPUB والبريد الإلكتروني والبيانات المنظمة (CSV وJSON/JSONL وXML) · تتطلب التنسيقات المعقدة محلل مستندات · حتى 15 ميغابايت لكل ملف",
|
||||
"newKnowledge.documentUploadPartial": "بدأت معالجة {{accepted}} مستندًا؛ تعذرت إضافة {{excluded}}: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "لم يتم قبول أي مستند: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "بدأت معالجة المستندات.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "Arbeitsbereichskontingent überschritten",
|
||||
"newKnowledge.documentUploadExclusion.target": "Dokumentziel ist nicht mehr gültig",
|
||||
"newKnowledge.documentUploadFailed": "Wir konnten diese Dokumente nicht hochladen. Versuchen Sie es erneut.",
|
||||
"newKnowledge.documentUploadFormats": "Unterstützt TXT, Markdown, PDF, HTML, XLSX, CSV und JSONL · jeweils bis zu 15 MB",
|
||||
"newKnowledge.documentUploadFormats": "Unterstützt Text, Markdown, HTML, PDF, Office, EPUB, E-Mail und strukturierte Daten (CSV, JSON/JSONL, XML) · komplexe Formate erfordern einen Dokumentparser · bis zu 15 MB pro Datei",
|
||||
"newKnowledge.documentUploadPartial": "{{accepted}} Dokumente wurden gestartet; {{excluded}} konnten nicht hinzugefügt werden: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Keine Dokumente wurden akzeptiert: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Die Dokumentenverarbeitung wurde gestartet.",
|
||||
|
||||
@ -264,7 +264,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "workspace quota exceeded",
|
||||
"newKnowledge.documentUploadExclusion.target": "document target is no longer valid",
|
||||
"newKnowledge.documentUploadFailed": "We couldn't upload these documents. Try again.",
|
||||
"newKnowledge.documentUploadFormats": "Supports TXT, Markdown, PDF, HTML, XLSX, CSV, and JSONL · up to 15MB each",
|
||||
"newKnowledge.documentUploadFormats": "Supports text, Markdown, HTML, PDF, Office, EPUB, email, and structured data (CSV, JSON/JSONL, XML) · complex formats require a document parser · up to 15MB each",
|
||||
"newKnowledge.documentUploadPartial": "{{accepted}} documents started; {{excluded}} could not be added: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "No documents were accepted: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Document processing started.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "se superó la cuota del espacio de trabajo",
|
||||
"newKnowledge.documentUploadExclusion.target": "el destino del documento ya no es válido",
|
||||
"newKnowledge.documentUploadFailed": "No pudimos cargar estos documentos. Intentar otra vez.",
|
||||
"newKnowledge.documentUploadFormats": "Admite TXT, Markdown, PDF, HTML, XLSX, CSV y JSONL · hasta 15 MB cada uno",
|
||||
"newKnowledge.documentUploadFormats": "Admite texto, Markdown, HTML, PDF, Office, EPUB, correo electrónico y datos estructurados (CSV, JSON/JSONL, XML) · los formatos complejos requieren un analizador de documentos · hasta 15 MB por archivo",
|
||||
"newKnowledge.documentUploadPartial": "Se inició el procesamiento de {{accepted}} documentos; no se pudieron añadir {{excluded}}: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "No se aceptó ningún documento: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Se inició el procesamiento de documentos.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "سهمیه فضای کاری رد شده است",
|
||||
"newKnowledge.documentUploadExclusion.target": "هدف سند دیگر معتبر نیست",
|
||||
"newKnowledge.documentUploadFailed": "ما نتوانستیم این اسناد را آپلود کنیم. دوباره امتحان کنید.",
|
||||
"newKnowledge.documentUploadFormats": "پشتیبانی از TXT، Markdown، PDF، HTML، XLSX، CSV و JSONL · هر فایل تا ۱۵ مگابایت",
|
||||
"newKnowledge.documentUploadFormats": "پشتیبانی از متن، Markdown، HTML، PDF، Office، EPUB، ایمیل و دادههای ساختیافته (CSV، JSON/JSONL، XML) · قالبهای پیچیده به تجزیهگر سند نیاز دارند · هر فایل تا ۱۵ مگابایت",
|
||||
"newKnowledge.documentUploadPartial": "پردازش {{accepted}} سند آغاز شد؛ {{excluded}} سند افزوده نشد: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "هیچ سندی پذیرفته نشد: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "پردازش اسناد آغاز شد.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "quota de l’espace de travail dépassé",
|
||||
"newKnowledge.documentUploadExclusion.target": "la cible du document n’est plus valide",
|
||||
"newKnowledge.documentUploadFailed": "Nous n'avons pas pu télécharger ces documents. Essayer à nouveau.",
|
||||
"newKnowledge.documentUploadFormats": "Formats pris en charge : TXT, Markdown, PDF, HTML, XLSX, CSV et JSONL · 15 Mo max par fichier",
|
||||
"newKnowledge.documentUploadFormats": "Formats pris en charge : texte, Markdown, HTML, PDF, Office, EPUB, e-mail et données structurées (CSV, JSON/JSONL, XML) · les formats complexes nécessitent un analyseur de documents · 15 Mo max par fichier",
|
||||
"newKnowledge.documentUploadPartial": "Le traitement de {{accepted}} documents a démarré ; {{excluded}} n’ont pas pu être ajoutés : {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Aucun document n’a été accepté : {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Le traitement du document a commencé.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "कार्यस्थान कोटा पार हो गया",
|
||||
"newKnowledge.documentUploadExclusion.target": "दस्तावेज़ लक्ष्य अब मान्य नहीं है",
|
||||
"newKnowledge.documentUploadFailed": "हम ये दस्तावेज़ अपलोड नहीं कर सके. पुनः प्रयास करें।",
|
||||
"newKnowledge.documentUploadFormats": "TXT, Markdown, PDF, HTML, XLSX, CSV और JSONL समर्थित · प्रत्येक फ़ाइल अधिकतम 15 MB",
|
||||
"newKnowledge.documentUploadFormats": "टेक्स्ट, Markdown, HTML, PDF, Office, EPUB, ईमेल और संरचित डेटा (CSV, JSON/JSONL, XML) समर्थित · जटिल फ़ॉर्मैट के लिए दस्तावेज़ पार्सर आवश्यक है · प्रत्येक फ़ाइल अधिकतम 15 MB",
|
||||
"newKnowledge.documentUploadPartial": "{{accepted}} दस्तावेज़ों की प्रोसेसिंग शुरू हुई; {{excluded}} जोड़े नहीं जा सके: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "कोई दस्तावेज़ स्वीकार नहीं किया गया: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "दस्तावेज़ प्रसंस्करण शुरू हुआ.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "kuota ruang kerja terlampaui",
|
||||
"newKnowledge.documentUploadExclusion.target": "target dokumen tidak lagi valid",
|
||||
"newKnowledge.documentUploadFailed": "Kami tidak dapat mengunggah dokumen-dokumen ini. Coba lagi.",
|
||||
"newKnowledge.documentUploadFormats": "Mendukung TXT, Markdown, PDF, HTML, XLSX, CSV, dan JSONL · masing-masing hingga 15 MB",
|
||||
"newKnowledge.documentUploadFormats": "Mendukung teks, Markdown, HTML, PDF, Office, EPUB, email, dan data terstruktur (CSV, JSON/JSONL, XML) · format kompleks memerlukan pengurai dokumen · hingga 15 MB per file",
|
||||
"newKnowledge.documentUploadPartial": "Pemrosesan {{accepted}} dokumen dimulai; {{excluded}} tidak dapat ditambahkan: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Tidak ada dokumen yang diterima: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Pemrosesan dokumen dimulai.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "quota dell'area di lavoro superata",
|
||||
"newKnowledge.documentUploadExclusion.target": "la destinazione del documento non è più valida",
|
||||
"newKnowledge.documentUploadFailed": "Non è stato possibile caricare questi documenti. Riprova.",
|
||||
"newKnowledge.documentUploadFormats": "Supporta TXT, Markdown, PDF, HTML, XLSX, CSV e JSONL · fino a 15 MB ciascuno",
|
||||
"newKnowledge.documentUploadFormats": "Supporta testo, Markdown, HTML, PDF, Office, EPUB, e-mail e dati strutturati (CSV, JSON/JSONL, XML) · i formati complessi richiedono un parser di documenti · fino a 15 MB per file",
|
||||
"newKnowledge.documentUploadPartial": "Elaborazione avviata per {{accepted}} documenti; impossibile aggiungerne {{excluded}}: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Nessun documento è stato accettato: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "È iniziata l'elaborazione del documento.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "ワークスペースのクォータを超えています",
|
||||
"newKnowledge.documentUploadExclusion.target": "ドキュメントの対象が無効になっています",
|
||||
"newKnowledge.documentUploadFailed": "これらの書類をアップロードできませんでした。もう一度やり直してください。",
|
||||
"newKnowledge.documentUploadFormats": "TXT、Markdown、PDF、HTML、XLSX、CSV、JSONL に対応 · 1ファイル最大15 MB",
|
||||
"newKnowledge.documentUploadFormats": "テキスト、Markdown、HTML、PDF、Office、EPUB、メール、構造化データ(CSV、JSON/JSONL、XML)に対応 · 複雑な形式にはドキュメントパーサーが必要 · 1ファイル最大15 MB",
|
||||
"newKnowledge.documentUploadPartial": "{{accepted}} 件のドキュメントの処理を開始しました。{{excluded}} 件は追加できませんでした:{{details}}",
|
||||
"newKnowledge.documentUploadRejected": "ドキュメントを受け付けられませんでした:{{details}}",
|
||||
"newKnowledge.documentUploadStarted": "文書処理が開始されました。",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "워크스페이스 할당량을 초과했습니다",
|
||||
"newKnowledge.documentUploadExclusion.target": "문서 대상이 더 이상 유효하지 않습니다",
|
||||
"newKnowledge.documentUploadFailed": "이 문서를 업로드할 수 없습니다. 다시 시도해 보세요.",
|
||||
"newKnowledge.documentUploadFormats": "TXT, Markdown, PDF, HTML, XLSX, CSV, JSONL 지원 · 파일당 최대 15 MB",
|
||||
"newKnowledge.documentUploadFormats": "텍스트, Markdown, HTML, PDF, Office, EPUB, 이메일 및 구조화 데이터(CSV, JSON/JSONL, XML) 지원 · 복잡한 형식에는 문서 파서 필요 · 파일당 최대 15 MB",
|
||||
"newKnowledge.documentUploadPartial": "문서 {{accepted}}개의 처리를 시작했습니다. {{excluded}}개는 추가하지 못했습니다: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "수락된 문서가 없습니다: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "문서 처리가 시작되었습니다.",
|
||||
|
||||
@ -253,7 +253,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "ເກີນໂຄຕ້າພື້ນທີ່ເຮັດວຽກ",
|
||||
"newKnowledge.documentUploadExclusion.target": "ເປົ້າໝາຍເອກະສານບໍ່ຖືກຕ້ອງອີກຕໍ່ໄປ",
|
||||
"newKnowledge.documentUploadFailed": "ພວກເຮົາບໍ່ສາມາດອັບໂຫລດເອກະສານເຫຼົ່ານີ້ໄດ້. ລອງອີກຄັ້ງ.",
|
||||
"newKnowledge.documentUploadFormats": "ຮອງຮັບ TXT, Markdown, PDF, HTML, XLSX, CSV ແລະ JSONL · ສູງສຸດ 15 MB ຕໍ່ໄຟລ໌",
|
||||
"newKnowledge.documentUploadFormats": "ຮອງຮັບຂໍ້ຄວາມ, Markdown, HTML, PDF, Office, EPUB, ອີເມວ ແລະຂໍ້ມູນທີ່ມີໂຄງສ້າງ (CSV, JSON/JSONL, XML) · ຮູບແບບທີ່ຊັບຊ້ອນຕ້ອງໃຊ້ຕົວແຍກວິເຄາະເອກະສານ · ສູງສຸດ 15 MB ຕໍ່ໄຟລ໌",
|
||||
"newKnowledge.documentUploadPartial": "ເອກະສານ {{accepted}} ລາຍການເລີ່ມແລ້ວ; ບໍ່ສາມາດເພີ່ມ {{excluded}} ລາຍການໄດ້: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "ບໍ່ມີເອກະສານໄດ້ຮັບການຍອມຮັບ: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "ການປະມວນຜົນເອກະສານໄດ້ເລີ່ມຕົ້ນ.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "werkruimtequotum overschreden",
|
||||
"newKnowledge.documentUploadExclusion.target": "documentdoel is niet meer geldig",
|
||||
"newKnowledge.documentUploadFailed": "We konden deze documenten niet uploaden. Probeer het opnieuw.",
|
||||
"newKnowledge.documentUploadFormats": "Ondersteunt TXT, Markdown, PDF, HTML, XLSX, CSV en JSONL · maximaal 15 MB per bestand",
|
||||
"newKnowledge.documentUploadFormats": "Ondersteunt tekst, Markdown, HTML, PDF, Office, EPUB, e-mail en gestructureerde gegevens (CSV, JSON/JSONL, XML) · complexe indelingen vereisen een documentparser · maximaal 15 MB per bestand",
|
||||
"newKnowledge.documentUploadPartial": "Verwerking van {{accepted}} documenten gestart; {{excluded}} konden niet worden toegevoegd: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Er zijn geen documenten geaccepteerd: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Documentverwerking gestart.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "przekroczono limit obszaru roboczego",
|
||||
"newKnowledge.documentUploadExclusion.target": "miejsce docelowe dokumentu jest już nieprawidłowe",
|
||||
"newKnowledge.documentUploadFailed": "Nie mogliśmy przesłać tych dokumentów. Spróbuj ponownie.",
|
||||
"newKnowledge.documentUploadFormats": "Obsługuje TXT, Markdown, PDF, HTML, XLSX, CSV i JSONL · do 15 MB na plik",
|
||||
"newKnowledge.documentUploadFormats": "Obsługuje tekst, Markdown, HTML, PDF, Office, EPUB, e-mail i dane strukturalne (CSV, JSON/JSONL, XML) · złożone formaty wymagają parsera dokumentów · do 15 MB na plik",
|
||||
"newKnowledge.documentUploadPartial": "Rozpoczęto przetwarzanie {{accepted}} dokumentów; nie udało się dodać {{excluded}}: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Nie zaakceptowano żadnych dokumentów: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Rozpoczęto przetwarzanie dokumentu.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "cota do espaço de trabalho excedida",
|
||||
"newKnowledge.documentUploadExclusion.target": "o destino do documento não é mais válido",
|
||||
"newKnowledge.documentUploadFailed": "Não foi possível fazer upload desses documentos. Tente novamente.",
|
||||
"newKnowledge.documentUploadFormats": "Compatível com TXT, Markdown, PDF, HTML, XLSX, CSV e JSONL · até 15 MB cada",
|
||||
"newKnowledge.documentUploadFormats": "Compatível com texto, Markdown, HTML, PDF, Office, EPUB, e-mail e dados estruturados (CSV, JSON/JSONL, XML) · formatos complexos exigem um analisador de documentos · até 15 MB por arquivo",
|
||||
"newKnowledge.documentUploadPartial": "O processamento de {{accepted}} documentos foi iniciado; não foi possível adicionar {{excluded}}: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Nenhum documento foi aceito: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "O processamento do documento foi iniciado.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "cota spațiului de lucru a fost depășită",
|
||||
"newKnowledge.documentUploadExclusion.target": "destinația documentului nu mai este validă",
|
||||
"newKnowledge.documentUploadFailed": "Nu am putut încărca aceste documente. Încearcă din nou.",
|
||||
"newKnowledge.documentUploadFormats": "Acceptă TXT, Markdown, PDF, HTML, XLSX, CSV și JSONL · maximum 15 MB fiecare",
|
||||
"newKnowledge.documentUploadFormats": "Acceptă text, Markdown, HTML, PDF, Office, EPUB, e-mail și date structurate (CSV, JSON/JSONL, XML) · formatele complexe necesită un analizor de documente · maximum 15 MB per fișier",
|
||||
"newKnowledge.documentUploadPartial": "Procesarea a {{accepted}} documente a început; {{excluded}} nu au putut fi adăugate: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Nu a fost acceptat niciun document: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Procesarea documentelor a început.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "превышена квота рабочей области",
|
||||
"newKnowledge.documentUploadExclusion.target": "назначение документа больше недействительно",
|
||||
"newKnowledge.documentUploadFailed": "Нам не удалось загрузить эти документы. Попробуйте еще раз.",
|
||||
"newKnowledge.documentUploadFormats": "Поддерживаются TXT, Markdown, PDF, HTML, XLSX, CSV и JSONL · до 15 МБ на файл",
|
||||
"newKnowledge.documentUploadFormats": "Поддерживаются текст, Markdown, HTML, PDF, Office, EPUB, электронная почта и структурированные данные (CSV, JSON/JSONL, XML) · для сложных форматов требуется анализатор документов · до 15 МБ на файл",
|
||||
"newKnowledge.documentUploadPartial": "Начата обработка {{accepted}} документов; не удалось добавить {{excluded}}: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Ни один документ не принят: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Началась обработка документов.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "kvota delovnega prostora je presežena",
|
||||
"newKnowledge.documentUploadExclusion.target": "cilj dokumenta ni več veljaven",
|
||||
"newKnowledge.documentUploadFailed": "Teh dokumentov nismo mogli naložiti. poskusi ponovno",
|
||||
"newKnowledge.documentUploadFormats": "Podpira TXT, Markdown, PDF, HTML, XLSX, CSV in JSONL · do 15 MB na datoteko",
|
||||
"newKnowledge.documentUploadFormats": "Podpira besedilo, Markdown, HTML, PDF, Office, EPUB, e-pošto in strukturirane podatke (CSV, JSON/JSONL, XML) · zapletene oblike zahtevajo razčlenjevalnik dokumentov · do 15 MB na datoteko",
|
||||
"newKnowledge.documentUploadPartial": "Obdelava {{accepted}} dokumentov se je začela; {{excluded}} jih ni bilo mogoče dodati: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Noben dokument ni bil sprejet: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Začela se je obdelava dokumentov.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "เกินโควตาพื้นที่ทำงาน",
|
||||
"newKnowledge.documentUploadExclusion.target": "เป้าหมายเอกสารไม่ถูกต้องอีกต่อไป",
|
||||
"newKnowledge.documentUploadFailed": "เราไม่สามารถอัปโหลดเอกสารเหล่านี้ได้ ลองอีกครั้ง",
|
||||
"newKnowledge.documentUploadFormats": "รองรับ TXT, Markdown, PDF, HTML, XLSX, CSV และ JSONL · สูงสุดไฟล์ละ 15 MB",
|
||||
"newKnowledge.documentUploadFormats": "รองรับข้อความ, Markdown, HTML, PDF, Office, EPUB, อีเมล และข้อมูลที่มีโครงสร้าง (CSV, JSON/JSONL, XML) · รูปแบบที่ซับซ้อนต้องใช้ตัวแยกวิเคราะห์เอกสาร · สูงสุดไฟล์ละ 15 MB",
|
||||
"newKnowledge.documentUploadPartial": "เริ่มประมวลผลเอกสาร {{accepted}} รายการแล้ว; เพิ่มไม่ได้ {{excluded}} รายการ: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "ไม่มีเอกสารที่ได้รับการยอมรับ: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "การประมวลผลเอกสารเริ่มต้นขึ้น",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "çalışma alanı kotası aşıldı",
|
||||
"newKnowledge.documentUploadExclusion.target": "belge hedefi artık geçerli değil",
|
||||
"newKnowledge.documentUploadFailed": "Bu belgeleri yükleyemedik. Tekrar deneyin.",
|
||||
"newKnowledge.documentUploadFormats": "TXT, Markdown, PDF, HTML, XLSX, CSV ve JSONL destekler · dosya başına en fazla 15 MB",
|
||||
"newKnowledge.documentUploadFormats": "Metin, Markdown, HTML, PDF, Office, EPUB, e-posta ve yapılandırılmış verileri (CSV, JSON/JSONL, XML) destekler · karmaşık biçimler belge ayrıştırıcısı gerektirir · dosya başına en fazla 15 MB",
|
||||
"newKnowledge.documentUploadPartial": "{{accepted}} belgenin işlenmesi başladı; {{excluded}} belge eklenemedi: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Hiçbir belge kabul edilmedi: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Evrak işlemleri başlatıldı.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "перевищено квоту робочої області",
|
||||
"newKnowledge.documentUploadExclusion.target": "ціль документа більше недійсна",
|
||||
"newKnowledge.documentUploadFailed": "Нам не вдалося завантажити ці документи. Спробуйте знову.",
|
||||
"newKnowledge.documentUploadFormats": "Підтримуються TXT, Markdown, PDF, HTML, XLSX, CSV і JSONL · до 15 МБ на файл",
|
||||
"newKnowledge.documentUploadFormats": "Підтримуються текст, Markdown, HTML, PDF, Office, EPUB, електронна пошта та структуровані дані (CSV, JSON/JSONL, XML) · для складних форматів потрібен аналізатор документів · до 15 МБ на файл",
|
||||
"newKnowledge.documentUploadPartial": "Розпочато обробку {{accepted}} документів; не вдалося додати {{excluded}}: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Жодного документа не прийнято: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Розпочато обробку документів.",
|
||||
|
||||
@ -260,7 +260,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "đã vượt quá hạn mức không gian làm việc",
|
||||
"newKnowledge.documentUploadExclusion.target": "đích tài liệu không còn hợp lệ",
|
||||
"newKnowledge.documentUploadFailed": "Chúng tôi không thể tải lên những tài liệu này. Hãy thử lại.",
|
||||
"newKnowledge.documentUploadFormats": "Hỗ trợ TXT, Markdown, PDF, HTML, XLSX, CSV và JSONL · tối đa 15 MB mỗi tệp",
|
||||
"newKnowledge.documentUploadFormats": "Hỗ trợ văn bản, Markdown, HTML, PDF, Office, EPUB, email và dữ liệu có cấu trúc (CSV, JSON/JSONL, XML) · định dạng phức tạp cần trình phân tích tài liệu · tối đa 15 MB mỗi tệp",
|
||||
"newKnowledge.documentUploadPartial": "Đã bắt đầu xử lý {{accepted}} tài liệu; không thể thêm {{excluded}} tài liệu: {{details}}",
|
||||
"newKnowledge.documentUploadRejected": "Không có tài liệu nào được chấp nhận: {{details}}",
|
||||
"newKnowledge.documentUploadStarted": "Quá trình xử lý tài liệu bắt đầu.",
|
||||
|
||||
@ -264,7 +264,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "超出工作区配额",
|
||||
"newKnowledge.documentUploadExclusion.target": "文档目标已失效",
|
||||
"newKnowledge.documentUploadFailed": "无法上传这些文档,请重试。",
|
||||
"newKnowledge.documentUploadFormats": "支持 TXT、Markdown、PDF、HTML、XLSX、CSV 和 JSONL · 每个文件不超过 15 MB",
|
||||
"newKnowledge.documentUploadFormats": "支持文本、Markdown、HTML、PDF、Office、EPUB、邮件及结构化数据(CSV、JSON/JSONL、XML)· 复杂格式需要文档解析服务 · 每个文件不超过 15 MB",
|
||||
"newKnowledge.documentUploadPartial": "已开始处理 {{accepted}} 个文档;{{excluded}} 个无法添加:{{details}}",
|
||||
"newKnowledge.documentUploadRejected": "所有文档均未能添加:{{details}}",
|
||||
"newKnowledge.documentUploadStarted": "文档处理已开始。",
|
||||
|
||||
@ -263,7 +263,7 @@
|
||||
"newKnowledge.documentUploadExclusion.quota": "超出工作區配額",
|
||||
"newKnowledge.documentUploadExclusion.target": "文件目標已失效",
|
||||
"newKnowledge.documentUploadFailed": "無法上傳這些文件,請再試一次。",
|
||||
"newKnowledge.documentUploadFormats": "支援 TXT、Markdown、PDF、HTML、XLSX、CSV 和 JSONL · 每個檔案不超過 15 MB",
|
||||
"newKnowledge.documentUploadFormats": "支援文字、Markdown、HTML、PDF、Office、EPUB、郵件及結構化資料(CSV、JSON/JSONL、XML)· 複雜格式需要文件解析服務 · 每個檔案不超過 15 MB",
|
||||
"newKnowledge.documentUploadPartial": "已開始處理 {{accepted}} 個文件;{{excluded}} 個無法新增:{{details}}",
|
||||
"newKnowledge.documentUploadRejected": "未接受任何文件:{{details}}",
|
||||
"newKnowledge.documentUploadStarted": "文件處理已開始。",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user