mirror of
https://github.com/langgenius/dify.git
synced 2026-09-03 23:47:16 +08:00
fix: truncate document section paths to the KnowledgeFS virtual path limit
This commit is contained in:
parent
cc10ca8198
commit
60bf39efd5
@ -2,6 +2,7 @@ import {
|
||||
DocumentAssetSchema,
|
||||
type DocumentMultimodalManifest,
|
||||
type DocumentOutline,
|
||||
KNOWLEDGE_FS_VIRTUAL_PATH_MAX_LENGTH,
|
||||
} from "@knowledge/core";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
@ -221,6 +222,29 @@ describe("document KnowledgeFS paths", () => {
|
||||
}),
|
||||
]);
|
||||
|
||||
const longSectionPath = buildDocumentSectionKnowledgePaths({
|
||||
asset,
|
||||
generateId: sequenceIds(["018f0d60-7a49-7cc2-9c1b-5b36f18f2c49"]),
|
||||
outline: {
|
||||
...documentOutline(asset.id, asset.knowledgeSpaceId),
|
||||
nodes: [
|
||||
{
|
||||
...documentOutline(asset.id, asset.knowledgeSpaceId).nodes[0]!,
|
||||
id: "outline-long-section",
|
||||
sectionPath: ["部署手册".repeat(100), "索引与检索设置".repeat(100)],
|
||||
title: "索引与检索设置".repeat(100),
|
||||
},
|
||||
],
|
||||
},
|
||||
tenantId: "tenant-dev",
|
||||
})[0]!;
|
||||
|
||||
expect(longSectionPath.virtualPath).toHaveLength(KNOWLEDGE_FS_VIRTUAL_PATH_MAX_LENGTH);
|
||||
expect(longSectionPath.virtualPath).toMatch(/--outlinel\.md$/u);
|
||||
expect(longSectionPath.metadata.filename).toBe(
|
||||
longSectionPath.virtualPath.split("/").at(-1),
|
||||
);
|
||||
|
||||
const publicationGenerationId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c60";
|
||||
const candidatePaths = [
|
||||
buildDocumentKnowledgePath({
|
||||
|
||||
@ -5,6 +5,7 @@ import {
|
||||
type DocumentOutline,
|
||||
type DocumentOutlineNode,
|
||||
type KnowledgePath,
|
||||
KNOWLEDGE_FS_VIRTUAL_PATH_MAX_LENGTH,
|
||||
KnowledgePathSchema,
|
||||
PublicationGenerationIdSchema,
|
||||
} from "@knowledge/core";
|
||||
@ -286,7 +287,13 @@ export function buildDocumentSectionKnowledgePaths({
|
||||
const documentPath = `${KNOWLEDGE_FS_DOCS_ROOT}/${documentFilenamePathSegment(asset.filename, asset.id)}`;
|
||||
|
||||
return flattenOutlineNodes(outline.nodes).map((node) => {
|
||||
const virtualPath = `${documentPath}/sections/${documentSectionFilename(node)}.md`;
|
||||
const sectionPrefix = `${documentPath}/sections/`;
|
||||
const extension = ".md";
|
||||
const sectionFilename = documentSectionFilename(
|
||||
node,
|
||||
KNOWLEDGE_FS_VIRTUAL_PATH_MAX_LENGTH - sectionPrefix.length - extension.length,
|
||||
);
|
||||
const virtualPath = `${sectionPrefix}${sectionFilename}${extension}`;
|
||||
|
||||
return KnowledgePathSchema.parse({
|
||||
id: generationScopedKnowledgePathId({
|
||||
@ -297,7 +304,7 @@ export function buildDocumentSectionKnowledgePaths({
|
||||
knowledgeSpaceId: asset.knowledgeSpaceId,
|
||||
metadata: {
|
||||
contentKind: "document-section",
|
||||
filename: `${documentSectionFilename(node)}.md`,
|
||||
filename: `${sectionFilename}${extension}`,
|
||||
mimeType: "text/markdown",
|
||||
outlineId: outline.id,
|
||||
outlineNodeId: node.id,
|
||||
@ -410,7 +417,7 @@ function flattenOutlineNodes(nodes: readonly DocumentOutlineNode[]): DocumentOut
|
||||
return nodes.flatMap((node) => [node, ...flattenOutlineNodes(node.children)]);
|
||||
}
|
||||
|
||||
function documentSectionFilename(node: DocumentOutlineNode): string {
|
||||
function documentSectionFilename(node: DocumentOutlineNode, maxLength: number): string {
|
||||
const titleSlug =
|
||||
node.sectionPath
|
||||
.map((segment) => segment.trim())
|
||||
@ -421,8 +428,13 @@ function documentSectionFilename(node: DocumentOutlineNode): string {
|
||||
.replaceAll(/-+/gu, "-")
|
||||
.replaceAll(/^-|-$/gu, "") || "section";
|
||||
const shortId = node.id.replaceAll("-", "").slice(0, 8);
|
||||
const suffix = `--${shortId}`;
|
||||
const titleBudget = Math.max(1, maxLength - suffix.length);
|
||||
let boundedTitle = titleSlug.slice(0, titleBudget);
|
||||
if (/[\uD800-\uDBFF]$/u.test(boundedTitle)) boundedTitle = boundedTitle.slice(0, -1);
|
||||
boundedTitle = boundedTitle.replaceAll(/-+$/gu, "") || "section".slice(0, titleBudget);
|
||||
|
||||
return `${titleSlug}--${shortId}`;
|
||||
return `${boundedTitle}${suffix}`;
|
||||
}
|
||||
|
||||
function documentMultimodalAssetFilename(item: DocumentMultimodalItem): string {
|
||||
|
||||
@ -112,9 +112,10 @@ export type KnowledgeFsLeaseTargetType = z.infer<typeof KnowledgeFsLeaseTargetTy
|
||||
export const KnowledgeFsLeaseStatusSchema = z.enum(["active", "released", "expired", "failed"]);
|
||||
export type KnowledgeFsLeaseStatus = z.infer<typeof KnowledgeFsLeaseStatusSchema>;
|
||||
|
||||
export const KNOWLEDGE_FS_VIRTUAL_PATH_MAX_LENGTH = 384;
|
||||
const KnowledgeFsLeaseVirtualPathSchema = z
|
||||
.string()
|
||||
.max(384)
|
||||
.max(KNOWLEDGE_FS_VIRTUAL_PATH_MAX_LENGTH)
|
||||
.regex(new RegExp(`^/(?:${KnowledgeFsNamespaceValues.join("|")})(?:/[^/\\s]+)*$`));
|
||||
|
||||
export const KnowledgeFsLeaseSchema = z.object({
|
||||
@ -1266,7 +1267,7 @@ const KnowledgeFsNamespaceSpecs = {
|
||||
const knowledgeFsNamespacePattern = KnowledgeFsNamespaceSchema.options.join("|");
|
||||
const KnowledgeFsVirtualPathSchema = z
|
||||
.string()
|
||||
.max(384)
|
||||
.max(KNOWLEDGE_FS_VIRTUAL_PATH_MAX_LENGTH)
|
||||
.regex(new RegExp(`^/(?:${knowledgeFsNamespacePattern})(?:/[^/\\s]+)*$`));
|
||||
|
||||
export function getKnowledgeFsNamespaceSpec(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user