opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-09-02 22:36:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(knowledge-fs): resolve published document outlines

Browse Source
This commit is contained in:
Jyong 2026-08-12 05:17:00 -04:00
parent ef3c321002
commit 804cbf0fe4
7 changed files with 339 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/knowledge-fs-contract.lock.json
View File

@ -1,6 +1,6 @@
{
"schemaVersion": 5,
"subtreeTree": "8e3ec841b8c1b99f05704738ee7a479dd842bc1c",
"subtreeTree": "8f44ca568f8b0f20459e4436883898b6679e460d",
"openapiSha256": "3d844b2cdf46ddb140803def327533dde4b108f9ac08b0bc7374b58e4ee81ab0",
"capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7",
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",

92
knowledge-fs/packages/api/src/document-read-handlers-coverage.test.ts
View File

@ -1,11 +1,14 @@
import { createNodePlatformAdapter } from "@knowledge/adapters/node";
import { ParseArtifactSchema } from "@knowledge/core";
import { DocumentOutlineSchema, ParseArtifactSchema } from "@knowledge/core";
import type { ParserAdapter } from "@knowledge/parsers";
import { describe, expect, it } from "vitest";
import {
createInMemoryDocumentAssetRepository,
createInMemoryDocumentOutlineRepository,
createInMemoryKnowledgeSpaceRepository,
createInMemoryProjectionSetPublicationMemberRepository,
createInMemoryProjectionSetPublicationRepository,
createKnowledgeGateway,
createStaticAuthVerifier,
} from "./index";
@ -18,6 +21,10 @@ const artifactId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c45";
const bareDocumentId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c99";
const unknownSpaceId = "00000000-0000-4000-8000-00000000dead";
const unknownDocumentId = "00000000-0000-4000-8000-00000000beef";
const publishedOutlineId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2d10";
const publicationGenerationId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2d11";
const publicationId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2d12";
const publicationFingerprint = `projection-set-sha256:${"d".repeat(64)}`;
const maxAssetReadBytes = 25 * 1024 * 1024;
const assetPrefix = `tenant-1/spaces/${spaceId}/documents/${documentId}/assets`;
@ -139,12 +146,22 @@ function createAdapterWithStorageOverrides() {
interface TestHarness {
app: ReturnType<typeof createKnowledgeGateway>;
documentOutlines: ReturnType<typeof createInMemoryDocumentOutlineRepository>;
itemIdByElement: Map<string, string>;
publicationMembers: ReturnType<typeof createInMemoryProjectionSetPublicationMemberRepository>;
publications: ReturnType<typeof createInMemoryProjectionSetPublicationRepository>;
}
async function createHarness(options: { enhance?: boolean } = {}): Promise<TestHarness> {
const { adapter, baseAdapter } = createAdapterWithStorageOverrides();
const documentAssets = createInMemoryDocumentAssetRepository({ maxAssets: 10 });
const documentOutlines = createInMemoryDocumentOutlineRepository({ maxOutlines: 10 });
const publications = createInMemoryProjectionSetPublicationRepository({ maxPublications: 10 });
const publicationMembers = createInMemoryProjectionSetPublicationMemberRepository({
maxListLimit: 20,
maxMembers: 100,
publications,
});
const enhancerCalls: string[] = [];
const app = createKnowledgeGateway({
adapter,
@ -163,12 +180,15 @@ async function createHarness(options: { enhance?: boolean } = {}): Promise<TestH
}
: {}),
generateDocumentAssetId: () => documentId,
documentOutlines,
knowledgeSpaces: createInMemoryKnowledgeSpaceRepository({
generateId: () => spaceId,
maxListLimit: 10,
maxSpaces: 10,
}),
parser: createFixtureParser(),
projectionSetPublicationMembers: publicationMembers,
projectionSetPublications: publications,
});
const createSpace = await app.request("/knowledge-spaces", {
@ -226,7 +246,7 @@ async function createHarness(options: { enhance?: boolean } = {}): Promise<TestH
expect(enhancerCalls).toContain(documentId);
}
return { app, itemIdByElement };
return { app, documentOutlines, itemIdByElement, publicationMembers, publications };
}
function assetUrl(itemId: string, variant?: string) {
@ -289,6 +309,74 @@ describe("document read handlers coverage", () => {
});
});
it("returns the outline owned by the current published generation", async () => {
const { app, documentOutlines, publicationMembers, publications } = await createHarness();
const outline = await documentOutlines.create(
DocumentOutlineSchema.parse({
artifactHash: "d".repeat(64),
createdAt: "2026-08-12T09:00:00.000Z",
documentAssetId: bareDocumentId,
id: publishedOutlineId,
knowledgeSpaceId: spaceId,
metadata: {},
nodes: [
{
id: "published-outline-node",
level: 1,
metadata: {},
title: "Published outline",
tocSource: "native-toc",
},
],
outlineVersion: "v1",
parseArtifactId: artifactId,
publicationGenerationId,
version: 1,
}),
);
await publications.createCandidate({
createdAt: "2026-08-12T09:00:01.000Z",
fingerprint: publicationFingerprint,
id: publicationId,
knowledgeSpaceId: spaceId,
projectionVersion: 1,
tenantId: "tenant-1",
});
await publicationMembers.replaceDocumentComponents({
candidateFingerprint: publicationFingerprint,
components: [
{
componentKey: outline.id,
componentType: "document-outline",
generationId: publicationGenerationId,
},
],
createdAt: "2026-08-12T09:00:02.000Z",
documentAssetId: bareDocumentId,
expectedHeadRevision: 0,
knowledgeSpaceId: spaceId,
tenantId: "tenant-1",
});
await publications.publish({
expectedHeadRevision: 0,
fingerprint: publicationFingerprint,
knowledgeSpaceId: spaceId,
tenantId: "tenant-1",
updatedAt: "2026-08-12T09:00:03.000Z",
});
const response = await app.request(
`/knowledge-spaces/${spaceId}/documents/${bareDocumentId}/outline`,
{ headers: bearer(readToken) },
);
expect(response.status).toBe(200);
await expect(response.json()).resolves.toMatchObject({
id: publishedOutlineId,
nodes: [{ title: "Published outline" }],
});
});
it("guards multimodal asset reads with item, variant, tenant, and size checks", async () => {
const { app, itemIdByElement } = await createHarness();
const itemId = (element: string) => {

71
knowledge-fs/packages/api/src/document-read-handlers.ts
View File

@ -24,8 +24,15 @@ import { DocumentOutlineResponseSchema } from "./document-response-schemas";
import type { KnowledgeGatewayEnv } from "./gateway-openapi-contracts";
import type { KnowledgeSpaceRepository } from "./knowledge-space-repository";
import type { ParseArtifactRepository } from "./parse-artifact-repository";
import type { ProjectionSetPublicationMemberRepository } from "./projection-publication-member-repository";
import type { ProjectionSetPublicationRepository } from "./projection-publication-repository";
import type { DocumentAsset, DocumentMultimodalManifest, PlatformAdapter } from "@knowledge/core";
import type {
DocumentAsset,
DocumentMultimodalManifest,
DocumentOutline,
PlatformAdapter,
} from "@knowledge/core";
export interface RegisterDocumentReadHandlersOptions {
readonly app: OpenAPIHono<KnowledgeGatewayEnv>;
@ -36,6 +43,10 @@ export interface RegisterDocumentReadHandlersOptions {
readonly multimodalManifests: DocumentMultimodalManifestRepository;
readonly objectStorage: PlatformAdapter["objectStorage"];
readonly outlines: DocumentOutlineRepository;
readonly publicationMembers?:
| Pick<ProjectionSetPublicationMemberRepository, "getDocumentComponent">
| undefined;
readonly publications?: Pick<ProjectionSetPublicationRepository, "getPublished"> | undefined;
readonly spaces: KnowledgeSpaceRepository;
/** Max bytes served from the multimodal asset route before returning 413. */
readonly assetMaxReadBytes?: number | undefined;
@ -63,6 +74,8 @@ export function registerDocumentReadHandlers({
multimodalManifests,
objectStorage,
outlines,
publicationMembers,
publications,
spaces,
}: RegisterDocumentReadHandlersOptions): void {
app.openapi(listDocumentAssetsRoute, async (context) => {
@ -224,9 +237,12 @@ export function registerDocumentReadHandlers({
return context.json({ error: "Document outline not found" }, 404);
}
const outline = await outlines.getByDocumentVersion({
documentAssetId: asset.id,
version: asset.version,
const outline = await resolveReadableDocumentOutline({
asset,
outlines,
publicationMembers,
publications,
tenantId: subject.tenantId,
});
if (!outline) {
@ -441,6 +457,53 @@ async function buildReadableDocumentMultimodalManifest({
return multimodalManifests.upsert(deterministicManifest);
}
async function resolveReadableDocumentOutline({
asset,
outlines,
publicationMembers,
publications,
tenantId,
}: {
readonly asset: DocumentAsset;
readonly outlines: DocumentOutlineRepository;
readonly publicationMembers?:
| Pick<ProjectionSetPublicationMemberRepository, "getDocumentComponent">
| undefined;
readonly publications?: Pick<ProjectionSetPublicationRepository, "getPublished"> | undefined;
readonly tenantId: string;
}): Promise<DocumentOutline | null> {
if (publicationMembers && publications) {
const published = await publications.getPublished({
knowledgeSpaceId: asset.knowledgeSpaceId,
tenantId,
});
if (published) {
const member = await publicationMembers.getDocumentComponent({
componentType: "document-outline",
documentAssetId: asset.id,
knowledgeSpaceId: asset.knowledgeSpaceId,
publicationId: published.id,
tenantId,
});
if (member) {
const outline = await outlines.getByDocumentVersion({
documentAssetId: asset.id,
publicationGenerationId: member.generationId,
version: asset.version,
});
return outline?.id === member.componentKey ? outline : null;
}
}
}
// Compatibility for documents compiled before immutable publication generations were added.
return outlines.getByDocumentVersion({
documentAssetId: asset.id,
version: asset.version,
});
}
function isTenantKnowledgeSpaceObjectKey({
knowledgeSpaceId,
objectKey,

5
knowledge-fs/packages/api/src/index.ts
View File

@ -785,6 +785,7 @@ export function createKnowledgeGateway({
parseArtifacts,
parser,
projections,
projectionSetPublicationMembers,
publishedGraph,
runtimeSnapshotResolver,
projectionSetPublications,
@ -1744,6 +1745,10 @@ export function createKnowledgeGateway({
multimodalManifests: multimodalManifestRepository,
objectStorage: adapter.objectStorage,
outlines: outlineRepository,
...(projectionSetPublicationMembers
? { publicationMembers: projectionSetPublicationMembers }
: {}),
...(projectionSetPublications ? { publications: projectionSetPublications } : {}),
spaces,
});

41
knowledge-fs/packages/api/src/projection-publication-member-database-repository.test.ts
View File

@ -74,6 +74,47 @@ describe("database projection publication member repository", () => {
]);
});
it("resolves a document-owned outline with the document membership index", async () => {
const fake = createFakeMemberDatabase({
memberRows: [memberRow({ component_type: "document-outline" })],
});
const repository = createDatabaseProjectionSetPublicationMemberRepository({
database: fake.database,
maxBatchSize: 10,
maxListLimit: 10,
});
await expect(
repository.getDocumentComponent({
componentType: "document-outline",
documentAssetId,
knowledgeSpaceId,
publicationId: candidatePublicationId,
tenantId,
}),
).resolves.toMatchObject({
componentKey: componentA,
componentType: "document-outline",
generationId,
});
expect(fake.calls).toHaveLength(1);
expect(fake.calls[0]?.input).toMatchObject({
maxRows: 2,
operation: "select",
params: [
tenantId,
knowledgeSpaceId,
candidatePublicationId,
documentAssetId,
"document-outline",
],
tableName: "projection_set_publication_members",
});
expect(fake.calls[0]?.input.sql).toContain('"document_asset_id" = $4');
expect(fake.calls[0]?.input.sql).toContain('"component_type" = $5');
expect(fake.calls[0]?.input.sql).toContain("LIMIT 2");
});
it("filters only requested keys in one bounded publication-member query", async () => {
const fake = createFakeMemberDatabase({ memberRows: [memberRow()] });
const repository = createDatabaseProjectionSetPublicationMemberRepository({

48
knowledge-fs/packages/api/src/projection-publication-member-repository.test.ts
View File

@ -39,6 +39,54 @@ const componentE = "018f0d60-7a49-7cc2-9c1b-5b36f18f2f05";
const componentF = "018f0d60-7a49-7cc2-9c1b-5b36f18f2f06";
describe("in-memory projection publication member repository", () => {
it("resolves a document-owned outline without loading the publication snapshot", async () => {
const publications = createInMemoryProjectionSetPublicationRepository({ maxPublications: 10 });
const members = createInMemoryProjectionSetPublicationMemberRepository({
maxListLimit: 20,
maxMembers: 100,
publications,
});
await publications.createCandidate(candidate(fingerprintA, publicationIdA));
await members.replaceDocumentComponents({
...mutation(fingerprintA, 0),
components: [
{
componentKey: componentA,
componentType: "document-outline",
generationId: generationA,
},
{
componentKey: componentB,
componentType: "multimodal-manifest",
generationId: generationA,
},
],
documentAssetId: documentIdA,
});
await expect(
members.getDocumentComponent({
componentType: "document-outline",
documentAssetId: documentIdA,
knowledgeSpaceId,
publicationId: publicationIdA,
tenantId,
}),
).resolves.toMatchObject({
componentKey: componentA,
generationId: generationA,
});
await expect(
members.getDocumentComponent({
componentType: "document-outline",
documentAssetId: documentIdB,
knowledgeSpaceId,
publicationId: publicationIdA,
tenantId,
}),
).resolves.toBeNull();
});
it("checks a bounded set of component keys without loading the publication", async () => {
const publications = createInMemoryProjectionSetPublicationRepository({ maxPublications: 10 });
const members = createInMemoryProjectionSetPublicationMemberRepository({

87
knowledge-fs/packages/api/src/projection-publication-member-repository.ts
View File

@ -104,6 +104,14 @@ export interface FilterProjectionSetPublicationMemberKeysInput {
readonly tenantId: string;
}
export interface ProjectionSetPublicationDocumentComponentLookupInput {
readonly componentType: "document-outline" | "multimodal-manifest";
readonly documentAssetId: string;
readonly knowledgeSpaceId: string;
readonly publicationId: string;
readonly tenantId: string;
}
/**
* Rebuilds an attempt-exclusive candidate from the current published snapshot and one document's
* complete component set. Implementations must treat this as one logical mutation: inherited
@ -130,6 +138,10 @@ export interface ProjectionSetPublicationMemberRepository {
filterComponentKeys(
input: FilterProjectionSetPublicationMemberKeysInput,
): Promise<readonly string[]>;
/** Resolves a singleton document-owned component without loading the full publication snapshot. */
getDocumentComponent(
input: ProjectionSetPublicationDocumentComponentLookupInput,
): Promise<ProjectionSetPublicationMember | null>;
inheritFromPublished(input: InheritProjectionSetPublicationMembersInput): Promise<number>;
listByFingerprint(
input: ProjectionSetPublicationLookupInput,
@ -173,6 +185,14 @@ export class ProjectionSetPublicationMemberBatchSizeExceededError extends Error
}
}
export class ProjectionSetPublicationDocumentComponentConflictError extends Error {
constructor(input: ProjectionSetPublicationDocumentComponentLookupInput) {
super(
`Projection set publication=${input.publicationId} document=${input.documentAssetId} has multiple ${input.componentType} components`,
);
}
}
export class ProjectionSetPublicationMemberIdentityConflictError extends Error {
constructor(componentKey: string, generationId: string) {
super(
@ -361,6 +381,22 @@ export function createInMemoryProjectionSetPublicationMemberRepository({
return normalized.componentKeys.filter((componentKey) => allowed.has(componentKey));
},
getDocumentComponent: async (input) => {
const normalized = normalizeDocumentComponentLookup(input);
const matches = sortedMembers(members.values()).filter(
(member) =>
member.tenantId === normalized.tenantId &&
member.knowledgeSpaceId === normalized.knowledgeSpaceId &&
member.publicationId === normalized.publicationId &&
member.documentAssetId === normalized.documentAssetId &&
member.componentType === normalized.componentType,
);
if (matches.length > 1) {
throw new ProjectionSetPublicationDocumentComponentConflictError(normalized);
}
return matches[0] ? cloneMember(matches[0]) : null;
},
inheritFromPublished: async (input) => {
const normalized = normalizeCandidateMutation(input);
const excludedComponentKeys = new Set(
@ -619,6 +655,45 @@ export function createDatabaseProjectionSetPublicationMemberRepository({
return normalized.componentKeys.filter((componentKey) => allowed.has(componentKey));
},
getDocumentComponent: async (input) => {
const normalized = normalizeDocumentComponentLookup(input);
const result = await database.execute({
maxRows: 2,
operation: "select",
params: [
normalized.tenantId,
normalized.knowledgeSpaceId,
normalized.publicationId,
normalized.documentAssetId,
normalized.componentType,
],
sql: `SELECT * FROM ${quoteDatabaseIdentifier(
database,
memberTableName,
)} WHERE ${quoteDatabaseIdentifier(database, "tenant_id")} = ${databasePlaceholder(
database,
1,
)} AND ${quoteDatabaseIdentifier(
database,
"knowledge_space_id",
)} = ${databasePlaceholder(database, 2)} AND ${quoteDatabaseIdentifier(
database,
"publication_id",
)} = ${databasePlaceholder(database, 3)} AND ${quoteDatabaseIdentifier(
database,
"document_asset_id",
)} = ${databasePlaceholder(database, 4)} AND ${quoteDatabaseIdentifier(
database,
"component_type",
)} = ${databasePlaceholder(database, 5)} LIMIT 2;`,
tableName: memberTableName,
});
if (result.rows.length > 1) {
throw new ProjectionSetPublicationDocumentComponentConflictError(normalized);
}
return result.rows[0] ? mapMemberRow(result.rows[0]) : null;
},
inheritFromPublished: async (input) => {
const normalized = normalizeCandidateMutation(input);
const excludedComponentKeys = normalizeExcludedComponentKeys(
@ -1452,6 +1527,18 @@ function normalizeFilterMemberKeysInput(
};
}
function normalizeDocumentComponentLookup(
input: ProjectionSetPublicationDocumentComponentLookupInput,
): ProjectionSetPublicationDocumentComponentLookupInput {
return {
componentType: input.componentType,
documentAssetId: UuidSchema.parse(input.documentAssetId),
knowledgeSpaceId: UuidSchema.parse(input.knowledgeSpaceId),
publicationId: UuidSchema.parse(input.publicationId),
tenantId: normalizeTenantId(input.tenantId),
};
}
function candidateLookup(input: NormalizedCandidateMutation): ProjectionSetPublicationLookupInput {
return {
fingerprint: input.candidateFingerprint,