opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-09-05 08:48:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/deploy/konwledge' into deploy/konwledge

Browse Source
This commit is contained in:
Stephen Zhou 2026-09-03 17:01:02 +08:00
commit 45d89701c3
No known key found for this signature in database
14 changed files with 103 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/knowledge-fs-contract.lock.json
View File

@ -1,6 +1,6 @@
{
"schemaVersion": 5,
"subtreeTree": "3f7d65b2322dfb13528f9c8a9b3a68e0b6a9b80a",
"subtreeTree": "958ff662b58baf8efd4a0e5a2112156f703596e1",
"openapiSha256": "bb5c1f6529ee216059b4aa1e01d8079a0bb175bb6adba7a0821fec8b6fa707f1",
"capabilityV2AuthManifestSha256": "e322a2fa779d1f40b95c54c1021cffecaec77abbd7b34899573dcdf4ff353109",
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",

2
knowledge-fs/apps/api/src/index.ts
View File

@ -472,7 +472,7 @@ const documentSemanticChunker = createLlmSemanticChunker({
: {}),
maxConcurrentWindows: ingestionModelRuntimeOptions.semanticExtractionMaxConcurrency,
maxNodes: 20_000,
maxProviderOutputRetries: 1,
maxProviderOutputRetries: 3,
maxWindowChars: ingestionModelRuntimeOptions.semanticChunkingMaxWindowChars,
metrics: operationalMetrics.ingestionModelCalls,
modelRequestGate: ingestionModelRuntimeOptions.modelRequestGate,

6
knowledge-fs/apps/api/src/migrate.test.ts
View File

@ -147,6 +147,8 @@ describe("runApiDatabaseMigrations", () => {
"insert",
"schema",
"insert",
"schema",
"insert",
]);
expect(migrationSql).toHaveLength(expectedPostgresMigrationIds.length);
expect(migrationSql[2]).toContain("-- Migration id: 0003_projection_set_publications\n");
@ -224,6 +226,10 @@ describe("runApiDatabaseMigrations", () => {
expect(migrationSql[47]).toContain(
'CREATE INDEX IF NOT EXISTS "deletion_tombstones_active_scope_idx"',
);
expect(migrationSql[48]).toContain("-- Migration id: 0049_answer_trace_query_images\n");
expect(migrationSql[49]).toContain(
"-- Migration id: 0050_deletion_job_capability_provenance\n",
);
expect(closed).toBe(true);
});

2
knowledge-fs/packages/adapters/src/migration-runner.test.ts
View File

@ -57,6 +57,7 @@ const removeProviderSyncPolicyMigrationId = "0046_remove_provider_sync_policy";
const parseArtifactCheckpointsMigrationId = "0047_parse_artifact_checkpoints";
const deletionActiveScopeIndexesMigrationId = "0048_deletion_active_scope_indexes";
const answerTraceQueryImagesMigrationId = "0049_answer_trace_query_images";
const deletionJobCapabilityProvenanceMigrationId = "0050_deletion_job_capability_provenance";
const migrationsAfterDurableDeletion = [
versionedSpaceProfilesMigrationId,
profilePublicationBindingsMigrationId,
@ -90,6 +91,7 @@ const migrationsAfterDurableDeletion = [
parseArtifactCheckpointsMigrationId,
deletionActiveScopeIndexesMigrationId,
answerTraceQueryImagesMigrationId,
deletionJobCapabilityProvenanceMigrationId,
] as const;
const migrationsAfterTidbBaselineRepair = [
spaceAccessControlMigrationId,

6
knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts
View File

@ -2920,6 +2920,12 @@ describe("LLM semantic chunker", () => {
expect(nodes).toHaveLength(1);
expect(provider.calls).toHaveLength(2);
expect(provider.calls[0]?.messages[0]?.content).not.toContain(
"previous response failed JSON schema validation",
);
expect(provider.calls[1]?.messages[0]?.content).toContain(
"previous response failed JSON schema validation",
);
});
it("accepts JSON wrapped in provider prose but strictly caps joint extraction arrays", async () => {

8
knowledge-fs/packages/api/src/llm-semantic-chunker.ts
View File

@ -497,6 +497,7 @@ export function createLlmSemanticChunker({
maxChunkChars: effectiveConfig.maxChunkChars,
maxEntitiesPerChunk,
maxRelationsPerChunk,
retryCount,
window,
});
const callStartedAt = Date.now();
@ -2368,6 +2369,7 @@ function semanticChunkingMessages({
maxChunkChars,
maxEntitiesPerChunk,
maxRelationsPerChunk,
retryCount,
window,
}: {
readonly enableGraph: boolean;
@ -2375,6 +2377,7 @@ function semanticChunkingMessages({
readonly maxChunkChars: number;
readonly maxEntitiesPerChunk: number;
readonly maxRelationsPerChunk: number;
readonly retryCount: number;
readonly window: SemanticWindow;
}): readonly SemanticChunkingLlmMessage[] {
const carriesParserProvenance = window.planningVersion !== "v1";
@ -2389,6 +2392,11 @@ function semanticChunkingMessages({
? "PageIndex is enabled: assign a concise semantic sectionPath and sectionSummary to every chunk."
: "PageIndex is disabled: preserve only the supplied sectionPath, omit sectionSummary, and do not invent child section levels.",
"Return strict JSON only. Never return, rewrite, summarize, correct, or duplicate source text.",
...(retryCount > 0
? [
"Your previous response failed JSON schema validation. Correct the structure and return exactly one JSON object matching Output shape; do not include prose or Markdown fences.",
]
: []),
"The units field is the core: cover every core unit exactly once, in order, by contiguous inclusive ranges.",
...(usesFixedCoreBoundary(window.planningVersion)
? [

11
knowledge-fs/packages/database/migrations/0050_deletion_job_capability_provenance.postgres.sql Normal file
View File

@ -0,0 +1,11 @@
-- Knowledge Platform schema migration
-- Migration id: 0050_deletion_job_capability_provenance
-- Dialect: postgres
-- A space deletion job outlives the space-owned grant that authorized it. Preserve the grant id
-- as immutable audit provenance without a live FK that blocks the terminal space delete.
ALTER TABLE "deletion_jobs"
DROP CONSTRAINT IF EXISTS "deletion_jobs_capability_grant_fk";
ALTER TABLE "deletion_retry_audits"
DROP CONSTRAINT IF EXISTS "deletion_retry_audits_capability_grant_fk";

11
knowledge-fs/packages/database/migrations/0050_deletion_job_capability_provenance.tidb.sql Normal file
View File

@ -0,0 +1,11 @@
-- Knowledge Platform schema migration
-- Migration id: 0050_deletion_job_capability_provenance
-- Dialect: tidb
-- A space deletion job outlives the space-owned grant that authorized it. Preserve the grant id
-- as immutable audit provenance without a live FK that blocks the terminal space delete.
ALTER TABLE `deletion_jobs`
DROP FOREIGN KEY `deletion_jobs_capability_grant_fk`;
ALTER TABLE `deletion_retry_audits`
DROP FOREIGN KEY `deletion_retry_audits_capability_grant_fk`;

4
knowledge-fs/packages/database/src/capability-job-provenance-migration.test.ts
View File

@ -59,7 +59,7 @@ describe("capability job provenance migration", () => {
foreignKey.referencedTable === "capability_grants" &&
foreignKey.columns.includes("capability_grant_id"),
),
).toBe(true);
).toBe(tableName !== "deletion_jobs");
expect(
schema.indexes.some(
(index) => index.tableName === tableName && index.columns.includes("capability_grant_id"),
@ -81,7 +81,7 @@ describe("capability job provenance migration", () => {
deletionRetryAudits?.foreignKeys?.some(
(foreignKey) => foreignKey.referencedTable === "capability_grants",
),
).toBe(true);
).toBe(false);
expect(
schema.indexes.some(
(index) =>

43
knowledge-fs/packages/database/src/deletion-job-capability-provenance-migration.test.ts Normal file
View File

@ -0,0 +1,43 @@
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { describe, expect, it } from "vitest";
import { getDatabaseSchema } from "./schema";
const root = resolve(import.meta.dirname, "../../..");
describe("deletion job capability provenance migration", () => {
it.each([
["postgres", "DROP CONSTRAINT IF EXISTS", '"'],
["tidb", "DROP FOREIGN KEY", "`"],
] as const)("drops the live grant foreign keys on %s", (dialect, statement, quote) => {
const sql = readFileSync(
resolve(
root,
`packages/database/migrations/0050_deletion_job_capability_provenance.${dialect}.sql`,
),
"utf8",
);
expect(sql).toContain(`${statement} ${quote}deletion_jobs_capability_grant_fk${quote}`);
expect(sql).toContain(`${statement} ${quote}deletion_retry_audits_capability_grant_fk${quote}`);
expect(sql).not.toContain("DROP COLUMN");
});
it("keeps the grant id as provenance without a live schema foreign key", () => {
const table = getDatabaseSchema().tables.find(
(candidate) => candidate.name === "deletion_jobs",
);
expect(table?.columns.some((column) => column.name === "capability_grant_id")).toBe(true);
expect(
table?.foreignKeys?.some((foreignKey) => foreignKey.columns.includes("capability_grant_id")),
).toBe(false);
const audits = getDatabaseSchema().tables.find(
(candidate) => candidate.name === "deletion_retry_audits",
);
expect(audits?.columns.some((column) => column.name === "capability_grant_id")).toBe(true);
expect(
audits?.foreignKeys?.some((foreignKey) => foreignKey.columns.includes("capability_grant_id")),
).toBe(false);
});
});

2
knowledge-fs/packages/database/src/migration-artifacts.generated.ts
View File

@ -101,4 +101,6 @@ export const migrationArtifacts = [
{ content: "-- Knowledge Platform schema migration\n-- Migration id: 0048_deletion_active_scope_indexes\n-- Dialect: tidb\n-- Keeps active deletion admission proportional to live fences instead of permanent audit history.\n\nCREATE INDEX IF NOT EXISTS `deletion_jobs_active_scope_idx`\n ON `deletion_jobs` (`tenant_id`, `knowledge_space_id`, `active_slot`, `target_type`, `target_id`);\n\nCREATE INDEX IF NOT EXISTS `deletion_tombstones_active_scope_idx`\n ON `deletion_tombstones` (`tenant_id`, `knowledge_space_id`, `state`);\n", path: "packages/database/migrations/0048_deletion_active_scope_indexes.tidb.sql" },
{ content: "-- Knowledge Platform schema migration\n-- Migration id: 0049_answer_trace_query_images\n-- Dialect: postgres\n-- Answer traces keep the query-image references they were run with so history can show them.\n\nALTER TABLE \"answer_traces\"\n ADD COLUMN IF NOT EXISTS \"query_images\" JSONB;\n", path: "packages/database/migrations/0049_answer_trace_query_images.postgres.sql" },
{ content: "-- Knowledge Platform schema migration\n-- Migration id: 0049_answer_trace_query_images\n-- Dialect: tidb\n-- Answer traces keep the query-image references they were run with so history can show them.\n\nALTER TABLE `answer_traces`\n ADD COLUMN IF NOT EXISTS `query_images` JSON NULL;\n", path: "packages/database/migrations/0049_answer_trace_query_images.tidb.sql" },
{ content: "-- Knowledge Platform schema migration\n-- Migration id: 0050_deletion_job_capability_provenance\n-- Dialect: postgres\n-- A space deletion job outlives the space-owned grant that authorized it. Preserve the grant id\n-- as immutable audit provenance without a live FK that blocks the terminal space delete.\n\nALTER TABLE \"deletion_jobs\"\n DROP CONSTRAINT IF EXISTS \"deletion_jobs_capability_grant_fk\";\n\nALTER TABLE \"deletion_retry_audits\"\n DROP CONSTRAINT IF EXISTS \"deletion_retry_audits_capability_grant_fk\";\n", path: "packages/database/migrations/0050_deletion_job_capability_provenance.postgres.sql" },
{ content: "-- Knowledge Platform schema migration\n-- Migration id: 0050_deletion_job_capability_provenance\n-- Dialect: tidb\n-- A space deletion job outlives the space-owned grant that authorized it. Preserve the grant id\n-- as immutable audit provenance without a live FK that blocks the terminal space delete.\n\nALTER TABLE `deletion_jobs`\n DROP FOREIGN KEY `deletion_jobs_capability_grant_fk`;\n\nALTER TABLE `deletion_retry_audits`\n DROP FOREIGN KEY `deletion_retry_audits_capability_grant_fk`;\n", path: "packages/database/migrations/0050_deletion_job_capability_provenance.tidb.sql" },
] as const satisfies readonly MigrationArtifact[];

4
knowledge-fs/packages/database/src/migration-file.test.ts
View File

@ -158,6 +158,8 @@ describe("migration file rendering", () => {
"packages/database/migrations/0048_deletion_active_scope_indexes.tidb.sql",
"packages/database/migrations/0049_answer_trace_query_images.postgres.sql",
"packages/database/migrations/0049_answer_trace_query_images.tidb.sql",
"packages/database/migrations/0050_deletion_job_capability_provenance.postgres.sql",
"packages/database/migrations/0050_deletion_job_capability_provenance.tidb.sql",
]);
const workflowCapturePostgres = artifacts.find(
(artifact) =>
@ -937,6 +939,7 @@ describe("migration file rendering", () => {
"packages/database/migrations/0047_parse_artifact_checkpoints.postgres.sql",
"packages/database/migrations/0048_deletion_active_scope_indexes.postgres.sql",
"packages/database/migrations/0049_answer_trace_query_images.postgres.sql",
"packages/database/migrations/0050_deletion_job_capability_provenance.postgres.sql",
]);
expect(
getPendingMigrationArtifacts({
@ -990,6 +993,7 @@ describe("migration file rendering", () => {
"0047_parse_artifact_checkpoints",
"0048_deletion_active_scope_indexes",
"0049_answer_trace_query_images",
"0050_deletion_job_capability_provenance",
],
dialect: "postgres",
}),

17
knowledge-fs/packages/database/src/schema.test.ts
View File

@ -697,12 +697,7 @@ describe("database schema catalog", () => {
const outbox = findTable(schema, "deletion_outbox");
const retryAudits = findTable(schema, "deletion_retry_audits");
expect(jobs.foreignKeys).toContainEqual({
columns: ["tenant_id", "knowledge_space_id", "capability_grant_id"],
onDelete: "RESTRICT",
referencedColumns: ["tenant_id", "knowledge_space_id", "grant_id"],
referencedTable: "capability_grants",
});
expect(jobs.foreignKeys ?? []).toEqual([]);
expect(tombstones.foreignKeys ?? []).toEqual([]);
expect(items.foreignKeys).toEqual([
{
@ -713,15 +708,7 @@ describe("database schema catalog", () => {
},
]);
expect(outbox.foreignKeys).toEqual(items.foreignKeys);
expect(retryAudits.foreignKeys).toEqual([
...(items.foreignKeys ?? []),
{
columns: ["tenant_id", "knowledge_space_id", "capability_grant_id"],
onDelete: "RESTRICT",
referencedColumns: ["tenant_id", "knowledge_space_id", "grant_id"],
referencedTable: "capability_grants",
},
]);
expect(retryAudits.foreignKeys).toEqual(items.foreignKeys);
expect(jobs.columns.map((column) => column.name)).toEqual(
expect.arrayContaining([
"target_revision",

18
knowledge-fs/packages/database/src/schema.ts
View File

@ -3639,14 +3639,10 @@ const tables = [
name: "deletion_jobs_lease_ck",
},
],
foreignKeys: [
{
columns: ["tenant_id", "knowledge_space_id", "capability_grant_id"],
onDelete: "RESTRICT",
referencedColumns: ["tenant_id", "knowledge_space_id", "grant_id"],
referencedTable: "capability_grants",
},
],
// capability_grant_id is immutable authorization provenance. It deliberately has no live FK:
// the referenced grant is owned by the space and is cascaded while this deletion job must
// survive long enough to commit its terminal state.
foreignKeys: [],
columns: [
idColumn(),
varcharColumn("tenant_id", 255),
@ -3946,12 +3942,6 @@ const tables = [
referencedColumns: ["id"],
referencedTable: "deletion_jobs",
},
{
columns: ["tenant_id", "knowledge_space_id", "capability_grant_id"],
onDelete: "RESTRICT",
referencedColumns: ["tenant_id", "knowledge_space_id", "grant_id"],
referencedTable: "capability_grants",
},
],
columns: [
idColumn(),