From 644a9b96a65b23573f12c1b9c14ba0ec4f2006af Mon Sep 17 00:00:00 2001 From: FFXN Date: Thu, 3 Sep 2026 16:59:57 +0800 Subject: [PATCH 1/2] fix(knowledge-fs): harden semantic chunking and deletion cleanup - retry semantic chunking when the LLM returns an invalid schema - add corrective schema instructions to retry prompts - remove capability grant foreign keys from durable deletion records - preserve capability grant IDs as immutable audit provenance - add migration and regression coverage --- api/services/knowledge_fs/product_dto.py | 3 ++ knowledge-fs/apps/api/src/index.ts | 2 +- knowledge-fs/apps/api/src/migrate.test.ts | 6 +++ .../adapters/src/migration-runner.test.ts | 2 + .../api/src/llm-semantic-chunker.test.ts | 6 +++ .../packages/api/src/llm-semantic-chunker.ts | 8 ++++ ...ion_job_capability_provenance.postgres.sql | 11 +++++ ...eletion_job_capability_provenance.tidb.sql | 11 +++++ ...apability-job-provenance-migration.test.ts | 4 +- ...ob-capability-provenance-migration.test.ts | 43 +++++++++++++++++++ .../src/migration-artifacts.generated.ts | 2 + .../database/src/migration-file.test.ts | 4 ++ .../packages/database/src/schema.test.ts | 17 +------- knowledge-fs/packages/database/src/schema.ts | 18 ++------ 14 files changed, 105 insertions(+), 32 deletions(-) create mode 100644 knowledge-fs/packages/database/migrations/0050_deletion_job_capability_provenance.postgres.sql create mode 100644 knowledge-fs/packages/database/migrations/0050_deletion_job_capability_provenance.tidb.sql create mode 100644 knowledge-fs/packages/database/src/deletion-job-capability-provenance-migration.test.ts diff --git a/api/services/knowledge_fs/product_dto.py b/api/services/knowledge_fs/product_dto.py index f0328ff5ede..1638411a7fd 100644 --- a/api/services/knowledge_fs/product_dto.py +++ b/api/services/knowledge_fs/product_dto.py @@ -239,6 +239,9 @@ class KnowledgeFSInitialWebsiteSourcePayload(KnowledgeFSInitialSyncPolicyPayload @model_validator(mode="after") def validate_selection(self) -> KnowledgeFSInitialWebsiteSourcePayload: + source_urls = [item.canonical_url for item in self.selection] + if len(set(source_urls)) != len(source_urls): + raise ValueError("initial website selection canonical URLs must be unique") if self.crawl_options.limit < len(self.selection): raise ValueError("initial website crawl limit must cover every selected URL") if ( diff --git a/knowledge-fs/apps/api/src/index.ts b/knowledge-fs/apps/api/src/index.ts index 5db60feb9d6..629c5045436 100644 --- a/knowledge-fs/apps/api/src/index.ts +++ b/knowledge-fs/apps/api/src/index.ts @@ -472,7 +472,7 @@ const documentSemanticChunker = createLlmSemanticChunker({ : {}), maxConcurrentWindows: ingestionModelRuntimeOptions.semanticExtractionMaxConcurrency, maxNodes: 20_000, - maxProviderOutputRetries: 1, + maxProviderOutputRetries: 3, maxWindowChars: ingestionModelRuntimeOptions.semanticChunkingMaxWindowChars, metrics: operationalMetrics.ingestionModelCalls, modelRequestGate: ingestionModelRuntimeOptions.modelRequestGate, diff --git a/knowledge-fs/apps/api/src/migrate.test.ts b/knowledge-fs/apps/api/src/migrate.test.ts index c496f0b5aa6..ebed39f0bd7 100644 --- a/knowledge-fs/apps/api/src/migrate.test.ts +++ b/knowledge-fs/apps/api/src/migrate.test.ts @@ -147,6 +147,8 @@ describe("runApiDatabaseMigrations", () => { "insert", "schema", "insert", + "schema", + "insert", ]); expect(migrationSql).toHaveLength(expectedPostgresMigrationIds.length); expect(migrationSql[2]).toContain("-- Migration id: 0003_projection_set_publications\n"); @@ -224,6 +226,10 @@ describe("runApiDatabaseMigrations", () => { expect(migrationSql[47]).toContain( 'CREATE INDEX IF NOT EXISTS "deletion_tombstones_active_scope_idx"', ); + expect(migrationSql[48]).toContain("-- Migration id: 0049_answer_trace_query_images\n"); + expect(migrationSql[49]).toContain( + "-- Migration id: 0050_deletion_job_capability_provenance\n", + ); expect(closed).toBe(true); }); diff --git a/knowledge-fs/packages/adapters/src/migration-runner.test.ts b/knowledge-fs/packages/adapters/src/migration-runner.test.ts index e0de3dc0a83..f6df30db9c5 100644 --- a/knowledge-fs/packages/adapters/src/migration-runner.test.ts +++ b/knowledge-fs/packages/adapters/src/migration-runner.test.ts @@ -57,6 +57,7 @@ const removeProviderSyncPolicyMigrationId = "0046_remove_provider_sync_policy"; const parseArtifactCheckpointsMigrationId = "0047_parse_artifact_checkpoints"; const deletionActiveScopeIndexesMigrationId = "0048_deletion_active_scope_indexes"; const answerTraceQueryImagesMigrationId = "0049_answer_trace_query_images"; +const deletionJobCapabilityProvenanceMigrationId = "0050_deletion_job_capability_provenance"; const migrationsAfterDurableDeletion = [ versionedSpaceProfilesMigrationId, profilePublicationBindingsMigrationId, @@ -90,6 +91,7 @@ const migrationsAfterDurableDeletion = [ parseArtifactCheckpointsMigrationId, deletionActiveScopeIndexesMigrationId, answerTraceQueryImagesMigrationId, + deletionJobCapabilityProvenanceMigrationId, ] as const; const migrationsAfterTidbBaselineRepair = [ spaceAccessControlMigrationId, diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts index 46e4a3aae3b..e204d9fab9d 100644 --- a/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts @@ -2920,6 +2920,12 @@ describe("LLM semantic chunker", () => { expect(nodes).toHaveLength(1); expect(provider.calls).toHaveLength(2); + expect(provider.calls[0]?.messages[0]?.content).not.toContain( + "previous response failed JSON schema validation", + ); + expect(provider.calls[1]?.messages[0]?.content).toContain( + "previous response failed JSON schema validation", + ); }); it("accepts JSON wrapped in provider prose but strictly caps joint extraction arrays", async () => { diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker.ts index 0c13d002865..5eba1cefc71 100644 --- a/knowledge-fs/packages/api/src/llm-semantic-chunker.ts +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker.ts @@ -497,6 +497,7 @@ export function createLlmSemanticChunker({ maxChunkChars: effectiveConfig.maxChunkChars, maxEntitiesPerChunk, maxRelationsPerChunk, + retryCount, window, }); const callStartedAt = Date.now(); @@ -2368,6 +2369,7 @@ function semanticChunkingMessages({ maxChunkChars, maxEntitiesPerChunk, maxRelationsPerChunk, + retryCount, window, }: { readonly enableGraph: boolean; @@ -2375,6 +2377,7 @@ function semanticChunkingMessages({ readonly maxChunkChars: number; readonly maxEntitiesPerChunk: number; readonly maxRelationsPerChunk: number; + readonly retryCount: number; readonly window: SemanticWindow; }): readonly SemanticChunkingLlmMessage[] { const carriesParserProvenance = window.planningVersion !== "v1"; @@ -2389,6 +2392,11 @@ function semanticChunkingMessages({ ? "PageIndex is enabled: assign a concise semantic sectionPath and sectionSummary to every chunk." : "PageIndex is disabled: preserve only the supplied sectionPath, omit sectionSummary, and do not invent child section levels.", "Return strict JSON only. Never return, rewrite, summarize, correct, or duplicate source text.", + ...(retryCount > 0 + ? [ + "Your previous response failed JSON schema validation. Correct the structure and return exactly one JSON object matching Output shape; do not include prose or Markdown fences.", + ] + : []), "The units field is the core: cover every core unit exactly once, in order, by contiguous inclusive ranges.", ...(usesFixedCoreBoundary(window.planningVersion) ? [ diff --git a/knowledge-fs/packages/database/migrations/0050_deletion_job_capability_provenance.postgres.sql b/knowledge-fs/packages/database/migrations/0050_deletion_job_capability_provenance.postgres.sql new file mode 100644 index 00000000000..97c3843e946 --- /dev/null +++ b/knowledge-fs/packages/database/migrations/0050_deletion_job_capability_provenance.postgres.sql @@ -0,0 +1,11 @@ +-- Knowledge Platform schema migration +-- Migration id: 0050_deletion_job_capability_provenance +-- Dialect: postgres +-- A space deletion job outlives the space-owned grant that authorized it. Preserve the grant id +-- as immutable audit provenance without a live FK that blocks the terminal space delete. + +ALTER TABLE "deletion_jobs" + DROP CONSTRAINT IF EXISTS "deletion_jobs_capability_grant_fk"; + +ALTER TABLE "deletion_retry_audits" + DROP CONSTRAINT IF EXISTS "deletion_retry_audits_capability_grant_fk"; diff --git a/knowledge-fs/packages/database/migrations/0050_deletion_job_capability_provenance.tidb.sql b/knowledge-fs/packages/database/migrations/0050_deletion_job_capability_provenance.tidb.sql new file mode 100644 index 00000000000..7028707ce3b --- /dev/null +++ b/knowledge-fs/packages/database/migrations/0050_deletion_job_capability_provenance.tidb.sql @@ -0,0 +1,11 @@ +-- Knowledge Platform schema migration +-- Migration id: 0050_deletion_job_capability_provenance +-- Dialect: tidb +-- A space deletion job outlives the space-owned grant that authorized it. Preserve the grant id +-- as immutable audit provenance without a live FK that blocks the terminal space delete. + +ALTER TABLE `deletion_jobs` + DROP FOREIGN KEY `deletion_jobs_capability_grant_fk`; + +ALTER TABLE `deletion_retry_audits` + DROP FOREIGN KEY `deletion_retry_audits_capability_grant_fk`; diff --git a/knowledge-fs/packages/database/src/capability-job-provenance-migration.test.ts b/knowledge-fs/packages/database/src/capability-job-provenance-migration.test.ts index 325c9118d49..9633924bd52 100644 --- a/knowledge-fs/packages/database/src/capability-job-provenance-migration.test.ts +++ b/knowledge-fs/packages/database/src/capability-job-provenance-migration.test.ts @@ -59,7 +59,7 @@ describe("capability job provenance migration", () => { foreignKey.referencedTable === "capability_grants" && foreignKey.columns.includes("capability_grant_id"), ), - ).toBe(true); + ).toBe(tableName !== "deletion_jobs"); expect( schema.indexes.some( (index) => index.tableName === tableName && index.columns.includes("capability_grant_id"), @@ -81,7 +81,7 @@ describe("capability job provenance migration", () => { deletionRetryAudits?.foreignKeys?.some( (foreignKey) => foreignKey.referencedTable === "capability_grants", ), - ).toBe(true); + ).toBe(false); expect( schema.indexes.some( (index) => diff --git a/knowledge-fs/packages/database/src/deletion-job-capability-provenance-migration.test.ts b/knowledge-fs/packages/database/src/deletion-job-capability-provenance-migration.test.ts new file mode 100644 index 00000000000..aa9809e6edb --- /dev/null +++ b/knowledge-fs/packages/database/src/deletion-job-capability-provenance-migration.test.ts @@ -0,0 +1,43 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { getDatabaseSchema } from "./schema"; + +const root = resolve(import.meta.dirname, "../../.."); + +describe("deletion job capability provenance migration", () => { + it.each([ + ["postgres", "DROP CONSTRAINT IF EXISTS", '"'], + ["tidb", "DROP FOREIGN KEY", "`"], + ] as const)("drops the live grant foreign keys on %s", (dialect, statement, quote) => { + const sql = readFileSync( + resolve( + root, + `packages/database/migrations/0050_deletion_job_capability_provenance.${dialect}.sql`, + ), + "utf8", + ); + expect(sql).toContain(`${statement} ${quote}deletion_jobs_capability_grant_fk${quote}`); + expect(sql).toContain(`${statement} ${quote}deletion_retry_audits_capability_grant_fk${quote}`); + expect(sql).not.toContain("DROP COLUMN"); + }); + + it("keeps the grant id as provenance without a live schema foreign key", () => { + const table = getDatabaseSchema().tables.find( + (candidate) => candidate.name === "deletion_jobs", + ); + expect(table?.columns.some((column) => column.name === "capability_grant_id")).toBe(true); + expect( + table?.foreignKeys?.some((foreignKey) => foreignKey.columns.includes("capability_grant_id")), + ).toBe(false); + const audits = getDatabaseSchema().tables.find( + (candidate) => candidate.name === "deletion_retry_audits", + ); + expect(audits?.columns.some((column) => column.name === "capability_grant_id")).toBe(true); + expect( + audits?.foreignKeys?.some((foreignKey) => foreignKey.columns.includes("capability_grant_id")), + ).toBe(false); + }); +}); diff --git a/knowledge-fs/packages/database/src/migration-artifacts.generated.ts b/knowledge-fs/packages/database/src/migration-artifacts.generated.ts index 67f5bd34103..b8e49d86ac9 100644 --- a/knowledge-fs/packages/database/src/migration-artifacts.generated.ts +++ b/knowledge-fs/packages/database/src/migration-artifacts.generated.ts @@ -101,4 +101,6 @@ export const migrationArtifacts = [ { content: "-- Knowledge Platform schema migration\n-- Migration id: 0048_deletion_active_scope_indexes\n-- Dialect: tidb\n-- Keeps active deletion admission proportional to live fences instead of permanent audit history.\n\nCREATE INDEX IF NOT EXISTS `deletion_jobs_active_scope_idx`\n ON `deletion_jobs` (`tenant_id`, `knowledge_space_id`, `active_slot`, `target_type`, `target_id`);\n\nCREATE INDEX IF NOT EXISTS `deletion_tombstones_active_scope_idx`\n ON `deletion_tombstones` (`tenant_id`, `knowledge_space_id`, `state`);\n", path: "packages/database/migrations/0048_deletion_active_scope_indexes.tidb.sql" }, { content: "-- Knowledge Platform schema migration\n-- Migration id: 0049_answer_trace_query_images\n-- Dialect: postgres\n-- Answer traces keep the query-image references they were run with so history can show them.\n\nALTER TABLE \"answer_traces\"\n ADD COLUMN IF NOT EXISTS \"query_images\" JSONB;\n", path: "packages/database/migrations/0049_answer_trace_query_images.postgres.sql" }, { content: "-- Knowledge Platform schema migration\n-- Migration id: 0049_answer_trace_query_images\n-- Dialect: tidb\n-- Answer traces keep the query-image references they were run with so history can show them.\n\nALTER TABLE `answer_traces`\n ADD COLUMN IF NOT EXISTS `query_images` JSON NULL;\n", path: "packages/database/migrations/0049_answer_trace_query_images.tidb.sql" }, + { content: "-- Knowledge Platform schema migration\n-- Migration id: 0050_deletion_job_capability_provenance\n-- Dialect: postgres\n-- A space deletion job outlives the space-owned grant that authorized it. Preserve the grant id\n-- as immutable audit provenance without a live FK that blocks the terminal space delete.\n\nALTER TABLE \"deletion_jobs\"\n DROP CONSTRAINT IF EXISTS \"deletion_jobs_capability_grant_fk\";\n\nALTER TABLE \"deletion_retry_audits\"\n DROP CONSTRAINT IF EXISTS \"deletion_retry_audits_capability_grant_fk\";\n", path: "packages/database/migrations/0050_deletion_job_capability_provenance.postgres.sql" }, + { content: "-- Knowledge Platform schema migration\n-- Migration id: 0050_deletion_job_capability_provenance\n-- Dialect: tidb\n-- A space deletion job outlives the space-owned grant that authorized it. Preserve the grant id\n-- as immutable audit provenance without a live FK that blocks the terminal space delete.\n\nALTER TABLE `deletion_jobs`\n DROP FOREIGN KEY `deletion_jobs_capability_grant_fk`;\n\nALTER TABLE `deletion_retry_audits`\n DROP FOREIGN KEY `deletion_retry_audits_capability_grant_fk`;\n", path: "packages/database/migrations/0050_deletion_job_capability_provenance.tidb.sql" }, ] as const satisfies readonly MigrationArtifact[]; diff --git a/knowledge-fs/packages/database/src/migration-file.test.ts b/knowledge-fs/packages/database/src/migration-file.test.ts index 77733df30fe..10a0f9b0d7e 100644 --- a/knowledge-fs/packages/database/src/migration-file.test.ts +++ b/knowledge-fs/packages/database/src/migration-file.test.ts @@ -158,6 +158,8 @@ describe("migration file rendering", () => { "packages/database/migrations/0048_deletion_active_scope_indexes.tidb.sql", "packages/database/migrations/0049_answer_trace_query_images.postgres.sql", "packages/database/migrations/0049_answer_trace_query_images.tidb.sql", + "packages/database/migrations/0050_deletion_job_capability_provenance.postgres.sql", + "packages/database/migrations/0050_deletion_job_capability_provenance.tidb.sql", ]); const workflowCapturePostgres = artifacts.find( (artifact) => @@ -937,6 +939,7 @@ describe("migration file rendering", () => { "packages/database/migrations/0047_parse_artifact_checkpoints.postgres.sql", "packages/database/migrations/0048_deletion_active_scope_indexes.postgres.sql", "packages/database/migrations/0049_answer_trace_query_images.postgres.sql", + "packages/database/migrations/0050_deletion_job_capability_provenance.postgres.sql", ]); expect( getPendingMigrationArtifacts({ @@ -990,6 +993,7 @@ describe("migration file rendering", () => { "0047_parse_artifact_checkpoints", "0048_deletion_active_scope_indexes", "0049_answer_trace_query_images", + "0050_deletion_job_capability_provenance", ], dialect: "postgres", }), diff --git a/knowledge-fs/packages/database/src/schema.test.ts b/knowledge-fs/packages/database/src/schema.test.ts index 61462c11700..225b50cc7b8 100644 --- a/knowledge-fs/packages/database/src/schema.test.ts +++ b/knowledge-fs/packages/database/src/schema.test.ts @@ -697,12 +697,7 @@ describe("database schema catalog", () => { const outbox = findTable(schema, "deletion_outbox"); const retryAudits = findTable(schema, "deletion_retry_audits"); - expect(jobs.foreignKeys).toContainEqual({ - columns: ["tenant_id", "knowledge_space_id", "capability_grant_id"], - onDelete: "RESTRICT", - referencedColumns: ["tenant_id", "knowledge_space_id", "grant_id"], - referencedTable: "capability_grants", - }); + expect(jobs.foreignKeys ?? []).toEqual([]); expect(tombstones.foreignKeys ?? []).toEqual([]); expect(items.foreignKeys).toEqual([ { @@ -713,15 +708,7 @@ describe("database schema catalog", () => { }, ]); expect(outbox.foreignKeys).toEqual(items.foreignKeys); - expect(retryAudits.foreignKeys).toEqual([ - ...(items.foreignKeys ?? []), - { - columns: ["tenant_id", "knowledge_space_id", "capability_grant_id"], - onDelete: "RESTRICT", - referencedColumns: ["tenant_id", "knowledge_space_id", "grant_id"], - referencedTable: "capability_grants", - }, - ]); + expect(retryAudits.foreignKeys).toEqual(items.foreignKeys); expect(jobs.columns.map((column) => column.name)).toEqual( expect.arrayContaining([ "target_revision", diff --git a/knowledge-fs/packages/database/src/schema.ts b/knowledge-fs/packages/database/src/schema.ts index 47115d4e4f6..e58112054ec 100644 --- a/knowledge-fs/packages/database/src/schema.ts +++ b/knowledge-fs/packages/database/src/schema.ts @@ -3639,14 +3639,10 @@ const tables = [ name: "deletion_jobs_lease_ck", }, ], - foreignKeys: [ - { - columns: ["tenant_id", "knowledge_space_id", "capability_grant_id"], - onDelete: "RESTRICT", - referencedColumns: ["tenant_id", "knowledge_space_id", "grant_id"], - referencedTable: "capability_grants", - }, - ], + // capability_grant_id is immutable authorization provenance. It deliberately has no live FK: + // the referenced grant is owned by the space and is cascaded while this deletion job must + // survive long enough to commit its terminal state. + foreignKeys: [], columns: [ idColumn(), varcharColumn("tenant_id", 255), @@ -3946,12 +3942,6 @@ const tables = [ referencedColumns: ["id"], referencedTable: "deletion_jobs", }, - { - columns: ["tenant_id", "knowledge_space_id", "capability_grant_id"], - onDelete: "RESTRICT", - referencedColumns: ["tenant_id", "knowledge_space_id", "grant_id"], - referencedTable: "capability_grants", - }, ], columns: [ idColumn(), From 25277f66d60345f5ee7a5734241e07329f0899d6 Mon Sep 17 00:00:00 2001 From: FFXN Date: Thu, 3 Sep 2026 17:00:17 +0800 Subject: [PATCH 2/2] fix(knowledge-fs): harden semantic chunking and deletion cleanup - retry semantic chunking when the LLM returns an invalid schema - add corrective schema instructions to retry prompts - remove capability grant foreign keys from durable deletion records - preserve capability grant IDs as immutable audit provenance - add migration and regression coverage --- api/knowledge-fs-contract.lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/knowledge-fs-contract.lock.json b/api/knowledge-fs-contract.lock.json index 38af3da423a..e2ea6504053 100644 --- a/api/knowledge-fs-contract.lock.json +++ b/api/knowledge-fs-contract.lock.json @@ -1,6 +1,6 @@ { "schemaVersion": 5, - "subtreeTree": "3f7d65b2322dfb13528f9c8a9b3a68e0b6a9b80a", + "subtreeTree": "958ff662b58baf8efd4a0e5a2112156f703596e1", "openapiSha256": "bb5c1f6529ee216059b4aa1e01d8079a0bb175bb6adba7a0821fec8b6fa707f1", "capabilityV2AuthManifestSha256": "e322a2fa779d1f40b95c54c1021cffecaec77abbd7b34899573dcdf4ff353109", "capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",