import { KnowledgeSpaceEmbeddingProfileSchema, type KnowledgeSpaceManifest, type KnowledgeSpacePendingModelConfiguration, createKnowledgeSpaceRetrievalProfile, hasRequiredKnowledgeSpaceRetrievalModels, updateKnowledgeSpaceEmbeddingProfile, } from "@knowledge/core"; import type { DocumentCompilationProfileReference } from "./document-compilation-attempt-repository"; import { type DocumentCompilationExecutionContext, DocumentCompilationProcessingError, } from "./document-compilation-runtime"; import { knowledgeFsFailureForCode } from "./knowledge-fs-errors"; import type { KnowledgeSpaceManifestRepository } from "./knowledge-space-manifest-repository"; import type { KnowledgeSpaceProfileHead, KnowledgeSpaceProfileRepository, KnowledgeSpaceUnpublishedProfileActivationRepository, } from "./knowledge-space-profile-repository"; import { type ModelCapabilityPreflight, ModelCapabilityPreflightError, type ModelCapabilitySnapshot, } from "./model-capability-preflight"; export interface DocumentCompilationInitialProfileCoordinator { /** Ensures the leased attempt is bound to an immutable, verified profile tuple. */ ensureReady(execution: DocumentCompilationExecutionContext): Promise; } export interface DocumentCompilationInitialProfileCoordinatorOptions { readonly activations: KnowledgeSpaceUnpublishedProfileActivationRepository; readonly manifests: Pick; readonly now?: (() => string) | undefined; readonly preflight: ModelCapabilityPreflight; readonly profiles: Pick; } interface VerifiedInitialProfiles { readonly embedding?: { readonly capability: ModelCapabilitySnapshot; readonly profile: ReturnType; }; readonly retrieval: { readonly capability: Readonly>; readonly profile: ReturnType; }; } type ModelConfigurationField = "embedding" | "reasoning" | "rerank"; class ModelConfigurationValidationError extends Error { readonly field: ModelConfigurationField; constructor(field: ModelConfigurationField, cause: unknown) { super(`The ${field} model failed validation`, { cause }); this.name = "ModelConfigurationValidationError"; this.field = field; } } /** * Lazy profile activation for an empty knowledge space. Space creation persists selections * immediately; the first durable compilation observes only the embedding dimension needed by the * vector store and trusts Dify's tenant-active catalog for reasoning and rerank identities. * * The local promise map suppresses duplicate activation work in one API process. Database manifest/profile * CAS remains the cross-process authority, so duplicate workers can only converge on the same * immutable snapshots and can never publish an unverified tuple. */ export function createDocumentCompilationInitialProfileCoordinator({ activations, manifests, now = () => new Date().toISOString(), preflight, profiles, }: DocumentCompilationInitialProfileCoordinatorOptions): DocumentCompilationInitialProfileCoordinator { const inFlight = new Map>(); return { ensureReady: async (execution) => { let existingHeads: Awaited>; let manifest: KnowledgeSpaceManifest | null; try { [existingHeads, manifest] = await Promise.all([ activeProfileHeads(profiles, execution.attempt), manifests.get(execution.attempt), ]); } catch (error) { throw compilationError( "MODEL_PROFILE_STATE_READ_FAILED", "Knowledge-space model state could not be read and will be retried", true, error, ); } if (!manifest) { throw compilationError( "KNOWLEDGE_SPACE_MANIFEST_NOT_FOUND", "Knowledge-space model configuration is unavailable", false, ); } if ( existingHeads.retrieval && !manifest.pendingModelConfiguration && activeTupleSupportsCompilation(existingHeads) ) { if (execution.attempt.retrievalProfile) return; await bindInitialProfiles(execution, { ...(existingHeads.embedding ? { embeddingProfile: profileReference(existingHeads.embedding) } : {}), retrievalProfile: profileReference(existingHeads.retrieval), }); return; } if (existingHeads.retrieval && !manifest.pendingModelConfiguration) { throw compilationError( "MODEL_PROFILE_ACTIVATION_INCOMPLETE", "Retrieval requires an active embedding profile and enabled rerank model", false, ); } const pending = requireCompletePendingConfiguration(manifest); const key = `${manifest.tenantId}:${manifest.knowledgeSpaceId}:${pending.digest}`; let activation = inFlight.get(key); if (!activation) { activation = activatePendingConfiguration({ activations, execution, manifests, now, pending, preflight, }).finally(() => inFlight.delete(key)); inFlight.set(key, activation); } await activation; let heads: Awaited>; try { heads = await activeProfileHeads(profiles, execution.attempt); } catch (error) { throw compilationError( "MODEL_PROFILE_STATE_READ_FAILED", "Activated model state could not be read and will be retried", true, error, ); } if (!heads.retrieval) { throw compilationError( "MODEL_PROFILE_ACTIVATION_INCOMPLETE", "Knowledge-space model validation has not completed", true, ); } await bindInitialProfiles(execution, { ...(heads.embedding ? { embeddingProfile: profileReference(heads.embedding) } : {}), retrievalProfile: profileReference(heads.retrieval), }); }, }; } async function bindInitialProfiles( execution: DocumentCompilationExecutionContext, profiles: { readonly embeddingProfile?: DocumentCompilationProfileReference | undefined; readonly retrievalProfile: DocumentCompilationProfileReference; }, ): Promise { try { await execution.bindInitialProfiles(profiles); } catch (error) { if (error instanceof DocumentCompilationProcessingError) throw error; throw compilationError( "MODEL_PROFILE_BIND_FAILED", "The verified model tuple could not be bound and will be retried", true, error, ); } } async function activatePendingConfiguration(input: { readonly activations: KnowledgeSpaceUnpublishedProfileActivationRepository; readonly execution: DocumentCompilationExecutionContext; readonly manifests: Pick; readonly now: () => string; readonly pending: KnowledgeSpacePendingModelConfiguration; readonly preflight: ModelCapabilityPreflight; }): Promise { const { attempt } = input.execution; const permission = initialActivationPermission(attempt); let verified: VerifiedInitialProfiles; try { verified = await verifyPendingConfiguration( input.preflight, input.pending, attempt.tenantId, input.execution.signal, ); } catch (error) { const classified = classifyInitialProfileError(error); if (!classified.retryable) { let recorded: "recorded" | "stale"; try { recorded = await recordValidationFailure(input, classified, permission); } catch (persistenceError) { throw compilationError( "MODEL_VALIDATION_FAILURE_PERSISTENCE_FAILED", "Model validation status could not be persisted and will be retried", true, persistenceError, ); } if (recorded === "stale") { throw compilationError( "MODEL_CONFIGURATION_STALE", "The selected model configuration changed while it was being validated", true, error, ); } } throw compilationError(classified.code, classified.message, classified.retryable, error); } try { const currentManifest = await input.manifests.get(input.execution.attempt); if (!currentManifest) { throw compilationError( "KNOWLEDGE_SPACE_MANIFEST_NOT_FOUND", "Knowledge-space model configuration is unavailable", false, ); } await input.activations.activateInitialTuple({ createdBySubjectId: "capabilityGrantId" in permission ? `capability-grant:${permission.capabilityGrantId}` : permission.requestedBySubjectId, ...(verified.embedding ? { embedding: { capabilitySnapshot: verified.embedding.capability, snapshot: verified.embedding.profile, }, } : {}), expectedManifestVersion: currentManifest.manifestVersion, expectedPendingConfiguration: { digest: input.pending.digest, revision: input.pending.revision, }, knowledgeSpaceId: input.execution.attempt.knowledgeSpaceId, now: input.now(), permission, requiredAccess: "write", retrieval: { capabilitySnapshot: verified.retrieval.capability, snapshot: verified.retrieval.profile, }, tenantId: input.execution.attempt.tenantId, }); } catch (error) { if (error instanceof DocumentCompilationProcessingError) throw error; throw compilationError( "MODEL_PROFILE_ACTIVATION_CONFLICT", "Knowledge-space model activation will be retried", true, error, ); } } async function verifyPendingConfiguration( preflight: ModelCapabilityPreflight, pending: KnowledgeSpacePendingModelConfiguration, tenantId: string, signal: AbortSignal, ): Promise { const retrievalInput = pending.retrievalProfile; if (!retrievalInput) { throw compilationError( "KNOWLEDGE_SPACE_MODEL_CONFIGURATION_REQUIRED", "A retrieval model must be configured before documents can be compiled", false, ); } const embeddingSelection = pending.embeddingSelection; const rerankSelection = retrievalInput.rerank.model; if ( !embeddingSelection || !rerankSelection || !hasRequiredKnowledgeSpaceRetrievalModels(retrievalInput) ) { throw compilationError( "KNOWLEDGE_SPACE_MODEL_CONFIGURATION_REQUIRED", "Embedding, reasoning, and rerank models must be configured before documents can be compiled", false, ); } const [embedding, reasoning, rerank] = await Promise.all([ verifyPendingModel(preflight, "embedding", { kind: "embedding", selection: embeddingSelection, signal, tenantId, }), resolveConfiguredModel(preflight, "reasoning", { kind: "reasoning", selection: retrievalInput.reasoningModel, signal, tenantId, }), resolveConfiguredModel(preflight, "rerank", { kind: "rerank", selection: rerankSelection, signal, tenantId, }), ]); if (!pending.embeddingSelection || !embedding) { throw new ModelCapabilityPreflightError( "MODEL_CAPABILITY_MISMATCH", "Fast, Deep, and Research retrieval require an embedding model for this knowledge space", ); } if (!reasoning) { throw new ModelCapabilityPreflightError( "MODEL_CAPABILITY_MISMATCH", "The reasoning model did not return a verified capability snapshot", ); } if (!rerank) { throw new ModelCapabilityPreflightError( "MODEL_CAPABILITY_MISMATCH", "The rerank model did not return a verified capability snapshot", ); } let verifiedEmbedding: VerifiedInitialProfiles["embedding"]; if (pending.embeddingSelection && embedding) { if ( embedding.kind !== "embedding" || embedding.dimension === undefined || !embedding.distanceMetric ) { throw new ModelCapabilityPreflightError( "EMBEDDING_DIMENSION_INVALID", "The embedding model did not return a usable vector dimension", ); } const profile = await updateKnowledgeSpaceEmbeddingProfile( undefined, pending.embeddingSelection, { capabilityDigest: embedding.capabilityDigest, dimension: embedding.dimension, distanceMetric: embedding.distanceMetric, pluginUniqueIdentifier: embedding.pluginUniqueIdentifier, schemaFingerprint: embedding.schemaFingerprint, }, ); verifiedEmbedding = { capability: embedding, profile: KnowledgeSpaceEmbeddingProfileSchema.parse({ ...profile, dimension: embedding.dimension, }), }; } return { ...(verifiedEmbedding ? { embedding: verifiedEmbedding } : {}), retrieval: { capability: { reasoning, rerank: rerank ?? null, verification: "verified", }, profile: createKnowledgeSpaceRetrievalProfile(retrievalInput), }, }; } async function resolveConfiguredModel( preflight: ModelCapabilityPreflight, field: Exclude, input: Parameters[0], ): Promise { try { return preflight.resolveConfigured ? await preflight.resolveConfigured(input) : await preflight.verify(input); } catch (error) { throw new ModelConfigurationValidationError(field, error); } } async function verifyPendingModel( preflight: ModelCapabilityPreflight, field: ModelConfigurationField, input: Parameters[0], ): Promise { try { return await preflight.verify(input); } catch (error) { throw new ModelConfigurationValidationError(field, error); } } async function recordValidationFailure( input: { readonly execution: DocumentCompilationExecutionContext; readonly manifests: Pick; readonly now: () => string; readonly pending: KnowledgeSpacePendingModelConfiguration; }, failure: { readonly code: string; readonly field?: ModelConfigurationField | undefined; readonly retryable: boolean; }, permission: Parameters< KnowledgeSpaceUnpublishedProfileActivationRepository["activate"] >[0]["permission"], ): Promise<"recorded" | "stale"> { const scope = input.execution.attempt; for (let retry = 0; retry < 5; retry += 1) { const manifest = await input.manifests.get(scope); if ( !manifest?.pendingModelConfiguration || manifest.pendingModelConfiguration.digest !== input.pending.digest || manifest.pendingModelConfiguration.revision !== input.pending.revision ) { return "stale"; } const timestamp = input.now(); const updated = await input.manifests.update({ expectedManifestVersion: manifest.manifestVersion, knowledgeSpaceId: scope.knowledgeSpaceId, permission: { fence: permission, now: timestamp, requiredAccess: "write" }, patch: { manifestVersion: manifest.manifestVersion + 1, pendingModelConfiguration: { ...manifest.pendingModelConfiguration, failure: { code: failure.code.slice(0, 64), failedAt: timestamp, ...(failure.field ? { field: failure.field } : {}), retryable: failure.retryable, }, state: "validation-failed", }, updatedAt: timestamp, }, tenantId: scope.tenantId, }); if (updated) return "recorded"; } throw new Error("Model validation failure manifest CAS retries were exhausted"); } function requireCompletePendingConfiguration( manifest: KnowledgeSpaceManifest, ): KnowledgeSpacePendingModelConfiguration { const pending = manifest.pendingModelConfiguration; if (!pending?.retrievalProfile) { throw compilationError( "KNOWLEDGE_SPACE_MODEL_CONFIGURATION_REQUIRED", "A retrieval model must be configured before documents can be compiled", false, ); } return pending; } async function activeProfileHeads( profiles: Pick, scope: { readonly knowledgeSpaceId: string; readonly tenantId: string }, ): Promise<{ readonly embedding: KnowledgeSpaceProfileHead | null; readonly retrieval: KnowledgeSpaceProfileHead | null; }> { const [embedding, retrieval] = await Promise.all([ profiles.getHead({ ...scope, kind: "embedding" }), profiles.getHead({ ...scope, kind: "retrieval" }), ]); return { embedding, retrieval }; } function profileReference(head: KnowledgeSpaceProfileHead): DocumentCompilationProfileReference { return { kind: head.kind, revision: head.activeRevision, revisionId: head.profileRevisionId, snapshotDigest: head.profile.snapshotDigest, }; } function activeTupleSupportsCompilation(heads: { readonly embedding: KnowledgeSpaceProfileHead | null; readonly retrieval: KnowledgeSpaceProfileHead | null; }): boolean { if (!heads.retrieval) return false; const retrieval = heads.retrieval.profile.snapshot; return ( "defaultMode" in retrieval && heads.embedding !== null && hasRequiredKnowledgeSpaceRetrievalModels(retrieval) ); } function initialActivationPermission( attempt: DocumentCompilationExecutionContext["attempt"], ): Parameters[0]["permission"] { if (attempt.capabilityGrantId) { return { capabilityGrantId: attempt.capabilityGrantId, knowledgeSpaceId: attempt.knowledgeSpaceId, tenantId: attempt.tenantId, }; } if (!attempt.permissionSnapshot || !attempt.requestedBySubjectId) { throw compilationError( "MODEL_PROFILE_ACTIVATION_PERMISSION_REQUIRED", "A durable write permission is required to activate the model configuration", false, ); } return { accessChannel: attempt.permissionSnapshot.accessChannel, knowledgeSpaceId: attempt.knowledgeSpaceId, permissionSnapshotId: attempt.permissionSnapshot.id, permissionSnapshotRevision: attempt.permissionSnapshot.revision, requestedBySubjectId: attempt.requestedBySubjectId, tenantId: attempt.tenantId, }; } function classifyInitialProfileError(error: unknown): { readonly code: string; readonly field?: ModelConfigurationField | undefined; readonly message: string; readonly retryable: boolean; } { if (error instanceof ModelConfigurationValidationError) { return { ...classifyInitialProfileError(error.cause), field: error.field }; } if (error instanceof DocumentCompilationProcessingError) { const failure = knowledgeFsFailureForCode(error.code, { stage: "model_preflight" }); return { code: failure.code, message: failure.message, retryable: error.retryable }; } if (error instanceof ModelCapabilityPreflightError) { const failure = knowledgeFsFailureForCode(error.code, { stage: "model_preflight" }); return { code: failure.code, message: failure.message, retryable: error.retryable, }; } const failure = knowledgeFsFailureForCode("MODEL_PREFLIGHT_FAILED", { stage: "model_preflight", }); return { code: failure.code, message: failure.message, retryable: true, }; } function compilationError( code: string, message: string, retryable: boolean, cause?: unknown, ): DocumentCompilationProcessingError { return new DocumentCompilationProcessingError(message, { ...(cause === undefined ? {} : { cause }), code, retryable, }); }