mirror of
https://github.com/langgenius/dify.git
synced 2026-09-02 22:36:15 +08:00
2169 lines
76 KiB
TypeScript
2169 lines
76 KiB
TypeScript
import { createNodePlatformAdapter } from "@knowledge/adapters/node";
|
|
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import { DurableDeletionServiceError } from "./durable-deletion-service";
|
|
import {
|
|
createAcceptingDurableDeletionService,
|
|
createAllowingDurableDeletionSafetyOptions,
|
|
} from "./durable-deletion-test-utils";
|
|
import {
|
|
type KnowledgeSpaceRepository,
|
|
type OnlineDocumentConnector,
|
|
type OnlineDriveConnector,
|
|
SOURCE_OPERATION_FAILURES,
|
|
SourceCredentialMutationError,
|
|
type SourceCredentialService,
|
|
type SourceCredentialTester,
|
|
type SourceDocumentMaterializer,
|
|
type SourceProductWorkflowRepository,
|
|
type SourceRepository,
|
|
type SourceSecretStore,
|
|
type WebsiteCrawlConnector,
|
|
createInMemoryKnowledgeSpaceRepository,
|
|
createInMemorySourceRepository,
|
|
createInMemorySourceRetiredSecretCleanupRepository,
|
|
createKnowledgeGateway,
|
|
createKnowledgeGatewayApp,
|
|
createSourceCredentialFingerprinter,
|
|
createSourceCredentialService,
|
|
createStaticAuthVerifier,
|
|
mimeTypeForFilename,
|
|
onlineDocumentFilename,
|
|
readImportedFilesState,
|
|
readImportedState,
|
|
readSourceCredentialConfig,
|
|
registerSourceHandlers,
|
|
} from "./index";
|
|
|
|
const readToken = "read-token";
|
|
const writeToken = "write-token";
|
|
const otherTenantToken = "other-tenant-token";
|
|
const missingSourceId = "00000000-0000-4000-8000-00000000dead";
|
|
|
|
function bearer(token: string) {
|
|
return { authorization: `Bearer ${token}` };
|
|
}
|
|
|
|
function json(token: string) {
|
|
return { ...bearer(token), "content-type": "application/json" };
|
|
}
|
|
|
|
interface GatewayOptions {
|
|
inlineSourceCredentialsAllowed?: boolean;
|
|
onlineDocumentConnector?: OnlineDocumentConnector;
|
|
onlineDriveConnector?: OnlineDriveConnector;
|
|
sourceCredentials?: SourceCredentialService;
|
|
sourceCredentialTester?: SourceCredentialTester;
|
|
sources?: SourceRepository;
|
|
websiteCrawlConnector?: WebsiteCrawlConnector;
|
|
}
|
|
|
|
function createApp(options: GatewayOptions = {}) {
|
|
return createKnowledgeGateway({
|
|
...createAllowingDurableDeletionSafetyOptions(),
|
|
adapter: createNodePlatformAdapter({ env: {} }),
|
|
auth: createStaticAuthVerifier({
|
|
subjectsByToken: {
|
|
[otherTenantToken]: {
|
|
scopes: ["knowledge-spaces:*"],
|
|
subjectId: "u2",
|
|
tenantId: "tenant-2",
|
|
},
|
|
[readToken]: { scopes: ["knowledge-spaces:read"], subjectId: "u1", tenantId: "tenant-1" },
|
|
[writeToken]: { scopes: ["knowledge-spaces:*"], subjectId: "u1", tenantId: "tenant-1" },
|
|
},
|
|
}),
|
|
durableDeletions: createAcceptingDurableDeletionService({
|
|
requestSourceDeletion: async () => {
|
|
throw new DurableDeletionServiceError(
|
|
"DURABLE_DELETION_NOT_FOUND",
|
|
"Deletion target not found",
|
|
);
|
|
},
|
|
}),
|
|
...(options.inlineSourceCredentialsAllowed === undefined
|
|
? {}
|
|
: { inlineSourceCredentialsAllowed: options.inlineSourceCredentialsAllowed }),
|
|
...(options.onlineDocumentConnector
|
|
? { onlineDocumentConnector: options.onlineDocumentConnector }
|
|
: {}),
|
|
...(options.onlineDriveConnector ? { onlineDriveConnector: options.onlineDriveConnector } : {}),
|
|
...(options.sourceCredentials ? { sourceCredentials: options.sourceCredentials } : {}),
|
|
...(options.sourceCredentialTester
|
|
? { sourceCredentialTester: options.sourceCredentialTester }
|
|
: {}),
|
|
...(options.sources ? { sources: options.sources } : {}),
|
|
...(options.websiteCrawlConnector
|
|
? { websiteCrawlConnector: options.websiteCrawlConnector }
|
|
: {}),
|
|
});
|
|
}
|
|
|
|
describe("Dify-managed source credentials", () => {
|
|
it("rejects inline create, rotate, and revoke mutations", async () => {
|
|
const app = createApp({ inlineSourceCredentialsAllowed: false });
|
|
const spaceId = await createSpace(app);
|
|
const rejectedCreate = await app.request(`/knowledge-spaces/${spaceId}/sources`, {
|
|
body: JSON.stringify({
|
|
credentials: { token: "must-not-enter-kfs" },
|
|
name: "Dify source",
|
|
type: "web",
|
|
uri: "https://example.com",
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
|
|
expect(rejectedCreate.status).toBe(400);
|
|
await expect(rejectedCreate.json()).resolves.toEqual({
|
|
error: "Datasource credentials are managed by Dify",
|
|
});
|
|
|
|
const sourceId = await createWebSource(app, spaceId);
|
|
const update = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}`, {
|
|
body: JSON.stringify({ metadata: { credentials: { token: "nope" } } }),
|
|
headers: json(writeToken),
|
|
method: "PATCH",
|
|
});
|
|
expect(update.status).toBe(400);
|
|
await expect(update.json()).resolves.toEqual({
|
|
error: "Datasource credentials are managed by Dify",
|
|
});
|
|
|
|
const rotate = await app.request(
|
|
`/knowledge-spaces/${spaceId}/sources/${sourceId}/credentials`,
|
|
{
|
|
body: JSON.stringify({ credentials: { token: "nope" }, expectedVersion: 1 }),
|
|
headers: json(writeToken),
|
|
method: "PUT",
|
|
},
|
|
);
|
|
expect(rotate.status).toBe(409);
|
|
await expect(rotate.json()).resolves.toEqual({
|
|
error: "Datasource credentials are managed by Dify",
|
|
});
|
|
|
|
const revoke = await app.request(
|
|
`/knowledge-spaces/${spaceId}/sources/${sourceId}/credentials?expectedVersion=1`,
|
|
{ headers: bearer(writeToken), method: "DELETE" },
|
|
);
|
|
expect(revoke.status).toBe(409);
|
|
await expect(revoke.json()).resolves.toEqual({
|
|
error: "Datasource credentials are managed by Dify",
|
|
});
|
|
});
|
|
});
|
|
|
|
function createMemorySourceSecretStore(): SourceSecretStore {
|
|
const records = new Map<
|
|
string,
|
|
{ credentials: Record<string, unknown>; fingerprint: string; ref: string }
|
|
>();
|
|
let sequence = 0;
|
|
const key = (input: {
|
|
knowledgeSpaceId: string;
|
|
ref: string;
|
|
sourceId: string;
|
|
tenantId: string;
|
|
}) => `${input.tenantId}/${input.knowledgeSpaceId}/${input.sourceId}/${input.ref}`;
|
|
const fingerprint = createSourceCredentialFingerprinter(new Uint8Array(32).fill(43));
|
|
|
|
return {
|
|
delete: async (input) => {
|
|
records.delete(key(input));
|
|
},
|
|
fingerprint,
|
|
get: async (input) => {
|
|
const record = records.get(key(input));
|
|
|
|
return record ? { ...record, credentials: { ...record.credentials } } : null;
|
|
},
|
|
put: async (input) => {
|
|
sequence += 1;
|
|
const ref = input.ref ?? `source-secret:test:${sequence.toString().padStart(16, "0")}`;
|
|
const record = {
|
|
credentials: { ...input.credentials },
|
|
fingerprint: fingerprint({
|
|
credentials: input.credentials,
|
|
knowledgeSpaceId: input.knowledgeSpaceId,
|
|
sourceId: input.sourceId,
|
|
tenantId: input.tenantId,
|
|
}),
|
|
ref,
|
|
};
|
|
records.set(key({ ...input, ref }), record);
|
|
|
|
return { ...record, credentials: { ...record.credentials } };
|
|
},
|
|
};
|
|
}
|
|
|
|
async function createSpace(app: ReturnType<typeof createApp>): Promise<string> {
|
|
const response = await app.request("/knowledge-spaces", {
|
|
body: JSON.stringify({ name: "Space", slug: "space" }),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(response.status).toBe(201);
|
|
|
|
return (await response.json()).id;
|
|
}
|
|
|
|
async function createWebSource(
|
|
app: ReturnType<typeof createApp>,
|
|
spaceId: string,
|
|
name = "Docs crawl",
|
|
): Promise<string> {
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources`, {
|
|
body: JSON.stringify({
|
|
metadata: { provider: "firecrawl" },
|
|
name,
|
|
type: "web",
|
|
uri: "https://example.com",
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(response.status).toBe(201);
|
|
|
|
return (await response.json()).id;
|
|
}
|
|
|
|
async function createConnectorSource(
|
|
app: ReturnType<typeof createApp>,
|
|
spaceId: string,
|
|
): Promise<string> {
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources`, {
|
|
body: JSON.stringify({
|
|
metadata: {
|
|
datasource: "notion_datasource",
|
|
pluginId: "langgenius/notion_datasource",
|
|
provider: "notion_datasource",
|
|
},
|
|
name: "Notion",
|
|
type: "connector",
|
|
uri: "workspace-1",
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(response.status).toBe(201);
|
|
|
|
return (await response.json()).id;
|
|
}
|
|
|
|
async function getSource(
|
|
app: ReturnType<typeof createApp>,
|
|
spaceId: string,
|
|
sourceId: string,
|
|
): Promise<{ metadata: Record<string, unknown>; status: string }> {
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}`, {
|
|
headers: bearer(writeToken),
|
|
});
|
|
expect(response.status).toBe(200);
|
|
|
|
return response.json();
|
|
}
|
|
|
|
describe("source creation edge branches", () => {
|
|
it("accepts a valid metadata.syncPolicy and rejects an invalid one", async () => {
|
|
const app = createApp();
|
|
const spaceId = await createSpace(app);
|
|
|
|
const created = await app.request(`/knowledge-spaces/${spaceId}/sources`, {
|
|
body: JSON.stringify({
|
|
metadata: { syncPolicy: { everyHours: 6 } },
|
|
name: "Scheduled",
|
|
type: "web",
|
|
uri: "https://example.com",
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(created.status).toBe(201);
|
|
const source = await created.json();
|
|
expect(source.metadata.syncPolicy).toEqual({ everyHours: 6 });
|
|
// The owning tenant is stamped for the background sync scheduler.
|
|
expect(source.metadata.tenantId).toBe("tenant-1");
|
|
|
|
const invalid = await app.request(`/knowledge-spaces/${spaceId}/sources`, {
|
|
body: JSON.stringify({
|
|
metadata: { syncPolicy: { everyHours: 0 } },
|
|
name: "Broken schedule",
|
|
type: "web",
|
|
uri: "https://example.com",
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(invalid.status).toBe(400);
|
|
expect((await invalid.json()).error).toContain("Invalid source syncPolicy");
|
|
});
|
|
|
|
it("returns 429 when the source repository capacity is exceeded", async () => {
|
|
const app = createApp({ sources: createInMemorySourceRepository({ maxSources: 1 }) });
|
|
const spaceId = await createSpace(app);
|
|
await createWebSource(app, spaceId);
|
|
|
|
const overflow = await app.request(`/knowledge-spaces/${spaceId}/sources`, {
|
|
body: JSON.stringify({ name: "Too many", type: "web", uri: "https://example.com/2" }),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(overflow.status).toBe(429);
|
|
expect((await overflow.json()).error).toContain("maxSources=1 exceeded");
|
|
});
|
|
|
|
it("rethrows unexpected repository failures as 500", async () => {
|
|
const inner = createInMemorySourceRepository({ maxSources: 10 });
|
|
const sources: SourceRepository = {
|
|
...inner,
|
|
create: async () => {
|
|
throw new Error("create exploded");
|
|
},
|
|
};
|
|
const app = createApp({ sources });
|
|
const spaceId = await createSpace(app);
|
|
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources`, {
|
|
body: JSON.stringify({ name: "Boom", type: "web", uri: "https://example.com" }),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(response.status).toBe(500);
|
|
});
|
|
});
|
|
|
|
describe("source list pagination", () => {
|
|
it("pages sources with cursor and nextCursor", async () => {
|
|
const app = createApp();
|
|
const spaceId = await createSpace(app);
|
|
await createWebSource(app, spaceId, "One");
|
|
await createWebSource(app, spaceId, "Two");
|
|
|
|
const first = await app.request(`/knowledge-spaces/${spaceId}/sources?limit=1`, {
|
|
headers: bearer(readToken),
|
|
});
|
|
expect(first.status).toBe(200);
|
|
const firstPage = await first.json();
|
|
expect(firstPage.items).toHaveLength(1);
|
|
expect(firstPage.nextCursor).toBeDefined();
|
|
|
|
const second = await app.request(
|
|
`/knowledge-spaces/${spaceId}/sources?limit=1&cursor=${firstPage.nextCursor}`,
|
|
{ headers: bearer(readToken) },
|
|
);
|
|
expect(second.status).toBe(200);
|
|
const secondPage = await second.json();
|
|
expect(secondPage.items).toHaveLength(1);
|
|
expect(secondPage.items[0].id).not.toBe(firstPage.items[0].id);
|
|
expect(secondPage.nextCursor).toBeUndefined();
|
|
});
|
|
});
|
|
|
|
describe("source response credential redaction", () => {
|
|
it("removes secret-bearing metadata recursively from create, list, get, and update responses", async () => {
|
|
const sources = createInMemorySourceRepository({ maxSources: 10 });
|
|
const secretStore = createMemorySourceSecretStore();
|
|
const retiredSecrets = createInMemorySourceRetiredSecretCleanupRepository({
|
|
maxClaimBatchSize: 10,
|
|
maxJobs: 100,
|
|
sources,
|
|
});
|
|
const sourceCredentials = createSourceCredentialService({
|
|
retiredSecrets,
|
|
secretStore,
|
|
sources,
|
|
});
|
|
const app = createApp({ sourceCredentials, sources });
|
|
const spaceId = await createSpace(app);
|
|
const created = await app.request(`/knowledge-spaces/${spaceId}/sources`, {
|
|
body: JSON.stringify({
|
|
metadata: {
|
|
auth: {
|
|
accessToken: "access-secret",
|
|
mode: "oauth",
|
|
nested: [{ integration_password: "password-secret", label: "primary" }],
|
|
},
|
|
credentials: { apiKey: "credential-secret" },
|
|
endpoint: "https://crawler.example.com",
|
|
provider: "firecrawl",
|
|
secretRotationAt: "2026-07-13T00:00:00.000Z",
|
|
tokenCount: 42,
|
|
},
|
|
name: "Secret source",
|
|
type: "web",
|
|
uri: "https://example.com",
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
|
|
expect(created.status).toBe(201);
|
|
const createdBody = await created.json();
|
|
expect(createdBody.credentialConfigured).toBe(true);
|
|
expect(createdBody.metadata).toEqual({
|
|
auth: { mode: "oauth", nested: [{ label: "primary" }] },
|
|
endpoint: "https://crawler.example.com",
|
|
provider: "firecrawl",
|
|
secretRotationAt: "2026-07-13T00:00:00.000Z",
|
|
tenantId: "tenant-1",
|
|
tokenCount: 42,
|
|
});
|
|
|
|
const sourceId = createdBody.id as string;
|
|
const storedAfterCreate = await sources.get({ id: sourceId, knowledgeSpaceId: spaceId });
|
|
expect(storedAfterCreate?.credentialRef).toBeDefined();
|
|
expect(storedAfterCreate?.metadata).not.toHaveProperty("credentials");
|
|
expect(storedAfterCreate?.metadata.auth).toEqual({
|
|
mode: "oauth",
|
|
nested: [{ label: "primary" }],
|
|
});
|
|
const credentialRef = storedAfterCreate?.credentialRef;
|
|
expect(credentialRef).toBeDefined();
|
|
expect(
|
|
await secretStore.get({
|
|
knowledgeSpaceId: spaceId,
|
|
ref: credentialRef as string,
|
|
sourceId,
|
|
tenantId: "tenant-1",
|
|
}),
|
|
).toMatchObject({ credentials: { apiKey: "credential-secret" } });
|
|
|
|
const listed = await app.request(`/knowledge-spaces/${spaceId}/sources`, {
|
|
headers: bearer(readToken),
|
|
});
|
|
expect(listed.status).toBe(200);
|
|
expect((await listed.json()).items[0].metadata).toEqual(createdBody.metadata);
|
|
|
|
const fetched = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}`, {
|
|
headers: bearer(readToken),
|
|
});
|
|
expect(fetched.status).toBe(200);
|
|
expect((await fetched.json()).metadata).toEqual(createdBody.metadata);
|
|
|
|
const updated = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}`, {
|
|
body: JSON.stringify({
|
|
metadata: {
|
|
"api-key": "updated-api-secret",
|
|
connection: {
|
|
authorizationHeader: "Bearer updated-token",
|
|
region: "us-east-1",
|
|
},
|
|
githubToken: "updated-token",
|
|
note: "visible",
|
|
password: "updated-password",
|
|
},
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "PATCH",
|
|
});
|
|
expect(updated.status).toBe(200);
|
|
expect((await updated.json()).metadata).toEqual({
|
|
auth: { mode: "oauth", nested: [{ label: "primary" }] },
|
|
connection: { region: "us-east-1" },
|
|
endpoint: "https://crawler.example.com",
|
|
note: "visible",
|
|
provider: "firecrawl",
|
|
secretRotationAt: "2026-07-13T00:00:00.000Z",
|
|
tenantId: "tenant-1",
|
|
tokenCount: 42,
|
|
});
|
|
|
|
const storedAfterUpdate = await sources.get({ id: sourceId, knowledgeSpaceId: spaceId });
|
|
expect(storedAfterUpdate?.credentialRef).toBe(credentialRef);
|
|
expect(storedAfterUpdate?.metadata).not.toHaveProperty("credentials");
|
|
expect(storedAfterUpdate?.metadata.auth).toEqual({
|
|
mode: "oauth",
|
|
nested: [{ label: "primary" }],
|
|
});
|
|
expect(storedAfterUpdate?.metadata.connection).toEqual({ region: "us-east-1" });
|
|
expect(storedAfterUpdate?.metadata).not.toHaveProperty("api-key");
|
|
expect(storedAfterUpdate?.metadata).not.toHaveProperty("githubToken");
|
|
expect(storedAfterUpdate?.metadata).not.toHaveProperty("password");
|
|
});
|
|
|
|
it("preserves stored credentials when redacted GET metadata is patched back", async () => {
|
|
const sources = createInMemorySourceRepository({ maxSources: 10 });
|
|
const secretStore = createMemorySourceSecretStore();
|
|
const retiredSecrets = createInMemorySourceRetiredSecretCleanupRepository({
|
|
maxClaimBatchSize: 10,
|
|
maxJobs: 100,
|
|
sources,
|
|
});
|
|
const sourceCredentials = createSourceCredentialService({
|
|
retiredSecrets,
|
|
secretStore,
|
|
sources,
|
|
});
|
|
let testedCredentials: Record<string, unknown> | undefined;
|
|
const app = createApp({
|
|
sourceCredentials,
|
|
sourceCredentialTester: {
|
|
test: async ({ source }) => {
|
|
testedCredentials = readSourceCredentialConfig(source).credentials;
|
|
return { valid: testedCredentials.apiKey === "stored-api-key" };
|
|
},
|
|
},
|
|
sources,
|
|
});
|
|
const spaceId = await createSpace(app);
|
|
const created = await app.request(`/knowledge-spaces/${spaceId}/sources`, {
|
|
body: JSON.stringify({
|
|
metadata: {
|
|
credentials: { apiKey: "stored-api-key" },
|
|
pluginId: "langgenius/firecrawl",
|
|
profiles: [{ name: "primary", token: "nested-token" }],
|
|
provider: "firecrawl",
|
|
},
|
|
name: "Credential round trip",
|
|
type: "connector",
|
|
uri: "workspace-1",
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(created.status).toBe(201);
|
|
const sourceId = (await created.json()).id as string;
|
|
|
|
const fetched = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}`, {
|
|
headers: bearer(readToken),
|
|
});
|
|
expect(fetched.status).toBe(200);
|
|
const publicSource = await fetched.json();
|
|
expect(publicSource.metadata).toEqual({
|
|
pluginId: "langgenius/firecrawl",
|
|
profiles: [{ name: "primary" }],
|
|
provider: "firecrawl",
|
|
tenantId: "tenant-1",
|
|
});
|
|
|
|
const patched = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}`, {
|
|
body: JSON.stringify({ metadata: { ...publicSource.metadata, label: "updated" } }),
|
|
headers: json(writeToken),
|
|
method: "PATCH",
|
|
});
|
|
expect(patched.status).toBe(200);
|
|
expect((await patched.json()).metadata).toEqual({
|
|
label: "updated",
|
|
pluginId: "langgenius/firecrawl",
|
|
profiles: [{ name: "primary" }],
|
|
provider: "firecrawl",
|
|
tenantId: "tenant-1",
|
|
});
|
|
|
|
const stored = await sources.get({ id: sourceId, knowledgeSpaceId: spaceId });
|
|
expect(stored?.credentialRef).toBeDefined();
|
|
expect(stored?.metadata).not.toHaveProperty("credentials");
|
|
expect(stored?.metadata.profiles).toEqual([{ name: "primary" }]);
|
|
expect(
|
|
await secretStore.get({
|
|
knowledgeSpaceId: spaceId,
|
|
ref: stored?.credentialRef as string,
|
|
sourceId,
|
|
tenantId: "tenant-1",
|
|
}),
|
|
).toMatchObject({ credentials: { apiKey: "stored-api-key" } });
|
|
|
|
const tested = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}/test`, {
|
|
headers: bearer(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(tested.status).toBe(200);
|
|
expect(await tested.json()).toEqual({ valid: true });
|
|
expect(testedCredentials).toEqual({ apiKey: "stored-api-key" });
|
|
});
|
|
});
|
|
|
|
describe("source credential lifecycle handlers", () => {
|
|
it("rotates and revokes credentials with version fences and redacted responses", async () => {
|
|
const sources = createInMemorySourceRepository({ maxSources: 10 });
|
|
const secretStore = createMemorySourceSecretStore();
|
|
const retiredSecrets = createInMemorySourceRetiredSecretCleanupRepository({
|
|
maxClaimBatchSize: 10,
|
|
maxJobs: 100,
|
|
sources,
|
|
});
|
|
const sourceCredentials = createSourceCredentialService({
|
|
retiredSecrets,
|
|
secretStore,
|
|
sources,
|
|
});
|
|
const app = createApp({ sourceCredentials, sources });
|
|
const spaceId = await createSpace(app);
|
|
const created = await app.request(`/knowledge-spaces/${spaceId}/sources`, {
|
|
body: JSON.stringify({
|
|
credentials: { apiKey: "initial-secret" },
|
|
name: "Credential source",
|
|
type: "web",
|
|
uri: "https://example.com",
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(created.status).toBe(201);
|
|
const initial = await created.json();
|
|
const base = `/knowledge-spaces/${spaceId}/sources/${initial.id}/credentials`;
|
|
|
|
const rotated = await app.request(base, {
|
|
body: JSON.stringify({
|
|
credentials: { apiKey: "rotated-secret" },
|
|
expectedVersion: initial.version,
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "PUT",
|
|
});
|
|
expect(rotated.status).toBe(200);
|
|
const rotatedBody = await rotated.json();
|
|
expect(rotatedBody).toMatchObject({ credentialConfigured: true, version: initial.version + 1 });
|
|
expect(rotatedBody.metadata).not.toHaveProperty("credentials");
|
|
|
|
const staleRotate = await app.request(base, {
|
|
body: JSON.stringify({
|
|
credentials: { apiKey: "stale-secret" },
|
|
expectedVersion: initial.version,
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "PUT",
|
|
});
|
|
expect(staleRotate.status).toBe(409);
|
|
expect(await staleRotate.json()).toMatchObject({ code: "SOURCE_VERSION_CONFLICT" });
|
|
|
|
const staleRevoke = await app.request(`${base}?expectedVersion=${initial.version}`, {
|
|
headers: bearer(writeToken),
|
|
method: "DELETE",
|
|
});
|
|
expect(staleRevoke.status).toBe(409);
|
|
|
|
const revoked = await app.request(`${base}?expectedVersion=${rotatedBody.version}`, {
|
|
headers: bearer(writeToken),
|
|
method: "DELETE",
|
|
});
|
|
expect(revoked.status).toBe(200);
|
|
expect(await revoked.json()).not.toHaveProperty("credentialConfigured");
|
|
});
|
|
|
|
it("maps unavailable, null, and unexpected credential mutations without leaking secrets", async () => {
|
|
const sources = createInMemorySourceRepository({ maxSources: 10 });
|
|
const secretStore = createMemorySourceSecretStore();
|
|
const retiredSecrets = createInMemorySourceRetiredSecretCleanupRepository({
|
|
maxClaimBatchSize: 10,
|
|
maxJobs: 100,
|
|
sources,
|
|
});
|
|
const delegate = createSourceCredentialService({ retiredSecrets, secretStore, sources });
|
|
type Mode =
|
|
| "revoke-mutation"
|
|
| "revoke-null"
|
|
| "revoke-unexpected"
|
|
| "rotate-mutation"
|
|
| "rotate-null"
|
|
| "rotate-unexpected";
|
|
let mode: Mode = "rotate-null";
|
|
const sourceCredentials: SourceCredentialService = {
|
|
create: (input) => delegate.create(input),
|
|
resolve: (input) => delegate.resolve(input),
|
|
revoke: async (input) => {
|
|
if (mode === "revoke-null") return null;
|
|
if (mode === "revoke-mutation") {
|
|
throw new SourceCredentialMutationError("credential revoke unavailable");
|
|
}
|
|
if (mode === "revoke-unexpected") throw new Error("unexpected revoke failure");
|
|
return delegate.revoke(input);
|
|
},
|
|
rotate: async (input) => {
|
|
if (mode === "rotate-null") return null;
|
|
if (mode === "rotate-mutation") {
|
|
throw new SourceCredentialMutationError("credential rotate unavailable");
|
|
}
|
|
if (mode === "rotate-unexpected") throw new Error("unexpected rotate failure");
|
|
return delegate.rotate(input);
|
|
},
|
|
};
|
|
const app = createApp({ sourceCredentials, sources });
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createWebSource(app, spaceId);
|
|
const base = `/knowledge-spaces/${spaceId}/sources/${sourceId}/credentials`;
|
|
const rotate = () =>
|
|
app.request(base, {
|
|
body: JSON.stringify({ credentials: { token: "never-returned" }, expectedVersion: 1 }),
|
|
headers: json(writeToken),
|
|
method: "PUT",
|
|
});
|
|
const revoke = () =>
|
|
app.request(`${base}?expectedVersion=1`, {
|
|
headers: bearer(writeToken),
|
|
method: "DELETE",
|
|
});
|
|
|
|
mode = "rotate-null";
|
|
expect((await rotate()).status).toBe(404);
|
|
mode = "rotate-mutation";
|
|
const rotateUnavailable = await rotate();
|
|
expect(rotateUnavailable.status).toBe(503);
|
|
expect(await rotateUnavailable.json()).toEqual({ error: "credential rotate unavailable" });
|
|
mode = "rotate-unexpected";
|
|
expect((await rotate()).status).toBe(500);
|
|
|
|
mode = "revoke-null";
|
|
expect((await revoke()).status).toBe(404);
|
|
mode = "revoke-mutation";
|
|
const revokeUnavailable = await revoke();
|
|
expect(revokeUnavailable.status).toBe(503);
|
|
expect(await revokeUnavailable.json()).toEqual({ error: "credential revoke unavailable" });
|
|
mode = "revoke-unexpected";
|
|
expect((await revoke()).status).toBe(500);
|
|
|
|
const appWithoutSecrets = createApp();
|
|
const plainSpaceId = await createSpace(appWithoutSecrets);
|
|
const plainSourceId = await createWebSource(appWithoutSecrets, plainSpaceId);
|
|
const plainBase = `/knowledge-spaces/${plainSpaceId}/sources/${plainSourceId}/credentials`;
|
|
expect(
|
|
(
|
|
await appWithoutSecrets.request(plainBase, {
|
|
body: JSON.stringify({ credentials: { token: "unused" }, expectedVersion: 1 }),
|
|
headers: json(writeToken),
|
|
method: "PUT",
|
|
})
|
|
).status,
|
|
).toBe(503);
|
|
expect(
|
|
(
|
|
await appWithoutSecrets.request(`${plainBase}?expectedVersion=1`, {
|
|
headers: bearer(writeToken),
|
|
method: "DELETE",
|
|
})
|
|
).status,
|
|
).toBe(503);
|
|
});
|
|
});
|
|
|
|
describe("tenant and existence guards on every source endpoint", () => {
|
|
it("returns 404 when the space belongs to another tenant", async () => {
|
|
const app = createApp();
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createWebSource(app, spaceId);
|
|
const base = `/knowledge-spaces/${spaceId}/sources/${sourceId}`;
|
|
|
|
expect((await app.request(base, { headers: bearer(otherTenantToken) })).status).toBe(404);
|
|
expect(
|
|
(
|
|
await app.request(base, {
|
|
body: JSON.stringify({ name: "Nope" }),
|
|
headers: json(otherTenantToken),
|
|
method: "PATCH",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
expect(
|
|
(
|
|
await app.request(`${base}/credentials`, {
|
|
body: JSON.stringify({ credentials: { token: "cross-tenant" }, expectedVersion: 1 }),
|
|
headers: json(otherTenantToken),
|
|
method: "PUT",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
expect(
|
|
(
|
|
await app.request(`${base}/credentials?expectedVersion=1`, {
|
|
headers: bearer(otherTenantToken),
|
|
method: "DELETE",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
expect(
|
|
(
|
|
await app.request(base, {
|
|
body: JSON.stringify({ expectedRevision: 1 }),
|
|
headers: {
|
|
...json(otherTenantToken),
|
|
"idempotency-key": "cross-tenant-source-delete",
|
|
},
|
|
method: "DELETE",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
expect(
|
|
(await app.request(`${base}/crawl`, { headers: bearer(otherTenantToken), method: "POST" }))
|
|
.status,
|
|
).toBe(404);
|
|
expect((await app.request(`${base}/pages`, { headers: bearer(otherTenantToken) })).status).toBe(
|
|
404,
|
|
);
|
|
expect(
|
|
(
|
|
await app.request(`${base}/import`, {
|
|
body: JSON.stringify({ pages: [{ pageId: "p1", type: "page", workspaceId: "w1" }] }),
|
|
headers: json(otherTenantToken),
|
|
method: "POST",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
expect(
|
|
(await app.request(`${base}/test`, { headers: bearer(otherTenantToken), method: "POST" }))
|
|
.status,
|
|
).toBe(404);
|
|
expect((await app.request(`${base}/files`, { headers: bearer(otherTenantToken) })).status).toBe(
|
|
404,
|
|
);
|
|
expect(
|
|
(
|
|
await app.request(`${base}/import-files`, {
|
|
body: JSON.stringify({ files: [{ id: "f1", name: "a.txt" }] }),
|
|
headers: json(otherTenantToken),
|
|
method: "POST",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
});
|
|
|
|
it("returns 404 when the source id is unknown", async () => {
|
|
const app = createApp();
|
|
const spaceId = await createSpace(app);
|
|
const base = `/knowledge-spaces/${spaceId}/sources/${missingSourceId}`;
|
|
|
|
expect(
|
|
(
|
|
await app.request(base, {
|
|
body: JSON.stringify({ name: "Nope" }),
|
|
headers: json(writeToken),
|
|
method: "PATCH",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
expect(
|
|
(
|
|
await app.request(base, {
|
|
body: JSON.stringify({ expectedRevision: 1 }),
|
|
headers: { ...json(writeToken), "idempotency-key": "missing-source-delete" },
|
|
method: "DELETE",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
expect(
|
|
(
|
|
await app.request(`${base}/credentials`, {
|
|
body: JSON.stringify({ credentials: { token: "missing" }, expectedVersion: 1 }),
|
|
headers: json(writeToken),
|
|
method: "PUT",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
expect(
|
|
(
|
|
await app.request(`${base}/credentials?expectedVersion=1`, {
|
|
headers: bearer(writeToken),
|
|
method: "DELETE",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
expect(
|
|
(await app.request(`${base}/crawl`, { headers: bearer(writeToken), method: "POST" })).status,
|
|
).toBe(404);
|
|
expect((await app.request(`${base}/pages`, { headers: bearer(readToken) })).status).toBe(404);
|
|
expect(
|
|
(
|
|
await app.request(`${base}/import`, {
|
|
body: JSON.stringify({ pages: [{ pageId: "p1", type: "page", workspaceId: "w1" }] }),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
expect(
|
|
(await app.request(`${base}/test`, { headers: bearer(writeToken), method: "POST" })).status,
|
|
).toBe(404);
|
|
expect((await app.request(`${base}/files`, { headers: bearer(readToken) })).status).toBe(404);
|
|
expect(
|
|
(
|
|
await app.request(`${base}/import-files`, {
|
|
body: JSON.stringify({ files: [{ id: "f1", name: "a.txt" }] }),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
})
|
|
).status,
|
|
).toBe(404);
|
|
});
|
|
});
|
|
|
|
describe("source update edge branches", () => {
|
|
it("validates metadata.syncPolicy on update and re-stamps the tenant", async () => {
|
|
const app = createApp();
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createWebSource(app, spaceId);
|
|
|
|
const invalid = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}`, {
|
|
body: JSON.stringify({ metadata: { syncPolicy: { dailyAt: [] } } }),
|
|
headers: json(writeToken),
|
|
method: "PATCH",
|
|
});
|
|
expect(invalid.status).toBe(400);
|
|
expect((await invalid.json()).error).toContain("Invalid source syncPolicy");
|
|
|
|
const valid = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}`, {
|
|
body: JSON.stringify({ metadata: { note: "kept", syncPolicy: { dailyAt: ["03:00"] } } }),
|
|
headers: json(writeToken),
|
|
method: "PATCH",
|
|
});
|
|
expect(valid.status).toBe(200);
|
|
const updated = await valid.json();
|
|
expect(updated.metadata).toEqual({
|
|
note: "kept",
|
|
provider: "firecrawl",
|
|
syncPolicy: { dailyAt: ["03:00"] },
|
|
tenantId: "tenant-1",
|
|
});
|
|
|
|
// Metadata without a syncPolicy merges into the existing metadata and skips policy validation.
|
|
const plain = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}`, {
|
|
body: JSON.stringify({ metadata: { plain: true } }),
|
|
headers: json(writeToken),
|
|
method: "PATCH",
|
|
});
|
|
expect(plain.status).toBe(200);
|
|
expect((await plain.json()).metadata).toEqual({
|
|
note: "kept",
|
|
plain: true,
|
|
provider: "firecrawl",
|
|
syncPolicy: { dailyAt: ["03:00"] },
|
|
tenantId: "tenant-1",
|
|
});
|
|
});
|
|
|
|
it("rethrows unexpected update failures as 500", async () => {
|
|
const inner = createInMemorySourceRepository({ maxSources: 10 });
|
|
const sources: SourceRepository = {
|
|
...inner,
|
|
update: async () => {
|
|
throw new Error("update exploded");
|
|
},
|
|
};
|
|
const app = createApp({ sources });
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createWebSource(app, spaceId);
|
|
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}`, {
|
|
body: JSON.stringify({ name: "Renamed" }),
|
|
headers: json(writeToken),
|
|
method: "PATCH",
|
|
});
|
|
expect(response.status).toBe(500);
|
|
});
|
|
});
|
|
|
|
describe("website crawl failure mapping", () => {
|
|
it("maps a non-Error crawl failure to the fallback messages", async () => {
|
|
const nonErrorFailure: unknown = "daemon exploded without an Error";
|
|
const app = createApp({
|
|
websiteCrawlConnector: {
|
|
crawl: async () => {
|
|
throw nonErrorFailure;
|
|
},
|
|
},
|
|
});
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createWebSource(app, spaceId);
|
|
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}/crawl`, {
|
|
headers: bearer(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(response.status).toBe(502);
|
|
expect(await response.json()).toEqual({
|
|
code: SOURCE_OPERATION_FAILURES.websiteCrawl.code,
|
|
error: SOURCE_OPERATION_FAILURES.websiteCrawl.message,
|
|
});
|
|
|
|
const source = await getSource(app, spaceId, sourceId);
|
|
expect(source.status).toBe("error");
|
|
expect(source.metadata.sync).toEqual({
|
|
error: SOURCE_OPERATION_FAILURES.websiteCrawl.message,
|
|
errorCode: SOURCE_OPERATION_FAILURES.websiteCrawl.code,
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("online document listing shape branches", () => {
|
|
it("passes through optional page fields and omits absent workspace fields", async () => {
|
|
const connector: OnlineDocumentConnector = {
|
|
getPageContent: async ({ page }) => ({ content: `# ${page.pageId}`, pageId: page.pageId }),
|
|
listPages: async () => ({
|
|
workspaces: [
|
|
{
|
|
pages: [
|
|
{
|
|
lastEditedTime: "2026-07-01T00:00:00.000Z",
|
|
pageId: "p1",
|
|
pageName: "One",
|
|
parentId: "root",
|
|
type: "page",
|
|
},
|
|
],
|
|
},
|
|
],
|
|
}),
|
|
};
|
|
const app = createApp({ onlineDocumentConnector: connector });
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createConnectorSource(app, spaceId);
|
|
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}/pages`, {
|
|
headers: bearer(readToken),
|
|
});
|
|
expect(response.status).toBe(200);
|
|
const body = await response.json();
|
|
expect(body.workspaces).toEqual([
|
|
{
|
|
pages: [
|
|
{
|
|
lastEditedTime: "2026-07-01T00:00:00.000Z",
|
|
pageId: "p1",
|
|
pageName: "One",
|
|
parentId: "root",
|
|
type: "page",
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
});
|
|
|
|
it("maps listing failures to 502", async () => {
|
|
const connector: OnlineDocumentConnector = {
|
|
getPageContent: async ({ page }) => ({ content: "x", pageId: page.pageId }),
|
|
listPages: async () => {
|
|
throw new Error("notion down: Authorization Bearer credential-secret");
|
|
},
|
|
};
|
|
const app = createApp({ onlineDocumentConnector: connector });
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createConnectorSource(app, spaceId);
|
|
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}/pages`, {
|
|
headers: bearer(readToken),
|
|
});
|
|
expect(response.status).toBe(502);
|
|
const errorBody = await response.json();
|
|
expect(errorBody).toEqual({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDocumentRequest.code,
|
|
error: SOURCE_OPERATION_FAILURES.onlineDocumentRequest.message,
|
|
});
|
|
expect(JSON.stringify(errorBody)).not.toContain("credential-secret");
|
|
|
|
// A non-Error failure falls back to the generic message.
|
|
const nonErrorFailure: unknown = "rejected without an Error";
|
|
const nonErrorApp = createApp({
|
|
onlineDocumentConnector: {
|
|
getPageContent: async ({ page }) => ({ content: "x", pageId: page.pageId }),
|
|
listPages: async () => {
|
|
throw nonErrorFailure;
|
|
},
|
|
},
|
|
});
|
|
const spaceB = await createSpace(nonErrorApp);
|
|
const sourceB = await createConnectorSource(nonErrorApp, spaceB);
|
|
const fallback = await nonErrorApp.request(
|
|
`/knowledge-spaces/${spaceB}/sources/${sourceB}/pages`,
|
|
{ headers: bearer(readToken) },
|
|
);
|
|
expect(fallback.status).toBe(502);
|
|
expect(await fallback.json()).toEqual({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDocumentRequest.code,
|
|
error: SOURCE_OPERATION_FAILURES.onlineDocumentRequest.message,
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("online document import edge branches", () => {
|
|
it("returns 400 for a non-connector source and 501 without a connector", async () => {
|
|
const noConnectorApp = createApp();
|
|
const spaceA = await createSpace(noConnectorApp);
|
|
const connectorSourceId = await createConnectorSource(noConnectorApp, spaceA);
|
|
const notConfigured = await noConnectorApp.request(
|
|
`/knowledge-spaces/${spaceA}/sources/${connectorSourceId}/import`,
|
|
{
|
|
body: JSON.stringify({ pages: [{ pageId: "p1", type: "page", workspaceId: "w1" }] }),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
},
|
|
);
|
|
expect(notConfigured.status).toBe(501);
|
|
|
|
const connector: OnlineDocumentConnector = {
|
|
getPageContent: async ({ page }) => ({ content: "x", pageId: page.pageId }),
|
|
listPages: async () => ({ workspaces: [] }),
|
|
};
|
|
const app = createApp({ onlineDocumentConnector: connector });
|
|
const spaceB = await createSpace(app);
|
|
const webSourceId = await createWebSource(app, spaceB);
|
|
const wrongType = await app.request(
|
|
`/knowledge-spaces/${spaceB}/sources/${webSourceId}/import`,
|
|
{
|
|
body: JSON.stringify({ pages: [{ pageId: "p1", type: "page", workspaceId: "w1" }] }),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
},
|
|
);
|
|
expect(wrongType.status).toBe(400);
|
|
});
|
|
|
|
it("isolates per-page fetch failures into the failed list", async () => {
|
|
const nonErrorFailure: unknown = "page fetch rejected without an Error";
|
|
const connector: OnlineDocumentConnector = {
|
|
getPageContent: async ({ page }) => {
|
|
if (page.pageId === "p2") {
|
|
throw new Error("page fetch denied");
|
|
}
|
|
|
|
if (page.pageId === "p3") {
|
|
throw nonErrorFailure;
|
|
}
|
|
|
|
return { content: `# ${page.pageId}`, pageId: page.pageId };
|
|
},
|
|
listPages: async () => ({ workspaces: [] }),
|
|
};
|
|
const app = createApp({ onlineDocumentConnector: connector });
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createConnectorSource(app, spaceId);
|
|
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}/import`, {
|
|
body: JSON.stringify({
|
|
pages: [
|
|
{ name: "One", pageId: "p1", type: "page", workspaceId: "w1" },
|
|
{ name: "Two", pageId: "p2", type: "page", workspaceId: "w1" },
|
|
{ name: "Three", pageId: "p3", type: "page", workspaceId: "w1" },
|
|
],
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(response.status).toBe(200);
|
|
const body = await response.json();
|
|
expect(body.documents).toHaveLength(1);
|
|
expect(body.failed).toEqual([
|
|
{
|
|
code: SOURCE_OPERATION_FAILURES.onlineDocumentPageFetch.code,
|
|
error: "A page from the online-document source could not be loaded.",
|
|
failure: expect.objectContaining({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDocumentPageFetch.code,
|
|
stage: "source-document-import",
|
|
}),
|
|
filename: "Two-p2.md",
|
|
},
|
|
{
|
|
code: SOURCE_OPERATION_FAILURES.onlineDocumentPageFetch.code,
|
|
error: "A page from the online-document source could not be loaded.",
|
|
failure: expect.objectContaining({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDocumentPageFetch.code,
|
|
stage: "source-document-import",
|
|
}),
|
|
filename: "Three-p3.md",
|
|
},
|
|
]);
|
|
expect(body.skipped).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe("source credential test result mapping", () => {
|
|
it("maps a tester error to a stable response without leaking it", async () => {
|
|
const app = createApp({
|
|
sourceCredentialTester: {
|
|
test: async () => ({ error: "expired token credential-secret", valid: false }),
|
|
},
|
|
});
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createWebSource(app, spaceId);
|
|
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}/test`, {
|
|
headers: bearer(writeToken),
|
|
method: "POST",
|
|
});
|
|
expect(response.status).toBe(200);
|
|
const body = await response.json();
|
|
expect(body).toEqual({
|
|
code: SOURCE_OPERATION_FAILURES.credentialTest.code,
|
|
error: "The source credential could not be validated.",
|
|
failure: expect.objectContaining({
|
|
code: SOURCE_OPERATION_FAILURES.credentialTest.code,
|
|
stage: "credential-test",
|
|
}),
|
|
valid: false,
|
|
});
|
|
expect(JSON.stringify(body)).not.toContain("credential-secret");
|
|
});
|
|
|
|
it("maps a thrown tester error to a stable response without leaking it", async () => {
|
|
const app = createApp({
|
|
sourceCredentialTester: {
|
|
test: async () => {
|
|
throw new Error("tester unavailable credential-secret");
|
|
},
|
|
},
|
|
});
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createWebSource(app, spaceId);
|
|
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}/test`, {
|
|
headers: bearer(writeToken),
|
|
method: "POST",
|
|
});
|
|
|
|
expect(response.status).toBe(502);
|
|
const body = await response.json();
|
|
expect(body).toEqual({
|
|
code: SOURCE_OPERATION_FAILURES.credentialTest.code,
|
|
error: "The source credential could not be validated.",
|
|
failure: expect.objectContaining({
|
|
code: SOURCE_OPERATION_FAILURES.credentialTest.code,
|
|
stage: "credential-test",
|
|
}),
|
|
});
|
|
expect(JSON.stringify(body)).not.toContain("credential-secret");
|
|
});
|
|
});
|
|
|
|
describe("online drive browse edge branches", () => {
|
|
it("forwards maxKeys/prefix without a bucket and maps truncated anonymous buckets", async () => {
|
|
const browseCalls: {
|
|
bucket: string | undefined;
|
|
maxKeys: number | undefined;
|
|
prefix: string | undefined;
|
|
}[] = [];
|
|
const connector: OnlineDriveConnector = {
|
|
browse: async ({ bucket, maxKeys, prefix }) => {
|
|
browseCalls.push({ bucket, maxKeys, prefix });
|
|
|
|
return {
|
|
buckets: [{ files: [{ id: "f1", name: "a.txt", type: "file" }], isTruncated: true }],
|
|
};
|
|
},
|
|
download: async () => ({ body: new TextEncoder().encode("x") }),
|
|
};
|
|
const app = createApp({ onlineDriveConnector: connector });
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createConnectorSource(app, spaceId);
|
|
|
|
const response = await app.request(
|
|
`/knowledge-spaces/${spaceId}/sources/${sourceId}/files?maxKeys=5&prefix=docs`,
|
|
{ headers: bearer(readToken) },
|
|
);
|
|
expect(response.status).toBe(200);
|
|
expect(await response.json()).toEqual({
|
|
buckets: [
|
|
{
|
|
files: [{ id: "f1", name: "a.txt", type: "file" }],
|
|
isTruncated: true,
|
|
},
|
|
],
|
|
});
|
|
expect(browseCalls).toEqual([{ bucket: undefined, maxKeys: 5, prefix: "docs" }]);
|
|
});
|
|
|
|
it("maps browse failures to 502", async () => {
|
|
const connector: OnlineDriveConnector = {
|
|
browse: async () => {
|
|
throw new Error("drive down: signedUrl=https://secret.example/credential-secret");
|
|
},
|
|
download: async () => ({ body: new Uint8Array() }),
|
|
};
|
|
const app = createApp({ onlineDriveConnector: connector });
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createConnectorSource(app, spaceId);
|
|
|
|
const response = await app.request(`/knowledge-spaces/${spaceId}/sources/${sourceId}/files`, {
|
|
headers: bearer(readToken),
|
|
});
|
|
expect(response.status).toBe(502);
|
|
const errorBody = await response.json();
|
|
expect(errorBody).toEqual({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDriveRequest.code,
|
|
error: SOURCE_OPERATION_FAILURES.onlineDriveRequest.message,
|
|
});
|
|
expect(JSON.stringify(errorBody)).not.toContain("credential-secret");
|
|
|
|
// A non-Error failure falls back to the generic message.
|
|
const nonErrorFailure: unknown = "rejected without an Error";
|
|
const nonErrorApp = createApp({
|
|
onlineDriveConnector: {
|
|
browse: async () => {
|
|
throw nonErrorFailure;
|
|
},
|
|
download: async () => ({ body: new Uint8Array() }),
|
|
},
|
|
});
|
|
const spaceB = await createSpace(nonErrorApp);
|
|
const sourceB = await createConnectorSource(nonErrorApp, spaceB);
|
|
const fallback = await nonErrorApp.request(
|
|
`/knowledge-spaces/${spaceB}/sources/${sourceB}/files`,
|
|
{ headers: bearer(readToken) },
|
|
);
|
|
expect(fallback.status).toBe(502);
|
|
expect(await fallback.json()).toEqual({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDriveRequest.code,
|
|
error: SOURCE_OPERATION_FAILURES.onlineDriveRequest.message,
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("online drive import edge branches", () => {
|
|
it("returns 400 for a non-connector source and 501 without a connector", async () => {
|
|
const noConnectorApp = createApp();
|
|
const spaceA = await createSpace(noConnectorApp);
|
|
const connectorSourceId = await createConnectorSource(noConnectorApp, spaceA);
|
|
const notConfigured = await noConnectorApp.request(
|
|
`/knowledge-spaces/${spaceA}/sources/${connectorSourceId}/import-files`,
|
|
{
|
|
body: JSON.stringify({ files: [{ id: "f1", name: "a.txt" }] }),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
},
|
|
);
|
|
expect(notConfigured.status).toBe(501);
|
|
|
|
const connector: OnlineDriveConnector = {
|
|
browse: async () => ({ buckets: [] }),
|
|
download: async () => ({ body: new Uint8Array() }),
|
|
};
|
|
const app = createApp({ onlineDriveConnector: connector });
|
|
const spaceB = await createSpace(app);
|
|
const webSourceId = await createWebSource(app, spaceB);
|
|
const wrongType = await app.request(
|
|
`/knowledge-spaces/${spaceB}/sources/${webSourceId}/import-files`,
|
|
{
|
|
body: JSON.stringify({ files: [{ id: "f1", name: "a.txt" }] }),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
},
|
|
);
|
|
expect(wrongType.status).toBe(400);
|
|
});
|
|
|
|
it("imports bucketless files with explicit mime types and isolates download failures", async () => {
|
|
const nonErrorFailure: unknown = "download rejected without an Error";
|
|
const connector: OnlineDriveConnector = {
|
|
browse: async () => ({ buckets: [] }),
|
|
download: async ({ file }) => {
|
|
if (file.id === "bad") {
|
|
throw new Error("download denied");
|
|
}
|
|
|
|
if (file.id === "worse") {
|
|
throw nonErrorFailure;
|
|
}
|
|
|
|
return { body: new TextEncoder().encode(`content of ${file.id}`) };
|
|
},
|
|
};
|
|
const app = createApp({ onlineDriveConnector: connector });
|
|
const spaceId = await createSpace(app);
|
|
const sourceId = await createConnectorSource(app, spaceId);
|
|
|
|
const response = await app.request(
|
|
`/knowledge-spaces/${spaceId}/sources/${sourceId}/import-files`,
|
|
{
|
|
body: JSON.stringify({
|
|
files: [
|
|
{ id: "ok", mimeType: "text/plain", name: "plain.txt" },
|
|
{ id: "bad", name: "broken.bin" },
|
|
{ id: "worse", name: "worse.bin" },
|
|
],
|
|
}),
|
|
headers: json(writeToken),
|
|
method: "POST",
|
|
},
|
|
);
|
|
expect(response.status).toBe(200);
|
|
const body = await response.json();
|
|
expect(body.documents).toEqual([
|
|
{ documentAssetId: expect.any(String), filename: "plain.txt" },
|
|
]);
|
|
expect(body.failed).toEqual([
|
|
{
|
|
code: SOURCE_OPERATION_FAILURES.onlineDriveFileDownload.code,
|
|
error: "A file from the online-drive source could not be downloaded.",
|
|
failure: expect.objectContaining({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDriveFileDownload.code,
|
|
stage: "source-document-import",
|
|
}),
|
|
filename: "broken.bin",
|
|
},
|
|
{
|
|
code: SOURCE_OPERATION_FAILURES.onlineDriveFileDownload.code,
|
|
error: "A file from the online-drive source could not be downloaded.",
|
|
failure: expect.objectContaining({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDriveFileDownload.code,
|
|
stage: "source-document-import",
|
|
}),
|
|
filename: "worse.bin",
|
|
},
|
|
]);
|
|
|
|
// The imported-files provenance omits absent buckets and keeps explicit mime types.
|
|
const source = await getSource(app, spaceId, sourceId);
|
|
expect(source.metadata.importedFiles).toEqual({
|
|
ok: { mimeType: "text/plain", name: "plain.txt" },
|
|
});
|
|
expect(source.metadata.sync).toMatchObject({ failed: 2, imported: 1, requested: 3 });
|
|
});
|
|
});
|
|
|
|
describe("source handlers without optional collaborators", () => {
|
|
interface BareAppOptions {
|
|
authorizationMismatch?: boolean;
|
|
candidateGrants?: readonly string[];
|
|
legacyMutationEndpointsEnabled?: boolean;
|
|
onlineDocumentConnector?: OnlineDocumentConnector;
|
|
onlineDriveConnector?: OnlineDriveConnector;
|
|
sourceDocumentMaterializer?: SourceDocumentMaterializer;
|
|
sourceProductWorkflows?: Pick<
|
|
SourceProductWorkflowRepository,
|
|
"listLatestSyncCompletions" | "listLatestSyncRuns" | "listSyncPolicies"
|
|
>;
|
|
sources?: SourceRepository;
|
|
websiteCrawlConnector?: WebsiteCrawlConnector;
|
|
}
|
|
|
|
function createBareApp(options: BareAppOptions = {}) {
|
|
const app = createKnowledgeGatewayApp();
|
|
app.use("*", async (context, next) => {
|
|
const knowledgeSpaceId = context.req.path.split("/")[2] ?? "";
|
|
context.set("subject", {
|
|
scopes: ["knowledge-spaces:*"],
|
|
subjectId: "u1",
|
|
tenantId: "tenant-1",
|
|
});
|
|
context.set("authorizationDecision", {
|
|
accessContext: {} as never,
|
|
permissionSnapshot: {
|
|
apiAccessRevision: 1,
|
|
callerKind: "interactive",
|
|
candidateGrants: options.candidateGrants ?? [],
|
|
issuedAt: "2026-07-14T00:00:00.000Z",
|
|
knowledgeSpaceId: options.authorizationMismatch ? missingSourceId : knowledgeSpaceId,
|
|
memberRevision: 1,
|
|
memberRole: "owner",
|
|
policyRevision: 1,
|
|
subjectId: "u1",
|
|
tenantId: "tenant-1",
|
|
},
|
|
});
|
|
context.set("traceId", "trace-source-coverage");
|
|
context.set("rateLimitChecked", true);
|
|
await next();
|
|
});
|
|
|
|
const spaces = createInMemoryKnowledgeSpaceRepository({ maxListLimit: 100, maxSpaces: 10 });
|
|
const sources = options.sources ?? createInMemorySourceRepository({ maxSources: 100 });
|
|
registerSourceHandlers({
|
|
app,
|
|
...(options.legacyMutationEndpointsEnabled === undefined
|
|
? {}
|
|
: { legacyMutationEndpointsEnabled: options.legacyMutationEndpointsEnabled }),
|
|
...(options.onlineDocumentConnector
|
|
? { onlineDocumentConnector: options.onlineDocumentConnector }
|
|
: {}),
|
|
...(options.onlineDriveConnector
|
|
? { onlineDriveConnector: options.onlineDriveConnector }
|
|
: {}),
|
|
...(options.sourceDocumentMaterializer
|
|
? { sourceDocumentMaterializer: options.sourceDocumentMaterializer }
|
|
: {}),
|
|
...(options.sourceProductWorkflows
|
|
? { sourceProductWorkflows: options.sourceProductWorkflows }
|
|
: {}),
|
|
sources,
|
|
spaces,
|
|
...(options.websiteCrawlConnector
|
|
? { websiteCrawlConnector: options.websiteCrawlConnector }
|
|
: {}),
|
|
});
|
|
|
|
return { app, sources, spaces };
|
|
}
|
|
|
|
async function seedSource(
|
|
repos: { sources: SourceRepository; spaces: KnowledgeSpaceRepository },
|
|
type: "connector" | "web",
|
|
): Promise<{ sourceId: string; spaceId: string }> {
|
|
const space = await repos.spaces.create({
|
|
name: "Space",
|
|
slug: `space-${type}`,
|
|
tenantId: "tenant-1",
|
|
});
|
|
const source = await repos.sources.create({
|
|
knowledgeSpaceId: space.id,
|
|
name: "Seeded",
|
|
type,
|
|
uri: "https://example.com",
|
|
});
|
|
|
|
return { sourceId: source.id, spaceId: space.id };
|
|
}
|
|
|
|
it("enriches a source list with product sync details in bulk lookups", async () => {
|
|
const listSyncPolicies = vi.fn();
|
|
const listLatestSyncCompletions = vi.fn();
|
|
const listLatestSyncRuns = vi.fn();
|
|
const bare = createBareApp({
|
|
sourceProductWorkflows: {
|
|
listLatestSyncCompletions,
|
|
listLatestSyncRuns,
|
|
listSyncPolicies,
|
|
},
|
|
});
|
|
const { sourceId, spaceId } = await seedSource(bare, "web");
|
|
listSyncPolicies.mockResolvedValue([
|
|
{
|
|
accessChannel: "interactive",
|
|
createdAt: "2026-07-14T00:00:00.000Z",
|
|
enabled: true,
|
|
expectedSourceVersion: 1,
|
|
id: "00000000-0000-4000-8000-000000000111",
|
|
knowledgeSpaceId: spaceId,
|
|
mode: "provider",
|
|
permissionSnapshotId: "permission-1",
|
|
permissionSnapshotRevision: 1,
|
|
requestedBySubjectId: "u1",
|
|
requiredPermissionScope: [],
|
|
revision: 1,
|
|
sourceId,
|
|
tenantId: "tenant-1",
|
|
updatedAt: "2026-07-14T00:00:00.000Z",
|
|
},
|
|
]);
|
|
listLatestSyncCompletions.mockResolvedValue([
|
|
{ completedAt: "2026-07-14T01:00:00.000Z", sourceId },
|
|
]);
|
|
listLatestSyncRuns.mockResolvedValue([
|
|
{
|
|
activeSlot: 1,
|
|
checkpoint: "provider-read",
|
|
createdAt: "2026-07-14T02:00:00.000Z",
|
|
executionAttempts: 1,
|
|
id: "00000000-0000-4000-8000-000000000222",
|
|
idempotencyKey: "source-sync-active",
|
|
knowledgeSpaceId: spaceId,
|
|
kind: "sync",
|
|
maxExecutionAttempts: 5,
|
|
payload: {},
|
|
progressCompleted: 2,
|
|
progressFailed: 0,
|
|
progressSkipped: 0,
|
|
progressTotal: 5,
|
|
rowVersion: 2,
|
|
sourceId,
|
|
state: "syncing",
|
|
tenantId: "tenant-1",
|
|
updatedAt: "2026-07-14T02:01:00.000Z",
|
|
},
|
|
]);
|
|
|
|
const response = await bare.app.request(`/knowledge-spaces/${spaceId}/sources`);
|
|
|
|
expect(response.status).toBe(200);
|
|
await expect(response.json()).resolves.toMatchObject({
|
|
items: [
|
|
{
|
|
syncWorkflow: {
|
|
id: "00000000-0000-4000-8000-000000000222",
|
|
progressCompleted: 2,
|
|
progressTotal: 5,
|
|
sourceId,
|
|
state: "syncing",
|
|
},
|
|
id: sourceId,
|
|
lastSyncedAt: "2026-07-14T01:00:00.000Z",
|
|
status: "syncing",
|
|
syncPolicy: {
|
|
enabled: true,
|
|
mode: "provider",
|
|
sourceId,
|
|
},
|
|
},
|
|
],
|
|
});
|
|
expect(listSyncPolicies).toHaveBeenCalledOnce();
|
|
expect(listSyncPolicies).toHaveBeenCalledWith({
|
|
knowledgeSpaceId: spaceId,
|
|
sourceIds: [sourceId],
|
|
tenantId: "tenant-1",
|
|
});
|
|
expect(listLatestSyncCompletions).toHaveBeenCalledOnce();
|
|
expect(listLatestSyncCompletions).toHaveBeenCalledWith({
|
|
knowledgeSpaceId: spaceId,
|
|
sourceIds: [sourceId],
|
|
tenantId: "tenant-1",
|
|
});
|
|
expect(listLatestSyncRuns).toHaveBeenCalledOnce();
|
|
expect(listLatestSyncRuns).toHaveBeenCalledWith({
|
|
candidateGrants: [],
|
|
knowledgeSpaceId: spaceId,
|
|
sourceIds: [sourceId],
|
|
tenantId: "tenant-1",
|
|
});
|
|
});
|
|
|
|
it("keeps the latest terminal sync result in the source list", async () => {
|
|
const listLatestSyncRuns = vi.fn();
|
|
const bare = createBareApp({
|
|
sourceProductWorkflows: {
|
|
listLatestSyncCompletions: vi.fn().mockResolvedValue([]),
|
|
listLatestSyncRuns,
|
|
listSyncPolicies: vi.fn().mockResolvedValue([]),
|
|
},
|
|
});
|
|
const { sourceId, spaceId } = await seedSource(bare, "web");
|
|
listLatestSyncRuns.mockResolvedValue([
|
|
{
|
|
checkpoint: "provider-read",
|
|
createdAt: "2030-07-14T02:00:00.000Z",
|
|
executionAttempts: 1,
|
|
id: "00000000-0000-4000-8000-000000000333",
|
|
idempotencyKey: "source-sync-failed",
|
|
knowledgeSpaceId: spaceId,
|
|
kind: "sync",
|
|
lastErrorCode: "PROVIDER_FAILED",
|
|
maxExecutionAttempts: 5,
|
|
payload: {},
|
|
progressCompleted: 2,
|
|
progressFailed: 1,
|
|
progressSkipped: 0,
|
|
progressTotal: 5,
|
|
rowVersion: 3,
|
|
sourceId,
|
|
state: "failed",
|
|
tenantId: "tenant-1",
|
|
updatedAt: "2030-07-14T02:01:00.000Z",
|
|
},
|
|
]);
|
|
|
|
const response = await bare.app.request(`/knowledge-spaces/${spaceId}/sources`);
|
|
|
|
expect(response.status).toBe(200);
|
|
await expect(response.json()).resolves.toMatchObject({
|
|
items: [
|
|
{
|
|
id: sourceId,
|
|
status: "error",
|
|
syncWorkflow: {
|
|
failure: { code: "KNOWLEDGE_FS_INTERNAL_ERROR" },
|
|
id: "00000000-0000-4000-8000-000000000333",
|
|
lastErrorCode: "KNOWLEDGE_FS_INTERNAL_ERROR",
|
|
state: "failed",
|
|
},
|
|
},
|
|
],
|
|
});
|
|
});
|
|
|
|
it("keeps a disabled source disabled while exposing its active sync run", async () => {
|
|
const listLatestSyncRuns = vi.fn();
|
|
const bare = createBareApp({
|
|
sourceProductWorkflows: {
|
|
listLatestSyncCompletions: vi.fn().mockResolvedValue([]),
|
|
listLatestSyncRuns,
|
|
listSyncPolicies: vi.fn().mockResolvedValue([]),
|
|
},
|
|
});
|
|
const { sourceId, spaceId } = await seedSource(bare, "web");
|
|
const source = await bare.sources.get({ id: sourceId, knowledgeSpaceId: spaceId });
|
|
expect(source).not.toBeNull();
|
|
await bare.sources.update({
|
|
expectedVersion: source?.version,
|
|
id: sourceId,
|
|
knowledgeSpaceId: spaceId,
|
|
status: "disabled",
|
|
});
|
|
listLatestSyncRuns.mockResolvedValue([
|
|
{
|
|
activeSlot: 1,
|
|
checkpoint: "provider-read",
|
|
createdAt: "2030-07-14T02:00:00.000Z",
|
|
executionAttempts: 1,
|
|
id: "00000000-0000-4000-8000-000000000444",
|
|
idempotencyKey: "source-sync-active-disabled",
|
|
knowledgeSpaceId: spaceId,
|
|
kind: "sync",
|
|
maxExecutionAttempts: 5,
|
|
payload: {},
|
|
progressCompleted: 2,
|
|
progressFailed: 0,
|
|
progressSkipped: 0,
|
|
progressTotal: 5,
|
|
rowVersion: 2,
|
|
sourceId,
|
|
state: "syncing",
|
|
tenantId: "tenant-1",
|
|
updatedAt: "2030-07-14T02:01:00.000Z",
|
|
},
|
|
]);
|
|
|
|
const response = await bare.app.request(`/knowledge-spaces/${spaceId}/sources`);
|
|
|
|
expect(response.status).toBe(200);
|
|
await expect(response.json()).resolves.toMatchObject({
|
|
items: [
|
|
{
|
|
id: sourceId,
|
|
status: "disabled",
|
|
syncWorkflow: {
|
|
id: "00000000-0000-4000-8000-000000000444",
|
|
state: "syncing",
|
|
},
|
|
},
|
|
],
|
|
});
|
|
});
|
|
|
|
it("enforces handler-local space, authorization, and candidate-scope fences", async () => {
|
|
const bare = createBareApp();
|
|
const missingBase = `/knowledge-spaces/${missingSourceId}/sources/${missingSourceId}`;
|
|
const missingResponses = await Promise.all([
|
|
bare.app.request(`/knowledge-spaces/${missingSourceId}/sources`, {
|
|
body: JSON.stringify({ name: "Missing", type: "web", uri: "https://example.com" }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
}),
|
|
bare.app.request(`/knowledge-spaces/${missingSourceId}/sources`),
|
|
bare.app.request(missingBase),
|
|
bare.app.request(missingBase, {
|
|
body: JSON.stringify({ name: "Missing" }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "PATCH",
|
|
}),
|
|
bare.app.request(`${missingBase}/credentials`, {
|
|
body: JSON.stringify({ credentials: { token: "unused" }, expectedVersion: 1 }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "PUT",
|
|
}),
|
|
bare.app.request(`${missingBase}/credentials?expectedVersion=1`, { method: "DELETE" }),
|
|
bare.app.request(`${missingBase}/crawl`, { method: "POST" }),
|
|
bare.app.request(`${missingBase}/pages`),
|
|
bare.app.request(`${missingBase}/import`, {
|
|
body: JSON.stringify({ pages: [{ pageId: "p1", type: "page", workspaceId: "w1" }] }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
}),
|
|
bare.app.request(`${missingBase}/test`, { method: "POST" }),
|
|
bare.app.request(`${missingBase}/files`),
|
|
bare.app.request(`${missingBase}/import-files`, {
|
|
body: JSON.stringify({ files: [{ id: "f1", name: "a.txt" }] }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
}),
|
|
]);
|
|
expect(missingResponses.map((response) => response.status)).toEqual(Array(12).fill(404));
|
|
|
|
const mismatched = createBareApp({ authorizationMismatch: true });
|
|
const seeded = await seedSource(mismatched, "web");
|
|
const base = `/knowledge-spaces/${seeded.spaceId}/sources/${seeded.sourceId}`;
|
|
const denied = await Promise.all([
|
|
mismatched.app.request(`/knowledge-spaces/${seeded.spaceId}/sources`, {
|
|
body: JSON.stringify({ name: "Denied", type: "web", uri: "https://example.com" }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
}),
|
|
mismatched.app.request(`/knowledge-spaces/${seeded.spaceId}/sources`),
|
|
mismatched.app.request(base),
|
|
mismatched.app.request(base, {
|
|
body: JSON.stringify({ name: "Denied" }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "PATCH",
|
|
}),
|
|
]);
|
|
expect(denied.map((response) => response.status)).toEqual([403, 403, 403, 403]);
|
|
expect((await mismatched.app.request(`${base}/crawl`, { method: "POST" })).status).toBe(404);
|
|
|
|
const scoped = createBareApp();
|
|
const space = await scoped.spaces.create({
|
|
name: "Scoped",
|
|
slug: "scoped",
|
|
tenantId: "tenant-1",
|
|
});
|
|
const overbroad = await scoped.app.request(`/knowledge-spaces/${space.id}/sources`, {
|
|
body: JSON.stringify({
|
|
name: "Overbroad",
|
|
permissionScope: ["private"],
|
|
type: "web",
|
|
uri: "https://example.com",
|
|
}),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
});
|
|
expect(overbroad.status).toBe(403);
|
|
const restricted = await scoped.sources.create({
|
|
knowledgeSpaceId: space.id,
|
|
name: "Restricted",
|
|
permissionScope: ["private"],
|
|
type: "web",
|
|
uri: "https://example.com",
|
|
});
|
|
expect(
|
|
(await scoped.app.request(`/knowledge-spaces/${space.id}/sources/${restricted.id}`)).status,
|
|
).toBe(404);
|
|
});
|
|
|
|
it("maps a bounded candidate-visibility scan exhaustion to 503", async () => {
|
|
const repository = createInMemorySourceRepository({ maxSources: 10 });
|
|
const state: { hidden?: Awaited<ReturnType<SourceRepository["create"]>> } = {};
|
|
const scanningRepository: SourceRepository = {
|
|
...repository,
|
|
list: async () => ({
|
|
items: state.hidden ? [state.hidden] : [],
|
|
nextCursor: { id: state.hidden?.id ?? missingSourceId },
|
|
}),
|
|
};
|
|
const bare = createBareApp({ sources: scanningRepository });
|
|
const space = await bare.spaces.create({
|
|
name: "Scan budget",
|
|
slug: "scan-budget",
|
|
tenantId: "tenant-1",
|
|
});
|
|
state.hidden = await repository.create({
|
|
knowledgeSpaceId: space.id,
|
|
name: "Hidden",
|
|
permissionScope: ["private"],
|
|
type: "web",
|
|
uri: "https://example.com",
|
|
});
|
|
|
|
const response = await bare.app.request(`/knowledge-spaces/${space.id}/sources?limit=1`);
|
|
expect(response.status).toBe(503);
|
|
expect(await response.json()).toEqual({
|
|
code: "CANDIDATE_VISIBILITY_SCAN_BUDGET_EXCEEDED",
|
|
error: "Candidate visibility scan budget exceeded",
|
|
});
|
|
});
|
|
|
|
it("rejects incompatible or unconfigured create-time credential bindings", async () => {
|
|
const bare = createBareApp();
|
|
const space = await bare.spaces.create({
|
|
name: "Credential boundaries",
|
|
slug: "credential-boundaries",
|
|
tenantId: "tenant-1",
|
|
});
|
|
const create = (body: Record<string, unknown>) =>
|
|
bare.app.request(`/knowledge-spaces/${space.id}/sources`, {
|
|
body: JSON.stringify({
|
|
name: "Credential source",
|
|
type: "connector",
|
|
uri: "workspace-1",
|
|
...body,
|
|
}),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
});
|
|
|
|
expect(
|
|
(
|
|
await create({
|
|
connectionId: missingSourceId,
|
|
credentials: { token: "mutually-exclusive" },
|
|
})
|
|
).status,
|
|
).toBe(400);
|
|
expect((await create({ connectionId: missingSourceId })).status).toBe(503);
|
|
expect((await create({ credentials: { token: "secret-store-required" } })).status).toBe(503);
|
|
});
|
|
|
|
it("crawls without a materializer: raw pages come back and sync counters are null", async () => {
|
|
const bare = createBareApp({
|
|
websiteCrawlConnector: {
|
|
crawl: async () => ({
|
|
pages: [{ content: "# A", description: "About A", sourceUrl: "https://example.com/a" }],
|
|
}),
|
|
},
|
|
});
|
|
const { sourceId, spaceId } = await seedSource(bare, "web");
|
|
|
|
const response = await bare.app.request(
|
|
`/knowledge-spaces/${spaceId}/sources/${sourceId}/crawl`,
|
|
{ method: "POST" },
|
|
);
|
|
expect(response.status).toBe(200);
|
|
const body = await response.json();
|
|
expect(body.pages).toEqual([
|
|
{ content: "# A", description: "About A", sourceUrl: "https://example.com/a" },
|
|
]);
|
|
// No materializer: no import bookkeeping fields on the response.
|
|
expect(body.failed).toBeUndefined();
|
|
expect(body.imported).toBeUndefined();
|
|
expect(body.replaced).toBeUndefined();
|
|
expect(body.skipped).toBeUndefined();
|
|
|
|
const source = await bare.sources.get({ id: sourceId, knowledgeSpaceId: spaceId });
|
|
expect(source?.status).toBe("active");
|
|
expect(source?.metadata.sync).toEqual({
|
|
completed: null,
|
|
failed: null,
|
|
imported: null,
|
|
pageCount: 1,
|
|
replaced: null,
|
|
skipped: null,
|
|
status: null,
|
|
total: null,
|
|
});
|
|
});
|
|
|
|
it("rejects all legacy synchronous mutation endpoints when durable Source product is enabled", async () => {
|
|
const crawl = vi.fn(async () => ({ pages: [] }));
|
|
const bare = createBareApp({
|
|
legacyMutationEndpointsEnabled: false,
|
|
onlineDocumentConnector: {
|
|
getPageContent: vi.fn(async () => ({ content: "x", pageId: "p1" })),
|
|
listPages: vi.fn(async () => ({ workspaces: [] })),
|
|
},
|
|
onlineDriveConnector: {
|
|
browse: vi.fn(async () => ({ buckets: [] })),
|
|
download: vi.fn(async () => ({ body: new Uint8Array() })),
|
|
},
|
|
websiteCrawlConnector: { crawl },
|
|
});
|
|
const web = await seedSource(bare, "web");
|
|
const connector = await seedSource(bare, "connector");
|
|
|
|
const responses = await Promise.all([
|
|
bare.app.request(`/knowledge-spaces/${web.spaceId}/sources/${web.sourceId}/crawl`, {
|
|
method: "POST",
|
|
}),
|
|
bare.app.request(
|
|
`/knowledge-spaces/${connector.spaceId}/sources/${connector.sourceId}/import`,
|
|
{
|
|
body: JSON.stringify({
|
|
pages: [{ pageId: "p1", type: "page", workspaceId: "workspace-1" }],
|
|
}),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
},
|
|
),
|
|
bare.app.request(
|
|
`/knowledge-spaces/${connector.spaceId}/sources/${connector.sourceId}/import-files`,
|
|
{
|
|
body: JSON.stringify({ files: [{ id: "f1", name: "f1.txt" }] }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
},
|
|
),
|
|
]);
|
|
|
|
expect(responses.map((response) => response.status)).toEqual([409, 409, 409]);
|
|
expect(crawl).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("returns 501 for page and file imports when the materializer is missing", async () => {
|
|
const bare = createBareApp({
|
|
onlineDocumentConnector: {
|
|
getPageContent: async ({ page }) => ({ content: "x", pageId: page.pageId }),
|
|
listPages: async () => ({ workspaces: [] }),
|
|
},
|
|
onlineDriveConnector: {
|
|
browse: async () => ({ buckets: [] }),
|
|
download: async () => ({ body: new Uint8Array() }),
|
|
},
|
|
});
|
|
const { sourceId, spaceId } = await seedSource(bare, "connector");
|
|
|
|
const pageImport = await bare.app.request(
|
|
`/knowledge-spaces/${spaceId}/sources/${sourceId}/import`,
|
|
{
|
|
body: JSON.stringify({ pages: [{ pageId: "p1", type: "page", workspaceId: "w1" }] }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
},
|
|
);
|
|
expect(pageImport.status).toBe(501);
|
|
|
|
const fileImport = await bare.app.request(
|
|
`/knowledge-spaces/${spaceId}/sources/${sourceId}/import-files`,
|
|
{
|
|
body: JSON.stringify({ files: [{ id: "f1", name: "a.txt" }] }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
},
|
|
);
|
|
expect(fileImport.status).toBe(501);
|
|
});
|
|
|
|
it("marks the source errored and returns 502 when materialization fails", async () => {
|
|
const failingMaterializer: SourceDocumentMaterializer = {
|
|
compensate: async () => undefined,
|
|
materialize: async () => {
|
|
throw new Error("materialize exploded with credential-secret");
|
|
},
|
|
};
|
|
const bare = createBareApp({
|
|
onlineDocumentConnector: {
|
|
getPageContent: async ({ page }) => ({ content: "c", pageId: page.pageId }),
|
|
listPages: async () => ({ workspaces: [] }),
|
|
},
|
|
onlineDriveConnector: {
|
|
browse: async () => ({ buckets: [] }),
|
|
download: async () => ({ body: new TextEncoder().encode("b") }),
|
|
},
|
|
sourceDocumentMaterializer: failingMaterializer,
|
|
});
|
|
const { sourceId, spaceId } = await seedSource(bare, "connector");
|
|
|
|
const pageImport = await bare.app.request(
|
|
`/knowledge-spaces/${spaceId}/sources/${sourceId}/import`,
|
|
{
|
|
body: JSON.stringify({ pages: [{ pageId: "p1", type: "page", workspaceId: "w1" }] }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
},
|
|
);
|
|
expect(pageImport.status).toBe(502);
|
|
const pageError = await pageImport.json();
|
|
expect(pageError).toEqual({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDocumentImport.code,
|
|
error: SOURCE_OPERATION_FAILURES.onlineDocumentImport.message,
|
|
});
|
|
expect(JSON.stringify(pageError)).not.toContain("credential-secret");
|
|
|
|
const afterPages = await bare.sources.get({ id: sourceId, knowledgeSpaceId: spaceId });
|
|
expect(afterPages?.status).toBe("error");
|
|
expect(afterPages?.metadata.sync).toEqual({
|
|
error: SOURCE_OPERATION_FAILURES.onlineDocumentImport.message,
|
|
errorCode: SOURCE_OPERATION_FAILURES.onlineDocumentImport.code,
|
|
});
|
|
expect(JSON.stringify(afterPages?.metadata)).not.toContain("credential-secret");
|
|
|
|
const fileImport = await bare.app.request(
|
|
`/knowledge-spaces/${spaceId}/sources/${sourceId}/import-files`,
|
|
{
|
|
body: JSON.stringify({ files: [{ id: "f1", name: "a.txt" }] }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
},
|
|
);
|
|
expect(fileImport.status).toBe(502);
|
|
const fileError = await fileImport.json();
|
|
expect(fileError).toEqual({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDriveImport.code,
|
|
error: SOURCE_OPERATION_FAILURES.onlineDriveImport.message,
|
|
});
|
|
expect(JSON.stringify(fileError)).not.toContain("credential-secret");
|
|
|
|
const afterFiles = await bare.sources.get({ id: sourceId, knowledgeSpaceId: spaceId });
|
|
expect(afterFiles?.status).toBe("error");
|
|
expect(afterFiles?.metadata.sync).toEqual({
|
|
error: SOURCE_OPERATION_FAILURES.onlineDriveImport.message,
|
|
errorCode: SOURCE_OPERATION_FAILURES.onlineDriveImport.code,
|
|
});
|
|
expect(JSON.stringify(afterFiles?.metadata)).not.toContain("credential-secret");
|
|
});
|
|
|
|
it("falls back to generic import failure messages when materialization throws a non-Error", async () => {
|
|
const nonErrorFailure: unknown = "materialize rejected without an Error";
|
|
const bare = createBareApp({
|
|
onlineDocumentConnector: {
|
|
getPageContent: async ({ page }) => ({ content: "c", pageId: page.pageId }),
|
|
listPages: async () => ({ workspaces: [] }),
|
|
},
|
|
onlineDriveConnector: {
|
|
browse: async () => ({ buckets: [] }),
|
|
download: async () => ({ body: new TextEncoder().encode("b") }),
|
|
},
|
|
sourceDocumentMaterializer: {
|
|
compensate: async () => undefined,
|
|
materialize: async () => {
|
|
throw nonErrorFailure;
|
|
},
|
|
},
|
|
});
|
|
const { sourceId, spaceId } = await seedSource(bare, "connector");
|
|
|
|
const pageImport = await bare.app.request(
|
|
`/knowledge-spaces/${spaceId}/sources/${sourceId}/import`,
|
|
{
|
|
body: JSON.stringify({ pages: [{ pageId: "p1", type: "page", workspaceId: "w1" }] }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
},
|
|
);
|
|
expect(pageImport.status).toBe(502);
|
|
expect(await pageImport.json()).toEqual({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDocumentImport.code,
|
|
error: SOURCE_OPERATION_FAILURES.onlineDocumentImport.message,
|
|
});
|
|
|
|
const afterPages = await bare.sources.get({ id: sourceId, knowledgeSpaceId: spaceId });
|
|
expect(afterPages?.metadata.sync).toEqual({
|
|
error: SOURCE_OPERATION_FAILURES.onlineDocumentImport.message,
|
|
errorCode: SOURCE_OPERATION_FAILURES.onlineDocumentImport.code,
|
|
});
|
|
|
|
const fileImport = await bare.app.request(
|
|
`/knowledge-spaces/${spaceId}/sources/${sourceId}/import-files`,
|
|
{
|
|
body: JSON.stringify({ files: [{ id: "f1", name: "a.txt" }] }),
|
|
headers: { "content-type": "application/json" },
|
|
method: "POST",
|
|
},
|
|
);
|
|
expect(fileImport.status).toBe(502);
|
|
expect(await fileImport.json()).toEqual({
|
|
code: SOURCE_OPERATION_FAILURES.onlineDriveImport.code,
|
|
error: SOURCE_OPERATION_FAILURES.onlineDriveImport.message,
|
|
});
|
|
|
|
const afterFiles = await bare.sources.get({ id: sourceId, knowledgeSpaceId: spaceId });
|
|
expect(afterFiles?.status).toBe("error");
|
|
expect(afterFiles?.metadata.sync).toEqual({
|
|
error: SOURCE_OPERATION_FAILURES.onlineDriveImport.message,
|
|
errorCode: SOURCE_OPERATION_FAILURES.onlineDriveImport.code,
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("source handler helper functions", () => {
|
|
it("builds unique online-document filenames with degraded slugs", () => {
|
|
expect(onlineDocumentFilename("Hello World", "p 1")).toBe("Hello-World-p-1.md");
|
|
// Page id slugs to nothing: fall back to the base name alone.
|
|
expect(onlineDocumentFilename("Notes", "///")).toBe("Notes.md");
|
|
// Nothing slugs to anything: fall back to the generic page name.
|
|
expect(onlineDocumentFilename("###", "$$$")).toBe("page.md");
|
|
});
|
|
|
|
it("maps filenames to mime types with an octet-stream fallback", () => {
|
|
expect(mimeTypeForFilename("report.PDF")).toBe("application/pdf");
|
|
expect(mimeTypeForFilename("archive.zip")).toBe("application/octet-stream");
|
|
expect(mimeTypeForFilename("README")).toBe("application/octet-stream");
|
|
});
|
|
|
|
it("reads imported page state defensively", () => {
|
|
expect(readImportedState({})).toEqual({});
|
|
expect(readImportedState({ imported: ["nope"] })).toEqual({});
|
|
expect(
|
|
readImportedState({
|
|
imported: {
|
|
a: "not an object",
|
|
b: { documentAssetId: 42, lastEditedTime: 42 },
|
|
c: { documentAssetId: "d1", lastEditedTime: "t1" },
|
|
d: null,
|
|
e: ["array"],
|
|
},
|
|
}),
|
|
).toEqual({ b: {}, c: { documentAssetId: "d1", lastEditedTime: "t1" } });
|
|
});
|
|
|
|
it("reads imported file state defensively", () => {
|
|
expect(readImportedFilesState({})).toEqual({});
|
|
expect(readImportedFilesState({ importedFiles: "nope" })).toEqual({});
|
|
expect(
|
|
readImportedFilesState({
|
|
importedFiles: {
|
|
a: { name: "" },
|
|
b: { bucket: 3, mimeType: "text/plain", name: "f.txt" },
|
|
c: ["array"],
|
|
d: { bucket: "b1", name: "g.txt" },
|
|
},
|
|
}),
|
|
).toEqual({
|
|
b: { mimeType: "text/plain", name: "f.txt" },
|
|
d: { bucket: "b1", name: "g.txt" },
|
|
});
|
|
});
|
|
});
|