From 42aecd649c65176b2fc3caad17a631848a2d58c2 Mon Sep 17 00:00:00 2001 From: FFXN Date: Sat, 8 Aug 2026 20:15:21 +0800 Subject: [PATCH] fix(knowledge_fs): add diagnostics for empty content and frame metadata logging --- ...-08-08-dify-online-document-text-stream.md | 5 ++ .../api/src/online-document-options.test.ts | 36 ++++++++++++- .../apps/api/src/online-document-options.ts | 52 +++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/knowledge-fs/.harness/changes/2026-08-08-dify-online-document-text-stream.md b/knowledge-fs/.harness/changes/2026-08-08-dify-online-document-text-stream.md index 8c7027f63c6..178e11e58d3 100644 --- a/knowledge-fs/.harness/changes/2026-08-08-dify-online-document-text-stream.md +++ b/knowledge-fs/.harness/changes/2026-08-08-dify-online-document-text-stream.md @@ -10,6 +10,9 @@ Date: 2026-08-08 `{ result: { content } }` envelopes. - Added regression coverage using the real Dify datasource message shape plus compatibility coverage for the structured envelope. +- Added one safe content-fetch diagnostic containing only bounded frame types, message field names, + recognized-frame count, and final UTF-8 byte length. Empty documents remain valid connector + results; neither content nor credentials are logged. ## Why @@ -22,6 +25,8 @@ zero knowledge nodes, and a terminal compilation failure because no FTS projecti - RED: `pnpm --filter @knowledge/api-app test -- online-document-options.test.ts` reproduced an empty content result for two Dify text messages. - GREEN: the same command passed all 211 tests across 42 API-app test files after the fix. +- RED: the empty-document diagnostic regression test confirmed that empty content already returned + successfully but no structural frame diagnostic was emitted. - `pnpm check` passed, including typecheck, 4,269 API tests plus the remaining workspace tests, coverage gates, evaluation gates, migration checks, Compose checks, and smoke tests. - `pnpm build` passed all 12 workspace package builds. diff --git a/knowledge-fs/apps/api/src/online-document-options.test.ts b/knowledge-fs/apps/api/src/online-document-options.test.ts index 1fa2a72455e..63c48764df9 100644 --- a/knowledge-fs/apps/api/src/online-document-options.test.ts +++ b/knowledge-fs/apps/api/src/online-document-options.test.ts @@ -1,5 +1,5 @@ import type { OnlineDocumentListInput } from "@knowledge/api"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { ApiDatasourceInvocationClient, @@ -139,4 +139,38 @@ describe("createApiOnlineDocumentConnector", () => { expect(content).toEqual({ content: "# Page One", pageId: "p1", workspaceId: "w1" }); }); + + it("keeps empty content valid and logs only bounded frame metadata", async () => { + const calls: ApiDatasourceInvocationInput[] = []; + const client = clientYielding( + [ + { message: { text: "" }, meta: { credential: "must-not-log" }, type: "text" }, + { message: { json_object: { ignored: true } }, type: "json" }, + ], + calls, + ); + const info = vi.spyOn(console, "info").mockImplementation(() => undefined); + + const content = await createApiOnlineDocumentConnector({ client }).getPageContent({ + page: { pageId: "p1", type: "page", workspaceId: "w1" }, + source: SOURCE, + tenantId: "tenant-1", + }); + + expect(content).toEqual({ content: "", pageId: "p1" }); + expect(info).toHaveBeenCalledOnce(); + const diagnostic = String(info.mock.calls[0]?.[0]); + expect(JSON.parse(diagnostic)).toMatchObject({ + contentBytes: 0, + event: "knowledge_fs.online_document.content_frames", + frameCount: 2, + frameTypes: { json: 1, text: 1 }, + messageKeys: ["json_object", "text"], + pageId: "p1", + recognizedFrames: 1, + sourceId: SOURCE.id, + }); + expect(diagnostic).not.toContain("must-not-log"); + info.mockRestore(); + }); }); diff --git a/knowledge-fs/apps/api/src/online-document-options.ts b/knowledge-fs/apps/api/src/online-document-options.ts index 0e31f0058a3..303cc107a1e 100644 --- a/knowledge-fs/apps/api/src/online-document-options.ts +++ b/knowledge-fs/apps/api/src/online-document-options.ts @@ -18,6 +18,8 @@ interface ListingEnvelopeMetadata { nextCursor?: string | undefined; } +const MAX_DIAGNOSTIC_KEYS = 20; + /** * Online-document connector backed by the deployment-selected datasource runtime. Page listing accumulates the * streamed workspaces/pages (deduped); page content concatenates Dify text messages while retaining support for @@ -29,8 +31,12 @@ export function createApiOnlineDocumentConnector(input: { return { getPageContent: async ({ page, signal, source, tenantId, userId }) => { let content = ""; + let frameCount = 0; let pageId = page.pageId; + let recognizedFrames = 0; let workspaceId: string | undefined; + const frameTypes = new Map(); + const messageKeys = new Set(); for await (const raw of input.client.dispatch({ operation: "get_online_document_page_content", @@ -40,12 +46,16 @@ export function createApiOnlineDocumentConnector(input: { ...(userId ? { userId } : {}), ...(signal ? { signal } : {}), })) { + frameCount += 1; + accumulateContentFrameMetadata(raw, frameTypes, messageKeys); const parsed = parseContentEnvelope(raw); if (!parsed) { continue; } + recognizedFrames += 1; + if (parsed.content) { content = parsed.append ? content + parsed.content : parsed.content; } @@ -59,6 +69,21 @@ export function createApiOnlineDocumentConnector(input: { } } + console.info( + JSON.stringify({ + contentBytes: new TextEncoder().encode(content).byteLength, + event: "knowledge_fs.online_document.content_frames", + frameCount, + frameTypes: Object.fromEntries( + Array.from(frameTypes.entries()).sort(([a], [b]) => a.localeCompare(b)), + ), + messageKeys: Array.from(messageKeys).sort(), + pageId, + recognizedFrames, + sourceId: source.id, + }), + ); + const result: OnlineDocumentPageContent = { content, pageId, @@ -105,6 +130,33 @@ export function createApiOnlineDocumentConnector(input: { }; } +function accumulateContentFrameMetadata( + raw: unknown, + frameTypes: Map, + messageKeys: Set, +): void { + if (!raw || typeof raw !== "object") { + incrementBoundedFrameType(frameTypes, typeof raw); + return; + } + + const envelope = raw as Record; + const type = typeof envelope.type === "string" && envelope.type ? envelope.type : "unknown"; + incrementBoundedFrameType(frameTypes, type); + + if (envelope.message && typeof envelope.message === "object") { + for (const key of Object.keys(envelope.message)) { + if (messageKeys.size >= MAX_DIAGNOSTIC_KEYS) break; + messageKeys.add(key); + } + } +} + +function incrementBoundedFrameType(frameTypes: Map, type: string): void { + const key = frameTypes.has(type) || frameTypes.size < MAX_DIAGNOSTIC_KEYS ? type : "other"; + frameTypes.set(key, (frameTypes.get(key) ?? 0) + 1); +} + export function createApiOnlineDocumentOptions(input: { readonly client: ApiDatasourceInvocationClient; }): {