From e911af2192faa0f20e1e60e2dfcac330413fa5eb Mon Sep 17 00:00:00 2001 From: matevip Date: Tue, 12 May 2026 14:11:02 +0800 Subject: [PATCH] =?UTF-8?q?feat(wiki):=20JSON=20output=20mode=20=E2=80=94?= =?UTF-8?q?=20structured=20transformation=20output=20for=20programmatic=20?= =?UTF-8?q?downstream?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../wiki/model/WikiTransformationEntity.java | 10 +++ .../service/WikiTransformationExecutor.java | 72 ++++++++++++++++++- .../service/WikiTransformationService.java | 14 ++++ ...109__wiki_transformation_output_format.sql | 8 +++ ...109__wiki_transformation_output_format.sql | 11 +++ .../wiki/transformation-system-json.txt | 16 +++++ mateclaw-ui/src/api/index.ts | 2 + mateclaw-ui/src/i18n/locales/en-US.ts | 4 ++ mateclaw-ui/src/i18n/locales/zh-CN.ts | 4 ++ .../Wiki/components/TransformationsPanel.vue | 22 ++++++ 10 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 mateclaw-server/src/main/resources/db/migration/h2/V109__wiki_transformation_output_format.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/mysql/V109__wiki_transformation_output_format.sql create mode 100644 mateclaw-server/src/main/resources/prompts/wiki/transformation-system-json.txt diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationEntity.java index 83c959cc..7a734ba5 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationEntity.java @@ -66,6 +66,16 @@ public class WikiTransformationEntity { */ private String outputTarget; + /** + * Declared shape of the LLM output. {@code markdown} (default) accepts + * any text and stores it verbatim. {@code json} asks the LLM for a + * single JSON document; the executor parses it, retries once on parse + * failure, and marks the run failed if both attempts fail. JSON output + * is stored as a fenced ```json block in the run row so the existing + * markdown rendering path stays compatible. + */ + private String outputFormat; + @TableField(fill = FieldFill.INSERT) private LocalDateTime createTime; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java index bd4c4c12..44e764b7 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java @@ -199,6 +199,10 @@ public class WikiTransformationExecutor { * Shared prompt-render + LLM-call + output-cleanup stage used by both * raw-input and page-input entry points. Sets {@code run.modelId} as a * side-effect so the run row reflects which model produced the output. + *

+ * When the template declares {@code outputFormat=json}, the response is + * parsed as JSON; on parse failure the LLM is asked once more with a + * stricter "return only JSON" reminder before the run is failed. */ private String renderAndCallLlm(WikiTransformationEntity transformation, Long kbId, String sourceTitle, String sourceText, @@ -207,7 +211,10 @@ public class WikiTransformationExecutor { ? sourceText.substring(0, MAX_INPUT_CHARS) + "\n…(truncated)" : sourceText; - String systemPrompt = PromptLoader.loadPrompt("wiki/transformation-system"); + boolean wantJson = "json".equalsIgnoreCase(transformation.getOutputFormat()); + + String systemPrompt = PromptLoader.loadPrompt( + wantJson ? "wiki/transformation-system-json" : "wiki/transformation-system"); String instruction = (transformation.getPromptTemplate() == null ? "" : transformation.getPromptTemplate()) .replace("{input_text}", trimmedInput) .replace("{title}", sourceTitle); @@ -220,6 +227,31 @@ public class WikiTransformationExecutor { ChatModel chatModel = buildChatModel(resolvedModelId); run.setModelId(resolvedModelId); + String output = callOnce(chatModel, systemPrompt, userPrompt); + if (wantJson) { + String coerced = coerceToJson(output); + if (coerced != null) { + // Wrap in a fenced block so UI rendering and save-as-page + // keep the existing markdown contract. The raw JSON is the + // first thing inside the block, so downstream tools can grep. + return "```json\n" + coerced + "\n```"; + } + // One retry with an explicit nudge. + log.info("[WikiTransformation] JSON parse failed for template={}; retrying with stricter reminder", + transformation.getName()); + String retryUserPrompt = userPrompt + "\n\n---\n\n上一次回复不是合法 JSON。请只返回一个合法 JSON 文档，前后不要有任何文字或代码块标记。"; + String retry = callOnce(chatModel, systemPrompt, retryUserPrompt); + String coercedRetry = coerceToJson(retry); + if (coercedRetry != null) { + return "```json\n" + coercedRetry + "\n```"; + } + throw new IllegalStateException("LLM output is not valid JSON after one retry"); + } + return output; + } + + /** One LLM call, returns the cleaned output. Throws when the call yields blank. */ + private String callOnce(ChatModel chatModel, String systemPrompt, String userPrompt) { ChatResponse resp = chatModel.call(new Prompt(List.of( new SystemMessage(systemPrompt), new UserMessage(userPrompt)))); String rawOutput = (resp == null || resp.getResult() == null @@ -235,6 +267,44 @@ public class WikiTransformationExecutor { return output; } + private static final com.fasterxml.jackson.databind.ObjectMapper JSON_MAPPER = + new com.fasterxml.jackson.databind.ObjectMapper(); + + /** + * Try to parse the output as JSON. If the LLM wrapped it in a fenced + * block or sprinkled prose around it, fall back to finding the outer + * '{' / '[' brackets and try again. Returns the normalized JSON string + * on success, {@code null} on failure. + */ + private static String coerceToJson(String text) { + if (text == null || text.isBlank()) return null; + String candidate = text.trim(); + try { + JSON_MAPPER.readTree(candidate); + return candidate; + } catch (Exception ignored) { + // fall through to bracket-trim attempt + } + int objStart = candidate.indexOf('{'); + int arrStart = candidate.indexOf('['); + int start; + char open; + if (objStart < 0) { start = arrStart; open = '['; } + else if (arrStart < 0) { start = objStart; open = '{'; } + else { start = Math.min(objStart, arrStart); open = candidate.charAt(start); } + if (start < 0) return null; + char close = open == '{' ? '}' : ']'; + int end = candidate.lastIndexOf(close); + if (end <= start) return null; + String trimmed = candidate.substring(start, end + 1); + try { + JSON_MAPPER.readTree(trimmed); + return trimmed; + } catch (Exception e) { + return null; + } + } + /** * Run the transformation against the given raw material and persist * the outcome. The returned entity is the persisted run row, regardless diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationService.java index 30d15445..a131697a 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationService.java @@ -110,6 +110,7 @@ public class WikiTransformationService { // create and update paths accept the same payload from the UI. entity.setModelId(input.getModelId() != null && input.getModelId() < 0 ? null : input.getModelId()); entity.setOutputTarget(normalizeOutputTarget(input.getOutputTarget())); + entity.setOutputFormat(normalizeOutputFormat(input.getOutputFormat())); transformationMapper.insert(entity); log.info("[WikiTransformation] created id={} name={} kbId={}", entity.getId(), entity.getName(), entity.getKbId()); @@ -135,6 +136,9 @@ public class WikiTransformationService { if (patch.getOutputTarget() != null) { entity.setOutputTarget(normalizeOutputTarget(patch.getOutputTarget())); } + if (patch.getOutputFormat() != null) { + entity.setOutputFormat(normalizeOutputFormat(patch.getOutputFormat())); + } transformationMapper.updateById(entity); return entity; } @@ -149,6 +153,16 @@ public class WikiTransformationService { }; } + /** Whitelist incoming outputFormat; unknown / null = "markdown". */ + private static String normalizeOutputFormat(String raw) { + if (raw == null) return "markdown"; + String trimmed = raw.trim().toLowerCase(); + return switch (trimmed) { + case "json" -> "json"; + default -> "markdown"; + }; + } + @Transactional public void delete(Long id) { transformationMapper.deleteById(id); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V109__wiki_transformation_output_format.sql b/mateclaw-server/src/main/resources/db/migration/h2/V109__wiki_transformation_output_format.sql new file mode 100644 index 00000000..e1fe5520 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V109__wiki_transformation_output_format.sql @@ -0,0 +1,8 @@ +-- Output format declared on the template so the executor can validate the +-- LLM's response shape. 'markdown' (default) keeps the legacy behaviour +-- where output is treated as Markdown and saved as page content; 'json' +-- asks the LLM for a single JSON object and the executor parses + validates +-- before persisting. Future formats (table, yaml) can extend this column. + +ALTER TABLE mate_wiki_transformation + ADD COLUMN IF NOT EXISTS output_format VARCHAR(16) NOT NULL DEFAULT 'markdown'; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V109__wiki_transformation_output_format.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V109__wiki_transformation_output_format.sql new file mode 100644 index 00000000..331b969b --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V109__wiki_transformation_output_format.sql @@ -0,0 +1,11 @@ +-- Output format declared on the template. See h2 sibling for the prose +-- explanation. MySQL needs the INFORMATION_SCHEMA guard pattern. + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_transformation' + AND COLUMN_NAME = 'output_format'); +SET @s := IF(@c = 0, + 'ALTER TABLE mate_wiki_transformation ADD COLUMN output_format VARCHAR(16) NOT NULL DEFAULT ''markdown''', + 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/prompts/wiki/transformation-system-json.txt b/mateclaw-server/src/main/resources/prompts/wiki/transformation-system-json.txt new file mode 100644 index 00000000..34f1be05 --- /dev/null +++ b/mateclaw-server/src/main/resources/prompts/wiki/transformation-system-json.txt @@ -0,0 +1,16 @@ +You are a content transformation worker producing structured JSON. The user +supplies (a) a transformation instruction and (b) a source text. Follow +the instruction precisely and return exactly one valid JSON document. + +Rules: +- Return ONLY a JSON document — no prose, no commentary, no markdown code + fences. The first character of your reply must be `{` or `[`. +- Do not add framing like "Here is the JSON:" — emit the JSON object alone. +- Use only JSON-valid escapes; double-quote all strings. +- Do not invent facts beyond the supplied source text. If the source is + empty, return `{"error": "empty source"}`. +- Preserve the language of the source text in string values unless the + instruction explicitly says otherwise. +- If the instruction describes a schema (fields, arrays, types), follow + it exactly; missing values get an empty string or `null` per JSON + convention rather than being omitted entirely. diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index 4438969f..68e97bff 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -691,6 +691,7 @@ export const wikiApi = { enabled?: boolean modelId?: number | null outputTarget?: 'none' | 'page' + outputFormat?: 'markdown' | 'json' }) => http.post('/wiki/transformations', data), updateTransformation: (id: number, data: { @@ -701,6 +702,7 @@ export const wikiApi = { enabled?: boolean modelId?: number | null outputTarget?: 'none' | 'page' + outputFormat?: 'markdown' | 'json' }) => http.put(`/wiki/transformations/${id}`, data), deleteTransformation: (id: number) => diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index a4b39f63..9f1595db 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -1818,6 +1818,10 @@ export default { outputTargetNone: 'None — output stays in run history', outputTargetPage: 'Save as wiki page (searchable / agent-accessible / linkable)', outputTargetPageBadge: 'Auto-save to page', + outputFormatLabel: 'Output format', + outputFormatMarkdown: 'Markdown (default, human-readable)', + outputFormatJson: 'JSON (structured for downstream tools; one auto-retry on parse failure)', + outputFormatJsonBadge: 'JSON', saveAsPageBtn: 'Save as page', saving: 'Saving…', savedAsPage: 'Saved as:', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 6dda20e9..355aa3bf 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -1830,6 +1830,10 @@ export default { outputTargetNone: '不保存（仅留在运行历史）', outputTargetPage: '保存为 Wiki 页面（可被搜索 / Agent / 关系图引用）', outputTargetPageBadge: '自动落页', + outputFormatLabel: '输出格式', + outputFormatMarkdown: 'Markdown（默认，适合人读）', + outputFormatJson: 'JSON（结构化，便于下游程序消费；失败会自动重试 1 次）', + outputFormatJsonBadge: 'JSON', saveAsPageBtn: '保存为页面', saving: '保存中…', savedAsPage: '已保存：', diff --git a/mateclaw-ui/src/views/Wiki/components/TransformationsPanel.vue b/mateclaw-ui/src/views/Wiki/components/TransformationsPanel.vue index 8739416d..80e046fe 100644 --- a/mateclaw-ui/src/views/Wiki/components/TransformationsPanel.vue +++ b/mateclaw-ui/src/views/Wiki/components/TransformationsPanel.vue @@ -41,6 +41,9 @@ {{ t('wiki.transformations.outputTargetPageBadge') }} + + {{ t('wiki.transformations.outputFormatJsonBadge') }} + {{ modelLabelFor(tpl.modelId) }} @@ -232,6 +235,18 @@ {{ t('wiki.transformations.outputTargetPage') }} + +

+ {{ t('wiki.transformations.outputFormatLabel') }} + + +
@@ -265,6 +280,7 @@ interface WikiTransformation { enabled: boolean modelId: number | null outputTarget: 'none' | 'page' | null + outputFormat: 'markdown' | 'json' | null } interface WikiTransformationRun { @@ -314,6 +330,7 @@ const form = reactive<{ applyDefault: boolean enabled: boolean outputTarget: 'none' | 'page' + outputFormat: 'markdown' | 'json' modelId: number | null }>({ name: '', @@ -323,6 +340,7 @@ const form = reactive<{ applyDefault: false, enabled: true, outputTarget: 'none', + outputFormat: 'markdown', modelId: null, }) @@ -409,6 +427,7 @@ function openCreate() { form.applyDefault = false form.enabled = true form.outputTarget = 'none' + form.outputFormat = 'markdown' form.modelId = null editorOpen.value = true ensureModelsLoaded() @@ -423,6 +442,7 @@ function openEdit(tpl: WikiTransformation) { form.applyDefault = tpl.applyDefault form.enabled = tpl.enabled !== false form.outputTarget = tpl.outputTarget === 'page' ? 'page' : 'none' + form.outputFormat = tpl.outputFormat === 'json' ? 'json' : 'markdown' form.modelId = tpl.modelId ?? null editorOpen.value = true ensureModelsLoaded() @@ -451,6 +471,7 @@ async function onSave() { applyDefault: form.applyDefault, enabled: form.enabled, outputTarget: form.outputTarget, + outputFormat: form.outputFormat, modelId: updateModelId, }) } else { @@ -463,6 +484,7 @@ async function onSave() { applyDefault: form.applyDefault, enabled: form.enabled, outputTarget: form.outputTarget, + outputFormat: form.outputFormat, modelId: form.modelId, }) }