feat(wiki): JSON output mode — structured transformation output for programmatic downstream

mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationEntity.java

@@ -66,6 +66,16 @@ public class WikiTransformationEntity {
      */
     private String outputTarget;
 
+    /**
+     * Declared shape of the LLM output. {@code markdown} (default) accepts
+     * any text and stores it verbatim. {@code json} asks the LLM for a
+     * single JSON document; the executor parses it, retries once on parse
+     * failure, and marks the run failed if both attempts fail. JSON output
+     * is stored as a fenced ```json block in the run row so the existing
+     * markdown rendering path stays compatible.
+     */
+    private String outputFormat;
+
     @TableField(fill = FieldFill.INSERT)
     private LocalDateTime createTime;
 

mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java

@@ -199,6 +199,10 @@ public class WikiTransformationExecutor {
      * Shared prompt-render + LLM-call + output-cleanup stage used by both
      * raw-input and page-input entry points. Sets {@code run.modelId} as a
      * side-effect so the run row reflects which model produced the output.
+     * <p>
+     * When the template declares {@code outputFormat=json}, the response is
+     * parsed as JSON; on parse failure the LLM is asked once more with a
+     * stricter "return only JSON" reminder before the run is failed.
      */
     private String renderAndCallLlm(WikiTransformationEntity transformation, Long kbId,
                                      String sourceTitle, String sourceText,
@@ -207,7 +211,10 @@ public class WikiTransformationExecutor {
                 ? sourceText.substring(0, MAX_INPUT_CHARS) + "\n…(truncated)"
                 : sourceText;
 
-        String systemPrompt = PromptLoader.loadPrompt("wiki/transformation-system");
+        boolean wantJson = "json".equalsIgnoreCase(transformation.getOutputFormat());
+
+        String systemPrompt = PromptLoader.loadPrompt(
+                wantJson ? "wiki/transformation-system-json" : "wiki/transformation-system");
         String instruction = (transformation.getPromptTemplate() == null ? "" : transformation.getPromptTemplate())
                 .replace("{input_text}", trimmedInput)
                 .replace("{title}", sourceTitle);
@@ -220,6 +227,31 @@ public class WikiTransformationExecutor {
         ChatModel chatModel = buildChatModel(resolvedModelId);
         run.setModelId(resolvedModelId);
 
+        String output = callOnce(chatModel, systemPrompt, userPrompt);
+        if (wantJson) {
+            String coerced = coerceToJson(output);
+            if (coerced != null) {
+                // Wrap in a fenced block so UI rendering and save-as-page
+                // keep the existing markdown contract. The raw JSON is the
+                // first thing inside the block, so downstream tools can grep.
+                return "```json\n" + coerced + "\n```";
+            }
+            // One retry with an explicit nudge.
+            log.info("[WikiTransformation] JSON parse failed for template={}; retrying with stricter reminder",
+                    transformation.getName());
+            String retryUserPrompt = userPrompt + "\n\n---\n\n上一次回复不是合法 JSON。请只返回一个合法 JSON 文档，前后不要有任何文字或代码块标记。";
+            String retry = callOnce(chatModel, systemPrompt, retryUserPrompt);
+            String coercedRetry = coerceToJson(retry);
+            if (coercedRetry != null) {
+                return "```json\n" + coercedRetry + "\n```";
+            }
+            throw new IllegalStateException("LLM output is not valid JSON after one retry");
+        }
+        return output;
+    }
+
+    /** One LLM call, returns the cleaned output. Throws when the call yields blank. */
+    private String callOnce(ChatModel chatModel, String systemPrompt, String userPrompt) {
         ChatResponse resp = chatModel.call(new Prompt(List.of(
                 new SystemMessage(systemPrompt), new UserMessage(userPrompt))));
         String rawOutput = (resp == null || resp.getResult() == null
@@ -235,6 +267,44 @@ public class WikiTransformationExecutor {
         return output;
     }
 
+    private static final com.fasterxml.jackson.databind.ObjectMapper JSON_MAPPER =
+            new com.fasterxml.jackson.databind.ObjectMapper();
+
+    /**
+     * Try to parse the output as JSON. If the LLM wrapped it in a fenced
+     * block or sprinkled prose around it, fall back to finding the outer
+     * '{' / '[' brackets and try again. Returns the normalized JSON string
+     * on success, {@code null} on failure.
+     */
+    private static String coerceToJson(String text) {
+        if (text == null || text.isBlank()) return null;
+        String candidate = text.trim();
+        try {
+            JSON_MAPPER.readTree(candidate);
+            return candidate;
+        } catch (Exception ignored) {
+            // fall through to bracket-trim attempt
+        }
+        int objStart = candidate.indexOf('{');
+        int arrStart = candidate.indexOf('[');
+        int start;
+        char open;
+        if (objStart < 0) { start = arrStart; open = '['; }
+        else if (arrStart < 0) { start = objStart; open = '{'; }
+        else { start = Math.min(objStart, arrStart); open = candidate.charAt(start); }
+        if (start < 0) return null;
+        char close = open == '{' ? '}' : ']';
+        int end = candidate.lastIndexOf(close);
+        if (end <= start) return null;
+        String trimmed = candidate.substring(start, end + 1);
+        try {
+            JSON_MAPPER.readTree(trimmed);
+            return trimmed;
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
     /**
      * Run the transformation against the given raw material and persist
      * the outcome. The returned entity is the persisted run row, regardless

mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationService.java

@@ -110,6 +110,7 @@ public class WikiTransformationService {
         // create and update paths accept the same payload from the UI.
         entity.setModelId(input.getModelId() != null && input.getModelId() < 0 ? null : input.getModelId());
         entity.setOutputTarget(normalizeOutputTarget(input.getOutputTarget()));
+        entity.setOutputFormat(normalizeOutputFormat(input.getOutputFormat()));
         transformationMapper.insert(entity);
         log.info("[WikiTransformation] created id={} name={} kbId={}",
                 entity.getId(), entity.getName(), entity.getKbId());
@@ -135,6 +136,9 @@ public class WikiTransformationService {
         if (patch.getOutputTarget() != null) {
             entity.setOutputTarget(normalizeOutputTarget(patch.getOutputTarget()));
         }
+        if (patch.getOutputFormat() != null) {
+            entity.setOutputFormat(normalizeOutputFormat(patch.getOutputFormat()));
+        }
         transformationMapper.updateById(entity);
         return entity;
     }
@@ -149,6 +153,16 @@ public class WikiTransformationService {
         };
     }
 
+    /** Whitelist incoming outputFormat; unknown / null = "markdown". */
+    private static String normalizeOutputFormat(String raw) {
+        if (raw == null) return "markdown";
+        String trimmed = raw.trim().toLowerCase();
+        return switch (trimmed) {
+            case "json" -> "json";
+            default -> "markdown";
+        };
+    }
+
     @Transactional
     public void delete(Long id) {
         transformationMapper.deleteById(id);

mateclaw-server/src/main/resources/db/migration/h2/V109__wiki_transformation_output_format.sql

@@ -0,0 +1,8 @@
+-- Output format declared on the template so the executor can validate the
+-- LLM's response shape. 'markdown' (default) keeps the legacy behaviour
+-- where output is treated as Markdown and saved as page content; 'json'
+-- asks the LLM for a single JSON object and the executor parses + validates
+-- before persisting. Future formats (table, yaml) can extend this column.
+
+ALTER TABLE mate_wiki_transformation
+    ADD COLUMN IF NOT EXISTS output_format VARCHAR(16) NOT NULL DEFAULT 'markdown';

mateclaw-server/src/main/resources/db/migration/mysql/V109__wiki_transformation_output_format.sql

@@ -0,0 +1,11 @@
+-- Output format declared on the template. See h2 sibling for the prose
+-- explanation. MySQL needs the INFORMATION_SCHEMA guard pattern.
+
+SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
+           WHERE TABLE_SCHEMA = DATABASE()
+             AND TABLE_NAME = 'mate_wiki_transformation'
+             AND COLUMN_NAME = 'output_format');
+SET @s := IF(@c = 0,
+    'ALTER TABLE mate_wiki_transformation ADD COLUMN output_format VARCHAR(16) NOT NULL DEFAULT ''markdown''',
+    'SELECT 1');
+PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;

mateclaw-server/src/main/resources/prompts/wiki/transformation-system-json.txt

@@ -0,0 +1,16 @@
+You are a content transformation worker producing structured JSON. The user
+supplies (a) a transformation instruction and (b) a source text. Follow
+the instruction precisely and return exactly one valid JSON document.
+
+Rules:
+- Return ONLY a JSON document — no prose, no commentary, no markdown code
+  fences. The first character of your reply must be `{` or `[`.
+- Do not add framing like "Here is the JSON:" — emit the JSON object alone.
+- Use only JSON-valid escapes; double-quote all strings.
+- Do not invent facts beyond the supplied source text. If the source is
+  empty, return `{"error": "empty source"}`.
+- Preserve the language of the source text in string values unless the
+  instruction explicitly says otherwise.
+- If the instruction describes a schema (fields, arrays, types), follow
+  it exactly; missing values get an empty string or `null` per JSON
+  convention rather than being omitted entirely.

mateclaw-ui/src/api/index.ts

@@ -691,6 +691,7 @@ export const wikiApi = {
     enabled?: boolean
     modelId?: number | null
     outputTarget?: 'none' | 'page'
+    outputFormat?: 'markdown' | 'json'
   }) =>
     http.post('/wiki/transformations', data),
   updateTransformation: (id: number, data: {
@@ -701,6 +702,7 @@ export const wikiApi = {
     enabled?: boolean
     modelId?: number | null
     outputTarget?: 'none' | 'page'
+    outputFormat?: 'markdown' | 'json'
   }) =>
     http.put(`/wiki/transformations/${id}`, data),
   deleteTransformation: (id: number) =>

mateclaw-ui/src/i18n/locales/en-US.ts

@@ -1818,6 +1818,10 @@ export default {
       outputTargetNone: 'None — output stays in run history',
       outputTargetPage: 'Save as wiki page (searchable / agent-accessible / linkable)',
       outputTargetPageBadge: 'Auto-save to page',
+      outputFormatLabel: 'Output format',
+      outputFormatMarkdown: 'Markdown (default, human-readable)',
+      outputFormatJson: 'JSON (structured for downstream tools; one auto-retry on parse failure)',
+      outputFormatJsonBadge: 'JSON',
       saveAsPageBtn: 'Save as page',
       saving: 'Saving…',
       savedAsPage: 'Saved as:',

mateclaw-ui/src/i18n/locales/zh-CN.ts

@@ -1830,6 +1830,10 @@ export default {
       outputTargetNone: '不保存（仅留在运行历史）',
       outputTargetPage: '保存为 Wiki 页面（可被搜索 / Agent / 关系图引用）',
       outputTargetPageBadge: '自动落页',
+      outputFormatLabel: '输出格式',
+      outputFormatMarkdown: 'Markdown（默认，适合人读）',
+      outputFormatJson: 'JSON（结构化，便于下游程序消费；失败会自动重试 1 次）',
+      outputFormatJsonBadge: 'JSON',
       saveAsPageBtn: '保存为页面',
       saving: '保存中…',
       savedAsPage: '已保存：',

mateclaw-ui/src/views/Wiki/components/TransformationsPanel.vue

@@ -41,6 +41,9 @@
             <span v-if="tpl.outputTarget === 'page'" class="flag flag--on">
               {{ t('wiki.transformations.outputTargetPageBadge') }}
             </span>
+            <span v-if="tpl.outputFormat === 'json'" class="flag flag--scope">
+              {{ t('wiki.transformations.outputFormatJsonBadge') }}
+            </span>
             <span v-if="tpl.modelId" class="flag flag--scope">
               {{ modelLabelFor(tpl.modelId) }}
             </span>
@@ -232,6 +235,18 @@
               <span>{{ t('wiki.transformations.outputTargetPage') }}</span>
             </label>
           </fieldset>
+
+          <fieldset class="field field--group">
+            <legend class="field-label">{{ t('wiki.transformations.outputFormatLabel') }}</legend>
+            <label class="radio-row">
+              <input type="radio" value="markdown" v-model="form.outputFormat" />
+              <span>{{ t('wiki.transformations.outputFormatMarkdown') }}</span>
+            </label>
+            <label class="radio-row">
+              <input type="radio" value="json" v-model="form.outputFormat" />
+              <span>{{ t('wiki.transformations.outputFormatJson') }}</span>
+            </label>
+          </fieldset>
         </div>
 
         <div class="modal-actions">
@@ -265,6 +280,7 @@ interface WikiTransformation {
   enabled: boolean
   modelId: number | null
   outputTarget: 'none' | 'page' | null
+  outputFormat: 'markdown' | 'json' | null
 }
 
 interface WikiTransformationRun {
@@ -314,6 +330,7 @@ const form = reactive<{
   applyDefault: boolean
   enabled: boolean
   outputTarget: 'none' | 'page'
+  outputFormat: 'markdown' | 'json'
   modelId: number | null
 }>({
   name: '',
@@ -323,6 +340,7 @@ const form = reactive<{
   applyDefault: false,
   enabled: true,
   outputTarget: 'none',
+  outputFormat: 'markdown',
   modelId: null,
 })
 
@@ -409,6 +427,7 @@ function openCreate() {
   form.applyDefault = false
   form.enabled = true
   form.outputTarget = 'none'
+  form.outputFormat = 'markdown'
   form.modelId = null
   editorOpen.value = true
   ensureModelsLoaded()
@@ -423,6 +442,7 @@ function openEdit(tpl: WikiTransformation) {
   form.applyDefault = tpl.applyDefault
   form.enabled = tpl.enabled !== false
   form.outputTarget = tpl.outputTarget === 'page' ? 'page' : 'none'
+  form.outputFormat = tpl.outputFormat === 'json' ? 'json' : 'markdown'
   form.modelId = tpl.modelId ?? null
   editorOpen.value = true
   ensureModelsLoaded()
@@ -451,6 +471,7 @@ async function onSave() {
         applyDefault: form.applyDefault,
         enabled: form.enabled,
         outputTarget: form.outputTarget,
+        outputFormat: form.outputFormat,
         modelId: updateModelId,
       })
     } else {
@@ -463,6 +484,7 @@ async function onSave() {
         applyDefault: form.applyDefault,
         enabled: form.enabled,
         outputTarget: form.outputTarget,
+        outputFormat: form.outputFormat,
         modelId: form.modelId,
       })
     }