diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiTransformationController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiTransformationController.java new file mode 100644 index 00000000..d2d78c91 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiTransformationController.java @@ -0,0 +1,221 @@ +package vip.mate.wiki.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.*; +import vip.mate.common.result.R; +import vip.mate.exception.MateClawException; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.model.WikiTransformationEntity; +import vip.mate.wiki.model.WikiTransformationRunEntity; +import vip.mate.wiki.service.WikiKnowledgeBaseService; +import vip.mate.wiki.service.WikiTransformationExecutor; +import vip.mate.wiki.service.WikiTransformationService; +import vip.mate.workspace.core.annotation.RequireWorkspaceRole; + +import java.util.List; +import java.util.Map; + +/** + * Management surface for user-defined wiki transformation templates and + * their execution history. Templates live under the workspace; a template + * with non-null {@code kbId} is pinned to a single KB, otherwise it is + * available to every KB in the workspace. + */ +@Slf4j +@Tag(name = "Wiki Transformations", + description = "User-defined prompt templates run over wiki raw materials") +@RestController +@RequestMapping("/api/v1/wiki/transformations") +@RequiredArgsConstructor +public class WikiTransformationController { + + private final WikiTransformationService transformationService; + private final WikiTransformationExecutor executor; + private final WikiKnowledgeBaseService kbService; + + // ==================== Templates ==================== + + @RequireWorkspaceRole("viewer") + @Operation(summary = "List transformations available to a KB", + description = "Returns templates pinned to the KB plus workspace-wide templates.") + @GetMapping + public R> list( + @RequestParam(required = false) Long kbId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + long wsId = workspaceId != null ? workspaceId : 1L; + if (kbId != null) { + verifyKBWorkspace(kbId, wsId); + return R.ok(transformationService.listForKb(kbId, wsId)); + } + return R.ok(transformationService.listByWorkspace(wsId)); + } + + @RequireWorkspaceRole("viewer") + @GetMapping("/{id}") + public R get(@PathVariable Long id, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationEntity t = transformationService.getById(id); + if (t == null) return R.fail("Transformation not found"); + verifyTemplateWorkspace(t, workspaceId); + return R.ok(t); + } + + @RequireWorkspaceRole("member") + @PostMapping + public R create(@RequestBody WikiTransformationEntity body, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + long wsId = workspaceId != null ? workspaceId : 1L; + if (body.getKbId() != null) { + verifyKBWorkspace(body.getKbId(), wsId); + } + body.setWorkspaceId(wsId); + WikiTransformationEntity created = transformationService.create(body); + return R.ok(created); + } + + @RequireWorkspaceRole("member") + @PutMapping("/{id}") + public R update(@PathVariable Long id, + @RequestBody WikiTransformationEntity body, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationEntity existing = transformationService.getById(id); + if (existing == null) return R.fail("Transformation not found"); + verifyTemplateWorkspace(existing, workspaceId); + return R.ok(transformationService.update(id, body)); + } + + @RequireWorkspaceRole("member") + @DeleteMapping("/{id}") + public R delete(@PathVariable Long id, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationEntity existing = transformationService.getById(id); + if (existing != null) { + verifyTemplateWorkspace(existing, workspaceId); + transformationService.delete(id); + } + return R.ok(); + } + + // ==================== Apply ==================== + + @RequireWorkspaceRole("member") + @Operation(summary = "Run a transformation against a raw material", + description = "Set sync=true to block until the LLM call returns " + + "(the response carries the populated run). " + + "When false (default) the call returns immediately with the pending run row.") + @PostMapping("/{id}/apply") + public R apply(@PathVariable Long id, + @RequestBody Map body, + @RequestParam(defaultValue = "false") boolean sync, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationEntity t = transformationService.getById(id); + if (t == null) return R.fail("Transformation not found"); + verifyTemplateWorkspace(t, workspaceId); + + Object rawIdRaw = body == null ? null : body.get("rawId"); + if (rawIdRaw == null) { + return R.fail("rawId is required"); + } + Long rawId = Long.valueOf(rawIdRaw.toString()); + + if (sync) { + return R.ok(executor.runOnRawSync(t, rawId, "manual")); + } + executor.runOnRawAsync(t, rawId, "manual"); + // Async path — caller polls /runs to see the result land. + return R.ok(); + } + + // ==================== Runs ==================== + + @RequireWorkspaceRole("viewer") + @GetMapping("/runs/{runId}") + public R getRun(@PathVariable Long runId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationRunEntity run = transformationService.getRun(runId); + if (run == null) return R.fail("Run not found"); + verifyKBWorkspace(run.getKbId(), workspaceId != null ? workspaceId : 1L); + return R.ok(run); + } + + @RequireWorkspaceRole("viewer") + @GetMapping("/runs") + public R> listRuns( + @RequestParam(required = false) Long rawId, + @RequestParam(required = false) Long kbId, + @RequestParam(required = false) Long transformationId, + @RequestParam(defaultValue = "50") int limit, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + long wsId = workspaceId != null ? workspaceId : 1L; + if (rawId != null) { + return R.ok(transformationService.listRunsByRaw(rawId, limit)); + } + if (transformationId != null) { + WikiTransformationEntity t = transformationService.getById(transformationId); + if (t == null) return R.fail("Transformation not found"); + verifyTemplateWorkspace(t, wsId); + return R.ok(transformationService.listRunsByTransformation(transformationId, limit)); + } + if (kbId != null) { + verifyKBWorkspace(kbId, wsId); + return R.ok(transformationService.listRunsByKb(kbId, limit)); + } + return R.fail("One of rawId / kbId / transformationId is required"); + } + + @RequireWorkspaceRole("member") + @Operation(summary = "Save a completed run's output as a synthesis wiki page", + description = "Idempotent: re-saving an already-saved run updates the same page slug.") + @PostMapping("/runs/{runId}/save-as-page") + public R> saveRunAsPage(@PathVariable Long runId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationRunEntity run = transformationService.getRun(runId); + if (run == null) return R.fail("Run not found"); + verifyKBWorkspace(run.getKbId(), workspaceId != null ? workspaceId : 1L); + try { + var page = executor.manualSaveRunAsPage(runId); + if (page == null) return R.fail("Page service unavailable"); + return R.ok(Map.of( + "pageId", page.getId(), + "slug", page.getSlug(), + "title", page.getTitle())); + } catch (IllegalStateException | IllegalArgumentException e) { + return R.fail(e.getMessage()); + } + } + + @RequireWorkspaceRole("member") + @DeleteMapping("/runs/{runId}") + public R deleteRun(@PathVariable Long runId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationRunEntity run = transformationService.getRun(runId); + if (run != null) { + verifyKBWorkspace(run.getKbId(), workspaceId != null ? workspaceId : 1L); + transformationService.deleteRun(runId); + } + return R.ok(); + } + + // ==================== helpers ==================== + + private void verifyKBWorkspace(Long kbId, Long workspaceId) { + WikiKnowledgeBaseEntity kb = kbService.getById(kbId); + if (kb == null) { + throw new MateClawException("Knowledge base not found"); + } + long wsId = workspaceId != null ? workspaceId : 1L; + if (kb.getWorkspaceId() != null && !kb.getWorkspaceId().equals(wsId)) { + throw new MateClawException("err.common.wrong_workspace", "Resource does not belong to current workspace"); + } + } + + private void verifyTemplateWorkspace(WikiTransformationEntity t, Long workspaceId) { + long wsId = workspaceId != null ? workspaceId : 1L; + if (t.getWorkspaceId() != null && !t.getWorkspaceId().equals(wsId)) { + throw new MateClawException("err.common.wrong_workspace", "Resource does not belong to current workspace"); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationEntity.java new file mode 100644 index 00000000..83c959cc --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationEntity.java @@ -0,0 +1,77 @@ +package vip.mate.wiki.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * User-defined prompt template applied to a knowledge base's raw materials + * (and, eventually, pages). One template + one source = one + * {@link WikiTransformationRunEntity}. + * + *

Template body supports the placeholders {@code {input_text}} and + * {@code {title}}, replaced by the executor before the LLM call. + */ +@Data +@TableName("mate_wiki_transformation") +public class WikiTransformationEntity { + + @TableId(type = IdType.AUTO) + private Long id; + + /** + * Pinned KB. {@code null} means the template is available to every KB + * in the workspace. + */ + private Long kbId; + + private Long workspaceId; + + /** Stable short identifier; unique per {@code kbId}. */ + private String name; + + private String title; + + private String description; + + /** Prompt body with {@code {input_text}} / {@code {title}} placeholders. */ + private String promptTemplate; + + /** + * When true, the ingestion pipeline fires this template automatically + * for every raw material that lands in {@code completed} for a matching + * KB. + */ + private Boolean applyDefault; + + /** Optional explicit model override; {@code null} = use KB default. */ + private Long modelId; + + private Boolean enabled; + + /** + * Where the output of a successful run lands. + *

    + *
  • {@code none} — output stays in the run history only (default).
    • + *
  • {@code page} — output is upserted as a synthesis wiki page on the + * same KB; subsequent runs against the same source raw material + * update the same page rather than spawning duplicates.
    • + *
+ */ + private String outputTarget; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + @TableLogic + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationRunEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationRunEntity.java new file mode 100644 index 00000000..25ef9f99 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationRunEntity.java @@ -0,0 +1,69 @@ +package vip.mate.wiki.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * One execution of a {@link WikiTransformationEntity} against a source + * (raw material today; pages in a follow-up). Output is stored inline so + * the UI can render the result without re-running the LLM. + */ +@Data +@TableName("mate_wiki_transformation_run") +public class WikiTransformationRunEntity { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long transformationId; + private Long kbId; + private Long workspaceId; + + /** {@code raw} | {@code page} | {@code text}. */ + private String inputKind; + + private Long rawId; + private Long pageId; + + /** {@code pending} | {@code running} | {@code completed} | {@code failed}. */ + private String status; + + /** LLM output; treat as Markdown unless the prompt asked for JSON. */ + private String output; + + private String error; + + /** Model that actually produced the output after routing fallback. */ + private Long modelId; + + /** {@code apply_default} | {@code manual} | {@code agent_tool}. */ + private String triggeredBy; + + private LocalDateTime startedAt; + private LocalDateTime completedAt; + private Long durationMs; + + /** + * Set when the run was persisted as a synthesis wiki page (either + * automatically because the template's {@code outputTarget} is {@code page}, + * or manually via the save-as-page endpoint). Points at + * {@code mate_wiki_page.id}. + */ + private Long outputPageId; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + @TableLogic + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationMapper.java b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationMapper.java new file mode 100644 index 00000000..667b8c3c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationMapper.java @@ -0,0 +1,9 @@ +package vip.mate.wiki.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.wiki.model.WikiTransformationEntity; + +@Mapper +public interface WikiTransformationMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationRunMapper.java b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationRunMapper.java new file mode 100644 index 00000000..e08bdc7e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationRunMapper.java @@ -0,0 +1,9 @@ +package vip.mate.wiki.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.wiki.model.WikiTransformationRunEntity; + +@Mapper +public interface WikiTransformationRunMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index d23761f9..996e7e4c 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -114,6 +114,15 @@ public class WikiProcessingService { @org.springframework.beans.factory.annotation.Autowired(required = false) private WikiLogService logService; + /** + * Optional. When present, every successful ingest triggers an async sweep + * of the KB's apply-default transformation templates. Missing in the + * legacy unit tests that wire this service directly. + */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + @org.springframework.context.annotation.Lazy + private WikiTransformationExecutor transformationExecutor; + /** Parallel chunk / material processing executor (JDK 21 virtual threads) */ public static final ExecutorService WIKI_EXECUTOR = Executors.newVirtualThreadPerTaskExecutor(); @@ -412,6 +421,14 @@ public class WikiProcessingService { eventPublisher.publishEvent(new vip.mate.wiki.event.WikiKbDirtyEvent(this, kb.getId())); } + // Run apply-default transformation templates against the newly + // ingested raw material. Fire-and-forget; failures are logged + // inside the executor and do not affect the ingest outcome. + if (transformationExecutor != null && nonTerminalSideEffects) { + Long wsId = kb.getWorkspaceId() == null ? 1L : kb.getWorkspaceId(); + transformationExecutor.runDefaultsAsync(kb.getId(), wsId, rawId, "apply_default"); + } + log.info("[Wiki] Processing completed for raw={}, kbId={}, generatedPages={}, totalPages={}", rawId, kb.getId(), totalPages, pageCount); diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java new file mode 100644 index 00000000..1ae7dff0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java @@ -0,0 +1,323 @@ +package vip.mate.wiki.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import vip.mate.agent.prompt.PromptLoader; +import vip.mate.wiki.job.WikiJobStep; +import vip.mate.wiki.job.WikiModelRoutingService; +import vip.mate.wiki.metrics.WikiMetrics; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.model.WikiRawMaterialEntity; +import vip.mate.wiki.model.WikiTransformationEntity; +import vip.mate.wiki.model.WikiTransformationRunEntity; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Runs a single {@link WikiTransformationEntity} against a source raw + * material: substitutes placeholders into the user-defined prompt, calls + * the configured chat model, and persists the run row with the output. + * + *

Sync entry point: {@link #runOnRawSync}. Async fire-and-forget + * helpers (used by the ingest-pipeline hook and the controller's "apply" + * endpoint when the caller doesn't want to block) wrap that on a virtual + * thread executor. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiTransformationExecutor { + + /** Virtual-thread pool — matches the WIKI_EXECUTOR pattern used elsewhere in the module. */ + private static final ExecutorService WORKER = Executors.newVirtualThreadPerTaskExecutor(); + + /** Hard cap on input text fed into the prompt (defensive against multi-MB extracted PDFs). */ + private static final int MAX_INPUT_CHARS = 60_000; + + private final WikiTransformationService transformationService; + private final WikiRawMaterialService rawService; + private final WikiMetrics metrics; + + @Autowired(required = false) + private WikiModelRoutingService modelRoutingService; + + /** Optional. When wired, completed runs whose template has + * {@code outputTarget=page} are persisted as a synthesis wiki page. */ + @Autowired(required = false) + private WikiPageService pageService; + + private final com.fasterxml.jackson.databind.ObjectMapper objectMapper = + new com.fasterxml.jackson.databind.ObjectMapper(); + + public CompletableFuture runDefaultsAsync(Long kbId, Long workspaceId, Long rawId, String triggeredBy) { + return CompletableFuture.runAsync(() -> { + try { + List defaults = + transformationService.listApplyDefaultsForKb(kbId, workspaceId); + for (WikiTransformationEntity t : defaults) { + try { + runOnRawSync(t, rawId, triggeredBy); + } catch (Exception e) { + log.warn("[WikiTransformation] default run failed transformation={} rawId={}: {}", + t.getName(), rawId, e.getMessage()); + } + } + } catch (Exception e) { + log.warn("[WikiTransformation] default sweep failed kbId={} rawId={}: {}", + kbId, rawId, e.getMessage()); + } + }, WORKER); + } + + public CompletableFuture runOnRawAsync( + WikiTransformationEntity transformation, Long rawId, String triggeredBy) { + return CompletableFuture.supplyAsync( + () -> runOnRawSync(transformation, rawId, triggeredBy), WORKER); + } + + /** + * Run the transformation against the given raw material and persist + * the outcome. The returned entity is the persisted run row, regardless + * of success or failure (failure leaves {@code status=failed} and + * {@code error} populated). + */ + public WikiTransformationRunEntity runOnRawSync( + WikiTransformationEntity transformation, Long rawId, String triggeredBy) { + if (transformation == null) { + throw new IllegalArgumentException("transformation is required"); + } + if (rawId == null) { + throw new IllegalArgumentException("rawId is required"); + } + WikiRawMaterialEntity raw = rawService.getById(rawId); + if (raw == null) { + throw new IllegalArgumentException("Raw material not found: " + rawId); + } + if (Boolean.FALSE.equals(transformation.getEnabled())) { + log.debug("[WikiTransformation] skipping disabled template id={} name={}", + transformation.getId(), transformation.getName()); + return null; + } + + long startNanos = System.nanoTime(); + WikiTransformationRunEntity run = new WikiTransformationRunEntity(); + run.setTransformationId(transformation.getId()); + run.setKbId(raw.getKbId()); + run.setWorkspaceId(transformation.getWorkspaceId()); + run.setInputKind("raw"); + run.setRawId(rawId); + run.setStatus("running"); + run.setTriggeredBy(triggeredBy == null ? "manual" : triggeredBy); + run.setStartedAt(LocalDateTime.now()); + transformationService.insertRun(run); + + try { + String inputText = rawService.getTextContent(raw); + if (inputText == null || inputText.isBlank()) { + throw new IllegalStateException("Raw material has no extractable text yet"); + } + String trimmedInput = inputText.length() > MAX_INPUT_CHARS + ? inputText.substring(0, MAX_INPUT_CHARS) + "\n…(truncated)" + : inputText; + + String systemPrompt = PromptLoader.loadPrompt("wiki/transformation-system"); + String userPrompt = PromptLoader.loadPrompt("wiki/transformation-user") + .replace("{instruction}", renderTemplate(transformation.getPromptTemplate(), raw, trimmedInput)) + .replace("{source_title}", safeTitle(raw)) + .replace("{source_text}", trimmedInput); + + Long resolvedModelId = resolveModelId(transformation, raw.getKbId()); + ChatModel chatModel = buildChatModel(resolvedModelId); + run.setModelId(resolvedModelId); + + ChatResponse resp = chatModel.call(new Prompt(List.of( + new SystemMessage(systemPrompt), new UserMessage(userPrompt)))); + String output = (resp == null || resp.getResult() == null + || resp.getResult().getOutput() == null) + ? null : resp.getResult().getOutput().getText(); + if (output == null || output.isBlank()) { + throw new IllegalStateException("LLM returned empty output"); + } + run.setOutput(output); + run.setStatus("completed"); + run.setCompletedAt(LocalDateTime.now()); + run.setDurationMs(Duration.ofNanos(System.nanoTime() - startNanos).toMillis()); + + // Persist as a synthesis wiki page when the template asks for it. + // Failures here are logged but do not flip the run back to failed: + // the LLM output is already valid, the page-write is best-effort. + if ("page".equalsIgnoreCase(transformation.getOutputTarget())) { + try { + WikiPageEntity page = saveRunAsPage(run, transformation, raw, output); + if (page != null) run.setOutputPageId(page.getId()); + } catch (Exception pe) { + log.warn("[WikiTransformation] auto-save as page failed run={}: {}", + run.getId(), pe.getMessage()); + } + } + transformationService.updateRun(run); + + metrics.recordCompileStage("transformation_run", raw.getKbId(), + Duration.ofNanos(System.nanoTime() - startNanos)); + log.info("[WikiTransformation] ok run={} transformation={} rawId={} kbId={} ({} ms, pageId={})", + run.getId(), transformation.getName(), rawId, raw.getKbId(), + run.getDurationMs(), run.getOutputPageId()); + } catch (Exception e) { + run.setStatus("failed"); + String msg = e.getMessage(); + run.setError(msg == null ? e.getClass().getSimpleName() : msg); + run.setCompletedAt(LocalDateTime.now()); + run.setDurationMs(Duration.ofNanos(System.nanoTime() - startNanos).toMillis()); + transformationService.updateRun(run); + log.warn("[WikiTransformation] failed run={} transformation={} rawId={}: {}", + run.getId(), transformation.getName(), rawId, msg); + } + return run; + } + + private String renderTemplate(String template, WikiRawMaterialEntity raw, String inputText) { + if (template == null) return ""; + String result = template; + result = result.replace("{input_text}", inputText); + result = result.replace("{title}", safeTitle(raw)); + return result; + } + + private Long resolveModelId(WikiTransformationEntity transformation, Long kbId) { + if (transformation.getModelId() != null) return transformation.getModelId(); + if (modelRoutingService == null) { + throw new IllegalStateException("No model bound on transformation and ModelRoutingService unavailable"); + } + return modelRoutingService.selectModelId(kbId, "heavy_ingest", WikiJobStep.CREATE_PAGE); + } + + private ChatModel buildChatModel(Long modelId) { + if (modelRoutingService == null) { + throw new IllegalStateException("ModelRoutingService unavailable; cannot run transformation"); + } + return modelRoutingService.buildChatModel(modelId); + } + + private static String safeTitle(WikiRawMaterialEntity raw) { + String t = raw.getTitle(); + return (t == null || t.isBlank()) ? ("raw#" + raw.getId()) : t; + } + + // ==================== Save-as-page ==================== + + /** + * Manual entry point used by the {@code POST /runs/{runId}/save-as-page} + * endpoint. Loads the run + its template + its source raw material, + * delegates to {@link #saveRunAsPage}, and updates the run row with the + * resulting page id so the UI can render a "saved as: …" affordance. + * + * @return the persisted page; never {@code null} on success + * @throws IllegalArgumentException when the run / raw / template is missing + * @throws IllegalStateException when the run is not completed or no output + */ + public WikiPageEntity manualSaveRunAsPage(Long runId) { + if (runId == null) throw new IllegalArgumentException("runId is required"); + WikiTransformationRunEntity run = transformationService.getRun(runId); + if (run == null) throw new IllegalArgumentException("Run not found: " + runId); + if (!"completed".equalsIgnoreCase(run.getStatus())) { + throw new IllegalStateException("Run is not completed (status=" + run.getStatus() + ")"); + } + if (run.getOutput() == null || run.getOutput().isBlank()) { + throw new IllegalStateException("Run has no output to save"); + } + if (run.getRawId() == null) { + throw new IllegalStateException("Run is not bound to a raw material"); + } + WikiTransformationEntity template = transformationService.getById(run.getTransformationId()); + if (template == null) { + throw new IllegalStateException("Transformation template no longer exists"); + } + WikiRawMaterialEntity raw = rawService.getById(run.getRawId()); + if (raw == null) { + throw new IllegalStateException("Source raw material no longer exists"); + } + WikiPageEntity page = saveRunAsPage(run, template, raw, run.getOutput()); + if (page != null) { + run.setOutputPageId(page.getId()); + transformationService.updateRun(run); + } + return page; + } + + /** + * Upsert the transformation output as a synthesis wiki page on the same + * KB. Slug is deterministic — {@code -} — + * so re-running an apply_default template against the same raw material + * updates the existing page in place rather than spawning duplicates. + */ + private WikiPageEntity saveRunAsPage(WikiTransformationRunEntity run, + WikiTransformationEntity template, + WikiRawMaterialEntity raw, + String output) { + if (pageService == null) { + log.warn("[WikiTransformation] save-as-page requested but WikiPageService not available"); + return null; + } + Long kbId = raw.getKbId(); + String slug = buildSlug(template, raw); + String title = template.getTitle() + " · " + safeTitle(raw); + String summary = deriveSummary(output); + String sourceRawIdsJson = toJsonArray(raw.getId()); + + WikiPageEntity existing = pageService.getBySlug(kbId, slug); + WikiPageEntity persisted; + if (existing == null) { + persisted = pageService.createPage(kbId, slug, title, output, summary, + sourceRawIdsJson, "synthesis"); + log.info("[WikiTransformation] saved run={} as new page slug={} pageId={}", + run.getId(), slug, persisted.getId()); + } else { + persisted = pageService.updatePageByAi(kbId, slug, output, summary, raw.getId()); + if (persisted == null) persisted = existing; + log.info("[WikiTransformation] updated existing synthesis page slug={} pageId={} from run={}", + slug, persisted.getId(), run.getId()); + } + return persisted; + } + + private static String buildSlug(WikiTransformationEntity template, WikiRawMaterialEntity raw) { + String rawPart = WikiPageService.toSlug(raw.getTitle()); + if (rawPart == null || rawPart.isBlank()) rawPart = "r" + raw.getId(); + return template.getName() + "-" + rawPart; + } + + /** First non-empty line of the output, capped to ~280 chars, used as page summary. */ + private static String deriveSummary(String output) { + if (output == null) return ""; + for (String line : output.split("\\n")) { + String trimmed = line.trim(); + if (trimmed.isEmpty()) continue; + if (trimmed.startsWith("#")) { + trimmed = trimmed.replaceAll("^#+\\s*", ""); + if (trimmed.isEmpty()) continue; + } + return trimmed.length() > 280 ? trimmed.substring(0, 280) + "…" : trimmed; + } + return ""; + } + + private String toJsonArray(Long rawId) { + try { + return objectMapper.writeValueAsString(java.util.List.of(rawId)); + } catch (Exception e) { + return "[" + rawId + "]"; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationService.java new file mode 100644 index 00000000..fecacb2f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationService.java @@ -0,0 +1,221 @@ +package vip.mate.wiki.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import vip.mate.wiki.model.WikiTransformationEntity; +import vip.mate.wiki.model.WikiTransformationRunEntity; +import vip.mate.wiki.repository.WikiTransformationMapper; +import vip.mate.wiki.repository.WikiTransformationRunMapper; + +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; + +/** + * CRUD + lookups for wiki transformation templates and their execution + * history. Pure persistence — the LLM call lives in + * {@link WikiTransformationExecutor}. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiTransformationService { + + private static final Pattern NAME_PATTERN = Pattern.compile("^[a-z0-9][a-z0-9-]{1,62}[a-z0-9]$"); + + private final WikiTransformationMapper transformationMapper; + private final WikiTransformationRunMapper runMapper; + + /** Templates visible to a KB: pinned to this KB plus workspace-wide ones (kb_id NULL). */ + public List listForKb(Long kbId, Long workspaceId) { + if (kbId == null) { + return List.of(); + } + return transformationMapper.selectList( + new LambdaQueryWrapper() + .and(w -> w.eq(WikiTransformationEntity::getKbId, kbId) + .or(g -> g.isNull(WikiTransformationEntity::getKbId) + .eq(WikiTransformationEntity::getWorkspaceId, workspaceId))) + .orderByDesc(WikiTransformationEntity::getUpdateTime)); + } + + public List listByWorkspace(Long workspaceId) { + return transformationMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiTransformationEntity::getWorkspaceId, workspaceId) + .orderByDesc(WikiTransformationEntity::getUpdateTime)); + } + + public WikiTransformationEntity getById(Long id) { + return transformationMapper.selectById(id); + } + + public Optional findByName(Long kbId, Long workspaceId, String name) { + if (name == null || name.isBlank()) return Optional.empty(); + // Prefer the KB-pinned record over a workspace-wide one of the same name. + WikiTransformationEntity pinned = transformationMapper.selectOne( + new LambdaQueryWrapper() + .eq(WikiTransformationEntity::getKbId, kbId) + .eq(WikiTransformationEntity::getName, name) + .last("LIMIT 1")); + if (pinned != null) return Optional.of(pinned); + WikiTransformationEntity global = transformationMapper.selectOne( + new LambdaQueryWrapper() + .isNull(WikiTransformationEntity::getKbId) + .eq(WikiTransformationEntity::getWorkspaceId, workspaceId) + .eq(WikiTransformationEntity::getName, name) + .last("LIMIT 1")); + return Optional.ofNullable(global); + } + + /** Default-apply templates that should run for a raw material in {@code kbId}. */ + public List listApplyDefaultsForKb(Long kbId, Long workspaceId) { + return listForKb(kbId, workspaceId).stream() + .filter(t -> Boolean.TRUE.equals(t.getApplyDefault())) + .filter(t -> !Boolean.FALSE.equals(t.getEnabled())) + .toList(); + } + + @Transactional + public WikiTransformationEntity create(WikiTransformationEntity input) { + validateName(input.getName()); + if (input.getTitle() == null || input.getTitle().isBlank()) { + throw new IllegalArgumentException("title is required"); + } + if (input.getPromptTemplate() == null || input.getPromptTemplate().isBlank()) { + throw new IllegalArgumentException("promptTemplate is required"); + } + Long workspaceId = input.getWorkspaceId() == null ? 1L : input.getWorkspaceId(); + + // Enforce uniqueness on (kbId, name) — including the NULL-kbId case + // where MySQL would otherwise allow duplicates. + findByExactScopeAndName(input.getKbId(), workspaceId, input.getName()) + .ifPresent(existing -> { + throw new IllegalArgumentException("Transformation already exists: " + input.getName()); + }); + + WikiTransformationEntity entity = new WikiTransformationEntity(); + entity.setKbId(input.getKbId()); + entity.setWorkspaceId(workspaceId); + entity.setName(input.getName()); + entity.setTitle(input.getTitle()); + entity.setDescription(input.getDescription()); + entity.setPromptTemplate(input.getPromptTemplate()); + entity.setApplyDefault(Boolean.TRUE.equals(input.getApplyDefault())); + entity.setEnabled(input.getEnabled() == null ? Boolean.TRUE : input.getEnabled()); + entity.setModelId(input.getModelId()); + entity.setOutputTarget(normalizeOutputTarget(input.getOutputTarget())); + transformationMapper.insert(entity); + log.info("[WikiTransformation] created id={} name={} kbId={}", + entity.getId(), entity.getName(), entity.getKbId()); + return entity; + } + + @Transactional + public WikiTransformationEntity update(Long id, WikiTransformationEntity patch) { + WikiTransformationEntity entity = transformationMapper.selectById(id); + if (entity == null) { + throw new IllegalArgumentException("Transformation not found: " + id); + } + if (patch.getTitle() != null) entity.setTitle(patch.getTitle()); + if (patch.getDescription() != null) entity.setDescription(patch.getDescription()); + if (patch.getPromptTemplate() != null) entity.setPromptTemplate(patch.getPromptTemplate()); + if (patch.getApplyDefault() != null) entity.setApplyDefault(patch.getApplyDefault()); + if (patch.getEnabled() != null) entity.setEnabled(patch.getEnabled()); + // modelId is allowed to be cleared via explicit -1 sentinel handled by controller; + // here we only honour non-null assignments. + if (patch.getModelId() != null) { + entity.setModelId(patch.getModelId() < 0 ? null : patch.getModelId()); + } + if (patch.getOutputTarget() != null) { + entity.setOutputTarget(normalizeOutputTarget(patch.getOutputTarget())); + } + transformationMapper.updateById(entity); + return entity; + } + + /** Whitelist incoming outputTarget; unknown / null = "none". */ + private static String normalizeOutputTarget(String raw) { + if (raw == null) return "none"; + String trimmed = raw.trim().toLowerCase(); + return switch (trimmed) { + case "page" -> "page"; + default -> "none"; + }; + } + + @Transactional + public void delete(Long id) { + transformationMapper.deleteById(id); + } + + // ==================== Runs ==================== + + public WikiTransformationRunEntity getRun(Long runId) { + return runMapper.selectById(runId); + } + + public List listRunsByRaw(Long rawId, int limit) { + return runMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiTransformationRunEntity::getRawId, rawId) + .orderByDesc(WikiTransformationRunEntity::getCreateTime) + .last("LIMIT " + Math.max(1, Math.min(limit, 200)))); + } + + public List listRunsByKb(Long kbId, int limit) { + return runMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiTransformationRunEntity::getKbId, kbId) + .orderByDesc(WikiTransformationRunEntity::getCreateTime) + .last("LIMIT " + Math.max(1, Math.min(limit, 200)))); + } + + public List listRunsByTransformation(Long transformationId, int limit) { + return runMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiTransformationRunEntity::getTransformationId, transformationId) + .orderByDesc(WikiTransformationRunEntity::getCreateTime) + .last("LIMIT " + Math.max(1, Math.min(limit, 200)))); + } + + @Transactional + public WikiTransformationRunEntity insertRun(WikiTransformationRunEntity run) { + runMapper.insert(run); + return run; + } + + @Transactional + public void updateRun(WikiTransformationRunEntity run) { + runMapper.updateById(run); + } + + @Transactional + public void deleteRun(Long runId) { + runMapper.deleteById(runId); + } + + // ==================== helpers ==================== + + private Optional findByExactScopeAndName(Long kbId, Long workspaceId, String name) { + LambdaQueryWrapper q = new LambdaQueryWrapper<>(); + if (kbId == null) { + q.isNull(WikiTransformationEntity::getKbId) + .eq(WikiTransformationEntity::getWorkspaceId, workspaceId); + } else { + q.eq(WikiTransformationEntity::getKbId, kbId); + } + q.eq(WikiTransformationEntity::getName, name).last("LIMIT 1"); + return Optional.ofNullable(transformationMapper.selectOne(q)); + } + + private static void validateName(String name) { + if (name == null || !NAME_PATTERN.matcher(name).matches()) { + throw new IllegalArgumentException( + "name must be 3-64 chars, lowercase letters / digits / hyphens (start and end alphanumeric)"); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java index 3d8e7d5f..cfc1c2fa 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java @@ -18,6 +18,8 @@ import vip.mate.wiki.job.model.WikiProcessingJobEntity; import vip.mate.wiki.model.WikiKnowledgeBaseEntity; import vip.mate.wiki.model.WikiPageEntity; import vip.mate.wiki.model.WikiRawMaterialEntity; +import vip.mate.wiki.model.WikiTransformationEntity; +import vip.mate.wiki.model.WikiTransformationRunEntity; import vip.mate.wiki.repository.WikiRawMaterialMapper; import vip.mate.wiki.service.*; @@ -59,6 +61,13 @@ public class WikiTool { @Autowired(required = false) private WikiCompileService compileService; + /** Optional transformation engine. Tools degrade with a clear error when missing. */ + @Autowired(required = false) + private WikiTransformationService transformationService; + + @Autowired(required = false) + private WikiTransformationExecutor transformationExecutor; + public WikiTool(WikiPageService pageService, WikiKnowledgeBaseService kbService, WikiRawMaterialService rawService, @@ -632,6 +641,78 @@ public class WikiTool { return "Wikilink enrichment queued for: " + slug; } + // ==================== Transformations ==================== + + @Tool(description = """ + List the transformation templates available to this agent's wiki KB. + Each result has a name (use it with wiki_apply_transformation), a + human title, and a description of what the prompt produces. + """) + public String wiki_list_transformations( + @ToolParam(description = "Agent ID") Long agentId) { + Long kbId = resolveKbId(agentId); + if (kbId == null) return error("No wiki knowledge base found for this agent"); + if (transformationService == null) return error("Transformations not available"); + + WikiKnowledgeBaseEntity kb = kbService.getById(kbId); + Long wsId = (kb == null || kb.getWorkspaceId() == null) ? 1L : kb.getWorkspaceId(); + + List templates = transformationService.listForKb(kbId, wsId); + JSONArray arr = new JSONArray(); + for (WikiTransformationEntity t : templates) { + if (Boolean.FALSE.equals(t.getEnabled())) continue; + arr.add(JSONUtil.createObj() + .set("name", t.getName()) + .set("title", t.getTitle()) + .set("description", t.getDescription()) + .set("applyDefault", Boolean.TRUE.equals(t.getApplyDefault()))); + } + return JSONUtil.createObj().set("kbId", kbId).set("transformations", arr).toString(); + } + + @Tool(description = """ + Run a transformation template against one raw material and return the + generated text. Use wiki_list_transformations first to discover names. + The run is also persisted so the result is visible in the wiki UI. + """) + public String wiki_apply_transformation( + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Transformation name (from wiki_list_transformations)") String name, + @ToolParam(description = "Raw material ID to run the transformation against") Long rawId) { + if (name == null || name.isBlank()) return error("name is required"); + if (rawId == null) return error("rawId is required"); + Long kbId = resolveKbId(agentId); + if (kbId == null) return error("No wiki knowledge base found for this agent"); + if (transformationService == null || transformationExecutor == null) { + return error("Transformations not available"); + } + + WikiKnowledgeBaseEntity kb = kbService.getById(kbId); + Long wsId = (kb == null || kb.getWorkspaceId() == null) ? 1L : kb.getWorkspaceId(); + + WikiTransformationEntity template = transformationService.findByName(kbId, wsId, name).orElse(null); + if (template == null) return error("Transformation not found: " + name); + + try { + WikiTransformationRunEntity run = transformationExecutor.runOnRawSync(template, rawId, "agent_tool"); + if (run == null) return error("Transformation is disabled: " + name); + if ("failed".equals(run.getStatus())) { + return error("Transformation failed: " + run.getError()); + } + return JSONUtil.createObj() + .set("ok", true) + .set("runId", run.getId()) + .set("transformation", template.getName()) + .set("output", run.getOutput()) + .toString(); + } catch (IllegalStateException | IllegalArgumentException e) { + return error(e.getMessage()); + } catch (Exception e) { + log.warn("[WikiTool] wiki_apply_transformation failed: {}", e.getMessage()); + return error("Apply failed: " + e.getMessage()); + } + } + // ==================== Helpers ==================== private Long resolveKbId(Long agentId) { diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V105__wiki_transformation.sql b/mateclaw-server/src/main/resources/db/migration/h2/V105__wiki_transformation.sql new file mode 100644 index 00000000..8cbb83e2 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V105__wiki_transformation.sql @@ -0,0 +1,90 @@ +-- Reusable user-defined prompt templates ("transformations") that run over +-- a raw material's extracted text and persist the LLM output as an artifact +-- on the knowledge base. Templates can be flagged apply_default so the +-- ingestion pipeline runs them automatically once a raw material reaches +-- the completed state. Manual / agent-tool runs are also supported. +-- +-- mate_wiki_transformation — the template (prompt + metadata) +-- mate_wiki_transformation_run — one row per execution attempt + +CREATE TABLE IF NOT EXISTS mate_wiki_transformation ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + + -- NULL = workspace-wide template available to every KB in the workspace. + -- Non-NULL = pinned to a single KB. + kb_id BIGINT NULL, + + workspace_id BIGINT NOT NULL DEFAULT 1, + + -- Short stable identifier (e.g. "risk-extract"). Used by agent tools to + -- target a transformation without exposing numeric IDs. + name VARCHAR(64) NOT NULL, + + -- Human-readable label shown in the UI. + title VARCHAR(255) NOT NULL, + + description VARCHAR(1024), + + -- Prompt body. Placeholders supported by the executor: + -- {input_text} — extracted text of the source raw material + -- {title} — title of the source raw material + prompt_template CLOB NOT NULL, + + -- When true, the executor fires this transformation automatically for + -- every raw material that reaches completed in the matching KB. + apply_default BOOLEAN NOT NULL DEFAULT FALSE, + + -- Optional explicit model override. NULL = fall back to the KB-bound + -- chat model (same routing chain WikiCompileService uses). + model_id BIGINT NULL, + + enabled BOOLEAN NOT NULL DEFAULT TRUE, + + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted INT NOT NULL DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_wtr_kb ON mate_wiki_transformation (kb_id, deleted); +CREATE INDEX IF NOT EXISTS idx_wtr_ws ON mate_wiki_transformation (workspace_id, deleted); +CREATE UNIQUE INDEX IF NOT EXISTS uk_wtr_kb_name ON mate_wiki_transformation (kb_id, name, deleted); + + +CREATE TABLE IF NOT EXISTS mate_wiki_transformation_run ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + + transformation_id BIGINT NOT NULL, + kb_id BIGINT NOT NULL, + workspace_id BIGINT NOT NULL DEFAULT 1, + + -- Either raw_id or page_id is set; input_kind says which. + input_kind VARCHAR(16) NOT NULL, + raw_id BIGINT NULL, + page_id BIGINT NULL, + + -- pending | running | completed | failed + status VARCHAR(16) NOT NULL DEFAULT 'pending', + + -- LLM output. Treat as Markdown unless the prompt asked for JSON. + output CLOB, + + error VARCHAR(2048), + + -- Model that actually produced the output (after routing). + model_id BIGINT NULL, + + -- apply_default | manual | agent_tool + triggered_by VARCHAR(32) NOT NULL DEFAULT 'manual', + + started_at TIMESTAMP NULL, + completed_at TIMESTAMP NULL, + duration_ms BIGINT NULL, + + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted INT NOT NULL DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_wtrn_tr ON mate_wiki_transformation_run (transformation_id, deleted); +CREATE INDEX IF NOT EXISTS idx_wtrn_kb ON mate_wiki_transformation_run (kb_id, deleted); +CREATE INDEX IF NOT EXISTS idx_wtrn_raw ON mate_wiki_transformation_run (raw_id, deleted); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V106__wiki_transformation_output_target.sql b/mateclaw-server/src/main/resources/db/migration/h2/V106__wiki_transformation_output_target.sql new file mode 100644 index 00000000..3fd5f75f --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V106__wiki_transformation_output_target.sql @@ -0,0 +1,19 @@ +-- Two-part follow-up to V105 so a transformation's output can flow back +-- into the KB as a first-class artifact: +-- +-- 1. mate_wiki_transformation.output_target — declarative target for the +-- template's output. `none` = legacy behaviour (output stays in the run +-- history only). `page` = after a successful run, persist the output as +-- a synthesis wiki page derived from the source raw material. Runs an +-- upsert against a deterministic slug so re-running is idempotent. +-- +-- 2. mate_wiki_transformation_run.output_page_id — when a run was saved as +-- a page (either via apply_default=page or the manual save-as-page +-- endpoint), this points at mate_wiki_page.id so the UI can render a +-- "saved as: " link without a join through sourceRawIds. + +ALTER TABLE mate_wiki_transformation + ADD COLUMN IF NOT EXISTS output_target VARCHAR(16) NOT NULL DEFAULT 'none'; + +ALTER TABLE mate_wiki_transformation_run + ADD COLUMN IF NOT EXISTS output_page_id BIGINT NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V105__wiki_transformation.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V105__wiki_transformation.sql new file mode 100644 index 00000000..55c28709 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V105__wiki_transformation.sql @@ -0,0 +1,68 @@ +-- Reusable user-defined prompt templates ("transformations") that run over +-- a raw material's extracted text and persist the LLM output as an artifact +-- on the knowledge base. Templates can be flagged apply_default so the +-- ingestion pipeline runs them automatically once a raw material reaches +-- the completed state. Manual / agent-tool runs are also supported. + +CREATE TABLE IF NOT EXISTS mate_wiki_transformation ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + + kb_id BIGINT NULL, + workspace_id BIGINT NOT NULL DEFAULT 1, + + name VARCHAR(64) NOT NULL, + title VARCHAR(255) NOT NULL, + description VARCHAR(1024), + + prompt_template MEDIUMTEXT NOT NULL, + + apply_default TINYINT(1) NOT NULL DEFAULT 0, + model_id BIGINT NULL, + enabled TINYINT(1) NOT NULL DEFAULT 1, + + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + deleted TINYINT NOT NULL DEFAULT 0, + + KEY idx_wtr_kb (kb_id, deleted), + KEY idx_wtr_ws (workspace_id, deleted) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- Unique name per KB (NULL kb_id rows compete in a shared "global" bucket). +-- MySQL treats NULL as distinct in unique indexes, so workspace-wide names +-- can technically collide; the service layer enforces uniqueness for the +-- NULL-kb_id case in software. +CREATE UNIQUE INDEX uk_wtr_kb_name ON mate_wiki_transformation (kb_id, name, deleted); + + +CREATE TABLE IF NOT EXISTS mate_wiki_transformation_run ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + + transformation_id BIGINT NOT NULL, + kb_id BIGINT NOT NULL, + workspace_id BIGINT NOT NULL DEFAULT 1, + + input_kind VARCHAR(16) NOT NULL, + raw_id BIGINT NULL, + page_id BIGINT NULL, + + status VARCHAR(16) NOT NULL DEFAULT 'pending', + + output MEDIUMTEXT, + error VARCHAR(2048), + model_id BIGINT NULL, + + triggered_by VARCHAR(32) NOT NULL DEFAULT 'manual', + + started_at DATETIME(3) NULL, + completed_at DATETIME(3) NULL, + duration_ms BIGINT NULL, + + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + deleted TINYINT NOT NULL DEFAULT 0, + + KEY idx_wtrn_tr (transformation_id, deleted), + KEY idx_wtrn_kb (kb_id, deleted), + KEY idx_wtrn_raw (raw_id, deleted) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V106__wiki_transformation_output_target.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V106__wiki_transformation_output_target.sql new file mode 100644 index 00000000..44b0435b --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V106__wiki_transformation_output_target.sql @@ -0,0 +1,22 @@ +-- Two-part follow-up to V105 so a transformation's output can flow back +-- into the KB as a first-class artifact. See the h2 sibling migration for +-- the prose explanation. MySQL lacks `ADD COLUMN IF NOT EXISTS`, so each +-- column is guarded by an INFORMATION_SCHEMA check + prepared statement. + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_transformation' + AND COLUMN_NAME = 'output_target'); +SET @s := IF(@c = 0, + 'ALTER TABLE mate_wiki_transformation ADD COLUMN output_target VARCHAR(16) NOT NULL DEFAULT ''none''', + 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_transformation_run' + AND COLUMN_NAME = 'output_page_id'); +SET @s := IF(@c = 0, + 'ALTER TABLE mate_wiki_transformation_run ADD COLUMN output_page_id BIGINT NULL', + 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/prompts/wiki/transformation-system.txt b/mateclaw-server/src/main/resources/prompts/wiki/transformation-system.txt new file mode 100644 index 00000000..f80e9c5f --- /dev/null +++ b/mateclaw-server/src/main/resources/prompts/wiki/transformation-system.txt @@ -0,0 +1,10 @@ +You are a content transformation worker. The user supplies (a) a transformation +instruction and (b) a source text. Follow the instruction precisely and return +only the transformed content. + +Rules: +- Do not add framing such as "Here is the result:" — emit only the transformation output. +- If the instruction asks for JSON, return exactly one valid JSON document and nothing else. +- Otherwise return Markdown. +- Do not invent facts beyond the supplied source text. If the source is empty, return a one-line note saying so. +- Preserve the language of the source text unless the instruction explicitly says otherwise. diff --git a/mateclaw-server/src/main/resources/prompts/wiki/transformation-user.txt b/mateclaw-server/src/main/resources/prompts/wiki/transformation-user.txt new file mode 100644 index 00000000..c04d7895 --- /dev/null +++ b/mateclaw-server/src/main/resources/prompts/wiki/transformation-user.txt @@ -0,0 +1,7 @@ +## Instruction + +{instruction} + +## Source — {source_title} + +{source_text} diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index 6ff54a20..461cccdc 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -675,6 +675,46 @@ export const wikiApi = { // RFC-032: Search preview searchPreview: (kbId: number, data: { query: string; mode?: string; topK?: number }) => http.post(`/wiki/kb/${kbId}/search-preview`, data), + + // Transformations: reusable prompt templates run over raw materials + listTransformations: (kbId?: number) => + http.get('/wiki/transformations', kbId != null ? { params: { kbId } } : undefined), + getTransformation: (id: number) => + http.get(`/wiki/transformations/${id}`), + createTransformation: (data: { + kbId?: number | null + name: string + title: string + description?: string + promptTemplate: string + applyDefault?: boolean + enabled?: boolean + modelId?: number | null + outputTarget?: 'none' | 'page' + }) => + http.post('/wiki/transformations', data), + updateTransformation: (id: number, data: { + title?: string + description?: string + promptTemplate?: string + applyDefault?: boolean + enabled?: boolean + modelId?: number | null + outputTarget?: 'none' | 'page' + }) => + http.put(`/wiki/transformations/${id}`, data), + deleteTransformation: (id: number) => + http.delete(`/wiki/transformations/${id}`), + applyTransformation: (id: number, rawId: number, sync = true) => + http.post(`/wiki/transformations/${id}/apply`, { rawId }, { params: { sync } }), + listTransformationRuns: (params: { rawId?: number; kbId?: number; transformationId?: number; limit?: number }) => + http.get('/wiki/transformations/runs', { params }), + getTransformationRun: (runId: number) => + http.get(`/wiki/transformations/runs/${runId}`), + deleteTransformationRun: (runId: number) => + http.delete(`/wiki/transformations/runs/${runId}`), + saveTransformationRunAsPage: (runId: number) => + http.post(`/wiki/transformations/runs/${runId}/save-as-page`), } // ==================== Workspace (Team) ==================== diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index 6fc3b178..687dcd3a 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -1796,6 +1796,57 @@ export default { rawMaterials: 'Raw Materials', pages: 'Wiki Pages', config: 'Config', + transformations: { + tab: 'Transformations', + title: 'Transformations', + desc: 'Reusable prompt templates. Each template runs against one raw material; the generated content is persisted here. Toggle "apply default" to fire it automatically after every successful ingest.', + empty: 'No transformations yet — create one to get started.', + createBtn: 'New transformation', + editBtn: 'Edit', + deleteBtn: 'Delete', + deleteConfirm: 'Delete this transformation? Past run history is kept.', + enabled: 'Enabled', + disabled: 'Disabled', + applyDefault: 'Apply by default', + applyDefaultOn: 'Runs automatically after each ingest completes', + applyDefaultOff: 'Manual or agent-tool runs only', + outputTargetLabel: 'Output target', + outputTargetNone: 'None — output stays in run history', + outputTargetPage: 'Save as wiki page (searchable / agent-accessible / linkable)', + outputTargetPageBadge: 'Auto-save to page', + saveAsPageBtn: 'Save as page', + saving: 'Saving…', + savedAsPage: 'Saved as:', + saveAsPageDone: 'Saved as wiki page', + saveAsPageFailed: 'Save as page failed', + openPage: 'Open page', + scope: 'Scope', + scopeKb: 'This knowledge base only', + scopeWorkspace: 'Workspace-wide', + name: 'Name', + namePlaceholder: 'e.g. risk-extract', + nameHelp: 'Used by agent tools; lowercase letters / digits / hyphens', + title2: 'Title', + titlePlaceholder: 'e.g. Contract risk extraction', + description: 'Description', + descriptionPlaceholder: 'One line about what this transformation produces', + prompt: 'Prompt template', + promptHelp: 'Supports {input_text} and {title} placeholders', + saveBtn: 'Save', + cancelBtn: 'Cancel', + runs: 'Run history', + runOn: 'Ran on', + runStatus: 'Status', + runDuration: 'Duration', + runOutput: 'Output', + runError: 'Error', + runApply: 'Run', + running: 'Running…', + runFailed: 'Run failed', + pickRaw: 'Pick a raw material', + pickRawHint: 'Select a raw material to use as input', + noRuns: 'No runs yet.', + }, hotCache: { tab: 'Hot Cache', title: 'Recent Activity Snapshot (Hot Cache)', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 6dcdc668..fbda6b1f 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -1808,6 +1808,57 @@ export default { rawMaterials: '原始材料', pages: 'Wiki 页面', config: '处理配置', + transformations: { + tab: '加工器', + title: '加工器（Transformations）', + desc: '可复用的 prompt 模板。每个模板针对单个原始材料运行，生成的内容会落到此处可查。开启「默认运行」后，材料处理完成会自动触发。', + empty: '尚无加工器模板，新建一个开始。', + createBtn: '新建加工器', + editBtn: '编辑', + deleteBtn: '删除', + deleteConfirm: '确认删除该加工器？历史运行记录会保留。', + enabled: '启用', + disabled: '已停用', + applyDefault: '默认运行', + applyDefaultOn: '每次新材料完成后自动跑', + applyDefaultOff: '仅在手动 / Agent 调用时运行', + outputTargetLabel: '输出去向', + outputTargetNone: '不保存（仅留在运行历史）', + outputTargetPage: '保存为 Wiki 页面（可被搜索 / Agent / 关系图引用）', + outputTargetPageBadge: '自动落页', + saveAsPageBtn: '保存为页面', + saving: '保存中…', + savedAsPage: '已保存：', + saveAsPageDone: '已保存为 Wiki 页面', + saveAsPageFailed: '保存为页面失败', + openPage: '查看页面', + scope: '范围', + scopeKb: '仅限此知识库', + scopeWorkspace: '工作区全局', + name: '标识名', + namePlaceholder: '例如：risk-extract', + nameHelp: '只用于 Agent 调用，小写字母 / 数字 / 连字符', + title2: '显示名', + titlePlaceholder: '例如：合同风险点提取', + description: '描述', + descriptionPlaceholder: '一句话说明这个加工器是做什么的', + prompt: '提示词模板', + promptHelp: '可使用 {input_text} 和 {title} 占位符', + saveBtn: '保存', + cancelBtn: '取消', + runs: '运行历史', + runOn: '运行于', + runStatus: '状态', + runDuration: '耗时', + runOutput: '输出', + runError: '错误', + runApply: '运行', + running: '运行中…', + runFailed: '运行失败', + pickRaw: '选择材料', + pickRawHint: '从知识库的原始材料中选择一个作为输入', + noRuns: '尚无运行记录。', + }, hotCache: { tab: '近况快照', title: '近况快照（Hot Cache）', diff --git a/mateclaw-ui/src/views/Wiki/components/TransformationsPanel.vue b/mateclaw-ui/src/views/Wiki/components/TransformationsPanel.vue new file mode 100644 index 00000000..42c87a5d --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/TransformationsPanel.vue @@ -0,0 +1,701 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/components/WikiWorkspace.vue b/mateclaw-ui/src/views/Wiki/components/WikiWorkspace.vue index 7b11af0f..6faea039 100644 --- a/mateclaw-ui/src/views/Wiki/components/WikiWorkspace.vue +++ b/mateclaw-ui/src/views/Wiki/components/WikiWorkspace.vue @@ -42,6 +42,10 @@

+ +
+ +
@@ -57,6 +61,7 @@ import WikiPageViewer from './WikiPageViewer.vue' import WikiConfig from './WikiConfig.vue' import WikiGraphView from './WikiGraphView.vue' import HotCachePanel from './HotCachePanel.vue' +import TransformationsPanel from './TransformationsPanel.vue' import WikiWorkspaceHeader from './WikiWorkspaceHeader.vue' import WikiPageSidebar from './WikiPageSidebar.vue' @@ -72,6 +77,7 @@ const tabs = computed(() => [ { key: 'pages', label: t('wiki.pages') }, { key: 'graph', label: t('wiki.graph.tab') }, { key: 'config', label: t('wiki.config') }, + { key: 'transformations', label: t('wiki.transformations.tab') }, { key: 'hotCache', label: t('wiki.hotCache.tab') }, ])