diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index cc12d971..a691c01f 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -16,6 +16,7 @@ import vip.mate.agent.prompt.PromptLoader; import vip.mate.llm.model.ModelConfigEntity; import vip.mate.llm.service.ModelConfigService; import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.job.WikiKbConfig; import vip.mate.wiki.model.WikiKnowledgeBaseEntity; import vip.mate.wiki.model.WikiPageEntity; import vip.mate.wiki.model.WikiRawMaterialEntity; @@ -146,6 +147,14 @@ public class WikiProcessingService { kbService.updateStatus(kb.getId(), "processing"); + // RFC-051 PR-1b: lazy ingest short-circuit. Per KB config, skip the heavy + // pipeline entirely: extract → chunk → embed → completed. 0 pages is the + // expected outcome, not a failure. ingestMode==null keeps existing behavior. + if ("lazy".equals(resolveIngestMode(kb))) { + processLazyIngest(kb, raw); + return; + } + // RFC-030 §9.1: create a processing job record and track its ID for stage transitions Long jobId = null; if (wikiJobService != null) { @@ -1625,4 +1634,97 @@ public class WikiProcessingService { return null; } } + + /** + * RFC-051 PR-1b: read {@code ingestMode} from KB config JSON. Returns null + * on any parse error or missing field so the caller falls through to eager. + */ + private String resolveIngestMode(WikiKnowledgeBaseEntity kb) { + if (kb == null || kb.getConfigContent() == null) return null; + try { + WikiKbConfig config = objectMapper.readValue(kb.getConfigContent(), WikiKbConfig.class); + return config.getIngestMode(); + } catch (Exception e) { + log.warn("[Wiki] Failed to parse KB config for ingest mode, falling back to eager: {}", e.getMessage()); + return null; + } + } + + /** + * RFC-051 PR-1b: lazy ingest — chunk + embed, no page generation. + *

+ * Intentionally minimal: reuses the legacy {@code persistChunks(List, offsets)} + * overload (no structural metadata; that lands in PR-1c with the preprocessor) + * and the existing {@code embedMissingChunks} entry point. Zero pages is the + * expected outcome, not a failure. + */ + private void processLazyIngest(WikiKnowledgeBaseEntity kb, WikiRawMaterialEntity raw) { + Long rawId = raw.getId(); + Long kbId = kb.getId(); + log.info("[Wiki] Lazy ingest starting for raw={}, kbId={}", rawId, kbId); + + rawService.updateProgress(rawId, "lazy", 0, 0); + progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_STARTED, + java.util.Map.of("rawId", rawId, "phase", "lazy")); + + try { + String textContent = rawService.getTextContent(raw); + if (textContent == null || textContent.isBlank()) { + rawService.updateProcessingStatus(rawId, "failed", "No text content available"); + kbService.updateStatus(kbId, "active"); + progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_FAILED, + java.util.Map.of("rawId", rawId, "error", "No text content available")); + return; + } + + List chunksWithOffset = splitIntoChunksWithOffsets(textContent); + List chunks = chunksWithOffset.stream().map(ChunkWithOffset::text).toList(); + List offsets = chunksWithOffset.stream() + .map(c -> new int[]{c.startOffset(), c.endOffset()}).toList(); + chunkService.persistChunks(kbId, rawId, chunks, offsets); + int totalChunks = chunks.size(); + log.info("[Wiki] Lazy ingest persisted {} chunks for raw={}", totalChunks, rawId); + + // Async embedding — mirror the eager path so a slow embedding model + // does not block the raw from reaching completed. + final Long fKbId = kbId; + WIKI_EXECUTOR.submit(() -> { + try { + int embedded = embeddingService.embedMissingChunks(fKbId); + if (embedded > 0) { + log.info("[Wiki] Lazy async embedding completed: kbId={}, embedded={}", fKbId, embedded); + } + } catch (Exception ex) { + log.warn("[Wiki] Lazy async embedding failed for kbId={}: {}", fKbId, ex.getMessage()); + } + }); + + rawService.updateProcessingStatus(rawId, "completed", null); + if (raw.getContentHash() != null) { + rawService.setLastProcessedHash(rawId, raw.getContentHash()); + } + rawService.updateProgress(rawId, "done", totalChunks, totalChunks); + int pageCount = pageService.countByKbId(kbId); + kbService.setPageCount(kbId, pageCount); + kbService.updateStatus(kbId, "active"); + + progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_COMPLETED, + java.util.Map.of( + "rawId", rawId, + "status", "completed", + "totalPages", 0, + "kbPageCount", pageCount, + "totalChunks", totalChunks)); + + log.info("[Wiki] Lazy processing completed for raw={}, kbId={}, chunks={}", + rawId, kbId, totalChunks); + } catch (Exception e) { + log.error("[Wiki] Lazy processing failed for raw={}: {}", rawId, e.getMessage(), e); + rawService.updateProcessingStatus(rawId, "failed", e.getMessage()); + kbService.updateStatus(kbId, "active"); + progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_FAILED, + java.util.Map.of("rawId", rawId, + "error", e.getMessage() == null ? "unknown" : e.getMessage())); + } + } } diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index 6e4af92b..6c09c4ad 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -1307,6 +1307,11 @@ export default { configPanel: { embeddingModel: 'Embedding Model', embeddingModelHint: 'Semantic search model for this KB; leave empty for system default', + ingestMode: 'Ingest Mode', + ingestModeEager: 'Eager', + ingestModeLazy: 'Lazy', + ingestModeEagerHint: 'Upload runs the full LLM pipeline to generate wiki pages immediately.', + ingestModeLazyHint: 'Upload only extracts, chunks, and embeds. No page generation; search works right away.', modelStrategy: 'Model Strategy', globalDefault: 'Global default', selectModel: 'Select a model…', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 063db738..c469b57f 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -1317,6 +1317,11 @@ export default { configPanel: { embeddingModel: '向量化模型', embeddingModelHint: '语义搜索模型，留空使用系统默认', + ingestMode: '入库模式', + ingestModeEager: '立即生成页面', + ingestModeLazy: '先入索引', + ingestModeEagerHint: '上传后立即调用 LLM 生成完整 Wiki 页面。', + ingestModeLazyHint: '上传仅抽取、切片、向量化，跳过页面生成；上传即可搜索，页面按需编译。', modelStrategy: '模型策略', globalDefault: '跟随全局默认', selectModel: '选择可用模型…', diff --git a/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue b/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue index 9d634470..e8e14f5d 100644 --- a/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue +++ b/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue @@ -19,6 +19,33 @@ + +

+
+
+
{{ t('wiki.configPanel.ingestMode') }}
+
+ {{ ingestMode === 'lazy' + ? t('wiki.configPanel.ingestModeLazyHint') + : t('wiki.configPanel.ingestModeEagerHint') }} +
+
+ +
+
+ + +
+
+
@@ -177,6 +204,29 @@ async function saveEmbeddingBinding() { } } +// ── Ingest mode (RFC-051 PR-1b) ── +// eager = legacy heavy pipeline (extract → chunk → route/create/merge LLM → pages). +// lazy = extract → chunk → embed → completed (0 pages is success). +const ingestMode = ref<'eager' | 'lazy'>('eager') +const savingIngestMode = ref(false) + +async function saveIngestMode() { + if (!store.currentKB) return + savingIngestMode.value = true + try { + let existingConfig: any = {} + try { + if (store.currentKB.configContent) existingConfig = JSON.parse(store.currentKB.configContent) + } catch { /* config may be plain text rules */ } + existingConfig.ingestMode = ingestMode.value + await wikiApi.updateConfig(store.currentKB.id, JSON.stringify(existingConfig, null, 2)) + } catch (e) { + console.error('[WikiConfig] Failed to save ingest mode', e) + } finally { + savingIngestMode.value = false + } +} + // ── Model strategy ── const stepKeys = ['route', 'create_page', 'merge_page', 'enrich', 'summary'] const stepModels = reactive>({}) @@ -192,6 +242,7 @@ function loadStepModels() { stepKeys.forEach(k => (stepModels[k] = '')) fallbackModelIds.value = [] wikiGlobalModelId.value = '' + ingestMode.value = 'eager' if (!store.currentKB) return try { const cfg = store.currentKB.configContent ? JSON.parse(store.currentKB.configContent) : null @@ -203,6 +254,7 @@ function loadStepModels() { } if (cfg?.fallbackModelIds) fallbackModelIds.value = cfg.fallbackModelIds.map(String) if (cfg?.wikiDefaultModelId) wikiGlobalModelId.value = String(cfg.wikiDefaultModelId) + if (cfg?.ingestMode === 'lazy') ingestMode.value = 'lazy' } catch { /* not JSON */ } } @@ -416,4 +468,28 @@ loadProviderNames().then(() => { } .btn-save:hover { opacity: 0.88; } .btn-save:disabled { background: var(--mc-border); cursor: not-allowed; } + +/* Ingest mode radio group */ +.ingest-mode-row { display: flex; gap: 8px; flex-wrap: wrap; } +.ingest-mode-option { + display: flex; + align-items: center; + gap: 6px; + padding: 6px 10px; + border: 1px solid var(--mc-border-light); + border-radius: 8px; + background: var(--mc-bg-elevated); + font-size: 12px; + color: var(--mc-text-secondary); + cursor: pointer; + transition: border-color 0.15s, color 0.15s, background 0.15s; +} +.ingest-mode-option input { margin: 0; cursor: pointer; } +.ingest-mode-option--active { + border-color: var(--mc-primary); + color: var(--mc-primary); + background: var(--mc-primary-bg); + font-weight: 600; +} +.ingest-mode-option__label { line-height: 1; }