feat(wiki): PR-1b lazy ingest — chunk+embed, no page generation

2026-09-14 11:37:31 +08:00 · 2026-04-25 09:56:18 +08:00 · 2026-04-25 09:56:18 +08:00 · 80725c9ac7
commit 80725c9ac7
parent 50d9ff2b3d
4 changed files with 188 additions and 0 deletions
--- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java
+++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java
@ -16,6 +16,7 @@ import vip.mate.agent.prompt.PromptLoader;
 import vip.mate.llm.model.ModelConfigEntity;
 import vip.mate.llm.service.ModelConfigService;
 import vip.mate.wiki.WikiProperties;
+import vip.mate.wiki.job.WikiKbConfig;
 import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
 import vip.mate.wiki.model.WikiPageEntity;
 import vip.mate.wiki.model.WikiRawMaterialEntity;
@ -146,6 +147,14 @@ public class WikiProcessingService {

        kbService.updateStatus(kb.getId(), "processing");

+        // RFC-051 PR-1b: lazy ingest short-circuit. Per KB config, skip the heavy
+        // pipeline entirely: extract → chunk → embed → completed. 0 pages is the
+        // expected outcome, not a failure. ingestMode==null keeps existing behavior.
+        if ("lazy".equals(resolveIngestMode(kb))) {
+            processLazyIngest(kb, raw);
+            return;
+        }
+
        // RFC-030 §9.1: create a processing job record and track its ID for stage transitions
        Long jobId = null;
        if (wikiJobService != null) {
@ -1625,4 +1634,97 @@ public class WikiProcessingService {
            return null;
        }
    }
+
+    /**
+     * RFC-051 PR-1b: read {@code ingestMode} from KB config JSON. Returns null
+     * on any parse error or missing field so the caller falls through to eager.
+     */
+    private String resolveIngestMode(WikiKnowledgeBaseEntity kb) {
+        if (kb == null || kb.getConfigContent() == null) return null;
+        try {
+            WikiKbConfig config = objectMapper.readValue(kb.getConfigContent(), WikiKbConfig.class);
+            return config.getIngestMode();
+        } catch (Exception e) {
+            log.warn("[Wiki] Failed to parse KB config for ingest mode, falling back to eager: {}", e.getMessage());
+            return null;
+        }
+    }
+
+    /**
+     * RFC-051 PR-1b: lazy ingest — chunk + embed, no page generation.
+     * <p>
+     * Intentionally minimal: reuses the legacy {@code persistChunks(List<String>, offsets)}
+     * overload (no structural metadata; that lands in PR-1c with the preprocessor)
+     * and the existing {@code embedMissingChunks} entry point. Zero pages is the
+     * expected outcome, not a failure.
+     */
+    private void processLazyIngest(WikiKnowledgeBaseEntity kb, WikiRawMaterialEntity raw) {
+        Long rawId = raw.getId();
+        Long kbId = kb.getId();
+        log.info("[Wiki] Lazy ingest starting for raw={}, kbId={}", rawId, kbId);
+
+        rawService.updateProgress(rawId, "lazy", 0, 0);
+        progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_STARTED,
+                java.util.Map.of("rawId", rawId, "phase", "lazy"));
+
+        try {
+            String textContent = rawService.getTextContent(raw);
+            if (textContent == null || textContent.isBlank()) {
+                rawService.updateProcessingStatus(rawId, "failed", "No text content available");
+                kbService.updateStatus(kbId, "active");
+                progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_FAILED,
+                        java.util.Map.of("rawId", rawId, "error", "No text content available"));
+                return;
+            }
+
+            List<ChunkWithOffset> chunksWithOffset = splitIntoChunksWithOffsets(textContent);
+            List<String> chunks = chunksWithOffset.stream().map(ChunkWithOffset::text).toList();
+            List<int[]> offsets = chunksWithOffset.stream()
+                    .map(c -> new int[]{c.startOffset(), c.endOffset()}).toList();
+            chunkService.persistChunks(kbId, rawId, chunks, offsets);
+            int totalChunks = chunks.size();
+            log.info("[Wiki] Lazy ingest persisted {} chunks for raw={}", totalChunks, rawId);
+
+            // Async embedding — mirror the eager path so a slow embedding model
+            // does not block the raw from reaching completed.
+            final Long fKbId = kbId;
+            WIKI_EXECUTOR.submit(() -> {
+                try {
+                    int embedded = embeddingService.embedMissingChunks(fKbId);
+                    if (embedded > 0) {
+                        log.info("[Wiki] Lazy async embedding completed: kbId={}, embedded={}", fKbId, embedded);
+                    }
+                } catch (Exception ex) {
+                    log.warn("[Wiki] Lazy async embedding failed for kbId={}: {}", fKbId, ex.getMessage());
+                }
+            });
+
+            rawService.updateProcessingStatus(rawId, "completed", null);
+            if (raw.getContentHash() != null) {
+                rawService.setLastProcessedHash(rawId, raw.getContentHash());
+            }
+            rawService.updateProgress(rawId, "done", totalChunks, totalChunks);
+            int pageCount = pageService.countByKbId(kbId);
+            kbService.setPageCount(kbId, pageCount);
+            kbService.updateStatus(kbId, "active");
+
+            progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_COMPLETED,
+                    java.util.Map.of(
+                            "rawId", rawId,
+                            "status", "completed",
+                            "totalPages", 0,
+                            "kbPageCount", pageCount,
+                            "totalChunks", totalChunks));
+
+            log.info("[Wiki] Lazy processing completed for raw={}, kbId={}, chunks={}",
+                    rawId, kbId, totalChunks);
+        } catch (Exception e) {
+            log.error("[Wiki] Lazy processing failed for raw={}: {}", rawId, e.getMessage(), e);
+            rawService.updateProcessingStatus(rawId, "failed", e.getMessage());
+            kbService.updateStatus(kbId, "active");
+            progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_FAILED,
+                    java.util.Map.of("rawId", rawId,
+                            "error", e.getMessage() == null ? "unknown" : e.getMessage()));
+        }
+    }
 }
--- a/mateclaw-ui/src/i18n/locales/en-US.ts
+++ b/mateclaw-ui/src/i18n/locales/en-US.ts
@ -1307,6 +1307,11 @@ export default {
    configPanel: {
      embeddingModel: 'Embedding Model',
      embeddingModelHint: 'Semantic search model for this KB; leave empty for system default',
+      ingestMode: 'Ingest Mode',
+      ingestModeEager: 'Eager',
+      ingestModeLazy: 'Lazy',
+      ingestModeEagerHint: 'Upload runs the full LLM pipeline to generate wiki pages immediately.',
+      ingestModeLazyHint: 'Upload only extracts, chunks, and embeds. No page generation; search works right away.',
      modelStrategy: 'Model Strategy',
      globalDefault: 'Global default',
      selectModel: 'Select a model…',
--- a/mateclaw-ui/src/i18n/locales/zh-CN.ts
+++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts
@ -1317,6 +1317,11 @@ export default {
    configPanel: {
      embeddingModel: '向量化模型',
      embeddingModelHint: '语义搜索模型，留空使用系统默认',
+      ingestMode: '入库模式',
+      ingestModeEager: '立即生成页面',
+      ingestModeLazy: '先入索引',
+      ingestModeEagerHint: '上传后立即调用 LLM 生成完整 Wiki 页面。',
+      ingestModeLazyHint: '上传仅抽取、切片、向量化，跳过页面生成；上传即可搜索，页面按需编译。',
      modelStrategy: '模型策略',
      globalDefault: '跟随全局默认',
      selectModel: '选择可用模型…',
--- a/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue
+++ b/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue
@ -19,6 +19,33 @@
      <WikiModelPicker v-model="embeddingModelId" :options="embeddingPickerOptions" :disabled="savingEmbedding" />
    </div>

+    <!-- ①b Ingest mode (RFC-051 PR-1b) -->
+    <div class="config-card">
+      <div class="config-card__head">
+        <div>
+          <div class="config-card__title">{{ t('wiki.configPanel.ingestMode') }}</div>
+          <div class="config-card__hint">
+            {{ ingestMode === 'lazy'
+              ? t('wiki.configPanel.ingestModeLazyHint')
+              : t('wiki.configPanel.ingestModeEagerHint') }}
+          </div>
+        </div>
+        <button class="btn-save" @click="saveIngestMode" :disabled="savingIngestMode">
+          {{ savingIngestMode ? t('wiki.saving') : t('common.save') }}
+        </button>
+      </div>
+      <div class="ingest-mode-row">
+        <label class="ingest-mode-option" :class="{ 'ingest-mode-option--active': ingestMode === 'eager' }">
+          <input type="radio" value="eager" v-model="ingestMode" :disabled="savingIngestMode" />
+          <span class="ingest-mode-option__label">{{ t('wiki.configPanel.ingestModeEager') }}</span>
+        </label>
+        <label class="ingest-mode-option" :class="{ 'ingest-mode-option--active': ingestMode === 'lazy' }">
+          <input type="radio" value="lazy" v-model="ingestMode" :disabled="savingIngestMode" />
+          <span class="ingest-mode-option__label">{{ t('wiki.configPanel.ingestModeLazy') }}</span>
+        </label>
+      </div>
+    </div>
+
    <!-- ② Model strategy -->
    <div class="config-card config-card--clickable" @click="modelsOpen = true">
      <div class="config-card__row">
@ -177,6 +204,29 @@ async function saveEmbeddingBinding() {
  }
 }

+// ── Ingest mode (RFC-051 PR-1b) ──
+// eager = legacy heavy pipeline (extract → chunk → route/create/merge LLM → pages).
+// lazy  = extract → chunk → embed → completed (0 pages is success).
+const ingestMode = ref<'eager' | 'lazy'>('eager')
+const savingIngestMode = ref(false)
+
+async function saveIngestMode() {
+  if (!store.currentKB) return
+  savingIngestMode.value = true
+  try {
+    let existingConfig: any = {}
+    try {
+      if (store.currentKB.configContent) existingConfig = JSON.parse(store.currentKB.configContent)
+    } catch { /* config may be plain text rules */ }
+    existingConfig.ingestMode = ingestMode.value
+    await wikiApi.updateConfig(store.currentKB.id, JSON.stringify(existingConfig, null, 2))
+  } catch (e) {
+    console.error('[WikiConfig] Failed to save ingest mode', e)
+  } finally {
+    savingIngestMode.value = false
+  }
+}
+
 // ── Model strategy ──
 const stepKeys = ['route', 'create_page', 'merge_page', 'enrich', 'summary']
 const stepModels = reactive<Record<string, string>>({})
@ -192,6 +242,7 @@ function loadStepModels() {
  stepKeys.forEach(k => (stepModels[k] = ''))
  fallbackModelIds.value = []
  wikiGlobalModelId.value = ''
+  ingestMode.value = 'eager'
  if (!store.currentKB) return
  try {
    const cfg = store.currentKB.configContent ? JSON.parse(store.currentKB.configContent) : null
@ -203,6 +254,7 @@ function loadStepModels() {
    }
    if (cfg?.fallbackModelIds) fallbackModelIds.value = cfg.fallbackModelIds.map(String)
    if (cfg?.wikiDefaultModelId) wikiGlobalModelId.value = String(cfg.wikiDefaultModelId)
+    if (cfg?.ingestMode === 'lazy') ingestMode.value = 'lazy'
  } catch { /* not JSON */ }
 }

@ -416,4 +468,28 @@ loadProviderNames().then(() => {
 }
 .btn-save:hover { opacity: 0.88; }
 .btn-save:disabled { background: var(--mc-border); cursor: not-allowed; }
+
+/* Ingest mode radio group */
+.ingest-mode-row { display: flex; gap: 8px; flex-wrap: wrap; }
+.ingest-mode-option {
+  display: flex;
+  align-items: center;
+  gap: 6px;
+  padding: 6px 10px;
+  border: 1px solid var(--mc-border-light);
+  border-radius: 8px;
+  background: var(--mc-bg-elevated);
+  font-size: 12px;
+  color: var(--mc-text-secondary);
+  cursor: pointer;
+  transition: border-color 0.15s, color 0.15s, background 0.15s;
+}
+.ingest-mode-option input { margin: 0; cursor: pointer; }
+.ingest-mode-option--active {
+  border-color: var(--mc-primary);
+  color: var(--mc-primary);
+  background: var(--mc-primary-bg);
+  font-weight: 600;
+}
+.ingest-mode-option__label { line-height: 1; }
 </style>