mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 19:23:42 +08:00
182 lines
7.4 KiB
Java
182 lines
7.4 KiB
Java
package vip.mate.wiki;
|
||
|
||
import lombok.Data;
|
||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||
|
||
/**
|
||
* Wiki 知识库配置
|
||
*
|
||
* @author MateClaw Team
|
||
*/
|
||
@Data
|
||
@ConfigurationProperties(prefix = "mate.wiki")
|
||
public class WikiProperties {
|
||
|
||
/** 是否启用 Wiki 知识库功能 */
|
||
private boolean enabled = true;
|
||
|
||
/**
|
||
* LLM 单次处理最大字符数（超过则分块）。
|
||
* <p>
|
||
* RFC-012：默认从 30000 下调到 15000 —— 单 chunk 输出 tokens 砍半、并行饱和度更高、
|
||
* 质量也更稳定。中端模型（qwen-plus/claude-sonnet）建议 12000-20000，
|
||
* 旗舰模型（qwen-max/claude-opus）可调大到 20000-30000。
|
||
*/
|
||
private int maxChunkSize = 15000;
|
||
|
||
/**
|
||
* 同一次 processAllPending 下同时处理的原始材料数上限。
|
||
* <p>
|
||
* RFC-012 Change 1：保护共享的 @Async 线程池（max=16）不被 wiki 长时间占满，
|
||
* 同时给材料级并发定一个可控的上限，避免 LLM 提供方触发限流。
|
||
*/
|
||
private int maxParallelRawMaterials = 3;
|
||
|
||
/**
|
||
* Max parallel chunks within a single raw material.
|
||
* <p>
|
||
* RFC-047 P3: Changed from 5 to 1. Parallel chunks all share the same existingPagesIndex
|
||
* snapshot, so chunk N cannot see pages created by chunk N-1 — causing duplicate pages and
|
||
* stale-index collisions. Serializing chunks eliminates this class of bug. Document-level
|
||
* parallelism (maxParallelRawMaterials) is preserved, so overall throughput is unchanged
|
||
* for multi-document batches.
|
||
*/
|
||
private int maxParallelChunks = 1;
|
||
|
||
/**
|
||
* 单个 chunk 内 phase B 阶段的 page 并行处理数上限。
|
||
* <p>
|
||
* RFC-012 follow-up #3：原实现 phase B 的 create / merge 循环逐页串行调用 LLM，
|
||
* 单 chunk N 个 page 就要串行 N 次 LLM 调用——一个卡超时整条流水线停摆。
|
||
* 改为受此 Semaphore 控制的并行，默认 3。结合 maxParallelRawMaterials × maxParallelChunks
|
||
* × maxParallelPhaseBPages = 3 × 5 × 3 = 45 的理论最大并发，实际按 LLM 限流为准。
|
||
*/
|
||
private int maxParallelPhaseBPages = 3;
|
||
|
||
/** 注入 agent prompt 的最大字符数 */
|
||
private int maxContextChars = 10000;
|
||
|
||
/** 单个原始材料最多生成的 Wiki 页面数 */
|
||
private int maxPagesPerRaw = 15;
|
||
|
||
/** 上传后是否自动触发处理 */
|
||
private boolean autoProcessOnUpload = true;
|
||
|
||
/** 上传文件存储目录 */
|
||
private String uploadDir = "./data/wiki-uploads";
|
||
|
||
/** 目录扫描最大文件数 */
|
||
private int maxScanFiles = 500;
|
||
|
||
/** 扫描时跳过大于此大小的文件（字节），默认 50MB */
|
||
private long maxScanFileSize = 50 * 1024 * 1024;
|
||
|
||
/**
|
||
* Wiki LLM 重试最大尝试次数（含首次）。
|
||
* <p>
|
||
* RFC-012 M1：旧实现无最大次数，遇到 nginx 504 这种"反复瞬时"错误会永远重试。
|
||
* 设为 5 后单 chunk 最多走 5 轮，配合 llmMaxTotalDurationMs 共同保证有界停止。
|
||
*/
|
||
private int llmMaxAttempts = 5;
|
||
|
||
/**
|
||
* Wiki LLM 重试总耗时上限（毫秒），从首次调用开始计时。
|
||
* <p>
|
||
* RFC-012 M1：单 chunk LLM 调用 + 重试的硬封顶，超过即放弃，让该 chunk 进入 failed 计数。
|
||
* 默认 4 分钟。
|
||
*/
|
||
private long llmMaxTotalDurationMs = 240_000;
|
||
|
||
/**
|
||
* RFC-047 P1: Max pages per BatchCreate LLM call.
|
||
* Pages planned by route are chunked into sub-batches of this size;
|
||
* a local liveIndex is updated between sub-batches so later pages can
|
||
* link to earlier ones created in the same chunk.
|
||
* Default 5: keeps output tokens per call predictable while covering
|
||
* the typical 3–5 creates per chunk in a single call.
|
||
*/
|
||
private int batchCreatePageSize = 3;
|
||
|
||
/**
|
||
* RFC-047: Minimum chunk length (chars) for the chunk-fallback mechanism.
|
||
* If route returns 0 create+update entries and the chunk exceeds this threshold,
|
||
* an overview page is auto-injected so no substantial content is silently dropped.
|
||
* Chunks shorter than this (e.g. TOC lines, blank pages) are allowed to produce nothing.
|
||
*/
|
||
private int chunkFallbackMinChars = 200;
|
||
|
||
/**
|
||
* 是否启用两阶段消化（路由 → 逐页 merge）。
|
||
* <p>
|
||
* RFC-012 M2：true 时单 chunk 的 LLM 输出量大幅缩减，避免 nginx 60s 网关超时。
|
||
* 默认 true（M2 上线）；遇问题可在 application.yml 配 mate.wiki.use-two-phase-digest=false 回退到旧行为。
|
||
*/
|
||
private boolean useTwoPhaseDigest = true;
|
||
|
||
// ==================== RFC-011: Embedding ====================
|
||
|
||
/** 嵌入模型名称（DashScope） */
|
||
private String embeddingModel = "text-embedding-v3";
|
||
|
||
/** 嵌入批量大小（一次 API 调用处理多少 chunk） */
|
||
private int embeddingBatchSize = 16;
|
||
|
||
/**
|
||
* Embedding 模型单段最大字符数，超过则子段拆分 + 向量均值。
|
||
* <p>
|
||
* 默认 6000（中文安全值，对应 ~4000 token，远小于 text-embedding-v3 的 8192 上限）。
|
||
* 纯英文场景可调大到 7500；其他 embedding 模型切换时按该模型的 token 限制调整。
|
||
*/
|
||
private int embeddingMaxChars = 6000;
|
||
|
||
/** 混合搜索默认模式：keyword / semantic / hybrid */
|
||
private String searchDefaultMode = "hybrid";
|
||
|
||
// ==================== RFC-031: Light processing tiers ====================
|
||
|
||
/** Whether to auto-dispatch a LIGHT_ENRICH job after heavy ingest completes */
|
||
private boolean lightEnrichEnabled = true;
|
||
|
||
/** Delay before light enrichment starts (ms) */
|
||
private long lightEnrichDelayMs = 2000;
|
||
|
||
/**
|
||
* Minimum ratio of enriched content length to original content length.
|
||
* If the LLM returns text shorter than this ratio, the enrichment is rejected.
|
||
*/
|
||
private double wikilinkMinContentRatio = 0.5;
|
||
|
||
/** Maximum characters for local repair single-page regeneration */
|
||
private int localRepairMaxChars = 8000;
|
||
|
||
/**
|
||
* Whether to run a document-level analysis pass before routing.
|
||
* When enabled, a single LLM call produces a concept map (topics + key_concepts)
|
||
* that is injected into every chunk's route prompt, giving the router global
|
||
* awareness of the document structure and reducing concept omissions.
|
||
* Adds ~1 LLM call and 10-20s per raw material.
|
||
*/
|
||
private boolean useDocumentAnalysis = true;
|
||
|
||
/**
|
||
* Max characters of document text fed to the analysis pass.
|
||
* Larger values improve coverage but increase prompt tokens.
|
||
* Default 15000 covers most documents while staying well within model limits.
|
||
*/
|
||
private int documentAnalysisSampleChars = 15000;
|
||
|
||
/**
|
||
* RFC-051 PR-6b: route-phase output binding. When {@code true}, the route
|
||
* LLM call uses a Spring AI {@code BeanOutputConverter<RouteResult>} —
|
||
* the format hint is injected into the user prompt and the response is
|
||
* parsed strictly into the DTO. Failures fall back to the legacy lenient
|
||
* JSON parser so a flaky model never blocks ingest.
|
||
* <p>
|
||
* Default {@code false} keeps existing behavior on first upgrade. Flip on
|
||
* once you've validated the route prompt against the models you actually
|
||
* run (DashScope / OpenAI / Anthropic / DeepSeek tend to be fine; Ollama
|
||
* and weaker models may need the fallback).
|
||
*/
|
||
private boolean useStructuredRoute = false;
|
||
}
|