feat(wiki): unify raw materials & source watcher into a Sources tab with per-KB auto-sync (#316)

* feat(wiki): unify raw materials & source watcher into a Sources tab with per-KB auto-sync

The raw-material directory scan and the Advanced "source watcher" sub-tab were
the same engine (same kb.sourceDirectory, same WikiDirectoryScanService) split
across two surfaces with two editable directory inputs. Merge them into one
"Sources" tab (upload / paste / directory manual scan + auto-sync toggle +
the raw-material list) and drop the watcher sub-tab from Advanced.

Auto-sync is now per-KB opt-in: a new watcher_enabled column (V146) gates the
periodic scan per knowledge base. The server-global mate.wiki.watcher-enabled
stays as an ops master switch — a KB is auto-scanned only when both are on
(AND). Manual scans are unaffected. Scan interval stays global for now
(tracked separately).

Closes matevip/mateclaw#314

* docs(wiki): document source-watcher global switch env vars

Expose MATE_WIKI_WATCHER_ENABLED / MATE_WIKI_WATCHER_INTERVAL_MS as
explicit placeholders in application-mysql.yml, .env.example and
docker-compose.yml, mirroring MATE_WIKI_ALLOWED_SOURCE_ROOTS. Notes the
AND semantics (global ops gate + per-KB toggle) so operators know the
global switch alone is not sufficient.
This commit is contained in:
倪程伟 2026-06-11 09:29:26 +08:00 committed by GitHub
parent 92d401bc1f
commit 18daad79b2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 195 additions and 64 deletions

View File

@ -87,6 +87,14 @@ MATECLAW_OAUTH_OPENAI_CALLBACK_BIND_HOST=
# - /your/host/path:/data/wiki
MATE_WIKI_ALLOWED_SOURCE_ROOTS=
# ── Wiki 知识源自动同步(变更监测)总开关 ────────────────────────
# 定时扫描各知识库的源目录、自动消化新文件。默认关闭,运维主动开启。
# AND 语义:全局这个开关开 *且* 某知识库自己的「自动同步」开关也开,
# 该库才会被定时扫描;手动「立即扫描」不受此开关影响。
# 间隔单位毫秒,默认 5 分钟(目前为全局,暂不支持按库配置)。
MATE_WIKI_WATCHER_ENABLED=false
MATE_WIKI_WATCHER_INTERVAL_MS=300000
# ── Skill 工作区目录 ─────────────────────────────────────────────
# 已安装的 skill、运行时积累的 LESSONS.md、skill 运行产物都落在这个目录。
# 默认(容器内)已指向 /app/data/skills由 docker-compose 的 server_data 卷

View File

@ -100,6 +100,11 @@ services:
# 示例MATE_WIKI_ALLOWED_SOURCE_ROOTS=/data/wiki,/opt/docs
# 记得同步在 volumes 里把宿主机路径挂进容器。
MATE_WIKI_ALLOWED_SOURCE_ROOTS: ${MATE_WIKI_ALLOWED_SOURCE_ROOTS:-}
# Wiki 知识源自动同步总开关运维总闸默认关。AND 语义:全局开关与
# 每个知识库自己的「自动同步」开关都开,该库才会被定时扫描。
# 间隔单位毫秒,默认 5 分钟。
MATE_WIKI_WATCHER_ENABLED: ${MATE_WIKI_WATCHER_ENABLED:-false}
MATE_WIKI_WATCHER_INTERVAL_MS: ${MATE_WIKI_WATCHER_INTERVAL_MS:-300000}
# Skill 工作区根目录。放在 /app/data 下,让现有的 server_data 卷一并持久化
# 已安装的 skill、运行时积累的 LESSONS.md 以及 skill 运行产物,容器重启不丢。
# 内置 skill 仍由 JAR classpath 每次启动现场释放,空卷不会丢内置文件。

View File

@ -345,7 +345,10 @@ public class WikiController {
if (kb == null) return R.fail(404, "Knowledge base not found");
vip.mate.wiki.source.WikiIngestSourceProvider provider = sourceWatcherService.providerFor(kb);
Map<String, Object> out = new LinkedHashMap<>();
// Global master switch (ops): gates the scheduler at all.
out.put("watcherEnabled", properties.isWatcherEnabled());
// Per-KB opt-in: auto-sync runs only when both are true (AND semantics).
out.put("kbWatcherEnabled", kb.getWatcherEnabled() != null && kb.getWatcherEnabled() == 1);
out.put("intervalMs", properties.getWatcherIntervalMs());
out.put("sourceDirectory", kb.getSourceDirectory());
out.put("sourceType", provider != null ? provider.sourceType() : null);
@ -354,6 +357,20 @@ public class WikiController {
return R.ok(out);
}
@RequireWorkspaceRole("member")
@Operation(summary = "开关知识库的自动同步(每库)")
@PutMapping("/knowledge-bases/{id}/source-watcher/enabled")
public R<Void> setWatcherEnabled(@PathVariable Long id, @RequestBody Map<String, Object> body,
@RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) {
verifyKBWorkspace(id, workspaceId);
WikiKnowledgeBaseEntity kb = kbService.getById(id);
if (kb == null) return R.fail(404, "Knowledge base not found");
Object v = body.get("enabled");
boolean enabled = (v instanceof Boolean b) ? b : Boolean.parseBoolean(String.valueOf(v));
kbService.updateWatcherEnabled(id, enabled);
return R.ok();
}
@RequireWorkspaceRole("member")
@Operation(summary = "手动触发一次源监听扫描")
@PostMapping("/knowledge-bases/{id}/source-watcher/scan")

View File

@ -33,6 +33,13 @@ public class WikiKnowledgeBaseEntity {
/** 关联的本地目录路径(可选,用于批量扫描导入) */
private String sourceDirectory;
/**
* 是否对该知识库启用自动同步周期扫描 sourceDirectory1=0=
* 自动扫描需"全局总闸 mate.wiki.watcher-enabled 开 且 本字段为 1"AND 语义
* 手动扫描不受此字段影响
*/
private Integer watcherEnabled;
/** 状态active / processing / error */
private String status;

View File

@ -383,6 +383,17 @@ public class WikiKnowledgeBaseService {
kbMapper.updateById(entity);
}
/** Toggle per-KB auto-sync (the periodic source-watcher scan). */
@Transactional
public void updateWatcherEnabled(Long id, boolean enabled) {
WikiKnowledgeBaseEntity entity = kbMapper.selectById(id);
if (entity == null) {
throw new IllegalArgumentException("Knowledge base not found: " + id);
}
entity.setWatcherEnabled(enabled ? 1 : 0);
kbMapper.updateById(entity);
}
@Transactional
public void decrementRawCount(Long kbId) {
WikiKnowledgeBaseEntity entity = kbMapper.selectById(kbId);

View File

@ -63,6 +63,12 @@ public class WikiSourceWatcherService {
public int runScanCycle() {
int totalAdded = 0;
for (WikiKnowledgeBaseEntity kb : kbService.listAll()) {
// Per-KB opt-in: the global master switch (checked in scheduledScan)
// gates the scheduler at all; this flag gates each KB. AND semantics
// a KB is auto-scanned only when both are on. Manual scans bypass this.
if (kb.getWatcherEnabled() == null || kb.getWatcherEnabled() != 1) {
continue;
}
vip.mate.wiki.source.WikiIngestSourceProvider provider = providerFor(kb);
if (provider == null) {
continue;

View File

@ -33,7 +33,14 @@ spring:
# Set MATE_WIKI_ALLOWED_SOURCE_ROOTS in .env (comma-separated paths):
# MATE_WIKI_ALLOWED_SOURCE_ROOTS=/data/wiki,/opt/docs
# The default profile (H2 / desktop / single-tenant) leaves this off.
# Source watcher master switch (ops gate). Off by default; operators opt in.
# AND semantics: a KB is auto-scanned only when this global switch AND that
# KB's own auto-sync toggle are both on. Manual scans are unaffected.
# MATE_WIKI_WATCHER_ENABLED=true
# MATE_WIKI_WATCHER_INTERVAL_MS=300000 # scan interval, default 5 min
mate:
wiki:
require-allowed-roots: true
allowed-source-roots: ${MATE_WIKI_ALLOWED_SOURCE_ROOTS:}
watcher-enabled: ${MATE_WIKI_WATCHER_ENABLED:false}
watcher-interval-ms: ${MATE_WIKI_WATCHER_INTERVAL_MS:300000}

View File

@ -0,0 +1,9 @@
-- Per-KB source-watcher toggle. Auto-sync (the periodic directory scan) was
-- previously gated only by the server-global `mate.wiki.watcher-enabled`.
-- This column makes the auto-sync opt-in per knowledge base: a KB is scanned
-- automatically only when the global master switch is on AND this flag is set
-- (AND semantics). Manual "scan now" is unaffected by this flag. Defaults to 0
-- so existing KBs are not auto-scanned until explicitly enabled.
ALTER TABLE mate_wiki_knowledge_base
ADD COLUMN IF NOT EXISTS watcher_enabled TINYINT(1) NOT NULL DEFAULT 0;

View File

@ -0,0 +1,12 @@
-- Per-KB source-watcher toggle. See the h2 sibling for the prose explanation.
-- MySQL lacks `ADD COLUMN IF NOT EXISTS`, so the column is guarded by an
-- INFORMATION_SCHEMA check + prepared statement (idempotent).
SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'mate_wiki_knowledge_base'
AND COLUMN_NAME = 'watcher_enabled');
SET @s := IF(@c = 0,
'ALTER TABLE mate_wiki_knowledge_base ADD COLUMN watcher_enabled TINYINT(1) NOT NULL DEFAULT 0',
'SELECT 1');
PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;

View File

@ -35,6 +35,8 @@ class WikiSourceWatcherServiceE2ETest {
private WikiKnowledgeBaseService kbService;
@Autowired
private WikiDirectoryScanService scanService;
@Autowired
private WikiRawMaterialService rawMaterialService;
private static final java.util.concurrent.atomic.AtomicLong SEQ =
new java.util.concurrent.atomic.AtomicLong(System.nanoTime());
@ -47,6 +49,8 @@ class WikiSourceWatcherServiceE2ETest {
WikiKnowledgeBaseEntity kb = kbService.create(
"watcher-" + SEQ.incrementAndGet(), "test", null);
kbService.updateSourceDirectory(kb.getId(), sourceDir.toString());
// Auto-sync is per-KB opt-in; enable it so the cycle scans this KB.
kbService.updateWatcherEnabled(kb.getId(), true);
// First cycle ingests both new files.
int firstAdded = watcherService.runScanCycle();
@ -109,4 +113,23 @@ class WikiSourceWatcherServiceE2ETest {
// Should complete without throwing (count is non-negative).
assertTrue(watcherService.runScanCycle() >= 0);
}
@Test
void disabledKb_isNotAutoScanned_untilEnabled(@TempDir Path sourceDir) throws IOException {
Files.writeString(sourceDir.resolve("note.md"), "# Note\n\ncontent");
WikiKnowledgeBaseEntity kb = kbService.create(
"watcher-off-" + SEQ.incrementAndGet(), "test", null);
kbService.updateSourceDirectory(kb.getId(), sourceDir.toString());
// watcher_enabled defaults to 0 the auto cycle must skip this KB.
watcherService.runScanCycle();
assertEquals(0, rawMaterialService.listByKbId(kb.getId()).size(),
"a KB with auto-sync disabled must not be auto-scanned");
// Once enabled, the same cycle ingests its file.
kbService.updateWatcherEnabled(kb.getId(), true);
watcherService.runScanCycle();
assertTrue(rawMaterialService.listByKbId(kb.getId()).size() >= 1,
"after enabling auto-sync the KB's file should be ingested");
}
}

View File

@ -946,6 +946,8 @@ export const wikiApi = {
http.get(`/wiki/knowledge-bases/${kbId}/source-watcher`),
triggerSourceWatcher: (kbId: string | number) =>
http.post(`/wiki/knowledge-bases/${kbId}/source-watcher/scan`),
setWatcherEnabled: (kbId: string | number, enabled: boolean) =>
http.put(`/wiki/knowledge-bases/${kbId}/source-watcher/enabled`, { enabled }),
// ---- Pipelines (REQ-5) ----
listPipelines: (kbId: string | number) =>

View File

@ -2091,6 +2091,13 @@ export default {
kbDescription: 'Description',
kbDescPlaceholder: 'Briefly describe the knowledge base purpose',
rawMaterials: 'Raw Materials',
sources: {
tab: 'Sources',
autoSync: 'Auto-sync',
autoSyncInterval: 'every {sec}s',
autoSyncGlobalOffHint: 'Auto-sync needs the global switch enabled by ops',
toggleFailed: 'Failed to toggle auto-sync',
},
pages: 'Wiki Pages',
config: 'Config',
transformations: {

View File

@ -2103,6 +2103,13 @@ export default {
kbDescription: '描述',
kbDescPlaceholder: '简要描述知识库用途',
rawMaterials: '原始材料',
sources: {
tab: '来源',
autoSync: '自动同步',
autoSyncInterval: '每 {sec} 秒',
autoSyncGlobalOffHint: '自动同步需运维开启全局开关',
toggleFailed: '切换自动同步失败',
},
pages: 'Wiki 页面',
config: '处理配置',
transformations: {

View File

@ -72,6 +72,26 @@
</div>
</div>
<!-- Auto-sync (per-KB source watcher): periodically scans the directory above -->
<div v-if="canManageWiki" class="auto-sync-row">
<label class="auto-sync-toggle" :class="{ disabled: !watcher.globalEnabled || watcher.busy }">
<input
type="checkbox"
:checked="watcher.kbEnabled"
:disabled="!watcher.globalEnabled || watcher.busy"
@change="toggleWatcher(($event.target as HTMLInputElement).checked)"
/>
<span>{{ t('wiki.sources.autoSync') }}</span>
</label>
<span v-if="watcher.globalEnabled && watcher.kbEnabled" class="auto-sync-meta">
{{ t('wiki.sources.autoSyncInterval', { sec: Math.round(watcher.intervalMs / 1000) }) }}
<template v-if="watcher.sourceType"> · {{ watcher.sourceType }}</template>
</span>
<span v-else-if="!watcher.globalEnabled" class="auto-sync-hint">
{{ t('wiki.sources.autoSyncGlobalOffHint') }}
</span>
</div>
<!-- Raw materials list -->
<div class="raw-list">
<h4 class="raw-list-title">
@ -502,6 +522,48 @@ const dirPath = ref(store.currentKB?.sourceDirectory || '')
const scanning = ref(false)
const scanResult = ref<{ scanned: number; added: number; skipped: number; errors?: string[] } | null>(null)
// Per-KB auto-sync (source watcher)
// Auto-sync periodically scans the directory above. It runs only when the
// server-global master switch (watcher.globalEnabled, ops-controlled) AND this
// KB's toggle (watcher.kbEnabled) are both on. When the global switch is off the
// toggle is disabled with a hint there's nothing a non-ops user can do here.
const watcher = reactive({
globalEnabled: false,
kbEnabled: false,
intervalMs: 0,
sourceType: null as string | null,
busy: false,
})
async function loadWatcher(kbId: number) {
try {
const res: any = await wikiApi.getSourceWatcher(kbId)
const d = res?.data ?? res
watcher.globalEnabled = !!d.watcherEnabled
watcher.kbEnabled = !!d.kbWatcherEnabled
watcher.intervalMs = d.intervalMs || 0
watcher.sourceType = d.sourceType || null
} catch { /* leave defaults; auto-sync UI just shows disabled */ }
}
async function toggleWatcher(next: boolean) {
if (!store.currentKB) return
watcher.busy = true
try {
await wikiApi.setWatcherEnabled(store.currentKB.id, next)
watcher.kbEnabled = next
mcToast.success(t('common.saved'))
} catch (e: any) {
mcToast.error(e?.response?.data?.message || t('wiki.sources.toggleFailed'))
} finally {
watcher.busy = false
}
}
watch(() => store.currentKB?.id, (id) => {
if (id) {
dirPath.value = store.currentKB?.sourceDirectory || ''
void loadWatcher(id as number)
}
}, { immediate: true })
// Drag-over state
const { isDragging, onDragEnter, onDragLeave, onDrop: handleDrop } = useFileDrop(uploadDroppedFiles)
@ -707,6 +769,14 @@ async function handleScanDir() {
/* Directory scan */
.dir-scan-row { display: flex; gap: 10px; align-items: center; }
/* Auto-sync (per-KB source watcher) */
.auto-sync-row { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; margin-top: -4px; }
.auto-sync-toggle { display: inline-flex; align-items: center; gap: 6px; font-size: 13px; color: var(--mc-text-secondary); cursor: pointer; }
.auto-sync-toggle.disabled { opacity: 0.55; cursor: not-allowed; }
.auto-sync-toggle input { cursor: inherit; }
.auto-sync-meta { font-size: 12px; color: var(--mc-text-tertiary); font-variant-numeric: tabular-nums; }
.auto-sync-hint { font-size: 12px; color: var(--mc-text-tertiary); }
.dir-input-wrap { flex: 1; display: flex; align-items: center; gap: 8px; padding: 8px 12px; border: 1px solid var(--mc-border); border-radius: 12px; background: var(--mc-bg-elevated); color: var(--mc-text-tertiary); }
.dir-input-wrap:focus-within { border-color: var(--mc-primary); box-shadow: 0 0 0 2px rgba(217,119,87,0.1); }
.dir-input { flex: 1; border: none; background: transparent; font-size: 13px; color: var(--mc-text-primary); outline: none; }

View File

@ -107,41 +107,7 @@
</template>
</section>
<!-- ===================== REQ-4: Source Watcher ===================== -->
<section v-else-if="section === 'watcher'" class="adv-section">
<header class="adv-head">
<div>
<h3>{{ t('wiki.adv.watcher.title') }}</h3>
<p class="adv-desc">{{ t('wiki.adv.watcher.desc') }}</p>
</div>
<button class="btn-ghost" @click="loadWatcher" :disabled="watcher.busy">{{ t('common.refresh') }}</button>
</header>
<div class="kv-grid">
<div class="kv"><span>{{ t('wiki.adv.watcher.enabled') }}</span><b>{{ watcher.data.watcherEnabled ? t('common.yes') : t('common.no') }}</b></div>
<div class="kv"><span>{{ t('wiki.adv.watcher.active') }}</span><b>{{ watcher.data.active ? t('common.yes') : t('common.no') }}</b></div>
<div class="kv"><span>{{ t('wiki.adv.watcher.interval') }}</span><b>{{ watcher.data.intervalMs ? (watcher.data.intervalMs / 1000) + 's' : '—' }}</b></div>
<div class="kv"><span>{{ t('wiki.adv.watcher.sourceType') }}</span><b>{{ watcher.data.sourceType || '—' }}</b></div>
</div>
<label class="field-label">{{ t('wiki.adv.watcher.directory') }}</label>
<textarea
v-model="watcher.dir"
class="code-editor dir-editor"
spellcheck="false"
:placeholder="t('wiki.adv.watcher.dirHint')"
></textarea>
<div class="adv-actions">
<button class="btn-ghost" @click="saveDirectory" :disabled="watcher.busy">{{ t('common.save') }}</button>
</div>
<p v-if="watcher.data.availableSourceTypes?.length" class="adv-desc">
{{ t('wiki.adv.watcher.availableTypes') }}: {{ watcher.data.availableSourceTypes.join(', ') }}
</p>
<div class="adv-actions">
<button class="btn-primary" @click="triggerScan" :disabled="watcher.busy || !watcher.data.active">{{ t('wiki.adv.watcher.scanNow') }}</button>
</div>
<div v-if="watcher.lastScan" class="issue-box ok">
{{ t('wiki.adv.watcher.scanResult', { scanned: watcher.lastScan.scanned, added: watcher.lastScan.added, skipped: watcher.lastScan.skipped, errors: watcher.lastScan.errors }) }}
</div>
</section>
<!-- Source-watcher moved to the unified Sources tab (RawMaterialPanel). -->
<!-- ===================== REQ-5: Pipeline ===================== -->
<section v-else-if="section === 'pipeline'" class="adv-section">
@ -227,12 +193,11 @@ const agentStore = useAgentStore()
const kbId = computed(() => (store.currentKB ? String(store.currentKB.id) : ''))
const agents = computed(() => agentStore.agents)
const section = ref<'profile' | 'layers' | 'permissions' | 'watcher' | 'pipeline'>('profile')
const section = ref<'profile' | 'layers' | 'permissions' | 'pipeline'>('profile')
const sections = computed(() => [
{ key: 'profile' as const, label: t('wiki.adv.profile.tab') },
{ key: 'layers' as const, label: t('wiki.adv.layers.tab') },
{ key: 'permissions' as const, label: t('wiki.adv.perm.tab') },
{ key: 'watcher' as const, label: t('wiki.adv.watcher.tab') },
{ key: 'pipeline' as const, label: t('wiki.adv.pipeline.tab') },
])
@ -243,7 +208,6 @@ function switchSection(key: typeof section.value) {
loaded[key] = true
if (key === 'layers') loadLayers()
else if (key === 'permissions') { if (agents.value.length === 0) agentStore.fetchAgents() }
else if (key === 'watcher') loadWatcher()
else if (key === 'pipeline') loadPipelines()
}
@ -315,31 +279,7 @@ async function deletePermission(row: any) {
function flag(v: any) { return v ? '✓' : '·' }
function policyClass(p?: string) { return p === 'allow' ? 'badge-ok' : p === 'deny' ? 'badge-warn' : 'badge-muted' }
// ---- REQ-4 Watcher ----
const watcher = reactive({ data: {} as any, dir: '', lastScan: null as any, busy: false })
async function loadWatcher() {
if (!kbId.value) return
watcher.busy = true
try {
watcher.data = unwrap(await wikiApi.getSourceWatcher(kbId.value)) || {}
watcher.dir = watcher.data.sourceDirectory || ''
} catch (e: any) { mcToast.error(errMsg(e, 'Load watcher failed')) } finally { watcher.busy = false }
}
async function saveDirectory() {
watcher.busy = true
try {
await wikiApi.setSourceDirectory(kbId.value, watcher.dir)
mcToast.success(t('common.saved'))
await loadWatcher()
} catch (e: any) { mcToast.error(errMsg(e, 'Save failed')) } finally { watcher.busy = false }
}
async function triggerScan() {
watcher.busy = true
try {
watcher.lastScan = unwrap(await wikiApi.triggerSourceWatcher(kbId.value))
mcToast.success(t('wiki.adv.watcher.scanDone'))
} catch (e: any) { mcToast.error(errMsg(e, 'Scan failed')) } finally { watcher.busy = false }
}
// REQ-4 Source-watcher moved to the unified Sources tab (RawMaterialPanel).
// ---- REQ-5 Pipeline ----
const pipeline = reactive({ defs: [] as any[], config: '', issues: [] as string[], runsFor: null as any, runs: [] as any[], busy: false })

View File

@ -147,7 +147,7 @@ watch(() => store.currentPage, (page) => {
const tabs = computed<{ key: WikiTab; label: string }[]>(() => {
if (!canManageWiki.value) return []
return [
{ key: 'raw', label: t('wiki.rawMaterials') },
{ key: 'raw', label: t('wiki.sources.tab') },
{ key: 'config', label: t('wiki.config') },
{ key: 'transformations', label: t('wiki.transformations.tab') },
{ key: 'advanced', label: t('wiki.adv.tab') },