From c627f898ec18c172e266f8f5aaff9045b4267910 Mon Sep 17 00:00:00 2001 From: matevip Date: Sun, 31 May 2026 07:59:34 +0800 Subject: [PATCH] feat(wiki): pluggable ingest-source SPI + source-watcher status API --- .../mate/wiki/controller/WikiController.java | 42 +++++++++++++++++++ .../service/WikiSourceWatcherService.java | 35 ++++++++++++---- .../wiki/source/FilesystemSourceProvider.java | 37 ++++++++++++++++ .../wiki/source/WikiIngestSourceProvider.java | 28 +++++++++++++ 4 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/source/FilesystemSourceProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/source/WikiIngestSourceProvider.java diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java index 060bb2f9..a65e41e7 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java @@ -61,6 +61,7 @@ public class WikiController { private final AuditEventService auditEventService; private final WikiPageTypeProfileService pageTypeProfileService; private final WikiSourcePathValidator pathValidator; + private final vip.mate.wiki.service.WikiSourceWatcherService sourceWatcherService; private final ObjectMapper objectMapper; // ==================== Knowledge Base ==================== @@ -302,6 +303,47 @@ public class WikiController { return R.ok(response); } + // ==================== Source Watcher ==================== + + @RequireWorkspaceRole("viewer") + @Operation(summary = "查看知识库源监听状态") + @GetMapping("/knowledge-bases/{id}/source-watcher") + public R> getSourceWatcher(@PathVariable Long id, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + verifyKBWorkspace(id, workspaceId); + WikiKnowledgeBaseEntity kb = kbService.getById(id); + if (kb == null) return R.fail(404, "Knowledge base not found"); + vip.mate.wiki.source.WikiIngestSourceProvider provider = sourceWatcherService.providerFor(kb); + Map out = new LinkedHashMap<>(); + out.put("watcherEnabled", properties.isWatcherEnabled()); + out.put("intervalMs", properties.getWatcherIntervalMs()); + out.put("sourceDirectory", kb.getSourceDirectory()); + out.put("sourceType", provider != null ? provider.sourceType() : null); + out.put("availableSourceTypes", sourceWatcherService.availableSourceTypes()); + out.put("active", provider != null); + return R.ok(out); + } + + @RequireWorkspaceRole("member") + @Operation(summary = "手动触发一次源监听扫描") + @PostMapping("/knowledge-bases/{id}/source-watcher/scan") + public R> triggerSourceWatcher(@PathVariable Long id, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + verifyKBWorkspace(id, workspaceId); + WikiKnowledgeBaseEntity kb = kbService.getById(id); + if (kb == null) return R.fail(404, "Knowledge base not found"); + vip.mate.wiki.source.WikiIngestSourceProvider provider = sourceWatcherService.providerFor(kb); + if (provider == null) return R.fail(400, "No source configured for this knowledge base"); + WikiDirectoryScanService.ScanResult result = provider.sync(kb); + Map out = new LinkedHashMap<>(); + out.put("sourceType", provider.sourceType()); + out.put("scanned", result.scanned()); + out.put("added", result.added()); + out.put("skipped", result.skipped()); + out.put("errors", result.errors()); + return R.ok(out); + } + // ==================== Raw Materials ==================== @RequireWorkspaceRole("viewer") diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiSourceWatcherService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiSourceWatcherService.java index 1fc9ef3a..2101a130 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiSourceWatcherService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiSourceWatcherService.java @@ -26,15 +26,20 @@ import vip.mate.wiki.model.WikiKnowledgeBaseEntity; public class WikiSourceWatcherService { private final WikiKnowledgeBaseService kbService; - private final WikiDirectoryScanService scanService; private final WikiProperties properties; + private final java.util.List sourceProviders; public WikiSourceWatcherService(WikiKnowledgeBaseService kbService, - WikiDirectoryScanService scanService, - WikiProperties properties) { + WikiProperties properties, + java.util.List sourceProviders) { this.kbService = kbService; - this.scanService = scanService; this.properties = properties; + this.sourceProviders = sourceProviders; + } + + /** The registered source-provider types (filesystem ships; api/mq pluggable later). */ + public java.util.List availableSourceTypes() { + return sourceProviders.stream().map(vip.mate.wiki.source.WikiIngestSourceProvider::sourceType).toList(); } /** Scheduled entry point — gated by config, serialized across instances. */ @@ -58,20 +63,32 @@ public class WikiSourceWatcherService { public int runScanCycle() { int totalAdded = 0; for (WikiKnowledgeBaseEntity kb : kbService.listAll()) { - String dir = kb.getSourceDirectory(); - if (dir == null || dir.isBlank()) { + vip.mate.wiki.source.WikiIngestSourceProvider provider = providerFor(kb); + if (provider == null) { continue; } try { - WikiDirectoryScanService.ScanResult result = scanService.scanDirectory(kb.getId(), dir); + WikiDirectoryScanService.ScanResult result = provider.sync(kb); totalAdded += result.added(); if (!result.errors().isEmpty()) { - log.warn("[WikiWatcher] KB {} scan reported issues: {}", kb.getId(), result.errors()); + log.warn("[WikiWatcher] KB {} ({}) sync reported issues: {}", + kb.getId(), provider.sourceType(), result.errors()); } } catch (Exception e) { - log.warn("[WikiWatcher] scan failed for KB {}: {}", kb.getId(), e.getMessage()); + log.warn("[WikiWatcher] sync failed for KB {} ({}): {}", + kb.getId(), provider.sourceType(), e.getMessage()); } } return totalAdded; } + + /** The first registered provider that supports the KB, or null. */ + public vip.mate.wiki.source.WikiIngestSourceProvider providerFor(WikiKnowledgeBaseEntity kb) { + for (vip.mate.wiki.source.WikiIngestSourceProvider p : sourceProviders) { + if (p.supports(kb)) { + return p; + } + } + return null; + } } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/source/FilesystemSourceProvider.java b/mateclaw-server/src/main/java/vip/mate/wiki/source/FilesystemSourceProvider.java new file mode 100644 index 00000000..9fc6d393 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/source/FilesystemSourceProvider.java @@ -0,0 +1,37 @@ +package vip.mate.wiki.source; + +import org.springframework.stereotype.Component; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.service.WikiDirectoryScanService; + +/** + * The built-in filesystem source: syncs a KB by scanning its configured source + * directory (path validation, symlink resolution and content-hash change + * detection live in the scan service). + * + * @author MateClaw Team + */ +@Component +public class FilesystemSourceProvider implements WikiIngestSourceProvider { + + private final WikiDirectoryScanService scanService; + + public FilesystemSourceProvider(WikiDirectoryScanService scanService) { + this.scanService = scanService; + } + + @Override + public String sourceType() { + return "filesystem"; + } + + @Override + public boolean supports(WikiKnowledgeBaseEntity kb) { + return kb != null && kb.getSourceDirectory() != null && !kb.getSourceDirectory().isBlank(); + } + + @Override + public WikiDirectoryScanService.ScanResult sync(WikiKnowledgeBaseEntity kb) { + return scanService.scanDirectory(kb.getId(), kb.getSourceDirectory()); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/source/WikiIngestSourceProvider.java b/mateclaw-server/src/main/java/vip/mate/wiki/source/WikiIngestSourceProvider.java new file mode 100644 index 00000000..d11d916e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/source/WikiIngestSourceProvider.java @@ -0,0 +1,28 @@ +package vip.mate.wiki.source; + +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.service.WikiDirectoryScanService; + +/** + * Pluggable source of raw material for a knowledge base. The watcher iterates + * KBs and asks each registered provider whether it {@link #supports} the KB, + * then {@link #sync}s it. The filesystem implementation ships today; API / + * message-queue sources can be added later by implementing this interface + * without touching the watcher. + * + * @author MateClaw Team + */ +public interface WikiIngestSourceProvider { + + /** Stable source-type id, e.g. {@code filesystem} / {@code api} / {@code mq}. */ + String sourceType(); + + /** Whether this provider can sync the given KB (e.g. it has the relevant config). */ + boolean supports(WikiKnowledgeBaseEntity kb); + + /** + * Pull new / changed material for the KB and ingest it, returning the + * scan-style result (scanned / added / skipped / errors). + */ + WikiDirectoryScanService.ScanResult sync(WikiKnowledgeBaseEntity kb); +}