opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-13 03:13:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(wiki): support multi-path and glob patterns for KB source directory

Browse Source 
Allow knowledge base source paths to be configured as a newline-separated
list of absolute paths or glob patterns rather than a single directory.

- Each non-blank, non-# line is treated as one path or glob pattern
- Plain paths (no wildcards) retain the existing recursive-scan behaviour
  filtered by SUPPORTED_EXTENSIONS
- Glob patterns (e.g. /data/ocr/**/*.txt) walk from the fixed-prefix base
  and apply Java's PathMatcher against each candidate's absolute path
- Patterns whose filename segment explicitly specifies an extension
  (*.txt, *.{xlsx,csv}) skip the SUPPORTED_EXTENSIONS secondary filter,
  respecting the user's explicit choice (key for OCR pipelines that
  produce .txt output and should ignore the original PDF scans)
- Candidates collected across multiple patterns are deduplicated by
  resolved absolute path so overlapping patterns don't double-count
- Symlink-escape check uses each pattern's own validated scan root
- parseSourcePatterns / extractBasePath moved into WikiSourcePathValidator
  to eliminate a static circular reference between the two services
- Frontend watcher panel: single-line <input> replaced with <textarea>
  supporting multiline editing; i18n updated with example patterns
- No DB schema change; fully backward-compatible with existing single-path
  configs stored in sourceDirectory

Closes #(pending issue)
This commit is contained in:
倪程伟 2026-06-05 12:45:20 +08:00 committed by matevip
parent 9baef57871
commit d2e1f9eb9c
6 changed files with 252 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java
View File

@ -285,7 +285,7 @@ public class WikiController {
String path = body.get("path");
if (path != null && !path.isBlank()) {
try {
pathValidator.validateDirectory(path);
pathValidator.validateSourcePatterns(path);
} catch (IllegalArgumentException e) {
return R.fail(400, e.getMessage());
}

214
mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java
View File

@ -5,7 +5,6 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import vip.mate.wiki.WikiProperties;
import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
import vip.mate.wiki.model.WikiRawMaterialEntity;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@ -18,6 +17,13 @@ import java.util.*;
* <p>
* 扫描本地目录中的文档文件，为每个文件创建原始材料。
* 基于 sourcePath 去重，避免重复导入。
* <p>
* sourceDirectory 支持换行分隔的多条记录，每条可以是：
* <ul>
* <li>普通目录路径（如 {@code /data/docs}）——递归扫描，按 SUPPORTED_EXTENSIONS 过滤</li>
* <li>Glob 模式（如 {@code /data/ocr/**}{@code /*.txt}）——从固定前缀出发，用 PathMatcher 过滤</li>
* </ul>
* 以 {@code #} 开头的行视为注释，忽略。路径解析与验证委托给 {@link WikiSourcePathValidator}。
*
* @author MateClaw Team
*/
@ -38,13 +44,16 @@ public class WikiDirectoryScanService {
private static final Set<String> TEXT_EXTENSIONS = Set.of("txt", "md", "csv");
/** 一个待处理候选文件及其所属的扫描根（用于符号链接逃逸检测）。 */
private record FileCandidate(Path file, Path scanRoot) {}
/**
* 扫描结果
*/
public record ScanResult(int scanned, int added, int skipped, List<String> errors) {}
/**
* 扫描指定知识库关联的目录
* 扫描指定知识库关联的目录（支持多路径 + glob）
*/
public ScanResult scan(Long kbId) {
WikiKnowledgeBaseEntity kb = kbService.getById(kbId);
@ -59,79 +68,42 @@ public class WikiDirectoryScanService {
}
/**
* 扫描指定目录，为每个支持的文件创建原始材料
* 扫描指定路径配置，支持换行分隔的多条路径/Glob 模式。
* 单条普通路径时与旧行为完全兼容。
*/
public ScanResult scanDirectory(Long kbId, String directoryPath) {
Path dir;
try {
// Canonicalize (resolving symlinks) and enforce allowed-roots so a
// scan cannot read outside the authorized area.
dir = pathValidator.validateDirectory(directoryPath);
} catch (IllegalArgumentException e) {
return new ScanResult(0, 0, 0, List.of(e.getMessage()));
List<String> patterns = WikiSourcePathValidator.parseSourcePatterns(directoryPath);
if (patterns.isEmpty()) {
return new ScanResult(0, 0, 0, List.of("No source directory configured"));
}
return scanWithPatterns(kbId, patterns);
}
if (!Files.exists(dir)) {
return new ScanResult(0, 0, 0, List.of("Directory does not exist: " + dir));
}
if (!Files.isDirectory(dir)) {
return new ScanResult(0, 0, 0, List.of("Path is not a directory: " + dir));
}
// ==================== private ====================
List<Path> files = new ArrayList<>();
private ScanResult scanWithPatterns(Long kbId, List<String> patterns) {
List<FileCandidate> candidates = new ArrayList<>();
List<String> errors = new ArrayList<>();
int maxFiles = properties.getMaxScanFiles();
long maxFileSize = properties.getMaxScanFileSize();
// 递归遍历目录
try {
Files.walkFileTree(dir, new SimpleFileVisitor<>() {
@Override
public FileVisitResult preVisitDirectory(Path d, BasicFileAttributes attrs) {
// 跳过隐藏目录
String name = d.getFileName().toString();
if (name.startsWith(".") && !d.equals(dir)) {
return FileVisitResult.SKIP_SUBTREE;
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
if (files.size() >= maxFiles) {
return FileVisitResult.TERMINATE;
}
String fileName = file.getFileName().toString();
// 跳过隐藏文件
if (fileName.startsWith(".")) return FileVisitResult.CONTINUE;
// 跳过过大文件
if (attrs.size() > maxFileSize) {
log.debug("[Wiki] Skipping large file: {} ({} bytes)", file, attrs.size());
return FileVisitResult.CONTINUE;
}
// 检查扩展名
String ext = getExtension(fileName);
if (SUPPORTED_EXTENSIONS.contains(ext)) {
files.add(file);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
errors.add("Cannot read: " + file.getFileName() + " (" + exc.getMessage() + ")");
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
return new ScanResult(0, 0, 0, List.of("Failed to scan directory: " + e.getMessage()));
for (String pattern : patterns) {
if (candidates.size() >= maxFiles) break;
collectCandidates(pattern, candidates, errors, maxFiles, maxFileSize);
}
int scanned = files.size();
// Deduplicate: the same file can be matched by multiple overlapping patterns.
// Keep first-match order; first-match scanRoot wins for the symlink escape check.
Set<Path> seen = new LinkedHashSet<>();
candidates.removeIf(c -> !seen.add(c.file().toAbsolutePath().normalize()));
int scanned = candidates.size();
int added = 0;
int skipped = 0;
for (Path file : files) {
for (FileCandidate candidate : candidates) {
Path file = candidate.file();
Path scanRoot = candidate.scanRoot();
try {
// Per-file symlink guard: a symlinked file inside an allowed
// directory could point outside it (e.g. secret.md ->
@ -143,7 +115,7 @@ public class WikiDirectoryScanService {
} catch (IOException e) {
realFile = file.toAbsolutePath().normalize();
}
if (!realFile.startsWith(dir)) {
if (!realFile.startsWith(scanRoot)) {
errors.add("Skipped symlink escaping the scan root: " + file.getFileName());
skipped++;
continue;
@ -210,17 +182,127 @@ public class WikiDirectoryScanService {
}
}
if (files.size() >= maxFiles) {
if (candidates.size() >= maxFiles) {
errors.add("Scan limit reached (" + maxFiles + " files). Some files may have been skipped.");
}
log.info("[Wiki] Directory scan completed: dir={}, scanned={}, added={}, skipped={}, errors={}",
directoryPath, scanned, added, skipped, errors.size());
log.info("[Wiki] Scan completed: patterns={}, scanned={}, added={}, skipped={}, errors={}",
patterns, scanned, added, skipped, errors.size());
return new ScanResult(scanned, added, skipped, errors);
}
private String getExtension(String fileName) {
private void collectCandidates(String pattern, List<FileCandidate> candidates,
List<String> errors, int maxFiles, long maxFileSize) {
boolean hasWildcard = containsWildcard(pattern);
Path scanRoot;
PathMatcher matcher;
boolean requireSupportedExt;
if (!hasWildcard) {
// Plain directory: walk recursively, filter by SUPPORTED_EXTENSIONS.
try {
scanRoot = pathValidator.validateDirectory(pattern);
} catch (IllegalArgumentException e) {
errors.add(e.getMessage());
return;
}
if (!Files.exists(scanRoot) || !Files.isDirectory(scanRoot)) {
errors.add("Not a directory: " + scanRoot);
return;
}
matcher = null;
requireSupportedExt = true;
} else {
// Glob pattern: validate the fixed-prefix base, then apply PathMatcher.
String basePath = WikiSourcePathValidator.extractBasePath(pattern);
try {
scanRoot = pathValidator.validateDirectory(basePath);
} catch (IllegalArgumentException e) {
errors.add(e.getMessage());
return;
}
if (!Files.exists(scanRoot)) {
errors.add("Base directory does not exist: " + scanRoot);
return;
}
try {
matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern);
} catch (IllegalArgumentException e) {
errors.add("Invalid glob pattern '" + pattern + "': " + e.getMessage());
return;
}
// If the filename segment already specifies an extension (e.g. *.txt),
// skip the secondary SUPPORTED_EXTENSIONS filter to respect the explicit choice.
requireSupportedExt = !patternSpecifiesExtension(pattern);
}
final Path finalScanRoot = scanRoot;
final PathMatcher finalMatcher = matcher;
final boolean finalRequireExt = requireSupportedExt;
try {
Files.walkFileTree(scanRoot, new SimpleFileVisitor<>() {
@Override
public FileVisitResult preVisitDirectory(Path d, BasicFileAttributes attrs) {
String name = d.getFileName().toString();
if (name.startsWith(".") && !d.equals(finalScanRoot)) {
return FileVisitResult.SKIP_SUBTREE;
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
if (candidates.size() >= maxFiles) return FileVisitResult.TERMINATE;
String fileName = file.getFileName().toString();
if (fileName.startsWith(".")) return FileVisitResult.CONTINUE;
if (attrs.size() > maxFileSize) {
log.debug("[Wiki] Skipping large file: {} ({} bytes)", file, attrs.size());
return FileVisitResult.CONTINUE;
}
String ext = getExtension(fileName);
boolean accept;
if (finalMatcher != null) {
accept = finalMatcher.matches(file.toAbsolutePath());
if (accept && finalRequireExt) {
accept = SUPPORTED_EXTENSIONS.contains(ext);
}
} else {
accept = SUPPORTED_EXTENSIONS.contains(ext);
}
if (accept) {
candidates.add(new FileCandidate(file, finalScanRoot));
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
errors.add("Cannot read: " + file.getFileName() + " (" + exc.getMessage() + ")");
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
errors.add("Failed to scan '" + pattern + "': " + e.getMessage());
}
}
/**
* 判断 glob 模式的文件名段是否已显式指定扩展名（如 {@code *.txt}、{@code *.{txt,md}}），
* 是则不再叠加 SUPPORTED_EXTENSIONS 过滤，以尊重用户的明确选择。
*/
private static boolean patternSpecifiesExtension(String pattern) {
int lastSlash = pattern.lastIndexOf('/');
String lastSeg = lastSlash >= 0 ? pattern.substring(lastSlash + 1) : pattern;
return lastSeg.contains(".") && containsWildcard(lastSeg);
}
private static boolean containsWildcard(String s) {
return s.contains("*") || s.contains("?") || s.contains("{") || s.contains("[");
}
private static String getExtension(String fileName) {
int dot = fileName.lastIndexOf('.');
return dot > 0 ? fileName.substring(dot + 1).toLowerCase() : "";
}

89
mateclaw-server/src/main/java/vip/mate/wiki/service/WikiSourcePathValidator.java
View File

@ -8,7 +8,10 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
/**
* Single point of truth for validating a KB source directory path, shared by
@ -22,6 +25,10 @@ import java.util.List;
* canonicalized (opt-in enforcement — existing single-tenant / desktop setups
* keep working, server operators can lock it down).
*
* <p>Also owns the parsing helpers for the multi-line source-paths config
* format so that both the validation endpoint and the scan service share a
* single implementation.
*
* @author MateClaw Team
*/
@Slf4j
@ -34,6 +41,48 @@ public class WikiSourcePathValidator {
this.properties = properties;
}
// ==================== parsing helpers (stateless, no Spring context) ====================
/**
* Parse the {@code sourceDirectory} field: split by newline, strip blank
* lines and lines starting with {@code #}.
*/
public static List<String> parseSourcePatterns(String raw) {
if (raw == null || raw.isBlank()) return List.of();
return Arrays.stream(raw.split("\n"))
.map(String::trim)
.filter(s -> !s.isBlank() && !s.startsWith("#"))
.collect(Collectors.toList());
}
/**
* Extract the fixed-prefix base directory from a glob pattern — the
* leading path segments before the first wildcard segment.
* <p>
* Examples:
* <ul>
* <li>{@code /data/ocr/**}{@code /*.txt} → {@code /data/ocr}</li>
* <li>{@code /data/*.txt} → {@code /data}</li>
* <li>{@code /data/docs} → {@code /data/docs} (no wildcard)</li>
* </ul>
*/
public static String extractBasePath(String pattern) {
if (!containsWildcard(pattern)) {
return pattern;
}
String[] segments = pattern.split("/", -1);
List<String> baseSegments = new ArrayList<>();
for (String seg : segments) {
if (containsWildcard(seg)) break;
baseSegments.add(seg);
}
if (baseSegments.isEmpty()) return "/";
String joined = String.join("/", baseSegments);
return joined.isEmpty() ? "/" : joined;
}
// ==================== validation ====================
/**
* Canonicalize and authorize a source directory path.
*
@ -69,6 +118,40 @@ public class WikiSourcePathValidator {
"Path is outside the allowed source roots: " + resolved);
}
/**
* Validate all patterns in a multi-line source-directory config. Each
* non-blank, non-comment line is validated; the first violation is thrown.
*
* @throws IllegalArgumentException describing which line failed and why
*/
public void validateSourcePatterns(String raw) {
List<String> patterns = parseSourcePatterns(raw);
for (String pattern : patterns) {
validatePatternBase(pattern);
}
}
/**
* Validate a single path-or-glob-pattern: for glob patterns the
* fixed-prefix base directory is extracted and validated; for plain paths
* the path itself is validated.
*
* @return the resolved base directory path
* @throws IllegalArgumentException when the base is outside the allowed roots
*/
public Path validatePatternBase(String pattern) {
if (pattern == null || pattern.isBlank()) {
throw new IllegalArgumentException("Pattern is blank");
}
String basePath = extractBasePath(pattern);
try {
return validateDirectory(basePath);
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(
"Pattern '" + pattern + "' has an invalid base path: " + e.getMessage(), e);
}
}
/** Whether a path passes validation, without throwing. */
public boolean isAllowed(String rawPath) {
try {
@ -79,6 +162,8 @@ public class WikiSourcePathValidator {
}
}
// ==================== private ====================
private Path canonicalize(Path path) {
Path abs = path.toAbsolutePath().normalize();
if (Files.exists(abs)) {
@ -90,4 +175,8 @@ public class WikiSourcePathValidator {
}
return abs;
}
private static boolean containsWildcard(String s) {
return s.contains("*") || s.contains("?") || s.contains("{") || s.contains("[");
}
}

6
mateclaw-ui/src/i18n/locales/en-US.ts
View File

@ -2203,13 +2203,13 @@ export default {
watcher: {
tab: 'Watcher',
title: 'Source Directory Watcher',
desc: 'Bind a local directory as a knowledge source; the watcher scans for new/changed files at an interval and ingests them automatically. You can also trigger a one-off scan.',
desc: 'Configure local scan paths (multiple paths and glob patterns supported); the watcher scans for new/changed files at an interval and ingests them automatically. You can also trigger a one-off scan.',
enabled: 'Watcher enabled',
active: 'Active',
interval: 'Scan interval',
sourceType: 'Source type',
directory: 'Source directory',
dirHint: 'absolute path, e.g. /data/docs',
directory: 'Scan paths',
dirHint: 'One path or glob pattern per line; lines starting with # are comments. Examples:\n/data/docs\n/data/ocr/**/*.txt\n/data/reports/*.{xlsx,csv}',
availableTypes: 'Available source types',
scanNow: 'Scan now',
scanDone: 'Scan complete',

6
mateclaw-ui/src/i18n/locales/zh-CN.ts
View File

@ -2215,13 +2215,13 @@ export default {
watcher: {
tab: '变更监测',
title: '源目录变更监测',
desc: '关联一个本地目录作为知识来源，监测器会按间隔扫描新增/变更文件并自动摄取。也可手动触发一次扫描。',
desc: '配置本地扫描路径（支持多路径、Glob 通配符），监测器会按间隔扫描新增/变更文件并自动摄取。也可手动触发一次扫描。',
enabled: '监测开关',
active: '已激活',
interval: '扫描间隔',
sourceType: '源类型',
directory: '源目录',
dirHint: '绝对路径，例如 /data/docs',
directory: '扫描路径',
dirHint: '每行一条路径或 Glob 模式，# 开头为注释。示例：\n/data/docs\n/data/ocr/**/*.txt\n/data/reports/*.{xlsx,csv}',
availableTypes: '可用源类型',
scanNow: '立即扫描',
scanDone: '扫描完成',

10
mateclaw-ui/src/views/Wiki/components/WikiAdvancedPanel.vue
View File

@ -140,8 +140,13 @@
<div class="kv"><span>{{ t('wiki.adv.watcher.sourceType') }}</span><b>{{ watcher.data.sourceType || '—' }}</b></div>
</div>
<label class="field-label">{{ t('wiki.adv.watcher.directory') }}</label>
<div class="dir-row">
<input v-model.trim="watcher.dir" class="form-input" :placeholder="t('wiki.adv.watcher.dirHint')" />
<textarea
v-model="watcher.dir"
class="code-editor dir-editor"
spellcheck="false"
:placeholder="t('wiki.adv.watcher.dirHint')"
></textarea>
<div class="adv-actions">
<button class="btn-ghost" @click="saveDirectory" :disabled="watcher.busy">{{ t('common.save') }}</button>
</div>
<p v-if="watcher.data.availableSourceTypes?.length" class="adv-desc">
@ -505,6 +510,7 @@ onMounted(() => { loaded.profile = true; loadProfile() })
.kv b { font-size: 14px; color: var(--mc-text-primary); }
.dir-row { display: flex; gap: 8px; }
.dir-row .form-input { flex: 1; }
.dir-editor { min-height: 100px; font-size: 12.5px; }
.runs-box { border: 1px solid var(--mc-border-light); border-radius: 12px; padding: 12px; background: var(--mc-bg-muted); }
.runs-head { display: flex; justify-content: space-between; align-items: center; margin-bottom: 8px; }