+ * 以 {@code #} 开头的行视为注释，忽略。路径解析与验证委托给 {@link WikiSourcePathValidator}。
*
* @author MateClaw Team
*/
@@ -38,13 +44,16 @@ public class WikiDirectoryScanService {
private static final Set TEXT_EXTENSIONS = Set.of("txt", "md", "csv");
+ /** 一个待处理候选文件及其所属的扫描根（用于符号链接逃逸检测）。 */
+ private record FileCandidate(Path file, Path scanRoot) {}
+
/**
* 扫描结果
*/
public record ScanResult(int scanned, int added, int skipped, List errors) {}
/**
- * 扫描指定知识库关联的目录
+ * 扫描指定知识库关联的目录（支持多路径 + glob）
*/
public ScanResult scan(Long kbId) {
WikiKnowledgeBaseEntity kb = kbService.getById(kbId);
@@ -59,79 +68,42 @@ public class WikiDirectoryScanService {
}
/**
- * 扫描指定目录，为每个支持的文件创建原始材料
+ * 扫描指定路径配置，支持换行分隔的多条路径/Glob 模式。
+ * 单条普通路径时与旧行为完全兼容。
*/
public ScanResult scanDirectory(Long kbId, String directoryPath) {
- Path dir;
- try {
- // Canonicalize (resolving symlinks) and enforce allowed-roots so a
- // scan cannot read outside the authorized area.
- dir = pathValidator.validateDirectory(directoryPath);
- } catch (IllegalArgumentException e) {
- return new ScanResult(0, 0, 0, List.of(e.getMessage()));
+ List patterns = WikiSourcePathValidator.parseSourcePatterns(directoryPath);
+ if (patterns.isEmpty()) {
+ return new ScanResult(0, 0, 0, List.of("No source directory configured"));
}
+ return scanWithPatterns(kbId, patterns);
+ }
- if (!Files.exists(dir)) {
- return new ScanResult(0, 0, 0, List.of("Directory does not exist: " + dir));
- }
- if (!Files.isDirectory(dir)) {
- return new ScanResult(0, 0, 0, List.of("Path is not a directory: " + dir));
- }
+ // ==================== private ====================
- List files = new ArrayList<>();
+ private ScanResult scanWithPatterns(Long kbId, List patterns) {
+ List candidates = new ArrayList<>();
List errors = new ArrayList<>();
int maxFiles = properties.getMaxScanFiles();
long maxFileSize = properties.getMaxScanFileSize();
- // 递归遍历目录
- try {
- Files.walkFileTree(dir, new SimpleFileVisitor<>() {
- @Override
- public FileVisitResult preVisitDirectory(Path d, BasicFileAttributes attrs) {
- // 跳过隐藏目录
- String name = d.getFileName().toString();
- if (name.startsWith(".") && !d.equals(dir)) {
- return FileVisitResult.SKIP_SUBTREE;
- }
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
- if (files.size() >= maxFiles) {
- return FileVisitResult.TERMINATE;
- }
- String fileName = file.getFileName().toString();
- // 跳过隐藏文件
- if (fileName.startsWith(".")) return FileVisitResult.CONTINUE;
- // 跳过过大文件
- if (attrs.size() > maxFileSize) {
- log.debug("[Wiki] Skipping large file: {} ({} bytes)", file, attrs.size());
- return FileVisitResult.CONTINUE;
- }
- // 检查扩展名
- String ext = getExtension(fileName);
- if (SUPPORTED_EXTENSIONS.contains(ext)) {
- files.add(file);
- }
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult visitFileFailed(Path file, IOException exc) {
- errors.add("Cannot read: " + file.getFileName() + " (" + exc.getMessage() + ")");
- return FileVisitResult.CONTINUE;
- }
- });
- } catch (IOException e) {
- return new ScanResult(0, 0, 0, List.of("Failed to scan directory: " + e.getMessage()));
+ for (String pattern : patterns) {
+ if (candidates.size() >= maxFiles) break;
+ collectCandidates(pattern, candidates, errors, maxFiles, maxFileSize);
}
- int scanned = files.size();
+ // Deduplicate: the same file can be matched by multiple overlapping patterns.
+ // Keep first-match order; first-match scanRoot wins for the symlink escape check.
+ Set seen = new LinkedHashSet<>();
+ candidates.removeIf(c -> !seen.add(c.file().toAbsolutePath().normalize()));
+
+ int scanned = candidates.size();
int added = 0;
int skipped = 0;
- for (Path file : files) {
+ for (FileCandidate candidate : candidates) {
+ Path file = candidate.file();
+ Path scanRoot = candidate.scanRoot();
try {
// Per-file symlink guard: a symlinked file inside an allowed
// directory could point outside it (e.g. secret.md ->
@@ -143,7 +115,7 @@ public class WikiDirectoryScanService {
} catch (IOException e) {
realFile = file.toAbsolutePath().normalize();
}
- if (!realFile.startsWith(dir)) {
+ if (!realFile.startsWith(scanRoot)) {
errors.add("Skipped symlink escaping the scan root: " + file.getFileName());
skipped++;
continue;
@@ -210,17 +182,127 @@ public class WikiDirectoryScanService {
}
}
- if (files.size() >= maxFiles) {
+ if (candidates.size() >= maxFiles) {
errors.add("Scan limit reached (" + maxFiles + " files). Some files may have been skipped.");
}
- log.info("[Wiki] Directory scan completed: dir={}, scanned={}, added={}, skipped={}, errors={}",
- directoryPath, scanned, added, skipped, errors.size());
+ log.info("[Wiki] Scan completed: patterns={}, scanned={}, added={}, skipped={}, errors={}",
+ patterns, scanned, added, skipped, errors.size());
return new ScanResult(scanned, added, skipped, errors);
}
- private String getExtension(String fileName) {
+ private void collectCandidates(String pattern, List candidates,
+ List errors, int maxFiles, long maxFileSize) {
+ boolean hasWildcard = containsWildcard(pattern);
+ Path scanRoot;
+ PathMatcher matcher;
+ boolean requireSupportedExt;
+
+ if (!hasWildcard) {
+ // Plain directory: walk recursively, filter by SUPPORTED_EXTENSIONS.
+ try {
+ scanRoot = pathValidator.validateDirectory(pattern);
+ } catch (IllegalArgumentException e) {
+ errors.add(e.getMessage());
+ return;
+ }
+ if (!Files.exists(scanRoot) || !Files.isDirectory(scanRoot)) {
+ errors.add("Not a directory: " + scanRoot);
+ return;
+ }
+ matcher = null;
+ requireSupportedExt = true;
+ } else {
+ // Glob pattern: validate the fixed-prefix base, then apply PathMatcher.
+ String basePath = WikiSourcePathValidator.extractBasePath(pattern);
+ try {
+ scanRoot = pathValidator.validateDirectory(basePath);
+ } catch (IllegalArgumentException e) {
+ errors.add(e.getMessage());
+ return;
+ }
+ if (!Files.exists(scanRoot)) {
+ errors.add("Base directory does not exist: " + scanRoot);
+ return;
+ }
+ try {
+ matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern);
+ } catch (IllegalArgumentException e) {
+ errors.add("Invalid glob pattern '" + pattern + "': " + e.getMessage());
+ return;
+ }
+ // If the filename segment already specifies an extension (e.g. *.txt),
+ // skip the secondary SUPPORTED_EXTENSIONS filter to respect the explicit choice.
+ requireSupportedExt = !patternSpecifiesExtension(pattern);
+ }
+
+ final Path finalScanRoot = scanRoot;
+ final PathMatcher finalMatcher = matcher;
+ final boolean finalRequireExt = requireSupportedExt;
+
+ try {
+ Files.walkFileTree(scanRoot, new SimpleFileVisitor<>() {
+ @Override
+ public FileVisitResult preVisitDirectory(Path d, BasicFileAttributes attrs) {
+ String name = d.getFileName().toString();
+ if (name.startsWith(".") && !d.equals(finalScanRoot)) {
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+ if (candidates.size() >= maxFiles) return FileVisitResult.TERMINATE;
+ String fileName = file.getFileName().toString();
+ if (fileName.startsWith(".")) return FileVisitResult.CONTINUE;
+ if (attrs.size() > maxFileSize) {
+ log.debug("[Wiki] Skipping large file: {} ({} bytes)", file, attrs.size());
+ return FileVisitResult.CONTINUE;
+ }
+ String ext = getExtension(fileName);
+ boolean accept;
+ if (finalMatcher != null) {
+ accept = finalMatcher.matches(file.toAbsolutePath());
+ if (accept && finalRequireExt) {
+ accept = SUPPORTED_EXTENSIONS.contains(ext);
+ }
+ } else {
+ accept = SUPPORTED_EXTENSIONS.contains(ext);
+ }
+ if (accept) {
+ candidates.add(new FileCandidate(file, finalScanRoot));
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFileFailed(Path file, IOException exc) {
+ errors.add("Cannot read: " + file.getFileName() + " (" + exc.getMessage() + ")");
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ } catch (IOException e) {
+ errors.add("Failed to scan '" + pattern + "': " + e.getMessage());
+ }
+ }
+
+ /**
+ * 判断 glob 模式的文件名段是否已显式指定扩展名（如 {@code *.txt}、{@code *.{txt,md}}），
+ * 是则不再叠加 SUPPORTED_EXTENSIONS 过滤，以尊重用户的明确选择。
+ */
+ private static boolean patternSpecifiesExtension(String pattern) {
+ int lastSlash = pattern.lastIndexOf('/');
+ String lastSeg = lastSlash >= 0 ? pattern.substring(lastSlash + 1) : pattern;
+ return lastSeg.contains(".") && containsWildcard(lastSeg);
+ }
+
+ private static boolean containsWildcard(String s) {
+ return s.contains("*") || s.contains("?") || s.contains("{") || s.contains("[");
+ }
+
+ private static String getExtension(String fileName) {
int dot = fileName.lastIndexOf('.');
return dot > 0 ? fileName.substring(dot + 1).toLowerCase() : "";
}
diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiSourcePathValidator.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiSourcePathValidator.java
index 593a9f31..72078f03 100644
--- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiSourcePathValidator.java
+++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiSourcePathValidator.java
@@ -8,7 +8,10 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.stream.Collectors;
/**
* Single point of truth for validating a KB source directory path, shared by
@@ -22,6 +25,10 @@ import java.util.List;
* canonicalized (opt-in enforcement — existing single-tenant / desktop setups
* keep working, server operators can lock it down).
*
+ *
Also owns the parsing helpers for the multi-line source-paths config
+ * format so that both the validation endpoint and the scan service share a
+ * single implementation.
+ *
* @author MateClaw Team
*/
@Slf4j
@@ -34,6 +41,48 @@ public class WikiSourcePathValidator {
this.properties = properties;
}
+ // ==================== parsing helpers (stateless, no Spring context) ====================
+
+ /**
+ * Parse the {@code sourceDirectory} field: split by newline, strip blank
+ * lines and lines starting with {@code #}.
+ */
+ public static List parseSourcePatterns(String raw) {
+ if (raw == null || raw.isBlank()) return List.of();
+ return Arrays.stream(raw.split("\n"))
+ .map(String::trim)
+ .filter(s -> !s.isBlank() && !s.startsWith("#"))
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Extract the fixed-prefix base directory from a glob pattern — the
+ * leading path segments before the first wildcard segment.
+ *