diff --git a/mateclaw-server/src/main/java/vip/mate/datasource/controller/DatasourceController.java b/mateclaw-server/src/main/java/vip/mate/datasource/controller/DatasourceController.java new file mode 100644 index 00000000..6b432246 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/datasource/controller/DatasourceController.java @@ -0,0 +1,71 @@ +package vip.mate.datasource.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.*; +import vip.mate.common.result.R; +import vip.mate.datasource.model.DatasourceEntity; +import vip.mate.datasource.service.DatasourceService; + +import java.util.List; +import java.util.Map; + +/** + * 数据源管理接口 + * + * @author MateClaw Team + */ +@Tag(name = "数据源管理") +@RestController +@RequestMapping("/api/v1/datasources") +@RequiredArgsConstructor +public class DatasourceController { + + private final DatasourceService datasourceService; + + @Operation(summary = "获取数据源列表") + @GetMapping + public R> list() { + return R.ok(datasourceService.listAll()); + } + + @Operation(summary = "获取数据源详情") + @GetMapping("/{id}") + public R get(@PathVariable Long id) { + return R.ok(datasourceService.getByIdMasked(id)); + } + + @Operation(summary = "创建数据源") + @PostMapping + public R create(@RequestBody DatasourceEntity entity) { + return R.ok(datasourceService.create(entity)); + } + + @Operation(summary = "更新数据源") + @PutMapping("/{id}") + public R update(@PathVariable Long id, @RequestBody DatasourceEntity entity) { + entity.setId(id); + return R.ok(datasourceService.update(entity)); + } + + @Operation(summary = "删除数据源") + @DeleteMapping("/{id}") + public R delete(@PathVariable Long id) { + datasourceService.delete(id); + return R.ok(); + } + + @Operation(summary = "测试数据源连接") + @PostMapping("/{id}/test") + public R> testConnection(@PathVariable Long id) { + boolean ok = datasourceService.testConnection(id); + return R.ok(Map.of("success", ok, "message", ok ? "连接成功" : "连接失败")); + } + + @Operation(summary = "启用/禁用数据源") + @PutMapping("/{id}/toggle") + public R toggle(@PathVariable Long id, @RequestParam boolean enabled) { + return R.ok(datasourceService.toggle(id, enabled)); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/datasource/model/DatasourceEntity.java b/mateclaw-server/src/main/java/vip/mate/datasource/model/DatasourceEntity.java new file mode 100644 index 00000000..1b11a92c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/datasource/model/DatasourceEntity.java @@ -0,0 +1,67 @@ +package vip.mate.datasource.model; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * 外部数据源实体（查数功能） + * + * @author MateClaw Team + */ +@Data +@TableName("mate_datasource") +public class DatasourceEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + /** 数据源名称 */ + private String name; + + /** 描述 */ + private String description; + + /** 数据库类型：mysql / postgresql / clickhouse */ + private String dbType; + + /** 主机地址 */ + private String host; + + /** 端口 */ + private Integer port; + + /** 数据库名称 */ + private String databaseName; + + /** 用户名 */ + private String username; + + /** 密码（AES 加密存储） */ + private String password; + + /** JDBC URL 附加参数 */ + private String extraParams; + + /** PostgreSQL schema */ + private String schemaName; + + /** 是否启用 */ + private Boolean enabled; + + /** 最近测试时间 */ + private LocalDateTime lastTestTime; + + /** 最近测试结果 */ + private Boolean lastTestOk; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + @TableLogic + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/datasource/repository/DatasourceMapper.java b/mateclaw-server/src/main/java/vip/mate/datasource/repository/DatasourceMapper.java new file mode 100644 index 00000000..ccb8d685 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/datasource/repository/DatasourceMapper.java @@ -0,0 +1,14 @@ +package vip.mate.datasource.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.datasource.model.DatasourceEntity; + +/** + * 数据源 Mapper + * + * @author MateClaw Team + */ +@Mapper +public interface DatasourceMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/datasource/service/DatasourceConnectionManager.java b/mateclaw-server/src/main/java/vip/mate/datasource/service/DatasourceConnectionManager.java new file mode 100644 index 00000000..e26d6640 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/datasource/service/DatasourceConnectionManager.java @@ -0,0 +1,149 @@ +package vip.mate.datasource.service; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.stereotype.Component; +import vip.mate.datasource.model.DatasourceEntity; +import vip.mate.exception.MateClawException; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 外部数据源连接池管理器 + *

+ * 每个数据源维护一个独立的 HikariCP 连接池（max=3）， + * 通过 ConcurrentHashMap 缓存，配置变更时自动失效。 + * + * @author MateClaw Team + */ +@Slf4j +@Component +public class DatasourceConnectionManager implements DisposableBean { + + private final ConcurrentHashMap pools = new ConcurrentHashMap<>(); + + /** + * 获取指定数据源的 JDBC 连接 + */ + public Connection getConnection(DatasourceEntity entity) { + HikariDataSource ds = pools.computeIfAbsent(entity.getId(), id -> createPool(entity)); + try { + return ds.getConnection(); + } catch (SQLException e) { + throw new MateClawException("获取数据库连接失败: " + e.getMessage()); + } + } + + /** + * 失效指定数据源的连接池（配置更新/删除时调用） + */ + public void invalidate(Long datasourceId) { + HikariDataSource ds = pools.remove(datasourceId); + if (ds != null && !ds.isClosed()) { + ds.close(); + log.info("已关闭数据源连接池: {}", datasourceId); + } + } + + /** + * 创建临时连接测试数据源连通性 + */ + public boolean testConnection(DatasourceEntity entity) { + HikariConfig config = buildConfig(entity); + config.setMaximumPoolSize(1); + config.setMinimumIdle(0); + config.setConnectionTimeout(5000); + try (HikariDataSource testDs = new HikariDataSource(config); + Connection conn = testDs.getConnection()) { + return conn.isValid(5); + } catch (Exception e) { + log.warn("数据源连接测试失败 [{}]: {}", entity.getName(), e.getMessage()); + return false; + } + } + + private HikariDataSource createPool(DatasourceEntity entity) { + HikariConfig config = buildConfig(entity); + config.setMaximumPoolSize(3); + config.setMinimumIdle(1); + config.setConnectionTimeout(10000); + config.setIdleTimeout(300000); + config.setMaxLifetime(600000); + config.setPoolName("mateclaw-ds-" + entity.getId()); + log.info("创建数据源连接池: {} ({})", entity.getName(), entity.getDbType()); + return new HikariDataSource(config); + } + + private HikariConfig buildConfig(DatasourceEntity entity) { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl(buildJdbcUrl(entity)); + if (entity.getUsername() != null) { + config.setUsername(entity.getUsername()); + } + if (entity.getPassword() != null) { + config.setPassword(entity.getPassword()); + } + // 安全：设置连接为只读模式 + config.setReadOnly(true); + return config; + } + + /** + * 根据数据库类型构建 JDBC URL + */ + public static String buildJdbcUrl(DatasourceEntity entity) { + String dbType = entity.getDbType().toLowerCase(); + String host = entity.getHost(); + int port = entity.getPort(); + String dbName = entity.getDatabaseName(); + String extra = entity.getExtraParams(); + + String baseUrl; + switch (dbType) { + case "mysql": + case "mariadb": + baseUrl = String.format("jdbc:mysql://%s:%d/%s", host, port, dbName); + if (extra == null || extra.isBlank()) { + extra = "useUnicode=true&characterEncoding=utf8&useSSL=false&allowPublicKeyRetrieval=true"; + } + break; + case "postgresql": + baseUrl = String.format("jdbc:postgresql://%s:%d/%s", host, port, dbName); + if (entity.getSchemaName() != null && !entity.getSchemaName().isBlank()) { + String schemaParam = "currentSchema=" + entity.getSchemaName(); + extra = (extra == null || extra.isBlank()) ? schemaParam : extra + "&" + schemaParam; + } + break; + case "clickhouse": + baseUrl = String.format("jdbc:clickhouse://%s:%d/%s", host, port, dbName); + break; + default: + throw new MateClawException("不支持的数据库类型: " + dbType); + } + + if (extra != null && !extra.isBlank()) { + // 安全检查：拒绝危险参数 + String lowerExtra = extra.toLowerCase(); + if (lowerExtra.contains("allowloadlocalinfile") || lowerExtra.contains("autodeserialize")) { + throw new MateClawException("JDBC 参数包含不安全选项"); + } + baseUrl += (baseUrl.contains("?") ? "&" : "?") + extra; + } + return baseUrl; + } + + @Override + public void destroy() { + log.info("关闭所有外部数据源连接池 (共 {} 个)", pools.size()); + pools.forEach((id, ds) -> { + if (!ds.isClosed()) { + ds.close(); + } + }); + pools.clear(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/datasource/service/DatasourceService.java b/mateclaw-server/src/main/java/vip/mate/datasource/service/DatasourceService.java new file mode 100644 index 00000000..3618115c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/datasource/service/DatasourceService.java @@ -0,0 +1,155 @@ +package vip.mate.datasource.service; + +import cn.hutool.crypto.SecureUtil; +import cn.hutool.crypto.symmetric.AES; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import vip.mate.datasource.model.DatasourceEntity; +import vip.mate.datasource.repository.DatasourceMapper; +import vip.mate.exception.MateClawException; + +import java.nio.charset.StandardCharsets; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; + +/** + * 数据源业务服务 + * + * @author MateClaw Team + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class DatasourceService { + + private final DatasourceMapper datasourceMapper; + private final DatasourceConnectionManager connectionManager; + + @Value("${mateclaw.datasource.encrypt-key:MateClaw@2024Key!}") + private String encryptKey; + + // ==================== CRUD ==================== + + public List listAll() { + List list = datasourceMapper.selectList( + new LambdaQueryWrapper().orderByDesc(DatasourceEntity::getCreateTime)); + list.forEach(this::maskPassword); + return list; + } + + public List listEnabled() { + return datasourceMapper.selectList( + new LambdaQueryWrapper() + .eq(DatasourceEntity::getEnabled, true) + .orderByAsc(DatasourceEntity::getName)); + } + + public DatasourceEntity getById(Long id) { + DatasourceEntity entity = datasourceMapper.selectById(id); + if (entity == null) { + throw new MateClawException("数据源不存在: " + id); + } + return entity; + } + + public DatasourceEntity getByIdMasked(Long id) { + DatasourceEntity entity = getById(id); + maskPassword(entity); + return entity; + } + + public DatasourceEntity create(DatasourceEntity entity) { + if (entity.getEnabled() == null) { + entity.setEnabled(true); + } + encryptPassword(entity); + datasourceMapper.insert(entity); + return entity; + } + + public DatasourceEntity update(DatasourceEntity entity) { + DatasourceEntity existing = getById(entity.getId()); + // 如果前端传回的密码是脱敏值，保留原密码 + if ("******".equals(entity.getPassword()) || entity.getPassword() == null) { + entity.setPassword(existing.getPassword()); + } else { + encryptPassword(entity); + } + datasourceMapper.updateById(entity); + // 失效连接池缓存 + connectionManager.invalidate(entity.getId()); + return entity; + } + + public void delete(Long id) { + datasourceMapper.deleteById(id); + connectionManager.invalidate(id); + } + + public DatasourceEntity toggle(Long id, boolean enabled) { + DatasourceEntity entity = getById(id); + entity.setEnabled(enabled); + datasourceMapper.updateById(entity); + if (!enabled) { + connectionManager.invalidate(id); + } + return entity; + } + + // ==================== 连接测试 ==================== + + public boolean testConnection(Long id) { + DatasourceEntity entity = getById(id); + decryptPassword(entity); + boolean ok = connectionManager.testConnection(entity); + // 更新测试结果 + entity.setLastTestTime(LocalDateTime.now()); + entity.setLastTestOk(ok); + datasourceMapper.updateById(entity); + return ok; + } + + // ==================== 内部方法供 Tool 使用 ==================== + + /** + * 获取解密密码后的实体（供 Tool 层获取连接用） + */ + public DatasourceEntity getDecrypted(Long id) { + DatasourceEntity entity = getById(id); + decryptPassword(entity); + return entity; + } + + // ==================== 加解密 ==================== + + private AES getAes() { + byte[] key = Arrays.copyOf(encryptKey.getBytes(StandardCharsets.UTF_8), 16); + return SecureUtil.aes(key); + } + + private void encryptPassword(DatasourceEntity entity) { + if (entity.getPassword() != null && !entity.getPassword().isBlank()) { + entity.setPassword(getAes().encryptHex(entity.getPassword())); + } + } + + private void decryptPassword(DatasourceEntity entity) { + if (entity.getPassword() != null && !entity.getPassword().isBlank()) { + try { + entity.setPassword(getAes().decryptStr(entity.getPassword())); + } catch (Exception e) { + log.warn("密码解密失败（可能是明文存储的旧数据）: {}", entity.getName()); + } + } + } + + private void maskPassword(DatasourceEntity entity) { + if (entity.getPassword() != null && !entity.getPassword().isBlank()) { + entity.setPassword("******"); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/datasource/service/SqlValidationService.java b/mateclaw-server/src/main/java/vip/mate/datasource/service/SqlValidationService.java new file mode 100644 index 00000000..0e9b3e0e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/datasource/service/SqlValidationService.java @@ -0,0 +1,91 @@ +package vip.mate.datasource.service; + +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.Statements; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.statement.select.Limit; +import org.springframework.stereotype.Service; +import vip.mate.exception.MateClawException; + +/** + * SQL 安全验证服务 + *

+ * 仅允许 SELECT 语句，拒绝一切写操作。 + * 无 LIMIT 时自动注入 LIMIT 500。 + * + * @author MateClaw Team + */ +@Slf4j +@Service +public class SqlValidationService { + + private static final long DEFAULT_LIMIT = 500; + + /** + * 验证并处理 SQL： + * 1. 仅允许单条 SELECT 语句 + * 2. 无 LIMIT 时自动注入 LIMIT 500 + * + * @param sql 原始 SQL + * @return 处理后的安全 SQL + */ + public String validateAndNormalize(String sql) { + if (sql == null || sql.isBlank()) { + throw new MateClawException("SQL 不能为空"); + } + + // 去除末尾分号 + sql = sql.strip(); + if (sql.endsWith(";")) { + sql = sql.substring(0, sql.length() - 1).strip(); + } + + Statement statement; + try { + // 尝试解析为多条语句，检查是否有多语句注入 + Statements stmts = CCJSqlParserUtil.parseStatements(sql); + if (stmts.getStatements().size() != 1) { + throw new MateClawException("仅允许执行单条 SQL 语句，检测到 " + stmts.getStatements().size() + " 条"); + } + statement = stmts.getStatements().get(0); + } catch (JSQLParserException e) { + throw new MateClawException("SQL 解析失败: " + e.getMessage()); + } + + // 仅允许 SELECT + if (!(statement instanceof Select)) { + throw new MateClawException("仅允许 SELECT 查询，检测到: " + statement.getClass().getSimpleName()); + } + + Select select = (Select) statement; + + // 注入 LIMIT（如果缺失） + injectLimitIfAbsent(select); + + return select.toString(); + } + + private void injectLimitIfAbsent(Select select) { + if (select instanceof PlainSelect) { + PlainSelect plain = (PlainSelect) select; + if (plain.getLimit() == null) { + Limit limit = new Limit(); + limit.setRowCount(new LongValue(DEFAULT_LIMIT)); + plain.setLimit(limit); + } + } else if (select instanceof SetOperationList) { + SetOperationList setOp = (SetOperationList) select; + if (setOp.getLimit() == null) { + Limit limit = new Limit(); + limit.setRowCount(new LongValue(DEFAULT_LIMIT)); + setOp.setLimit(limit); + } + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/DatasourceTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DatasourceTool.java new file mode 100644 index 00000000..64ae98cf --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DatasourceTool.java @@ -0,0 +1,183 @@ +package vip.mate.tool.builtin; + +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.stereotype.Component; +import vip.mate.datasource.model.DatasourceEntity; +import vip.mate.datasource.service.DatasourceConnectionManager; +import vip.mate.datasource.service.DatasourceService; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; + +/** + * 内置工具：数据源发现 + *

+ * 提供数据源列表查询、表列表查询、表结构查询三个动作， + * 供 Agent 在查数场景下发现可用数据源和表结构。 + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class DatasourceTool { + + private final DatasourceService datasourceService; + private final DatasourceConnectionManager connectionManager; + + @Tool(description = """ + 查询外部数据源的元数据。支持三种动作： + 1. action='list_datasources' — 列出所有可用数据源（无需其他参数） + 2. action='list_tables' — 列出指定数据源中的所有表（需要 datasourceId） + 3. action='describe_table' — 查看指定表的列详情（需要 datasourceId 和 tableName） + """) + public String query_datasource( + @ToolParam(description = "动作：list_datasources / list_tables / describe_table") String action, + @ToolParam(description = "数据源 ID（list_tables 和 describe_table 时必填）", required = false) Long datasourceId, + @ToolParam(description = "表名（describe_table 时必填）", required = false) String tableName) { + + try { + return switch (action) { + case "list_datasources" -> listDatasources(); + case "list_tables" -> listTables(datasourceId); + case "describe_table" -> describeTable(datasourceId, tableName); + default -> error("未知动作: " + action + "，支持: list_datasources / list_tables / describe_table"); + }; + } catch (Exception e) { + log.error("数据源查询失败: {}", e.getMessage(), e); + return error(e.getMessage()); + } + } + + private String listDatasources() { + List list = datasourceService.listEnabled(); + JSONArray arr = new JSONArray(); + for (DatasourceEntity ds : list) { + JSONObject obj = new JSONObject(); + obj.set("id", ds.getId()); + obj.set("name", ds.getName()); + obj.set("dbType", ds.getDbType()); + obj.set("databaseName", ds.getDatabaseName()); + obj.set("description", ds.getDescription()); + arr.add(obj); + } + JSONObject result = new JSONObject(); + result.set("datasources", arr); + result.set("count", arr.size()); + return result.toStringPretty(); + } + + private String listTables(Long datasourceId) throws SQLException { + if (datasourceId == null) { + return error("list_tables 需要 datasourceId 参数"); + } + DatasourceEntity entity = datasourceService.getDecrypted(datasourceId); + String dbType = entity.getDbType().toLowerCase(); + + String sql = switch (dbType) { + case "mysql", "mariadb" -> String.format( + "SELECT TABLE_NAME, TABLE_COMMENT, TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = '%s' ORDER BY TABLE_NAME", + entity.getDatabaseName()); + case "postgresql" -> String.format( + "SELECT tablename AS table_name, obj_description(c.oid) AS table_comment " + + "FROM pg_tables t LEFT JOIN pg_class c ON c.relname = t.tablename " + + "WHERE t.schemaname = '%s' ORDER BY tablename", + entity.getSchemaName() != null ? entity.getSchemaName() : "public"); + case "clickhouse" -> "SHOW TABLES"; + default -> throw new IllegalArgumentException("不支持的数据库类型: " + dbType); + }; + + try (Connection conn = connectionManager.getConnection(entity); + Statement stmt = conn.createStatement()) { + stmt.setQueryTimeout(15); + ResultSet rs = stmt.executeQuery(sql); + return formatResultSet(rs, 200); + } + } + + private String describeTable(Long datasourceId, String tableName) throws SQLException { + if (datasourceId == null || tableName == null || tableName.isBlank()) { + return error("describe_table 需要 datasourceId 和 tableName 参数"); + } + DatasourceEntity entity = datasourceService.getDecrypted(datasourceId); + String dbType = entity.getDbType().toLowerCase(); + + String sql = switch (dbType) { + case "mysql", "mariadb" -> String.format( + "SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_KEY, COLUMN_DEFAULT, COLUMN_COMMENT " + + "FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' ORDER BY ORDINAL_POSITION", + entity.getDatabaseName(), tableName); + case "postgresql" -> String.format( + "SELECT c.column_name, c.data_type, c.is_nullable, " + + "CASE WHEN pk.column_name IS NOT NULL THEN 'PRI' ELSE '' END AS column_key, " + + "c.column_default, pgd.description AS column_comment " + + "FROM information_schema.columns c " + + "LEFT JOIN (SELECT ku.column_name FROM information_schema.table_constraints tc " + + "JOIN information_schema.key_column_usage ku ON tc.constraint_name = ku.constraint_name " + + "WHERE tc.table_name = '%s' AND tc.constraint_type = 'PRIMARY KEY') pk ON c.column_name = pk.column_name " + + "LEFT JOIN pg_catalog.pg_statio_all_tables st ON st.relname = c.table_name " + + "LEFT JOIN pg_catalog.pg_description pgd ON pgd.objoid = st.relid AND pgd.objsubid = c.ordinal_position " + + "WHERE c.table_schema = '%s' AND c.table_name = '%s' ORDER BY c.ordinal_position", + tableName, entity.getSchemaName() != null ? entity.getSchemaName() : "public", tableName); + case "clickhouse" -> String.format("DESCRIBE TABLE %s", tableName); + default -> throw new IllegalArgumentException("不支持的数据库类型: " + dbType); + }; + + try (Connection conn = connectionManager.getConnection(entity); + Statement stmt = conn.createStatement()) { + stmt.setQueryTimeout(15); + ResultSet rs = stmt.executeQuery(sql); + return formatResultSet(rs, 500); + } + } + + /** + * 将 ResultSet 格式化为 Markdown 表格 + */ + private String formatResultSet(ResultSet rs, int maxRows) throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + int colCount = meta.getColumnCount(); + + // 表头 + StringBuilder sb = new StringBuilder(); + List headers = new ArrayList<>(); + for (int i = 1; i <= colCount; i++) { + headers.add(meta.getColumnLabel(i)); + } + sb.append("| ").append(String.join(" | ", headers)).append(" |\n"); + sb.append("| ").append("--- | ".repeat(colCount)).append("\n"); + + // 数据行 + int rowCount = 0; + while (rs.next() && rowCount < maxRows) { + sb.append("| "); + for (int i = 1; i <= colCount; i++) { + String val = rs.getString(i); + sb.append(val != null ? val.replace("|", "\\|") : "NULL"); + if (i < colCount) sb.append(" | "); + } + sb.append(" |\n"); + rowCount++; + } + + if (rowCount == 0) { + return "查询结果为空"; + } + sb.append("\n共 ").append(rowCount).append(" 条记录"); + if (rowCount >= maxRows) { + sb.append("（已截断，实际可能更多）"); + } + return sb.toString(); + } + + private String error(String message) { + return JSONUtil.toJsonStr(new JSONObject().set("error", message)); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/SqlQueryTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/SqlQueryTool.java new file mode 100644 index 00000000..3c5c836f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/SqlQueryTool.java @@ -0,0 +1,142 @@ +package vip.mate.tool.builtin; + +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.stereotype.Component; +import vip.mate.datasource.model.DatasourceEntity; +import vip.mate.datasource.service.DatasourceConnectionManager; +import vip.mate.datasource.service.DatasourceService; +import vip.mate.datasource.service.SqlValidationService; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; + +/** + * 内置工具：SQL 查询执行 + *

+ * 仅允许 SELECT 语句。自动注入 LIMIT 保护。 + * 查询超时 30 秒。结果格式化为 Markdown 表格或 JSON。 + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class SqlQueryTool { + + private final DatasourceService datasourceService; + private final DatasourceConnectionManager connectionManager; + private final SqlValidationService sqlValidationService; + + private static final int QUERY_TIMEOUT_SECONDS = 30; + private static final int MAX_ROWS = 500; + private static final int MARKDOWN_TABLE_THRESHOLD = 20; + + @Tool(description = """ + 在外部数据源上执行只读 SQL 查询。 + 仅允许 SELECT 语句，禁止 INSERT/UPDATE/DELETE/DROP 等写操作。 + 如果 SQL 没有 LIMIT 子句会自动添加 LIMIT 500。 + 返回查询结果（Markdown 表格或 JSON 格式）以及行数和执行耗时。 + """) + public String execute_sql( + @ToolParam(description = "目标数据源 ID") Long datasourceId, + @ToolParam(description = "要执行的 SQL 查询（仅允许 SELECT）") String sql) { + + try { + // 1. 验证并规范化 SQL + String safeSql = sqlValidationService.validateAndNormalize(sql); + log.info("执行 SQL 查询 [数据源 {}]: {}", datasourceId, safeSql); + + // 2. 获取数据源连接 + DatasourceEntity entity = datasourceService.getDecrypted(datasourceId); + + // 3. 执行查询 + long startTime = System.currentTimeMillis(); + try (Connection conn = connectionManager.getConnection(entity); + Statement stmt = conn.createStatement()) { + + stmt.setQueryTimeout(QUERY_TIMEOUT_SECONDS); + stmt.setMaxRows(MAX_ROWS); + ResultSet rs = stmt.executeQuery(safeSql); + + long elapsed = System.currentTimeMillis() - startTime; + return formatResult(rs, safeSql, elapsed); + } + } catch (Exception e) { + log.error("SQL 查询执行失败: {}", e.getMessage(), e); + JSONObject result = new JSONObject(); + result.set("error", e.getMessage()); + result.set("sql", sql); + return result.toStringPretty(); + } + } + + private String formatResult(ResultSet rs, String sql, long elapsedMs) throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + int colCount = meta.getColumnCount(); + + // 收集列名 + List columns = new ArrayList<>(); + for (int i = 1; i <= colCount; i++) { + columns.add(meta.getColumnLabel(i)); + } + + // 收集数据 + List> rows = new ArrayList<>(); + while (rs.next() && rows.size() < MAX_ROWS) { + List row = new ArrayList<>(); + for (int i = 1; i <= colCount; i++) { + String val = rs.getString(i); + row.add(val != null ? val : "NULL"); + } + rows.add(row); + } + + StringBuilder sb = new StringBuilder(); + sb.append("**SQL**: `").append(sql).append("`\n"); + sb.append("**结果**: ").append(rows.size()).append(" 行, ").append(colCount).append(" 列"); + sb.append(" (耗时 ").append(elapsedMs).append("ms)\n\n"); + + if (rows.isEmpty()) { + sb.append("查询结果为空。"); + return sb.toString(); + } + + if (rows.size() <= MARKDOWN_TABLE_THRESHOLD && colCount <= 10) { + // Markdown 表格格式 + sb.append("| ").append(String.join(" | ", columns)).append(" |\n"); + sb.append("| ").append("--- | ".repeat(colCount)).append("\n"); + for (List row : rows) { + sb.append("| "); + for (int i = 0; i < row.size(); i++) { + sb.append(row.get(i).replace("|", "\\|")); + if (i < row.size() - 1) sb.append(" | "); + } + sb.append(" |\n"); + } + } else { + // JSON 格式（大结果集） + JSONArray jsonRows = new JSONArray(); + for (List row : rows) { + JSONObject obj = new JSONObject(); + for (int i = 0; i < columns.size(); i++) { + obj.set(columns.get(i), row.get(i)); + } + jsonRows.add(obj); + } + sb.append(jsonRows.toStringPretty()); + } + + if (rows.size() >= MAX_ROWS) { + sb.append("\n\n> 结果已截断至 ").append(MAX_ROWS).append(" 行，实际数据可能更多。"); + } + + return sb.toString(); + } +} diff --git a/mateclaw-server/src/main/resources/db/data-en.sql b/mateclaw-server/src/main/resources/db/data-en.sql index 53711703..401de0cc 100644 --- a/mateclaw-server/src/main/resources/db/data-en.sql +++ b/mateclaw-server/src/main/resources/db/data-en.sql @@ -472,6 +472,10 @@ MERGE INTO mate_skill (id, name, description, skill_type, icon, version, author, KEY (id) VALUES (1000000013, 'mateclaw_source_index', 'Map user questions to MateClaw doc paths and source code entry points to reduce blind searching.', 'builtin', '🗂️', '1.0.0', 'MateClaw', '{"upstream":"mateclaw","entryFile":"SKILL.md"}', TRUE, TRUE, 'docs,index,source,qa', NOW(), NOW(), 0); +MERGE INTO mate_skill (id, name, description, skill_type, icon, version, author, config_json, enabled, builtin, tags, create_time, update_time, deleted) +KEY (id) +VALUES (1000000014, 'sql_query', 'Query databases using natural language. Discover schemas, generate SQL, and execute read-only queries against configured external datasources.', 'builtin', '📊', '1.0.0', 'MateClaw', '{"upstream":"mateclaw","entryFile":"SKILL.md"}', TRUE, TRUE, 'sql,database,query,data', NOW(), NOW(), 0); + -- Populate skill_content for key built-in skills (SKILL.md execution protocol) -- NOTE: For pdf/docx/pptx/xlsx/himalaya, the authoritative SKILL.md is bundled in -- classpath:skills/{name}/ and auto-synced to workspace on startup. diff --git a/mateclaw-server/src/main/resources/db/data-zh.sql b/mateclaw-server/src/main/resources/db/data-zh.sql index 677d721c..1ef1ef78 100644 --- a/mateclaw-server/src/main/resources/db/data-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-zh.sql @@ -476,6 +476,10 @@ MERGE INTO mate_skill (id, name, description, skill_type, icon, version, author, KEY (id) VALUES (1000000013, 'mateclaw_source_index', '将用户问题映射到 MateClaw 文档路径与源码入口，减少盲目搜索。', 'builtin', '🗂️', '1.0.0', 'MateClaw', '{"upstream":"mateclaw","entryFile":"SKILL.md"}', TRUE, TRUE, 'docs,index,source,qa', NOW(), NOW(), 0); +MERGE INTO mate_skill (id, name, description, skill_type, icon, version, author, config_json, enabled, builtin, tags, create_time, update_time, deleted) +KEY (id) +VALUES (1000000014, 'sql_query', '自然语言查询数据库。发现表结构、生成 SQL 并在外部数据源上执行只读查询。', 'builtin', '📊', '1.0.0', 'MateClaw', '{"upstream":"mateclaw","entryFile":"SKILL.md"}', TRUE, TRUE, 'sql,database,query,data,查数', NOW(), NOW(), 0); + -- 为关键 builtin skill 填充 skill_content（SKILL.md 执行协议） -- NOTE: For pdf/docx/pptx/xlsx/himalaya, the authoritative SKILL.md is bundled in -- classpath:skills/{name}/ and auto-synced to workspace on startup. diff --git a/mateclaw-server/src/main/resources/db/schema.sql b/mateclaw-server/src/main/resources/db/schema.sql index 6fb75a5f..9d37d31f 100644 --- a/mateclaw-server/src/main/resources/db/schema.sql +++ b/mateclaw-server/src/main/resources/db/schema.sql @@ -383,6 +383,27 @@ CREATE TABLE IF NOT EXISTS mate_tool_guard_audit_log ( deleted INT NOT NULL DEFAULT 0 ); +-- 外部数据源表（查数功能） +CREATE TABLE IF NOT EXISTS mate_datasource ( + id BIGINT NOT NULL PRIMARY KEY, + name VARCHAR(128) NOT NULL, + description VARCHAR(512), + db_type VARCHAR(32) NOT NULL, + host VARCHAR(256) NOT NULL, + port INT NOT NULL, + database_name VARCHAR(128) NOT NULL, + username VARCHAR(128), + password VARCHAR(512), + extra_params VARCHAR(512), + schema_name VARCHAR(128), + enabled BOOLEAN NOT NULL DEFAULT TRUE, + last_test_time DATETIME, + last_test_ok BOOLEAN, + create_time DATETIME NOT NULL, + update_time DATETIME NOT NULL, + deleted INT NOT NULL DEFAULT 0 +); + -- 为现有表添加 metadata 列（向后兼容，防止迁移时数据丢失） ALTER TABLE mate_message ADD COLUMN IF NOT EXISTS metadata JSON; diff --git a/mateclaw-server/src/main/resources/db/tools-sync.sql b/mateclaw-server/src/main/resources/db/tools-sync.sql index f39e28e8..c8c9346f 100644 --- a/mateclaw-server/src/main/resources/db/tools-sync.sql +++ b/mateclaw-server/src/main/resources/db/tools-sync.sql @@ -52,3 +52,11 @@ VALUES (1000000011, 'WorkspaceMemoryTool', '工作区记忆', '读写数据库 MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) KEY (id) VALUES (1000000014, 'DelegateAgentTool', 'Agent 委派', '委派任务给其他 Agent 执行，实现多 Agent 协作。支持按名称调用目标 Agent，在独立会话中运行并返回结果。', 'builtin', 'delegateAgentTool', '🤝', TRUE, TRUE, NOW(), NOW(), 0); + +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000015, 'DatasourceTool', '数据源查询', '查询外部数据源的元数据：列出可用数据源、查看表列表、查看表结构（列名/类型/注释）。支持 MySQL、PostgreSQL、ClickHouse。', 'builtin', 'datasourceTool', '🗄', TRUE, TRUE, NOW(), NOW(), 0); + +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000016, 'SqlQueryTool', 'SQL 查询', '在外部数据源上执行只读 SQL 查询。仅允许 SELECT 语句，自动添加 LIMIT 保护，结果格式化为表格展示。', 'builtin', 'sqlQueryTool', '📊', TRUE, TRUE, NOW(), NOW(), 0); diff --git a/mateclaw-server/src/main/resources/skills/sql_query/SKILL.md b/mateclaw-server/src/main/resources/skills/sql_query/SKILL.md new file mode 100644 index 00000000..8c1aa88f --- /dev/null +++ b/mateclaw-server/src/main/resources/skills/sql_query/SKILL.md @@ -0,0 +1,68 @@ +--- +name: sql_query +description: "当用户提出数据查询需求（如'查数'、'查一下订单量'、'有多少用户'、'帮我跑个SQL'等），使用数据源工具发现表结构，生成并执行只读 SQL 查询。" +dependencies: + tools: + - datasourceTool + - sqlQueryTool +--- + +# SQL 查询技能 + +当用户提出与数据查询相关的问题时，按照以下工作流程操作。 + +## 工作流程 + +### 第一步：发现数据源 +调用 `query_datasource(action='list_datasources')` 查看所有可用的外部数据源。 +- 如果只有一个数据源，直接使用它 +- 如果有多个数据源，根据用户问题推断最相关的数据源；不确定时询问用户 + +### 第二步：发现表结构 +调用 `query_datasource(action='list_tables', datasourceId=)` 查看数据源中的表列表。 +- 根据表名和注释判断与用户问题相关的表 +- 不要一次查看所有表的结构，只查看相关的 2-3 张表 + +### 第三步：查看列详情 +调用 `query_datasource(action='describe_table', datasourceId=, tableName='<表名>')` 查看列名、类型和注释。 +- 记住列名和类型，生成 SQL 时必须使用正确的列名 + +### 第四步：生成 SQL +根据表结构和用户问题，生成 SELECT SQL。遵循以下规则： + +**SQL 生成规则：** +1. **仅生成 SELECT 语句**，绝不生成 INSERT/UPDATE/DELETE/DROP 等写操作 +2. 多表查询时，所有字段必须用表别名限定（如 `t1.name`） +3. 聚合查询使用 COUNT/SUM/AVG/MAX/MIN 等函数 +4. 时间过滤使用数据库对应的日期函数 +5. 字符串匹配使用 LIKE 并注意大小写 +6. 结果默认按合理的顺序排序（如时间倒序） +7. 系统会自动注入 LIMIT 500，无需手动添加（除非用户指定了数量） + +**不同数据库的注意事项：** +- MySQL：日期用 `DATE_FORMAT()`、`NOW()`，字符串连接用 `CONCAT()` +- PostgreSQL：日期用 `TO_CHAR()`、`NOW()`，字符串连接用 `||` +- ClickHouse：日期用 `toDate()`、`today()`，注意不支持部分标准 SQL 语法 + +### 第五步：执行查询 +调用 `execute_sql(datasourceId=, sql='')` 执行查询。 + +### 第六步：解读结果 +- 用自然语言总结查询结果的要点 +- 如果结果为空，分析可能的原因（表名/列名/条件有误等） +- 如果需要，可以调整 SQL 重新查询 + +## 错误处理 + +如果 SQL 执行失败： +1. 仔细阅读错误信息 +2. 常见原因：列名拼写错误、类型不匹配、语法错误 +3. 根据错误修正 SQL 并重试一次 +4. 如果仍然失败，向用户说明错误原因 + +## 安全须知 + +- 本工具仅支持只读查询（SELECT），写操作会被系统拒绝 +- 查询结果默认限制为 500 行 +- 查询超时为 30 秒 +- 如果用户要求执行写操作，应礼貌拒绝并说明这是安全限制 diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index 52b10fcc..fc7e4b2c 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -159,6 +159,18 @@ export const skillInstallApi = { http.delete(`/skills/install/${skillName}`), } +// ==================== Datasource ==================== +export const datasourceApi = { + list: () => http.get('/datasources'), + get: (id: string | number) => http.get(`/datasources/${id}`), + create: (data: any) => http.post('/datasources', data), + update: (id: string | number, data: any) => http.put(`/datasources/${id}`, data), + delete: (id: string | number) => http.delete(`/datasources/${id}`), + toggle: (id: string | number, enabled: boolean) => + http.put(`/datasources/${id}/toggle?enabled=${enabled}`), + test: (id: string | number) => http.post(`/datasources/${id}/test`), +} + // ==================== Tool ==================== export const toolApi = { list: () => http.get('/tools'), diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index fdb2c48d..32c06d81 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -162,6 +162,7 @@ export default { workspace: 'Workspace', skills: 'Skills', tools: 'Tools', + datasources: 'Datasources', mcpServers: 'MCP Servers', settingsGroup: 'Settings', agents: 'Agents', @@ -721,6 +722,65 @@ export default { toggleFailed: 'Failed to toggle tool status', }, }, + datasources: { + title: 'Datasources', + desc: 'Manage external database connections for natural language queries', + addButton: 'Add Datasource', + empty: 'No datasources configured. Click Add to get started.', + columns: { + name: 'Name', + type: 'Type', + connection: 'Connection', + status: 'Status', + actions: 'Actions', + }, + sections: { + basic: 'Basic Info', + connection: 'Connection', + auth: 'Authentication', + advanced: 'Advanced', + }, + modal: { + editTitle: 'Edit Datasource', + newTitle: 'Add Datasource', + }, + fields: { + name: 'Name', + dbType: 'Database Type', + host: 'Host', + port: 'Port', + databaseName: 'Database Name', + schemaName: 'Schema', + username: 'Username', + password: 'Password', + description: 'Description', + extraParams: 'JDBC Parameters', + }, + placeholders: { + name: 'e.g. Business Database', + host: '127.0.0.1', + port: '3306', + databaseName: 'Database name', + schemaName: 'public (default)', + username: 'Database username', + password: 'Database password', + description: 'What this datasource is for (optional)', + extraParams: 'useSSL=false&serverTimezone=UTC', + }, + hints: { + extraParams: 'Additional JDBC URL parameters, separated by &', + }, + testButton: 'Test Connection', + messages: { + saveFailed: 'Failed to save datasource', + deleteConfirm: 'Are you sure you want to delete this datasource? The agent will no longer be able to query it.', + deleteTitle: 'Confirm Delete', + deleteFailed: 'Failed to delete datasource', + toggleFailed: 'Failed to toggle datasource status', + testSuccess: 'Connection successful', + testFailed: 'Connection failed. Please check configuration.', + }, + }, cronJobs: { title: 'Cron Jobs', desc: 'Schedule agents to run messages or goals on a timer', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index a74f54a4..d0390dce 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -162,6 +162,7 @@ export default { workspace: '工作区', skills: '技能', tools: '工具', + datasources: '数据源', mcpServers: 'MCP 服务', settingsGroup: '设置', agents: '智能体', @@ -721,6 +722,65 @@ export default { toggleFailed: '切换工具状态失败', }, }, + datasources: { + title: '数据源管理', + desc: '管理外部数据库连接，用于自然语言查数', + addButton: '添加数据源', + empty: '暂无数据源，点击添加开始配置', + columns: { + name: '名称', + type: '类型', + connection: '连接', + status: '状态', + actions: '操作', + }, + sections: { + basic: '基本信息', + connection: '连接信息', + auth: '认证', + advanced: '高级配置', + }, + modal: { + editTitle: '编辑数据源', + newTitle: '添加数据源', + }, + fields: { + name: '名称', + dbType: '数据库类型', + host: '主机', + port: '端口', + databaseName: '数据库名', + schemaName: 'Schema', + username: '用户名', + password: '密码', + description: '描述', + extraParams: 'JDBC 参数', + }, + placeholders: { + name: '例如：业务数据库', + host: '127.0.0.1', + port: '3306', + databaseName: '数据库名', + schemaName: 'public（默认）', + username: '数据库用户名', + password: '数据库密码', + description: '数据源用途说明（可选）', + extraParams: 'useSSL=false&serverTimezone=UTC', + }, + hints: { + extraParams: 'JDBC URL 附加参数，多个参数用 & 分隔', + }, + testButton: '测试连接', + messages: { + saveFailed: '保存数据源失败', + deleteConfirm: '确定要删除这个数据源吗？删除后 Agent 将无法访问此数据源。', + deleteTitle: '确认删除', + deleteFailed: '删除数据源失败', + toggleFailed: '切换数据源状态失败', + testSuccess: '连接成功', + testFailed: '连接失败，请检查配置', + }, + }, cronJobs: { title: '定时任务', desc: '定时触发 Agent 执行消息或目标任务', diff --git a/mateclaw-ui/src/router/index.ts b/mateclaw-ui/src/router/index.ts index 3e2c3ac0..723a53c5 100644 --- a/mateclaw-ui/src/router/index.ts +++ b/mateclaw-ui/src/router/index.ts @@ -50,6 +50,12 @@ const router = createRouter({ component: () => import('@/views/Tools.vue'), meta: { title: 'Tools' }, }, + { + path: 'datasources', + name: 'Datasources', + component: () => import('@/views/Datasources.vue'), + meta: { title: 'Datasources' }, + }, { path: 'mcp-servers', name: 'McpServers', diff --git a/mateclaw-ui/src/views/Datasources.vue b/mateclaw-ui/src/views/Datasources.vue new file mode 100644 index 00000000..4f9a6dfe --- /dev/null +++ b/mateclaw-ui/src/views/Datasources.vue @@ -0,0 +1,513 @@ + + + + + diff --git a/mateclaw-ui/src/views/layout/MainLayout.vue b/mateclaw-ui/src/views/layout/MainLayout.vue index 243b291b..dd0d90e8 100644 --- a/mateclaw-ui/src/views/layout/MainLayout.vue +++ b/mateclaw-ui/src/views/layout/MainLayout.vue @@ -221,6 +221,11 @@ const navGroups = computed(() => [ label: t('nav.tools'), icon: ``, }, + { + path: '/datasources', + label: t('nav.datasources'), + icon: ``, + }, { path: '/mcp-servers', label: t('nav.mcpServers'),