This commit is contained in:
没钱 2026-09-03 14:14:40 +08:00
parent 79bd1db16f
commit 6a3b4ed4c3
81 changed files with 8072 additions and 27 deletions

BIN
.DS_Store vendored Normal file

Binary file not shown.

9
.dockerignore Normal file
View File

@ -0,0 +1,9 @@
.git
.idea
.vscode
.pnpm-store
**/.DS_Store
**/node_modules
**/dist
**/target
**/*.log

View File

@ -0,0 +1 @@
../../../ui/data-sync-s3-vue

29
deploy/Dockerfile.backend Normal file
View File

@ -0,0 +1,29 @@
FROM docker.m.daocloud.io/library/maven:3.9.11-eclipse-temurin-21 AS builder
WORKDIR /build
COPY . .
RUN mvn -Pprod -DskipTests -pl ruoyi-admin -am clean package
FROM docker.m.daocloud.io/library/eclipse-temurin:21-jre-jammy
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates curl \
&& curl -fsSL --connect-timeout 10 --max-time 30 https://ghfast.top/https://raw.githubusercontent.com/DingTalk-Real-AI/dingtalk-workspace-cli/main/scripts/install.sh -o /tmp/install-dws.sh \
&& env DWS_GITEE_REPO=DingTalk-Real-AI/dingtalk-workspace-cli DWS_VERSION=v1.0.60 DWS_NO_SKILLS=1 DWS_INSTALL_DIR=/usr/local/bin sh /tmp/install-dws.sh \
&& dws version \
&& rm -rf /var/lib/apt/lists/* /tmp/install-dws.sh /root/.cache
WORKDIR /app
RUN mkdir -p /app/logs /app/temp /var/lib/ruoyi-sync/dws
COPY --from=builder /build/ruoyi-admin/target/ruoyi-admin.jar /app/app.jar
ENV TZ=Asia/Shanghai \
JAVA_OPTS="-XX:+UseZGC -XX:+HeapDumpOnOutOfMemoryError" \
DWS_EXECUTABLE=/usr/local/bin/dws \
DWS_PROFILE_ROOT=/var/lib/ruoyi-sync/dws
EXPOSE 8080
HEALTHCHECK --interval=15s --timeout=5s --start-period=60s --retries=10 \
CMD curl -fsS http://127.0.0.1:8080/ >/dev/null || exit 1
ENTRYPOINT ["sh", "-c", "exec java $JAVA_OPTS -jar /app/app.jar"]

15
deploy/Dockerfile.web Normal file
View File

@ -0,0 +1,15 @@
FROM docker.m.daocloud.io/library/node:22-bookworm-slim AS builder
WORKDIR /build
RUN corepack enable && corepack prepare pnpm@10.34.5 --activate
COPY ui/data-sync-s3-vue/package.json ui/data-sync-s3-vue/pnpm-lock.yaml ui/data-sync-s3-vue/pnpm-workspace.yaml ./
RUN pnpm install --frozen-lockfile
COPY ui/data-sync-s3-vue/ ./
RUN pnpm build:prod
FROM docker.m.daocloud.io/library/nginx:1.31.1-alpine
COPY deploy/nginx.conf /etc/nginx/nginx.conf
COPY --from=builder /build/dist/ /usr/share/nginx/html/
EXPOSE 80
HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=5 \
CMD wget -qO- http://127.0.0.1/ >/dev/null || exit 1

View File

@ -0,0 +1,103 @@
name: data-sync-s3
services:
mysql:
image: docker.m.daocloud.io/library/mysql:8.4.9
restart: unless-stopped
environment:
TZ: Asia/Shanghai
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}
MYSQL_DATABASE: ${MYSQL_DATABASE:-ry-vue}
command:
- --character-set-server=utf8mb4
- --collation-server=utf8mb4_general_ci
- --explicit_defaults_for_timestamp=true
- --lower_case_table_names=1
volumes:
- mysql-data:/var/lib/mysql
- ../script/sql/ry_vue.sql:/docker-entrypoint-initdb.d/01-ry-vue.sql:ro
- ../script/sql/ry_job.sql:/docker-entrypoint-initdb.d/02-ry-job.sql:ro
- ../script/sql/ry_workflow.sql:/docker-entrypoint-initdb.d/03-ry-workflow.sql:ro
- ../script/sql/ry_ai.sql:/docker-entrypoint-initdb.d/04-ry-ai.sql:ro
- ../script/sql/ry_sync.sql:/docker-entrypoint-initdb.d/05-ry-sync.sql:ro
healthcheck:
test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -p$$MYSQL_ROOT_PASSWORD --silent"]
interval: 10s
timeout: 5s
retries: 15
start_period: 30s
networks: [app]
redis:
image: docker.m.daocloud.io/library/redis:8.2.1-alpine
restart: unless-stopped
command: ["sh", "-c", "exec redis-server --appendonly yes --requirepass \"$$REDIS_PASSWORD\""]
environment:
TZ: Asia/Shanghai
REDIS_PASSWORD: ${REDIS_PASSWORD}
volumes:
- redis-data:/data
healthcheck:
test: ["CMD-SHELL", "redis-cli -a \"$$REDIS_PASSWORD\" ping | grep PONG"]
interval: 10s
timeout: 5s
retries: 10
networks: [app]
backend:
build:
context: ..
dockerfile: deploy/Dockerfile.backend
image: data-sync-s3/backend:latest
restart: unless-stopped
environment:
TZ: Asia/Shanghai
SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_URL: jdbc:mysql://mysql:3306/${MYSQL_DATABASE:-ry-vue}?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=Asia/Shanghai&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_USERNAME: root
SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_PASSWORD: ${MYSQL_ROOT_PASSWORD}
SPRING_DATA_REDIS_HOST: redis
SPRING_DATA_REDIS_PORT: 6379
SPRING_DATA_REDIS_PASSWORD: ${REDIS_PASSWORD}
MYBATIS_ENCRYPTOR_ENABLE: "true"
MYBATIS_ENCRYPTOR_PASSWORD: ${MYBATIS_ENCRYPTOR_PASSWORD}
DWS_EXECUTABLE: /usr/local/bin/dws
DWS_PROFILE_ROOT: /var/lib/ruoyi-sync/dws
DWS_KEYCHAIN_ROOT: /var/lib/ruoyi-sync/dws-keychain
SYNC_SCHEDULER_ENABLED: "true"
JAVA_OPTS: -Xms256m -Xmx1024m -XX:+UseZGC -XX:+HeapDumpOnOutOfMemoryError
volumes:
- backend-logs:/app/logs
- dws-profiles:/var/lib/ruoyi-sync/dws
- dws-keychains:/var/lib/ruoyi-sync/dws-keychain
depends_on:
mysql:
condition: service_healthy
redis:
condition: service_healthy
networks: [app]
web:
build:
context: ..
dockerfile: deploy/Dockerfile.web
image: data-sync-s3/web:latest
restart: unless-stopped
ports:
# The host already runs another Nginx on port 80. Expose this stack on
# 8080 and let the existing reverse proxy forward traffic here.
- "8080:80"
depends_on:
backend:
condition: service_healthy
networks: [app]
networks:
app:
driver: bridge
volumes:
mysql-data:
redis-data:
backend-logs:
dws-profiles:
dws-keychains:

55
deploy/nginx.conf Normal file
View File

@ -0,0 +1,55 @@
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 2048;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
client_max_body_size 100m;
gzip_static on;
server_tokens off;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
upstream backend {
server backend:8080;
}
server {
listen 80;
server_name _;
location / {
root /usr/share/nginx/html;
try_files $uri $uri/ /index.html;
index index.html;
}
location /prod-api/ {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_read_timeout 86400s;
proxy_buffering off;
proxy_cache off;
proxy_pass http://backend/;
}
location ~ ^(/[^/]*)?/actuator.*(/.*)?$ {
return 403;
}
}
}

View File

@ -429,6 +429,13 @@
<version>${revision}</version>
</dependency>
<!-- 数据同步模块 -->
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-sync</artifactId>
<version>${revision}</version>
</dependency>
<!-- 代码生成模块 -->
<dependency>
<groupId>org.dromara</groupId>

View File

@ -92,6 +92,12 @@
<artifactId>ruoyi-ai</artifactId>
</dependency>
<!-- 数据同步模块 -->
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-sync</artifactId>
</dependency>
<!-- demo模块 -->
<dependency>
<groupId>org.dromara</groupId>

View File

@ -12,9 +12,6 @@ import me.zhyd.oauth.utils.AuthStateUtils;
import org.dromara.common.core.constant.SystemConstants;
import org.dromara.common.core.domain.R;
import org.dromara.common.core.domain.model.LoginBody;
import org.dromara.common.core.enums.PushSourceEnum;
import org.dromara.common.core.enums.PushTypeEnum;
import org.dromara.common.core.utils.DateUtils;
import org.dromara.common.core.utils.MessageUtils;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.core.utils.ValidatorUtils;
@ -24,8 +21,6 @@ import org.dromara.common.satoken.utils.LoginHelper;
import org.dromara.common.social.config.properties.SocialLoginConfigProperties;
import org.dromara.common.social.config.properties.SocialProperties;
import org.dromara.common.social.utils.SocialUtils;
import org.dromara.system.api.MessageService;
import org.dromara.system.api.domain.PushPayloadDTO;
import org.dromara.system.api.model.RegisterBody;
import org.dromara.system.api.model.SocialLoginBody;
import org.dromara.system.domain.vo.SysClientVo;
@ -39,10 +34,6 @@ import org.dromara.web.service.SysRegisterService;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* 认证控制器提供登录注册社交绑定和退出能力
@ -62,8 +53,6 @@ public class AuthController {
private final ISysConfigService configService;
private final ISysSocialService socialUserService;
private final ISysClientService clientService;
private final ScheduledExecutorService scheduledExecutorService;
private final MessageService messageService;
/**
@ -91,18 +80,6 @@ public class AuthController {
// 登录
LoginVo loginVo = IAuthStrategy.login(body, client, grantType);
Long userId = LoginHelper.getUserId();
scheduledExecutorService.schedule(() -> {
messageService.publishMessage(
List.of(userId),
PushPayloadDTO.of(
PushTypeEnum.MESSAGE,
PushSourceEnum.BACKEND,
DateUtils.getTodayHour(new Date()) + "好,欢迎登录 RuoYi-Vue-Plus 后台管理系统",
null
)
);
}, 5, TimeUnit.SECONDS);
return R.ok(loginVo);
}

View File

@ -42,6 +42,32 @@ user:
# 密码锁定时间默认10分钟
lockTime: 10
# 数据同步配置
sync:
dingtalk:
# DingTalk Workspace CLI 可执行文件;生产环境可通过环境变量指定绝对路径
dws-executable: ${DWS_EXECUTABLE:dws}
# 每个连接使用独立 DWS 登录态目录,实际目录为该路径下的连接 ID
profile-root: ${DWS_PROFILE_ROOT:${user.home}/.ruoyi-sync/dws}
# DWS 加密 keychain 也按连接隔离;生产环境必须和 profile 一起持久化
keychain-root: ${DWS_KEYCHAIN_ROOT:${user.home}/.ruoyi-sync/dws-keychain}
auth:
# 是否允许通过管理后台发起钉钉设备授权
enabled: ${DWS_WEB_AUTH_ENABLED:true}
# 设备流最长等待时间DWS 服务端默认约 10 分钟)
timeout-seconds: ${DWS_WEB_AUTH_TIMEOUT_SECONDS:600}
# 启动接口等待 DWS 打印授权链接的最长时间;超时后前端继续轮询 STARTING
metadata-wait-seconds: ${DWS_WEB_AUTH_METADATA_WAIT_SECONDS:15}
# 单节点同时进行的设备授权数量上限
max-concurrent: ${DWS_WEB_AUTH_MAX_CONCURRENT:4}
# 终态会话保留时间,便于前端读取最终结果
retention-seconds: ${DWS_WEB_AUTH_RETENTION_SECONDS:900}
scheduler:
# 使用 SnailJob 等外部调度时应关闭内置调度器,避免重复触发
enabled: ${SYNC_SCHEDULER_ENABLED:true}
# Cron 同步计划扫描间隔(毫秒)
poll-interval-ms: ${SYNC_SCHEDULER_POLL_INTERVAL_MS:30000}
# Spring配置
spring:
application:
@ -133,13 +159,13 @@ mybatis-plus:
# 数据加密
mybatis-encryptor:
# 是否开启加密
enable: false
enable: ${MYBATIS_ENCRYPTOR_ENABLE:false}
# 默认加密算法
algorithm: BASE64
# 编码方式 BASE64/HEX。默认BASE64
encode: BASE64
# 安全秘钥 对称算法的秘钥 如AESSM4
password:
password: ${MYBATIS_ENCRYPTOR_PASSWORD:}
# 公私钥 非对称算法的公私钥 如SM2RSA
publicKey:
privateKey:

View File

@ -22,6 +22,7 @@
<module>ruoyi-system</module>
<module>ruoyi-workflow</module>
<module>ruoyi-ai</module>
<module>ruoyi-sync</module>
</modules>
</project>

View File

@ -0,0 +1,280 @@
# 数据同步后端V1
本模块把原来的“钉钉盘备份到 OSS”收敛为可扩展的数据同步内核。V1 支持:
- 源端:钉钉企业网盘 / 我的文件DingTalk Workspace CLI简称 DWS
- 目标端:标准 S3 兼容存储、阿里云 OSS复用项目原生 S3 客户端)
- 执行:手动运行、内置 Cron 调度、可选 SnailJob 入口
- 能力:全量或清单差异同步、在线文档导出、冲突策略、源端删除策略、连接测试、任务明细、对象清单和检查点
## 1. 架构与扩展点
```text
DingTalkSourceConnector
│ ScanResult / SourceContent
SyncExecutionWorker ── sync_job / sync_job_item / sync_object / sync_checkpoint
│ TargetWriteRequest
├──────────────► S3TargetConnector
└──────────────► AliyunOssTargetConnector
```
源、目标均通过接口隔离:
- `SourceConnector`:分页扫描、下载或导出源对象
- `TargetConnector`:检查对象、上传、删除目标对象
- `ConnectorRegistry`:按连接角色和类型选择实现
后续增加其他源或目标时,不需要修改计划、任务和对象清单的数据模型。
## 2. 数据库
先导入项目基础 SQL再导入 [`script/sql/ry_sync.sql`](../../script/sql/ry_sync.sql)。该脚本面向 MySQL包含菜单、权限和字典数据。
| 表 | 用途 |
|---|---|
| `sync_connection` | 源/目标连接及加密凭证 |
| `sync_plan` | 路径、策略、并发、Cron 和删除保护配置 |
| `sync_job` | 一次运行及汇总;数据库唯一键保证同一计划只有一个活动任务 |
| `sync_job_item` | 每个文件的上传、跳过或删除结果 |
| `sync_object` | 源对象与目标对象的持久清单,用于差异判断和删除发现 |
| `sync_checkpoint` | 已完成任务水位V1 不把未完成分页游标当作可提交检查点 |
| `sync_transfer_part` | 应用级分片续传的预留表V1 尚未写入 |
`delete_guard_percent` 默认是 50。若一次运行拟删除的目标文件比例超过阈值任务会失败且不会执行任何删除只有显式设为 100 才允许一次删除全部目标文件。
## 3. 必需的服务配置
生产环境必须启用字段加密,否则服务会拒绝保存 `secretJson`
```bash
MYBATIS_ENCRYPTOR_ENABLE=true
MYBATIS_ENCRYPTOR_PASSWORD=<由密钥管理系统注入的 AES 密钥>
DWS_EXECUTABLE=/opt/dws/bin/dws
DWS_PROFILE_ROOT=/var/lib/ruoyi-sync/dws
DWS_KEYCHAIN_ROOT=/var/lib/ruoyi-sync/dws-keychain
```
请不要把真实密钥写进 YAML、启动脚本或 Git。`SyncConnectionVo` 不含 `secretJson`,新增/修改接口的操作日志也排除了该字段。
后端运行时需要 JDK 21项目使用虚拟线程。DWS 建议固定到已验收版本;仓库提供的生产镜像默认使用 `v1.0.60`,升级 DWS 后应重新做一次设备登录和真实空间访问验收。
内置调度器默认开启。如改由 SnailJob 触发,应设置:
```bash
SYNC_SCHEDULER_ENABLED=false
```
SnailJob 执行器名为 `syncPlanJobExecutor`,任务参数是同步计划 ID。内置调度和 SnailJob 二选一。
## 4. 钉钉连接
### 4.1 DWS 运行契约
运行节点需要安装与验收 DWS并至少确认以下命令存在
```bash
dws drive list --help
dws drive download --help
dws doc export --help
dws wiki space list --help
```
本模块使用原子 `drive list` 保留版本、修改时间、扩展名等增量证据;普通文件使用原子 `drive download` 的分片、并发和断点能力;`adoc` 在线文档使用 `doc export`(自动提交、轮询并下载)导出为 `docx`
每个同步连接都有独立的 DWS 配置目录:
```text
${DWS_PROFILE_ROOT}/${connectionId}
```
登录 token 的加密 keychain 也按连接隔离,目录为:
```text
${DWS_KEYCHAIN_ROOT}/${connectionId}/dws-cli
```
所有业务命令还会显式传入该连接配置中的 `profile=corpId:userId`,不会依赖服务进程当前选中的账号。
### 4.2 登录态初始化
应用密钥不能代替用户 OAuth 登录态。现在首次登录可以完全从管理后台完成,不需要 SSH 到服务器:
1. 创建一个 `DINGTALK/SOURCE` 连接。`secretJson` 填写 AppKey/AppSecret`configJson` 可以暂时不填 `profile`
2. 网页调用 `POST /sync/connection/{connectionId}/auth/login/start`
3. 页面展示返回的 `verificationUriComplete`(可生成二维码),同时展示 `verificationUri``userCode` 作为备用方式。
4. 用户在自己的浏览器中打开链接并在钉钉完成确认;页面每 2 秒左右调用状态接口,直到 `SUCCESS`、`FAILED`、`EXPIRED` 或 `CANCELLED`
5. 授权成功后,服务器自动把 DWS 返回的稳定 `corpId:userId` 写入该连接的 `configJson.profile`,之后即可执行连接测试和同步。
启动接口返回的对象只含一次性授权链接、验证码和脱敏身份信息,绝不返回 access token、refresh token、device code 或 AppSecret。DWS 进程和加密登录态始终留在后端节点;浏览器只负责完成钉钉授权。
后端只接受 DWS 最终 JSON 中明确返回的 `corp_id`/`corpId` 与 `user_id`/`userId`(或明确的 `profile` 字段),缺少任一 ID 会安全失败不会根据普通日志文本猜测登录人。连接凭证属于创建者普通用户只能查看和操作自己创建的连接超级管理员可跨创建者管理登录操作也仅允许连接创建人或超级管理员发起、查询和取消。登录进行中禁止修改或删除该连接Profile 只能由 Web 授权回调变更,编辑旧表单不会清掉已完成的登录态。
接口契约:
| 方法 | 地址 | 说明 |
|---|---|---|
| `POST` | `/sync/connection/{id}/auth/login/start` | 启动设备流;可选查询参数 `expectedCorpId` 用于组织校验 |
| `GET` | `/sync/connection/{id}/auth/login/{sessionId}` | 查询会话状态和展示信息 |
| `POST` | `/sync/connection/{id}/auth/login/{sessionId}/cancel` | 取消会话并终止服务器上的 DWS 进程 |
DWS 设备流有服务端有效期(通常约 10 分钟过期后重新点击“登录”即可。DWS 会自动刷新已保存的登录态,日常同步不需要重复登录。组织必须允许 DWS/CLI 数据访问,且授权用户必须拥有目标空间的访问权限;否则会话会返回失败提示。
会话状态接口返回 `STARTING`、`WAITING_USER`、`FINALIZING`、`SUCCESS`、`FAILED`、`EXPIRED` 或 `CANCELLED`;进入 `FINALIZING` 后正在写回连接身份,页面应继续轮询并禁止重复发起登录。
生产部署必须持久化下面两个目录,否则重启后会丢失 DWS 登录态:
```text
${DWS_PROFILE_ROOT}
${DWS_KEYCHAIN_ROOT}
```
当前会话管理器保存在单节点内存中,因此 V1 要求登录页面的三次请求落到同一个后端实例(单实例部署天然满足;多实例请配置会话亲和或单独的认证节点)。后续需要无亲和的集群部署时,再把会话状态和进程协调迁移到 Redis/专用认证服务。
同一节点内Web 授权会独占该连接的 DWS 配置/keychain 锁;正在执行的扫描或下载会让登录请求稍后重试,登录期间新发起的 DWS 业务命令也会被拒绝。若同步 worker 和 Web/API 分布在多个后端节点,除会话亲和外还需要把这把连接锁迁移到 Redis 或数据库分布式锁。
如需运维兜底,仍可在服务器上执行等价的设备流命令;这不是正常使用路径:
```bash
DWS_CONFIG_DIR=/var/lib/ruoyi-sync/dws/<connectionId> \
DWS_KEYCHAIN_DIR=/var/lib/ruoyi-sync/dws-keychain/<connectionId> \
DWS_CLIENT_ID=<与该连接 secretJson 一致的 AppKey> \
DWS_CLIENT_SECRET=<与该连接 secretJson 一致的 AppSecret> \
dws auth login --device --no-browser
```
### 4.3 连接示例
请求中的 JSON 字符串为展示方便做了格式化;实际字段仍是字符串。
```json
{
"connectionName": "钉钉企业盘",
"connectionRole": "SOURCE",
"connectionType": "DINGTALK",
"configJson": "{\"profile\":\"corpId:userId\",\"spaceType\":\"orgSpace\",\"spaceId\":\"可选\",\"downloadPartSize\":\"32MB\",\"downloadParallel\":\"4\"}",
"secretJson": "{\"clientId\":\"AppKey\",\"clientSecret\":\"AppSecret\"}",
"status": "0"
}
```
- `spaceId` 可选:填写时只同步该空间;不填时按 `spaceType` 发现空间。
- `spaceType``orgSpace` 或 `mySpace`,默认 `orgSpace`
- `sourceRoot``/` 表示空间根;指定子目录时必须填写该目录的 `dentryUuid`,且连接必须指定 `spaceId`
- `downloadParallel`1 至 8默认 4。
- `downloadPartSize`DWS 支持的容量字符串,例如 `32MB`
- `configDir` 不允许由连接指定,避免一个连接读取另一个连接的登录态。
V1 只导出 `adoc -> docx`。`axls`、`able`、`appt`、`amind`、`adraw` 会被识别为在线对象并明确失败,不会错误地按普通文件下载;相应导出适配器留到后续版本。
## 5. S3 / 阿里云 OSS 连接
标准 S3 示例:
```json
{
"connectionName": "备份 S3",
"connectionRole": "TARGET",
"connectionType": "S3",
"endpoint": "s3.example.com",
"region": "us-east-1",
"bucketName": "company-backup",
"basePath": "dingtalk",
"configJson": "{\"useHttps\":true,\"pathStyleAccess\":true}",
"secretJson": "{\"accessKey\":\"...\",\"secretKey\":\"...\"}",
"status": "0"
}
```
阿里云 OSS 示例:
```json
{
"connectionName": "阿里云 OSS",
"connectionRole": "TARGET",
"connectionType": "ALIYUN_OSS",
"endpoint": "oss-cn-hangzhou.aliyuncs.com",
"region": "cn-hangzhou",
"bucketName": "company-backup",
"basePath": "dingtalk",
"configJson": "{\"useHttps\":true,\"pathStyleAccess\":false}",
"secretJson": "{\"accessKey\":\"...\",\"secretKey\":\"...\"}",
"status": "0"
}
```
`region` 为可选配置。留空时与 ruoyi-vue-plus 原生 OSS 配置保持一致,客户端使用 `us-east-1` 作为默认 Region如已知存储桶所在地域仍建议填写实际值例如 `cn-hangzhou`)。
连接测试会真实执行 `HeadBucket`。上传完成后再执行 `HeadObject`以远端实际大小、ETag、版本 ID 和元数据作为任务结果。
## 6. 同步语义
计划请求示例:
```json
{
"planName": "钉钉企业盘每日备份",
"sourceConnectionId": 1,
"targetConnectionId": 2,
"sourceRoot": "/",
"targetPrefix": "daily",
"syncMode": "INCREMENTAL",
"scheduleType": "CRON",
"cronExpression": "0 0 2 * * *",
"conflictStrategy": "OVERWRITE",
"deleteStrategy": "KEEP",
"deleteGuardPercent": 50,
"verifyMode": "SHA256",
"maxConcurrency": 4,
"bandwidthLimitKbps": 0,
"status": "0"
}
```
- `FULL`:扫描并重新处理全部文件。
- `INCREMENTAL`:仍完整枚举权威目录树,然后用 `versionToken -> hash -> modifiedTime + size` 依次判断变化;证据不足时保守地重新传输,不会只凭相同大小跳过。
- `OVERWRITE`:写入目标键。
- `SKIP`:目标键已存在时跳过。
- `KEEP_BOTH`:目标键已存在时附加稳定版本后缀。
- `KEEP` / `MARK`:源端消失时保留目标,仅更新清单和任务记录。
- `DELETE`:通过删除比例保护后删除目标;删除失败会保留待重试状态,不会误记为已完成。
- `SIZE`:以 `HeadObject.contentLength` 校验。
- `ETAG`:要求目标端回读到 ETag不能把不同厂商/不同分片策略的 ETag 当作源端内容哈希。
- `SHA256`:本地计算摘要、写入对象元数据并通过 `HeadObject` 回读校验。
目录页缺少集合、条目缺少关键字段、部分错误、重复游标、重复对象 ID 或目录环路都会使任务失败。失败任务不会进入源删除发现阶段。
## 7. 后端接口
| 方法 | 地址 | 权限 | 用途 |
|---|---|---|---|
| `GET` | `/sync/connection/list` | `sync:connection:list` | 连接分页 |
| `GET` | `/sync/connection/{id}` | `sync:connection:query` | 连接详情(不返回凭证) |
| `POST` | `/sync/connection` | `sync:connection:add` | 新增连接 |
| `PUT` | `/sync/connection` | `sync:connection:edit` | 修改连接;空 `secretJson` 保留原凭证 |
| `DELETE` | `/sync/connection/{ids}` | `sync:connection:remove` | 删除未被计划引用的连接 |
| `POST` | `/sync/connection/test/{id}` | `sync:connection:test` | 真实连通性测试 |
| `POST` | `/sync/connection/{id}/auth/login/start` | `sync:connection:auth` | Web 端启动钉钉设备授权 |
| `GET` | `/sync/connection/{id}/auth/login/{sessionId}` | `sync:connection:auth` | 查询 Web 授权状态 |
| `POST` | `/sync/connection/{id}/auth/login/{sessionId}/cancel` | `sync:connection:auth` | 取消 Web 授权 |
| `GET` | `/sync/plan/list` | `sync:plan:list` | 计划分页 |
| `GET` | `/sync/plan/{id}` | `sync:plan:query` | 计划详情 |
| `POST` / `PUT` / `DELETE` | `/sync/plan` | 对应 add/edit/remove | 计划维护 |
| `POST` | `/sync/job/run/{planId}` | `sync:job:run` | 立即运行 |
| `POST` | `/sync/job/retry/{jobId}` | `sync:job:retry` | 重新枚举并重试失败对象 |
| `POST` | `/sync/job/cancel/{jobId}` | `sync:job:cancel` | 取消并中断本节点任务 |
| `GET` | `/sync/job/list` | `sync:job:list` | 任务分页 |
| `GET` | `/sync/job/{id}` | `sync:job:query` | 任务汇总 |
| `GET` | `/sync/job/{id}/items` | `sync:job:query` | 文件明细 |
| `DELETE` | `/sync/job/{ids}` | `sync:job:remove` | 删除非活动任务及明细 |
| `GET` | `/sync/object/list` | `sync:object:list` | 对象清单分页 |
| `GET` | `/sync/object/{id}` | `sync:object:query` | 对象详情 |
## 8. 当前版本边界
- Web 登录会话是单节点内存态;多实例部署需要网关会话亲和或专用认证节点。不会把 token 或 device code 写入数据库或返回给浏览器。
- `sync_transfer_part` 为应用级、跨任务分片续传预留。当前 DWS 下载和项目原生 S3 客户端各自使用其内部大文件机制,但应用数据库尚不接管 uploadId/part 状态。
- 带宽字段已预留,但 V1 会拒绝非 0 值,避免界面显示“已限速”而运行时未生效。
- 应用进程异常退出可能遗留 `PENDING/RUNNING/CANCELING` 任务。为避免多实例误接管仍在其他节点运行的任务V1 不做激进的启动自动恢复;运维确认原节点已停止后,需要由 DBA 将遗留任务核销为 `CANCELED`,再点重试。
- 取消会中断本节点 DWS 命令并在上传、校验、删除前复查数据库状态;对象存储 SDK 已经提交到远端的请求只能尽力取消,取消后重试仍按幂等对象键收敛。
- 未接真实钉钉账号、S3 或 OSS 凭证时只能完成编译和静态契约验证;上线前必须做真实空间枚举、空文件、大文件、在线文档、权限变化、分页和删除保护验收。

View File

@ -0,0 +1,59 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-modules</artifactId>
<version>${revision}</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>ruoyi-sync</artifactId>
<description>可扩展数据同步与备份模块</description>
<dependencies>
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-common-core</artifactId>
</dependency>
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-common-json</artifactId>
</dependency>
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-common-redis</artifactId>
</dependency>
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-common-mybatis</artifactId>
</dependency>
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-common-oss</artifactId>
</dependency>
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-common-log</artifactId>
</dependency>
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-common-security</artifactId>
</dependency>
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-common-web</artifactId>
</dependency>
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-common-encrypt</artifactId>
</dependency>
<dependency>
<groupId>org.dromara</groupId>
<artifactId>ruoyi-common-job</artifactId>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,16 @@
package org.dromara.sync.config;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableScheduling;
/**
* 数据同步内置调度配置
*
* <p>该配置不依赖 SnailJob确保仅启用内置调度器时 {@code @Scheduled} 仍然生效</p>
*/
@Configuration(proxyBeanMethods = false)
@EnableScheduling
@ConditionalOnProperty(prefix = "sync.scheduler", name = "enabled", havingValue = "true", matchIfMissing = true)
public class SyncSchedulingConfig {
}

View File

@ -0,0 +1,44 @@
package org.dromara.sync.connector;
import lombok.RequiredArgsConstructor;
import org.dromara.common.core.exception.ServiceException;
import org.dromara.sync.constant.SyncConstants;
import org.dromara.sync.domain.SyncConnection;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* 按连接角色和类型解析连接器
*/
@Component
@RequiredArgsConstructor
public class ConnectorRegistry {
private final List<SourceConnector> sourceConnectors;
private final List<TargetConnector> targetConnectors;
public SyncConnector resolve(SyncConnection connection) {
if (SyncConstants.ROLE_SOURCE.equals(connection.getConnectionRole())) {
return source(connection.getConnectionType());
}
if (SyncConstants.ROLE_TARGET.equals(connection.getConnectionRole())) {
return target(connection.getConnectionType());
}
throw new ServiceException("不支持的连接角色:{}", connection.getConnectionRole());
}
public SourceConnector source(String type) {
return sourceConnectors.stream()
.filter(connector -> connector.type().equals(type))
.findFirst()
.orElseThrow(() -> new ServiceException("未找到源端连接器:{}", type));
}
public TargetConnector target(String type) {
return targetConnectors.stream()
.filter(connector -> connector.type().equals(type))
.findFirst()
.orElseThrow(() -> new ServiceException("未找到目标端连接器:{}", type));
}
}

View File

@ -0,0 +1,17 @@
package org.dromara.sync.connector;
import org.dromara.sync.connector.model.ScanRequest;
import org.dromara.sync.connector.model.ScanResult;
import org.dromara.sync.connector.model.SourceContent;
import org.dromara.sync.connector.model.SourceObject;
import org.dromara.sync.domain.SyncConnection;
/**
* 源端连接器契约
*/
public interface SourceConnector extends SyncConnector {
ScanResult scan(SyncConnection connection, ScanRequest request);
SourceContent download(SyncConnection connection, SourceObject object);
}

View File

@ -0,0 +1,25 @@
package org.dromara.sync.connector;
import org.dromara.sync.connector.model.ConnectorTestResult;
import org.dromara.sync.domain.SyncConnection;
/**
* 数据同步连接器基础契约
*/
public interface SyncConnector {
/**
* 获取连接器类型编码
*
* @return 连接类型
*/
String type();
/**
* 测试连接
*
* @param connection 连接配置
* @return 测试结果
*/
ConnectorTestResult test(SyncConnection connection);
}

View File

@ -0,0 +1,31 @@
package org.dromara.sync.connector;
import org.dromara.sync.connector.model.TargetWriteRequest;
import org.dromara.sync.connector.model.TargetWriteResult;
import org.dromara.sync.domain.SyncConnection;
/**
* 目标端连接器契约
*/
public interface TargetConnector extends SyncConnector {
TargetWriteResult upload(SyncConnection connection, TargetWriteRequest request);
/**
* 判断目标对象是否存在供冲突策略使用
*
* @param connection 目标连接
* @param objectKey 对象键
* @return 对象存在返回 {@code true}
*/
boolean exists(SyncConnection connection, String objectKey);
/**
* 删除目标对象只有同步计划明确配置删除策略时才允许调用
*
* @param connection 目标连接
* @param objectKey 对象键
* @return 是否删除成功
*/
boolean delete(SyncConnection connection, String objectKey);
}

View File

@ -0,0 +1,737 @@
package org.dromara.sync.connector.dingtalk;
import lombok.RequiredArgsConstructor;
import org.dromara.common.core.exception.ServiceException;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.json.utils.JsonUtils;
import org.dromara.sync.connector.SourceConnector;
import org.dromara.sync.connector.model.*;
import org.dromara.sync.constant.SyncConstants;
import org.dromara.sync.domain.SyncConnection;
import org.springframework.stereotype.Component;
import tools.jackson.databind.JsonNode;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.charset.StandardCharsets;
import java.io.IOException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeParseException;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import java.util.zip.ZipInputStream;
/**
* 基于 DingTalk Workspace CLI 的钉钉网盘源连接器
*/
@Component
@RequiredArgsConstructor
public class DingTalkSourceConnector implements SourceConnector {
private static final int PAGE_SIZE = 50;
private static final Set<String> ONLINE_EXTENSIONS = Set.of("adoc", "axls", "able", "appt", "amind", "adraw");
private static final String XLSX_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private final DwsCommandRunner commandRunner;
@Override
public String type() {
return SyncConstants.TYPE_DINGTALK;
}
@Override
public ConnectorTestResult test(SyncConnection connection) {
try {
Map<String, Object> config = commandRunner.config(connection);
String spaceId = value(config, "spaceId");
if (StringUtils.isNotBlank(spaceId)) {
DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, List.of(
"drive", "list", "--space-id", spaceId, "--limit", "1", "--format", "json"
), Duration.ofSeconds(60));
parseScanResult(result.stdout(), spaceId);
return ConnectorTestResult.success("钉钉空间访问成功");
}
List<String> arguments = new ArrayList<>(List.of(
"wiki", "space", "list", "--type", spaceType(config), "--format", "json"
));
if ("orgSpace".equals(spaceType(config))) {
arguments.add("--limit");
arguments.add("1");
}
DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, arguments, Duration.ofSeconds(60));
parseSpaceResult(result.stdout());
return ConnectorTestResult.success("钉钉空间发现成功");
} catch (Exception e) {
return ConnectorTestResult.failure(e.getMessage());
}
}
@Override
public ScanResult scan(SyncConnection connection, ScanRequest request) {
Map<String, Object> config = commandRunner.config(connection);
String spaceId = StringUtils.firstNonBlank(request.scopeId(), value(config, "spaceId"));
if (StringUtils.isBlank(spaceId)) {
if (StringUtils.isNotBlank(request.parentObjectId())) {
throw new ServiceException("指定钉盘子目录时必须在连接配置中提供 spaceId");
}
return scanSpaces(connection, config, request.cursor());
}
List<String> arguments = new ArrayList<>(List.of(
"drive", "list", "--limit", String.valueOf(PAGE_SIZE), "--format", "json"
));
addOption(arguments, "--space-id", spaceId);
addOption(arguments, "--folder", request.parentObjectId());
addOption(arguments, "--cursor", request.cursor());
DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, arguments, Duration.ofMinutes(5));
return parseScanResult(result.stdout(), spaceId);
}
@Override
public SourceContent download(SyncConnection connection, SourceObject object) {
if (object.isFolder()) {
throw new ServiceException("文件夹不能下载:{}", object.name());
}
String extension = normalizeExtension(object.extension());
if ("dlink".equalsIgnoreCase(extension)) {
throw new ServiceException("钉钉多维表链接(.dlink不支持按普通文件下载");
}
if ("amind".equalsIgnoreCase(extension) || "adraw".equalsIgnoreCase(extension)) {
throw new ServiceException("钉钉官方 DWS 暂不支持自动导出:{}", extension);
}
Path directory = null;
try {
directory = Files.createTempDirectory("ruoyi-sync-dingtalk-");
Path output = directory.resolve(resolveOutputName(object));
if ("dlink".equalsIgnoreCase(extension)) {
exportAITable(connection, object, output);
return new SourceContent(output, Files.size(output), XLSX_CONTENT_TYPE, true);
}
List<String> arguments = buildDownloadArguments(connection, object, output);
try {
commandRunner.run(connection, arguments, Duration.ofHours(2), directory);
} catch (ServiceException exportFailure) {
// DWS v1.0.60 cannot export an empty ALIDOC and reports the
// asynchronous submit as doc_write_commit_unknown. Confirm
// the document is truly empty before applying the fallback;
// all non-empty documents and unrelated failures must retain
// the original error.
if (!SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType())
|| !"adoc".equalsIgnoreCase(normalizeExtension(object.extension()))
|| !isEmptyOnlineDocument(connection, object)) {
throw exportFailure;
}
writeEmptyDocx(output);
}
if (!Files.isRegularFile(output)) {
throw new ServiceException("DWS 执行成功但未生成文件:{}", object.name());
}
String contentType = SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType())
? onlineDocumentContentType(object)
: object.contentType();
return new SourceContent(output, Files.size(output), contentType, true);
} catch (ServiceException e) {
cleanDirectory(directory);
throw e;
} catch (Exception e) {
cleanDirectory(directory);
throw new ServiceException("下载钉钉对象失败:{}", object.name(), e);
}
}
private boolean isEmptyOnlineDocument(SyncConnection connection, SourceObject object) {
try {
DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, List.of(
"doc", "+fetch", "--node", object.objectId(), "--format", "json", "--detail", "full"
), Duration.ofMinutes(2));
JsonNode root = JsonUtils.getJsonMapper().readTree(result.stdout());
JsonNode content = root.path("content.jsonml");
return !containsNonBlankText(content);
} catch (Exception ignored) {
return false;
}
}
private boolean containsNonBlankText(JsonNode node) {
if (node == null || node.isNull()) {
return false;
}
if (node.isTextual()) {
return StringUtils.isNotBlank(node.asString());
}
if (node.isArray() || node.isObject()) {
for (JsonNode child : node) {
if (containsNonBlankText(child)) {
return true;
}
}
}
return false;
}
private void writeEmptyDocx(Path output) throws Exception {
Files.deleteIfExists(output);
try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(output), StandardCharsets.UTF_8)) {
putZipEntry(zip, "[Content_Types].xml", """
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
<Default Extension="xml" ContentType="application/xml"/>
<Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
<Override PartName="/word/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml"/>
</Types>
""");
putZipEntry(zip, "_rels/.rels", """
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
</Relationships>
""");
putZipEntry(zip, "word/document.xml", """
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
<w:body><w:p/><w:sectPr/></w:body>
</w:document>
""");
putZipEntry(zip, "word/styles.xml", """
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<w:styles xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:docDefaults/></w:styles>
""");
}
}
private void putZipEntry(ZipOutputStream zip, String name, String content) throws IOException {
zip.putNextEntry(new ZipEntry(name));
zip.write(content.getBytes(StandardCharsets.UTF_8));
zip.closeEntry();
}
private List<String> buildDownloadArguments(SyncConnection connection, SourceObject object, Path output) {
if (SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType())) {
String extension = normalizeExtension(object.extension());
if ("axls".equalsIgnoreCase(extension)) {
return List.of(
"sheet", "export", "--node", object.objectId(), "--output", output.getFileName().toString(), "--yes"
);
}
if ("amind".equalsIgnoreCase(extension) || "adraw".equalsIgnoreCase(extension)) {
throw new ServiceException("钉钉官方 DWS 暂不支持自动导出:{}", extension);
}
if (!"adoc".equalsIgnoreCase(extension)) {
throw new ServiceException("当前版本暂不支持导出钉钉在线文档类型:{}", object.extension());
}
// Use the consolidated `doc export` command. The `+export`
// shortcut maps to the low-level submit operation in some DWS
// releases (including v1.0.60), which can return
// `doc_write_commit_unknown` before polling/downloading the
// completed export job. `doc export` performs submit, polling
// and download atomically and only returns after the output file
// has been written.
return List.of(
"doc", "export", "--node", object.objectId(), "--output", output.getFileName().toString(),
"--export-format", "docx", "--yes"
);
}
Map<String, Object> config = commandRunner.config(connection);
List<String> arguments = new ArrayList<>(List.of(
"drive", "download", "--node", object.objectId(),
"--output", output.getFileName().toString(), "--format", "json"
));
addOption(arguments, "--space-id",
StringUtils.firstNonBlank(metadataValue(object, "spaceId"), value(config, "spaceId")));
String partSize = value(config, "downloadPartSize");
addOption(arguments, "--part-size", StringUtils.isBlank(partSize) ? "32MB" : partSize);
String parallel = value(config, "downloadParallel");
addOption(arguments, "--parallel", StringUtils.isBlank(parallel) ? "4" : parallel);
return arguments;
}
private void exportAITable(SyncConnection connection, SourceObject object, Path output) throws Exception {
String url = StringUtils.firstNonBlank(metadataValue(object, "sourceUrl"),
object.name() != null && object.name().startsWith("http") ? object.name() : null,
object.objectId() != null && object.objectId().startsWith("http") ? object.objectId() : null);
if (StringUtils.isBlank(url) || !url.startsWith("http")) {
throw new ServiceException("无法解析钉钉多维表链接(.dlink缺少完整 alidocs/dlink URL");
}
DwsCommandRunner.DwsCommandResult resolved = commandRunner.run(connection, List.of(
"aitable", "+url-resolve", "--url", url, "--format", "json"
), Duration.ofMinutes(2));
JsonNode ids = JsonUtils.getJsonMapper().readTree(resolved.stdout());
String baseId = findText(ids, "baseId", "baseID");
String tableId = findText(ids, "tableId", "datasheetId", "sheetId");
if (StringUtils.isAnyBlank(baseId, tableId)) {
throw new ServiceException("无法解析钉钉多维表链接(.dlink未找到 baseId/tableId");
}
JsonNode result;
try {
DwsCommandRunner.DwsCommandResult exported = commandRunner.run(connection, List.of(
"aitable", "+export-data", "--base-id", baseId, "--scope", "table", "--format", "excel",
"--table-id", tableId, "--timeout-ms", "120000"
), Duration.ofMinutes(5));
String exportOutput = exported.stdout();
result = JsonUtils.getJsonMapper().readTree(exportOutput);
if (result == null && exportOutput != null && exportOutput.trim().startsWith("http")) {
result = JsonUtils.getJsonMapper().createObjectNode().put("downloadUrl", exportOutput.trim());
}
} catch (Exception exportFailure) {
result = null;
}
String downloadUrl = findText(result, "downloadUrl", "url", "fileUrl", "downloadURL");
if (StringUtils.isBlank(downloadUrl)) {
DwsCommandRunner.DwsCommandResult records = commandRunner.run(connection, List.of(
"aitable", "record", "query", "--base-id", baseId, "--table-id", tableId,
"--all", "--page-limit", "100"
), Duration.ofMinutes(5));
writeAITableWorkbook(output, minimalWorkbook(JsonUtils.getJsonMapper().readTree(records.stdout())),
url, object, baseId, tableId);
return;
}
HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build();
HttpRequest request = HttpRequest.newBuilder(URI.create(downloadUrl)).GET().build();
HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
if (response.statusCode() < 200 || response.statusCode() >= 300 || response.body().length == 0) {
throw new ServiceException("下载钉钉多维表导出文件失败HTTP {}", response.statusCode());
}
writeAITableWorkbook(output, response.body(), url, object, baseId, tableId);
}
private byte[] minimalWorkbook(JsonNode data) throws IOException {
StringBuilder rows = new StringBuilder();
int rowNumber = 1;
JsonNode records = data == null ? null : (data.isArray() ? data : data.path("records"));
if (records != null && records.isArray()) {
for (JsonNode record : records) {
rows.append(row(rowNumber++, "record", xmlEscape(record.toString())));
}
}
if (rows.isEmpty()) rows.append(row(1, "data", ""));
Map<String, byte[]> entries = new LinkedHashMap<>();
entries.put("[Content_Types].xml", "<?xml version=\"1.0\"?><Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\"><Default Extension=\"rels\" ContentType=\"application/vnd.openxmlformats-package.relationships+xml\"/><Default Extension=\"xml\" ContentType=\"application/xml\"/><Override PartName=\"/xl/workbook.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml\"/><Override PartName=\"/xl/worksheets/sheet1.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml\"/></Types>".getBytes(StandardCharsets.UTF_8));
entries.put("_rels/.rels", "<?xml version=\"1.0\"?><Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\"><Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument\" Target=\"xl/workbook.xml\"/></Relationships>".getBytes(StandardCharsets.UTF_8));
entries.put("xl/workbook.xml", "<?xml version=\"1.0\"?><workbook xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\" xmlns:r=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships\"><sheets><sheet name=\"Sheet1\" sheetId=\"1\" r:id=\"rId1\"/></sheets></workbook>".getBytes(StandardCharsets.UTF_8));
entries.put("xl/_rels/workbook.xml.rels", "<?xml version=\"1.0\"?><Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\"><Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet\" Target=\"worksheets/sheet1.xml\"/></Relationships>".getBytes(StandardCharsets.UTF_8));
entries.put("xl/worksheets/sheet1.xml", ("<?xml version=\"1.0\"?><worksheet xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\"><sheetData>" + rows + "</sheetData></worksheet>").getBytes(StandardCharsets.UTF_8));
ByteArrayOutputStream out = new ByteArrayOutputStream();
try (ZipOutputStream zip = new ZipOutputStream(out)) {
for (Map.Entry<String, byte[]> entry : entries.entrySet()) { zip.putNextEntry(new ZipEntry(entry.getKey())); zip.write(entry.getValue()); zip.closeEntry(); }
}
return out.toByteArray();
}
private String findText(JsonNode node, String... names) {
if (node == null || node.isNull()) return null;
for (String name : names) {
JsonNode value = node.path(name);
if (value.isTextual() && StringUtils.isNotBlank(value.asString())) return value.asString();
}
if (node.isArray() || node.isObject()) {
for (JsonNode child : node) {
String found = findText(child, names);
if (StringUtils.isNotBlank(found)) return found;
}
}
return null;
}
private void writeAITableWorkbook(Path output, byte[] workbook, String url, SourceObject object,
String baseId, String tableId) throws IOException {
// Keep the official exported workbook as Sheet1 and add a metadata Sheet2.
Map<String, byte[]> entries = new LinkedHashMap<>();
try (ZipInputStream zip = new ZipInputStream(new ByteArrayInputStream(workbook))) {
ZipEntry entry;
while ((entry = zip.getNextEntry()) != null) {
entries.put(entry.getName(), zip.readAllBytes());
}
}
byte[] workbookXml = entries.get("xl/workbook.xml");
byte[] rels = entries.get("xl/_rels/workbook.xml.rels");
if (workbookXml == null || rels == null) {
throw new IOException("导出的文件不是有效 XLSX");
}
String sheetName = "Sheet2";
String relId = "rId" + (entries.size() + 1);
String xml = new String(workbookXml, StandardCharsets.UTF_8);
int sheetsEnd = xml.indexOf("</sheets>");
if (sheetsEnd < 0) throw new IOException("XLSX 缺少 sheets 节点");
xml = xml.substring(0, sheetsEnd) + "<sheet name=\"Sheet2\" sheetId=\"2\" r:id=\"" + relId + "\"/>" + xml.substring(sheetsEnd);
String relXml = new String(rels, StandardCharsets.UTF_8);
int relEnd = relXml.indexOf("</Relationships>");
relXml = relXml.substring(0, relEnd) + "<Relationship Id=\"" + relId + "\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet\" Target=\"worksheets/sheet2.xml\"/>" + relXml.substring(relEnd);
entries.put("xl/workbook.xml", xml.getBytes(StandardCharsets.UTF_8));
entries.put("xl/_rels/workbook.xml.rels", relXml.getBytes(StandardCharsets.UTF_8));
String contentTypes = new String(entries.get("[Content_Types].xml"), StandardCharsets.UTF_8);
int ctEnd = contentTypes.indexOf("</Types>");
contentTypes = contentTypes.substring(0, ctEnd) + "<Override PartName=\"/xl/worksheets/sheet2.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml\"/>" + contentTypes.substring(ctEnd);
entries.put("[Content_Types].xml", contentTypes.getBytes(StandardCharsets.UTF_8));
String escapedUrl = xmlEscape(url);
String escapedName = xmlEscape(object.name());
String sheet2 = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><worksheet xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\"><sheetData>"
+ row(1, "Field", "Value") + row(2, "sourceUrl", escapedUrl) + row(3, "sourceObjectId", xmlEscape(object.objectId()))
+ row(4, "name", escapedName) + row(5, "baseId", xmlEscape(baseId)) + row(6, "tableId", xmlEscape(tableId))
+ row(7, "exportedAt", xmlEscape(Instant.now().toString())) + "</sheetData></worksheet>";
entries.put("xl/worksheets/sheet2.xml", sheet2.getBytes(StandardCharsets.UTF_8));
try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(output))) {
for (Map.Entry<String, byte[]> entry : entries.entrySet()) {
zip.putNextEntry(new ZipEntry(entry.getKey())); zip.write(entry.getValue()); zip.closeEntry();
}
}
}
private String row(int row, String key, String value) {
return "<row r=\"" + row + "\"><c r=\"A" + row + "\" t=\"inlineStr\"><is><t>" + key + "</t></is></c><c r=\"B" + row + "\" t=\"inlineStr\"><is><t>" + value + "</t></is></c></row>";
}
private String xmlEscape(String value) {
return value == null ? "" : value.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\"", "&quot;");
}
private String onlineDocumentContentType(SourceObject object) {
String extension = normalizeExtension(object.extension());
if ("axls".equalsIgnoreCase(extension)) {
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
}
if ("amind".equalsIgnoreCase(extension) || "adraw".equalsIgnoreCase(extension)) {
return "application/pdf";
}
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
}
private ScanResult scanSpaces(SyncConnection connection, Map<String, Object> config, String cursor) {
String spaceType = spaceType(config);
List<String> arguments = new ArrayList<>(List.of(
"wiki", "space", "list", "--type", spaceType, "--format", "json"
));
if ("orgSpace".equals(spaceType)) {
arguments.add("--limit");
arguments.add(String.valueOf(PAGE_SIZE));
addOption(arguments, "--cursor", cursor);
}
DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, arguments, Duration.ofMinutes(2));
return parseSpaceResult(result.stdout());
}
private ScanResult parseScanResult(String json, String spaceId) {
JsonNode root = JsonUtils.getJsonMapper().readTree(json);
validateEnvelope(root);
JsonNode payload = root.has("body") ? root.path("body") : root;
JsonNode array = findObjectArray(payload);
if (array == null) {
throw new ServiceException("无法识别 DWS 目录列表结果,已停止同步以避免误判删除");
}
List<SourceObject> objects = new ArrayList<>();
for (JsonNode item : array) {
objects.add(mapObject(item, spaceId));
}
String cursor = nestedText(payload, "nextCursor", "nextToken");
boolean hasMore = nestedBoolean(payload, "hasMore") || StringUtils.isNotBlank(cursor);
return new ScanResult(objects, cursor, hasMore);
}
private ScanResult parseSpaceResult(String json) {
JsonNode root = JsonUtils.getJsonMapper().readTree(json);
validateEnvelope(root);
JsonNode payload = root.has("body") ? root.path("body") : root;
JsonNode array = findObjectArray(payload);
if (array == null) {
throw new ServiceException("无法识别 DWS 空间列表结果,已停止同步");
}
List<SourceObject> spaces = new ArrayList<>();
for (JsonNode item : array) {
String spaceId = text(item, "spaceId", "id");
String rootFolderId = text(item, "rootFolderId", "rootDentryUuid", "rootNodeId");
String name = text(item, "spaceName", "name", "title");
if (StringUtils.isAnyBlank(spaceId, rootFolderId, name)) {
throw new ServiceException("DWS 空间列表包含缺少 spaceId、rootFolderId 或名称的条目,已停止同步");
}
validateObjectIdentity(rootFolderId, name);
validateLength("空间ID", spaceId, 255);
Map<String, String> metadata = new HashMap<>();
metadata.put("spaceId", spaceId);
putIfPresent(metadata, "spaceType", text(item, "spaceType", "type"));
spaces.add(new SourceObject(rootFolderId, null, name, SyncConstants.OBJECT_FOLDER,
"", null, 0L, null, spaceId, null, metadata));
}
String cursor = nestedText(payload, "nextToken", "nextCursor");
return new ScanResult(spaces, cursor, StringUtils.isNotBlank(cursor));
}
private void validateEnvelope(JsonNode root) {
if (root == null || root.isNull()) {
throw new ServiceException("DWS 返回了空结果");
}
if (root.path("success").isBoolean() && !root.path("success").asBoolean()) {
throw new ServiceException("DWS 返回失败:{}", root.path("message").asString("未知错误"));
}
JsonNode errors = findField(root, "errors");
if (errors != null && errors.isArray() && !errors.isEmpty()) {
throw new ServiceException("DWS 返回了 {} 项目录读取错误,已停止同步", errors.size());
}
}
private JsonNode findObjectArray(JsonNode payload) {
if (payload == null) {
return null;
}
if (payload.isArray()) {
return payload;
}
for (String name : List.of("items", "files", "dentries", "entries", "fileList", "spaces",
"list", "data", "result", "content", "body")) {
JsonNode candidate = payload.path(name);
if (candidate.isArray()) {
return candidate;
}
if (candidate.isObject()) {
JsonNode nested = findObjectArray(candidate);
if (nested != null) {
return nested;
}
}
}
return null;
}
private SourceObject mapObject(JsonNode item, String scanSpaceId) {
String objectId = text(item, "fileId", "dentryUuid", "nodeId", "id");
String name = text(item, "name", "fileName", "title");
if (StringUtils.isBlank(objectId) || StringUtils.isBlank(name)) {
throw new ServiceException("DWS 目录列表包含缺少对象ID或名称的条目已停止同步");
}
validateObjectIdentity(objectId, name);
String extension = normalizeExtension(text(item, "extension", "fileExtension", "suffix"));
String dentryType = text(item, "dentryType", "nodeType", "type");
String contentType = text(item, "contentType", "mimeType");
String objectType = resolveObjectType(dentryType, contentType, extension);
Long listedSize = longValue(item, "size", "fileSize", "contentSize");
if (SyncConstants.OBJECT_FILE.equals(objectType) && listedSize == null) {
throw new ServiceException("DWS 普通文件缺少大小字段:{}", name);
}
long size = listedSize == null ? 0L : listedSize;
if (size < 0) {
throw new ServiceException("DWS 对象大小不能为负数:{}", name);
}
LocalDateTime modifiedTime = timeValue(item, "modifyTime", "modifiedTime", "updateTime", "updatedAt");
String versionToken = text(item, "versionToken", "version", "versionNumber", "etag", "eTag");
if (StringUtils.isBlank(versionToken) && modifiedTime != null) {
versionToken = modifiedTime.toString() + ':' + size;
}
Map<String, String> metadata = new HashMap<>();
putIfPresent(metadata, "spaceId", StringUtils.firstNonBlank(text(item, "spaceId"), scanSpaceId));
putIfPresent(metadata, "dentryId", text(item, "dentryId"));
putIfPresent(metadata, "extension", extension);
putIfPresent(metadata, "sourceUrl", text(item, "url", "link", "dlink", "dlinkUrl", "alidocsUrl", "shareUrl", "sourceUrl"));
if (!metadata.containsKey("sourceUrl") && "dlink".equalsIgnoreCase(extension)
&& name.startsWith("http")) {
metadata.put("sourceUrl", name);
}
String parentObjectId = text(item, "parentId", "parentFileId", "parentDentryUuid");
String hash = text(item, "sha256", "hash", "etag", "eTag");
validateLength("父对象ID", parentObjectId, 255);
validateLength("版本标识", versionToken, 512);
validateLength("源端ETag", hash, 255);
validateLength("内容类型", contentType, 255);
return new SourceObject(
objectId,
parentObjectId,
name,
objectType,
extension,
contentType,
size,
modifiedTime,
versionToken,
hash,
metadata
);
}
private void validateObjectIdentity(String objectId, String name) {
validateLength("对象ID", objectId, 255);
validateLength("对象名称", name, 1024);
if (".".equals(name) || "..".equals(name)
|| name.indexOf('/') >= 0 || name.indexOf('\\') >= 0 || name.indexOf('\0') >= 0) {
throw new ServiceException("DWS 返回了不安全的对象名称,已停止同步");
}
for (int index = 0; index < name.length(); index++) {
if (Character.isISOControl(name.charAt(index))) {
throw new ServiceException("DWS 对象名称包含控制字符,已停止同步");
}
}
}
private void validateLength(String fieldName, String value, int maxLength) {
if (value != null && value.length() > maxLength) {
throw new ServiceException("DWS {}超过数据库长度上限{}", fieldName, maxLength);
}
}
private String resolveObjectType(String dentryType, String contentType, String extension) {
if ("folder".equalsIgnoreCase(dentryType) || "FOLDER".equalsIgnoreCase(dentryType)) {
return SyncConstants.OBJECT_FOLDER;
}
if ("ALIDOC".equalsIgnoreCase(contentType) || ONLINE_EXTENSIONS.contains(extension)) {
return SyncConstants.OBJECT_ONLINE_DOCUMENT;
}
if ("file".equalsIgnoreCase(dentryType) || "FILE".equalsIgnoreCase(dentryType)) {
return SyncConstants.OBJECT_FILE;
}
throw new ServiceException("无法识别 DWS 对象类型:{}", StringUtils.defaultIfBlank(dentryType, ""));
}
private String resolveOutputName(SourceObject object) {
if ("dlink".equalsIgnoreCase(normalizeExtension(object.extension()))) {
return "payload-" + UUID.randomUUID() + ".xlsx";
}
if (SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType())) {
String sourceExtension = normalizeExtension(object.extension());
String extension = "axls".equalsIgnoreCase(sourceExtension) ? "xlsx"
: ("amind".equalsIgnoreCase(sourceExtension) || "adraw".equalsIgnoreCase(sourceExtension)
? "pdf" : "docx");
return "payload-" + UUID.randomUUID() + "." + extension;
}
String extension = normalizeExtension(object.extension());
if (!extension.matches("[a-z0-9]{1,16}")) {
extension = "";
}
return "payload-" + UUID.randomUUID() + (extension.isEmpty() ? "" : "." + extension);
}
private LocalDateTime timeValue(JsonNode node, String... names) {
String value = text(node, names);
if (StringUtils.isBlank(value)) {
return null;
}
try {
long timestamp = Long.parseLong(value);
if (timestamp < 10_000_000_000L) {
timestamp *= 1000;
}
return LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.systemDefault());
} catch (NumberFormatException ignored) {
try {
return LocalDateTime.ofInstant(Instant.parse(value), ZoneId.systemDefault());
} catch (DateTimeParseException ignoredAgain) {
try {
return LocalDateTime.parse(value);
} catch (DateTimeParseException ignoredLast) {
throw new ServiceException("DWS 返回了无法解析的时间字段:{}", value);
}
}
}
}
private Long longValue(JsonNode node, String... names) {
String value = text(node, names);
if (StringUtils.isBlank(value)) {
return null;
}
try {
return Long.parseLong(value);
} catch (NumberFormatException ignored) {
throw new ServiceException("DWS 返回了无法解析的大小字段:{}", value);
}
}
private String nestedText(JsonNode node, String... names) {
JsonNode value = findField(node, names);
return value == null || value.isObject() || value.isArray() ? null : value.asString();
}
private boolean nestedBoolean(JsonNode node, String name) {
JsonNode value = findField(node, name);
return value != null && (value.isBoolean() ? value.asBoolean()
: Boolean.parseBoolean(value.asString("false")));
}
private JsonNode findField(JsonNode node, String... names) {
if (node == null || node.isNull() || node.isArray()) {
return null;
}
for (String name : names) {
JsonNode value = node.get(name);
if (value != null && !value.isNull()) {
return value;
}
}
for (JsonNode child : node) {
if (child.isObject()) {
JsonNode value = findField(child, names);
if (value != null) {
return value;
}
}
}
return null;
}
private String text(JsonNode node, String... names) {
for (String name : names) {
JsonNode value = node.path(name);
if (!value.isMissingNode() && !value.isNull()) {
String text = value.asString();
if (StringUtils.isNotBlank(text)) {
return text;
}
}
}
return null;
}
private String value(Map<String, Object> values, String key) {
Object value = values.get(key);
return value == null ? null : String.valueOf(value);
}
private String metadataValue(SourceObject object, String key) {
return object.metadata() == null ? null : object.metadata().get(key);
}
private void addOption(List<String> arguments, String name, String value) {
if (StringUtils.isNotBlank(value)) {
arguments.add(name);
arguments.add(value);
}
}
private String normalizeExtension(String extension) {
if (StringUtils.isBlank(extension)) {
return "";
}
return extension.startsWith(".") ? extension.substring(1).toLowerCase(Locale.ROOT)
: extension.toLowerCase(Locale.ROOT);
}
private String spaceType(Map<String, Object> config) {
String type = value(config, "spaceType");
return "mySpace".equals(type) ? "mySpace" : "orgSpace";
}
private void cleanDirectory(Path directory) {
if (directory == null) {
return;
}
try (var children = Files.list(directory)) {
for (Path child : children.toList()) {
Files.deleteIfExists(child);
}
Files.deleteIfExists(directory);
} catch (Exception ignored) {
// 下载失败后的临时目录采用尽力清理
}
}
private void putIfPresent(Map<String, String> target, String key, String value) {
if (StringUtils.isNotBlank(value)) {
target.put(key, value);
}
}
}

View File

@ -0,0 +1,16 @@
package org.dromara.sync.connector.dingtalk;
/**
* DWS 登录完成后返回的最小身份信息
*
* @param corpId 组织 ID
* @param corpName 组织名称
* @param userId 钉钉用户 ID
* @param userName 用户名称
*/
public record DwsAuthIdentity(String corpId, String corpName, String userId, String userName) {
public String profile() {
return corpId + ":" + userId;
}
}

View File

@ -0,0 +1,912 @@
package org.dromara.sync.connector.dingtalk;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.dromara.common.core.exception.ServiceException;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.json.utils.JsonUtils;
import org.dromara.sync.domain.SyncConnection;
import org.dromara.sync.domain.vo.DwsAuthSessionVo;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import tools.jackson.databind.JsonNode;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* 管理 DWS Web 设备授权会话
*
* <p>DWS 本身是本地 CLI且设备流需要进程持续轮询此组件把该进程
* 放在后端节点运行只把一次性授权链接和验证码返回给 Web 页面会话
* 采用内存保存进程句柄适用于单节点部署多节点部署时应通过网关会话
* 亲和或专用认证节点保证后续状态请求落到同一节点</p>
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class DwsAuthSessionManager {
private static final Pattern URL_PATTERN = Pattern.compile("https://[^\\s<>\\\"']+");
private static final Pattern USER_CODE_PATTERN = Pattern.compile(
"(?i)(?:授权码|user\\s*code|authorization\\s*code|verification\\s*code|code)\\s*[:=]\\s*"
+ "([A-Za-z0-9][A-Za-z0-9_-]{2,63})");
private static final Pattern POLL_INTERVAL_PATTERN = Pattern.compile(
"(?i)(?:每|every)\\s*([1-9]\\d*)\\s*(?:秒|seconds?)");
private static final Pattern PROFILE_PATTERN = Pattern.compile("^([^\\s:]+):([^\\s:]+)$");
private static final Pattern ANSI_PATTERN = Pattern.compile("\\u001B\\[[;\\d]*m");
private static final int MAX_OUTPUT_BYTES = 512 * 1024;
private static final int MAX_OUTPUT_LINE_CHARS = 16 * 1024;
private static final int MAX_ERROR_LENGTH = 2000;
private static final long DEFAULT_TIMEOUT_SECONDS = 600;
private static final long DEFAULT_METADATA_WAIT_SECONDS = 15;
private static final int DEFAULT_MAX_CONCURRENT = 4;
private final DwsCommandRunner commandRunner;
@Value("${sync.dingtalk.auth.enabled:true}")
private boolean enabled;
@Value("${sync.dingtalk.auth.timeout-seconds:600}")
private long timeoutSeconds;
@Value("${sync.dingtalk.auth.metadata-wait-seconds:15}")
private long metadataWaitSeconds;
@Value("${sync.dingtalk.auth.max-concurrent:4}")
private int maxConcurrent;
@Value("${sync.dingtalk.auth.retention-seconds:900}")
private long retentionSeconds;
private final ConcurrentMap<String, Session> sessions = new ConcurrentHashMap<>();
private final ConcurrentMap<Long, String> activeByConnection = new ConcurrentHashMap<>();
private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();
private final ScheduledExecutorService cleanupExecutor = Executors.newSingleThreadScheduledExecutor();
private volatile Semaphore permits;
/**
* 初始化并启动过期会话清理任务
*/
@PostConstruct
public void init() {
// Keep operator-provided values useful while preventing an accidental
// overflow/indefinite HTTP request from turning a login slot into a
// permanent resource leak.
timeoutSeconds = boundedPositive(timeoutSeconds, DEFAULT_TIMEOUT_SECONDS, 1800);
metadataWaitSeconds = Math.min(timeoutSeconds,
boundedPositive(metadataWaitSeconds, DEFAULT_METADATA_WAIT_SECONDS, 30));
maxConcurrent = (int) Math.max(1, Math.min(maxConcurrent, 32));
retentionSeconds = boundedPositive(retentionSeconds, 900, 86400);
permits = new Semaphore(maxConcurrent);
cleanupExecutor.scheduleAtFixedRate(this::cleanupFinishedSessions, 1, 1, TimeUnit.MINUTES);
}
/**
* 停止所有活动会话并释放执行器
*/
@PreDestroy
public void destroy() {
sessions.values().stream().filter(session -> !session.isTerminal()).forEach(session ->
cancelProcess(session, DwsAuthSessionState.CANCELLED, "服务正在关闭,授权会话已取消"));
cleanupExecutor.shutdownNow();
// Let the watcher finish its finally block and let a just-started
// profile write complete before interrupting the virtual-thread pool.
// This avoids clearing the connection reference while FINALIZING is
// still invoking the persistence callback.
executor.shutdown();
try {
if (!executor.awaitTermination(10, TimeUnit.SECONDS)) {
executor.shutdownNow();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
executor.shutdownNow();
}
sessions.values().stream().filter(Session::isTerminal).forEach(Session::clearSensitiveReferences);
}
/**
* 启动一个 Web 设备授权会话
*
* @param connection 钉钉连接
* @param ownerId 发起用户 ID
* @param expectedCorpId 可选的组织 ID用于防止授权到错误组织
* @param successHandler 授权成功后的身份持久化回调
* @return 会话快照
*/
public DwsAuthSessionVo start(SyncConnection connection, Long ownerId, String expectedCorpId,
Consumer<DwsAuthIdentity> successHandler) {
if (!enabled) {
throw new ServiceException("Web 钉钉登录功能已被管理员关闭");
}
if (connection == null || connection.getConnectionId() == null) {
throw new ServiceException("钉钉连接必须先保存后才能登录");
}
String normalizedExpectedCorpId = normalize(expectedCorpId);
if (normalizedExpectedCorpId != null && !isSafeIdentityPart(normalizedExpectedCorpId)) {
throw new ServiceException("expectedCorpId 格式不正确");
}
Semaphore currentPermits = permits;
if (currentPermits == null || !currentPermits.tryAcquire()) {
throw new ServiceException("当前登录请求较多,请稍后重试");
}
String sessionId = newSessionId();
Session session = new Session(sessionId, connection.getConnectionId(), ownerId,
normalizedExpectedCorpId, System.currentTimeMillis() + timeoutSeconds * 1000L, successHandler);
session.connection = connection;
// Put the session and its per-connection index under one lock. A
// putIfAbsent followed by sessions.put leaves a small window in which
// a concurrent status request can observe an index with no session and
// accidentally remove a newly-started login.
synchronized (activeByConnection) {
String previous = activeByConnection.get(connection.getConnectionId());
if (previous != null) {
Session existing = sessions.get(previous);
if (existing != null && !existing.isTerminal()) {
currentPermits.release();
throw new ServiceException("该钉钉连接已有一个正在进行的登录会话");
}
activeByConnection.remove(connection.getConnectionId(), previous);
}
sessions.put(sessionId, session);
activeByConnection.put(connection.getConnectionId(), sessionId);
}
Process process = null;
Future<?> stdoutReader = null;
Future<?> stderrReader = null;
DwsCommandRunner.ConnectionLockLease authLock = null;
boolean handedOff = false;
try {
authLock = commandRunner.tryAcquireAuthLock(connection);
if (authLock == null) {
throw new ServiceException("该钉钉连接正在执行同步命令,请等待任务结束后再登录");
}
// Cancellation can arrive while the short lock acquisition is
// waiting for an in-flight download/scan. Publish the lease only
// after rechecking the session state; process publication below is
// serialized on the same session monitor.
synchronized (session) {
if (session.isTerminal()) {
authLock.close();
authLock = null;
throw new IllegalStateException("授权会话在取得连接锁前已结束");
}
session.authLock = authLock;
}
ProcessBuilder builder = commandRunner.newProcessBuilderWithoutProfile(connection, List.of(
"auth", "login", "--device", "--no-browser", "--yes", "--format", "json"));
/*
* Serialize process publication with cancel/expire/destroy. The
* session is visible before the CLI is started so a duplicate
* login can be rejected, but that also means a concurrent cancel
* can otherwise observe a null process and let an orphan process
* escape. Holding this lock through publication closes that
* window: cancellation either happens before start, or sees and
* stops the fully-published process.
*/
synchronized (session) {
if (session.isTerminal()) {
throw new IllegalStateException("授权会话在 DWS 进程启动前已结束");
}
process = builder.start();
process.getOutputStream().close();
session.process = process;
Process runningProcess = process;
Future<?> startedStdoutReader = executor.submit(
() -> readStream(session, runningProcess.getInputStream(), false));
Future<?> startedStderrReader = executor.submit(
() -> readStream(session, runningProcess.getErrorStream(), true));
stdoutReader = startedStdoutReader;
stderrReader = startedStderrReader;
Future<?> task = executor.submit(
() -> awaitProcess(session, runningProcess, startedStdoutReader, startedStderrReader));
session.task = task;
// From this point awaitProcess owns the process and semaphore.
// A later HTTP response error must not release the same permit
// twice. The lock also guarantees cancel cannot run between
// process publication and this hand-off marker.
handedOff = true;
}
// 通常 DWS 会在首次请求完成后立即打印二维码/链接短暂等待可以让
// 前端首个响应直接得到可展示的数据网络慢时则返回 STARTING
try {
session.metadataReady.await(metadataWaitSeconds, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
return snapshot(session);
} catch (Exception e) {
if (!handedOff) {
cancelFuture(stdoutReader);
cancelFuture(stderrReader);
commandRunner.stop(process);
session.process = null;
sessions.remove(sessionId, session);
activeByConnection.remove(connection.getConnectionId(), sessionId);
session.clearOutputs();
session.clearSensitiveReferences();
DwsCommandRunner.ConnectionLockLease publishedLock = session.authLock;
session.authLock = null;
if (publishedLock != null) {
publishedLock.close();
} else if (authLock != null) {
authLock.close();
}
currentPermits.release();
}
if (e instanceof ServiceException serviceException) {
throw serviceException;
}
throw new ServiceException("无法启动 DWS Web 登录,请确认服务器已安装可执行的 DWS", e);
}
}
/**
* 读取指定用户的授权会话状态
*
* @param connectionId 连接 ID
* @param sessionId 会话 ID
* @param ownerId 当前用户 ID
* @return 会话快照
*/
public DwsAuthSessionVo get(Long connectionId, String sessionId, Long ownerId) {
return get(connectionId, sessionId, ownerId, false);
}
/**
* 查询会话状态可由超级管理员接管查看其他用户发起的会话
*
* @param connectionId 连接 ID
* @param sessionId 会话 ID
* @param ownerId 当前用户 ID
* @param privileged 是否为超级管理员
* @return 会话快照
*/
public DwsAuthSessionVo get(Long connectionId, String sessionId, Long ownerId, boolean privileged) {
Session session = requireSession(connectionId, sessionId, ownerId, privileged);
if (!session.isTerminal() && System.currentTimeMillis() >= session.expiresAt) {
expire(session);
}
return snapshot(session);
}
/**
* 取消指定用户的授权会话
*
* @param connectionId 连接 ID
* @param sessionId 会话 ID
* @param ownerId 当前用户 ID
* @return 会话快照
*/
public DwsAuthSessionVo cancel(Long connectionId, String sessionId, Long ownerId) {
return cancel(connectionId, sessionId, ownerId, false);
}
/**
* 取消会话可由超级管理员接管取消其他用户发起的会话
*
* @param connectionId 连接 ID
* @param sessionId 会话 ID
* @param ownerId 当前用户 ID
* @param privileged 是否为超级管理员
* @return 取消后的会话快照
*/
public DwsAuthSessionVo cancel(Long connectionId, String sessionId, Long ownerId, boolean privileged) {
Session session = requireSession(connectionId, sessionId, ownerId, privileged);
if (!session.isTerminal()) {
cancelProcess(session, DwsAuthSessionState.CANCELLED, "已取消钉钉登录");
}
return snapshot(session);
}
/**
* 判断连接是否仍有一个 Web 登录会话占用登录态目录
*
* <p>编辑或删除连接前调用此方法可以避免回调把 profile 写回已经
* 变更/删除的记录</p>
*/
public boolean hasActive(Long connectionId) {
if (connectionId == null) {
return false;
}
String sessionId = activeByConnection.get(connectionId);
if (sessionId == null) {
return false;
}
Session session = sessions.get(sessionId);
if (session == null || session.isTerminal()) {
activeByConnection.remove(connectionId, sessionId);
return false;
}
return true;
}
private void awaitProcess(Session session, Process process, Future<?> stdoutReader, Future<?> stderrReader) {
try {
if (!process.waitFor(Math.max(1, timeoutSeconds), TimeUnit.SECONDS)) {
expire(session);
return;
}
waitReader(stdoutReader);
waitReader(stderrReader);
if (session.state == DwsAuthSessionState.CANCELLED
|| session.state == DwsAuthSessionState.EXPIRED
|| session.state == DwsAuthSessionState.FINALIZING
|| session.state == DwsAuthSessionState.SUCCESS) {
return;
}
if (process.exitValue() != 0) {
fail(session, diagnostic(session.errorOutput(), session.stdoutOutput(),
"DWS 登录命令执行失败"));
return;
}
complete(session);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
if (!session.isTerminal()) {
fail(session, "DWS 登录进程被中断");
}
} catch (Exception e) {
if (!session.isTerminal()) {
fail(session, diagnostic(e.getMessage(), null, "DWS 登录失败"));
}
} finally {
// A shutdown/interruption can happen while waitFor is blocked.
// Always terminate the native process before dropping the handle;
// otherwise the watcher could leave an orphan DWS process behind.
commandRunner.stop(process);
Semaphore currentPermits = permits;
if (currentPermits != null) {
currentPermits.release();
}
activeByConnection.remove(session.connectionId, session.id);
session.process = null;
session.terminalAt = System.currentTimeMillis();
// CLI output can contain the one-time user code and verification
// URL. Do not retain it for the whole terminal-session TTL.
session.clearOutputs();
session.clearSensitiveReferences();
cancelFuture(stdoutReader);
cancelFuture(stderrReader);
DwsCommandRunner.ConnectionLockLease authLock = session.authLock;
session.authLock = null;
if (authLock != null) {
authLock.close();
}
}
}
private void complete(Session session) {
synchronized (session) {
if (session.isTerminal() || session.state == DwsAuthSessionState.FINALIZING) {
return;
}
// Once DWS has returned success, make cancellation deterministic:
// the session is finalizing its database write and cannot be
// changed back to CANCELLED halfway through that write.
session.state = DwsAuthSessionState.FINALIZING;
session.clearDevicePrompt();
session.metadataReady.countDown();
}
String stdout = session.stdoutOutput();
DwsAuthIdentity identity = parseIdentity(stdout);
session.clearOutputs();
if (identity == null) {
fail(session, "DWS 登录成功但未返回完整 corpId:userId请固定已验收版本后重试");
return;
}
if (StringUtils.isNotBlank(session.expectedCorpId)
&& !Objects.equals(session.expectedCorpId, identity.corpId())) {
fail(session, "授权组织与填写的组织 ID 不一致,请确认后重试");
return;
}
try {
if (session.successHandler != null) {
session.successHandler.accept(identity);
}
synchronized (session) {
if (session.state != DwsAuthSessionState.FINALIZING) {
return;
}
session.corpId = identity.corpId();
session.corpName = identity.corpName();
session.userId = identity.userId();
session.userName = identity.userName();
session.profile = identity.profile();
session.message = "钉钉登录成功,已保存登录态";
session.state = DwsAuthSessionState.SUCCESS;
}
} catch (Exception e) {
fail(session, diagnostic(e.getMessage(), null, "登录成功,但保存连接身份失败"));
}
}
private void readStream(Session session, InputStream inputStream, boolean errorStream) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
char[] buffer = new char[4096];
StringBuilder line = new StringBuilder(Math.min(MAX_OUTPUT_LINE_CHARS, 4096));
int length;
while ((length = reader.read(buffer)) >= 0) {
for (int index = 0; index < length; index++) {
char character = buffer[index];
if (character == '\n') {
appendOutputLine(session, line.toString(), errorStream);
line.setLength(0);
} else if (character != '\r') {
if (line.length() < MAX_OUTPUT_LINE_CHARS) {
line.append(character);
} else {
// Flush bounded chunks from a pathological line so
// read() never accumulates unbounded attacker/CLI
// output in memory. Prompt lines are far shorter.
appendOutputLine(session, line.toString(), errorStream);
line.setLength(0);
}
}
}
}
if (line.length() > 0) {
appendOutputLine(session, line.toString(), errorStream);
}
} catch (IOException e) {
if (!session.isTerminal()) {
session.followUpError = diagnostic(e.getMessage(), session.followUpError, "读取 DWS 登录输出失败");
}
}
}
private void appendOutputLine(Session session, String line, boolean errorStream) {
String clean = stripAnsi(line);
if (errorStream) {
session.appendError(clean);
} else {
session.appendStdout(clean);
}
parseDevicePrompt(session, clean);
}
private void parseDevicePrompt(Session session, String line) {
if (StringUtils.isBlank(line)) {
return;
}
synchronized (session) {
if (session.isTerminal() || session.state == DwsAuthSessionState.FINALIZING) {
return;
}
Matcher urlMatcher = URL_PATTERN.matcher(line);
while (urlMatcher.find()) {
String url = trimUrl(urlMatcher.group());
if (!isAllowedVerificationUrl(url)) {
continue;
}
if (url.contains("?") || url.contains("&")) {
session.verificationUriComplete = url;
} else if (StringUtils.isBlank(session.verificationUri)) {
session.verificationUri = url;
}
}
Matcher codeMatcher = USER_CODE_PATTERN.matcher(line);
if (codeMatcher.find()) {
session.userCode = codeMatcher.group(1);
}
Matcher intervalMatcher = POLL_INTERVAL_PATTERN.matcher(line);
if (intervalMatcher.find()) {
try {
int interval = Integer.parseInt(intervalMatcher.group(1));
session.pollInterval = Math.max(1, Math.min(interval, 30));
} catch (NumberFormatException ignored) {
// Keep the safe default when a localized line is malformed.
}
}
if (StringUtils.isNotBlank(session.verificationUriComplete)
|| (StringUtils.isNotBlank(session.verificationUri) && StringUtils.isNotBlank(session.userCode))) {
if (StringUtils.isBlank(session.verificationUri)) {
session.verificationUri = session.verificationUriComplete;
}
session.state = DwsAuthSessionState.WAITING_USER;
session.metadataReady.countDown();
}
}
}
private DwsAuthIdentity parseIdentity(String output) {
if (StringUtils.isBlank(output)) {
return null;
}
String json = extractJson(stripAnsi(output));
if (json == null) {
return null;
}
try {
JsonNode root = JsonUtils.getJsonMapper().readTree(json);
return findIdentity(root);
} catch (Exception ignored) {
return null;
}
}
private DwsAuthIdentity findIdentity(JsonNode node) {
if (node == null || node.isNull()) {
return null;
}
if (node.isObject()) {
// Prefer explicit IDs. A profile/display name may contain a
// colon too (for example "企业名:张三") and must not outrank the
// authoritative corp_id/user_id fields returned by DWS.
String corpId = text(node, "corpId", "corp_id", "organizationId", "orgId", "org_id");
String userId = text(node, "userId", "user_id", "userid");
if (StringUtils.isNotBlank(corpId) && StringUtils.isNotBlank(userId)) {
DwsAuthIdentity identity = validIdentity(corpId, text(node, "corpName", "corp_name", "organizationName"),
userId, text(node, "userName", "user_name", "name", "nick"));
if (identity != null) {
return identity;
}
}
String profile = text(node, "profile", "account", "accountProfile");
DwsAuthIdentity byProfile = identityFromProfile(profile, node);
if (byProfile != null) {
return byProfile;
}
for (var field : node.properties()) {
DwsAuthIdentity nested = findIdentity(field.getValue());
if (nested != null) {
return nested;
}
}
} else if (node.isArray()) {
for (JsonNode child : node) {
DwsAuthIdentity nested = findIdentity(child);
if (nested != null) {
return nested;
}
}
}
return null;
}
private DwsAuthIdentity identityFromProfile(String profile, JsonNode source) {
if (StringUtils.isBlank(profile)) {
return null;
}
Matcher matcher = PROFILE_PATTERN.matcher(profile.trim());
if (!matcher.matches()) {
return null;
}
String corpId = matcher.group(1);
String userId = matcher.group(2);
return validIdentity(corpId,
source == null ? null : text(source, "corpName", "corp_name", "organizationName"),
userId,
source == null ? null : text(source, "userName", "user_name", "name", "nick"));
}
private DwsAuthIdentity validIdentity(String corpId, String corpName, String userId, String userName) {
if (!isSafeIdentityPart(corpId) || !isSafeIdentityPart(userId)) {
return null;
}
return new DwsAuthIdentity(corpId, normalize(corpName), userId, normalize(userName));
}
private boolean isSafeIdentityPart(String value) {
return StringUtils.isNotBlank(value)
&& value.length() <= 255
&& value.chars().noneMatch(character -> Character.isWhitespace(character)
|| Character.isISOControl(character))
&& value.indexOf(':') < 0
&& value.indexOf('/') < 0
&& value.indexOf('\\') < 0
&& value.indexOf('?') < 0
&& value.indexOf('&') < 0
&& value.indexOf('"') < 0
&& value.indexOf('\'') < 0;
}
private Session requireSession(Long connectionId, String sessionId, Long ownerId, boolean privileged) {
if (connectionId == null || StringUtils.isBlank(sessionId)) {
throw new ServiceException("授权会话参数不能为空");
}
Session session = sessions.get(sessionId);
if (session == null || !Objects.equals(session.connectionId, connectionId)
|| (!privileged && !Objects.equals(session.ownerId, ownerId))) {
throw new ServiceException("授权会话不存在或已失效");
}
return session;
}
private void expire(Session session) {
if (session.isTerminal()) {
return;
}
cancelProcess(session, DwsAuthSessionState.EXPIRED, "授权链接已过期,请重新发起登录");
}
private void fail(Session session, String message) {
synchronized (session) {
if (session.isTerminal()) {
return;
}
session.state = DwsAuthSessionState.FAILED;
session.message = diagnostic(message, null, "DWS 登录失败,请重试");
session.clearDevicePrompt();
session.metadataReady.countDown();
session.clearSensitiveReferences();
}
}
private void cancelProcess(Session session, DwsAuthSessionState state, String message) {
synchronized (session) {
if (session.isTerminal() || session.state == DwsAuthSessionState.FINALIZING) {
return;
}
session.state = state;
session.message = message;
session.clearDevicePrompt();
Process process = session.process;
if (process != null) {
commandRunner.stop(process);
}
session.metadataReady.countDown();
session.clearOutputs();
session.clearSensitiveReferences();
}
}
private DwsAuthSessionVo snapshot(Session session) {
long remaining = Math.max(0, (session.expiresAt - System.currentTimeMillis()) / 1000L);
if (session.isTerminal() && session.state != DwsAuthSessionState.SUCCESS) {
remaining = 0;
}
boolean exposeDevicePrompt = session.state == DwsAuthSessionState.STARTING
|| session.state == DwsAuthSessionState.WAITING_USER;
return new DwsAuthSessionVo(
session.id,
session.connectionId,
session.state.name(),
exposeDevicePrompt ? session.verificationUri : null,
exposeDevicePrompt ? session.verificationUriComplete : null,
exposeDevicePrompt ? session.userCode : null,
session.expiresAt,
remaining,
session.pollInterval,
session.corpId,
session.corpName,
session.userId,
session.userName,
session.profile,
session.message);
}
private void cleanupFinishedSessions() {
long threshold = System.currentTimeMillis() - retentionSeconds * 1000L;
sessions.entrySet().removeIf(entry -> {
Session session = entry.getValue();
return session.isTerminal() && session.terminalAt > 0 && session.terminalAt < threshold;
});
}
private void waitReader(Future<?> reader) throws Exception {
if (reader != null) {
reader.get(15, TimeUnit.SECONDS);
}
}
private void cancelFuture(Future<?> future) {
if (future != null && !future.isDone()) {
future.cancel(true);
}
}
private String diagnostic(String primary, String secondary, String fallback) {
String value = StringUtils.firstNonBlank(primary, secondary, fallback);
if (value == null) {
return fallback;
}
String sanitized = value
.replaceAll("(?i)(access[_-]?token|refresh[_-]?token|device[_-]?code|client[_-]?secret|app[_-]?secret)"
+ "([\\\"'\\s:=]+)[^\\s,;\\\"']+", "$1$2***")
.replaceAll("(?i)(授权码|user\\s*code|authorization\\s*code|verification\\s*code|code)"
+ "\\s*[:=]\\s*[A-Za-z0-9][A-Za-z0-9_-]{2,63}", "$1:***")
.replaceAll("(?i)https?://[^\\s\\\"']+", "[授权链接已隐藏]")
.trim();
return sanitized.length() <= MAX_ERROR_LENGTH ? sanitized : sanitized.substring(0, MAX_ERROR_LENGTH);
}
private String extractJson(String output) {
int start = output.indexOf('{');
int end = output.lastIndexOf('}');
if (start < 0 || end <= start) {
start = output.indexOf('[');
end = output.lastIndexOf(']');
}
return start >= 0 && end > start ? output.substring(start, end + 1) : null;
}
private String text(JsonNode node, String... names) {
for (String name : names) {
JsonNode value = node.path(name);
if ((value.isTextual() || value.isNumber()) && StringUtils.isNotBlank(value.asString())) {
return value.asString().trim();
}
}
return null;
}
private String trimUrl(String value) {
return value.replaceAll("[\\.,;:!?\\)\\]\\},。;:!?)】》]+$", "");
}
private boolean isAllowedVerificationUrl(String value) {
try {
URI uri = URI.create(value);
String host = uri.getHost();
if (!"https".equalsIgnoreCase(uri.getScheme()) || StringUtils.isBlank(host)) {
return false;
}
String normalizedHost = host.toLowerCase(Locale.ROOT);
return "dingtalk.com".equals(normalizedHost)
|| normalizedHost.endsWith(".dingtalk.com");
} catch (Exception e) {
return false;
}
}
private String stripAnsi(String value) {
return value == null ? null : ANSI_PATTERN.matcher(value).replaceAll("");
}
private String normalize(String value) {
return StringUtils.isBlank(value) ? null : value.trim();
}
private long positive(long value, long fallback) {
return value > 0 ? value : fallback;
}
private long boundedPositive(long value, long fallback, long maximum) {
return Math.min(positive(value, fallback), maximum);
}
private String newSessionId() {
byte[] bytes = new byte[24];
SecureRandomHolder.INSTANCE.nextBytes(bytes);
return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes);
}
private static final class SecureRandomHolder {
private static final java.security.SecureRandom INSTANCE = new java.security.SecureRandom();
}
private static final class Session {
private final String id;
private final Long connectionId;
private final Long ownerId;
private final String expectedCorpId;
private final long expiresAt;
private final CountDownLatch metadataReady = new CountDownLatch(1);
private final LimitedOutput stdout = new LimitedOutput();
private final LimitedOutput stderr = new LimitedOutput();
private volatile SyncConnection connection;
private volatile Process process;
private volatile Future<?> task;
private volatile DwsCommandRunner.ConnectionLockLease authLock;
private volatile Consumer<DwsAuthIdentity> successHandler;
private volatile DwsAuthSessionState state = DwsAuthSessionState.STARTING;
private volatile String verificationUri;
private volatile String verificationUriComplete;
private volatile String userCode;
private volatile int pollInterval = 2;
private volatile String corpId;
private volatile String corpName;
private volatile String userId;
private volatile String userName;
private volatile String profile;
private volatile String message;
private volatile String followUpError;
private volatile long terminalAt;
private Session(String id, Long connectionId, Long ownerId, String expectedCorpId,
long expiresAt, Consumer<DwsAuthIdentity> successHandler) {
this.id = id;
this.connectionId = connectionId;
this.ownerId = ownerId;
this.expectedCorpId = expectedCorpId;
this.expiresAt = expiresAt;
this.successHandler = successHandler;
}
private boolean isTerminal() {
return switch (state) {
case SUCCESS, FAILED, EXPIRED, CANCELLED -> true;
default -> false;
};
}
private void appendStdout(String line) {
stdout.append(line);
}
private void appendError(String line) {
stderr.append(line);
}
private String stdoutOutput() {
return stdout.value();
}
private String errorOutput() {
return stderr.value();
}
private void clearOutputs() {
stdout.clear();
stderr.clear();
}
private void clearDevicePrompt() {
verificationUri = null;
verificationUriComplete = null;
userCode = null;
}
private void clearSensitiveReferences() {
connection = null;
successHandler = null;
task = null;
}
}
private static final class LimitedOutput {
private final StringBuilder value = new StringBuilder();
private int bytes;
private synchronized void append(String line) {
if (bytes >= MAX_OUTPUT_BYTES) {
return;
}
String text = line + System.lineSeparator();
byte[] encoded = text.getBytes(StandardCharsets.UTF_8);
int remaining = MAX_OUTPUT_BYTES - bytes;
if (encoded.length <= remaining) {
value.append(text);
bytes += encoded.length;
} else {
value.append(new String(encoded, 0, remaining, StandardCharsets.UTF_8));
bytes = MAX_OUTPUT_BYTES;
}
}
private synchronized String value() {
return value.toString();
}
private synchronized void clear() {
value.setLength(0);
bytes = 0;
}
}
}

View File

@ -0,0 +1,15 @@
package org.dromara.sync.connector.dingtalk;
/**
* DWS Web 设备授权会话状态
*/
enum DwsAuthSessionState {
STARTING,
WAITING_USER,
/** DWS 已完成授权,后端正在把身份写回连接配置。 */
FINALIZING,
SUCCESS,
FAILED,
EXPIRED,
CANCELLED
}

View File

@ -0,0 +1,459 @@
package org.dromara.sync.connector.dingtalk;
import org.dromara.common.core.exception.ServiceException;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.json.utils.JsonUtils;
import org.dromara.sync.domain.SyncConnection;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import tools.jackson.core.type.TypeReference;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.LinkOption;
import java.nio.file.attribute.PosixFilePermission;
import java.time.Duration;
import java.util.ArrayList;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.StampedLock;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
/**
* 安全执行 DingTalk Workspace CLI不通过命令行参数传递凭证
*/
@Component
class DwsCommandRunner {
private static final int MAX_CAPTURE_BYTES = 8 * 1024 * 1024;
private static final long COMMAND_LOCK_WAIT_MILLIS = 2000;
private static final long AUTH_LOCK_WAIT_MILLIS = 1000;
/**
* Environment variables needed by a native CLI/network stack. In
* particular, do not inherit the whole Spring process environment: it may
* contain database, object-storage, or encryption secrets.
*/
private static final List<String> SAFE_ENVIRONMENT_KEYS = List.of(
"PATH", "USER", "LOGNAME", "LANG", "LC_ALL", "TZ", "TMPDIR", "TMP", "TEMP",
"HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY",
"http_proxy", "https_proxy", "all_proxy", "no_proxy",
"SSL_CERT_FILE", "SSL_CERT_DIR", "DWS_LANG", "DWS_TRUSTED_DOMAINS",
"DWS_ALLOW_HTTP_ENDPOINTS", "DWS_CHANNEL", "DWS_AGENT_PRODUCT");
@Value("${sync.dingtalk.dws-executable:dws}")
private String executable;
@Value("${sync.dingtalk.profile-root:}")
private String profileRoot;
/**
* DWS 加密 keychain 的根目录DWS_CONFIG_DIR 只隔离配置文件不能保证
* 不同连接之间的 token/key 不串用因此这里也按连接隔离 keychain
*/
@Value("${sync.dingtalk.keychain-root:}")
private String keychainRoot;
/**
* A DWS config/keychain directory is mutable state. Keep business
* commands as readers so normal parallel downloads remain possible, while
* a Web login takes the exclusive writer lock for the whole device flow.
* This is intentionally JVM-local; a multi-node deployment still needs
* sticky routing or a distributed lock around DWS operations.
*/
private final ConcurrentMap<Long, StampedLock> connectionLocks = new ConcurrentHashMap<>();
DwsCommandResult run(SyncConnection connection, List<String> arguments, Duration timeout) {
return run(connection, arguments, timeout, null);
}
DwsCommandResult run(SyncConnection connection, List<String> arguments, Duration timeout, Path workingDirectory) {
return runInternal(connection, arguments, timeout, workingDirectory, true);
}
/**
* 执行不需要 profile DWS 命令例如 auth status/profile list
*
* @param connection 连接
* @param arguments 命令参数
* @param timeout 超时时间
* @return 命令结果
*/
DwsCommandResult runWithoutProfile(SyncConnection connection, List<String> arguments, Duration timeout) {
return runInternal(connection, arguments, timeout, null, false);
}
/**
* 尝试为 Web 登录取得连接级独占锁
*
* @param connection 连接
* @return 成功时的锁租约忙时返回 {@code null}
*/
ConnectionLockLease tryAcquireAuthLock(SyncConnection connection) {
return tryAcquire(connection, true, AUTH_LOCK_WAIT_MILLIS);
}
/**
* Web 设备登录创建进程构造器该进程不会带 profile登录完成后
* 才能由 DWS 返回实际的 corpId:userId
*
* @param connection 连接
* @param arguments 命令参数
* @return 已设置安全环境变量的进程构造器
*/
ProcessBuilder newProcessBuilderWithoutProfile(SyncConnection connection, List<String> arguments) {
return buildProcessBuilder(connection, arguments, null, false);
}
/**
* 终止正在运行的 DWS 子进程及其子孙进程
*
* @param process 进程
*/
void stop(Process process) {
terminate(process);
}
private DwsCommandResult runInternal(SyncConnection connection, List<String> arguments, Duration timeout,
Path workingDirectory, boolean includeProfile) {
Map<String, Object> secret = parseJson(connection.getSecretJson());
String clientId = value(secret, "clientId", null);
String clientSecret = value(secret, "clientSecret", null);
if (StringUtils.isAnyBlank(clientId, clientSecret)) {
throw new ServiceException("钉钉连接 clientId、clientSecret 不能为空");
}
if (StringUtils.isBlank(executable)) {
throw new ServiceException("sync.dingtalk.dws-executable 不能为空");
}
ExecutorService executor = null;
ConnectionLockLease commandLock = null;
Process process = null;
try {
commandLock = tryAcquire(connection, false, COMMAND_LOCK_WAIT_MILLIS);
if (commandLock == null) {
throw new ServiceException("钉钉连接正在进行 Web 登录,请登录结束后重试");
}
executor = java.util.concurrent.Executors.newVirtualThreadPerTaskExecutor();
ProcessBuilder processBuilder = buildProcessBuilder(connection, arguments, workingDirectory, includeProfile);
process = processBuilder.start();
process.getOutputStream().close();
Process startedProcess = process;
Future<CapturedOutput> stdoutFuture = executor.submit(() -> read(startedProcess.getInputStream()));
Future<CapturedOutput> stderrFuture = executor.submit(() -> read(startedProcess.getErrorStream()));
if (!process.waitFor(timeout.toMillis(), TimeUnit.MILLISECONDS)) {
terminate(process);
throw new ServiceException("DWS 命令执行超时({} 秒)", timeout.toSeconds());
}
CapturedOutput stdoutResult = stdoutFuture.get(30, TimeUnit.SECONDS);
CapturedOutput stderrResult = stderrFuture.get(30, TimeUnit.SECONDS);
if (stdoutResult.truncated() || stderrResult.truncated()) {
throw new ServiceException("DWS 命令输出超过 {} MiB 安全上限", MAX_CAPTURE_BYTES / 1024 / 1024);
}
String stdout = stdoutResult.value();
String stderr = stderrResult.value();
if (process.exitValue() != 0) {
String message = StringUtils.isNotBlank(stderr) ? stderr : stdout;
throw new ServiceException("DWS 命令执行失败:{}",
abbreviate(sanitizeDiagnostic(message, clientId, clientSecret)));
}
return new DwsCommandResult(stdout, stderr);
} catch (ServiceException e) {
throw e;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new ServiceException("DWS 命令执行被中断", e);
} catch (Exception e) {
throw new ServiceException("无法执行 DWS请确认 DingTalk Workspace CLI 已安装并可用", e);
} finally {
terminate(process);
if (executor != null) {
executor.shutdownNow();
}
if (commandLock != null) {
commandLock.close();
}
}
}
private ConnectionLockLease tryAcquire(SyncConnection connection, boolean exclusive, long waitMillis) {
if (connection == null || connection.getConnectionId() == null) {
throw new ServiceException("钉钉连接必须先保存后才能执行 DWS 命令");
}
StampedLock lock = connectionLocks.computeIfAbsent(connection.getConnectionId(),
ignored -> new StampedLock());
try {
long stamp = exclusive
? lock.tryWriteLock(waitMillis, TimeUnit.MILLISECONDS)
: lock.tryReadLock(waitMillis, TimeUnit.MILLISECONDS);
if (stamp == 0L) {
return null;
}
return new ConnectionLockLease(lock, stamp, exclusive);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new ServiceException("等待钉钉连接执行锁时被中断", e);
}
}
private ProcessBuilder buildProcessBuilder(SyncConnection connection, List<String> arguments,
Path workingDirectory, boolean includeProfile) {
Map<String, Object> config = parseJson(connection.getConfigJson());
Map<String, Object> secret = parseJson(connection.getSecretJson());
String clientId = value(secret, "clientId", null);
String clientSecret = value(secret, "clientSecret", null);
if (StringUtils.isAnyBlank(clientId, clientSecret)) {
throw new ServiceException("钉钉连接 clientId、clientSecret 不能为空");
}
if (StringUtils.isBlank(executable)) {
throw new ServiceException("sync.dingtalk.dws-executable 不能为空");
}
List<String> command = new ArrayList<>(arguments.size() + (includeProfile ? 3 : 1));
command.add(executable);
if (includeProfile) {
String profile = value(config, "profile", null);
if (StringUtils.isBlank(profile)) {
throw new ServiceException("钉钉连接尚未完成登录,请先在网页端完成设备授权");
}
command.add("--profile");
command.add(profile);
}
command.addAll(arguments);
Path configDirectory = resolveConfigDirectory(connection);
Path keychainDirectory = resolveKeychainDirectory(connection);
ProcessBuilder processBuilder = new ProcessBuilder(command);
restrictEnvironment(processBuilder, configDirectory);
putEnvironment(processBuilder, "DWS_CLIENT_ID", clientId);
putEnvironment(processBuilder, "DWS_CLIENT_SECRET", clientSecret);
putEnvironment(processBuilder, "DWS_CONFIG_DIR", configDirectory.toString());
putEnvironment(processBuilder, "DWS_KEYCHAIN_DIR", keychainDirectory.toString());
if (workingDirectory != null) {
processBuilder.directory(workingDirectory.toFile());
}
return processBuilder;
}
private void restrictEnvironment(ProcessBuilder processBuilder, Path configDirectory) {
Map<String, String> inherited = new HashMap<>();
Map<String, String> environment = processBuilder.environment();
boolean windowsHome = environment.containsKey("USERPROFILE");
for (String key : SAFE_ENVIRONMENT_KEYS) {
String value = environment.get(key);
if (StringUtils.isNotBlank(value)) {
inherited.put(key, value);
}
}
environment.clear();
environment.putAll(inherited);
// DWS_CONFIG_DIR and DWS_KEYCHAIN_DIR are explicit below. Point HOME
// at the connection directory as an additional guard against the CLI
// reading a global ~/.dws cache or writing credentials there.
environment.put("HOME", configDirectory.toString());
if (windowsHome) {
environment.put("USERPROFILE", configDirectory.toString());
}
}
Map<String, Object> config(SyncConnection connection) {
return parseJson(connection.getConfigJson());
}
private void putEnvironment(ProcessBuilder builder, String key, String value) {
if (StringUtils.isNotBlank(value)) {
builder.environment().put(key, value);
}
}
Path resolveConfigDirectory(SyncConnection connection) {
if (connection.getConnectionId() == null) {
throw new ServiceException("钉钉连接必须先保存后才能执行 DWS 命令");
}
if (StringUtils.isBlank(profileRoot)) {
throw new ServiceException("sync.dingtalk.profile-root 不能为空");
}
try {
return resolvePrivateDirectory(profileRoot, connection.getConnectionId(), "DWS 配置");
} catch (ServiceException e) {
throw e;
} catch (Exception e) {
throw new ServiceException("无法创建该连接专属的 DWS 配置目录", e);
}
}
private Path resolveKeychainDirectory(SyncConnection connection) {
if (connection.getConnectionId() == null) {
throw new ServiceException("钉钉连接必须先保存后才能执行 DWS 命令");
}
String rootValue = StringUtils.isNotBlank(keychainRoot)
? keychainRoot : (StringUtils.isNotBlank(profileRoot) ? profileRoot + "-keychain" : null);
if (StringUtils.isBlank(rootValue)) {
throw new ServiceException("sync.dingtalk.keychain-root 不能为空");
}
try {
return resolvePrivateDirectory(rootValue, connection.getConnectionId(), "DWS keychain");
} catch (ServiceException e) {
throw e;
} catch (Exception e) {
throw new ServiceException("无法创建该连接专属的 DWS keychain 目录", e);
}
}
private Path resolvePrivateDirectory(String rootValue, Long connectionId, String label) throws IOException {
Path root = Path.of(rootValue).toAbsolutePath().normalize();
if (Files.isSymbolicLink(root)) {
throw new ServiceException(label + "根目录不能是符号链接");
}
createPrivateDirectory(root);
Path realRoot = root.toRealPath(LinkOption.NOFOLLOW_LINKS);
Path directory = realRoot.resolve(String.valueOf(connectionId)).normalize();
if (!directory.startsWith(realRoot)) {
throw new ServiceException(label + "目录越界");
}
if (Files.isSymbolicLink(directory)) {
throw new ServiceException(label + "目录不能是符号链接");
}
createPrivateDirectory(directory);
Path realDirectory = directory.toRealPath(LinkOption.NOFOLLOW_LINKS);
if (!realDirectory.startsWith(realRoot) || Files.isSymbolicLink(directory)) {
throw new ServiceException(label + "目录越界或包含符号链接");
}
return realDirectory;
}
private void createPrivateDirectory(Path directory) throws IOException {
if (Files.isSymbolicLink(directory)) {
throw new IOException("directory is a symbolic link: " + directory);
}
Files.createDirectories(directory);
if (Files.isSymbolicLink(directory)) {
throw new IOException("directory became a symbolic link: " + directory);
}
try {
Set<PosixFilePermission> permissions = EnumSet.of(
PosixFilePermission.OWNER_READ,
PosixFilePermission.OWNER_WRITE,
PosixFilePermission.OWNER_EXECUTE);
Files.setPosixFilePermissions(directory, permissions);
} catch (UnsupportedOperationException ignored) {
// Windows 文件系统没有 POSIX 权限模型依赖进程账户 ACL
}
}
private CapturedOutput read(java.io.InputStream inputStream) throws IOException {
ByteArrayOutputStream captured = new ByteArrayOutputStream();
byte[] buffer = new byte[8192];
int length;
long total = 0;
while ((length = inputStream.read(buffer)) >= 0) {
total += length;
int remaining = MAX_CAPTURE_BYTES - captured.size();
if (remaining > 0) {
captured.write(buffer, 0, Math.min(length, remaining));
}
}
return new CapturedOutput(captured.toString(StandardCharsets.UTF_8), total > MAX_CAPTURE_BYTES);
}
private Map<String, Object> parseJson(String json) {
if (StringUtils.isBlank(json)) {
return Map.of();
}
try {
return JsonUtils.parseObject(json, new TypeReference<>() {
});
} catch (Exception e) {
throw new ServiceException("钉钉连接配置不是合法 JSON", e);
}
}
private String value(Map<String, Object> values, String key, String defaultValue) {
Object value = values.get(key);
return value == null ? defaultValue : String.valueOf(value);
}
private String abbreviate(String value) {
if (value == null) {
return "未知错误";
}
String trimmed = value.trim();
return trimmed.length() <= 2000 ? trimmed : trimmed.substring(0, 2000);
}
private String sanitizeDiagnostic(String value, String... secrets) {
String redacted = value;
if (redacted == null) {
return null;
}
for (String secret : secrets) {
if (StringUtils.isNotBlank(secret)) {
redacted = redacted.replace(secret, "***");
}
}
redacted = redacted.replaceAll("(?i)(access[_-]?token|refresh[_-]?token|client[_-]?secret|app[_-]?secret|authorization)([\\\"'\\s:=]+)[^\\s,;\\\"']+", "$1$2***");
return redacted.replaceAll("(?i)https?://[^\\s\\\"']+\\?[^\\s\\\"']+", "[signed-url-redacted]");
}
private void terminate(Process process) {
if (process == null || !process.isAlive()) {
return;
}
boolean interrupted = Thread.interrupted();
try {
process.descendants().forEach(ProcessHandle::destroy);
process.destroy();
if (!process.waitFor(2, TimeUnit.SECONDS)) {
process.descendants().forEach(ProcessHandle::destroyForcibly);
process.destroyForcibly();
process.waitFor(2, TimeUnit.SECONDS);
}
} catch (InterruptedException e) {
interrupted = true;
process.descendants().forEach(ProcessHandle::destroyForcibly);
process.destroyForcibly();
} finally {
if (interrupted) {
Thread.currentThread().interrupt();
}
}
}
/** A one-shot owner for a connection read/write lock. */
static final class ConnectionLockLease implements AutoCloseable {
private final StampedLock lock;
private final long stamp;
private final boolean exclusive;
private boolean closed;
private ConnectionLockLease(StampedLock lock, long stamp, boolean exclusive) {
this.lock = lock;
this.stamp = stamp;
this.exclusive = exclusive;
}
@Override
public synchronized void close() {
if (!closed) {
closed = true;
if (exclusive) {
lock.unlockWrite(stamp);
} else {
lock.unlockRead(stamp);
}
}
}
}
record DwsCommandResult(String stdout, String stderr) {
}
private record CapturedOutput(String value, boolean truncated) {
}
}

View File

@ -0,0 +1,18 @@
package org.dromara.sync.connector.model;
/**
* 连接器连通性测试结果
*
* @param success 是否成功
* @param message 结果说明
*/
public record ConnectorTestResult(boolean success, String message) {
public static ConnectorTestResult success(String message) {
return new ConnectorTestResult(true, message);
}
public static ConnectorTestResult failure(String message) {
return new ConnectorTestResult(false, message);
}
}

View File

@ -0,0 +1,11 @@
package org.dromara.sync.connector.model;
/**
* 源端单页扫描请求
*
* @param parentObjectId 父对象标识为空表示根目录
* @param cursor 分页游标
* @param scopeId 源端作用域标识例如钉盘 spaceId
*/
public record ScanRequest(String parentObjectId, String cursor, String scopeId) {
}

View File

@ -0,0 +1,13 @@
package org.dromara.sync.connector.model;
import java.util.List;
/**
* 源端单页扫描结果
*
* @param objects 当前页对象
* @param nextCursor 下一页游标
* @param hasMore 是否还有下一页
*/
public record ScanResult(List<SourceObject> objects, String nextCursor, boolean hasMore) {
}

View File

@ -0,0 +1,36 @@
package org.dromara.sync.connector.model;
import java.nio.file.Files;
import java.nio.file.Path;
/**
* 下载或导出到本地临时文件的源内容
*
* @param path 本地路径
* @param size 实际大小
* @param contentType 内容类型
* @param deleteOnClose 关闭后是否清理文件及临时目录
*/
public record SourceContent(Path path, long size, String contentType, boolean deleteOnClose) implements AutoCloseable {
@Override
public void close() {
if (!deleteOnClose) {
return;
}
try {
Files.deleteIfExists(path);
Path parent = path.getParent();
if (parent != null) {
try (var children = Files.list(parent)) {
for (Path child : children.toList()) {
Files.deleteIfExists(child);
}
}
Files.deleteIfExists(parent);
}
} catch (Exception ignored) {
// 临时文件清理失败不能把已完成的远端上传回滚成失败状态
}
}
}

View File

@ -0,0 +1,38 @@
package org.dromara.sync.connector.model;
import java.time.LocalDateTime;
import java.util.Map;
/**
* 不依赖具体源厂商的统一数据对象
*
* @param objectId 源端稳定对象标识
* @param parentObjectId 父对象标识
* @param name 对象名称
* @param objectType FILEFOLDER ONLINE_DOCUMENT
* @param extension 文件扩展名
* @param contentType 内容类型
* @param size 文件大小
* @param modifiedTime 修改时间
* @param versionToken 源端版本标识
* @param hash 源端校验值
* @param metadata 源端扩展元数据
*/
public record SourceObject(
String objectId,
String parentObjectId,
String name,
String objectType,
String extension,
String contentType,
long size,
LocalDateTime modifiedTime,
String versionToken,
String hash,
Map<String, String> metadata
) {
public boolean isFolder() {
return "FOLDER".equals(objectType);
}
}

View File

@ -0,0 +1,22 @@
package org.dromara.sync.connector.model;
import java.nio.file.Path;
import java.util.Map;
/**
* 目标端写入请求
*
* @param objectKey 目标对象键
* @param path 待上传本地文件
* @param size 文件大小
* @param contentType 内容类型
* @param metadata 对象元数据
*/
public record TargetWriteRequest(
String objectKey,
Path path,
long size,
String contentType,
Map<String, String> metadata
) {
}

View File

@ -0,0 +1,23 @@
package org.dromara.sync.connector.model;
import java.util.Map;
/**
* 目标端写入结果
*
* @param objectKey 目标对象键
* @param versionId 目标版本标识
* @param eTag ETag
* @param size 已写入大小
* @param url 对象访问地址
* @param metadata 从目标端回读的对象元数据
*/
public record TargetWriteResult(
String objectKey,
String versionId,
String eTag,
long size,
String url,
Map<String, String> metadata
) {
}

View File

@ -0,0 +1,151 @@
package org.dromara.sync.connector.s3;
import jakarta.annotation.PreDestroy;
import org.dromara.common.core.exception.ServiceException;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.json.utils.JsonUtils;
import org.dromara.common.oss.client.OssClient;
import org.dromara.common.oss.config.OssClientConfig;
import org.dromara.common.oss.model.Options;
import org.dromara.common.oss.model.PutObjectResult;
import org.dromara.sync.connector.TargetConnector;
import org.dromara.sync.connector.model.ConnectorTestResult;
import org.dromara.sync.connector.model.TargetWriteRequest;
import org.dromara.sync.connector.model.TargetWriteResult;
import org.dromara.sync.domain.SyncConnection;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import tools.jackson.core.type.TypeReference;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* 基于仓库原生 S3 客户端的目标连接器基础实现
*/
public abstract class AbstractS3TargetConnector implements TargetConnector {
private final Map<Long, SyncS3OssClient> clients = new ConcurrentHashMap<>();
@Override
public ConnectorTestResult test(SyncConnection connection) {
try {
client(connection).headBucket(connection.getBucketName());
return ConnectorTestResult.success("目标存储桶访问成功");
} catch (Exception e) {
return ConnectorTestResult.failure(e.getMessage());
}
}
@Override
public TargetWriteResult upload(SyncConnection connection, TargetWriteRequest request) {
Options options = Options.builder()
.setLength(request.size())
.setContentType(request.contentType())
.setMetadata(request.metadata());
SyncS3OssClient client = client(connection);
PutObjectResult result = client.upload(request.objectKey(), request.path(), options);
HeadObjectResponse head = client.headObject(connection.getBucketName(), result.key());
return new TargetWriteResult(result.key(), head.versionId(), head.eTag(),
head.contentLength(), result.url(), head.metadata());
}
@Override
public boolean exists(SyncConnection connection, String objectKey) {
return client(connection).objectExists(connection.getBucketName(), objectKey);
}
@Override
public boolean delete(SyncConnection connection, String objectKey) {
return client(connection).delete(objectKey);
}
/**
* 获取不同厂商的默认路径访问风格
*/
protected boolean defaultPathStyleAccess() {
return false;
}
private SyncS3OssClient client(SyncConnection connection) {
OssClientConfig expected = buildConfig(connection);
return clients.compute(connection.getConnectionId(), (id, current) -> {
if (current != null && current.verifyConfig(expected)) {
return current;
}
closeQuietly(current);
return new SyncS3OssClient("sync-" + type().toLowerCase() + "-" + id, expected);
});
}
private OssClientConfig buildConfig(SyncConnection connection) {
if (StringUtils.isBlank(connection.getEndpoint())) {
throw new ServiceException("目标连接 endpoint 不能为空");
}
if (StringUtils.isBlank(connection.getBucketName())) {
throw new ServiceException("目标连接 bucketName 不能为空");
}
Map<String, Object> secret = parseJson(connection.getSecretJson());
String accessKey = value(secret, "accessKey");
String secretKey = value(secret, "secretKey");
if (StringUtils.isBlank(accessKey) || StringUtils.isBlank(secretKey)) {
throw new ServiceException("目标连接 accessKey、secretKey 不能为空");
}
Map<String, Object> config = parseJson(connection.getConfigJson());
boolean useHttps = booleanValue(config, "useHttps", true);
boolean pathStyle = booleanValue(config, "pathStyleAccess", defaultPathStyleAccess());
String domain = value(config, "domain");
Region region = StringUtils.isBlank(connection.getRegion())
? Region.US_EAST_1 : Region.of(connection.getRegion());
return OssClientConfig.builder()
.endpoint(connection.getEndpoint())
.domain(domain)
.useHttps(useHttps)
.usePathStyleAccess(pathStyle)
.accessKey(accessKey)
.secretKey(secretKey)
.bucket(connection.getBucketName())
.region(region)
.prefix(connection.getBasePath())
.build();
}
private Map<String, Object> parseJson(String json) {
if (StringUtils.isBlank(json)) {
return Map.of();
}
try {
return JsonUtils.parseObject(json, new TypeReference<>() {
});
} catch (Exception e) {
throw new ServiceException("连接配置不是合法 JSON", e);
}
}
private String value(Map<String, Object> values, String key) {
Object value = values.get(key);
return value == null ? null : String.valueOf(value);
}
private boolean booleanValue(Map<String, Object> values, String key, boolean defaultValue) {
Object value = values.get(key);
return value == null ? defaultValue : Boolean.parseBoolean(String.valueOf(value));
}
@PreDestroy
public void destroy() {
clients.values().forEach(this::closeQuietly);
clients.clear();
}
private void closeQuietly(OssClient client) {
if (client == null) {
return;
}
try {
client.close();
} catch (Exception ignored) {
// 客户端替换或容器销毁时不再抛出关闭异常
}
}
}

View File

@ -0,0 +1,19 @@
package org.dromara.sync.connector.s3;
import org.dromara.sync.constant.SyncConstants;
import org.springframework.stereotype.Component;
/**
* 阿里云 OSS 目标适配器
*
* <p>第一版复用仓库现有的 S3 协议客户端厂商差异保留在独立连接器中
* 后续可在不影响同步业务层的情况下切换阿里云原生 SDK</p>
*/
@Component
public class AliyunOssTargetConnector extends AbstractS3TargetConnector {
@Override
public String type() {
return SyncConstants.TYPE_ALIYUN_OSS;
}
}

View File

@ -0,0 +1,16 @@
package org.dromara.sync.connector.s3;
import org.dromara.sync.constant.SyncConstants;
import org.springframework.stereotype.Component;
/**
* S3 标准兼容目标连接器
*/
@Component
public class S3TargetConnector extends AbstractS3TargetConnector {
@Override
public String type() {
return SyncConstants.TYPE_S3;
}
}

View File

@ -0,0 +1,44 @@
package org.dromara.sync.connector.s3;
import org.dromara.common.oss.client.DefaultOssClientImpl;
import org.dromara.common.oss.config.OssClientConfig;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.S3Exception;
import java.util.concurrent.CompletionException;
/**
* 为同步连接器补充只读的存储桶连通性探测
*/
class SyncS3OssClient extends DefaultOssClientImpl {
SyncS3OssClient(String clientId, OssClientConfig config) {
super(clientId, config);
}
void headBucket(String bucket) {
s3AsyncClient.headBucket(builder -> builder.bucket(bucket)).join();
}
HeadObjectResponse headObject(String bucket, String key) {
return s3AsyncClient.headObject(builder -> builder.bucket(bucket).key(key)).join();
}
boolean objectExists(String bucket, String key) {
try {
headObject(bucket, key);
return true;
} catch (CompletionException e) {
Throwable cause = e.getCause();
if (cause instanceof S3Exception s3Exception && s3Exception.statusCode() == 404) {
return false;
}
throw e;
} catch (S3Exception e) {
if (e.statusCode() == 404) {
return false;
}
throw e;
}
}
}

View File

@ -0,0 +1,35 @@
package org.dromara.sync.constant;
/**
* 数据同步模块常量
*/
public interface SyncConstants {
String ROLE_SOURCE = "SOURCE";
String ROLE_TARGET = "TARGET";
String TYPE_DINGTALK = "DINGTALK";
String TYPE_S3 = "S3";
String TYPE_ALIYUN_OSS = "ALIYUN_OSS";
String STATUS_NORMAL = "0";
String STATUS_DISABLED = "1";
String JOB_PENDING = "PENDING";
String JOB_RUNNING = "RUNNING";
String JOB_CANCELING = "CANCELING";
String JOB_SUCCESS = "SUCCESS";
String JOB_PARTIAL_FAILED = "PARTIAL_FAILED";
String JOB_FAILED = "FAILED";
String JOB_CANCELED = "CANCELED";
String ITEM_PENDING = "PENDING";
String ITEM_RUNNING = "RUNNING";
String ITEM_SUCCESS = "SUCCESS";
String ITEM_SKIPPED = "SKIPPED";
String ITEM_FAILED = "FAILED";
String OBJECT_FILE = "FILE";
String OBJECT_FOLDER = "FOLDER";
String OBJECT_ONLINE_DOCUMENT = "ONLINE_DOCUMENT";
}

View File

@ -0,0 +1,185 @@
package org.dromara.sync.controller;
import cn.dev33.satoken.annotation.SaCheckPermission;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.RequiredArgsConstructor;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.core.domain.R;
import org.dromara.common.core.validate.AddGroup;
import org.dromara.common.core.validate.EditGroup;
import org.dromara.common.core.validate.QueryGroup;
import org.dromara.common.log.annotation.Log;
import org.dromara.common.log.enums.BusinessType;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.common.redis.annotation.RepeatSubmit;
import org.dromara.common.satoken.utils.LoginHelper;
import org.dromara.common.web.core.BaseController;
import org.dromara.sync.connector.model.ConnectorTestResult;
import org.dromara.sync.domain.bo.SyncConnectionBo;
import org.dromara.sync.domain.vo.DwsAuthSessionVo;
import org.dromara.sync.domain.vo.SyncConnectionVo;
import org.dromara.sync.service.ISyncConnectionService;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* 同步连接管理
*
* @author Codex
* @date 2026-09-02
*/
@Validated
@RequiredArgsConstructor
@RestController
@RequestMapping("/sync/connection")
public class SyncConnectionController extends BaseController {
private final ISyncConnectionService connectionService;
/**
* 分页查询同步连接列表
*
* @param bo 查询条件
* @param pageQuery 分页参数
* @return 同步连接分页列表
*/
@SaCheckPermission("sync:connection:list")
@GetMapping("/list")
public R<PageResult<SyncConnectionVo>> list(@Validated(QueryGroup.class) SyncConnectionBo bo,
PageQuery pageQuery) {
return R.ok(connectionService.queryPageList(bo, pageQuery));
}
/**
* 获取同步连接详情
*
* @param connectionId 连接主键
* @return 同步连接详情
*/
@SaCheckPermission("sync:connection:query")
@GetMapping("/{connectionId}")
public R<SyncConnectionVo> getInfo(@NotNull(message = "连接主键不能为空")
@PathVariable Long connectionId) {
return R.ok(connectionService.queryById(connectionId));
}
/**
* 新增同步连接
*
* @param bo 同步连接业务对象
* @return 操作结果
*/
@SaCheckPermission("sync:connection:add")
@Log(title = "同步连接", businessType = BusinessType.INSERT, excludeParamNames = "secretJson")
@RepeatSubmit()
@PostMapping()
public R<Void> add(@Validated(AddGroup.class) @RequestBody SyncConnectionBo bo) {
return toAjax(connectionService.insertByBo(bo));
}
/**
* 修改同步连接
*
* @param bo 同步连接业务对象
* @return 操作结果
*/
@SaCheckPermission("sync:connection:edit")
@Log(title = "同步连接", businessType = BusinessType.UPDATE, excludeParamNames = "secretJson")
@RepeatSubmit()
@PutMapping()
public R<Void> edit(@Validated(EditGroup.class) @RequestBody SyncConnectionBo bo) {
return toAjax(connectionService.updateByBo(bo));
}
/**
* 批量删除同步连接
*
* @param connectionIds 连接主键数组
* @return 操作结果
*/
@SaCheckPermission("sync:connection:remove")
@Log(title = "同步连接", businessType = BusinessType.DELETE)
@RepeatSubmit()
@DeleteMapping("/{connectionIds}")
public R<Void> remove(@NotEmpty(message = "连接主键不能为空")
@PathVariable Long[] connectionIds) {
return toAjax(connectionService.deleteWithValidByIds(List.of(connectionIds), true));
}
/**
* 校验同步连接配置并执行连通性测试
*
* @param connectionId 连接主键
* @return 操作结果
*/
@SaCheckPermission("sync:connection:test")
@Log(title = "同步连接测试", businessType = BusinessType.OTHER)
@RepeatSubmit()
@PostMapping("/test/{connectionId}")
public R<ConnectorTestResult> testConnection(@NotNull(message = "连接主键不能为空")
@PathVariable Long connectionId) {
return R.ok(connectionService.testConnection(connectionId));
}
/**
* Web 端启动钉钉设备授权服务器负责运行 DWS浏览器只接收一次性
* 授权链接和验证码不会接触任何 token 或应用密钥
*
* @param connectionId 钉钉连接主键
* @param expectedCorpId 可选的组织 ID用于防止授权到错误组织
* @return 授权会话及展示信息
*/
@SaCheckPermission("sync:connection:auth")
@Log(title = "钉钉 Web 登录", businessType = BusinessType.OTHER)
@RepeatSubmit()
@PostMapping("/{connectionId}/auth/login/start")
public R<DwsAuthSessionVo> startDingTalkAuth(
@NotNull(message = "连接主键不能为空") @PathVariable Long connectionId,
@RequestParam(required = false) String expectedCorpId) {
return R.ok(connectionService.startDingTalkAuth(connectionId, LoginHelper.getUserId(), expectedCorpId));
}
/**
* 查询钉钉 Web 设备授权状态
*
* @param connectionId 连接主键
* @param sessionId 授权会话 ID
* @return 授权状态
*/
@SaCheckPermission("sync:connection:auth")
@GetMapping("/{connectionId}/auth/login/{sessionId}")
public R<DwsAuthSessionVo> getDingTalkAuth(
@NotNull(message = "连接主键不能为空") @PathVariable Long connectionId,
@PathVariable String sessionId) {
return R.ok(connectionService.getDingTalkAuth(connectionId, sessionId, LoginHelper.getUserId()));
}
/**
* 取消钉钉 Web 设备授权并终止服务器上的 DWS 进程
*
* @param connectionId 连接主键
* @param sessionId 授权会话 ID
* @return 取消后的授权状态
*/
@SaCheckPermission("sync:connection:auth")
@Log(title = "取消钉钉 Web 登录", businessType = BusinessType.UPDATE)
@RepeatSubmit()
@PostMapping("/{connectionId}/auth/login/{sessionId}/cancel")
public R<DwsAuthSessionVo> cancelDingTalkAuth(
@NotNull(message = "连接主键不能为空") @PathVariable Long connectionId,
@PathVariable String sessionId) {
return R.ok(connectionService.cancelDingTalkAuth(connectionId, sessionId, LoginHelper.getUserId()));
}
}

View File

@ -0,0 +1,83 @@
package org.dromara.sync.controller;
import cn.dev33.satoken.annotation.SaCheckPermission;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.RequiredArgsConstructor;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.core.domain.R;
import org.dromara.common.log.annotation.Log;
import org.dromara.common.log.enums.BusinessType;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.common.redis.annotation.RepeatSubmit;
import org.dromara.common.web.core.BaseController;
import org.dromara.sync.domain.bo.SyncJobBo;
import org.dromara.sync.domain.bo.SyncJobItemBo;
import org.dromara.sync.domain.vo.SyncJobItemVo;
import org.dromara.sync.domain.vo.SyncJobVo;
import org.dromara.sync.service.ISyncJobService;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import java.util.List;
/**
* 同步任务管理接口
*/
@Validated
@RestController
@RequiredArgsConstructor
@RequestMapping("/sync/job")
public class SyncJobController extends BaseController {
private final ISyncJobService jobService;
@SaCheckPermission("sync:job:list")
@GetMapping("/list")
public R<PageResult<SyncJobVo>> list(SyncJobBo bo, PageQuery pageQuery) {
return R.ok(jobService.queryPageList(bo, pageQuery));
}
@SaCheckPermission("sync:job:query")
@GetMapping("/{jobId}")
public R<SyncJobVo> getInfo(@NotNull(message = "任务ID不能为空") @PathVariable Long jobId) {
return R.ok(jobService.queryById(jobId));
}
@SaCheckPermission("sync:job:query")
@GetMapping("/{jobId}/items")
public R<PageResult<SyncJobItemVo>> items(@PathVariable Long jobId, SyncJobItemBo bo, PageQuery pageQuery) {
bo.setJobId(jobId);
return R.ok(jobService.queryItemPageList(bo, pageQuery));
}
@SaCheckPermission("sync:job:run")
@Log(title = "同步任务", businessType = BusinessType.OTHER)
@RepeatSubmit
@PostMapping("/run/{planId}")
public R<Long> run(@NotNull(message = "计划ID不能为空") @PathVariable Long planId) {
return R.ok(jobService.startPlan(planId, "MANUAL", null));
}
@SaCheckPermission("sync:job:retry")
@Log(title = "同步任务重试", businessType = BusinessType.OTHER)
@RepeatSubmit
@PostMapping("/retry/{jobId}")
public R<Long> retry(@NotNull(message = "任务ID不能为空") @PathVariable Long jobId) {
return R.ok(jobService.retry(jobId));
}
@SaCheckPermission("sync:job:cancel")
@Log(title = "取消同步任务", businessType = BusinessType.UPDATE)
@PostMapping("/cancel/{jobId}")
public R<Void> cancel(@NotNull(message = "任务ID不能为空") @PathVariable Long jobId) {
return toAjax(jobService.cancel(jobId));
}
@SaCheckPermission("sync:job:remove")
@Log(title = "同步任务", businessType = BusinessType.DELETE)
@DeleteMapping("/{jobIds}")
public R<Void> remove(@NotEmpty(message = "任务ID不能为空") @PathVariable Long[] jobIds) {
return toAjax(jobService.deleteWithValidByIds(List.of(jobIds), true));
}
}

View File

@ -0,0 +1,38 @@
package org.dromara.sync.controller;
import cn.dev33.satoken.annotation.SaCheckPermission;
import jakarta.validation.constraints.NotNull;
import lombok.RequiredArgsConstructor;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.core.domain.R;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.common.web.core.BaseController;
import org.dromara.sync.domain.bo.SyncObjectBo;
import org.dromara.sync.domain.vo.SyncObjectVo;
import org.dromara.sync.service.ISyncObjectService;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
/**
* 同步对象清单接口
*/
@Validated
@RestController
@RequiredArgsConstructor
@RequestMapping("/sync/object")
public class SyncObjectController extends BaseController {
private final ISyncObjectService objectService;
@SaCheckPermission("sync:object:list")
@GetMapping("/list")
public R<PageResult<SyncObjectVo>> list(SyncObjectBo bo, PageQuery pageQuery) {
return R.ok(objectService.queryPageList(bo, pageQuery));
}
@SaCheckPermission("sync:object:query")
@GetMapping("/{objectId}")
public R<SyncObjectVo> getInfo(@NotNull(message = "对象ID不能为空") @PathVariable Long objectId) {
return R.ok(objectService.queryById(objectId));
}
}

View File

@ -0,0 +1,121 @@
package org.dromara.sync.controller;
import cn.dev33.satoken.annotation.SaCheckPermission;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.RequiredArgsConstructor;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.core.domain.R;
import org.dromara.common.core.validate.AddGroup;
import org.dromara.common.core.validate.EditGroup;
import org.dromara.common.core.validate.QueryGroup;
import org.dromara.common.log.annotation.Log;
import org.dromara.common.log.enums.BusinessType;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.common.redis.annotation.RepeatSubmit;
import org.dromara.common.web.core.BaseController;
import org.dromara.sync.domain.bo.SyncPlanBo;
import org.dromara.sync.domain.vo.SyncPlanVo;
import org.dromara.sync.service.ISyncPlanService;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* 同步计划管理
*
* @author Codex
* @date 2026-09-02
*/
@Validated
@RequiredArgsConstructor
@RestController
@RequestMapping("/sync/plan")
public class SyncPlanController extends BaseController {
private final ISyncPlanService planService;
/**
* 分页查询同步计划列表
*
* @param bo 查询条件
* @param pageQuery 分页参数
* @return 同步计划分页数据
*/
@SaCheckPermission("sync:plan:list")
@GetMapping("/list")
public R<PageResult<SyncPlanVo>> list(@Validated(QueryGroup.class) SyncPlanBo bo, PageQuery pageQuery) {
return R.ok(planService.queryPageList(bo, pageQuery));
}
/**
* 获取同步计划详细信息
*
* @param planId 计划主键
* @return 同步计划详情
*/
@SaCheckPermission("sync:plan:query")
@GetMapping("/{planId}")
public R<SyncPlanVo> getInfo(@NotNull(message = "计划主键不能为空")
@PathVariable Long planId) {
return R.ok(planService.queryById(planId));
}
/**
* 新增同步计划
*
* @param bo 同步计划
* @return 操作结果
*/
@SaCheckPermission("sync:plan:add")
@Log(title = "同步计划", businessType = BusinessType.INSERT)
@RepeatSubmit()
@PostMapping()
public R<Void> add(@Validated(AddGroup.class) @RequestBody SyncPlanBo bo) {
if (!planService.checkPlanNameUnique(bo)) {
return R.fail("新增同步计划'" + bo.getPlanName() + "'失败,计划名称已存在");
}
return toAjax(planService.insertByBo(bo));
}
/**
* 修改同步计划
*
* @param bo 同步计划
* @return 操作结果
*/
@SaCheckPermission("sync:plan:edit")
@Log(title = "同步计划", businessType = BusinessType.UPDATE)
@RepeatSubmit()
@PutMapping()
public R<Void> edit(@Validated(EditGroup.class) @RequestBody SyncPlanBo bo) {
if (!planService.checkPlanNameUnique(bo)) {
return R.fail("修改同步计划'" + bo.getPlanName() + "'失败,计划名称已存在");
}
return toAjax(planService.updateByBo(bo));
}
/**
* 批量删除同步计划
*
* @param planIds 计划主键集合
* @return 操作结果
*/
@SaCheckPermission("sync:plan:remove")
@Log(title = "同步计划", businessType = BusinessType.DELETE)
@RepeatSubmit()
@DeleteMapping("/{planIds}")
public R<Void> remove(@NotEmpty(message = "计划主键不能为空")
@PathVariable Long[] planIds) {
return toAjax(planService.deleteWithValidByIds(List.of(planIds), true));
}
}

View File

@ -0,0 +1,34 @@
package org.dromara.sync.domain;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.baomidou.mybatisplus.annotation.Version;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.dromara.common.mybatis.core.domain.BaseEntity;
import java.io.Serial;
import java.time.LocalDateTime;
/**
* 同步增量与分页断点 sync_checkpoint
*/
@Data
@EqualsAndHashCode(callSuper = true)
@TableName("sync_checkpoint")
public class SyncCheckpoint extends BaseEntity {
@Serial
private static final long serialVersionUID = 1L;
@TableId("checkpoint_id")
private Long checkpointId;
private Long planId;
private String checkpointType;
private String checkpointKey;
private String checkpointValue;
private LocalDateTime watermarkTime;
private Long lastJobId;
@Version
private Long version;
}

View File

@ -0,0 +1,96 @@
package org.dromara.sync.domain;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableLogic;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.dromara.common.encrypt.annotation.EncryptField;
import org.dromara.common.encrypt.enums.AlgorithmType;
import org.dromara.common.mybatis.core.domain.BaseEntity;
import java.io.Serial;
/**
* 同步连接对象 sync_connection
*
* @author Codex
* @date 2026-09-02
*/
@Data
@EqualsAndHashCode(callSuper = true)
@TableName("sync_connection")
public class SyncConnection extends BaseEntity {
@Serial
private static final long serialVersionUID = 1L;
/**
* 连接主键
*/
@TableId(value = "connection_id")
private Long connectionId;
/**
* 连接名称
*/
private String connectionName;
/**
* 连接角色SOURCE 源端TARGET 目标端
*/
private String connectionRole;
/**
* 连接类型DINGTALKS3ALIYUN_OSS
*/
private String connectionType;
/**
* 服务端点
*/
private String endpoint;
/**
* 服务区域
*/
private String region;
/**
* 存储桶名称
*/
private String bucketName;
/**
* 目标基础路径
*/
private String basePath;
/**
* 非敏感扩展配置 JSON
*/
private String configJson;
/**
* 敏感凭证配置 JSON持久化时由 MyBatis 加密处理器加密
*/
@EncryptField(algorithm = AlgorithmType.AES)
private String secretJson;
/**
* 状态0 正常1 停用
*/
private String status;
/**
* 删除标志0 存在1 删除
*/
@TableLogic
private String delFlag;
/**
* 备注
*/
private String remark;
}

View File

@ -0,0 +1,42 @@
package org.dromara.sync.domain;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.dromara.common.mybatis.core.domain.BaseEntity;
import java.io.Serial;
import java.time.LocalDateTime;
/**
* 同步任务运行实例 sync_job
*/
@Data
@EqualsAndHashCode(callSuper = true)
@TableName("sync_job")
public class SyncJob extends BaseEntity {
@Serial
private static final long serialVersionUID = 1L;
@TableId("job_id")
private Long jobId;
private Long planId;
private String triggerType;
private String runType;
private String status;
private String checkpointBefore;
private String checkpointAfter;
private Long totalCount;
private Long processedCount;
private Long successCount;
private Long failedCount;
private Long skippedCount;
private Long deletedCount;
private Long totalBytes;
private Long transferredBytes;
private LocalDateTime startTime;
private LocalDateTime finishTime;
private String errorMessage;
}

View File

@ -0,0 +1,45 @@
package org.dromara.sync.domain;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.dromara.common.mybatis.core.domain.BaseEntity;
import java.io.Serial;
import java.time.LocalDateTime;
/**
* 同步任务文件明细 sync_job_item
*/
@Data
@EqualsAndHashCode(callSuper = true)
@TableName("sync_job_item")
public class SyncJobItem extends BaseEntity {
@Serial
private static final long serialVersionUID = 1L;
@TableId("item_id")
private Long itemId;
private Long jobId;
private String sourceObjectId;
private String parentObjectId;
private String sourcePath;
private String targetKey;
private String objectType;
private String actionType;
private String status;
private Long size;
private Long transferredBytes;
private String versionToken;
private String sourceEtag;
private String sourceSha256;
private String targetEtag;
private String targetVersionId;
private Integer retryCount;
private LocalDateTime startTime;
private LocalDateTime finishTime;
private String errorCode;
private String errorMessage;
}

View File

@ -0,0 +1,51 @@
package org.dromara.sync.domain;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.FieldStrategy;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.dromara.common.mybatis.core.domain.BaseEntity;
import java.io.Serial;
import java.time.LocalDateTime;
/**
* 源对象与目标对象映射清单 sync_object
*/
@Data
@EqualsAndHashCode(callSuper = true)
@TableName("sync_object")
public class SyncObject extends BaseEntity {
@Serial
private static final long serialVersionUID = 1L;
@TableId("object_id")
private Long objectId;
private Long planId;
private String sourceObjectId;
private String parentObjectId;
private String sourcePath;
private String objectName;
private String objectType;
private Long size;
private LocalDateTime modifiedTime;
private String versionToken;
private String sourceEtag;
private String sourceSha256;
private String contentType;
private String metadataJson;
private String targetKey;
private String targetVersionId;
private String targetEtag;
private String syncStatus;
private String sourceDeleted;
private Long firstSeenJobId;
private Long lastSeenJobId;
private Long lastSyncJobId;
private LocalDateTime lastSyncTime;
@TableField(updateStrategy = FieldStrategy.ALWAYS)
private String lastErrorMessage;
}

View File

@ -0,0 +1,132 @@
package org.dromara.sync.domain;
import com.baomidou.mybatisplus.annotation.FieldStrategy;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableLogic;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.dromara.common.mybatis.core.domain.BaseEntity;
import java.io.Serial;
import java.time.LocalDateTime;
/**
* 同步计划对象 sync_plan
*
* @author Codex
* @date 2026-09-02
*/
@Data
@EqualsAndHashCode(callSuper = true)
@TableName("sync_plan")
public class SyncPlan extends BaseEntity {
@Serial
private static final long serialVersionUID = 1L;
/**
* 计划主键
*/
@TableId(value = "plan_id")
private Long planId;
/**
* 计划名称
*/
private String planName;
/**
* 源端连接主键
*/
private Long sourceConnectionId;
/**
* 目标端连接主键
*/
private Long targetConnectionId;
/**
* 源端同步根路径或对象标识
*/
private String sourceRoot;
/**
* 目标端对象键前缀
*/
private String targetPrefix;
/**
* 同步模式FULL 全量INCREMENTAL 增量
*/
private String syncMode;
/**
* 调度类型MANUAL 手动CRON 定时
*/
private String scheduleType;
/**
* CRON 表达式切换为手动调度时需要显式清空
*/
@TableField(updateStrategy = FieldStrategy.ALWAYS)
private String cronExpression;
/**
* 冲突处理策略OVERWRITE 覆盖SKIP 跳过KEEP_BOTH 两者保留
*/
private String conflictStrategy;
/**
* 源端删除处理策略KEEP 保留MARK 标记DELETE 删除
*/
private String deleteStrategy;
/**
* 单次目标删除比例保护阈值范围 0 100
*/
private Integer deleteGuardPercent;
/**
* 完整性校验方式SIZEETAGSHA256
*/
private String verifyMode;
/**
* 最大并发传输数
*/
private Integer maxConcurrency;
/**
* 带宽上限单位 KB/s0 表示不限速
*/
private Long bandwidthLimitKbps;
/**
* 状态0 正常1 停用
*/
private String status;
/**
* 最近一次运行时间
*/
private LocalDateTime lastRunTime;
/**
* 下一次计划运行时间
*/
private LocalDateTime nextRunTime;
/**
* 删除标志0 存在1 删除
*/
@TableLogic
private String delFlag;
/**
* 备注
*/
private String remark;
}

View File

@ -0,0 +1,38 @@
package org.dromara.sync.domain;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.dromara.common.mybatis.core.domain.BaseEntity;
import java.io.Serial;
import java.time.LocalDateTime;
/**
* 大文件分片传输断点 sync_transfer_part
*/
@Data
@EqualsAndHashCode(callSuper = true)
@TableName("sync_transfer_part")
public class SyncTransferPart extends BaseEntity {
@Serial
private static final long serialVersionUID = 1L;
@TableId("part_id")
private Long partId;
private Long jobItemId;
private String uploadId;
private Integer partNumber;
private Long partOffset;
private Long partSize;
private Long transferredBytes;
private String partEtag;
private String checksumSha256;
private String status;
private Integer retryCount;
private LocalDateTime startTime;
private LocalDateTime finishTime;
private String errorMessage;
}

View File

@ -0,0 +1,105 @@
package org.dromara.sync.domain.bo;
import io.github.linpeilie.annotations.AutoMapper;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Pattern;
import lombok.Data;
import org.dromara.common.core.validate.AddGroup;
import org.dromara.common.core.validate.EditGroup;
import org.dromara.common.json.validate.JsonPattern;
import org.dromara.common.json.validate.JsonType;
import org.dromara.sync.domain.SyncConnection;
import java.io.Serial;
import java.io.Serializable;
/**
* 同步连接业务对象 sync_connection
*
* @author Codex
* @date 2026-09-02
*/
@Data
@AutoMapper(target = SyncConnection.class, reverseConvertGenerate = false)
public class SyncConnectionBo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
/**
* 连接主键
*/
@NotNull(message = "连接主键不能为空", groups = EditGroup.class)
private Long connectionId;
/**
* 连接名称
*/
@NotBlank(message = "连接名称不能为空", groups = {AddGroup.class, EditGroup.class})
private String connectionName;
/**
* 连接角色SOURCE 源端TARGET 目标端
*/
@NotBlank(message = "连接角色不能为空", groups = {AddGroup.class, EditGroup.class})
@Pattern(regexp = "SOURCE|TARGET", message = "连接角色只支持 SOURCE 或 TARGET",
groups = {AddGroup.class, EditGroup.class})
private String connectionRole;
/**
* 连接类型DINGTALKS3ALIYUN_OSS
*/
@NotBlank(message = "连接类型不能为空", groups = {AddGroup.class, EditGroup.class})
@Pattern(regexp = "DINGTALK|S3|ALIYUN_OSS", message = "连接类型不受支持",
groups = {AddGroup.class, EditGroup.class})
private String connectionType;
/**
* 服务端点
*/
private String endpoint;
/**
* 服务区域
*/
private String region;
/**
* 存储桶名称
*/
private String bucketName;
/**
* 目标基础路径
*/
private String basePath;
/**
* 非敏感扩展配置 JSON
*/
@JsonPattern(type = JsonType.OBJECT, message = "扩展配置必须是 JSON 对象",
groups = {AddGroup.class, EditGroup.class})
private String configJson;
/**
* 敏感凭证配置 JSON修改时留空表示保留原值
*/
@JsonPattern(type = JsonType.OBJECT, message = "敏感配置必须是 JSON 对象",
groups = {AddGroup.class, EditGroup.class})
private String secretJson;
/**
* 状态0 正常1 停用
*/
@NotBlank(message = "连接状态不能为空", groups = {AddGroup.class, EditGroup.class})
@Pattern(regexp = "[01]", message = "连接状态只支持 0 或 1",
groups = {AddGroup.class, EditGroup.class})
private String status;
/**
* 备注
*/
private String remark;
}

View File

@ -0,0 +1,24 @@
package org.dromara.sync.domain.bo;
import lombok.Data;
import java.io.Serial;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
/**
* 同步任务查询对象
*/
@Data
public class SyncJobBo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
private Long planId;
private String triggerType;
private String runType;
private String status;
private Map<String, Object> params = new HashMap<>();
}

View File

@ -0,0 +1,22 @@
package org.dromara.sync.domain.bo;
import lombok.Data;
import java.io.Serial;
import java.io.Serializable;
/**
* 同步任务明细查询对象
*/
@Data
public class SyncJobItemBo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
private Long jobId;
private String sourcePath;
private String objectType;
private String actionType;
private String status;
}

View File

@ -0,0 +1,23 @@
package org.dromara.sync.domain.bo;
import lombok.Data;
import java.io.Serial;
import java.io.Serializable;
/**
* 同步对象清单查询对象
*/
@Data
public class SyncObjectBo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
private Long planId;
private String sourceObjectId;
private String sourcePath;
private String objectType;
private String syncStatus;
private String sourceDeleted;
}

View File

@ -0,0 +1,165 @@
package org.dromara.sync.domain.bo;
import io.github.linpeilie.annotations.AutoMapper;
import jakarta.validation.constraints.Max;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Pattern;
import jakarta.validation.constraints.PositiveOrZero;
import jakarta.validation.constraints.Size;
import lombok.Data;
import org.dromara.common.core.validate.AddGroup;
import org.dromara.common.core.validate.EditGroup;
import org.dromara.sync.domain.SyncPlan;
import java.io.Serial;
import java.io.Serializable;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.Map;
/**
* 同步计划业务对象 sync_plan
*
* @author Codex
* @date 2026-09-02
*/
@Data
@AutoMapper(target = SyncPlan.class, reverseConvertGenerate = false)
public class SyncPlanBo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
/**
* 计划主键
*/
@NotNull(message = "计划主键不能为空", groups = EditGroup.class)
private Long planId;
/**
* 计划名称
*/
@NotBlank(message = "计划名称不能为空", groups = {AddGroup.class, EditGroup.class})
@Size(max = 100, message = "计划名称不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class})
private String planName;
/**
* 源端连接主键
*/
@NotNull(message = "源端连接不能为空", groups = {AddGroup.class, EditGroup.class})
private Long sourceConnectionId;
/**
* 目标端连接主键
*/
@NotNull(message = "目标端连接不能为空", groups = {AddGroup.class, EditGroup.class})
private Long targetConnectionId;
/**
* 源端同步根路径或对象标识
*/
@Size(max = 1024, message = "源端同步根路径不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class})
private String sourceRoot;
/**
* 目标端对象键前缀
*/
@Size(max = 1024, message = "目标端对象键前缀不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class})
private String targetPrefix;
/**
* 同步模式FULLINCREMENTAL
*/
@NotBlank(message = "同步模式不能为空", groups = {AddGroup.class, EditGroup.class})
@Pattern(regexp = "FULL|INCREMENTAL", message = "同步模式不正确", groups = {AddGroup.class, EditGroup.class})
private String syncMode;
/**
* 调度类型MANUALCRON
*/
@NotBlank(message = "调度类型不能为空", groups = {AddGroup.class, EditGroup.class})
@Pattern(regexp = "MANUAL|CRON", message = "调度类型不正确", groups = {AddGroup.class, EditGroup.class})
private String scheduleType;
/**
* CRON 表达式
*/
@Size(max = 128, message = "CRON表达式不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class})
private String cronExpression;
/**
* 冲突处理策略OVERWRITESKIPKEEP_BOTH
*/
@NotBlank(message = "冲突处理策略不能为空", groups = {AddGroup.class, EditGroup.class})
@Pattern(regexp = "OVERWRITE|SKIP|KEEP_BOTH", message = "冲突处理策略不正确",
groups = {AddGroup.class, EditGroup.class})
private String conflictStrategy;
/**
* 源端删除处理策略KEEPMARKDELETE
*/
@NotBlank(message = "删除处理策略不能为空", groups = {AddGroup.class, EditGroup.class})
@Pattern(regexp = "KEEP|MARK|DELETE", message = "删除处理策略不正确", groups = {AddGroup.class, EditGroup.class})
private String deleteStrategy;
/**
* 单次目标删除比例保护阈值100 表示允许一次删除全部对象
*/
@NotNull(message = "删除保护阈值不能为空", groups = {AddGroup.class, EditGroup.class})
@Min(value = 0, message = "删除保护阈值不能小于{value}", groups = {AddGroup.class, EditGroup.class})
@Max(value = 100, message = "删除保护阈值不能大于{value}", groups = {AddGroup.class, EditGroup.class})
private Integer deleteGuardPercent;
/**
* 完整性校验方式SIZEETAGSHA256
*/
@NotBlank(message = "校验方式不能为空", groups = {AddGroup.class, EditGroup.class})
@Pattern(regexp = "SIZE|ETAG|SHA256", message = "校验方式不正确", groups = {AddGroup.class, EditGroup.class})
private String verifyMode;
/**
* 最大并发传输数范围为 1 100
*/
@NotNull(message = "最大并发数不能为空", groups = {AddGroup.class, EditGroup.class})
@Min(value = 1, message = "最大并发数不能小于{value}", groups = {AddGroup.class, EditGroup.class})
@Max(value = 100, message = "最大并发数不能大于{value}", groups = {AddGroup.class, EditGroup.class})
private Integer maxConcurrency;
/**
* 带宽上限单位 KB/s0 表示不限速
*/
@NotNull(message = "带宽上限不能为空", groups = {AddGroup.class, EditGroup.class})
@PositiveOrZero(message = "带宽上限不能小于0", groups = {AddGroup.class, EditGroup.class})
private Long bandwidthLimitKbps;
/**
* 状态0 正常1 停用
*/
@NotBlank(message = "状态不能为空", groups = {AddGroup.class, EditGroup.class})
@Pattern(regexp = "0|1", message = "状态值不正确", groups = {AddGroup.class, EditGroup.class})
private String status;
/**
* 最近一次运行时间
*/
private LocalDateTime lastRunTime;
/**
* 下一次计划运行时间
*/
private LocalDateTime nextRunTime;
/**
* 备注
*/
@Size(max = 500, message = "备注不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class})
private String remark;
/**
* 扩展查询参数
*/
private Map<String, Object> params = new HashMap<>();
}

View File

@ -0,0 +1,48 @@
package org.dromara.sync.domain.vo;
import java.io.Serial;
import java.io.Serializable;
/**
* 钉钉 Web 设备授权会话视图
*
* <p>该对象只包含一次性授权链接验证码和脱敏身份信息绝不包含
* access tokenrefresh tokendevice code 或应用密钥</p>
*
* @param sessionId 授权会话 ID
* @param connectionId 同步连接 ID
* @param status 会话状态
* @param verificationUri 用户输入验证码的授权地址
* @param verificationUriComplete 可直接打开或生成二维码的完整授权地址
* @param userCode 一次性用户验证码
* @param expiresAt 会话过期时间Unix 毫秒
* @param remainingSeconds 距离过期的秒数
* @param pollInterval 前端建议轮询间隔
* @param corpId 授权后的组织 ID
* @param corpName 授权后的组织名称
* @param userId 授权后的钉钉用户 ID
* @param userName 授权后的用户名称
* @param profile DWS 稳定 profilecorpId:userId
* @param message 面向用户的脱敏提示
*/
public record DwsAuthSessionVo(
String sessionId,
Long connectionId,
String status,
String verificationUri,
String verificationUriComplete,
String userCode,
Long expiresAt,
long remainingSeconds,
int pollInterval,
String corpId,
String corpName,
String userId,
String userName,
String profile,
String message
) implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
}

View File

@ -0,0 +1,29 @@
package org.dromara.sync.domain.vo;
import io.github.linpeilie.annotations.AutoMapper;
import lombok.Data;
import org.dromara.sync.domain.SyncCheckpoint;
import java.io.Serial;
import java.io.Serializable;
import java.time.LocalDateTime;
/**
* 同步断点视图对象
*/
@Data
@AutoMapper(target = SyncCheckpoint.class)
public class SyncCheckpointVo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
private Long checkpointId;
private Long planId;
private String checkpointType;
private String checkpointKey;
private String checkpointValue;
private LocalDateTime watermarkTime;
private Long lastJobId;
private Long version;
}

View File

@ -0,0 +1,106 @@
package org.dromara.sync.domain.vo;
import io.github.linpeilie.annotations.AutoMapper;
import lombok.Data;
import org.dromara.sync.domain.SyncConnection;
import java.io.Serial;
import java.io.Serializable;
import java.time.LocalDateTime;
/**
* 同步连接视图对象 sync_connection
*
* <p>敏感凭证字段不在该视图对象中声明任何查询接口均不会返回凭证内容</p>
*
* @author Codex
* @date 2026-09-02
*/
@Data
@AutoMapper(target = SyncConnection.class)
public class SyncConnectionVo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
/**
* 连接主键
*/
private Long connectionId;
/**
* 连接名称
*/
private String connectionName;
/**
* 连接角色SOURCE 源端TARGET 目标端
*/
private String connectionRole;
/**
* 连接类型DINGTALKS3ALIYUN_OSS
*/
private String connectionType;
/**
* 服务端点
*/
private String endpoint;
/**
* 服务区域
*/
private String region;
/**
* 存储桶名称
*/
private String bucketName;
/**
* 目标基础路径
*/
private String basePath;
/**
* 非敏感扩展配置 JSON
*/
private String configJson;
/**
* 状态0 正常1 停用
*/
private String status;
/**
* 备注
*/
private String remark;
/**
* 创建部门
*/
private Long createDept;
/**
* 创建者
*/
private Long createBy;
/**
* 创建时间
*/
private LocalDateTime createTime;
/**
* 更新者
*/
private Long updateBy;
/**
* 更新时间
*/
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,42 @@
package org.dromara.sync.domain.vo;
import io.github.linpeilie.annotations.AutoMapper;
import lombok.Data;
import org.dromara.sync.domain.SyncJobItem;
import java.io.Serial;
import java.io.Serializable;
import java.time.LocalDateTime;
/**
* 同步任务明细视图对象
*/
@Data
@AutoMapper(target = SyncJobItem.class)
public class SyncJobItemVo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
private Long itemId;
private Long jobId;
private String sourceObjectId;
private String parentObjectId;
private String sourcePath;
private String targetKey;
private String objectType;
private String actionType;
private String status;
private Long size;
private Long transferredBytes;
private String versionToken;
private String sourceEtag;
private String sourceSha256;
private String targetEtag;
private String targetVersionId;
private Integer retryCount;
private LocalDateTime startTime;
private LocalDateTime finishTime;
private String errorCode;
private String errorMessage;
}

View File

@ -0,0 +1,40 @@
package org.dromara.sync.domain.vo;
import io.github.linpeilie.annotations.AutoMapper;
import lombok.Data;
import org.dromara.sync.domain.SyncJob;
import java.io.Serial;
import java.io.Serializable;
import java.time.LocalDateTime;
/**
* 同步任务视图对象
*/
@Data
@AutoMapper(target = SyncJob.class)
public class SyncJobVo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
private Long jobId;
private Long planId;
private String triggerType;
private String runType;
private String status;
private String checkpointBefore;
private String checkpointAfter;
private Long totalCount;
private Long processedCount;
private Long successCount;
private Long failedCount;
private Long skippedCount;
private Long deletedCount;
private Long totalBytes;
private Long transferredBytes;
private LocalDateTime startTime;
private LocalDateTime finishTime;
private String errorMessage;
private LocalDateTime createTime;
}

View File

@ -0,0 +1,47 @@
package org.dromara.sync.domain.vo;
import io.github.linpeilie.annotations.AutoMapper;
import lombok.Data;
import org.dromara.sync.domain.SyncObject;
import java.io.Serial;
import java.io.Serializable;
import java.time.LocalDateTime;
/**
* 同步对象清单视图对象
*/
@Data
@AutoMapper(target = SyncObject.class)
public class SyncObjectVo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
private Long objectId;
private Long planId;
private String sourceObjectId;
private String parentObjectId;
private String sourcePath;
private String objectName;
private String objectType;
private Long size;
private LocalDateTime modifiedTime;
private String versionToken;
private String sourceEtag;
private String sourceSha256;
private String contentType;
private String metadataJson;
private String targetKey;
private String targetVersionId;
private String targetEtag;
private String syncStatus;
private String sourceDeleted;
private Long firstSeenJobId;
private Long lastSeenJobId;
private Long lastSyncJobId;
private LocalDateTime lastSyncTime;
private String lastErrorMessage;
private LocalDateTime createTime;
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,144 @@
package org.dromara.sync.domain.vo;
import io.github.linpeilie.annotations.AutoMapper;
import lombok.Data;
import org.dromara.sync.domain.SyncPlan;
import java.io.Serial;
import java.io.Serializable;
import java.time.LocalDateTime;
/**
* 同步计划视图对象 sync_plan
*
* @author Codex
* @date 2026-09-02
*/
@Data
@AutoMapper(target = SyncPlan.class)
public class SyncPlanVo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
/**
* 计划主键
*/
private Long planId;
/**
* 计划名称
*/
private String planName;
/**
* 源端连接主键
*/
private Long sourceConnectionId;
/**
* 目标端连接主键
*/
private Long targetConnectionId;
/**
* 源端同步根路径或对象标识
*/
private String sourceRoot;
/**
* 目标端对象键前缀
*/
private String targetPrefix;
/**
* 同步模式FULLINCREMENTAL
*/
private String syncMode;
/**
* 调度类型MANUALCRON
*/
private String scheduleType;
/**
* CRON 表达式
*/
private String cronExpression;
/**
* 冲突处理策略OVERWRITESKIPKEEP_BOTH
*/
private String conflictStrategy;
/**
* 源端删除处理策略KEEPMARKDELETE
*/
private String deleteStrategy;
/**
* 单次目标删除比例保护阈值
*/
private Integer deleteGuardPercent;
/**
* 完整性校验方式SIZEETAGSHA256
*/
private String verifyMode;
/**
* 最大并发传输数
*/
private Integer maxConcurrency;
/**
* 带宽上限单位 KB/s0 表示不限速
*/
private Long bandwidthLimitKbps;
/**
* 状态0 正常1 停用
*/
private String status;
/**
* 最近一次运行时间
*/
private LocalDateTime lastRunTime;
/**
* 下一次计划运行时间
*/
private LocalDateTime nextRunTime;
/**
* 备注
*/
private String remark;
/**
* 创建部门
*/
private Long createDept;
/**
* 创建者
*/
private Long createBy;
/**
* 创建时间
*/
private LocalDateTime createTime;
/**
* 更新者
*/
private Long updateBy;
/**
* 更新时间
*/
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,35 @@
package org.dromara.sync.domain.vo;
import io.github.linpeilie.annotations.AutoMapper;
import lombok.Data;
import org.dromara.sync.domain.SyncTransferPart;
import java.io.Serial;
import java.io.Serializable;
import java.time.LocalDateTime;
/**
* 传输分片视图对象
*/
@Data
@AutoMapper(target = SyncTransferPart.class)
public class SyncTransferPartVo implements Serializable {
@Serial
private static final long serialVersionUID = 1L;
private Long partId;
private Long jobItemId;
private String uploadId;
private Integer partNumber;
private Long partOffset;
private Long partSize;
private Long transferredBytes;
private String partEtag;
private String checksumSha256;
private String status;
private Integer retryCount;
private LocalDateTime startTime;
private LocalDateTime finishTime;
private String errorMessage;
}

View File

@ -0,0 +1,31 @@
package org.dromara.sync.job;
import cn.hutool.core.convert.Convert;
import com.aizuda.snailjob.client.job.core.annotation.JobExecutor;
import com.aizuda.snailjob.client.job.core.dto.JobArgs;
import com.aizuda.snailjob.model.dto.ExecuteResult;
import lombok.RequiredArgsConstructor;
import org.dromara.sync.service.ISyncJobService;
import org.springframework.stereotype.Component;
/**
* SnailJob 同步计划执行入口
*
* <p>执行器名称为 {@code syncPlanJobExecutor}任务参数填写同步计划 ID</p>
*/
@Component
@RequiredArgsConstructor
@JobExecutor(name = "syncPlanJobExecutor")
public class SyncPlanJobExecutor {
private final ISyncJobService jobService;
public ExecuteResult jobExecute(JobArgs jobArgs) {
Long planId = Convert.toLong(jobArgs.getJobParams());
if (planId == null) {
return ExecuteResult.failure("任务参数必须是同步计划 ID");
}
Long jobId = jobService.startPlan(planId, "SCHEDULE", null);
return ExecuteResult.success("同步任务已提交jobId=" + jobId);
}
}

View File

@ -0,0 +1,75 @@
package org.dromara.sync.job;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.dromara.sync.constant.SyncConstants;
import org.dromara.sync.domain.SyncPlan;
import org.dromara.sync.mapper.SyncPlanMapper;
import org.dromara.sync.service.ISyncJobService;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.scheduling.support.CronExpression;
import org.springframework.stereotype.Component;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.util.List;
/**
* 从同步计划表领取到期的 Cron 计划
*
* <p>通过 next_run_time 条件更新完成多实例抢占同一触发时刻只有一个应用实例能够提交任务</p>
*/
@Slf4j
@Component
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "sync.scheduler", name = "enabled", havingValue = "true", matchIfMissing = true)
public class SyncPlanScheduler {
private final SyncPlanMapper planMapper;
private final ISyncJobService jobService;
@Scheduled(fixedDelayString = "${sync.scheduler.poll-interval-ms:30000}")
public void dispatchDuePlans() {
LocalDateTime now = LocalDateTime.now();
List<SyncPlan> plans = planMapper.selectList(Wrappers.<SyncPlan>lambdaQuery()
.eq(SyncPlan::getStatus, SyncConstants.STATUS_NORMAL)
.eq(SyncPlan::getScheduleType, "CRON")
.isNotNull(SyncPlan::getNextRunTime)
.le(SyncPlan::getNextRunTime, now)
.orderByAsc(SyncPlan::getNextRunTime)
.last("limit 100"));
for (SyncPlan plan : plans) {
dispatch(plan, now);
}
}
private void dispatch(SyncPlan plan, LocalDateTime now) {
try {
LocalDateTime nextRunTime = nextRunTime(plan.getCronExpression(), now);
boolean claimed = planMapper.lambda()
.set(SyncPlan::getNextRunTime, nextRunTime)
.eq(SyncPlan::getPlanId, plan.getPlanId())
.eq(SyncPlan::getNextRunTime, plan.getNextRunTime())
.eq(SyncPlan::getCronExpression, plan.getCronExpression())
.eq(SyncPlan::getScheduleType, "CRON")
.eq(SyncPlan::getStatus, SyncConstants.STATUS_NORMAL)
.update();
if (!claimed) {
return;
}
jobService.startPlan(plan.getPlanId(), "SCHEDULE", null);
} catch (Exception e) {
log.warn("到期同步计划提交失败planId={},原因={}", plan.getPlanId(), e.getMessage());
}
}
private LocalDateTime nextRunTime(String cronExpression, LocalDateTime now) {
ZonedDateTime next = CronExpression.parse(cronExpression).next(now.atZone(java.time.ZoneId.systemDefault()));
if (next == null) {
throw new IllegalArgumentException("Cron 表达式没有下一次触发时间");
}
return next.toLocalDateTime();
}
}

View File

@ -0,0 +1,11 @@
package org.dromara.sync.mapper;
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
import org.dromara.sync.domain.SyncCheckpoint;
import org.dromara.sync.domain.vo.SyncCheckpointVo;
/**
* 同步断点 Mapper
*/
public interface SyncCheckpointMapper extends BaseMapperPlus<SyncCheckpoint, SyncCheckpointVo> {
}

View File

@ -0,0 +1,15 @@
package org.dromara.sync.mapper;
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
import org.dromara.sync.domain.SyncConnection;
import org.dromara.sync.domain.vo.SyncConnectionVo;
/**
* 同步连接 Mapper 接口
*
* @author Codex
* @date 2026-09-02
*/
public interface SyncConnectionMapper extends BaseMapperPlus<SyncConnection, SyncConnectionVo> {
}

View File

@ -0,0 +1,11 @@
package org.dromara.sync.mapper;
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
import org.dromara.sync.domain.SyncJobItem;
import org.dromara.sync.domain.vo.SyncJobItemVo;
/**
* 同步任务明细 Mapper
*/
public interface SyncJobItemMapper extends BaseMapperPlus<SyncJobItem, SyncJobItemVo> {
}

View File

@ -0,0 +1,11 @@
package org.dromara.sync.mapper;
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
import org.dromara.sync.domain.SyncJob;
import org.dromara.sync.domain.vo.SyncJobVo;
/**
* 同步任务 Mapper
*/
public interface SyncJobMapper extends BaseMapperPlus<SyncJob, SyncJobVo> {
}

View File

@ -0,0 +1,11 @@
package org.dromara.sync.mapper;
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
import org.dromara.sync.domain.SyncObject;
import org.dromara.sync.domain.vo.SyncObjectVo;
/**
* 同步对象清单 Mapper
*/
public interface SyncObjectMapper extends BaseMapperPlus<SyncObject, SyncObjectVo> {
}

View File

@ -0,0 +1,15 @@
package org.dromara.sync.mapper;
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
import org.dromara.sync.domain.SyncPlan;
import org.dromara.sync.domain.vo.SyncPlanVo;
/**
* 同步计划 Mapper 接口
*
* @author Codex
* @date 2026-09-02
*/
public interface SyncPlanMapper extends BaseMapperPlus<SyncPlan, SyncPlanVo> {
}

View File

@ -0,0 +1,11 @@
package org.dromara.sync.mapper;
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
import org.dromara.sync.domain.SyncTransferPart;
import org.dromara.sync.domain.vo.SyncTransferPartVo;
/**
* 同步传输分片 Mapper
*/
public interface SyncTransferPartMapper extends BaseMapperPlus<SyncTransferPart, SyncTransferPartVo> {
}

View File

@ -0,0 +1,109 @@
package org.dromara.sync.service;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.sync.connector.model.ConnectorTestResult;
import org.dromara.sync.domain.bo.SyncConnectionBo;
import org.dromara.sync.domain.vo.DwsAuthSessionVo;
import org.dromara.sync.domain.vo.SyncConnectionVo;
import java.util.Collection;
import java.util.List;
/**
* 同步连接 Service 接口
*
* @author Codex
* @date 2026-09-02
*/
public interface ISyncConnectionService {
/**
* 根据主键查询同步连接
*
* @param connectionId 连接主键
* @return 同步连接详情
*/
SyncConnectionVo queryById(Long connectionId);
/**
* 分页查询同步连接列表
*
* @param bo 查询条件
* @param pageQuery 分页参数
* @return 同步连接分页列表
*/
PageResult<SyncConnectionVo> queryPageList(SyncConnectionBo bo, PageQuery pageQuery);
/**
* 查询符合条件的同步连接列表
*
* @param bo 查询条件
* @return 同步连接列表
*/
List<SyncConnectionVo> queryList(SyncConnectionBo bo);
/**
* 新增同步连接
*
* @param bo 同步连接业务对象
* @return 是否新增成功
*/
Boolean insertByBo(SyncConnectionBo bo);
/**
* 修改同步连接
*
* @param bo 同步连接业务对象
* @return 是否修改成功
*/
Boolean updateByBo(SyncConnectionBo bo);
/**
* 校验并批量删除同步连接
*
* @param ids 待删除的连接主键集合
* @param isValid 是否执行删除前校验
* @return 是否删除成功
*/
Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid);
/**
* 校验配置并调用对应连接器执行连通性测试
*
* @param connectionId 连接主键
* @return 连通性测试结果
*/
ConnectorTestResult testConnection(Long connectionId);
/**
* 启动钉钉 Web 设备授权
*
* @param connectionId 钉钉连接主键
* @param ownerId 发起授权的系统用户
* @param expectedCorpId 可选的组织 ID用于防止误授权到其他组织
* @return 授权会话快照
*/
DwsAuthSessionVo startDingTalkAuth(Long connectionId, Long ownerId, String expectedCorpId);
/**
* 查询钉钉 Web 设备授权状态
*
* @param connectionId 钉钉连接主键
* @param sessionId 授权会话 ID
* @param ownerId 发起授权的系统用户
* @return 授权会话快照
*/
DwsAuthSessionVo getDingTalkAuth(Long connectionId, String sessionId, Long ownerId);
/**
* 取消钉钉 Web 设备授权
*
* @param connectionId 钉钉连接主键
* @param sessionId 授权会话 ID
* @param ownerId 发起授权的系统用户
* @return 取消后的会话快照
*/
DwsAuthSessionVo cancelDingTalkAuth(Long connectionId, String sessionId, Long ownerId);
}

View File

@ -0,0 +1,30 @@
package org.dromara.sync.service;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.sync.domain.bo.SyncJobBo;
import org.dromara.sync.domain.bo.SyncJobItemBo;
import org.dromara.sync.domain.vo.SyncJobItemVo;
import org.dromara.sync.domain.vo.SyncJobVo;
import java.util.Collection;
/**
* 同步任务服务
*/
public interface ISyncJobService {
SyncJobVo queryById(Long jobId);
PageResult<SyncJobVo> queryPageList(SyncJobBo bo, PageQuery pageQuery);
PageResult<SyncJobItemVo> queryItemPageList(SyncJobItemBo bo, PageQuery pageQuery);
Long startPlan(Long planId, String triggerType, String runType);
Long retry(Long jobId);
Boolean cancel(Long jobId);
Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid);
}

View File

@ -0,0 +1,16 @@
package org.dromara.sync.service;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.sync.domain.bo.SyncObjectBo;
import org.dromara.sync.domain.vo.SyncObjectVo;
/**
* 同步对象清单服务
*/
public interface ISyncObjectService {
SyncObjectVo queryById(Long objectId);
PageResult<SyncObjectVo> queryPageList(SyncObjectBo bo, PageQuery pageQuery);
}

View File

@ -0,0 +1,77 @@
package org.dromara.sync.service;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.sync.domain.bo.SyncPlanBo;
import org.dromara.sync.domain.vo.SyncPlanVo;
import java.util.Collection;
import java.util.List;
/**
* 同步计划 Service 接口
*
* @author Codex
* @date 2026-09-02
*/
public interface ISyncPlanService {
/**
* 根据主键查询同步计划
*
* @param planId 计划主键
* @return 同步计划详情
*/
SyncPlanVo queryById(Long planId);
/**
* 分页查询同步计划列表
*
* @param bo 查询条件
* @param pageQuery 分页参数
* @return 同步计划分页列表
*/
PageResult<SyncPlanVo> queryPageList(SyncPlanBo bo, PageQuery pageQuery);
/**
* 查询符合条件的同步计划列表
*
* @param bo 查询条件
* @return 同步计划列表
*/
List<SyncPlanVo> queryList(SyncPlanBo bo);
/**
* 校验计划名称是否唯一
*
* @param bo 同步计划
* @return 名称未被占用返回 {@code true}
*/
boolean checkPlanNameUnique(SyncPlanBo bo);
/**
* 新增同步计划
*
* @param bo 同步计划
* @return 是否新增成功
*/
Boolean insertByBo(SyncPlanBo bo);
/**
* 修改同步计划
*
* @param bo 同步计划
* @return 是否修改成功
*/
Boolean updateByBo(SyncPlanBo bo);
/**
* 校验并批量删除同步计划
*
* @param ids 计划主键集合
* @param isValid 是否执行删除前业务校验
* @return 是否删除成功
*/
Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid);
}

View File

@ -0,0 +1,713 @@
package org.dromara.sync.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.RequiredArgsConstructor;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.core.exception.ServiceException;
import org.dromara.common.core.utils.MapstructUtils;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.json.utils.JsonUtils;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.common.mybatis.core.query.QueryBuilder;
import org.dromara.common.satoken.utils.LoginHelper;
import org.dromara.sync.connector.ConnectorRegistry;
import org.dromara.sync.connector.dingtalk.DwsAuthIdentity;
import org.dromara.sync.connector.dingtalk.DwsAuthSessionManager;
import org.dromara.sync.connector.model.ConnectorTestResult;
import org.dromara.sync.constant.SyncConstants;
import org.dromara.sync.domain.SyncConnection;
import org.dromara.sync.domain.SyncJob;
import org.dromara.sync.domain.SyncPlan;
import org.dromara.sync.domain.bo.SyncConnectionBo;
import org.dromara.sync.domain.vo.DwsAuthSessionVo;
import org.dromara.sync.domain.vo.SyncConnectionVo;
import org.dromara.sync.mapper.SyncConnectionMapper;
import org.dromara.sync.mapper.SyncJobMapper;
import org.dromara.sync.mapper.SyncPlanMapper;
import org.dromara.sync.service.ISyncConnectionService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* 同步连接 Service 业务层处理
*
* @author Codex
* @date 2026-09-02
*/
@RequiredArgsConstructor
@Service
public class SyncConnectionServiceImpl implements ISyncConnectionService {
private static final Pattern DOWNLOAD_PART_SIZE_PATTERN = Pattern.compile("(?i)^([1-9]\\d*)(KB|MB|GB)$");
private static final long ONE_MIB = 1024L * 1024L;
private static final long ONE_GIB = 1024L * 1024L * 1024L;
private final SyncConnectionMapper connectionMapper;
private final SyncPlanMapper planMapper;
private final SyncJobMapper jobMapper;
private final ConnectorRegistry connectorRegistry;
private final DwsAuthSessionManager dwsAuthSessionManager;
/**
* Serializes connection mutations with the asynchronous DWS profile
* callback. Without this small critical section an edit could pass the
* active-session check just before login starts and then overwrite the
* profile written by the callback (or vice versa).
*/
private final Object connectionMutationLock = new Object();
@Value("${mybatis-encryptor.enable:false}")
private boolean fieldEncryptionEnabled;
@Value("${mybatis-encryptor.password:}")
private String fieldEncryptionPassword;
/**
* 根据主键查询同步连接详情
*
* @param connectionId 连接主键
* @return 同步连接详情
*/
@Override
public SyncConnectionVo queryById(Long connectionId) {
ensureConnectionAccess(connectionId);
return connectionMapper.selectVoById(connectionId);
}
/**
* 分页查询同步连接列表
*
* @param bo 查询条件
* @param pageQuery 分页参数
* @return 同步连接分页列表
*/
@Override
public PageResult<SyncConnectionVo> queryPageList(SyncConnectionBo bo, PageQuery pageQuery) {
LambdaQueryWrapper<SyncConnection> lqw = buildQueryWrapper(bo);
Page<SyncConnectionVo> result = connectionMapper.selectVoPage(pageQuery.build(), lqw);
return PageResult.build(result.getRecords(), result.getTotal());
}
/**
* 查询符合条件的同步连接列表
*
* @param bo 查询条件
* @return 同步连接列表
*/
@Override
public List<SyncConnectionVo> queryList(SyncConnectionBo bo) {
return connectionMapper.selectVoList(buildQueryWrapper(bo));
}
/**
* 构造同步连接动态查询条件
*
* @param bo 查询条件
* @return 查询条件包装器
*/
private LambdaQueryWrapper<SyncConnection> buildQueryWrapper(SyncConnectionBo bo) {
LambdaQueryWrapper<SyncConnection> wrapper = QueryBuilder.lambda(SyncConnection.class)
.eqIfPresent(SyncConnection::getConnectionId, bo.getConnectionId())
.likeIfText(SyncConnection::getConnectionName, bo.getConnectionName())
.eqIfText(SyncConnection::getConnectionRole, bo.getConnectionRole())
.eqIfText(SyncConnection::getConnectionType, bo.getConnectionType())
.likeIfText(SyncConnection::getEndpoint, bo.getEndpoint())
.likeIfText(SyncConnection::getBucketName, bo.getBucketName())
.eqIfText(SyncConnection::getStatus, bo.getStatus())
.orderByAsc(SyncConnection::getConnectionId)
.build();
// Connections contain credentials and are creator-owned resources. Do
// not rely on the menu permission alone for row-level isolation.
if (!LoginHelper.isSuperAdmin()) {
wrapper.eq(SyncConnection::getCreateBy, requireCurrentUserId());
}
return wrapper;
}
/**
* 新增同步连接
*
* @param bo 同步连接业务对象
* @return 是否新增成功
*/
@Override
public Boolean insertByBo(SyncConnectionBo bo) {
requireCurrentUserId();
SyncConnection add = MapstructUtils.convert(bo, SyncConnection.class);
validEntityBeforeSave(add);
boolean flag;
try {
flag = connectionMapper.insert(add) > 0;
} catch (DuplicateKeyException e) {
throw new ServiceException("连接名称【{}】已存在", add.getConnectionName());
}
if (flag) {
bo.setConnectionId(add.getConnectionId());
}
return flag;
}
/**
* 修改同步连接未填写敏感配置时不更新数据库中的原凭证
*
* @param bo 同步连接业务对象
* @return 是否修改成功
*/
@Override
public Boolean updateByBo(SyncConnectionBo bo) {
synchronized (connectionMutationLock) {
SyncConnection current = connectionMapper.selectById(bo.getConnectionId());
if (current == null) {
throw new ServiceException("同步连接不存在");
}
ensureConnectionAccess(current);
if (hasActiveJob(current.getConnectionId())) {
throw new ServiceException("连接正在被同步任务使用,任务结束后才能修改");
}
ensureNoDingTalkAuth(current.getConnectionId());
SyncConnection update = MapstructUtils.convert(bo, SyncConnection.class);
boolean connectorChanged = !Objects.equals(current.getConnectionRole(), update.getConnectionRole())
|| !Objects.equals(current.getConnectionType(), update.getConnectionType());
if (connectorChanged && isReferenced(update.getConnectionId())) {
throw new ServiceException("连接已被同步计划引用,不能修改连接角色或类型");
}
if (connectorChanged && StringUtils.isBlank(update.getSecretJson())) {
throw new ServiceException("修改连接角色或类型时必须重新填写敏感凭证");
}
preserveDingTalkProfile(current, update);
validEntityBeforeSave(update);
if (StringUtils.isBlank(update.getSecretJson())) {
update.setSecretJson(null);
}
try {
return connectionMapper.updateById(update) > 0;
} catch (DuplicateKeyException e) {
throw new ServiceException("连接名称【{}】已存在", update.getConnectionName());
}
}
}
/**
* 执行保存前的唯一性枚举组合与 JSON 格式校验
*
* @param entity 待保存的同步连接
*/
private void validEntityBeforeSave(SyncConnection entity) {
if (StringUtils.isNotBlank(entity.getSecretJson())
&& (!fieldEncryptionEnabled || StringUtils.isBlank(fieldEncryptionPassword))) {
throw new ServiceException("保存敏感凭证前必须启用 mybatis-encryptor 并配置 MYBATIS_ENCRYPTOR_PASSWORD");
}
entity.setConnectionName(StringUtils.trim(entity.getConnectionName()));
entity.setConnectionRole(StringUtils.trim(entity.getConnectionRole()));
entity.setConnectionType(StringUtils.trim(entity.getConnectionType()));
entity.setStatus(StringUtils.trim(entity.getStatus()));
entity.setEndpoint(StringUtils.trim(entity.getEndpoint()));
entity.setRegion(StringUtils.trim(entity.getRegion()));
entity.setBucketName(StringUtils.trim(entity.getBucketName()));
entity.setBasePath(StringUtils.trim(entity.getBasePath()));
requireText(entity.getConnectionName(), "连接名称不能为空");
validateLength("连接名称", entity.getConnectionName(), 100);
validateLength("服务端点", entity.getEndpoint(), 512);
validateLength("区域", entity.getRegion(), 64);
validateLength("存储桶名称", entity.getBucketName(), 255);
validateLength("基础路径", entity.getBasePath(), 1024);
validateLength("备注", entity.getRemark(), 500);
validateObjectKeyPrefix(entity.getBasePath(), "基础路径");
validateRoleAndType(entity);
validateStatus(entity.getStatus());
validateJsonObjectIfPresent(entity.getConfigJson(), "扩展配置");
validateJsonObjectIfPresent(entity.getSecretJson(), "敏感配置");
validateRequiredConfig(entity);
if (!isConnectionNameUnique(entity)) {
throw new ServiceException("连接名称【{}】已存在", entity.getConnectionName());
}
}
/**
* 校验连接名称唯一性编辑时排除当前记录
*
* @param entity 同步连接
* @return 名称唯一返回 {@code true}
*/
private boolean isConnectionNameUnique(SyncConnection entity) {
if (StringUtils.isBlank(entity.getConnectionName())) {
return true;
}
LambdaQueryWrapper<SyncConnection> lqw = QueryBuilder.lambda(SyncConnection.class)
.eq(SyncConnection::getConnectionName, entity.getConnectionName())
.neIfPresent(SyncConnection::getConnectionId, entity.getConnectionId())
.build();
return !connectionMapper.exists(lqw);
}
/**
* 校验连接角色与连接类型组合
*
* @param connection 同步连接
*/
private void validateRoleAndType(SyncConnection connection) {
String role = connection.getConnectionRole();
String type = connection.getConnectionType();
if (SyncConstants.TYPE_DINGTALK.equals(type)) {
if (!SyncConstants.ROLE_SOURCE.equals(role)) {
throw new ServiceException("DINGTALK 连接只能配置为 SOURCE 角色");
}
return;
}
if (SyncConstants.TYPE_S3.equals(type) || SyncConstants.TYPE_ALIYUN_OSS.equals(type)) {
if (!SyncConstants.ROLE_TARGET.equals(role)) {
throw new ServiceException("{} 连接只能配置为 TARGET 角色", type);
}
return;
}
throw new ServiceException("不支持的连接类型:{}", type);
}
/**
* 校验连接状态取值
*
* @param status 连接状态
*/
private void validateStatus(String status) {
if (!SyncConstants.STATUS_NORMAL.equals(status) && !SyncConstants.STATUS_DISABLED.equals(status)) {
throw new ServiceException("连接状态只支持 0 或 1");
}
}
private void validateRequiredConfig(SyncConnection connection) {
validateRequiredConfig(connection, false);
}
/**
* 校验连接运行所需配置
*
* @param connection 连接
* @param requireProfile 是否要求已经完成 DWS profile 配置
*/
private void validateRequiredConfig(SyncConnection connection, boolean requireProfile) {
boolean credentialRequired = connection.getConnectionId() == null
|| StringUtils.isNotBlank(connection.getSecretJson());
Map<String, Object> secrets = credentialRequired
? parseSecretJson(connection.getSecretJson()) : Map.of();
if (SyncConstants.TYPE_DINGTALK.equals(connection.getConnectionType())) {
if (credentialRequired) {
requireSecret(secrets, "clientId", "钉钉连接 clientId 不能为空");
requireSecret(secrets, "clientSecret", "钉钉连接 clientSecret 不能为空");
}
Map<String, Object> config = parseOptionalJson(connection.getConfigJson());
String profile = stringValue(config, "profile");
if (requireProfile && StringUtils.isBlank(profile)) {
throw new ServiceException("钉钉连接尚未完成登录,请先在网页端完成设备授权");
}
if (StringUtils.isNotBlank(profile) && !profile.matches("[^\\s:]+:[^\\s:]+")) {
throw new ServiceException("钉钉 profile 必须使用 corpId:userId 稳定标识");
}
if (config.containsKey("configDir")) {
throw new ServiceException("钉钉 configDir 由服务端按连接隔离分配,不能在连接配置中指定");
}
Object spaceType = config.get("spaceType");
if (spaceType != null && !List.of("orgSpace", "mySpace").contains(String.valueOf(spaceType))) {
throw new ServiceException("钉钉 spaceType 只支持 orgSpace 或 mySpace");
}
validateDownloadOptions(config);
return;
}
requireText(connection.getEndpoint(), "目标连接 endpoint 不能为空");
requireText(connection.getBucketName(), "目标连接 bucketName 不能为空");
if (credentialRequired) {
requireSecret(secrets, "accessKey", "目标连接 accessKey 不能为空");
requireSecret(secrets, "secretKey", "目标连接 secretKey 不能为空");
}
}
/**
* 校验可选 JSON 字段格式
*
* @param value JSON 字符串
* @param fieldName 字段名称
*/
private void validateJsonObjectIfPresent(String value, String fieldName) {
if (StringUtils.isNotBlank(value) && !JsonUtils.isJsonObject(value)) {
throw new ServiceException("{}必须是 JSON 对象", fieldName);
}
}
private Map<String, Object> parseOptionalJson(String json) {
return StringUtils.isBlank(json) ? Map.of() : JsonUtils.parseMap(json);
}
private void validateLength(String fieldName, String value, int maxLength) {
if (value != null && value.length() > maxLength) {
throw new ServiceException("{}不能超过{}个字符", fieldName, maxLength);
}
}
private void validateObjectKeyPrefix(String value, String fieldName) {
if (StringUtils.isBlank(value)) {
return;
}
for (String segment : value.replace('\\', '/').split("/")) {
if (".".equals(segment) || "..".equals(segment)) {
throw new ServiceException("{}不能包含 . 或 .. 路径段", fieldName);
}
}
if (value.chars().anyMatch(Character::isISOControl)) {
throw new ServiceException("{}不能包含控制字符", fieldName);
}
}
/**
* 校验并批量删除同步连接
*
* @param ids 待删除的连接主键集合
* @param isValid 是否执行删除前校验
* @return 是否删除成功
*/
@Override
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
synchronized (connectionMutationLock) {
if (isValid && (ids == null || ids.isEmpty())) {
throw new ServiceException("待删除的连接主键不能为空");
}
if (ids != null) {
for (Long connectionId : ids) {
ensureConnectionAccess(connectionId);
if (isValid) {
ensureNoDingTalkAuth(connectionId);
}
}
}
if (isValid && planMapper.exists(com.baomidou.mybatisplus.core.toolkit.Wrappers.<SyncPlan>lambdaQuery()
.and(wrapper -> wrapper.in(SyncPlan::getSourceConnectionId, ids)
.or().in(SyncPlan::getTargetConnectionId, ids)))) {
throw new ServiceException("连接已被同步计划引用,不能删除");
}
return connectionMapper.deleteByIds(ids) > 0;
}
}
/**
* 校验连接配置并调用对应连接器执行连通性测试
*
* @param connectionId 连接主键
* @return 连通性测试结果
*/
@Override
public ConnectorTestResult testConnection(Long connectionId) {
SyncConnection connection = connectionMapper.selectById(connectionId);
if (connection == null) {
throw new ServiceException("同步连接不存在");
}
ensureConnectionAccess(connection);
requireText(connection.getConnectionName(), "连接名称不能为空");
validateRoleAndType(connection);
validateStatus(connection.getStatus());
validateJsonObjectIfPresent(connection.getConfigJson(), "扩展配置");
validateJsonObjectIfPresent(connection.getSecretJson(), "敏感配置");
validateRequiredConfig(connection, true);
return connectorRegistry.resolve(connection).test(connection);
}
/**
* 启动钉钉 Web 设备授权登录完成后由回调把 DWS 返回的稳定 profile
* 合并回连接的非敏感配置不会覆盖其他配置项或敏感凭证
*
* @param connectionId 连接主键
* @param ownerId 当前系统用户
* @param expectedCorpId 可选组织 ID
* @return 授权会话快照
*/
@Override
public DwsAuthSessionVo startDingTalkAuth(Long connectionId, Long ownerId, String expectedCorpId) {
synchronized (connectionMutationLock) {
SyncConnection connection = requireDingTalkConnection(connectionId, ownerId);
if (hasActiveJob(connectionId)) {
throw new ServiceException("连接正在被同步任务使用,任务结束后才能重新登录");
}
validateJsonObjectIfPresent(connection.getConfigJson(), "扩展配置");
validateJsonObjectIfPresent(connection.getSecretJson(), "敏感配置");
// 保存时允许 profile 为空启动 Web 登录前仍必须有应用凭证
validateRequiredConfig(connection, false);
return dwsAuthSessionManager.start(connection, ownerId, expectedCorpId,
identity -> saveDingTalkProfile(connectionId, ownerId, identity));
}
}
/**
* 查询钉钉 Web 设备授权状态
*
* @param connectionId 连接主键
* @param sessionId 会话 ID
* @param ownerId 当前系统用户
* @return 授权会话快照
*/
@Override
public DwsAuthSessionVo getDingTalkAuth(Long connectionId, String sessionId, Long ownerId) {
requireDingTalkConnection(connectionId, ownerId);
return dwsAuthSessionManager.get(connectionId, sessionId, ownerId, LoginHelper.isSuperAdmin(ownerId));
}
/**
* 取消钉钉 Web 设备授权
*
* @param connectionId 连接主键
* @param sessionId 会话 ID
* @param ownerId 当前系统用户
* @return 取消后的授权会话快照
*/
@Override
public DwsAuthSessionVo cancelDingTalkAuth(Long connectionId, String sessionId, Long ownerId) {
requireDingTalkConnection(connectionId, ownerId);
return dwsAuthSessionManager.cancel(connectionId, sessionId, ownerId, LoginHelper.isSuperAdmin(ownerId));
}
private void ensureNoDingTalkAuth(Long connectionId) {
if (connectionId != null && dwsAuthSessionManager.hasActive(connectionId)) {
throw new ServiceException("连接正在进行钉钉 Web 登录,登录结束后才能修改或删除");
}
}
private SyncConnection requireDingTalkConnection(Long connectionId) {
if (connectionId == null) {
throw new ServiceException("连接主键不能为空");
}
SyncConnection connection = connectionMapper.selectById(connectionId);
if (connection == null) {
throw new ServiceException("同步连接不存在");
}
if (!SyncConstants.TYPE_DINGTALK.equals(connection.getConnectionType())
|| !SyncConstants.ROLE_SOURCE.equals(connection.getConnectionRole())) {
throw new ServiceException("只有 SOURCE 角色的 DINGTALK 连接支持 Web 登录");
}
validateStatus(connection.getStatus());
return connection;
}
/**
* Ensures that the current user may access a connection row. The
* super-admin is the only account allowed to operate across owners.
*
* @param connectionId connection id
*/
private void ensureConnectionAccess(Long connectionId) {
if (connectionId == null) {
throw new ServiceException("连接主键不能为空");
}
SyncConnection connection = connectionMapper.selectById(connectionId);
if (connection == null) {
throw new ServiceException("同步连接不存在");
}
ensureConnectionAccess(connection);
}
/**
* Ensures that the current user may access a loaded connection row.
*
* @param connection connection row
*/
private void ensureConnectionAccess(SyncConnection connection) {
Long userId = requireCurrentUserId();
if (!LoginHelper.isSuperAdmin(userId) && !Objects.equals(connection.getCreateBy(), userId)) {
throw new ServiceException("无权访问该同步连接");
}
}
/**
* Gets the authenticated system user id for service-layer access checks.
*
* @return current user id
*/
private Long requireCurrentUserId() {
Long userId = LoginHelper.getUserId();
if (userId == null) {
throw new ServiceException("当前登录用户无效,请重新登录系统");
}
return userId;
}
private SyncConnection requireDingTalkConnection(Long connectionId, Long ownerId) {
SyncConnection connection = requireDingTalkConnection(connectionId);
if (ownerId == null) {
throw new ServiceException("当前登录用户无效,请重新登录系统");
}
if (!LoginHelper.isSuperAdmin(ownerId) && !Objects.equals(connection.getCreateBy(), ownerId)) {
throw new ServiceException("只有连接创建者或超级管理员可以发起/查看钉钉 Web 登录");
}
return connection;
}
private void saveDingTalkProfile(Long connectionId, Long ownerId, DwsAuthIdentity identity) {
synchronized (connectionMutationLock) {
SyncConnection current = requireDingTalkConnection(connectionId);
if (ownerId == null || (!LoginHelper.isSuperAdmin(ownerId)
&& !Objects.equals(current.getCreateBy(), ownerId))) {
throw new ServiceException("连接归属已变化,不能保存钉钉登录身份");
}
Map<String, Object> config = parseOptionalJson(current.getConfigJson());
if (config.isEmpty()) {
config = new java.util.LinkedHashMap<>();
} else {
config = new java.util.LinkedHashMap<>(config);
}
config.put("profile", identity.profile());
String configJson = JsonUtils.toJsonString(config);
com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper<SyncConnection> wrapper =
com.baomidou.mybatisplus.core.toolkit.Wrappers.lambdaUpdate(SyncConnection.class)
.eq(SyncConnection::getConnectionId, connectionId)
.eq(SyncConnection::getConnectionType, SyncConstants.TYPE_DINGTALK)
.eq(SyncConnection::getConnectionRole, SyncConstants.ROLE_SOURCE)
.eq(SyncConnection::getStatus, SyncConstants.STATUS_NORMAL)
.set(SyncConnection::getConfigJson, configJson)
.set(SyncConnection::getUpdateBy, ownerId);
if (!LoginHelper.isSuperAdmin(ownerId)) {
wrapper.eq(SyncConnection::getCreateBy, ownerId);
}
if (connectionMapper.update(null, wrapper) <= 0) {
throw new ServiceException("保存钉钉登录身份失败");
}
}
}
/**
* The DWS profile is an authentication result, not editable connection
* metadata. Keep it server-authoritative so a normal PUT cannot redirect
* a connection to another user's/org's profile (or clear a valid profile
* by submitting a form that omits the read-only field).
*
* @param current persisted connection
* @param update client update
*/
private void preserveDingTalkProfile(SyncConnection current, SyncConnection update) {
if (!SyncConstants.TYPE_DINGTALK.equals(current.getConnectionType())
|| !SyncConstants.ROLE_SOURCE.equals(current.getConnectionRole())
|| !SyncConstants.TYPE_DINGTALK.equals(update.getConnectionType())
|| !SyncConstants.ROLE_SOURCE.equals(update.getConnectionRole())) {
return;
}
Map<String, Object> currentConfig = parseOptionalJson(current.getConfigJson());
String currentProfile = stringValue(currentConfig, "profile");
Map<String, Object> requestedConfig = StringUtils.isBlank(update.getConfigJson())
? new java.util.LinkedHashMap<>(currentConfig)
: new java.util.LinkedHashMap<>(parseOptionalJson(update.getConfigJson()));
String requestedProfile = stringValue(requestedConfig, "profile");
if (StringUtils.isNotBlank(requestedProfile)
&& !Objects.equals(currentProfile, requestedProfile)) {
throw new ServiceException("钉钉 profile 只能通过 Web 登录授权变更");
}
if (StringUtils.isNotBlank(currentProfile)) {
requestedConfig.put("profile", currentProfile);
} else {
requestedConfig.remove("profile");
}
update.setConfigJson(requestedConfig.isEmpty() ? null : JsonUtils.toJsonString(requestedConfig));
}
/**
* 解析敏感配置 JSON
*
* @param secretJson 敏感配置 JSON
* @return 敏感配置键值
*/
private Map<String, Object> parseSecretJson(String secretJson) {
if (StringUtils.isBlank(secretJson)) {
throw new ServiceException("敏感配置不能为空");
}
return JsonUtils.parseMap(secretJson);
}
private boolean isReferenced(Long connectionId) {
return planMapper.exists(com.baomidou.mybatisplus.core.toolkit.Wrappers.<SyncPlan>lambdaQuery()
.and(wrapper -> wrapper.eq(SyncPlan::getSourceConnectionId, connectionId)
.or().eq(SyncPlan::getTargetConnectionId, connectionId)));
}
private boolean hasActiveJob(Long connectionId) {
List<Long> planIds = planMapper.selectList(com.baomidou.mybatisplus.core.toolkit.Wrappers.<SyncPlan>lambdaQuery()
.select(SyncPlan::getPlanId)
.and(wrapper -> wrapper.eq(SyncPlan::getSourceConnectionId, connectionId)
.or().eq(SyncPlan::getTargetConnectionId, connectionId)))
.stream().map(SyncPlan::getPlanId).toList();
return !planIds.isEmpty() && jobMapper.exists(
com.baomidou.mybatisplus.core.toolkit.Wrappers.<SyncJob>lambdaQuery()
.in(SyncJob::getPlanId, planIds)
.in(SyncJob::getStatus, SyncConstants.JOB_PENDING,
SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING));
}
private void validateDownloadOptions(Map<String, Object> config) {
String parallel = stringValue(config, "downloadParallel");
if (StringUtils.isNotBlank(parallel)) {
try {
int value = Integer.parseInt(parallel);
if (value < 1 || value > 8) {
throw new ServiceException("钉钉 downloadParallel 必须在1到8之间");
}
} catch (NumberFormatException e) {
throw new ServiceException("钉钉 downloadParallel 必须是整数");
}
}
String partSize = stringValue(config, "downloadPartSize");
if (StringUtils.isNotBlank(partSize)) {
Matcher matcher = DOWNLOAD_PART_SIZE_PATTERN.matcher(partSize);
if (!matcher.matches()) {
throw new ServiceException("钉钉 downloadPartSize 格式不正确,例如 32MB");
}
try {
long multiplier = switch (matcher.group(2).toUpperCase(Locale.ROOT)) {
case "KB" -> 1024L;
case "MB" -> ONE_MIB;
case "GB" -> ONE_GIB;
default -> throw new IllegalStateException("无法识别容量单位");
};
long bytes = Math.multiplyExact(Long.parseLong(matcher.group(1)), multiplier);
if (bytes < ONE_MIB || bytes > ONE_GIB) {
throw new ServiceException("钉钉 downloadPartSize 必须在1MB到1GB之间");
}
} catch (NumberFormatException | ArithmeticException e) {
throw new ServiceException("钉钉 downloadPartSize 超出有效范围");
}
}
}
private String stringValue(Map<String, Object> values, String key) {
Object value = values.get(key);
return value == null ? null : String.valueOf(value);
}
/**
* 校验敏感配置中的必填项
*
* @param secrets 敏感配置
* @param key 配置键
* @param message 校验失败提示
*/
private void requireSecret(Map<String, Object> secrets, String key, String message) {
Object value = secrets.get(key);
if (value == null || StringUtils.isBlank(String.valueOf(value))) {
throw new ServiceException(message);
}
}
/**
* 校验必填文本
*
* @param value 文本值
* @param message 校验失败提示
*/
private void requireText(String value, String message) {
if (StringUtils.isBlank(value)) {
throw new ServiceException(message);
}
}
}

View File

@ -0,0 +1,199 @@
package org.dromara.sync.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.RequiredArgsConstructor;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.core.exception.ServiceException;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.sync.constant.SyncConstants;
import org.dromara.sync.domain.SyncJob;
import org.dromara.sync.domain.SyncJobItem;
import org.dromara.sync.domain.SyncPlan;
import org.dromara.sync.domain.SyncTransferPart;
import org.dromara.sync.domain.bo.SyncJobBo;
import org.dromara.sync.domain.bo.SyncJobItemBo;
import org.dromara.sync.domain.vo.SyncJobItemVo;
import org.dromara.sync.domain.vo.SyncJobVo;
import org.dromara.sync.mapper.SyncJobItemMapper;
import org.dromara.sync.mapper.SyncJobMapper;
import org.dromara.sync.mapper.SyncPlanMapper;
import org.dromara.sync.mapper.SyncTransferPartMapper;
import org.dromara.sync.service.ISyncJobService;
import org.dromara.sync.worker.SyncExecutionWorker;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.util.Collection;
import java.util.List;
/**
* 同步任务服务实现
*/
@Service
@RequiredArgsConstructor
public class SyncJobServiceImpl implements ISyncJobService {
private final SyncJobMapper jobMapper;
private final SyncJobItemMapper itemMapper;
private final SyncTransferPartMapper transferPartMapper;
private final SyncPlanMapper planMapper;
private final SyncExecutionWorker executionWorker;
@Override
public SyncJobVo queryById(Long jobId) {
return jobMapper.selectVoById(jobId);
}
@Override
public PageResult<SyncJobVo> queryPageList(SyncJobBo bo, PageQuery pageQuery) {
LambdaQueryWrapper<SyncJob> lqw = Wrappers.lambdaQuery();
lqw.eq(bo.getPlanId() != null, SyncJob::getPlanId, bo.getPlanId());
lqw.eq(StringUtils.isNotBlank(bo.getTriggerType()), SyncJob::getTriggerType, bo.getTriggerType());
lqw.eq(StringUtils.isNotBlank(bo.getRunType()), SyncJob::getRunType, bo.getRunType());
lqw.eq(StringUtils.isNotBlank(bo.getStatus()), SyncJob::getStatus, bo.getStatus());
Object beginTime = bo.getParams().get("beginTime");
Object endTime = bo.getParams().get("endTime");
lqw.ge(beginTime != null, SyncJob::getCreateTime, beginTime);
lqw.le(endTime != null, SyncJob::getCreateTime, endTime);
lqw.orderByDesc(SyncJob::getJobId);
Page<SyncJobVo> page = jobMapper.selectVoPage(pageQuery.build(), lqw);
return PageResult.build(page.getRecords(), page.getTotal());
}
@Override
public PageResult<SyncJobItemVo> queryItemPageList(SyncJobItemBo bo, PageQuery pageQuery) {
LambdaQueryWrapper<SyncJobItem> lqw = Wrappers.lambdaQuery();
lqw.eq(bo.getJobId() != null, SyncJobItem::getJobId, bo.getJobId());
lqw.like(StringUtils.isNotBlank(bo.getSourcePath()), SyncJobItem::getSourcePath, bo.getSourcePath());
lqw.eq(StringUtils.isNotBlank(bo.getObjectType()), SyncJobItem::getObjectType, bo.getObjectType());
lqw.eq(StringUtils.isNotBlank(bo.getActionType()), SyncJobItem::getActionType, bo.getActionType());
lqw.eq(StringUtils.isNotBlank(bo.getStatus()), SyncJobItem::getStatus, bo.getStatus());
lqw.orderByAsc(SyncJobItem::getItemId);
Page<SyncJobItemVo> page = itemMapper.selectVoPage(pageQuery.build(), lqw);
return PageResult.build(page.getRecords(), page.getTotal());
}
@Override
public Long startPlan(Long planId, String triggerType, String runType) {
SyncPlan plan = planMapper.selectById(planId);
if (plan == null) {
throw new ServiceException("同步计划不存在");
}
if (!SyncConstants.STATUS_NORMAL.equals(plan.getStatus())) {
throw new ServiceException("同步计划已停用");
}
boolean running = jobMapper.exists(Wrappers.<SyncJob>lambdaQuery()
.eq(SyncJob::getPlanId, planId)
.in(SyncJob::getStatus, SyncConstants.JOB_PENDING,
SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING));
if (running) {
throw new ServiceException("该同步计划已有运行中任务");
}
String actualTriggerType = StringUtils.isBlank(triggerType) ? "MANUAL" : triggerType;
String actualRunType = StringUtils.isBlank(runType) ? plan.getSyncMode() : runType;
if (!List.of("MANUAL", "SCHEDULE", "RETRY").contains(actualTriggerType)) {
throw new ServiceException("不支持的任务触发类型:{}", actualTriggerType);
}
if (!List.of("FULL", "INCREMENTAL").contains(actualRunType)) {
throw new ServiceException("不支持的任务运行类型:{}", actualRunType);
}
SyncJob job = new SyncJob();
job.setPlanId(planId);
job.setTriggerType(actualTriggerType);
job.setRunType(actualRunType);
job.setStatus(SyncConstants.JOB_PENDING);
job.setTotalCount(0L);
job.setProcessedCount(0L);
job.setSuccessCount(0L);
job.setFailedCount(0L);
job.setSkippedCount(0L);
job.setDeletedCount(0L);
job.setTotalBytes(0L);
job.setTransferredBytes(0L);
try {
if (jobMapper.insert(job) <= 0) {
throw new ServiceException("创建同步任务失败");
}
} catch (DuplicateKeyException e) {
throw new ServiceException("该同步计划已有等待或运行中的任务");
}
try {
executionWorker.executeAsync(job.getJobId());
} catch (RuntimeException e) {
jobMapper.lambda()
.set(SyncJob::getStatus, SyncConstants.JOB_FAILED)
.set(SyncJob::getFinishTime, LocalDateTime.now())
.set(SyncJob::getErrorMessage, "同步任务未能提交到后台执行器")
.eq(SyncJob::getJobId, job.getJobId())
.eq(SyncJob::getStatus, SyncConstants.JOB_PENDING)
.update();
throw new ServiceException("同步任务未能提交到后台执行器", e);
}
return job.getJobId();
}
@Override
public Long retry(Long jobId) {
SyncJob oldJob = jobMapper.selectById(jobId);
if (oldJob == null) {
throw new ServiceException("原同步任务不存在");
}
if (List.of(SyncConstants.JOB_PENDING, SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING)
.contains(oldJob.getStatus())) {
throw new ServiceException("运行中的任务不能重试");
}
return startPlan(oldJob.getPlanId(), "RETRY", "INCREMENTAL");
}
@Override
public Boolean cancel(Long jobId) {
SyncJob job = jobMapper.selectById(jobId);
if (job == null) {
throw new ServiceException("同步任务不存在");
}
if (!List.of(SyncConstants.JOB_PENDING, SyncConstants.JOB_RUNNING).contains(job.getStatus())) {
throw new ServiceException("只有等待或运行中的任务可以取消");
}
String targetStatus = SyncConstants.JOB_PENDING.equals(job.getStatus())
? SyncConstants.JOB_CANCELED : SyncConstants.JOB_CANCELING;
boolean canceled = jobMapper.lambda()
.set(SyncJob::getStatus, targetStatus)
.set(SyncConstants.JOB_CANCELED.equals(targetStatus), SyncJob::getFinishTime, LocalDateTime.now())
.eq(SyncJob::getJobId, jobId)
.eq(SyncJob::getStatus, job.getStatus())
.update();
if (canceled && SyncConstants.JOB_CANCELING.equals(targetStatus)) {
executionWorker.cancel(jobId);
}
return canceled;
}
@Override
@Transactional(rollbackFor = Exception.class)
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
if (isValid) {
boolean active = jobMapper.exists(Wrappers.<SyncJob>lambdaQuery()
.in(SyncJob::getJobId, ids)
.in(SyncJob::getStatus, SyncConstants.JOB_PENDING,
SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING));
if (active) {
throw new ServiceException("等待或运行中的同步任务不能删除");
}
}
List<Long> itemIds = itemMapper.selectList(Wrappers.<SyncJobItem>lambdaQuery()
.select(SyncJobItem::getItemId)
.in(SyncJobItem::getJobId, ids))
.stream().map(SyncJobItem::getItemId).toList();
if (!itemIds.isEmpty()) {
transferPartMapper.delete(Wrappers.<SyncTransferPart>lambdaQuery()
.in(SyncTransferPart::getJobItemId, itemIds));
}
itemMapper.delete(Wrappers.<SyncJobItem>lambdaQuery().in(SyncJobItem::getJobId, ids));
return jobMapper.deleteByIds(ids) > 0;
}
}

View File

@ -0,0 +1,44 @@
package org.dromara.sync.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.RequiredArgsConstructor;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.sync.domain.SyncObject;
import org.dromara.sync.domain.bo.SyncObjectBo;
import org.dromara.sync.domain.vo.SyncObjectVo;
import org.dromara.sync.mapper.SyncObjectMapper;
import org.dromara.sync.service.ISyncObjectService;
import org.springframework.stereotype.Service;
/**
* 同步对象清单服务实现
*/
@Service
@RequiredArgsConstructor
public class SyncObjectServiceImpl implements ISyncObjectService {
private final SyncObjectMapper objectMapper;
@Override
public SyncObjectVo queryById(Long objectId) {
return objectMapper.selectVoById(objectId);
}
@Override
public PageResult<SyncObjectVo> queryPageList(SyncObjectBo bo, PageQuery pageQuery) {
LambdaQueryWrapper<SyncObject> lqw = Wrappers.lambdaQuery();
lqw.eq(bo.getPlanId() != null, SyncObject::getPlanId, bo.getPlanId());
lqw.eq(StringUtils.isNotBlank(bo.getSourceObjectId()), SyncObject::getSourceObjectId, bo.getSourceObjectId());
lqw.like(StringUtils.isNotBlank(bo.getSourcePath()), SyncObject::getSourcePath, bo.getSourcePath());
lqw.eq(StringUtils.isNotBlank(bo.getObjectType()), SyncObject::getObjectType, bo.getObjectType());
lqw.eq(StringUtils.isNotBlank(bo.getSyncStatus()), SyncObject::getSyncStatus, bo.getSyncStatus());
lqw.eq(StringUtils.isNotBlank(bo.getSourceDeleted()), SyncObject::getSourceDeleted, bo.getSourceDeleted());
lqw.orderByAsc(SyncObject::getSourcePath);
Page<SyncObjectVo> page = objectMapper.selectVoPage(pageQuery.build(), lqw);
return PageResult.build(page.getRecords(), page.getTotal());
}
}

View File

@ -0,0 +1,380 @@
package org.dromara.sync.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.RequiredArgsConstructor;
import org.dromara.common.core.domain.PageResult;
import org.dromara.common.core.exception.ServiceException;
import org.dromara.common.core.utils.MapstructUtils;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.mybatis.core.page.PageQuery;
import org.dromara.common.mybatis.core.query.QueryBuilder;
import org.dromara.sync.constant.SyncConstants;
import org.dromara.sync.domain.SyncConnection;
import org.dromara.sync.domain.SyncJob;
import org.dromara.sync.domain.SyncPlan;
import org.dromara.sync.domain.bo.SyncPlanBo;
import org.dromara.sync.domain.vo.SyncPlanVo;
import org.dromara.sync.mapper.SyncConnectionMapper;
import org.dromara.sync.mapper.SyncJobMapper;
import org.dromara.sync.mapper.SyncPlanMapper;
import org.dromara.sync.service.ISyncPlanService;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.scheduling.support.CronExpression;
import org.springframework.stereotype.Service;
import java.time.ZonedDateTime;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* 同步计划 Service 业务层处理
*
* @author Codex
* @date 2026-09-02
*/
@RequiredArgsConstructor
@Service
public class SyncPlanServiceImpl implements ISyncPlanService {
private static final String SCHEDULE_MANUAL = "MANUAL";
private static final String SCHEDULE_CRON = "CRON";
private static final Set<String> SYNC_MODES = Set.of("FULL", "INCREMENTAL");
private static final Set<String> SCHEDULE_TYPES = Set.of(SCHEDULE_MANUAL, SCHEDULE_CRON);
private static final Set<String> CONFLICT_STRATEGIES = Set.of("OVERWRITE", "SKIP", "KEEP_BOTH");
private static final Set<String> DELETE_STRATEGIES = Set.of("KEEP", "MARK", "DELETE");
private static final Set<String> VERIFY_MODES = Set.of("SIZE", "ETAG", "SHA256");
private static final Set<String> PLAN_STATUSES = Set.of(SyncConstants.STATUS_NORMAL, SyncConstants.STATUS_DISABLED);
private final SyncPlanMapper planMapper;
private final SyncConnectionMapper connectionMapper;
private final SyncJobMapper jobMapper;
/**
* 根据主键查询同步计划
*
* @param planId 计划主键
* @return 同步计划详情
*/
@Override
public SyncPlanVo queryById(Long planId) {
return planMapper.selectVoById(planId);
}
/**
* 分页查询同步计划列表
*
* @param bo 查询条件
* @param pageQuery 分页参数
* @return 同步计划分页列表
*/
@Override
public PageResult<SyncPlanVo> queryPageList(SyncPlanBo bo, PageQuery pageQuery) {
LambdaQueryWrapper<SyncPlan> lqw = buildQueryWrapper(bo);
Page<SyncPlanVo> result = planMapper.selectVoPage(pageQuery.build(), lqw);
return PageResult.build(result.getRecords(), result.getTotal());
}
/**
* 查询符合条件的同步计划列表
*
* @param bo 查询条件
* @return 同步计划列表
*/
@Override
public List<SyncPlanVo> queryList(SyncPlanBo bo) {
return planMapper.selectVoList(buildQueryWrapper(bo));
}
/**
* 构建同步计划动态查询条件
*
* @param bo 查询条件
* @return 查询条件包装器
*/
private LambdaQueryWrapper<SyncPlan> buildQueryWrapper(SyncPlanBo bo) {
Map<String, Object> params = bo.getParams();
return QueryBuilder.lambda(SyncPlan.class)
.eqIfPresent(SyncPlan::getPlanId, bo.getPlanId())
.likeIfText(SyncPlan::getPlanName, bo.getPlanName())
.eqIfPresent(SyncPlan::getSourceConnectionId, bo.getSourceConnectionId())
.eqIfPresent(SyncPlan::getTargetConnectionId, bo.getTargetConnectionId())
.likeIfText(SyncPlan::getSourceRoot, bo.getSourceRoot())
.likeIfText(SyncPlan::getTargetPrefix, bo.getTargetPrefix())
.eqIfText(SyncPlan::getSyncMode, bo.getSyncMode())
.eqIfText(SyncPlan::getScheduleType, bo.getScheduleType())
.eqIfText(SyncPlan::getConflictStrategy, bo.getConflictStrategy())
.eqIfText(SyncPlan::getDeleteStrategy, bo.getDeleteStrategy())
.eqIfPresent(SyncPlan::getDeleteGuardPercent, bo.getDeleteGuardPercent())
.eqIfText(SyncPlan::getVerifyMode, bo.getVerifyMode())
.eqIfPresent(SyncPlan::getMaxConcurrency, bo.getMaxConcurrency())
.eqIfPresent(SyncPlan::getBandwidthLimitKbps, bo.getBandwidthLimitKbps())
.eqIfText(SyncPlan::getStatus, bo.getStatus())
.betweenParams(SyncPlan::getLastRunTime, params, "beginLastRunTime", "endLastRunTime")
.betweenParams(SyncPlan::getNextRunTime, params, "beginNextRunTime", "endNextRunTime")
.orderByAsc(SyncPlan::getPlanId)
.build();
}
/**
* 校验计划名称是否唯一
*
* @param bo 同步计划
* @return 名称未被占用返回 {@code true}
*/
@Override
public boolean checkPlanNameUnique(SyncPlanBo bo) {
String planName = StringUtils.trim(bo.getPlanName());
if (StringUtils.isBlank(planName)) {
return true;
}
return !planMapper.lambda()
.eq(SyncPlan::getPlanName, planName)
.neIfPresent(SyncPlan::getPlanId, bo.getPlanId())
.exists();
}
/**
* 新增同步计划
*
* @param bo 同步计划
* @return 是否新增成功
*/
@Override
public Boolean insertByBo(SyncPlanBo bo) {
SyncPlan add = MapstructUtils.convert(bo, SyncPlan.class);
validEntityBeforeSave(add);
boolean flag;
try {
flag = planMapper.insert(add) > 0;
} catch (DuplicateKeyException e) {
throw new ServiceException("计划名称'{}'已存在", add.getPlanName());
}
if (flag) {
bo.setPlanId(add.getPlanId());
}
return flag;
}
/**
* 修改同步计划
*
* @param bo 同步计划
* @return 是否修改成功
*/
@Override
public Boolean updateByBo(SyncPlanBo bo) {
SyncPlan current = planMapper.selectById(bo.getPlanId());
if (current == null) {
throw new ServiceException("同步计划不存在");
}
boolean hasActiveJob = jobMapper.lambda()
.eq(SyncJob::getPlanId, bo.getPlanId())
.in(SyncJob::getStatus, SyncConstants.JOB_PENDING,
SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING)
.exists();
if (hasActiveJob) {
throw new ServiceException("计划存在等待或运行中的任务,任务结束后才能修改");
}
SyncPlan update = MapstructUtils.convert(bo, SyncPlan.class);
validEntityBeforeSave(update);
try {
return planMapper.updateById(update) > 0;
} catch (DuplicateKeyException e) {
throw new ServiceException("计划名称'{}'已存在", update.getPlanName());
}
}
/**
* 执行同步计划保存前的业务校验和数据归一化
*
* @param entity 同步计划实体
*/
private void validEntityBeforeSave(SyncPlan entity) {
entity.setPlanName(StringUtils.trim(entity.getPlanName()));
entity.setSourceRoot(StringUtils.trim(entity.getSourceRoot()));
entity.setTargetPrefix(StringUtils.trim(entity.getTargetPrefix()));
if (StringUtils.isBlank(entity.getPlanName())) {
throw new ServiceException("计划名称不能为空");
}
validateLength("计划名称", entity.getPlanName(), 100);
validateLength("源端同步根路径", entity.getSourceRoot(), 1024);
validateLength("目标端对象键前缀", entity.getTargetPrefix(), 1024);
validateLength("备注", entity.getRemark(), 500);
validateObjectKeyPrefix(entity.getTargetPrefix(), "目标端对象键前缀");
boolean nameExists = planMapper.lambda()
.eq(SyncPlan::getPlanName, entity.getPlanName())
.neIfPresent(SyncPlan::getPlanId, entity.getPlanId())
.exists();
if (nameExists) {
throw new ServiceException("计划名称'{}'已存在", entity.getPlanName());
}
validateConnection(entity.getSourceConnectionId(), SyncConstants.ROLE_SOURCE, "源端");
validateConnection(entity.getTargetConnectionId(), SyncConstants.ROLE_TARGET, "目标端");
entity.setSyncMode(normalizeOption(entity.getSyncMode()));
entity.setScheduleType(normalizeOption(entity.getScheduleType()));
entity.setConflictStrategy(normalizeOption(entity.getConflictStrategy()));
entity.setDeleteStrategy(normalizeOption(entity.getDeleteStrategy()));
entity.setVerifyMode(normalizeOption(entity.getVerifyMode()));
entity.setStatus(normalizeOption(entity.getStatus()));
validateOption("同步模式", entity.getSyncMode(), SYNC_MODES);
validateOption("调度类型", entity.getScheduleType(), SCHEDULE_TYPES);
validateOption("冲突处理策略", entity.getConflictStrategy(), CONFLICT_STRATEGIES);
validateOption("删除处理策略", entity.getDeleteStrategy(), DELETE_STRATEGIES);
validateOption("校验方式", entity.getVerifyMode(), VERIFY_MODES);
validateOption("状态", entity.getStatus(), PLAN_STATUSES);
if (entity.getDeleteGuardPercent() == null
|| entity.getDeleteGuardPercent() < 0
|| entity.getDeleteGuardPercent() > 100) {
throw new ServiceException("删除保护阈值必须在0到100之间");
}
if (entity.getMaxConcurrency() == null
|| entity.getMaxConcurrency() < 1
|| entity.getMaxConcurrency() > 100) {
throw new ServiceException("最大并发数必须在1到100之间");
}
if (entity.getBandwidthLimitKbps() == null || entity.getBandwidthLimitKbps() < 0) {
throw new ServiceException("带宽上限不能小于0");
}
if (entity.getBandwidthLimitKbps() > 0) {
throw new ServiceException("当前版本尚未启用带宽限速请将带宽上限设置为0");
}
if (SCHEDULE_CRON.equals(entity.getScheduleType())) {
String expression = StringUtils.trim(entity.getCronExpression());
if (StringUtils.isBlank(expression)) {
throw new ServiceException("CRON调度的表达式不能为空");
}
validateLength("CRON表达式", expression, 128);
try {
CronExpression.parse(expression);
} catch (IllegalArgumentException ex) {
throw new ServiceException("CRON表达式格式不正确");
}
entity.setCronExpression(expression);
ZonedDateTime next = CronExpression.parse(expression).next(ZonedDateTime.now());
entity.setNextRunTime(next == null ? null : next.toLocalDateTime());
} else {
entity.setCronExpression(null);
entity.setNextRunTime(null);
}
}
/**
* 校验连接存在启用且角色与计划端点一致
*
* @param connectionId 连接主键
* @param expectedRole 期望角色
* @param endpointName 端点名称
*/
private void validateConnection(Long connectionId, String expectedRole, String endpointName) {
if (connectionId == null) {
throw new ServiceException("{}连接不能为空", endpointName);
}
SyncConnection connection = connectionMapper.selectById(connectionId);
if (connection == null) {
throw new ServiceException("{}连接不存在或已删除", endpointName);
}
if (!SyncConstants.STATUS_NORMAL.equals(connection.getStatus())) {
throw new ServiceException("{}连接'{}'已停用", endpointName, connection.getConnectionName());
}
if (!expectedRole.equals(connection.getConnectionRole())) {
throw new ServiceException("{}连接'{}'的角色必须为{}", endpointName,
connection.getConnectionName(), expectedRole);
}
if (SyncConstants.ROLE_SOURCE.equals(expectedRole)
&& !SyncConstants.TYPE_DINGTALK.equals(connection.getConnectionType())) {
throw new ServiceException("首版源端连接只支持 DINGTALK");
}
if (SyncConstants.ROLE_TARGET.equals(expectedRole)
&& !Set.of(SyncConstants.TYPE_S3, SyncConstants.TYPE_ALIYUN_OSS)
.contains(connection.getConnectionType())) {
throw new ServiceException("首版目标端连接只支持 S3 或 ALIYUN_OSS");
}
}
/**
* 去除枚举型文本首尾空白
*
* @param value 原始值
* @return 归一化后的值
*/
private String normalizeOption(String value) {
return StringUtils.trim(value);
}
/**
* 校验枚举型配置值
*
* @param fieldName 字段名称
* @param value 字段值
* @param options 允许值
*/
private void validateOption(String fieldName, String value, Set<String> options) {
if (!options.contains(value)) {
throw new ServiceException("{}不正确,可选值为{}", fieldName, String.join(",", options));
}
}
/**
* 校验文本长度不超过数据库字段上限
*
* @param fieldName 字段名称
* @param value 字段值
* @param maxLength 最大长度
*/
private void validateLength(String fieldName, String value, int maxLength) {
if (value != null && value.length() > maxLength) {
throw new ServiceException("{}不能超过{}个字符", fieldName, maxLength);
}
}
private void validateObjectKeyPrefix(String value, String fieldName) {
if (StringUtils.isBlank(value)) {
return;
}
for (String segment : value.replace('\\', '/').split("/")) {
if (".".equals(segment) || "..".equals(segment)) {
throw new ServiceException("{}不能包含 . 或 .. 路径段", fieldName);
}
}
if (value.chars().anyMatch(Character::isISOControl)) {
throw new ServiceException("{}不能包含控制字符", fieldName);
}
}
/**
* 校验并批量删除同步计划
*
* @param ids 计划主键集合
* @param isValid 是否执行删除前业务校验
* @return 是否删除成功
*/
@Override
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
if (isValid && (ids == null || ids.isEmpty())) {
throw new ServiceException("待删除的计划主键不能为空");
}
if (isValid) {
boolean hasRunningJob = jobMapper.lambda()
.in(SyncJob::getPlanId, ids)
.in(SyncJob::getStatus, List.of(SyncConstants.JOB_PENDING,
SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING))
.exists();
if (hasRunningJob) {
throw new ServiceException("存在等待中或运行中的同步任务,不能删除计划");
}
}
return planMapper.deleteByIds(ids) > 0;
}
}

View File

@ -0,0 +1,966 @@
package org.dromara.sync.worker;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.dromara.common.core.exception.ServiceException;
import org.dromara.common.core.utils.StringUtils;
import org.dromara.common.json.utils.JsonUtils;
import org.dromara.sync.connector.ConnectorRegistry;
import org.dromara.sync.connector.SourceConnector;
import org.dromara.sync.connector.TargetConnector;
import org.dromara.sync.connector.model.*;
import org.dromara.sync.constant.SyncConstants;
import org.dromara.sync.domain.*;
import org.dromara.sync.mapper.*;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.support.CronExpression;
import org.springframework.stereotype.Component;
import java.io.InputStream;
import java.nio.file.Files;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* 同步任务后台执行器
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class SyncExecutionWorker {
private static final String CHECKPOINT_TYPE = "RUN_WATERMARK";
private static final String CHECKPOINT_KEY = "default";
private static final int MAX_PAGES_PER_FOLDER = 100_000;
private static final long MAX_OBJECTS_PER_RUN = 10_000_000L;
private final SyncJobMapper jobMapper;
private final SyncJobItemMapper itemMapper;
private final SyncPlanMapper planMapper;
private final SyncConnectionMapper connectionMapper;
private final SyncObjectMapper objectMapper;
private final SyncCheckpointMapper checkpointMapper;
private final ConnectorRegistry connectorRegistry;
private final Map<Long, Thread> activeExecutions = new ConcurrentHashMap<>();
/**
* 异步执行已创建的同步任务
*
* @param jobId 任务ID
*/
@Async
public void executeAsync(Long jobId) {
execute(jobId);
}
/**
* 同步执行任务供测试和调度适配器复用
*
* @param jobId 任务ID
*/
public void execute(Long jobId) {
if (!markRunning(jobId)) {
return;
}
Thread executionThread = Thread.currentThread();
activeExecutions.put(jobId, executionThread);
Stats stats = new Stats();
try {
SyncJob job = requireJob(jobId);
SyncPlan plan = requirePlan(job.getPlanId());
SyncConnection sourceConnection = requireConnection(plan.getSourceConnectionId(), SyncConstants.ROLE_SOURCE);
SyncConnection targetConnection = requireConnection(plan.getTargetConnectionId(), SyncConstants.ROLE_TARGET);
SourceConnector sourceConnector = connectorRegistry.source(sourceConnection.getConnectionType());
TargetConnector targetConnector = connectorRegistry.target(targetConnection.getConnectionType());
String checkpointBefore = loadCheckpoint(plan.getPlanId());
jobMapper.lambda().set(SyncJob::getCheckpointBefore, checkpointBefore)
.eq(SyncJob::getJobId, jobId).update();
scanAndTransfer(job, plan, sourceConnection, targetConnection, sourceConnector, targetConnector, stats);
checkCanceled(jobId);
if (hasFailures(stats)) {
log.warn("本次同步存在文件失败项跳过源端删除处理jobId={}", jobId);
} else {
processDeletedObjects(job, plan, targetConnection, targetConnector, stats);
}
checkCanceled(jobId);
String checkpointAfter = LocalDateTime.now() + ":" + jobId;
saveCheckpoint(plan.getPlanId(), jobId, checkpointAfter);
if (!finish(jobId, stats, checkpointAfter)) {
checkCanceled(jobId);
throw new ServiceException("同步任务状态已发生变化,不能提交完成状态");
}
planMapper.lambda()
.set(SyncPlan::getLastRunTime, LocalDateTime.now())
.set(SyncPlan::getNextRunTime, calculateNextRunSafely(plan))
.eq(SyncPlan::getPlanId, plan.getPlanId()).update();
} catch (JobCanceledException ignored) {
markCanceled(jobId);
log.info("同步任务已取消jobId={}", jobId);
} catch (Exception e) {
if (isCancellationRequested(jobId)) {
markCanceled(jobId);
log.info("同步任务已取消jobId={}", jobId);
} else {
log.error("同步任务执行失败jobId={}", jobId, e);
fail(jobId, stats, e.getMessage());
}
} finally {
activeExecutions.remove(jobId, executionThread);
}
}
/**
* 中断当前节点上正在执行的任务数据库状态仍是跨节点取消的最终依据
*
* @param jobId 任务ID
*/
public void cancel(Long jobId) {
Thread executionThread = activeExecutions.get(jobId);
if (executionThread != null) {
executionThread.interrupt();
}
}
private void scanAndTransfer(SyncJob job, SyncPlan plan, SyncConnection sourceConnection,
SyncConnection targetConnection, SourceConnector sourceConnector,
TargetConnector targetConnector, Stats stats) {
int configuredConcurrency = plan.getMaxConcurrency() == null ? 1 : plan.getMaxConcurrency();
int concurrency = Math.max(1, Math.min(configuredConcurrency, 100));
ExecutorService executor = Executors.newFixedThreadPool(concurrency,
Thread.ofVirtual().name("sync-transfer-" + job.getJobId() + "-", 0).factory());
CompletionService<Void> transfers = new ExecutorCompletionService<>(executor);
int outstanding = 0;
try {
Deque<FolderNode> folders = new ArrayDeque<>();
FolderNode root = new FolderNode(normalizeSourceRoot(plan.getSourceRoot()), "", null);
folders.add(root);
Set<FolderKey> discoveredFolders = new HashSet<>();
discoveredFolders.add(new FolderKey(root.scopeId(), root.objectId()));
Set<String> discoveredObjects = new HashSet<>();
long objectCount = 0;
while (!folders.isEmpty()) {
FolderNode folder = folders.removeFirst();
String cursor = null;
Set<String> visitedCursors = new HashSet<>();
int pageCount = 0;
do {
checkCanceled(job.getJobId());
if (++pageCount > MAX_PAGES_PER_FOLDER) {
throw new ServiceException("单个源目录分页超过安全上限,已停止同步");
}
String cursorKey = StringUtils.isBlank(cursor) ? "<FIRST_PAGE>" : cursor;
if (!visitedCursors.add(cursorKey)) {
throw new ServiceException("源端返回了重复分页游标,已停止同步");
}
ScanResult page = sourceConnector.scan(sourceConnection,
new ScanRequest(folder.objectId(), cursor, folder.scopeId()));
if (page == null || page.objects() == null) {
throw new ServiceException("源端返回了不完整的分页结果,已停止同步");
}
for (SourceObject sourceObject : page.objects()) {
checkCanceled(job.getJobId());
if (sourceObject == null || StringUtils.isBlank(sourceObject.objectId())) {
throw new ServiceException("源端返回了缺少对象ID的条目已停止同步");
}
if (!discoveredObjects.add(sourceObject.objectId())) {
throw new ServiceException("源端返回了重复对象ID{}", sourceObject.objectId());
}
if (++objectCount > MAX_OBJECTS_PER_RUN) {
throw new ServiceException("本次扫描对象数超过安全上限,已停止同步");
}
String sourcePath = joinPath(folder.path(), sourceObject.name());
if (sourcePath.length() > 2048) {
throw new ServiceException("源端路径超过数据库长度上限:{}", sourcePath.substring(0, 256));
}
SyncObject existing = findObject(plan.getPlanId(), sourceObject.objectId());
if (existing != null && !Objects.equals(existing.getObjectType(), sourceObject.objectType())) {
throw new ServiceException("源对象类型发生变化,需人工确认后再同步:{}", sourcePath);
}
boolean existed = existing != null;
boolean changed = isChanged(existing, sourcePath, sourceObject);
String previousTargetKey = existing == null ? null : existing.getTargetKey();
SyncObject current = saveSeenObject(existing, plan.getPlanId(), job.getJobId(),
sourcePath, sourceObject);
if (sourceObject.isFolder()) {
FolderNode child = new FolderNode(sourceObject.objectId(), sourcePath,
metadataValue(sourceObject, "spaceId"));
if (!discoveredFolders.add(new FolderKey(child.scopeId(), child.objectId()))) {
throw new ServiceException("源端目录结构存在重复或循环节点:{}", sourcePath);
}
folders.addLast(child);
continue;
}
while (outstanding >= concurrency * 2) {
awaitTransfer(transfers);
outstanding--;
}
transfers.submit(() -> {
checkCanceled(job.getJobId());
transferObject(job, plan, targetConnection, sourceConnector, targetConnector,
sourceConnection, sourceObject, existed, changed, previousTargetKey,
current, sourcePath, stats);
return null;
});
outstanding++;
}
cursor = page.hasMore() ? page.nextCursor() : null;
if (page.hasMore() && StringUtils.isBlank(cursor)) {
throw new ServiceException("源端返回 hasMore=true 但没有分页游标,已停止同步");
}
} while (StringUtils.isNotBlank(cursor));
}
while (outstanding > 0) {
awaitTransfer(transfers);
outstanding--;
}
} finally {
executor.shutdownNow();
boolean interrupted = Thread.interrupted();
long nextWarningAt = System.nanoTime() + TimeUnit.MINUTES.toNanos(1);
try {
while (!executor.isTerminated()) {
try {
if (executor.awaitTermination(10, TimeUnit.SECONDS)) {
break;
}
} catch (InterruptedException e) {
interrupted = true;
executor.shutdownNow();
}
if (System.nanoTime() >= nextWarningAt) {
log.warn("仍在等待同步传输线程安全退出jobId={}", job.getJobId());
nextWarningAt = System.nanoTime() + TimeUnit.MINUTES.toNanos(1);
}
}
} finally {
if (interrupted) {
Thread.currentThread().interrupt();
}
}
}
}
private void awaitTransfer(CompletionService<Void> transfers) {
try {
transfers.take().get();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new ServiceException("等待文件传输时线程被中断", e);
} catch (ExecutionException e) {
Throwable cause = e.getCause();
if (cause instanceof JobCanceledException canceledException) {
throw canceledException;
}
if (cause instanceof RuntimeException runtimeException) {
throw runtimeException;
}
throw new ServiceException("文件传输线程执行失败", cause);
}
}
private void transferObject(SyncJob job, SyncPlan plan, SyncConnection targetConnection,
SourceConnector sourceConnector, TargetConnector targetConnector,
SyncConnection sourceConnection, SourceObject sourceObject,
boolean existed, boolean changed, String previousTargetKey, SyncObject current,
String sourcePath, Stats stats) {
boolean full = "FULL".equalsIgnoreCase(job.getRunType());
boolean shouldTransfer = full || !existed || changed;
String action = !existed ? "CREATE" : changed || full ? "UPDATE" : "SKIP";
if (!shouldTransfer) {
action = "SKIP";
}
String targetKey = buildTargetKey(targetConnection, plan, sourcePath, sourceObject);
SyncJobItem item = createItem(job.getJobId(), sourceObject, sourcePath, targetKey, action);
synchronized (stats) {
stats.totalCount++;
}
String unsupportedMessage = unsupportedSourceMessage(sourceObject);
if (unsupportedMessage != null) {
completeUnsupported(item, current, unsupportedMessage);
synchronized (stats) {
stats.processedCount++;
stats.skippedCount++;
}
updateProgress(job.getJobId(), stats);
return;
}
if (!shouldTransfer) {
completeSkipped(item, current, false);
synchronized (stats) {
stats.processedCount++;
stats.skippedCount++;
}
updateProgress(job.getJobId(), stats);
return;
}
try {
if ("SKIP".equals(plan.getConflictStrategy()) || "KEEP_BOTH".equals(plan.getConflictStrategy())) {
checkCanceled(job.getJobId());
boolean targetExists = targetConnector.exists(targetConnection, targetKey);
checkCanceled(job.getJobId());
if ("SKIP".equals(plan.getConflictStrategy()) && targetExists) {
completeSkipped(item, current, true);
synchronized (stats) {
stats.skippedCount++;
}
return;
}
if ("KEEP_BOTH".equals(plan.getConflictStrategy()) && targetExists) {
targetKey = addVersionSuffix(targetKey, sourceVersionIdentity(sourceObject));
item.setTargetKey(targetKey);
itemMapper.updateById(item);
}
}
synchronized (stats) {
stats.totalBytes += Math.max(sourceObject.size(), 0L);
}
item.setStatus(SyncConstants.ITEM_RUNNING);
item.setStartTime(LocalDateTime.now());
itemMapper.updateById(item);
try (SourceContent content = sourceConnector.download(sourceConnection, sourceObject)) {
checkCanceled(job.getJobId());
if (content.size() != sourceObject.size()) {
synchronized (stats) {
stats.totalBytes += content.size() - Math.max(sourceObject.size(), 0L);
}
item.setSize(content.size());
}
String sha256 = "SHA256".equals(plan.getVerifyMode())
? sha256(content.path(), job.getJobId()) : sourceSha256(sourceObject.hash());
checkCanceled(job.getJobId());
Map<String, String> metadata = new HashMap<>();
metadata.put("sync-source-id", sourceObject.objectId());
if (StringUtils.isNotBlank(sha256)) {
metadata.put("sync-sha256", sha256);
}
TargetWriteResult result = targetConnector.upload(targetConnection,
new TargetWriteRequest(targetKey, content.path(), content.size(), content.contentType(), metadata));
checkCanceled(job.getJobId());
if (result.size() != content.size()) {
throw new ServiceException("上传后大小校验失败,源大小 {},目标大小 {}", content.size(), result.size());
}
if ("SHA256".equals(plan.getVerifyMode())
&& !Objects.equals(sha256, metadataValue(result.metadata(), "sync-sha256"))) {
throw new ServiceException("上传后 SHA-256 元数据回读校验失败");
}
if ("ETAG".equals(plan.getVerifyMode()) && StringUtils.isBlank(result.eTag())) {
throw new ServiceException("上传后未能读取目标 ETag");
}
item.setTransferredBytes(result.size());
if (StringUtils.isNotBlank(previousTargetKey)
&& !previousTargetKey.equals(result.objectKey())
&& "DELETE".equals(plan.getDeleteStrategy())
&& !"KEEP_BOTH".equals(plan.getConflictStrategy())) {
checkCanceled(job.getJobId());
targetConnector.delete(targetConnection, previousTargetKey);
}
item.setStatus(SyncConstants.ITEM_SUCCESS);
item.setSourceSha256(sha256);
item.setTargetEtag(result.eTag());
item.setTargetVersionId(result.versionId());
item.setFinishTime(LocalDateTime.now());
itemMapper.updateById(item);
current.setTargetKey(result.objectKey());
current.setTargetVersionId(result.versionId());
current.setTargetEtag(result.eTag());
current.setSourceSha256(sha256);
current.setSyncStatus("SYNCED");
current.setLastSyncJobId(job.getJobId());
current.setLastSyncTime(LocalDateTime.now());
current.setLastErrorMessage(null);
objectMapper.updateById(current);
synchronized (stats) {
stats.successCount++;
stats.transferredBytes += result.size();
}
}
} catch (JobCanceledException e) {
item.setStatus(SyncConstants.ITEM_FAILED);
item.setFinishTime(LocalDateTime.now());
item.setErrorCode("CANCELED");
item.setErrorMessage("任务已取消");
itemMapper.updateById(item);
throw e;
} catch (Exception e) {
if (isAITableLink(sourceObject)) {
String message = "钉钉多维表链接(.dlink导出失败" + abbreviate(e.getMessage());
item.setStatus(SyncConstants.ITEM_SKIPPED);
item.setFinishTime(LocalDateTime.now());
item.setErrorCode("UNSUPPORTED");
item.setErrorMessage(message);
itemMapper.updateById(item);
current.setSyncStatus("SKIPPED");
current.setLastErrorMessage(message);
objectMapper.updateById(current);
synchronized (stats) {
stats.skippedCount++;
}
} else {
item.setStatus(SyncConstants.ITEM_FAILED);
item.setFinishTime(LocalDateTime.now());
item.setErrorCode(e.getClass().getSimpleName());
item.setErrorMessage(abbreviate(e.getMessage()));
itemMapper.updateById(item);
current.setSyncStatus("FAILED");
current.setLastErrorMessage(abbreviate(e.getMessage()));
objectMapper.updateById(current);
synchronized (stats) {
stats.failedCount++;
}
}
} finally {
synchronized (stats) {
stats.processedCount++;
}
updateProgress(job.getJobId(), stats);
}
}
private void processDeletedObjects(SyncJob job, SyncPlan plan, SyncConnection targetConnection,
TargetConnector targetConnector, Stats stats) {
List<SyncObject> deleted = objectMapper.selectList(Wrappers.<SyncObject>lambdaQuery()
.eq(SyncObject::getPlanId, plan.getPlanId())
.eq(SyncObject::getSourceDeleted, "0")
.and(wrapper -> wrapper.isNull(SyncObject::getLastSeenJobId)
.or().ne(SyncObject::getLastSeenJobId, job.getJobId())));
validateDeleteGuard(plan, deleted);
for (SyncObject object : deleted) {
checkCanceled(job.getJobId());
if (SyncConstants.OBJECT_FOLDER.equals(object.getObjectType())) {
markSourceDeleted(object, job.getJobId());
synchronized (stats) {
stats.deletedCount++;
}
continue;
}
SyncJobItem item = createDeleteItem(job.getJobId(), object);
synchronized (stats) {
stats.totalCount++;
}
try {
checkCanceled(job.getJobId());
if ("DELETE".equals(plan.getDeleteStrategy()) && StringUtils.isNotBlank(object.getTargetKey())) {
targetConnector.delete(targetConnection, object.getTargetKey());
}
checkCanceled(job.getJobId());
item.setStatus(SyncConstants.ITEM_SUCCESS);
item.setFinishTime(LocalDateTime.now());
itemMapper.updateById(item);
markSourceDeleted(object, job.getJobId());
synchronized (stats) {
stats.successCount++;
stats.deletedCount++;
}
} catch (JobCanceledException e) {
item.setStatus(SyncConstants.ITEM_FAILED);
item.setFinishTime(LocalDateTime.now());
item.setErrorCode("CANCELED");
item.setErrorMessage("任务已取消");
itemMapper.updateById(item);
throw e;
} catch (Exception e) {
item.setStatus(SyncConstants.ITEM_FAILED);
item.setFinishTime(LocalDateTime.now());
item.setErrorCode(e.getClass().getSimpleName());
item.setErrorMessage(abbreviate(e.getMessage()));
itemMapper.updateById(item);
object.setSyncStatus("FAILED");
object.setLastErrorMessage(abbreviate(e.getMessage()));
objectMapper.updateById(object);
synchronized (stats) {
stats.failedCount++;
}
}
synchronized (stats) {
stats.processedCount++;
}
updateProgress(job.getJobId(), stats);
}
}
private void validateDeleteGuard(SyncPlan plan, List<SyncObject> deleted) {
if (!"DELETE".equals(plan.getDeleteStrategy())) {
return;
}
long destructiveCount = deleted.stream()
.filter(object -> !SyncConstants.OBJECT_FOLDER.equals(object.getObjectType()))
.filter(object -> StringUtils.isNotBlank(object.getTargetKey()))
.count();
if (destructiveCount == 0) {
return;
}
long activeFileCount = objectMapper.selectCount(Wrappers.<SyncObject>lambdaQuery()
.eq(SyncObject::getPlanId, plan.getPlanId())
.eq(SyncObject::getSourceDeleted, "0")
.ne(SyncObject::getObjectType, SyncConstants.OBJECT_FOLDER));
int allowedPercent = plan.getDeleteGuardPercent() == null ? 50 : plan.getDeleteGuardPercent();
if (activeFileCount > 0 && destructiveCount * 100L > activeFileCount * allowedPercent) {
throw new ServiceException(
"本次拟删除目标对象 {} 个,占现有文件 {} 个的比例超过保护阈值 {}%,已拒绝删除",
destructiveCount, activeFileCount, allowedPercent);
}
}
private void markSourceDeleted(SyncObject object, Long jobId) {
object.setSourceDeleted("1");
object.setSyncStatus("DELETED");
object.setLastSyncJobId(jobId);
object.setLastSyncTime(LocalDateTime.now());
object.setLastErrorMessage(null);
objectMapper.updateById(object);
}
private SyncObject saveSeenObject(SyncObject existing, Long planId, Long jobId,
String sourcePath, SourceObject sourceObject) {
SyncObject entity = existing == null ? new SyncObject() : existing;
entity.setPlanId(planId);
entity.setSourceObjectId(sourceObject.objectId());
entity.setParentObjectId(sourceObject.parentObjectId());
entity.setSourcePath(sourcePath);
entity.setObjectName(sourceObject.name());
entity.setObjectType(sourceObject.objectType());
entity.setSize(sourceObject.size());
entity.setModifiedTime(sourceObject.modifiedTime());
entity.setVersionToken(sourceObject.versionToken());
entity.setSourceEtag(sourceObject.hash());
entity.setContentType(sourceObject.contentType());
entity.setMetadataJson(JsonUtils.toJsonString(sourceObject.metadata()));
entity.setSourceDeleted("0");
entity.setLastSeenJobId(jobId);
if (sourceObject.isFolder()) {
entity.setSyncStatus("SYNCED");
} else if (existing == null) {
entity.setSyncStatus("PENDING");
}
if (existing == null) {
entity.setFirstSeenJobId(jobId);
objectMapper.insert(entity);
} else {
objectMapper.updateById(entity);
}
return entity;
}
private SyncJobItem createItem(Long jobId, SourceObject object, String sourcePath,
String targetKey, String action) {
SyncJobItem item = new SyncJobItem();
item.setJobId(jobId);
item.setSourceObjectId(object.objectId());
item.setParentObjectId(object.parentObjectId());
item.setSourcePath(sourcePath);
item.setTargetKey(targetKey);
item.setObjectType(object.objectType());
item.setActionType(action);
item.setStatus(SyncConstants.ITEM_PENDING);
item.setSize(object.size());
item.setTransferredBytes(0L);
item.setVersionToken(object.versionToken());
item.setSourceEtag(object.hash());
item.setRetryCount(0);
itemMapper.insert(item);
return item;
}
private SyncJobItem createDeleteItem(Long jobId, SyncObject object) {
SourceObject sourceObject = new SourceObject(object.getSourceObjectId(), object.getParentObjectId(),
object.getObjectName(), object.getObjectType(), null, null, object.getSize(),
object.getModifiedTime(), object.getVersionToken(), object.getSourceEtag(), Map.of());
SyncJobItem item = createItem(jobId, sourceObject, object.getSourcePath(), object.getTargetKey(), "DELETE");
item.setStartTime(LocalDateTime.now());
itemMapper.updateById(item);
return item;
}
private void completeSkipped(SyncJobItem item, SyncObject object, boolean conflictSkipped) {
item.setActionType("SKIP");
item.setStatus(SyncConstants.ITEM_SKIPPED);
item.setFinishTime(LocalDateTime.now());
itemMapper.updateById(item);
object.setSyncStatus(conflictSkipped ? "SKIPPED" : "SYNCED");
object.setLastErrorMessage(null);
objectMapper.updateById(object);
}
/**
* 标记源端当前版本暂不支持的对象避免把预期能力缺失计为任务失败
*/
private void completeUnsupported(SyncJobItem item, SyncObject object, String message) {
item.setActionType("SKIP");
item.setStatus(SyncConstants.ITEM_SKIPPED);
item.setFinishTime(LocalDateTime.now());
item.setErrorCode("UNSUPPORTED");
item.setErrorMessage(message);
itemMapper.updateById(item);
object.setSyncStatus("SKIPPED");
object.setLastErrorMessage(message);
objectMapper.updateById(object);
}
private String unsupportedSourceMessage(SourceObject sourceObject) {
String extension = sourceObject.extension();
if (extension == null) {
return null;
}
extension = extension.startsWith(".") ? extension.substring(1) : extension;
extension = extension.toLowerCase(Locale.ROOT);
if ("amind".equals(extension) || "adraw".equals(extension)) {
return "钉钉官方 DWS 暂不支持自动导出:" + extension;
}
return null;
}
private boolean isAITableLink(SourceObject sourceObject) {
String extension = sourceObject.extension();
if (extension == null) return false;
return "dlink".equalsIgnoreCase(extension.startsWith(".") ? extension.substring(1) : extension);
}
private boolean isChanged(SyncObject existing, String sourcePath, SourceObject sourceObject) {
if (existing == null) {
return true;
}
if (!"SYNCED".equals(existing.getSyncStatus())
|| "1".equals(existing.getSourceDeleted())
|| StringUtils.isBlank(existing.getTargetKey())
|| !Objects.equals(existing.getSourcePath(), sourcePath)) {
return true;
}
if (StringUtils.isNotBlank(sourceObject.versionToken()) || StringUtils.isNotBlank(existing.getVersionToken())) {
return !Objects.equals(existing.getVersionToken(), sourceObject.versionToken());
}
if (StringUtils.isNotBlank(sourceObject.hash()) || StringUtils.isNotBlank(existing.getSourceEtag())) {
return !Objects.equals(existing.getSourceEtag(), sourceObject.hash())
|| !Objects.equals(existing.getSize(), sourceObject.size());
}
if (sourceObject.modifiedTime() == null || existing.getModifiedTime() == null) {
return true;
}
return !Objects.equals(existing.getSize(), sourceObject.size())
|| !Objects.equals(existing.getModifiedTime(), sourceObject.modifiedTime());
}
private String buildTargetKey(SyncConnection targetConnection, SyncPlan plan, String sourcePath,
SourceObject sourceObject) {
String relativePath = sourcePath;
if ("dlink".equalsIgnoreCase(normalizeExtension(sourceObject.extension()))) {
relativePath = replaceExtension(relativePath, "xlsx");
} else if (SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(sourceObject.objectType())) {
if ("adoc".equalsIgnoreCase(sourceObject.extension())) {
relativePath = replaceExtension(relativePath, "docx");
} else if ("axls".equalsIgnoreCase(sourceObject.extension())) {
relativePath = replaceExtension(relativePath, "xlsx");
} else if ("amind".equalsIgnoreCase(sourceObject.extension())
|| "adraw".equalsIgnoreCase(sourceObject.extension())) {
relativePath = replaceExtension(relativePath, "pdf");
}
}
return joinObjectKey(targetConnection.getBasePath(), plan.getTargetPrefix(), relativePath);
}
private String joinObjectKey(String... values) {
List<String> segments = new ArrayList<>();
for (String value : values) {
if (StringUtils.isBlank(value)) {
continue;
}
for (String segment : value.replace('\\', '/').split("/")) {
if (StringUtils.isBlank(segment)) {
continue;
}
if (".".equals(segment) || "..".equals(segment)) {
throw new ServiceException("对象键路径不能包含 . 或 .. 段");
}
segments.add(segment);
}
}
if (segments.isEmpty()) {
throw new ServiceException("目标对象键不能为空");
}
String key = String.join("/", segments);
if (key.length() > 2048) {
throw new ServiceException("目标对象键超过数据库长度上限");
}
return key;
}
private String joinPath(String parent, String name) {
return StringUtils.isBlank(parent) ? name : parent + '/' + name;
}
private String normalizeSourceRoot(String sourceRoot) {
String value = StringUtils.trim(sourceRoot);
return StringUtils.isBlank(value) || "/".equals(value) ? null : value;
}
private LocalDateTime calculateNextRunSafely(SyncPlan plan) {
if (!"CRON".equals(plan.getScheduleType()) || StringUtils.isBlank(plan.getCronExpression())) {
return null;
}
try {
ZonedDateTime next = CronExpression.parse(plan.getCronExpression()).next(ZonedDateTime.now());
return next == null ? null : next.toLocalDateTime();
} catch (Exception e) {
log.warn("同步计划 Cron 表达式已失效planId={},原因={}", plan.getPlanId(), e.getMessage());
return null;
}
}
private String metadataValue(SourceObject sourceObject, String key) {
return sourceObject.metadata() == null ? null : sourceObject.metadata().get(key);
}
private String metadataValue(Map<String, String> metadata, String key) {
return metadata == null ? null : metadata.get(key);
}
private String sourceSha256(String hash) {
if (StringUtils.isBlank(hash) || !hash.matches("(?i)[0-9a-f]{64}")) {
return null;
}
return hash.toLowerCase(Locale.ROOT);
}
private String normalizeExtension(String extension) {
if (StringUtils.isBlank(extension)) return "";
String value = extension.startsWith(".") ? extension.substring(1) : extension;
return value.toLowerCase(Locale.ROOT);
}
private String replaceExtension(String path, String extension) {
int slash = path.lastIndexOf('/');
int dot = path.lastIndexOf('.');
return dot > slash ? path.substring(0, dot + 1) + extension : path + '.' + extension;
}
private String addVersionSuffix(String key, String versionToken) {
String suffix = StringUtils.isBlank(versionToken)
? String.valueOf(System.currentTimeMillis()) : versionToken.replaceAll("[^a-zA-Z0-9_-]", "_");
if (suffix.length() > 48) {
suffix = suffix.substring(0, 48);
}
int slash = key.lastIndexOf('/');
int dot = key.lastIndexOf('.');
String versionedKey = dot > slash
? key.substring(0, dot) + "__" + suffix + key.substring(dot) : key + "__" + suffix;
if (versionedKey.length() > 2048) {
throw new ServiceException("保留多版本后的目标对象键超过数据库长度上限");
}
return versionedKey;
}
private String sourceVersionIdentity(SourceObject sourceObject) {
if (StringUtils.isNotBlank(sourceObject.versionToken())) {
return sourceObject.versionToken();
}
if (StringUtils.isNotBlank(sourceObject.hash())) {
return sourceObject.hash();
}
if (sourceObject.modifiedTime() != null) {
return sourceObject.modifiedTime() + "_" + sourceObject.size();
}
return "size_" + sourceObject.size();
}
private String sha256(java.nio.file.Path path, Long jobId) throws Exception {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
try (InputStream input = Files.newInputStream(path)) {
byte[] buffer = new byte[1024 * 1024];
int length;
while ((length = input.read(buffer)) >= 0) {
if (Thread.currentThread().isInterrupted()) {
checkCanceled(jobId);
throw new ServiceException("计算文件摘要时线程被中断");
}
if (length > 0) {
digest.update(buffer, 0, length);
}
}
}
checkCanceled(jobId);
return HexFormat.of().formatHex(digest.digest());
}
private SyncObject findObject(Long planId, String sourceObjectId) {
return objectMapper.selectOne(Wrappers.<SyncObject>lambdaQuery()
.eq(SyncObject::getPlanId, planId)
.eq(SyncObject::getSourceObjectId, sourceObjectId));
}
private String loadCheckpoint(Long planId) {
SyncCheckpoint checkpoint = checkpointMapper.selectOne(Wrappers.<SyncCheckpoint>lambdaQuery()
.eq(SyncCheckpoint::getPlanId, planId)
.eq(SyncCheckpoint::getCheckpointType, CHECKPOINT_TYPE)
.eq(SyncCheckpoint::getCheckpointKey, CHECKPOINT_KEY));
return checkpoint == null ? null : checkpoint.getCheckpointValue();
}
private void saveCheckpoint(Long planId, Long jobId, String value) {
SyncCheckpoint checkpoint = checkpointMapper.selectOne(Wrappers.<SyncCheckpoint>lambdaQuery()
.eq(SyncCheckpoint::getPlanId, planId)
.eq(SyncCheckpoint::getCheckpointType, CHECKPOINT_TYPE)
.eq(SyncCheckpoint::getCheckpointKey, CHECKPOINT_KEY));
if (checkpoint == null) {
checkpoint = new SyncCheckpoint();
checkpoint.setPlanId(planId);
checkpoint.setCheckpointType(CHECKPOINT_TYPE);
checkpoint.setCheckpointKey(CHECKPOINT_KEY);
checkpoint.setCheckpointValue(value);
checkpoint.setWatermarkTime(LocalDateTime.now());
checkpoint.setLastJobId(jobId);
checkpoint.setVersion(0L);
checkpointMapper.insert(checkpoint);
} else {
checkpoint.setCheckpointValue(value);
checkpoint.setWatermarkTime(LocalDateTime.now());
checkpoint.setLastJobId(jobId);
checkpointMapper.updateById(checkpoint);
}
}
private boolean markRunning(Long jobId) {
return jobMapper.lambda()
.set(SyncJob::getStatus, SyncConstants.JOB_RUNNING)
.set(SyncJob::getStartTime, LocalDateTime.now())
.eq(SyncJob::getJobId, jobId)
.eq(SyncJob::getStatus, SyncConstants.JOB_PENDING)
.update();
}
private void updateProgress(Long jobId, Stats stats) {
synchronized (stats) {
jobMapper.lambda()
.set(SyncJob::getTotalCount, stats.totalCount)
.set(SyncJob::getProcessedCount, stats.processedCount)
.set(SyncJob::getSuccessCount, stats.successCount)
.set(SyncJob::getFailedCount, stats.failedCount)
.set(SyncJob::getSkippedCount, stats.skippedCount)
.set(SyncJob::getDeletedCount, stats.deletedCount)
.set(SyncJob::getTotalBytes, stats.totalBytes)
.set(SyncJob::getTransferredBytes, stats.transferredBytes)
.eq(SyncJob::getJobId, jobId)
.update();
}
}
private boolean hasFailures(Stats stats) {
synchronized (stats) {
return stats.failedCount > 0;
}
}
private boolean finish(Long jobId, Stats stats, String checkpointAfter) {
String status;
synchronized (stats) {
status = stats.failedCount == 0 ? SyncConstants.JOB_SUCCESS
: stats.successCount > 0 || stats.skippedCount > 0
? SyncConstants.JOB_PARTIAL_FAILED : SyncConstants.JOB_FAILED;
}
updateProgress(jobId, stats);
return jobMapper.lambda()
.set(SyncJob::getStatus, status)
.set(SyncJob::getCheckpointAfter, checkpointAfter)
.set(SyncJob::getFinishTime, LocalDateTime.now())
.eq(SyncJob::getJobId, jobId)
.eq(SyncJob::getStatus, SyncConstants.JOB_RUNNING)
.update();
}
private void fail(Long jobId, Stats stats, String message) {
updateProgress(jobId, stats);
jobMapper.lambda()
.set(SyncJob::getStatus, SyncConstants.JOB_FAILED)
.set(SyncJob::getFinishTime, LocalDateTime.now())
.set(SyncJob::getErrorMessage, abbreviate(message))
.eq(SyncJob::getJobId, jobId)
.eq(SyncJob::getStatus, SyncConstants.JOB_RUNNING)
.update();
}
private boolean isCancellationRequested(Long jobId) {
SyncJob job = jobMapper.selectById(jobId);
return job != null && (SyncConstants.JOB_CANCELING.equals(job.getStatus())
|| SyncConstants.JOB_CANCELED.equals(job.getStatus()));
}
private void markCanceled(Long jobId) {
jobMapper.lambda()
.set(SyncJob::getStatus, SyncConstants.JOB_CANCELED)
.set(SyncJob::getFinishTime, LocalDateTime.now())
.eq(SyncJob::getJobId, jobId)
.eq(SyncJob::getStatus, SyncConstants.JOB_CANCELING)
.update();
}
private void checkCanceled(Long jobId) {
SyncJob job = requireJob(jobId);
if (SyncConstants.JOB_CANCELING.equals(job.getStatus())
|| SyncConstants.JOB_CANCELED.equals(job.getStatus())) {
throw new JobCanceledException();
}
}
private SyncJob requireJob(Long jobId) {
SyncJob job = jobMapper.selectById(jobId);
if (job == null) {
throw new ServiceException("同步任务不存在:{}", jobId);
}
return job;
}
private SyncPlan requirePlan(Long planId) {
SyncPlan plan = planMapper.selectById(planId);
if (plan == null) {
throw new ServiceException("同步计划不存在:{}", planId);
}
return plan;
}
private SyncConnection requireConnection(Long connectionId, String expectedRole) {
SyncConnection connection = connectionMapper.selectById(connectionId);
if (connection == null || !expectedRole.equals(connection.getConnectionRole())) {
throw new ServiceException("同步连接不存在或角色不正确:{}", connectionId);
}
if (!SyncConstants.STATUS_NORMAL.equals(connection.getStatus())) {
throw new ServiceException("同步连接已停用:{}", connection.getConnectionName());
}
return connection;
}
private String abbreviate(String message) {
if (message == null) {
return "未知错误";
}
return message.length() <= 2000 ? message : message.substring(0, 2000);
}
private record FolderNode(String objectId, String path, String scopeId) {
}
private record FolderKey(String scopeId, String objectId) {
}
private static final class Stats {
private long totalCount;
private long processedCount;
private long successCount;
private long failedCount;
private long skippedCount;
private long deletedCount;
private long totalBytes;
private long transferredBytes;
}
private static final class JobCanceledException extends RuntimeException {
private static final long serialVersionUID = 1L;
}
}

359
script/sql/ry_sync.sql Normal file
View File

@ -0,0 +1,359 @@
-- ----------------------------
-- 数据同步模块MySQL
-- 依赖 ry_vue.sql 中的 sys_menu、sys_dict_type、sys_dict_data 表
-- ----------------------------
-- ----------------------------
-- 1、连接配置表
-- ----------------------------
create table sync_connection
(
connection_id bigint(20) not null comment '连接ID',
connection_name varchar(100) not null comment '连接名称',
connection_role varchar(16) not null comment '连接角色SOURCE源端 TARGET目标端',
connection_type varchar(32) not null comment '连接类型DINGTALK钉钉 S3标准S3 ALIYUN_OSS阿里云OSS',
endpoint varchar(512) default null comment '服务端点',
region varchar(64) default null comment '区域',
bucket_name varchar(255) default null comment '存储桶名称',
base_path varchar(1024) default '' comment '连接默认根路径',
config_json text comment '非敏感扩展配置JSON',
secret_json text comment '加密后的敏感配置JSON禁止明文存储',
status char(1) default '0' comment '状态0正常 1停用',
del_flag char(1) default '0' comment '删除标志0代表存在 1代表删除',
active_name varchar(100) generated always as
(case when del_flag = '0' then connection_name else null end) stored
comment '有效连接名称(用于唯一约束)',
create_dept bigint(20) default null comment '创建部门',
create_by bigint(20) default null comment '创建者',
create_time datetime default null comment '创建时间',
update_by bigint(20) default null comment '更新者',
update_time datetime default null comment '更新时间',
remark varchar(500) default null comment '备注',
primary key (connection_id),
unique key uk_sync_connection_active_name (active_name),
key idx_sync_connection_name_del (connection_name, del_flag),
key idx_sync_connection_role_type (connection_role, connection_type),
key idx_sync_connection_status_del (status, del_flag)
) engine = innodb comment = '数据同步连接配置表';
-- ----------------------------
-- 2、同步计划表
-- ----------------------------
create table sync_plan
(
plan_id bigint(20) not null comment '同步计划ID',
plan_name varchar(100) not null comment '计划名称',
source_connection_id bigint(20) not null comment '源端连接ID',
target_connection_id bigint(20) not null comment '目标端连接ID',
source_root varchar(1024) default '/' comment '源端同步根路径或空间标识',
target_prefix varchar(1024) default '' comment '目标端对象键前缀',
sync_mode varchar(16) default 'INCREMENTAL' comment '同步模式FULL全量 INCREMENTAL增量',
schedule_type varchar(16) default 'MANUAL' comment '调度类型MANUAL手动 CRON定时',
cron_expression varchar(128) default null comment 'Cron表达式',
conflict_strategy varchar(16) default 'OVERWRITE' comment '冲突策略OVERWRITE覆盖 SKIP跳过 KEEP_BOTH两者保留',
delete_strategy varchar(16) default 'KEEP' comment '删除策略KEEP保留 MARK标记 DELETE删除目标',
delete_guard_percent int(11) default 50 comment '单次目标删除比例保护阈值0-100100表示允许全删',
verify_mode varchar(16) default 'SIZE' comment '校验方式SIZE大小 ETAG标识 SHA256摘要',
max_concurrency int(11) default 4 comment '最大并发传输数',
bandwidth_limit_kbps bigint(20) default 0 comment '带宽上限KB/s0表示不限速',
status char(1) default '0' comment '状态0正常 1停用',
last_run_time datetime default null comment '最近运行时间',
next_run_time datetime default null comment '下次运行时间',
del_flag char(1) default '0' comment '删除标志0代表存在 1代表删除',
active_name varchar(100) generated always as
(case when del_flag = '0' then plan_name else null end) stored
comment '有效计划名称(用于唯一约束)',
create_dept bigint(20) default null comment '创建部门',
create_by bigint(20) default null comment '创建者',
create_time datetime default null comment '创建时间',
update_by bigint(20) default null comment '更新者',
update_time datetime default null comment '更新时间',
remark varchar(500) default null comment '备注',
primary key (plan_id),
unique key uk_sync_plan_active_name (active_name),
key idx_sync_plan_name_del (plan_name, del_flag),
key idx_sync_plan_source_connection (source_connection_id),
key idx_sync_plan_target_connection (target_connection_id),
key idx_sync_plan_schedule (status, del_flag, schedule_type, next_run_time)
) engine = innodb comment = '数据同步计划表';
-- ----------------------------
-- 3、同步任务表
-- ----------------------------
create table sync_job
(
job_id bigint(20) not null comment '同步任务ID',
plan_id bigint(20) not null comment '同步计划ID',
trigger_type varchar(16) not null comment '触发类型MANUAL手动 SCHEDULE调度 RETRY重试',
run_type varchar(16) not null comment '运行类型FULL全量 INCREMENTAL增量',
status varchar(20) default 'PENDING' comment '任务状态PENDING待执行 RUNNING执行中 CANCELING取消中 SUCCESS成功 PARTIAL_FAILED部分失败 FAILED失败 CANCELED已取消',
active_plan_id bigint(20) generated always as
(case when status in ('PENDING', 'RUNNING', 'CANCELING') then plan_id else null end) stored
comment '活动任务计划ID用于保证单计划仅一个运行任务',
checkpoint_before text comment '运行前检查点快照',
checkpoint_after text comment '运行后检查点快照',
total_count bigint(20) default 0 comment '待处理对象总数',
processed_count bigint(20) default 0 comment '已处理对象数',
success_count bigint(20) default 0 comment '成功对象数',
failed_count bigint(20) default 0 comment '失败对象数',
skipped_count bigint(20) default 0 comment '跳过对象数',
deleted_count bigint(20) default 0 comment '删除或标记对象数',
total_bytes bigint(20) default 0 comment '待传输总字节数',
transferred_bytes bigint(20) default 0 comment '已传输字节数',
start_time datetime default null comment '开始时间',
finish_time datetime default null comment '结束时间',
error_message text comment '任务错误信息',
create_dept bigint(20) default null comment '创建部门',
create_by bigint(20) default null comment '创建者',
create_time datetime default null comment '创建时间',
update_by bigint(20) default null comment '更新者',
update_time datetime default null comment '更新时间',
primary key (job_id),
unique key uk_sync_job_active_plan (active_plan_id),
key idx_sync_job_plan_time (plan_id, create_time),
key idx_sync_job_status_time (status, create_time)
) engine = innodb comment = '数据同步任务表';
-- ----------------------------
-- 4、同步任务明细表
-- ----------------------------
create table sync_job_item
(
item_id bigint(20) not null comment '任务明细ID',
job_id bigint(20) not null comment '同步任务ID',
source_object_id varchar(255) not null comment '源端对象唯一标识',
parent_object_id varchar(255) default null comment '源端父对象标识',
source_path varchar(2048) not null comment '源端相对路径',
target_key varchar(2048) default null comment '目标端对象键',
object_type varchar(16) not null comment '对象类型FILE文件 FOLDER目录 ONLINE_DOCUMENT在线文档',
action_type varchar(16) not null comment '动作类型CREATE新增上传 UPDATE更新上传 SKIP跳过 DELETE删除',
status varchar(20) default 'PENDING' comment '明细状态PENDING待执行 RUNNING执行中 SUCCESS成功 SKIPPED已跳过 FAILED失败',
size bigint(20) default 0 comment '源对象字节数',
transferred_bytes bigint(20) default 0 comment '已传输字节数',
version_token varchar(512) default null comment '源端版本标识',
source_etag varchar(255) default null comment '源端ETag',
source_sha256 char(64) default null comment '源端SHA-256摘要',
target_etag varchar(255) default null comment '目标端ETag',
target_version_id varchar(512) default null comment '目标端版本ID',
retry_count int(11) default 0 comment '已重试次数',
start_time datetime default null comment '开始时间',
finish_time datetime default null comment '结束时间',
error_code varchar(64) default null comment '错误码',
error_message text comment '错误信息',
create_dept bigint(20) default null comment '创建部门',
create_by bigint(20) default null comment '创建者',
create_time datetime default null comment '创建时间',
update_by bigint(20) default null comment '更新者',
update_time datetime default null comment '更新时间',
primary key (item_id),
unique key uk_sync_job_item_job_object (job_id, source_object_id),
key idx_sync_job_item_job_status (job_id, status),
key idx_sync_job_item_retry (status, retry_count),
key idx_sync_job_item_parent (job_id, parent_object_id)
) engine = innodb comment = '数据同步任务明细表';
-- ----------------------------
-- 5、源对象状态表
-- ----------------------------
create table sync_object
(
object_id bigint(20) not null comment '对象记录ID',
plan_id bigint(20) not null comment '同步计划ID',
source_object_id varchar(255) not null comment '源端对象唯一标识',
parent_object_id varchar(255) default null comment '源端父对象标识',
object_name varchar(1024) not null comment '对象名称',
source_path varchar(2048) not null comment '相对同步根目录的源端路径',
object_type varchar(16) not null comment '对象类型FILE文件 FOLDER目录 ONLINE_DOCUMENT在线文档',
size bigint(20) default 0 comment '对象字节数',
modified_time datetime default null comment '源端最后修改时间',
version_token varchar(512) default null comment '源端版本标识',
source_etag varchar(255) default null comment '源端ETag',
source_sha256 char(64) default null comment '源端SHA-256摘要',
content_type varchar(255) default null comment '内容类型',
metadata_json text comment '源端扩展元数据JSON',
target_key varchar(2048) default null comment '最近成功同步的目标对象键',
target_version_id varchar(512) default null comment '最近成功同步的目标版本ID',
target_etag varchar(255) default null comment '最近成功同步的目标端ETag',
sync_status varchar(20) default 'DISCOVERED' comment '同步状态DISCOVERED已发现 PENDING待同步 SYNCED已同步 SKIPPED已跳过 FAILED失败 DELETED源端已删除',
source_deleted char(1) default '0' comment '源端删除标志0未删除 1已删除',
first_seen_job_id bigint(20) default null comment '首次发现任务ID',
last_seen_job_id bigint(20) default null comment '最近扫描发现任务ID',
last_sync_job_id bigint(20) default null comment '最近成功同步任务ID',
last_sync_time datetime default null comment '最近成功同步时间',
last_error_message text comment '最近同步错误信息',
create_dept bigint(20) default null comment '创建部门',
create_by bigint(20) default null comment '创建者',
create_time datetime default null comment '创建时间',
update_by bigint(20) default null comment '更新者',
update_time datetime default null comment '更新时间',
primary key (object_id),
unique key uk_sync_object_plan_source (plan_id, source_object_id),
key idx_sync_object_plan_status (plan_id, sync_status, source_deleted),
key idx_sync_object_last_seen (plan_id, last_seen_job_id),
key idx_sync_object_target_key (plan_id, target_key(191))
) engine = innodb comment = '数据同步源对象状态表';
-- ----------------------------
-- 6、增量检查点表
-- ----------------------------
create table sync_checkpoint
(
checkpoint_id bigint(20) not null comment '检查点ID',
plan_id bigint(20) not null comment '同步计划ID',
checkpoint_type varchar(32) not null comment '检查点类型PAGE_CURSOR分页游标 INCREMENTAL_WATERMARK增量水位 RUN_WATERMARK运行水位',
checkpoint_key varchar(255) not null comment '检查点作用域键(空间、目录或分区标识)',
checkpoint_value longtext comment '检查点值或JSON快照',
watermark_time datetime default null comment '增量水位时间',
last_job_id bigint(20) default null comment '最近提交检查点的任务ID',
version bigint(20) default 0 comment '乐观锁版本号',
create_dept bigint(20) default null comment '创建部门',
create_by bigint(20) default null comment '创建者',
create_time datetime default null comment '创建时间',
update_by bigint(20) default null comment '更新者',
update_time datetime default null comment '更新时间',
primary key (checkpoint_id),
unique key uk_sync_checkpoint_plan_type_key (plan_id, checkpoint_type, checkpoint_key),
key idx_sync_checkpoint_last_job (last_job_id)
) engine = innodb comment = '数据同步增量检查点表';
-- ----------------------------
-- 7、分片传输记录表
-- ----------------------------
create table sync_transfer_part
(
part_id bigint(20) not null comment '分片记录ID',
job_item_id bigint(20) not null comment '任务明细ID',
upload_id varchar(512) not null comment '目标端分片上传会话ID',
part_number int(11) not null comment '分片序号从1开始',
part_offset bigint(20) default 0 comment '分片起始字节偏移',
part_size bigint(20) default 0 comment '分片字节数',
transferred_bytes bigint(20) default 0 comment '分片已传输字节数',
part_etag varchar(255) default null comment '目标端分片ETag',
checksum_sha256 char(64) default null comment '分片SHA-256摘要',
status varchar(20) default 'PENDING' comment '分片状态PENDING待上传 UPLOADING上传中 SUCCESS成功 FAILED失败',
retry_count int(11) default 0 comment '已重试次数',
start_time datetime default null comment '开始时间',
finish_time datetime default null comment '结束时间',
error_message text comment '错误信息',
create_dept bigint(20) default null comment '创建部门',
create_by bigint(20) default null comment '创建者',
create_time datetime default null comment '创建时间',
update_by bigint(20) default null comment '更新者',
update_time datetime default null comment '更新时间',
primary key (part_id),
unique key uk_sync_transfer_part_item_number (job_item_id, part_number),
key idx_sync_transfer_part_upload_status (upload_id(191), status),
key idx_sync_transfer_part_status_retry (status, retry_count)
) engine = innodb comment = '数据同步分片传输记录表';
-- ----------------------------
-- 8、数据同步菜单
-- ----------------------------
insert into sys_menu
(menu_id, menu_name, parent_id, order_num, path, component, query_param, is_frame, is_cache, menu_type,
visible, status, perms, icon, active_menu, ext, create_dept, create_by, create_time, update_by, update_time, remark)
values
(1901400000000000001, '数据同步', 0, 2, 'sync', null, null, 'N', 'Y', 'M', '0', '0', null, 'sync', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步目录'),
(1901400000000000100, '连接管理', 1901400000000000001, 1, 'connection', 'sync/connection/index', null, 'N', 'Y', 'C', '0', '0', 'sync:connection:list', 'link', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步连接管理菜单'),
(1901400000000000101, '同步计划', 1901400000000000001, 2, 'plan', 'sync/plan/index', null, 'N', 'Y', 'C', '0', '0', 'sync:plan:list', 'calendar', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步计划菜单'),
(1901400000000000102, '同步任务', 1901400000000000001, 3, 'job', 'sync/job/index', null, 'N', 'Y', 'C', '0', '0', 'sync:job:list', 'job', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步任务菜单'),
(1901400000000000103, '对象清单', 1901400000000000001, 4, 'object', 'sync/object/index', null, 'N', 'Y', 'C', '0', '0', 'sync:object:list', 'list', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步对象清单菜单'),
(1901400000000001001, '连接查询', 1901400000000000100, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001002, '连接新增', 1901400000000000100, 2, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:add', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001003, '连接修改', 1901400000000000100, 3, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:edit', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001004, '连接删除', 1901400000000000100, 4, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:remove', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001006, '连接测试', 1901400000000000100, 5, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:test', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001007, '钉钉 Web 登录', 1901400000000000100, 6, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:auth', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001011, '计划查询', 1901400000000000101, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001012, '计划新增', 1901400000000000101, 2, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:add', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001013, '计划修改', 1901400000000000101, 3, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:edit', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001014, '计划删除', 1901400000000000101, 4, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:remove', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001016, '立即运行', 1901400000000000101, 5, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:run', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001021, '任务查询', 1901400000000000102, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001022, '任务删除', 1901400000000000102, 2, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:remove', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001024, '取消任务', 1901400000000000102, 3, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:cancel', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001025, '重试任务', 1901400000000000102, 4, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:retry', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
(1901400000000001031, '对象查询', 1901400000000000103, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:object:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '');
-- ----------------------------
-- 9、数据同步字典类型
-- ----------------------------
insert into sys_dict_type
(dict_id, dict_name, dict_type, create_dept, create_by, create_time, update_by, update_time, remark)
values
(1901500000000000001, '同步连接角色', 'sync_connection_role', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步连接角色'),
(1901500000000000002, '同步连接类型', 'sync_connection_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步连接类型'),
(1901500000000000003, '同步模式', 'sync_mode', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '全量或增量同步模式'),
(1901500000000000004, '同步调度类型', 'sync_schedule_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '手动或Cron调度'),
(1901500000000000005, '同步冲突策略', 'sync_conflict_strategy', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '目标对象冲突处理策略'),
(1901500000000000006, '同步删除策略', 'sync_delete_strategy', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端删除处理策略'),
(1901500000000000007, '同步校验方式', 'sync_verify_mode', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '传输完整性校验方式'),
(1901500000000000008, '同步触发类型', 'sync_trigger_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务触发类型'),
(1901500000000000009, '同步任务状态', 'sync_job_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务运行状态'),
(1901500000000000010, '同步明细状态', 'sync_job_item_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务明细状态'),
(1901500000000000011, '同步对象类型', 'sync_object_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步对象类型'),
(1901500000000000012, '同步动作类型', 'sync_action_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务明细动作类型'),
(1901500000000000013, '同步对象状态', 'sync_object_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源对象同步状态'),
(1901500000000000014, '同步分片状态', 'sync_transfer_part_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片上传状态'),
(1901500000000000015, '同步检查点类型', 'sync_checkpoint_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '增量检查点类型');
-- ----------------------------
-- 10、数据同步字典数据
-- ----------------------------
insert into sys_dict_data
(dict_code, dict_sort, dict_label, dict_value, dict_type, css_class, list_class, is_default,
create_dept, create_by, create_time, update_by, update_time, remark)
values
(1901600000000000001, 1, '源端', 'SOURCE', 'sync_connection_role', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据来源连接'),
(1901600000000000002, 2, '目标端', 'TARGET', 'sync_connection_role', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据目标连接'),
(1901600000000000003, 1, '钉钉', 'DINGTALK', 'sync_connection_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '钉钉企业网盘'),
(1901600000000000004, 2, '标准S3', 'S3', 'sync_connection_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, 'S3标准兼容存储'),
(1901600000000000005, 3, '阿里云OSS', 'ALIYUN_OSS', 'sync_connection_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '阿里云对象存储OSS'),
(1901600000000000006, 1, '全量同步', 'FULL', 'sync_mode', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '每次扫描全部对象'),
(1901600000000000007, 2, '增量同步', 'INCREMENTAL', 'sync_mode', '', 'success', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '完整枚举后按持久清单差异传输'),
(1901600000000000008, 1, '手动执行', 'MANUAL', 'sync_schedule_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '仅手动触发'),
(1901600000000000009, 2, 'Cron调度', 'CRON', 'sync_schedule_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按Cron表达式触发'),
(1901600000000000010, 1, '覆盖', 'OVERWRITE', 'sync_conflict_strategy', '', 'warning', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '覆盖目标对象'),
(1901600000000000011, 2, '跳过', 'SKIP', 'sync_conflict_strategy', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '保留目标对象并跳过'),
(1901600000000000012, 3, '两者保留', 'KEEP_BOTH', 'sync_conflict_strategy', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '重命名后保留两个对象'),
(1901600000000000013, 1, '保留目标', 'KEEP', 'sync_delete_strategy', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端删除时保留目标对象'),
(1901600000000000014, 2, '仅标记', 'MARK', 'sync_delete_strategy', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '仅记录源端删除状态'),
(1901600000000000015, 3, '删除目标', 'DELETE', 'sync_delete_strategy', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步删除目标对象'),
(1901600000000000016, 1, '文件大小', 'SIZE', 'sync_verify_mode', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按对象大小校验'),
(1901600000000000017, 2, 'ETag', 'ETAG', 'sync_verify_mode', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按ETag校验'),
(1901600000000000018, 3, 'SHA-256', 'SHA256', 'sync_verify_mode', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按SHA-256摘要校验'),
(1901600000000000019, 1, '手动触发', 'MANUAL', 'sync_trigger_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '用户手动触发任务'),
(1901600000000000020, 2, '调度触发', 'SCHEDULE', 'sync_trigger_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '计划调度触发任务'),
(1901600000000000021, 3, '重试触发', 'RETRY', 'sync_trigger_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '失败任务重试触发'),
(1901600000000000022, 1, '待执行', 'PENDING', 'sync_job_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '等待执行'),
(1901600000000000023, 2, '执行中', 'RUNNING', 'sync_job_status', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '正在执行'),
(1901600000000000024, 3, '成功', 'SUCCESS', 'sync_job_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '全部成功'),
(1901600000000000025, 4, '部分失败', 'PARTIAL_FAILED', 'sync_job_status', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '部分对象失败'),
(1901600000000000026, 5, '失败', 'FAILED', 'sync_job_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '任务失败'),
(1901600000000000027, 6, '已取消', 'CANCELED', 'sync_job_status', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '任务已取消'),
(1901600000000000053, 7, '取消中', 'CANCELING', 'sync_job_status', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '正在终止本节点传输'),
(1901600000000000028, 1, '待执行', 'PENDING', 'sync_job_item_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '等待执行'),
(1901600000000000029, 2, '执行中', 'RUNNING', 'sync_job_item_status', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '正在执行'),
(1901600000000000030, 3, '成功', 'SUCCESS', 'sync_job_item_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '对象同步成功'),
(1901600000000000031, 4, '已跳过', 'SKIPPED', 'sync_job_item_status', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '对象已跳过'),
(1901600000000000032, 5, '失败', 'FAILED', 'sync_job_item_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '对象同步失败'),
(1901600000000000033, 1, '文件', 'FILE', 'sync_object_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '普通文件'),
(1901600000000000034, 2, '目录', 'FOLDER', 'sync_object_type', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '目录对象'),
(1901600000000000035, 3, '在线文档', 'ONLINE_DOCUMENT', 'sync_object_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '需导出的在线文档'),
(1901600000000000036, 1, '新增上传', 'CREATE', 'sync_action_type', '', 'success', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '首次上传目标对象'),
(1901600000000000037, 2, '更新上传', 'UPDATE', 'sync_action_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '更新或覆盖目标对象'),
(1901600000000000038, 3, '跳过', 'SKIP', 'sync_action_type', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '跳过目标操作'),
(1901600000000000051, 4, '删除', 'DELETE', 'sync_action_type', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '删除目标对象'),
(1901600000000000039, 1, '已发现', 'DISCOVERED', 'sync_object_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '扫描已发现'),
(1901600000000000040, 2, '待同步', 'PENDING', 'sync_object_status', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '等待同步'),
(1901600000000000041, 3, '已同步', 'SYNCED', 'sync_object_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '最近同步成功'),
(1901600000000000042, 4, '已跳过', 'SKIPPED', 'sync_object_status', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按策略跳过'),
(1901600000000000043, 5, '失败', 'FAILED', 'sync_object_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '最近同步失败'),
(1901600000000000044, 6, '源端已删除', 'DELETED', 'sync_object_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端对象已删除'),
(1901600000000000045, 1, '待上传', 'PENDING', 'sync_transfer_part_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片等待上传'),
(1901600000000000046, 2, '上传中', 'UPLOADING', 'sync_transfer_part_status', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片正在上传'),
(1901600000000000047, 3, '成功', 'SUCCESS', 'sync_transfer_part_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片上传成功'),
(1901600000000000048, 4, '失败', 'FAILED', 'sync_transfer_part_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片上传失败'),
(1901600000000000049, 1, '分页游标', 'PAGE_CURSOR', 'sync_checkpoint_type', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端分页扫描游标'),
(1901600000000000050, 2, '增量水位', 'INCREMENTAL_WATERMARK', 'sync_checkpoint_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端增量同步水位'),
(1901600000000000052, 3, '运行水位', 'RUN_WATERMARK', 'sync_checkpoint_type', '', 'warning', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '当前版本任务运行水位');

View File

@ -82,7 +82,7 @@ create table sys_user (
user_id bigint(20) not null comment '用户ID',
dept_id bigint(20) default null comment '部门ID',
user_name varchar(30) not null comment '用户账号',
nick_name varchar(30) not null comment '用户昵称',
nick_name varchar(64) not null comment '用户昵称',
user_type varchar(10) default 'sys_user' comment '用户类型sys_user系统用户',
email varchar(50) default '' comment '用户邮箱',
phone_number varchar(11) default '' comment '手机号码',
@ -931,4 +931,3 @@ INSERT INTO test_tree VALUES (1762200000000000010, 1762200000000000007, 17610000
INSERT INTO test_tree VALUES (1762200000000000011, 1762200000000000007, 1761000000000000108, 1761100000000000003, '子节点77', 0, 1761000000000000103, sysdate(), 1761100000000000001, NULL, NULL, 0);
INSERT INTO test_tree VALUES (1762200000000000012, 1762200000000000010, 1761000000000000108, 1761100000000000003, '子节点88', 0, 1761000000000000103, sysdate(), 1761100000000000001, NULL, NULL, 0);
INSERT INTO test_tree VALUES (1762200000000000013, 1762200000000000010, 1761000000000000108, 1761100000000000003, '子节点99', 0, 1761000000000000103, sysdate(), 1761100000000000001, NULL, NULL, 0);

1
ui/data-sync-s3-vue Submodule

@ -0,0 +1 @@
Subproject commit a85fa0aee44f6f12dc35198126914ce722ee8622