mirror of
https://gitee.com/dromara/RuoYi-Vue-Plus.git
synced 2026-09-13 07:33:42 +08:00
Pre Merge pull request !886 from 没钱/feat/sync-s3-sse
This commit is contained in:
commit
4bf92c505d
9
.dockerignore
Normal file
9
.dockerignore
Normal file
@ -0,0 +1,9 @@
|
||||
.git
|
||||
.idea
|
||||
.vscode
|
||||
.pnpm-store
|
||||
**/.DS_Store
|
||||
**/node_modules
|
||||
**/dist
|
||||
**/target
|
||||
**/*.log
|
||||
1
.pnpm-store/v10/projects/8f418c8c0283cb0e672809522b013ebf
Symbolic link
1
.pnpm-store/v10/projects/8f418c8c0283cb0e672809522b013ebf
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ui/data-sync-s3-vue
|
||||
29
deploy/Dockerfile.backend
Normal file
29
deploy/Dockerfile.backend
Normal file
@ -0,0 +1,29 @@
|
||||
FROM docker.m.daocloud.io/library/maven:3.9.11-eclipse-temurin-21 AS builder
|
||||
|
||||
WORKDIR /build
|
||||
COPY . .
|
||||
RUN mvn -Pprod -DskipTests -pl ruoyi-admin -am clean package
|
||||
|
||||
FROM docker.m.daocloud.io/library/eclipse-temurin:21-jre-jammy
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl \
|
||||
&& curl -fsSL --connect-timeout 10 --max-time 30 https://ghfast.top/https://raw.githubusercontent.com/DingTalk-Real-AI/dingtalk-workspace-cli/main/scripts/install.sh -o /tmp/install-dws.sh \
|
||||
&& env DWS_GITEE_REPO=DingTalk-Real-AI/dingtalk-workspace-cli DWS_VERSION=v1.0.60 DWS_NO_SKILLS=1 DWS_INSTALL_DIR=/usr/local/bin sh /tmp/install-dws.sh \
|
||||
&& dws version \
|
||||
&& rm -rf /var/lib/apt/lists/* /tmp/install-dws.sh /root/.cache
|
||||
|
||||
WORKDIR /app
|
||||
RUN mkdir -p /app/logs /app/temp /var/lib/ruoyi-sync/dws
|
||||
COPY --from=builder /build/ruoyi-admin/target/ruoyi-admin.jar /app/app.jar
|
||||
|
||||
ENV TZ=Asia/Shanghai \
|
||||
JAVA_OPTS="-XX:+UseZGC -XX:+HeapDumpOnOutOfMemoryError" \
|
||||
DWS_EXECUTABLE=/usr/local/bin/dws \
|
||||
DWS_PROFILE_ROOT=/var/lib/ruoyi-sync/dws
|
||||
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=15s --timeout=5s --start-period=60s --retries=10 \
|
||||
CMD curl -fsS http://127.0.0.1:8080/ >/dev/null || exit 1
|
||||
|
||||
ENTRYPOINT ["sh", "-c", "exec java $JAVA_OPTS -jar /app/app.jar"]
|
||||
15
deploy/Dockerfile.web
Normal file
15
deploy/Dockerfile.web
Normal file
@ -0,0 +1,15 @@
|
||||
FROM docker.m.daocloud.io/library/node:22-bookworm-slim AS builder
|
||||
|
||||
WORKDIR /build
|
||||
RUN corepack enable && corepack prepare pnpm@10.34.5 --activate
|
||||
COPY ui/data-sync-s3-vue/package.json ui/data-sync-s3-vue/pnpm-lock.yaml ui/data-sync-s3-vue/pnpm-workspace.yaml ./
|
||||
RUN pnpm install --frozen-lockfile
|
||||
COPY ui/data-sync-s3-vue/ ./
|
||||
RUN pnpm build:prod
|
||||
|
||||
FROM docker.m.daocloud.io/library/nginx:1.31.1-alpine
|
||||
COPY deploy/nginx.conf /etc/nginx/nginx.conf
|
||||
COPY --from=builder /build/dist/ /usr/share/nginx/html/
|
||||
EXPOSE 80
|
||||
HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=5 \
|
||||
CMD wget -qO- http://127.0.0.1/ >/dev/null || exit 1
|
||||
44
deploy/docker-compose.backend.yml
Normal file
44
deploy/docker-compose.backend.yml
Normal file
@ -0,0 +1,44 @@
|
||||
name: data-sync-s3
|
||||
|
||||
services:
|
||||
backend:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: deploy/Dockerfile.backend
|
||||
image: data-sync-s3/backend:latest
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "38080:8080"
|
||||
extra_hosts:
|
||||
# Allows containers to reach services published on the Docker host.
|
||||
- "host.docker.internal:host-gateway"
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_URL: jdbc:mysql://${MYSQL_HOST:-host.docker.internal}:${MYSQL_PORT:-3306}/${MYSQL_DATABASE:-ry-vue}?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=Asia/Shanghai&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
|
||||
SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_USERNAME: ${MYSQL_USERNAME:-root}
|
||||
SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_PASSWORD: ${MYSQL_PASSWORD}
|
||||
SPRING_DATA_REDIS_HOST: ${REDIS_HOST:-host.docker.internal}
|
||||
SPRING_DATA_REDIS_PORT: ${REDIS_PORT:-6379}
|
||||
SPRING_DATA_REDIS_PASSWORD: ${REDIS_PASSWORD}
|
||||
MYBATIS_ENCRYPTOR_ENABLE: "true"
|
||||
MYBATIS_ENCRYPTOR_PASSWORD: ${MYBATIS_ENCRYPTOR_PASSWORD}
|
||||
DWS_EXECUTABLE: /usr/local/bin/dws
|
||||
DWS_PROFILE_ROOT: /var/lib/ruoyi-sync/dws
|
||||
DWS_KEYCHAIN_ROOT: /var/lib/ruoyi-sync/dws-keychain
|
||||
SYNC_SCHEDULER_ENABLED: "true"
|
||||
JAVA_OPTS: -Xms256m -Xmx1024m -XX:+UseZGC -XX:+HeapDumpOnOutOfMemoryError
|
||||
volumes:
|
||||
- backend-logs:/app/logs
|
||||
- dws-profiles:/var/lib/ruoyi-sync/dws
|
||||
- dws-keychains:/var/lib/ruoyi-sync/dws-keychain
|
||||
networks:
|
||||
- 1panel-network
|
||||
|
||||
networks:
|
||||
1panel-network:
|
||||
external: true
|
||||
|
||||
volumes:
|
||||
backend-logs:
|
||||
dws-profiles:
|
||||
dws-keychains:
|
||||
103
deploy/docker-compose.prod.yml
Normal file
103
deploy/docker-compose.prod.yml
Normal file
@ -0,0 +1,103 @@
|
||||
name: data-sync-s3
|
||||
|
||||
services:
|
||||
mysql:
|
||||
image: docker.m.daocloud.io/library/mysql:8.4.9
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}
|
||||
MYSQL_DATABASE: ${MYSQL_DATABASE:-ry-vue}
|
||||
command:
|
||||
- --character-set-server=utf8mb4
|
||||
- --collation-server=utf8mb4_general_ci
|
||||
- --explicit_defaults_for_timestamp=true
|
||||
- --lower_case_table_names=1
|
||||
volumes:
|
||||
- mysql-data:/var/lib/mysql
|
||||
- ../script/sql/ry_vue.sql:/docker-entrypoint-initdb.d/01-ry-vue.sql:ro
|
||||
- ../script/sql/ry_job.sql:/docker-entrypoint-initdb.d/02-ry-job.sql:ro
|
||||
- ../script/sql/ry_workflow.sql:/docker-entrypoint-initdb.d/03-ry-workflow.sql:ro
|
||||
- ../script/sql/ry_ai.sql:/docker-entrypoint-initdb.d/04-ry-ai.sql:ro
|
||||
- ../script/sql/ry_sync.sql:/docker-entrypoint-initdb.d/05-ry-sync.sql:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -p$$MYSQL_ROOT_PASSWORD --silent"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 15
|
||||
start_period: 30s
|
||||
networks: [app]
|
||||
|
||||
redis:
|
||||
image: docker.m.daocloud.io/library/redis:8.2.1-alpine
|
||||
restart: unless-stopped
|
||||
command: ["sh", "-c", "exec redis-server --appendonly yes --requirepass \"$$REDIS_PASSWORD\""]
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD}
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "redis-cli -a \"$$REDIS_PASSWORD\" ping | grep PONG"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
networks: [app]
|
||||
|
||||
backend:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: deploy/Dockerfile.backend
|
||||
image: data-sync-s3/backend:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_URL: jdbc:mysql://mysql:3306/${MYSQL_DATABASE:-ry-vue}?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=Asia/Shanghai&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
|
||||
SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_USERNAME: root
|
||||
SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_PASSWORD: ${MYSQL_ROOT_PASSWORD}
|
||||
SPRING_DATA_REDIS_HOST: redis
|
||||
SPRING_DATA_REDIS_PORT: 6379
|
||||
SPRING_DATA_REDIS_PASSWORD: ${REDIS_PASSWORD}
|
||||
MYBATIS_ENCRYPTOR_ENABLE: "true"
|
||||
MYBATIS_ENCRYPTOR_PASSWORD: ${MYBATIS_ENCRYPTOR_PASSWORD}
|
||||
DWS_EXECUTABLE: /usr/local/bin/dws
|
||||
DWS_PROFILE_ROOT: /var/lib/ruoyi-sync/dws
|
||||
DWS_KEYCHAIN_ROOT: /var/lib/ruoyi-sync/dws-keychain
|
||||
SYNC_SCHEDULER_ENABLED: "true"
|
||||
JAVA_OPTS: -Xms256m -Xmx1024m -XX:+UseZGC -XX:+HeapDumpOnOutOfMemoryError
|
||||
volumes:
|
||||
- backend-logs:/app/logs
|
||||
- dws-profiles:/var/lib/ruoyi-sync/dws
|
||||
- dws-keychains:/var/lib/ruoyi-sync/dws-keychain
|
||||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
networks: [app]
|
||||
|
||||
web:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: deploy/Dockerfile.web
|
||||
image: data-sync-s3/web:latest
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
# The host already runs another Nginx on port 80. Expose this stack on
|
||||
# 8080 and let the existing reverse proxy forward traffic here.
|
||||
- "38080:80"
|
||||
depends_on:
|
||||
backend:
|
||||
condition: service_healthy
|
||||
networks: [app]
|
||||
|
||||
networks:
|
||||
app:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
mysql-data:
|
||||
redis-data:
|
||||
backend-logs:
|
||||
dws-profiles:
|
||||
dws-keychains:
|
||||
55
deploy/nginx.conf
Normal file
55
deploy/nginx.conf
Normal file
@ -0,0 +1,55 @@
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 2048;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
client_max_body_size 100m;
|
||||
gzip_static on;
|
||||
server_tokens off;
|
||||
|
||||
map $http_upgrade $connection_upgrade {
|
||||
default upgrade;
|
||||
'' close;
|
||||
}
|
||||
|
||||
upstream backend {
|
||||
server backend:8080;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
location / {
|
||||
root /usr/share/nginx/html;
|
||||
try_files $uri $uri/ /index.html;
|
||||
index index.html;
|
||||
}
|
||||
|
||||
location /prod-api/ {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
proxy_read_timeout 86400s;
|
||||
proxy_buffering off;
|
||||
proxy_cache off;
|
||||
proxy_pass http://backend/;
|
||||
}
|
||||
|
||||
location ~ ^(/[^/]*)?/actuator.*(/.*)?$ {
|
||||
return 403;
|
||||
}
|
||||
}
|
||||
}
|
||||
7
pom.xml
7
pom.xml
@ -429,6 +429,13 @@
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- 数据同步模块 -->
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-sync</artifactId>
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- 代码生成模块 -->
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
|
||||
BIN
ruoyi-admin/.DS_Store
vendored
Normal file
BIN
ruoyi-admin/.DS_Store
vendored
Normal file
Binary file not shown.
@ -92,6 +92,12 @@
|
||||
<artifactId>ruoyi-ai</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- 数据同步模块 -->
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-sync</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- demo模块 -->
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
|
||||
@ -12,9 +12,6 @@ import me.zhyd.oauth.utils.AuthStateUtils;
|
||||
import org.dromara.common.core.constant.SystemConstants;
|
||||
import org.dromara.common.core.domain.R;
|
||||
import org.dromara.common.core.domain.model.LoginBody;
|
||||
import org.dromara.common.core.enums.PushSourceEnum;
|
||||
import org.dromara.common.core.enums.PushTypeEnum;
|
||||
import org.dromara.common.core.utils.DateUtils;
|
||||
import org.dromara.common.core.utils.MessageUtils;
|
||||
import org.dromara.common.core.utils.StringUtils;
|
||||
import org.dromara.common.core.utils.ValidatorUtils;
|
||||
@ -24,8 +21,6 @@ import org.dromara.common.satoken.utils.LoginHelper;
|
||||
import org.dromara.common.social.config.properties.SocialLoginConfigProperties;
|
||||
import org.dromara.common.social.config.properties.SocialProperties;
|
||||
import org.dromara.common.social.utils.SocialUtils;
|
||||
import org.dromara.system.api.MessageService;
|
||||
import org.dromara.system.api.domain.PushPayloadDTO;
|
||||
import org.dromara.system.api.model.RegisterBody;
|
||||
import org.dromara.system.api.model.SocialLoginBody;
|
||||
import org.dromara.system.domain.vo.SysClientVo;
|
||||
@ -39,10 +34,6 @@ import org.dromara.web.service.SysRegisterService;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 认证控制器,提供登录、注册、社交绑定和退出能力。
|
||||
@ -62,8 +53,6 @@ public class AuthController {
|
||||
private final ISysConfigService configService;
|
||||
private final ISysSocialService socialUserService;
|
||||
private final ISysClientService clientService;
|
||||
private final ScheduledExecutorService scheduledExecutorService;
|
||||
private final MessageService messageService;
|
||||
|
||||
|
||||
/**
|
||||
@ -91,18 +80,6 @@ public class AuthController {
|
||||
// 登录
|
||||
LoginVo loginVo = IAuthStrategy.login(body, client, grantType);
|
||||
|
||||
Long userId = LoginHelper.getUserId();
|
||||
scheduledExecutorService.schedule(() -> {
|
||||
messageService.publishMessage(
|
||||
List.of(userId),
|
||||
PushPayloadDTO.of(
|
||||
PushTypeEnum.MESSAGE,
|
||||
PushSourceEnum.BACKEND,
|
||||
DateUtils.getTodayHour(new Date()) + "好,欢迎登录 RuoYi-Vue-Plus 后台管理系统",
|
||||
null
|
||||
)
|
||||
);
|
||||
}, 5, TimeUnit.SECONDS);
|
||||
return R.ok(loginVo);
|
||||
}
|
||||
|
||||
|
||||
@ -103,9 +103,9 @@ spring:
|
||||
driverClassName: com.mysql.cj.jdbc.Driver
|
||||
# jdbc 所有参数配置参考 https://lionli.blog.csdn.net/article/details/122018562
|
||||
# rewriteBatchedStatements=true 批处理优化 大幅提升批量插入更新删除性能(对数据库有性能损耗 使用批量操作应考虑性能问题)
|
||||
url: jdbc:mysql://localhost:3306/ry-vue?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
|
||||
username: root
|
||||
password: root
|
||||
url: jdbc:mysql://kfn-data-sync-s3-mysql:33306/data-sync-s3?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
|
||||
username: data-sync-s3
|
||||
password: PSGrMaQdZ8z7AfiJ
|
||||
# # 从库数据源
|
||||
# slave:
|
||||
# lazy: true
|
||||
@ -152,13 +152,13 @@ spring:
|
||||
spring.data:
|
||||
redis:
|
||||
# 地址
|
||||
host: localhost
|
||||
host: kfn-data-sync-s3
|
||||
# 端口,默认为6379
|
||||
port: 6379
|
||||
port: 36379
|
||||
# 数据库索引
|
||||
database: 0
|
||||
# redis 密码必须配置
|
||||
password: ruoyi123
|
||||
password: redis_RpMMYK
|
||||
# 连接超时时间
|
||||
timeout: 10s
|
||||
# 是否开启ssl
|
||||
|
||||
@ -42,6 +42,32 @@ user:
|
||||
# 密码锁定时间(默认10分钟)
|
||||
lockTime: 10
|
||||
|
||||
# 数据同步配置
|
||||
sync:
|
||||
dingtalk:
|
||||
# DingTalk Workspace CLI 可执行文件;生产环境可通过环境变量指定绝对路径
|
||||
dws-executable: ${DWS_EXECUTABLE:dws}
|
||||
# 每个连接使用独立 DWS 登录态目录,实际目录为该路径下的连接 ID
|
||||
profile-root: ${DWS_PROFILE_ROOT:${user.home}/.ruoyi-sync/dws}
|
||||
# DWS 加密 keychain 也按连接隔离;生产环境必须和 profile 一起持久化
|
||||
keychain-root: ${DWS_KEYCHAIN_ROOT:${user.home}/.ruoyi-sync/dws-keychain}
|
||||
auth:
|
||||
# 是否允许通过管理后台发起钉钉设备授权
|
||||
enabled: ${DWS_WEB_AUTH_ENABLED:true}
|
||||
# 设备流最长等待时间(DWS 服务端默认约 10 分钟)
|
||||
timeout-seconds: ${DWS_WEB_AUTH_TIMEOUT_SECONDS:600}
|
||||
# 启动接口等待 DWS 打印授权链接的最长时间;超时后前端继续轮询 STARTING
|
||||
metadata-wait-seconds: ${DWS_WEB_AUTH_METADATA_WAIT_SECONDS:15}
|
||||
# 单节点同时进行的设备授权数量上限
|
||||
max-concurrent: ${DWS_WEB_AUTH_MAX_CONCURRENT:4}
|
||||
# 终态会话保留时间,便于前端读取最终结果
|
||||
retention-seconds: ${DWS_WEB_AUTH_RETENTION_SECONDS:900}
|
||||
scheduler:
|
||||
# 使用 SnailJob 等外部调度时应关闭内置调度器,避免重复触发
|
||||
enabled: ${SYNC_SCHEDULER_ENABLED:true}
|
||||
# Cron 同步计划扫描间隔(毫秒)
|
||||
poll-interval-ms: ${SYNC_SCHEDULER_POLL_INTERVAL_MS:30000}
|
||||
|
||||
# Spring配置
|
||||
spring:
|
||||
application:
|
||||
@ -67,9 +93,9 @@ spring:
|
||||
servlet:
|
||||
multipart:
|
||||
# 单个文件大小
|
||||
max-file-size: 10MB
|
||||
max-file-size: 1000MB
|
||||
# 设置总上传的文件大小
|
||||
max-request-size: 20MB
|
||||
max-request-size: 2000MB
|
||||
mvc:
|
||||
# 设置静态资源路径 防止所有请求都去查静态资源
|
||||
static-path-pattern: /static/**
|
||||
@ -133,13 +159,13 @@ mybatis-plus:
|
||||
# 数据加密
|
||||
mybatis-encryptor:
|
||||
# 是否开启加密
|
||||
enable: false
|
||||
enable: ${MYBATIS_ENCRYPTOR_ENABLE:false}
|
||||
# 默认加密算法
|
||||
algorithm: BASE64
|
||||
# 编码方式 BASE64/HEX。默认BASE64
|
||||
encode: BASE64
|
||||
# 安全秘钥 对称算法的秘钥 如:AES,SM4
|
||||
password:
|
||||
password: ${MYBATIS_ENCRYPTOR_PASSWORD:}
|
||||
# 公私钥 非对称算法的公私钥 如:SM2,RSA
|
||||
publicKey:
|
||||
privateKey:
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
<module>ruoyi-system</module>
|
||||
<module>ruoyi-workflow</module>
|
||||
<module>ruoyi-ai</module>
|
||||
<module>ruoyi-sync</module>
|
||||
</modules>
|
||||
|
||||
</project>
|
||||
|
||||
280
ruoyi-modules/ruoyi-sync/README.md
Normal file
280
ruoyi-modules/ruoyi-sync/README.md
Normal file
@ -0,0 +1,280 @@
|
||||
# 数据同步后端(V1)
|
||||
|
||||
本模块把原来的“钉钉盘备份到 OSS”收敛为可扩展的数据同步内核。V1 支持:
|
||||
|
||||
- 源端:钉钉企业网盘 / 我的文件(DingTalk Workspace CLI,简称 DWS)
|
||||
- 目标端:标准 S3 兼容存储、阿里云 OSS(复用项目原生 S3 客户端)
|
||||
- 执行:手动运行、内置 Cron 调度、可选 SnailJob 入口
|
||||
- 能力:全量或清单差异同步、在线文档导出、冲突策略、源端删除策略、连接测试、任务明细、对象清单和检查点
|
||||
|
||||
## 1. 架构与扩展点
|
||||
|
||||
```text
|
||||
DingTalkSourceConnector
|
||||
│ ScanResult / SourceContent
|
||||
▼
|
||||
SyncExecutionWorker ── sync_job / sync_job_item / sync_object / sync_checkpoint
|
||||
│ TargetWriteRequest
|
||||
├──────────────► S3TargetConnector
|
||||
└──────────────► AliyunOssTargetConnector
|
||||
```
|
||||
|
||||
源、目标均通过接口隔离:
|
||||
|
||||
- `SourceConnector`:分页扫描、下载或导出源对象
|
||||
- `TargetConnector`:检查对象、上传、删除目标对象
|
||||
- `ConnectorRegistry`:按连接角色和类型选择实现
|
||||
|
||||
后续增加其他源或目标时,不需要修改计划、任务和对象清单的数据模型。
|
||||
|
||||
## 2. 数据库
|
||||
|
||||
先导入项目基础 SQL,再导入 [`script/sql/ry_sync.sql`](../../script/sql/ry_sync.sql)。该脚本面向 MySQL,包含菜单、权限和字典数据。
|
||||
|
||||
| 表 | 用途 |
|
||||
|---|---|
|
||||
| `sync_connection` | 源/目标连接及加密凭证 |
|
||||
| `sync_plan` | 路径、策略、并发、Cron 和删除保护配置 |
|
||||
| `sync_job` | 一次运行及汇总;数据库唯一键保证同一计划只有一个活动任务 |
|
||||
| `sync_job_item` | 每个文件的上传、跳过或删除结果 |
|
||||
| `sync_object` | 源对象与目标对象的持久清单,用于差异判断和删除发现 |
|
||||
| `sync_checkpoint` | 已完成任务水位;V1 不把未完成分页游标当作可提交检查点 |
|
||||
| `sync_transfer_part` | 应用级分片续传的预留表,V1 尚未写入 |
|
||||
|
||||
`delete_guard_percent` 默认是 50。若一次运行拟删除的目标文件比例超过阈值,任务会失败且不会执行任何删除;只有显式设为 100 才允许一次删除全部目标文件。
|
||||
|
||||
## 3. 必需的服务配置
|
||||
|
||||
生产环境必须启用字段加密,否则服务会拒绝保存 `secretJson`:
|
||||
|
||||
```bash
|
||||
MYBATIS_ENCRYPTOR_ENABLE=true
|
||||
MYBATIS_ENCRYPTOR_PASSWORD=<由密钥管理系统注入的 AES 密钥>
|
||||
DWS_EXECUTABLE=/opt/dws/bin/dws
|
||||
DWS_PROFILE_ROOT=/var/lib/ruoyi-sync/dws
|
||||
DWS_KEYCHAIN_ROOT=/var/lib/ruoyi-sync/dws-keychain
|
||||
```
|
||||
|
||||
请不要把真实密钥写进 YAML、启动脚本或 Git。`SyncConnectionVo` 不含 `secretJson`,新增/修改接口的操作日志也排除了该字段。
|
||||
|
||||
后端运行时需要 JDK 21(项目使用虚拟线程)。DWS 建议固定到已验收版本;仓库提供的生产镜像默认使用 `v1.0.60`,升级 DWS 后应重新做一次设备登录和真实空间访问验收。
|
||||
|
||||
内置调度器默认开启。如改由 SnailJob 触发,应设置:
|
||||
|
||||
```bash
|
||||
SYNC_SCHEDULER_ENABLED=false
|
||||
```
|
||||
|
||||
SnailJob 执行器名为 `syncPlanJobExecutor`,任务参数是同步计划 ID。内置调度和 SnailJob 二选一。
|
||||
|
||||
## 4. 钉钉连接
|
||||
|
||||
### 4.1 DWS 运行契约
|
||||
|
||||
运行节点需要安装与验收 DWS,并至少确认以下命令存在:
|
||||
|
||||
```bash
|
||||
dws drive list --help
|
||||
dws drive download --help
|
||||
dws doc export --help
|
||||
dws wiki space list --help
|
||||
```
|
||||
|
||||
本模块使用原子 `drive list` 保留版本、修改时间、扩展名等增量证据;普通文件使用原子 `drive download` 的分片、并发和断点能力;`adoc` 在线文档使用 `doc export`(自动提交、轮询并下载)导出为 `docx`。
|
||||
|
||||
每个同步连接都有独立的 DWS 配置目录:
|
||||
|
||||
```text
|
||||
${DWS_PROFILE_ROOT}/${connectionId}
|
||||
```
|
||||
|
||||
登录 token 的加密 keychain 也按连接隔离,目录为:
|
||||
|
||||
```text
|
||||
${DWS_KEYCHAIN_ROOT}/${connectionId}/dws-cli
|
||||
```
|
||||
|
||||
所有业务命令还会显式传入该连接配置中的 `profile=corpId:userId`,不会依赖服务进程当前选中的账号。
|
||||
|
||||
### 4.2 登录态初始化
|
||||
|
||||
应用密钥不能代替用户 OAuth 登录态。现在首次登录可以完全从管理后台完成,不需要 SSH 到服务器:
|
||||
|
||||
1. 创建一个 `DINGTALK/SOURCE` 连接。`secretJson` 填写 AppKey/AppSecret;`configJson` 可以暂时不填 `profile`。
|
||||
2. 网页调用 `POST /sync/connection/{connectionId}/auth/login/start`。
|
||||
3. 页面展示返回的 `verificationUriComplete`(可生成二维码),同时展示 `verificationUri` 和 `userCode` 作为备用方式。
|
||||
4. 用户在自己的浏览器中打开链接并在钉钉完成确认;页面每 2 秒左右调用状态接口,直到 `SUCCESS`、`FAILED`、`EXPIRED` 或 `CANCELLED`。
|
||||
5. 授权成功后,服务器自动把 DWS 返回的稳定 `corpId:userId` 写入该连接的 `configJson.profile`,之后即可执行连接测试和同步。
|
||||
|
||||
启动接口返回的对象只含一次性授权链接、验证码和脱敏身份信息,绝不返回 access token、refresh token、device code 或 AppSecret。DWS 进程和加密登录态始终留在后端节点;浏览器只负责完成钉钉授权。
|
||||
|
||||
后端只接受 DWS 最终 JSON 中明确返回的 `corp_id`/`corpId` 与 `user_id`/`userId`(或明确的 `profile` 字段),缺少任一 ID 会安全失败,不会根据普通日志文本猜测登录人。连接凭证属于创建者,普通用户只能查看和操作自己创建的连接,超级管理员可跨创建者管理;登录操作也仅允许连接创建人或超级管理员发起、查询和取消。登录进行中禁止修改或删除该连接,Profile 只能由 Web 授权回调变更,编辑旧表单不会清掉已完成的登录态。
|
||||
|
||||
接口契约:
|
||||
|
||||
| 方法 | 地址 | 说明 |
|
||||
|---|---|---|
|
||||
| `POST` | `/sync/connection/{id}/auth/login/start` | 启动设备流;可选查询参数 `expectedCorpId` 用于组织校验 |
|
||||
| `GET` | `/sync/connection/{id}/auth/login/{sessionId}` | 查询会话状态和展示信息 |
|
||||
| `POST` | `/sync/connection/{id}/auth/login/{sessionId}/cancel` | 取消会话并终止服务器上的 DWS 进程 |
|
||||
|
||||
DWS 设备流有服务端有效期(通常约 10 分钟),过期后重新点击“登录”即可。DWS 会自动刷新已保存的登录态,日常同步不需要重复登录。组织必须允许 DWS/CLI 数据访问,且授权用户必须拥有目标空间的访问权限;否则会话会返回失败提示。
|
||||
|
||||
会话状态接口返回 `STARTING`、`WAITING_USER`、`FINALIZING`、`SUCCESS`、`FAILED`、`EXPIRED` 或 `CANCELLED`;进入 `FINALIZING` 后正在写回连接身份,页面应继续轮询并禁止重复发起登录。
|
||||
|
||||
生产部署必须持久化下面两个目录,否则重启后会丢失 DWS 登录态:
|
||||
|
||||
```text
|
||||
${DWS_PROFILE_ROOT}
|
||||
${DWS_KEYCHAIN_ROOT}
|
||||
```
|
||||
|
||||
当前会话管理器保存在单节点内存中,因此 V1 要求登录页面的三次请求落到同一个后端实例(单实例部署天然满足;多实例请配置会话亲和或单独的认证节点)。后续需要无亲和的集群部署时,再把会话状态和进程协调迁移到 Redis/专用认证服务。
|
||||
|
||||
同一节点内,Web 授权会独占该连接的 DWS 配置/keychain 锁;正在执行的扫描或下载会让登录请求稍后重试,登录期间新发起的 DWS 业务命令也会被拒绝。若同步 worker 和 Web/API 分布在多个后端节点,除会话亲和外还需要把这把连接锁迁移到 Redis 或数据库分布式锁。
|
||||
|
||||
如需运维兜底,仍可在服务器上执行等价的设备流命令;这不是正常使用路径:
|
||||
|
||||
```bash
|
||||
DWS_CONFIG_DIR=/var/lib/ruoyi-sync/dws/<connectionId> \
|
||||
DWS_KEYCHAIN_DIR=/var/lib/ruoyi-sync/dws-keychain/<connectionId> \
|
||||
DWS_CLIENT_ID=<与该连接 secretJson 一致的 AppKey> \
|
||||
DWS_CLIENT_SECRET=<与该连接 secretJson 一致的 AppSecret> \
|
||||
dws auth login --device --no-browser
|
||||
```
|
||||
|
||||
### 4.3 连接示例
|
||||
|
||||
请求中的 JSON 字符串为展示方便做了格式化;实际字段仍是字符串。
|
||||
|
||||
```json
|
||||
{
|
||||
"connectionName": "钉钉企业盘",
|
||||
"connectionRole": "SOURCE",
|
||||
"connectionType": "DINGTALK",
|
||||
"configJson": "{\"profile\":\"corpId:userId\",\"spaceType\":\"orgSpace\",\"spaceId\":\"可选\",\"downloadPartSize\":\"32MB\",\"downloadParallel\":\"4\"}",
|
||||
"secretJson": "{\"clientId\":\"AppKey\",\"clientSecret\":\"AppSecret\"}",
|
||||
"status": "0"
|
||||
}
|
||||
```
|
||||
|
||||
- `spaceId` 可选:填写时只同步该空间;不填时按 `spaceType` 发现空间。
|
||||
- `spaceType`:`orgSpace` 或 `mySpace`,默认 `orgSpace`。
|
||||
- `sourceRoot`:`/` 表示空间根;指定子目录时必须填写该目录的 `dentryUuid`,且连接必须指定 `spaceId`。
|
||||
- `downloadParallel`:1 至 8,默认 4。
|
||||
- `downloadPartSize`:DWS 支持的容量字符串,例如 `32MB`。
|
||||
- `configDir` 不允许由连接指定,避免一个连接读取另一个连接的登录态。
|
||||
|
||||
V1 只导出 `adoc -> docx`。`axls`、`able`、`appt`、`amind`、`adraw` 会被识别为在线对象并明确失败,不会错误地按普通文件下载;相应导出适配器留到后续版本。
|
||||
|
||||
## 5. S3 / 阿里云 OSS 连接
|
||||
|
||||
标准 S3 示例:
|
||||
|
||||
```json
|
||||
{
|
||||
"connectionName": "备份 S3",
|
||||
"connectionRole": "TARGET",
|
||||
"connectionType": "S3",
|
||||
"endpoint": "s3.example.com",
|
||||
"region": "us-east-1",
|
||||
"bucketName": "company-backup",
|
||||
"basePath": "dingtalk",
|
||||
"configJson": "{\"useHttps\":true,\"pathStyleAccess\":true}",
|
||||
"secretJson": "{\"accessKey\":\"...\",\"secretKey\":\"...\"}",
|
||||
"status": "0"
|
||||
}
|
||||
```
|
||||
|
||||
阿里云 OSS 示例:
|
||||
|
||||
```json
|
||||
{
|
||||
"connectionName": "阿里云 OSS",
|
||||
"connectionRole": "TARGET",
|
||||
"connectionType": "ALIYUN_OSS",
|
||||
"endpoint": "oss-cn-hangzhou.aliyuncs.com",
|
||||
"region": "cn-hangzhou",
|
||||
"bucketName": "company-backup",
|
||||
"basePath": "dingtalk",
|
||||
"configJson": "{\"useHttps\":true,\"pathStyleAccess\":false}",
|
||||
"secretJson": "{\"accessKey\":\"...\",\"secretKey\":\"...\"}",
|
||||
"status": "0"
|
||||
}
|
||||
```
|
||||
|
||||
`region` 为可选配置。留空时与 ruoyi-vue-plus 原生 OSS 配置保持一致,客户端使用 `us-east-1` 作为默认 Region;如已知存储桶所在地域,仍建议填写实际值(例如 `cn-hangzhou`)。
|
||||
|
||||
连接测试会真实执行 `HeadBucket`。上传完成后再执行 `HeadObject`,以远端实际大小、ETag、版本 ID 和元数据作为任务结果。
|
||||
|
||||
## 6. 同步语义
|
||||
|
||||
计划请求示例:
|
||||
|
||||
```json
|
||||
{
|
||||
"planName": "钉钉企业盘每日备份",
|
||||
"sourceConnectionId": 1,
|
||||
"targetConnectionId": 2,
|
||||
"sourceRoot": "/",
|
||||
"targetPrefix": "daily",
|
||||
"syncMode": "INCREMENTAL",
|
||||
"scheduleType": "CRON",
|
||||
"cronExpression": "0 0 2 * * *",
|
||||
"conflictStrategy": "OVERWRITE",
|
||||
"deleteStrategy": "KEEP",
|
||||
"deleteGuardPercent": 50,
|
||||
"verifyMode": "SHA256",
|
||||
"maxConcurrency": 4,
|
||||
"bandwidthLimitKbps": 0,
|
||||
"status": "0"
|
||||
}
|
||||
```
|
||||
|
||||
- `FULL`:扫描并重新处理全部文件。
|
||||
- `INCREMENTAL`:仍完整枚举权威目录树,然后用 `versionToken -> hash -> modifiedTime + size` 依次判断变化;证据不足时保守地重新传输,不会只凭相同大小跳过。
|
||||
- `OVERWRITE`:写入目标键。
|
||||
- `SKIP`:目标键已存在时跳过。
|
||||
- `KEEP_BOTH`:目标键已存在时附加稳定版本后缀。
|
||||
- `KEEP` / `MARK`:源端消失时保留目标,仅更新清单和任务记录。
|
||||
- `DELETE`:通过删除比例保护后删除目标;删除失败会保留待重试状态,不会误记为已完成。
|
||||
- `SIZE`:以 `HeadObject.contentLength` 校验。
|
||||
- `ETAG`:要求目标端回读到 ETag,不能把不同厂商/不同分片策略的 ETag 当作源端内容哈希。
|
||||
- `SHA256`:本地计算摘要、写入对象元数据并通过 `HeadObject` 回读校验。
|
||||
|
||||
目录页缺少集合、条目缺少关键字段、部分错误、重复游标、重复对象 ID 或目录环路都会使任务失败。失败任务不会进入源删除发现阶段。
|
||||
|
||||
## 7. 后端接口
|
||||
|
||||
| 方法 | 地址 | 权限 | 用途 |
|
||||
|---|---|---|---|
|
||||
| `GET` | `/sync/connection/list` | `sync:connection:list` | 连接分页 |
|
||||
| `GET` | `/sync/connection/{id}` | `sync:connection:query` | 连接详情(不返回凭证) |
|
||||
| `POST` | `/sync/connection` | `sync:connection:add` | 新增连接 |
|
||||
| `PUT` | `/sync/connection` | `sync:connection:edit` | 修改连接;空 `secretJson` 保留原凭证 |
|
||||
| `DELETE` | `/sync/connection/{ids}` | `sync:connection:remove` | 删除未被计划引用的连接 |
|
||||
| `POST` | `/sync/connection/test/{id}` | `sync:connection:test` | 真实连通性测试 |
|
||||
| `POST` | `/sync/connection/{id}/auth/login/start` | `sync:connection:auth` | Web 端启动钉钉设备授权 |
|
||||
| `GET` | `/sync/connection/{id}/auth/login/{sessionId}` | `sync:connection:auth` | 查询 Web 授权状态 |
|
||||
| `POST` | `/sync/connection/{id}/auth/login/{sessionId}/cancel` | `sync:connection:auth` | 取消 Web 授权 |
|
||||
| `GET` | `/sync/plan/list` | `sync:plan:list` | 计划分页 |
|
||||
| `GET` | `/sync/plan/{id}` | `sync:plan:query` | 计划详情 |
|
||||
| `POST` / `PUT` / `DELETE` | `/sync/plan` | 对应 add/edit/remove | 计划维护 |
|
||||
| `POST` | `/sync/job/run/{planId}` | `sync:job:run` | 立即运行 |
|
||||
| `POST` | `/sync/job/retry/{jobId}` | `sync:job:retry` | 重新枚举并重试失败对象 |
|
||||
| `POST` | `/sync/job/cancel/{jobId}` | `sync:job:cancel` | 取消并中断本节点任务 |
|
||||
| `GET` | `/sync/job/list` | `sync:job:list` | 任务分页 |
|
||||
| `GET` | `/sync/job/{id}` | `sync:job:query` | 任务汇总 |
|
||||
| `GET` | `/sync/job/{id}/items` | `sync:job:query` | 文件明细 |
|
||||
| `DELETE` | `/sync/job/{ids}` | `sync:job:remove` | 删除非活动任务及明细 |
|
||||
| `GET` | `/sync/object/list` | `sync:object:list` | 对象清单分页 |
|
||||
| `GET` | `/sync/object/{id}` | `sync:object:query` | 对象详情 |
|
||||
|
||||
## 8. 当前版本边界
|
||||
|
||||
- Web 登录会话是单节点内存态;多实例部署需要网关会话亲和或专用认证节点。不会把 token 或 device code 写入数据库或返回给浏览器。
|
||||
- `sync_transfer_part` 为应用级、跨任务分片续传预留。当前 DWS 下载和项目原生 S3 客户端各自使用其内部大文件机制,但应用数据库尚不接管 uploadId/part 状态。
|
||||
- 带宽字段已预留,但 V1 会拒绝非 0 值,避免界面显示“已限速”而运行时未生效。
|
||||
- 应用进程异常退出可能遗留 `PENDING/RUNNING/CANCELING` 任务。为避免多实例误接管仍在其他节点运行的任务,V1 不做激进的启动自动恢复;运维确认原节点已停止后,需要由 DBA 将遗留任务核销为 `CANCELED`,再点重试。
|
||||
- 取消会中断本节点 DWS 命令并在上传、校验、删除前复查数据库状态;对象存储 SDK 已经提交到远端的请求只能尽力取消,取消后重试仍按幂等对象键收敛。
|
||||
- 未接真实钉钉账号、S3 或 OSS 凭证时只能完成编译和静态契约验证;上线前必须做真实空间枚举、空文件、大文件、在线文档、权限变化、分页和删除保护验收。
|
||||
59
ruoyi-modules/ruoyi-sync/pom.xml
Normal file
59
ruoyi-modules/ruoyi-sync/pom.xml
Normal file
@ -0,0 +1,59 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-modules</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>ruoyi-sync</artifactId>
|
||||
|
||||
<description>可扩展数据同步与备份模块</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-common-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-common-json</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-common-redis</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-common-mybatis</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-common-oss</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-common-log</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-common-security</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-common-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-common-encrypt</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dromara</groupId>
|
||||
<artifactId>ruoyi-common-job</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@ -0,0 +1,16 @@
|
||||
package org.dromara.sync.config;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
|
||||
/**
|
||||
* 数据同步内置调度配置。
|
||||
*
|
||||
* <p>该配置不依赖 SnailJob,确保仅启用内置调度器时 {@code @Scheduled} 仍然生效。</p>
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@EnableScheduling
|
||||
@ConditionalOnProperty(prefix = "sync.scheduler", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class SyncSchedulingConfig {
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
package org.dromara.sync.connector;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.sync.constant.SyncConstants;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 按连接角色和类型解析连接器。
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class ConnectorRegistry {
|
||||
|
||||
private final List<SourceConnector> sourceConnectors;
|
||||
private final List<TargetConnector> targetConnectors;
|
||||
|
||||
public SyncConnector resolve(SyncConnection connection) {
|
||||
if (SyncConstants.ROLE_SOURCE.equals(connection.getConnectionRole())) {
|
||||
return source(connection.getConnectionType());
|
||||
}
|
||||
if (SyncConstants.ROLE_TARGET.equals(connection.getConnectionRole())) {
|
||||
return target(connection.getConnectionType());
|
||||
}
|
||||
throw new ServiceException("不支持的连接角色:{}", connection.getConnectionRole());
|
||||
}
|
||||
|
||||
public SourceConnector source(String type) {
|
||||
return sourceConnectors.stream()
|
||||
.filter(connector -> connector.type().equals(type))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new ServiceException("未找到源端连接器:{}", type));
|
||||
}
|
||||
|
||||
public TargetConnector target(String type) {
|
||||
return targetConnectors.stream()
|
||||
.filter(connector -> connector.type().equals(type))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new ServiceException("未找到目标端连接器:{}", type));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
package org.dromara.sync.connector;
|
||||
|
||||
import org.dromara.sync.connector.model.ScanRequest;
|
||||
import org.dromara.sync.connector.model.ScanResult;
|
||||
import org.dromara.sync.connector.model.SourceContent;
|
||||
import org.dromara.sync.connector.model.SourceObject;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
|
||||
/**
|
||||
* 源端连接器契约。
|
||||
*/
|
||||
public interface SourceConnector extends SyncConnector {
|
||||
|
||||
ScanResult scan(SyncConnection connection, ScanRequest request);
|
||||
|
||||
SourceContent download(SyncConnection connection, SourceObject object);
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
package org.dromara.sync.connector;
|
||||
|
||||
import org.dromara.sync.connector.model.ConnectorTestResult;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
|
||||
/**
|
||||
* 数据同步连接器基础契约。
|
||||
*/
|
||||
public interface SyncConnector {
|
||||
|
||||
/**
|
||||
* 获取连接器类型编码。
|
||||
*
|
||||
* @return 连接类型
|
||||
*/
|
||||
String type();
|
||||
|
||||
/**
|
||||
* 测试连接。
|
||||
*
|
||||
* @param connection 连接配置
|
||||
* @return 测试结果
|
||||
*/
|
||||
ConnectorTestResult test(SyncConnection connection);
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
package org.dromara.sync.connector;
|
||||
|
||||
import org.dromara.sync.connector.model.TargetWriteRequest;
|
||||
import org.dromara.sync.connector.model.TargetWriteResult;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
|
||||
/**
|
||||
* 目标端连接器契约。
|
||||
*/
|
||||
public interface TargetConnector extends SyncConnector {
|
||||
|
||||
TargetWriteResult upload(SyncConnection connection, TargetWriteRequest request);
|
||||
|
||||
/**
|
||||
* 判断目标对象是否存在,供冲突策略使用。
|
||||
*
|
||||
* @param connection 目标连接
|
||||
* @param objectKey 对象键
|
||||
* @return 对象存在返回 {@code true}
|
||||
*/
|
||||
boolean exists(SyncConnection connection, String objectKey);
|
||||
|
||||
/**
|
||||
* 删除目标对象。只有同步计划明确配置删除策略时才允许调用。
|
||||
*
|
||||
* @param connection 目标连接
|
||||
* @param objectKey 对象键
|
||||
* @return 是否删除成功
|
||||
*/
|
||||
boolean delete(SyncConnection connection, String objectKey);
|
||||
}
|
||||
@ -0,0 +1,737 @@
|
||||
package org.dromara.sync.connector.dingtalk;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.core.utils.StringUtils;
|
||||
import org.dromara.common.json.utils.JsonUtils;
|
||||
import org.dromara.sync.connector.SourceConnector;
|
||||
import org.dromara.sync.connector.model.*;
|
||||
import org.dromara.sync.constant.SyncConstants;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.io.IOException;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.*;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
/**
|
||||
* 基于 DingTalk Workspace CLI 的钉钉网盘源连接器。
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class DingTalkSourceConnector implements SourceConnector {
|
||||
|
||||
private static final int PAGE_SIZE = 50;
|
||||
private static final Set<String> ONLINE_EXTENSIONS = Set.of("adoc", "axls", "able", "appt", "amind", "adraw");
|
||||
private static final String XLSX_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
|
||||
private final DwsCommandRunner commandRunner;
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return SyncConstants.TYPE_DINGTALK;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorTestResult test(SyncConnection connection) {
|
||||
try {
|
||||
Map<String, Object> config = commandRunner.config(connection);
|
||||
String spaceId = value(config, "spaceId");
|
||||
if (StringUtils.isNotBlank(spaceId)) {
|
||||
DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, List.of(
|
||||
"drive", "list", "--space-id", spaceId, "--limit", "1", "--format", "json"
|
||||
), Duration.ofSeconds(60));
|
||||
parseScanResult(result.stdout(), spaceId);
|
||||
return ConnectorTestResult.success("钉钉空间访问成功");
|
||||
}
|
||||
List<String> arguments = new ArrayList<>(List.of(
|
||||
"wiki", "space", "list", "--type", spaceType(config), "--format", "json"
|
||||
));
|
||||
if ("orgSpace".equals(spaceType(config))) {
|
||||
arguments.add("--limit");
|
||||
arguments.add("1");
|
||||
}
|
||||
DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, arguments, Duration.ofSeconds(60));
|
||||
parseSpaceResult(result.stdout());
|
||||
return ConnectorTestResult.success("钉钉空间发现成功");
|
||||
} catch (Exception e) {
|
||||
return ConnectorTestResult.failure(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScanResult scan(SyncConnection connection, ScanRequest request) {
|
||||
Map<String, Object> config = commandRunner.config(connection);
|
||||
String spaceId = StringUtils.firstNonBlank(request.scopeId(), value(config, "spaceId"));
|
||||
if (StringUtils.isBlank(spaceId)) {
|
||||
if (StringUtils.isNotBlank(request.parentObjectId())) {
|
||||
throw new ServiceException("指定钉盘子目录时必须在连接配置中提供 spaceId");
|
||||
}
|
||||
return scanSpaces(connection, config, request.cursor());
|
||||
}
|
||||
List<String> arguments = new ArrayList<>(List.of(
|
||||
"drive", "list", "--limit", String.valueOf(PAGE_SIZE), "--format", "json"
|
||||
));
|
||||
addOption(arguments, "--space-id", spaceId);
|
||||
addOption(arguments, "--folder", request.parentObjectId());
|
||||
addOption(arguments, "--cursor", request.cursor());
|
||||
DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, arguments, Duration.ofMinutes(5));
|
||||
return parseScanResult(result.stdout(), spaceId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SourceContent download(SyncConnection connection, SourceObject object) {
|
||||
if (object.isFolder()) {
|
||||
throw new ServiceException("文件夹不能下载:{}", object.name());
|
||||
}
|
||||
String extension = normalizeExtension(object.extension());
|
||||
if ("dlink".equalsIgnoreCase(extension)) {
|
||||
throw new ServiceException("钉钉多维表链接(.dlink)不支持按普通文件下载");
|
||||
}
|
||||
if ("amind".equalsIgnoreCase(extension) || "adraw".equalsIgnoreCase(extension)) {
|
||||
throw new ServiceException("钉钉官方 DWS 暂不支持自动导出:{}", extension);
|
||||
}
|
||||
Path directory = null;
|
||||
try {
|
||||
directory = Files.createTempDirectory("ruoyi-sync-dingtalk-");
|
||||
Path output = directory.resolve(resolveOutputName(object));
|
||||
if ("dlink".equalsIgnoreCase(extension)) {
|
||||
exportAITable(connection, object, output);
|
||||
return new SourceContent(output, Files.size(output), XLSX_CONTENT_TYPE, true);
|
||||
}
|
||||
List<String> arguments = buildDownloadArguments(connection, object, output);
|
||||
try {
|
||||
commandRunner.run(connection, arguments, Duration.ofHours(2), directory);
|
||||
} catch (ServiceException exportFailure) {
|
||||
// DWS v1.0.60 cannot export an empty ALIDOC and reports the
|
||||
// asynchronous submit as doc_write_commit_unknown. Confirm
|
||||
// the document is truly empty before applying the fallback;
|
||||
// all non-empty documents and unrelated failures must retain
|
||||
// the original error.
|
||||
if (!SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType())
|
||||
|| !"adoc".equalsIgnoreCase(normalizeExtension(object.extension()))
|
||||
|| !isEmptyOnlineDocument(connection, object)) {
|
||||
throw exportFailure;
|
||||
}
|
||||
writeEmptyDocx(output);
|
||||
}
|
||||
if (!Files.isRegularFile(output)) {
|
||||
throw new ServiceException("DWS 执行成功但未生成文件:{}", object.name());
|
||||
}
|
||||
String contentType = SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType())
|
||||
? onlineDocumentContentType(object)
|
||||
: object.contentType();
|
||||
return new SourceContent(output, Files.size(output), contentType, true);
|
||||
} catch (ServiceException e) {
|
||||
cleanDirectory(directory);
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
cleanDirectory(directory);
|
||||
throw new ServiceException("下载钉钉对象失败:{}", object.name(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isEmptyOnlineDocument(SyncConnection connection, SourceObject object) {
|
||||
try {
|
||||
DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, List.of(
|
||||
"doc", "+fetch", "--node", object.objectId(), "--format", "json", "--detail", "full"
|
||||
), Duration.ofMinutes(2));
|
||||
JsonNode root = JsonUtils.getJsonMapper().readTree(result.stdout());
|
||||
JsonNode content = root.path("content.jsonml");
|
||||
return !containsNonBlankText(content);
|
||||
} catch (Exception ignored) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean containsNonBlankText(JsonNode node) {
|
||||
if (node == null || node.isNull()) {
|
||||
return false;
|
||||
}
|
||||
if (node.isTextual()) {
|
||||
return StringUtils.isNotBlank(node.asString());
|
||||
}
|
||||
if (node.isArray() || node.isObject()) {
|
||||
for (JsonNode child : node) {
|
||||
if (containsNonBlankText(child)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void writeEmptyDocx(Path output) throws Exception {
|
||||
Files.deleteIfExists(output);
|
||||
try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(output), StandardCharsets.UTF_8)) {
|
||||
putZipEntry(zip, "[Content_Types].xml", """
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
|
||||
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
|
||||
<Default Extension="xml" ContentType="application/xml"/>
|
||||
<Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
|
||||
<Override PartName="/word/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml"/>
|
||||
</Types>
|
||||
""");
|
||||
putZipEntry(zip, "_rels/.rels", """
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
|
||||
</Relationships>
|
||||
""");
|
||||
putZipEntry(zip, "word/document.xml", """
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
|
||||
<w:body><w:p/><w:sectPr/></w:body>
|
||||
</w:document>
|
||||
""");
|
||||
putZipEntry(zip, "word/styles.xml", """
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<w:styles xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:docDefaults/></w:styles>
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
private void putZipEntry(ZipOutputStream zip, String name, String content) throws IOException {
|
||||
zip.putNextEntry(new ZipEntry(name));
|
||||
zip.write(content.getBytes(StandardCharsets.UTF_8));
|
||||
zip.closeEntry();
|
||||
}
|
||||
|
||||
private List<String> buildDownloadArguments(SyncConnection connection, SourceObject object, Path output) {
|
||||
if (SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType())) {
|
||||
String extension = normalizeExtension(object.extension());
|
||||
if ("axls".equalsIgnoreCase(extension)) {
|
||||
return List.of(
|
||||
"sheet", "export", "--node", object.objectId(), "--output", output.getFileName().toString(), "--yes"
|
||||
);
|
||||
}
|
||||
if ("amind".equalsIgnoreCase(extension) || "adraw".equalsIgnoreCase(extension)) {
|
||||
throw new ServiceException("钉钉官方 DWS 暂不支持自动导出:{}", extension);
|
||||
}
|
||||
if (!"adoc".equalsIgnoreCase(extension)) {
|
||||
throw new ServiceException("当前版本暂不支持导出钉钉在线文档类型:{}", object.extension());
|
||||
}
|
||||
// Use the consolidated `doc export` command. The `+export`
|
||||
// shortcut maps to the low-level submit operation in some DWS
|
||||
// releases (including v1.0.60), which can return
|
||||
// `doc_write_commit_unknown` before polling/downloading the
|
||||
// completed export job. `doc export` performs submit, polling
|
||||
// and download atomically and only returns after the output file
|
||||
// has been written.
|
||||
return List.of(
|
||||
"doc", "export", "--node", object.objectId(), "--output", output.getFileName().toString(),
|
||||
"--export-format", "docx", "--yes"
|
||||
);
|
||||
}
|
||||
Map<String, Object> config = commandRunner.config(connection);
|
||||
List<String> arguments = new ArrayList<>(List.of(
|
||||
"drive", "download", "--node", object.objectId(),
|
||||
"--output", output.getFileName().toString(), "--format", "json"
|
||||
));
|
||||
addOption(arguments, "--space-id",
|
||||
StringUtils.firstNonBlank(metadataValue(object, "spaceId"), value(config, "spaceId")));
|
||||
String partSize = value(config, "downloadPartSize");
|
||||
addOption(arguments, "--part-size", StringUtils.isBlank(partSize) ? "32MB" : partSize);
|
||||
String parallel = value(config, "downloadParallel");
|
||||
addOption(arguments, "--parallel", StringUtils.isBlank(parallel) ? "4" : parallel);
|
||||
return arguments;
|
||||
}
|
||||
|
||||
private void exportAITable(SyncConnection connection, SourceObject object, Path output) throws Exception {
|
||||
String url = StringUtils.firstNonBlank(metadataValue(object, "sourceUrl"),
|
||||
object.name() != null && object.name().startsWith("http") ? object.name() : null,
|
||||
object.objectId() != null && object.objectId().startsWith("http") ? object.objectId() : null);
|
||||
if (StringUtils.isBlank(url) || !url.startsWith("http")) {
|
||||
throw new ServiceException("无法解析钉钉多维表链接(.dlink):缺少完整 alidocs/dlink URL");
|
||||
}
|
||||
DwsCommandRunner.DwsCommandResult resolved = commandRunner.run(connection, List.of(
|
||||
"aitable", "+url-resolve", "--url", url, "--format", "json"
|
||||
), Duration.ofMinutes(2));
|
||||
JsonNode ids = JsonUtils.getJsonMapper().readTree(resolved.stdout());
|
||||
String baseId = findText(ids, "baseId", "baseID");
|
||||
String tableId = findText(ids, "tableId", "datasheetId", "sheetId");
|
||||
if (StringUtils.isAnyBlank(baseId, tableId)) {
|
||||
throw new ServiceException("无法解析钉钉多维表链接(.dlink):未找到 baseId/tableId");
|
||||
}
|
||||
JsonNode result;
|
||||
try {
|
||||
DwsCommandRunner.DwsCommandResult exported = commandRunner.run(connection, List.of(
|
||||
"aitable", "+export-data", "--base-id", baseId, "--scope", "table", "--format", "excel",
|
||||
"--table-id", tableId, "--timeout-ms", "120000"
|
||||
), Duration.ofMinutes(5));
|
||||
String exportOutput = exported.stdout();
|
||||
result = JsonUtils.getJsonMapper().readTree(exportOutput);
|
||||
if (result == null && exportOutput != null && exportOutput.trim().startsWith("http")) {
|
||||
result = JsonUtils.getJsonMapper().createObjectNode().put("downloadUrl", exportOutput.trim());
|
||||
}
|
||||
} catch (Exception exportFailure) {
|
||||
result = null;
|
||||
}
|
||||
String downloadUrl = findText(result, "downloadUrl", "url", "fileUrl", "downloadURL");
|
||||
if (StringUtils.isBlank(downloadUrl)) {
|
||||
DwsCommandRunner.DwsCommandResult records = commandRunner.run(connection, List.of(
|
||||
"aitable", "record", "query", "--base-id", baseId, "--table-id", tableId,
|
||||
"--all", "--page-limit", "100"
|
||||
), Duration.ofMinutes(5));
|
||||
writeAITableWorkbook(output, minimalWorkbook(JsonUtils.getJsonMapper().readTree(records.stdout())),
|
||||
url, object, baseId, tableId);
|
||||
return;
|
||||
}
|
||||
HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build();
|
||||
HttpRequest request = HttpRequest.newBuilder(URI.create(downloadUrl)).GET().build();
|
||||
HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300 || response.body().length == 0) {
|
||||
throw new ServiceException("下载钉钉多维表导出文件失败,HTTP {}", response.statusCode());
|
||||
}
|
||||
writeAITableWorkbook(output, response.body(), url, object, baseId, tableId);
|
||||
}
|
||||
|
||||
private byte[] minimalWorkbook(JsonNode data) throws IOException {
|
||||
StringBuilder rows = new StringBuilder();
|
||||
int rowNumber = 1;
|
||||
JsonNode records = data == null ? null : (data.isArray() ? data : data.path("records"));
|
||||
if (records != null && records.isArray()) {
|
||||
for (JsonNode record : records) {
|
||||
rows.append(row(rowNumber++, "record", xmlEscape(record.toString())));
|
||||
}
|
||||
}
|
||||
if (rows.isEmpty()) rows.append(row(1, "data", ""));
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
entries.put("[Content_Types].xml", "<?xml version=\"1.0\"?><Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\"><Default Extension=\"rels\" ContentType=\"application/vnd.openxmlformats-package.relationships+xml\"/><Default Extension=\"xml\" ContentType=\"application/xml\"/><Override PartName=\"/xl/workbook.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml\"/><Override PartName=\"/xl/worksheets/sheet1.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml\"/></Types>".getBytes(StandardCharsets.UTF_8));
|
||||
entries.put("_rels/.rels", "<?xml version=\"1.0\"?><Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\"><Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument\" Target=\"xl/workbook.xml\"/></Relationships>".getBytes(StandardCharsets.UTF_8));
|
||||
entries.put("xl/workbook.xml", "<?xml version=\"1.0\"?><workbook xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\" xmlns:r=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships\"><sheets><sheet name=\"Sheet1\" sheetId=\"1\" r:id=\"rId1\"/></sheets></workbook>".getBytes(StandardCharsets.UTF_8));
|
||||
entries.put("xl/_rels/workbook.xml.rels", "<?xml version=\"1.0\"?><Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\"><Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet\" Target=\"worksheets/sheet1.xml\"/></Relationships>".getBytes(StandardCharsets.UTF_8));
|
||||
entries.put("xl/worksheets/sheet1.xml", ("<?xml version=\"1.0\"?><worksheet xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\"><sheetData>" + rows + "</sheetData></worksheet>").getBytes(StandardCharsets.UTF_8));
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
try (ZipOutputStream zip = new ZipOutputStream(out)) {
|
||||
for (Map.Entry<String, byte[]> entry : entries.entrySet()) { zip.putNextEntry(new ZipEntry(entry.getKey())); zip.write(entry.getValue()); zip.closeEntry(); }
|
||||
}
|
||||
return out.toByteArray();
|
||||
}
|
||||
|
||||
private String findText(JsonNode node, String... names) {
|
||||
if (node == null || node.isNull()) return null;
|
||||
for (String name : names) {
|
||||
JsonNode value = node.path(name);
|
||||
if (value.isTextual() && StringUtils.isNotBlank(value.asString())) return value.asString();
|
||||
}
|
||||
if (node.isArray() || node.isObject()) {
|
||||
for (JsonNode child : node) {
|
||||
String found = findText(child, names);
|
||||
if (StringUtils.isNotBlank(found)) return found;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void writeAITableWorkbook(Path output, byte[] workbook, String url, SourceObject object,
|
||||
String baseId, String tableId) throws IOException {
|
||||
// Keep the official exported workbook as Sheet1 and add a metadata Sheet2.
|
||||
Map<String, byte[]> entries = new LinkedHashMap<>();
|
||||
try (ZipInputStream zip = new ZipInputStream(new ByteArrayInputStream(workbook))) {
|
||||
ZipEntry entry;
|
||||
while ((entry = zip.getNextEntry()) != null) {
|
||||
entries.put(entry.getName(), zip.readAllBytes());
|
||||
}
|
||||
}
|
||||
byte[] workbookXml = entries.get("xl/workbook.xml");
|
||||
byte[] rels = entries.get("xl/_rels/workbook.xml.rels");
|
||||
if (workbookXml == null || rels == null) {
|
||||
throw new IOException("导出的文件不是有效 XLSX");
|
||||
}
|
||||
String sheetName = "Sheet2";
|
||||
String relId = "rId" + (entries.size() + 1);
|
||||
String xml = new String(workbookXml, StandardCharsets.UTF_8);
|
||||
int sheetsEnd = xml.indexOf("</sheets>");
|
||||
if (sheetsEnd < 0) throw new IOException("XLSX 缺少 sheets 节点");
|
||||
xml = xml.substring(0, sheetsEnd) + "<sheet name=\"Sheet2\" sheetId=\"2\" r:id=\"" + relId + "\"/>" + xml.substring(sheetsEnd);
|
||||
String relXml = new String(rels, StandardCharsets.UTF_8);
|
||||
int relEnd = relXml.indexOf("</Relationships>");
|
||||
relXml = relXml.substring(0, relEnd) + "<Relationship Id=\"" + relId + "\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet\" Target=\"worksheets/sheet2.xml\"/>" + relXml.substring(relEnd);
|
||||
entries.put("xl/workbook.xml", xml.getBytes(StandardCharsets.UTF_8));
|
||||
entries.put("xl/_rels/workbook.xml.rels", relXml.getBytes(StandardCharsets.UTF_8));
|
||||
String contentTypes = new String(entries.get("[Content_Types].xml"), StandardCharsets.UTF_8);
|
||||
int ctEnd = contentTypes.indexOf("</Types>");
|
||||
contentTypes = contentTypes.substring(0, ctEnd) + "<Override PartName=\"/xl/worksheets/sheet2.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml\"/>" + contentTypes.substring(ctEnd);
|
||||
entries.put("[Content_Types].xml", contentTypes.getBytes(StandardCharsets.UTF_8));
|
||||
String escapedUrl = xmlEscape(url);
|
||||
String escapedName = xmlEscape(object.name());
|
||||
String sheet2 = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><worksheet xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\"><sheetData>"
|
||||
+ row(1, "Field", "Value") + row(2, "sourceUrl", escapedUrl) + row(3, "sourceObjectId", xmlEscape(object.objectId()))
|
||||
+ row(4, "name", escapedName) + row(5, "baseId", xmlEscape(baseId)) + row(6, "tableId", xmlEscape(tableId))
|
||||
+ row(7, "exportedAt", xmlEscape(Instant.now().toString())) + "</sheetData></worksheet>";
|
||||
entries.put("xl/worksheets/sheet2.xml", sheet2.getBytes(StandardCharsets.UTF_8));
|
||||
try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(output))) {
|
||||
for (Map.Entry<String, byte[]> entry : entries.entrySet()) {
|
||||
zip.putNextEntry(new ZipEntry(entry.getKey())); zip.write(entry.getValue()); zip.closeEntry();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String row(int row, String key, String value) {
|
||||
return "<row r=\"" + row + "\"><c r=\"A" + row + "\" t=\"inlineStr\"><is><t>" + key + "</t></is></c><c r=\"B" + row + "\" t=\"inlineStr\"><is><t>" + value + "</t></is></c></row>";
|
||||
}
|
||||
|
||||
private String xmlEscape(String value) {
|
||||
return value == null ? "" : value.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """);
|
||||
}
|
||||
|
||||
private String onlineDocumentContentType(SourceObject object) {
|
||||
String extension = normalizeExtension(object.extension());
|
||||
if ("axls".equalsIgnoreCase(extension)) {
|
||||
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
}
|
||||
if ("amind".equalsIgnoreCase(extension) || "adraw".equalsIgnoreCase(extension)) {
|
||||
return "application/pdf";
|
||||
}
|
||||
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
|
||||
}
|
||||
|
||||
private ScanResult scanSpaces(SyncConnection connection, Map<String, Object> config, String cursor) {
|
||||
String spaceType = spaceType(config);
|
||||
List<String> arguments = new ArrayList<>(List.of(
|
||||
"wiki", "space", "list", "--type", spaceType, "--format", "json"
|
||||
));
|
||||
if ("orgSpace".equals(spaceType)) {
|
||||
arguments.add("--limit");
|
||||
arguments.add(String.valueOf(PAGE_SIZE));
|
||||
addOption(arguments, "--cursor", cursor);
|
||||
}
|
||||
DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, arguments, Duration.ofMinutes(2));
|
||||
return parseSpaceResult(result.stdout());
|
||||
}
|
||||
|
||||
private ScanResult parseScanResult(String json, String spaceId) {
|
||||
JsonNode root = JsonUtils.getJsonMapper().readTree(json);
|
||||
validateEnvelope(root);
|
||||
JsonNode payload = root.has("body") ? root.path("body") : root;
|
||||
JsonNode array = findObjectArray(payload);
|
||||
if (array == null) {
|
||||
throw new ServiceException("无法识别 DWS 目录列表结果,已停止同步以避免误判删除");
|
||||
}
|
||||
List<SourceObject> objects = new ArrayList<>();
|
||||
for (JsonNode item : array) {
|
||||
objects.add(mapObject(item, spaceId));
|
||||
}
|
||||
String cursor = nestedText(payload, "nextCursor", "nextToken");
|
||||
boolean hasMore = nestedBoolean(payload, "hasMore") || StringUtils.isNotBlank(cursor);
|
||||
return new ScanResult(objects, cursor, hasMore);
|
||||
}
|
||||
|
||||
private ScanResult parseSpaceResult(String json) {
|
||||
JsonNode root = JsonUtils.getJsonMapper().readTree(json);
|
||||
validateEnvelope(root);
|
||||
JsonNode payload = root.has("body") ? root.path("body") : root;
|
||||
JsonNode array = findObjectArray(payload);
|
||||
if (array == null) {
|
||||
throw new ServiceException("无法识别 DWS 空间列表结果,已停止同步");
|
||||
}
|
||||
List<SourceObject> spaces = new ArrayList<>();
|
||||
for (JsonNode item : array) {
|
||||
String spaceId = text(item, "spaceId", "id");
|
||||
String rootFolderId = text(item, "rootFolderId", "rootDentryUuid", "rootNodeId");
|
||||
String name = text(item, "spaceName", "name", "title");
|
||||
if (StringUtils.isAnyBlank(spaceId, rootFolderId, name)) {
|
||||
throw new ServiceException("DWS 空间列表包含缺少 spaceId、rootFolderId 或名称的条目,已停止同步");
|
||||
}
|
||||
validateObjectIdentity(rootFolderId, name);
|
||||
validateLength("空间ID", spaceId, 255);
|
||||
Map<String, String> metadata = new HashMap<>();
|
||||
metadata.put("spaceId", spaceId);
|
||||
putIfPresent(metadata, "spaceType", text(item, "spaceType", "type"));
|
||||
spaces.add(new SourceObject(rootFolderId, null, name, SyncConstants.OBJECT_FOLDER,
|
||||
"", null, 0L, null, spaceId, null, metadata));
|
||||
}
|
||||
String cursor = nestedText(payload, "nextToken", "nextCursor");
|
||||
return new ScanResult(spaces, cursor, StringUtils.isNotBlank(cursor));
|
||||
}
|
||||
|
||||
private void validateEnvelope(JsonNode root) {
|
||||
if (root == null || root.isNull()) {
|
||||
throw new ServiceException("DWS 返回了空结果");
|
||||
}
|
||||
if (root.path("success").isBoolean() && !root.path("success").asBoolean()) {
|
||||
throw new ServiceException("DWS 返回失败:{}", root.path("message").asString("未知错误"));
|
||||
}
|
||||
JsonNode errors = findField(root, "errors");
|
||||
if (errors != null && errors.isArray() && !errors.isEmpty()) {
|
||||
throw new ServiceException("DWS 返回了 {} 项目录读取错误,已停止同步", errors.size());
|
||||
}
|
||||
}
|
||||
|
||||
private JsonNode findObjectArray(JsonNode payload) {
|
||||
if (payload == null) {
|
||||
return null;
|
||||
}
|
||||
if (payload.isArray()) {
|
||||
return payload;
|
||||
}
|
||||
for (String name : List.of("items", "files", "dentries", "entries", "fileList", "spaces",
|
||||
"list", "data", "result", "content", "body")) {
|
||||
JsonNode candidate = payload.path(name);
|
||||
if (candidate.isArray()) {
|
||||
return candidate;
|
||||
}
|
||||
if (candidate.isObject()) {
|
||||
JsonNode nested = findObjectArray(candidate);
|
||||
if (nested != null) {
|
||||
return nested;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private SourceObject mapObject(JsonNode item, String scanSpaceId) {
|
||||
String objectId = text(item, "fileId", "dentryUuid", "nodeId", "id");
|
||||
String name = text(item, "name", "fileName", "title");
|
||||
if (StringUtils.isBlank(objectId) || StringUtils.isBlank(name)) {
|
||||
throw new ServiceException("DWS 目录列表包含缺少对象ID或名称的条目,已停止同步");
|
||||
}
|
||||
validateObjectIdentity(objectId, name);
|
||||
String extension = normalizeExtension(text(item, "extension", "fileExtension", "suffix"));
|
||||
String dentryType = text(item, "dentryType", "nodeType", "type");
|
||||
String contentType = text(item, "contentType", "mimeType");
|
||||
String objectType = resolveObjectType(dentryType, contentType, extension);
|
||||
Long listedSize = longValue(item, "size", "fileSize", "contentSize");
|
||||
if (SyncConstants.OBJECT_FILE.equals(objectType) && listedSize == null) {
|
||||
throw new ServiceException("DWS 普通文件缺少大小字段:{}", name);
|
||||
}
|
||||
long size = listedSize == null ? 0L : listedSize;
|
||||
if (size < 0) {
|
||||
throw new ServiceException("DWS 对象大小不能为负数:{}", name);
|
||||
}
|
||||
LocalDateTime modifiedTime = timeValue(item, "modifyTime", "modifiedTime", "updateTime", "updatedAt");
|
||||
String versionToken = text(item, "versionToken", "version", "versionNumber", "etag", "eTag");
|
||||
if (StringUtils.isBlank(versionToken) && modifiedTime != null) {
|
||||
versionToken = modifiedTime.toString() + ':' + size;
|
||||
}
|
||||
Map<String, String> metadata = new HashMap<>();
|
||||
putIfPresent(metadata, "spaceId", StringUtils.firstNonBlank(text(item, "spaceId"), scanSpaceId));
|
||||
putIfPresent(metadata, "dentryId", text(item, "dentryId"));
|
||||
putIfPresent(metadata, "extension", extension);
|
||||
putIfPresent(metadata, "sourceUrl", text(item, "url", "link", "dlink", "dlinkUrl", "alidocsUrl", "shareUrl", "sourceUrl"));
|
||||
if (!metadata.containsKey("sourceUrl") && "dlink".equalsIgnoreCase(extension)
|
||||
&& name.startsWith("http")) {
|
||||
metadata.put("sourceUrl", name);
|
||||
}
|
||||
String parentObjectId = text(item, "parentId", "parentFileId", "parentDentryUuid");
|
||||
String hash = text(item, "sha256", "hash", "etag", "eTag");
|
||||
validateLength("父对象ID", parentObjectId, 255);
|
||||
validateLength("版本标识", versionToken, 512);
|
||||
validateLength("源端ETag", hash, 255);
|
||||
validateLength("内容类型", contentType, 255);
|
||||
return new SourceObject(
|
||||
objectId,
|
||||
parentObjectId,
|
||||
name,
|
||||
objectType,
|
||||
extension,
|
||||
contentType,
|
||||
size,
|
||||
modifiedTime,
|
||||
versionToken,
|
||||
hash,
|
||||
metadata
|
||||
);
|
||||
}
|
||||
|
||||
private void validateObjectIdentity(String objectId, String name) {
|
||||
validateLength("对象ID", objectId, 255);
|
||||
validateLength("对象名称", name, 1024);
|
||||
if (".".equals(name) || "..".equals(name)
|
||||
|| name.indexOf('/') >= 0 || name.indexOf('\\') >= 0 || name.indexOf('\0') >= 0) {
|
||||
throw new ServiceException("DWS 返回了不安全的对象名称,已停止同步");
|
||||
}
|
||||
for (int index = 0; index < name.length(); index++) {
|
||||
if (Character.isISOControl(name.charAt(index))) {
|
||||
throw new ServiceException("DWS 对象名称包含控制字符,已停止同步");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void validateLength(String fieldName, String value, int maxLength) {
|
||||
if (value != null && value.length() > maxLength) {
|
||||
throw new ServiceException("DWS {}超过数据库长度上限{}", fieldName, maxLength);
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveObjectType(String dentryType, String contentType, String extension) {
|
||||
if ("folder".equalsIgnoreCase(dentryType) || "FOLDER".equalsIgnoreCase(dentryType)) {
|
||||
return SyncConstants.OBJECT_FOLDER;
|
||||
}
|
||||
if ("ALIDOC".equalsIgnoreCase(contentType) || ONLINE_EXTENSIONS.contains(extension)) {
|
||||
return SyncConstants.OBJECT_ONLINE_DOCUMENT;
|
||||
}
|
||||
if ("file".equalsIgnoreCase(dentryType) || "FILE".equalsIgnoreCase(dentryType)) {
|
||||
return SyncConstants.OBJECT_FILE;
|
||||
}
|
||||
throw new ServiceException("无法识别 DWS 对象类型:{}", StringUtils.defaultIfBlank(dentryType, "空"));
|
||||
}
|
||||
|
||||
private String resolveOutputName(SourceObject object) {
|
||||
if ("dlink".equalsIgnoreCase(normalizeExtension(object.extension()))) {
|
||||
return "payload-" + UUID.randomUUID() + ".xlsx";
|
||||
}
|
||||
if (SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType())) {
|
||||
String sourceExtension = normalizeExtension(object.extension());
|
||||
String extension = "axls".equalsIgnoreCase(sourceExtension) ? "xlsx"
|
||||
: ("amind".equalsIgnoreCase(sourceExtension) || "adraw".equalsIgnoreCase(sourceExtension)
|
||||
? "pdf" : "docx");
|
||||
return "payload-" + UUID.randomUUID() + "." + extension;
|
||||
}
|
||||
String extension = normalizeExtension(object.extension());
|
||||
if (!extension.matches("[a-z0-9]{1,16}")) {
|
||||
extension = "";
|
||||
}
|
||||
return "payload-" + UUID.randomUUID() + (extension.isEmpty() ? "" : "." + extension);
|
||||
}
|
||||
|
||||
private LocalDateTime timeValue(JsonNode node, String... names) {
|
||||
String value = text(node, names);
|
||||
if (StringUtils.isBlank(value)) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
long timestamp = Long.parseLong(value);
|
||||
if (timestamp < 10_000_000_000L) {
|
||||
timestamp *= 1000;
|
||||
}
|
||||
return LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.systemDefault());
|
||||
} catch (NumberFormatException ignored) {
|
||||
try {
|
||||
return LocalDateTime.ofInstant(Instant.parse(value), ZoneId.systemDefault());
|
||||
} catch (DateTimeParseException ignoredAgain) {
|
||||
try {
|
||||
return LocalDateTime.parse(value);
|
||||
} catch (DateTimeParseException ignoredLast) {
|
||||
throw new ServiceException("DWS 返回了无法解析的时间字段:{}", value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Long longValue(JsonNode node, String... names) {
|
||||
String value = text(node, names);
|
||||
if (StringUtils.isBlank(value)) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return Long.parseLong(value);
|
||||
} catch (NumberFormatException ignored) {
|
||||
throw new ServiceException("DWS 返回了无法解析的大小字段:{}", value);
|
||||
}
|
||||
}
|
||||
|
||||
private String nestedText(JsonNode node, String... names) {
|
||||
JsonNode value = findField(node, names);
|
||||
return value == null || value.isObject() || value.isArray() ? null : value.asString();
|
||||
}
|
||||
|
||||
private boolean nestedBoolean(JsonNode node, String name) {
|
||||
JsonNode value = findField(node, name);
|
||||
return value != null && (value.isBoolean() ? value.asBoolean()
|
||||
: Boolean.parseBoolean(value.asString("false")));
|
||||
}
|
||||
|
||||
private JsonNode findField(JsonNode node, String... names) {
|
||||
if (node == null || node.isNull() || node.isArray()) {
|
||||
return null;
|
||||
}
|
||||
for (String name : names) {
|
||||
JsonNode value = node.get(name);
|
||||
if (value != null && !value.isNull()) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
for (JsonNode child : node) {
|
||||
if (child.isObject()) {
|
||||
JsonNode value = findField(child, names);
|
||||
if (value != null) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String text(JsonNode node, String... names) {
|
||||
for (String name : names) {
|
||||
JsonNode value = node.path(name);
|
||||
if (!value.isMissingNode() && !value.isNull()) {
|
||||
String text = value.asString();
|
||||
if (StringUtils.isNotBlank(text)) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String value(Map<String, Object> values, String key) {
|
||||
Object value = values.get(key);
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
|
||||
private String metadataValue(SourceObject object, String key) {
|
||||
return object.metadata() == null ? null : object.metadata().get(key);
|
||||
}
|
||||
|
||||
private void addOption(List<String> arguments, String name, String value) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
arguments.add(name);
|
||||
arguments.add(value);
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizeExtension(String extension) {
|
||||
if (StringUtils.isBlank(extension)) {
|
||||
return "";
|
||||
}
|
||||
return extension.startsWith(".") ? extension.substring(1).toLowerCase(Locale.ROOT)
|
||||
: extension.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String spaceType(Map<String, Object> config) {
|
||||
String type = value(config, "spaceType");
|
||||
return "mySpace".equals(type) ? "mySpace" : "orgSpace";
|
||||
}
|
||||
|
||||
private void cleanDirectory(Path directory) {
|
||||
if (directory == null) {
|
||||
return;
|
||||
}
|
||||
try (var children = Files.list(directory)) {
|
||||
for (Path child : children.toList()) {
|
||||
Files.deleteIfExists(child);
|
||||
}
|
||||
Files.deleteIfExists(directory);
|
||||
} catch (Exception ignored) {
|
||||
// 下载失败后的临时目录采用尽力清理。
|
||||
}
|
||||
}
|
||||
|
||||
private void putIfPresent(Map<String, String> target, String key, String value) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
target.put(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
package org.dromara.sync.connector.dingtalk;
|
||||
|
||||
/**
|
||||
* DWS 登录完成后返回的最小身份信息。
|
||||
*
|
||||
* @param corpId 组织 ID
|
||||
* @param corpName 组织名称
|
||||
* @param userId 钉钉用户 ID
|
||||
* @param userName 用户名称
|
||||
*/
|
||||
public record DwsAuthIdentity(String corpId, String corpName, String userId, String userName) {
|
||||
|
||||
public String profile() {
|
||||
return corpId + ":" + userId;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,912 @@
|
||||
package org.dromara.sync.connector.dingtalk;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.core.utils.StringUtils;
|
||||
import org.dromara.common.json.utils.JsonUtils;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
import org.dromara.sync.domain.vo.DwsAuthSessionVo;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 管理 DWS Web 设备授权会话。
|
||||
*
|
||||
* <p>DWS 本身是本地 CLI,且设备流需要进程持续轮询。此组件把该进程
|
||||
* 放在后端节点运行,只把一次性授权链接和验证码返回给 Web 页面。会话
|
||||
* 采用内存保存进程句柄,适用于单节点部署;多节点部署时应通过网关会话
|
||||
* 亲和或专用认证节点保证后续状态请求落到同一节点。</p>
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class DwsAuthSessionManager {
|
||||
|
||||
private static final Pattern URL_PATTERN = Pattern.compile("https://[^\\s<>\\\"']+");
|
||||
private static final Pattern USER_CODE_PATTERN = Pattern.compile(
|
||||
"(?i)(?:授权码|user\\s*code|authorization\\s*code|verification\\s*code|code)\\s*[::=]\\s*"
|
||||
+ "([A-Za-z0-9][A-Za-z0-9_-]{2,63})");
|
||||
private static final Pattern POLL_INTERVAL_PATTERN = Pattern.compile(
|
||||
"(?i)(?:每|every)\\s*([1-9]\\d*)\\s*(?:秒|seconds?)");
|
||||
private static final Pattern PROFILE_PATTERN = Pattern.compile("^([^\\s:]+):([^\\s:]+)$");
|
||||
private static final Pattern ANSI_PATTERN = Pattern.compile("\\u001B\\[[;\\d]*m");
|
||||
private static final int MAX_OUTPUT_BYTES = 512 * 1024;
|
||||
private static final int MAX_OUTPUT_LINE_CHARS = 16 * 1024;
|
||||
private static final int MAX_ERROR_LENGTH = 2000;
|
||||
private static final long DEFAULT_TIMEOUT_SECONDS = 600;
|
||||
private static final long DEFAULT_METADATA_WAIT_SECONDS = 15;
|
||||
private static final int DEFAULT_MAX_CONCURRENT = 4;
|
||||
|
||||
private final DwsCommandRunner commandRunner;
|
||||
|
||||
@Value("${sync.dingtalk.auth.enabled:true}")
|
||||
private boolean enabled;
|
||||
|
||||
@Value("${sync.dingtalk.auth.timeout-seconds:600}")
|
||||
private long timeoutSeconds;
|
||||
|
||||
@Value("${sync.dingtalk.auth.metadata-wait-seconds:15}")
|
||||
private long metadataWaitSeconds;
|
||||
|
||||
@Value("${sync.dingtalk.auth.max-concurrent:4}")
|
||||
private int maxConcurrent;
|
||||
|
||||
@Value("${sync.dingtalk.auth.retention-seconds:900}")
|
||||
private long retentionSeconds;
|
||||
|
||||
private final ConcurrentMap<String, Session> sessions = new ConcurrentHashMap<>();
|
||||
private final ConcurrentMap<Long, String> activeByConnection = new ConcurrentHashMap<>();
|
||||
private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();
|
||||
private final ScheduledExecutorService cleanupExecutor = Executors.newSingleThreadScheduledExecutor();
|
||||
private volatile Semaphore permits;
|
||||
|
||||
/**
|
||||
* 初始化并启动过期会话清理任务。
|
||||
*/
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
// Keep operator-provided values useful while preventing an accidental
|
||||
// overflow/indefinite HTTP request from turning a login slot into a
|
||||
// permanent resource leak.
|
||||
timeoutSeconds = boundedPositive(timeoutSeconds, DEFAULT_TIMEOUT_SECONDS, 1800);
|
||||
metadataWaitSeconds = Math.min(timeoutSeconds,
|
||||
boundedPositive(metadataWaitSeconds, DEFAULT_METADATA_WAIT_SECONDS, 30));
|
||||
maxConcurrent = (int) Math.max(1, Math.min(maxConcurrent, 32));
|
||||
retentionSeconds = boundedPositive(retentionSeconds, 900, 86400);
|
||||
permits = new Semaphore(maxConcurrent);
|
||||
cleanupExecutor.scheduleAtFixedRate(this::cleanupFinishedSessions, 1, 1, TimeUnit.MINUTES);
|
||||
}
|
||||
|
||||
/**
|
||||
* 停止所有活动会话并释放执行器。
|
||||
*/
|
||||
@PreDestroy
|
||||
public void destroy() {
|
||||
sessions.values().stream().filter(session -> !session.isTerminal()).forEach(session ->
|
||||
cancelProcess(session, DwsAuthSessionState.CANCELLED, "服务正在关闭,授权会话已取消"));
|
||||
cleanupExecutor.shutdownNow();
|
||||
// Let the watcher finish its finally block and let a just-started
|
||||
// profile write complete before interrupting the virtual-thread pool.
|
||||
// This avoids clearing the connection reference while FINALIZING is
|
||||
// still invoking the persistence callback.
|
||||
executor.shutdown();
|
||||
try {
|
||||
if (!executor.awaitTermination(10, TimeUnit.SECONDS)) {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
executor.shutdownNow();
|
||||
}
|
||||
sessions.values().stream().filter(Session::isTerminal).forEach(Session::clearSensitiveReferences);
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动一个 Web 设备授权会话。
|
||||
*
|
||||
* @param connection 钉钉连接
|
||||
* @param ownerId 发起用户 ID
|
||||
* @param expectedCorpId 可选的组织 ID,用于防止授权到错误组织
|
||||
* @param successHandler 授权成功后的身份持久化回调
|
||||
* @return 会话快照
|
||||
*/
|
||||
public DwsAuthSessionVo start(SyncConnection connection, Long ownerId, String expectedCorpId,
|
||||
Consumer<DwsAuthIdentity> successHandler) {
|
||||
if (!enabled) {
|
||||
throw new ServiceException("Web 钉钉登录功能已被管理员关闭");
|
||||
}
|
||||
if (connection == null || connection.getConnectionId() == null) {
|
||||
throw new ServiceException("钉钉连接必须先保存后才能登录");
|
||||
}
|
||||
String normalizedExpectedCorpId = normalize(expectedCorpId);
|
||||
if (normalizedExpectedCorpId != null && !isSafeIdentityPart(normalizedExpectedCorpId)) {
|
||||
throw new ServiceException("expectedCorpId 格式不正确");
|
||||
}
|
||||
Semaphore currentPermits = permits;
|
||||
if (currentPermits == null || !currentPermits.tryAcquire()) {
|
||||
throw new ServiceException("当前登录请求较多,请稍后重试");
|
||||
}
|
||||
String sessionId = newSessionId();
|
||||
Session session = new Session(sessionId, connection.getConnectionId(), ownerId,
|
||||
normalizedExpectedCorpId, System.currentTimeMillis() + timeoutSeconds * 1000L, successHandler);
|
||||
session.connection = connection;
|
||||
// Put the session and its per-connection index under one lock. A
|
||||
// putIfAbsent followed by sessions.put leaves a small window in which
|
||||
// a concurrent status request can observe an index with no session and
|
||||
// accidentally remove a newly-started login.
|
||||
synchronized (activeByConnection) {
|
||||
String previous = activeByConnection.get(connection.getConnectionId());
|
||||
if (previous != null) {
|
||||
Session existing = sessions.get(previous);
|
||||
if (existing != null && !existing.isTerminal()) {
|
||||
currentPermits.release();
|
||||
throw new ServiceException("该钉钉连接已有一个正在进行的登录会话");
|
||||
}
|
||||
activeByConnection.remove(connection.getConnectionId(), previous);
|
||||
}
|
||||
sessions.put(sessionId, session);
|
||||
activeByConnection.put(connection.getConnectionId(), sessionId);
|
||||
}
|
||||
Process process = null;
|
||||
Future<?> stdoutReader = null;
|
||||
Future<?> stderrReader = null;
|
||||
DwsCommandRunner.ConnectionLockLease authLock = null;
|
||||
boolean handedOff = false;
|
||||
try {
|
||||
authLock = commandRunner.tryAcquireAuthLock(connection);
|
||||
if (authLock == null) {
|
||||
throw new ServiceException("该钉钉连接正在执行同步命令,请等待任务结束后再登录");
|
||||
}
|
||||
// Cancellation can arrive while the short lock acquisition is
|
||||
// waiting for an in-flight download/scan. Publish the lease only
|
||||
// after rechecking the session state; process publication below is
|
||||
// serialized on the same session monitor.
|
||||
synchronized (session) {
|
||||
if (session.isTerminal()) {
|
||||
authLock.close();
|
||||
authLock = null;
|
||||
throw new IllegalStateException("授权会话在取得连接锁前已结束");
|
||||
}
|
||||
session.authLock = authLock;
|
||||
}
|
||||
ProcessBuilder builder = commandRunner.newProcessBuilderWithoutProfile(connection, List.of(
|
||||
"auth", "login", "--device", "--no-browser", "--yes", "--format", "json"));
|
||||
/*
|
||||
* Serialize process publication with cancel/expire/destroy. The
|
||||
* session is visible before the CLI is started so a duplicate
|
||||
* login can be rejected, but that also means a concurrent cancel
|
||||
* can otherwise observe a null process and let an orphan process
|
||||
* escape. Holding this lock through publication closes that
|
||||
* window: cancellation either happens before start, or sees and
|
||||
* stops the fully-published process.
|
||||
*/
|
||||
synchronized (session) {
|
||||
if (session.isTerminal()) {
|
||||
throw new IllegalStateException("授权会话在 DWS 进程启动前已结束");
|
||||
}
|
||||
process = builder.start();
|
||||
process.getOutputStream().close();
|
||||
session.process = process;
|
||||
Process runningProcess = process;
|
||||
Future<?> startedStdoutReader = executor.submit(
|
||||
() -> readStream(session, runningProcess.getInputStream(), false));
|
||||
Future<?> startedStderrReader = executor.submit(
|
||||
() -> readStream(session, runningProcess.getErrorStream(), true));
|
||||
stdoutReader = startedStdoutReader;
|
||||
stderrReader = startedStderrReader;
|
||||
Future<?> task = executor.submit(
|
||||
() -> awaitProcess(session, runningProcess, startedStdoutReader, startedStderrReader));
|
||||
session.task = task;
|
||||
// From this point awaitProcess owns the process and semaphore.
|
||||
// A later HTTP response error must not release the same permit
|
||||
// twice. The lock also guarantees cancel cannot run between
|
||||
// process publication and this hand-off marker.
|
||||
handedOff = true;
|
||||
}
|
||||
|
||||
// 通常 DWS 会在首次请求完成后立即打印二维码/链接;短暂等待可以让
|
||||
// 前端首个响应直接得到可展示的数据,网络慢时则返回 STARTING。
|
||||
try {
|
||||
session.metadataReady.await(metadataWaitSeconds, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return snapshot(session);
|
||||
} catch (Exception e) {
|
||||
if (!handedOff) {
|
||||
cancelFuture(stdoutReader);
|
||||
cancelFuture(stderrReader);
|
||||
commandRunner.stop(process);
|
||||
session.process = null;
|
||||
sessions.remove(sessionId, session);
|
||||
activeByConnection.remove(connection.getConnectionId(), sessionId);
|
||||
session.clearOutputs();
|
||||
session.clearSensitiveReferences();
|
||||
DwsCommandRunner.ConnectionLockLease publishedLock = session.authLock;
|
||||
session.authLock = null;
|
||||
if (publishedLock != null) {
|
||||
publishedLock.close();
|
||||
} else if (authLock != null) {
|
||||
authLock.close();
|
||||
}
|
||||
currentPermits.release();
|
||||
}
|
||||
if (e instanceof ServiceException serviceException) {
|
||||
throw serviceException;
|
||||
}
|
||||
throw new ServiceException("无法启动 DWS Web 登录,请确认服务器已安装可执行的 DWS", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取指定用户的授权会话状态。
|
||||
*
|
||||
* @param connectionId 连接 ID
|
||||
* @param sessionId 会话 ID
|
||||
* @param ownerId 当前用户 ID
|
||||
* @return 会话快照
|
||||
*/
|
||||
public DwsAuthSessionVo get(Long connectionId, String sessionId, Long ownerId) {
|
||||
return get(connectionId, sessionId, ownerId, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询会话状态,可由超级管理员接管查看其他用户发起的会话。
|
||||
*
|
||||
* @param connectionId 连接 ID
|
||||
* @param sessionId 会话 ID
|
||||
* @param ownerId 当前用户 ID
|
||||
* @param privileged 是否为超级管理员
|
||||
* @return 会话快照
|
||||
*/
|
||||
public DwsAuthSessionVo get(Long connectionId, String sessionId, Long ownerId, boolean privileged) {
|
||||
Session session = requireSession(connectionId, sessionId, ownerId, privileged);
|
||||
if (!session.isTerminal() && System.currentTimeMillis() >= session.expiresAt) {
|
||||
expire(session);
|
||||
}
|
||||
return snapshot(session);
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消指定用户的授权会话。
|
||||
*
|
||||
* @param connectionId 连接 ID
|
||||
* @param sessionId 会话 ID
|
||||
* @param ownerId 当前用户 ID
|
||||
* @return 会话快照
|
||||
*/
|
||||
public DwsAuthSessionVo cancel(Long connectionId, String sessionId, Long ownerId) {
|
||||
return cancel(connectionId, sessionId, ownerId, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消会话,可由超级管理员接管取消其他用户发起的会话。
|
||||
*
|
||||
* @param connectionId 连接 ID
|
||||
* @param sessionId 会话 ID
|
||||
* @param ownerId 当前用户 ID
|
||||
* @param privileged 是否为超级管理员
|
||||
* @return 取消后的会话快照
|
||||
*/
|
||||
public DwsAuthSessionVo cancel(Long connectionId, String sessionId, Long ownerId, boolean privileged) {
|
||||
Session session = requireSession(connectionId, sessionId, ownerId, privileged);
|
||||
if (!session.isTerminal()) {
|
||||
cancelProcess(session, DwsAuthSessionState.CANCELLED, "已取消钉钉登录");
|
||||
}
|
||||
return snapshot(session);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断连接是否仍有一个 Web 登录会话占用登录态目录。
|
||||
*
|
||||
* <p>编辑或删除连接前调用此方法可以避免回调把 profile 写回已经
|
||||
* 变更/删除的记录。</p>
|
||||
*/
|
||||
public boolean hasActive(Long connectionId) {
|
||||
if (connectionId == null) {
|
||||
return false;
|
||||
}
|
||||
String sessionId = activeByConnection.get(connectionId);
|
||||
if (sessionId == null) {
|
||||
return false;
|
||||
}
|
||||
Session session = sessions.get(sessionId);
|
||||
if (session == null || session.isTerminal()) {
|
||||
activeByConnection.remove(connectionId, sessionId);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private void awaitProcess(Session session, Process process, Future<?> stdoutReader, Future<?> stderrReader) {
|
||||
try {
|
||||
if (!process.waitFor(Math.max(1, timeoutSeconds), TimeUnit.SECONDS)) {
|
||||
expire(session);
|
||||
return;
|
||||
}
|
||||
waitReader(stdoutReader);
|
||||
waitReader(stderrReader);
|
||||
if (session.state == DwsAuthSessionState.CANCELLED
|
||||
|| session.state == DwsAuthSessionState.EXPIRED
|
||||
|| session.state == DwsAuthSessionState.FINALIZING
|
||||
|| session.state == DwsAuthSessionState.SUCCESS) {
|
||||
return;
|
||||
}
|
||||
if (process.exitValue() != 0) {
|
||||
fail(session, diagnostic(session.errorOutput(), session.stdoutOutput(),
|
||||
"DWS 登录命令执行失败"));
|
||||
return;
|
||||
}
|
||||
complete(session);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
if (!session.isTerminal()) {
|
||||
fail(session, "DWS 登录进程被中断");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (!session.isTerminal()) {
|
||||
fail(session, diagnostic(e.getMessage(), null, "DWS 登录失败"));
|
||||
}
|
||||
} finally {
|
||||
// A shutdown/interruption can happen while waitFor is blocked.
|
||||
// Always terminate the native process before dropping the handle;
|
||||
// otherwise the watcher could leave an orphan DWS process behind.
|
||||
commandRunner.stop(process);
|
||||
Semaphore currentPermits = permits;
|
||||
if (currentPermits != null) {
|
||||
currentPermits.release();
|
||||
}
|
||||
activeByConnection.remove(session.connectionId, session.id);
|
||||
session.process = null;
|
||||
session.terminalAt = System.currentTimeMillis();
|
||||
// CLI output can contain the one-time user code and verification
|
||||
// URL. Do not retain it for the whole terminal-session TTL.
|
||||
session.clearOutputs();
|
||||
session.clearSensitiveReferences();
|
||||
cancelFuture(stdoutReader);
|
||||
cancelFuture(stderrReader);
|
||||
DwsCommandRunner.ConnectionLockLease authLock = session.authLock;
|
||||
session.authLock = null;
|
||||
if (authLock != null) {
|
||||
authLock.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void complete(Session session) {
|
||||
synchronized (session) {
|
||||
if (session.isTerminal() || session.state == DwsAuthSessionState.FINALIZING) {
|
||||
return;
|
||||
}
|
||||
// Once DWS has returned success, make cancellation deterministic:
|
||||
// the session is finalizing its database write and cannot be
|
||||
// changed back to CANCELLED halfway through that write.
|
||||
session.state = DwsAuthSessionState.FINALIZING;
|
||||
session.clearDevicePrompt();
|
||||
session.metadataReady.countDown();
|
||||
}
|
||||
|
||||
String stdout = session.stdoutOutput();
|
||||
DwsAuthIdentity identity = parseIdentity(stdout);
|
||||
session.clearOutputs();
|
||||
if (identity == null) {
|
||||
fail(session, "DWS 登录成功但未返回完整 corpId:userId,请固定已验收版本后重试");
|
||||
return;
|
||||
}
|
||||
if (StringUtils.isNotBlank(session.expectedCorpId)
|
||||
&& !Objects.equals(session.expectedCorpId, identity.corpId())) {
|
||||
fail(session, "授权组织与填写的组织 ID 不一致,请确认后重试");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (session.successHandler != null) {
|
||||
session.successHandler.accept(identity);
|
||||
}
|
||||
synchronized (session) {
|
||||
if (session.state != DwsAuthSessionState.FINALIZING) {
|
||||
return;
|
||||
}
|
||||
session.corpId = identity.corpId();
|
||||
session.corpName = identity.corpName();
|
||||
session.userId = identity.userId();
|
||||
session.userName = identity.userName();
|
||||
session.profile = identity.profile();
|
||||
session.message = "钉钉登录成功,已保存登录态";
|
||||
session.state = DwsAuthSessionState.SUCCESS;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
fail(session, diagnostic(e.getMessage(), null, "登录成功,但保存连接身份失败"));
|
||||
}
|
||||
}
|
||||
|
||||
private void readStream(Session session, InputStream inputStream, boolean errorStream) {
|
||||
try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
|
||||
char[] buffer = new char[4096];
|
||||
StringBuilder line = new StringBuilder(Math.min(MAX_OUTPUT_LINE_CHARS, 4096));
|
||||
int length;
|
||||
while ((length = reader.read(buffer)) >= 0) {
|
||||
for (int index = 0; index < length; index++) {
|
||||
char character = buffer[index];
|
||||
if (character == '\n') {
|
||||
appendOutputLine(session, line.toString(), errorStream);
|
||||
line.setLength(0);
|
||||
} else if (character != '\r') {
|
||||
if (line.length() < MAX_OUTPUT_LINE_CHARS) {
|
||||
line.append(character);
|
||||
} else {
|
||||
// Flush bounded chunks from a pathological line so
|
||||
// read() never accumulates unbounded attacker/CLI
|
||||
// output in memory. Prompt lines are far shorter.
|
||||
appendOutputLine(session, line.toString(), errorStream);
|
||||
line.setLength(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (line.length() > 0) {
|
||||
appendOutputLine(session, line.toString(), errorStream);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
if (!session.isTerminal()) {
|
||||
session.followUpError = diagnostic(e.getMessage(), session.followUpError, "读取 DWS 登录输出失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void appendOutputLine(Session session, String line, boolean errorStream) {
|
||||
String clean = stripAnsi(line);
|
||||
if (errorStream) {
|
||||
session.appendError(clean);
|
||||
} else {
|
||||
session.appendStdout(clean);
|
||||
}
|
||||
parseDevicePrompt(session, clean);
|
||||
}
|
||||
|
||||
private void parseDevicePrompt(Session session, String line) {
|
||||
if (StringUtils.isBlank(line)) {
|
||||
return;
|
||||
}
|
||||
synchronized (session) {
|
||||
if (session.isTerminal() || session.state == DwsAuthSessionState.FINALIZING) {
|
||||
return;
|
||||
}
|
||||
Matcher urlMatcher = URL_PATTERN.matcher(line);
|
||||
while (urlMatcher.find()) {
|
||||
String url = trimUrl(urlMatcher.group());
|
||||
if (!isAllowedVerificationUrl(url)) {
|
||||
continue;
|
||||
}
|
||||
if (url.contains("?") || url.contains("&")) {
|
||||
session.verificationUriComplete = url;
|
||||
} else if (StringUtils.isBlank(session.verificationUri)) {
|
||||
session.verificationUri = url;
|
||||
}
|
||||
}
|
||||
Matcher codeMatcher = USER_CODE_PATTERN.matcher(line);
|
||||
if (codeMatcher.find()) {
|
||||
session.userCode = codeMatcher.group(1);
|
||||
}
|
||||
Matcher intervalMatcher = POLL_INTERVAL_PATTERN.matcher(line);
|
||||
if (intervalMatcher.find()) {
|
||||
try {
|
||||
int interval = Integer.parseInt(intervalMatcher.group(1));
|
||||
session.pollInterval = Math.max(1, Math.min(interval, 30));
|
||||
} catch (NumberFormatException ignored) {
|
||||
// Keep the safe default when a localized line is malformed.
|
||||
}
|
||||
}
|
||||
if (StringUtils.isNotBlank(session.verificationUriComplete)
|
||||
|| (StringUtils.isNotBlank(session.verificationUri) && StringUtils.isNotBlank(session.userCode))) {
|
||||
if (StringUtils.isBlank(session.verificationUri)) {
|
||||
session.verificationUri = session.verificationUriComplete;
|
||||
}
|
||||
session.state = DwsAuthSessionState.WAITING_USER;
|
||||
session.metadataReady.countDown();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private DwsAuthIdentity parseIdentity(String output) {
|
||||
if (StringUtils.isBlank(output)) {
|
||||
return null;
|
||||
}
|
||||
String json = extractJson(stripAnsi(output));
|
||||
if (json == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
JsonNode root = JsonUtils.getJsonMapper().readTree(json);
|
||||
return findIdentity(root);
|
||||
} catch (Exception ignored) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private DwsAuthIdentity findIdentity(JsonNode node) {
|
||||
if (node == null || node.isNull()) {
|
||||
return null;
|
||||
}
|
||||
if (node.isObject()) {
|
||||
// Prefer explicit IDs. A profile/display name may contain a
|
||||
// colon too (for example "企业名:张三") and must not outrank the
|
||||
// authoritative corp_id/user_id fields returned by DWS.
|
||||
String corpId = text(node, "corpId", "corp_id", "organizationId", "orgId", "org_id");
|
||||
String userId = text(node, "userId", "user_id", "userid");
|
||||
if (StringUtils.isNotBlank(corpId) && StringUtils.isNotBlank(userId)) {
|
||||
DwsAuthIdentity identity = validIdentity(corpId, text(node, "corpName", "corp_name", "organizationName"),
|
||||
userId, text(node, "userName", "user_name", "name", "nick"));
|
||||
if (identity != null) {
|
||||
return identity;
|
||||
}
|
||||
}
|
||||
String profile = text(node, "profile", "account", "accountProfile");
|
||||
DwsAuthIdentity byProfile = identityFromProfile(profile, node);
|
||||
if (byProfile != null) {
|
||||
return byProfile;
|
||||
}
|
||||
for (var field : node.properties()) {
|
||||
DwsAuthIdentity nested = findIdentity(field.getValue());
|
||||
if (nested != null) {
|
||||
return nested;
|
||||
}
|
||||
}
|
||||
} else if (node.isArray()) {
|
||||
for (JsonNode child : node) {
|
||||
DwsAuthIdentity nested = findIdentity(child);
|
||||
if (nested != null) {
|
||||
return nested;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private DwsAuthIdentity identityFromProfile(String profile, JsonNode source) {
|
||||
if (StringUtils.isBlank(profile)) {
|
||||
return null;
|
||||
}
|
||||
Matcher matcher = PROFILE_PATTERN.matcher(profile.trim());
|
||||
if (!matcher.matches()) {
|
||||
return null;
|
||||
}
|
||||
String corpId = matcher.group(1);
|
||||
String userId = matcher.group(2);
|
||||
return validIdentity(corpId,
|
||||
source == null ? null : text(source, "corpName", "corp_name", "organizationName"),
|
||||
userId,
|
||||
source == null ? null : text(source, "userName", "user_name", "name", "nick"));
|
||||
}
|
||||
|
||||
private DwsAuthIdentity validIdentity(String corpId, String corpName, String userId, String userName) {
|
||||
if (!isSafeIdentityPart(corpId) || !isSafeIdentityPart(userId)) {
|
||||
return null;
|
||||
}
|
||||
return new DwsAuthIdentity(corpId, normalize(corpName), userId, normalize(userName));
|
||||
}
|
||||
|
||||
private boolean isSafeIdentityPart(String value) {
|
||||
return StringUtils.isNotBlank(value)
|
||||
&& value.length() <= 255
|
||||
&& value.chars().noneMatch(character -> Character.isWhitespace(character)
|
||||
|| Character.isISOControl(character))
|
||||
&& value.indexOf(':') < 0
|
||||
&& value.indexOf('/') < 0
|
||||
&& value.indexOf('\\') < 0
|
||||
&& value.indexOf('?') < 0
|
||||
&& value.indexOf('&') < 0
|
||||
&& value.indexOf('"') < 0
|
||||
&& value.indexOf('\'') < 0;
|
||||
}
|
||||
|
||||
private Session requireSession(Long connectionId, String sessionId, Long ownerId, boolean privileged) {
|
||||
if (connectionId == null || StringUtils.isBlank(sessionId)) {
|
||||
throw new ServiceException("授权会话参数不能为空");
|
||||
}
|
||||
Session session = sessions.get(sessionId);
|
||||
if (session == null || !Objects.equals(session.connectionId, connectionId)
|
||||
|| (!privileged && !Objects.equals(session.ownerId, ownerId))) {
|
||||
throw new ServiceException("授权会话不存在或已失效");
|
||||
}
|
||||
return session;
|
||||
}
|
||||
|
||||
private void expire(Session session) {
|
||||
if (session.isTerminal()) {
|
||||
return;
|
||||
}
|
||||
cancelProcess(session, DwsAuthSessionState.EXPIRED, "授权链接已过期,请重新发起登录");
|
||||
}
|
||||
|
||||
private void fail(Session session, String message) {
|
||||
synchronized (session) {
|
||||
if (session.isTerminal()) {
|
||||
return;
|
||||
}
|
||||
session.state = DwsAuthSessionState.FAILED;
|
||||
session.message = diagnostic(message, null, "DWS 登录失败,请重试");
|
||||
session.clearDevicePrompt();
|
||||
session.metadataReady.countDown();
|
||||
session.clearSensitiveReferences();
|
||||
}
|
||||
}
|
||||
|
||||
private void cancelProcess(Session session, DwsAuthSessionState state, String message) {
|
||||
synchronized (session) {
|
||||
if (session.isTerminal() || session.state == DwsAuthSessionState.FINALIZING) {
|
||||
return;
|
||||
}
|
||||
session.state = state;
|
||||
session.message = message;
|
||||
session.clearDevicePrompt();
|
||||
Process process = session.process;
|
||||
if (process != null) {
|
||||
commandRunner.stop(process);
|
||||
}
|
||||
session.metadataReady.countDown();
|
||||
session.clearOutputs();
|
||||
session.clearSensitiveReferences();
|
||||
}
|
||||
}
|
||||
|
||||
private DwsAuthSessionVo snapshot(Session session) {
|
||||
long remaining = Math.max(0, (session.expiresAt - System.currentTimeMillis()) / 1000L);
|
||||
if (session.isTerminal() && session.state != DwsAuthSessionState.SUCCESS) {
|
||||
remaining = 0;
|
||||
}
|
||||
boolean exposeDevicePrompt = session.state == DwsAuthSessionState.STARTING
|
||||
|| session.state == DwsAuthSessionState.WAITING_USER;
|
||||
return new DwsAuthSessionVo(
|
||||
session.id,
|
||||
session.connectionId,
|
||||
session.state.name(),
|
||||
exposeDevicePrompt ? session.verificationUri : null,
|
||||
exposeDevicePrompt ? session.verificationUriComplete : null,
|
||||
exposeDevicePrompt ? session.userCode : null,
|
||||
session.expiresAt,
|
||||
remaining,
|
||||
session.pollInterval,
|
||||
session.corpId,
|
||||
session.corpName,
|
||||
session.userId,
|
||||
session.userName,
|
||||
session.profile,
|
||||
session.message);
|
||||
}
|
||||
|
||||
private void cleanupFinishedSessions() {
|
||||
long threshold = System.currentTimeMillis() - retentionSeconds * 1000L;
|
||||
sessions.entrySet().removeIf(entry -> {
|
||||
Session session = entry.getValue();
|
||||
return session.isTerminal() && session.terminalAt > 0 && session.terminalAt < threshold;
|
||||
});
|
||||
}
|
||||
|
||||
private void waitReader(Future<?> reader) throws Exception {
|
||||
if (reader != null) {
|
||||
reader.get(15, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
private void cancelFuture(Future<?> future) {
|
||||
if (future != null && !future.isDone()) {
|
||||
future.cancel(true);
|
||||
}
|
||||
}
|
||||
|
||||
private String diagnostic(String primary, String secondary, String fallback) {
|
||||
String value = StringUtils.firstNonBlank(primary, secondary, fallback);
|
||||
if (value == null) {
|
||||
return fallback;
|
||||
}
|
||||
String sanitized = value
|
||||
.replaceAll("(?i)(access[_-]?token|refresh[_-]?token|device[_-]?code|client[_-]?secret|app[_-]?secret)"
|
||||
+ "([\\\"'\\s:=]+)[^\\s,;\\\"']+", "$1$2***")
|
||||
.replaceAll("(?i)(授权码|user\\s*code|authorization\\s*code|verification\\s*code|code)"
|
||||
+ "\\s*[::=]\\s*[A-Za-z0-9][A-Za-z0-9_-]{2,63}", "$1:***")
|
||||
.replaceAll("(?i)https?://[^\\s\\\"']+", "[授权链接已隐藏]")
|
||||
.trim();
|
||||
return sanitized.length() <= MAX_ERROR_LENGTH ? sanitized : sanitized.substring(0, MAX_ERROR_LENGTH);
|
||||
}
|
||||
|
||||
private String extractJson(String output) {
|
||||
int start = output.indexOf('{');
|
||||
int end = output.lastIndexOf('}');
|
||||
if (start < 0 || end <= start) {
|
||||
start = output.indexOf('[');
|
||||
end = output.lastIndexOf(']');
|
||||
}
|
||||
return start >= 0 && end > start ? output.substring(start, end + 1) : null;
|
||||
}
|
||||
|
||||
private String text(JsonNode node, String... names) {
|
||||
for (String name : names) {
|
||||
JsonNode value = node.path(name);
|
||||
if ((value.isTextual() || value.isNumber()) && StringUtils.isNotBlank(value.asString())) {
|
||||
return value.asString().trim();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String trimUrl(String value) {
|
||||
return value.replaceAll("[\\.,;:!?\\)\\]\\},。;:!?)】》]+$", "");
|
||||
}
|
||||
|
||||
private boolean isAllowedVerificationUrl(String value) {
|
||||
try {
|
||||
URI uri = URI.create(value);
|
||||
String host = uri.getHost();
|
||||
if (!"https".equalsIgnoreCase(uri.getScheme()) || StringUtils.isBlank(host)) {
|
||||
return false;
|
||||
}
|
||||
String normalizedHost = host.toLowerCase(Locale.ROOT);
|
||||
return "dingtalk.com".equals(normalizedHost)
|
||||
|| normalizedHost.endsWith(".dingtalk.com");
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private String stripAnsi(String value) {
|
||||
return value == null ? null : ANSI_PATTERN.matcher(value).replaceAll("");
|
||||
}
|
||||
|
||||
private String normalize(String value) {
|
||||
return StringUtils.isBlank(value) ? null : value.trim();
|
||||
}
|
||||
|
||||
private long positive(long value, long fallback) {
|
||||
return value > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
private long boundedPositive(long value, long fallback, long maximum) {
|
||||
return Math.min(positive(value, fallback), maximum);
|
||||
}
|
||||
|
||||
private String newSessionId() {
|
||||
byte[] bytes = new byte[24];
|
||||
SecureRandomHolder.INSTANCE.nextBytes(bytes);
|
||||
return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes);
|
||||
}
|
||||
|
||||
private static final class SecureRandomHolder {
|
||||
private static final java.security.SecureRandom INSTANCE = new java.security.SecureRandom();
|
||||
}
|
||||
|
||||
private static final class Session {
|
||||
private final String id;
|
||||
private final Long connectionId;
|
||||
private final Long ownerId;
|
||||
private final String expectedCorpId;
|
||||
private final long expiresAt;
|
||||
private final CountDownLatch metadataReady = new CountDownLatch(1);
|
||||
private final LimitedOutput stdout = new LimitedOutput();
|
||||
private final LimitedOutput stderr = new LimitedOutput();
|
||||
private volatile SyncConnection connection;
|
||||
private volatile Process process;
|
||||
private volatile Future<?> task;
|
||||
private volatile DwsCommandRunner.ConnectionLockLease authLock;
|
||||
private volatile Consumer<DwsAuthIdentity> successHandler;
|
||||
private volatile DwsAuthSessionState state = DwsAuthSessionState.STARTING;
|
||||
private volatile String verificationUri;
|
||||
private volatile String verificationUriComplete;
|
||||
private volatile String userCode;
|
||||
private volatile int pollInterval = 2;
|
||||
private volatile String corpId;
|
||||
private volatile String corpName;
|
||||
private volatile String userId;
|
||||
private volatile String userName;
|
||||
private volatile String profile;
|
||||
private volatile String message;
|
||||
private volatile String followUpError;
|
||||
private volatile long terminalAt;
|
||||
|
||||
private Session(String id, Long connectionId, Long ownerId, String expectedCorpId,
|
||||
long expiresAt, Consumer<DwsAuthIdentity> successHandler) {
|
||||
this.id = id;
|
||||
this.connectionId = connectionId;
|
||||
this.ownerId = ownerId;
|
||||
this.expectedCorpId = expectedCorpId;
|
||||
this.expiresAt = expiresAt;
|
||||
this.successHandler = successHandler;
|
||||
}
|
||||
|
||||
private boolean isTerminal() {
|
||||
return switch (state) {
|
||||
case SUCCESS, FAILED, EXPIRED, CANCELLED -> true;
|
||||
default -> false;
|
||||
};
|
||||
}
|
||||
|
||||
private void appendStdout(String line) {
|
||||
stdout.append(line);
|
||||
}
|
||||
|
||||
private void appendError(String line) {
|
||||
stderr.append(line);
|
||||
}
|
||||
|
||||
private String stdoutOutput() {
|
||||
return stdout.value();
|
||||
}
|
||||
|
||||
private String errorOutput() {
|
||||
return stderr.value();
|
||||
}
|
||||
|
||||
private void clearOutputs() {
|
||||
stdout.clear();
|
||||
stderr.clear();
|
||||
}
|
||||
|
||||
private void clearDevicePrompt() {
|
||||
verificationUri = null;
|
||||
verificationUriComplete = null;
|
||||
userCode = null;
|
||||
}
|
||||
|
||||
private void clearSensitiveReferences() {
|
||||
connection = null;
|
||||
successHandler = null;
|
||||
task = null;
|
||||
}
|
||||
}
|
||||
|
||||
private static final class LimitedOutput {
|
||||
private final StringBuilder value = new StringBuilder();
|
||||
private int bytes;
|
||||
|
||||
private synchronized void append(String line) {
|
||||
if (bytes >= MAX_OUTPUT_BYTES) {
|
||||
return;
|
||||
}
|
||||
String text = line + System.lineSeparator();
|
||||
byte[] encoded = text.getBytes(StandardCharsets.UTF_8);
|
||||
int remaining = MAX_OUTPUT_BYTES - bytes;
|
||||
if (encoded.length <= remaining) {
|
||||
value.append(text);
|
||||
bytes += encoded.length;
|
||||
} else {
|
||||
value.append(new String(encoded, 0, remaining, StandardCharsets.UTF_8));
|
||||
bytes = MAX_OUTPUT_BYTES;
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized String value() {
|
||||
return value.toString();
|
||||
}
|
||||
|
||||
private synchronized void clear() {
|
||||
value.setLength(0);
|
||||
bytes = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
package org.dromara.sync.connector.dingtalk;
|
||||
|
||||
/**
|
||||
* DWS Web 设备授权会话状态。
|
||||
*/
|
||||
enum DwsAuthSessionState {
|
||||
STARTING,
|
||||
WAITING_USER,
|
||||
/** DWS 已完成授权,后端正在把身份写回连接配置。 */
|
||||
FINALIZING,
|
||||
SUCCESS,
|
||||
FAILED,
|
||||
EXPIRED,
|
||||
CANCELLED
|
||||
}
|
||||
@ -0,0 +1,459 @@
|
||||
package org.dromara.sync.connector.dingtalk;
|
||||
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.core.utils.StringUtils;
|
||||
import org.dromara.common.json.utils.JsonUtils;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.LinkOption;
|
||||
import java.nio.file.attribute.PosixFilePermission;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.locks.StampedLock;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 安全执行 DingTalk Workspace CLI,不通过命令行参数传递凭证。
|
||||
*/
|
||||
@Component
|
||||
class DwsCommandRunner {
|
||||
|
||||
private static final int MAX_CAPTURE_BYTES = 8 * 1024 * 1024;
|
||||
private static final long COMMAND_LOCK_WAIT_MILLIS = 2000;
|
||||
private static final long AUTH_LOCK_WAIT_MILLIS = 1000;
|
||||
/**
|
||||
* Environment variables needed by a native CLI/network stack. In
|
||||
* particular, do not inherit the whole Spring process environment: it may
|
||||
* contain database, object-storage, or encryption secrets.
|
||||
*/
|
||||
private static final List<String> SAFE_ENVIRONMENT_KEYS = List.of(
|
||||
"PATH", "USER", "LOGNAME", "LANG", "LC_ALL", "TZ", "TMPDIR", "TMP", "TEMP",
|
||||
"HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY",
|
||||
"http_proxy", "https_proxy", "all_proxy", "no_proxy",
|
||||
"SSL_CERT_FILE", "SSL_CERT_DIR", "DWS_LANG", "DWS_TRUSTED_DOMAINS",
|
||||
"DWS_ALLOW_HTTP_ENDPOINTS", "DWS_CHANNEL", "DWS_AGENT_PRODUCT");
|
||||
|
||||
@Value("${sync.dingtalk.dws-executable:dws}")
|
||||
private String executable;
|
||||
|
||||
@Value("${sync.dingtalk.profile-root:}")
|
||||
private String profileRoot;
|
||||
|
||||
/**
|
||||
* DWS 加密 keychain 的根目录。DWS_CONFIG_DIR 只隔离配置文件,不能保证
|
||||
* 不同连接之间的 token/key 不串用,因此这里也按连接隔离 keychain。
|
||||
*/
|
||||
@Value("${sync.dingtalk.keychain-root:}")
|
||||
private String keychainRoot;
|
||||
|
||||
/**
|
||||
* A DWS config/keychain directory is mutable state. Keep business
|
||||
* commands as readers so normal parallel downloads remain possible, while
|
||||
* a Web login takes the exclusive writer lock for the whole device flow.
|
||||
* This is intentionally JVM-local; a multi-node deployment still needs
|
||||
* sticky routing or a distributed lock around DWS operations.
|
||||
*/
|
||||
private final ConcurrentMap<Long, StampedLock> connectionLocks = new ConcurrentHashMap<>();
|
||||
|
||||
DwsCommandResult run(SyncConnection connection, List<String> arguments, Duration timeout) {
|
||||
return run(connection, arguments, timeout, null);
|
||||
}
|
||||
|
||||
DwsCommandResult run(SyncConnection connection, List<String> arguments, Duration timeout, Path workingDirectory) {
|
||||
return runInternal(connection, arguments, timeout, workingDirectory, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行不需要 profile 的 DWS 命令(例如 auth status/profile list)。
|
||||
*
|
||||
* @param connection 连接
|
||||
* @param arguments 命令参数
|
||||
* @param timeout 超时时间
|
||||
* @return 命令结果
|
||||
*/
|
||||
DwsCommandResult runWithoutProfile(SyncConnection connection, List<String> arguments, Duration timeout) {
|
||||
return runInternal(connection, arguments, timeout, null, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 尝试为 Web 登录取得连接级独占锁。
|
||||
*
|
||||
* @param connection 连接
|
||||
* @return 成功时的锁租约,忙时返回 {@code null}
|
||||
*/
|
||||
ConnectionLockLease tryAcquireAuthLock(SyncConnection connection) {
|
||||
return tryAcquire(connection, true, AUTH_LOCK_WAIT_MILLIS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 为 Web 设备登录创建进程构造器。该进程不会带 profile,登录完成后
|
||||
* 才能由 DWS 返回实际的 corpId:userId。
|
||||
*
|
||||
* @param connection 连接
|
||||
* @param arguments 命令参数
|
||||
* @return 已设置安全环境变量的进程构造器
|
||||
*/
|
||||
ProcessBuilder newProcessBuilderWithoutProfile(SyncConnection connection, List<String> arguments) {
|
||||
return buildProcessBuilder(connection, arguments, null, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 终止正在运行的 DWS 子进程及其子孙进程。
|
||||
*
|
||||
* @param process 进程
|
||||
*/
|
||||
void stop(Process process) {
|
||||
terminate(process);
|
||||
}
|
||||
|
||||
private DwsCommandResult runInternal(SyncConnection connection, List<String> arguments, Duration timeout,
|
||||
Path workingDirectory, boolean includeProfile) {
|
||||
Map<String, Object> secret = parseJson(connection.getSecretJson());
|
||||
String clientId = value(secret, "clientId", null);
|
||||
String clientSecret = value(secret, "clientSecret", null);
|
||||
if (StringUtils.isAnyBlank(clientId, clientSecret)) {
|
||||
throw new ServiceException("钉钉连接 clientId、clientSecret 不能为空");
|
||||
}
|
||||
if (StringUtils.isBlank(executable)) {
|
||||
throw new ServiceException("sync.dingtalk.dws-executable 不能为空");
|
||||
}
|
||||
ExecutorService executor = null;
|
||||
ConnectionLockLease commandLock = null;
|
||||
Process process = null;
|
||||
try {
|
||||
commandLock = tryAcquire(connection, false, COMMAND_LOCK_WAIT_MILLIS);
|
||||
if (commandLock == null) {
|
||||
throw new ServiceException("钉钉连接正在进行 Web 登录,请登录结束后重试");
|
||||
}
|
||||
executor = java.util.concurrent.Executors.newVirtualThreadPerTaskExecutor();
|
||||
ProcessBuilder processBuilder = buildProcessBuilder(connection, arguments, workingDirectory, includeProfile);
|
||||
process = processBuilder.start();
|
||||
process.getOutputStream().close();
|
||||
Process startedProcess = process;
|
||||
Future<CapturedOutput> stdoutFuture = executor.submit(() -> read(startedProcess.getInputStream()));
|
||||
Future<CapturedOutput> stderrFuture = executor.submit(() -> read(startedProcess.getErrorStream()));
|
||||
if (!process.waitFor(timeout.toMillis(), TimeUnit.MILLISECONDS)) {
|
||||
terminate(process);
|
||||
throw new ServiceException("DWS 命令执行超时({} 秒)", timeout.toSeconds());
|
||||
}
|
||||
CapturedOutput stdoutResult = stdoutFuture.get(30, TimeUnit.SECONDS);
|
||||
CapturedOutput stderrResult = stderrFuture.get(30, TimeUnit.SECONDS);
|
||||
if (stdoutResult.truncated() || stderrResult.truncated()) {
|
||||
throw new ServiceException("DWS 命令输出超过 {} MiB 安全上限", MAX_CAPTURE_BYTES / 1024 / 1024);
|
||||
}
|
||||
String stdout = stdoutResult.value();
|
||||
String stderr = stderrResult.value();
|
||||
if (process.exitValue() != 0) {
|
||||
String message = StringUtils.isNotBlank(stderr) ? stderr : stdout;
|
||||
throw new ServiceException("DWS 命令执行失败:{}",
|
||||
abbreviate(sanitizeDiagnostic(message, clientId, clientSecret)));
|
||||
}
|
||||
return new DwsCommandResult(stdout, stderr);
|
||||
} catch (ServiceException e) {
|
||||
throw e;
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new ServiceException("DWS 命令执行被中断", e);
|
||||
} catch (Exception e) {
|
||||
throw new ServiceException("无法执行 DWS,请确认 DingTalk Workspace CLI 已安装并可用", e);
|
||||
} finally {
|
||||
terminate(process);
|
||||
if (executor != null) {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
if (commandLock != null) {
|
||||
commandLock.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private ConnectionLockLease tryAcquire(SyncConnection connection, boolean exclusive, long waitMillis) {
|
||||
if (connection == null || connection.getConnectionId() == null) {
|
||||
throw new ServiceException("钉钉连接必须先保存后才能执行 DWS 命令");
|
||||
}
|
||||
StampedLock lock = connectionLocks.computeIfAbsent(connection.getConnectionId(),
|
||||
ignored -> new StampedLock());
|
||||
try {
|
||||
long stamp = exclusive
|
||||
? lock.tryWriteLock(waitMillis, TimeUnit.MILLISECONDS)
|
||||
: lock.tryReadLock(waitMillis, TimeUnit.MILLISECONDS);
|
||||
if (stamp == 0L) {
|
||||
return null;
|
||||
}
|
||||
return new ConnectionLockLease(lock, stamp, exclusive);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new ServiceException("等待钉钉连接执行锁时被中断", e);
|
||||
}
|
||||
}
|
||||
|
||||
private ProcessBuilder buildProcessBuilder(SyncConnection connection, List<String> arguments,
|
||||
Path workingDirectory, boolean includeProfile) {
|
||||
Map<String, Object> config = parseJson(connection.getConfigJson());
|
||||
Map<String, Object> secret = parseJson(connection.getSecretJson());
|
||||
String clientId = value(secret, "clientId", null);
|
||||
String clientSecret = value(secret, "clientSecret", null);
|
||||
if (StringUtils.isAnyBlank(clientId, clientSecret)) {
|
||||
throw new ServiceException("钉钉连接 clientId、clientSecret 不能为空");
|
||||
}
|
||||
if (StringUtils.isBlank(executable)) {
|
||||
throw new ServiceException("sync.dingtalk.dws-executable 不能为空");
|
||||
}
|
||||
List<String> command = new ArrayList<>(arguments.size() + (includeProfile ? 3 : 1));
|
||||
command.add(executable);
|
||||
if (includeProfile) {
|
||||
String profile = value(config, "profile", null);
|
||||
if (StringUtils.isBlank(profile)) {
|
||||
throw new ServiceException("钉钉连接尚未完成登录,请先在网页端完成设备授权");
|
||||
}
|
||||
command.add("--profile");
|
||||
command.add(profile);
|
||||
}
|
||||
command.addAll(arguments);
|
||||
Path configDirectory = resolveConfigDirectory(connection);
|
||||
Path keychainDirectory = resolveKeychainDirectory(connection);
|
||||
ProcessBuilder processBuilder = new ProcessBuilder(command);
|
||||
restrictEnvironment(processBuilder, configDirectory);
|
||||
putEnvironment(processBuilder, "DWS_CLIENT_ID", clientId);
|
||||
putEnvironment(processBuilder, "DWS_CLIENT_SECRET", clientSecret);
|
||||
putEnvironment(processBuilder, "DWS_CONFIG_DIR", configDirectory.toString());
|
||||
putEnvironment(processBuilder, "DWS_KEYCHAIN_DIR", keychainDirectory.toString());
|
||||
if (workingDirectory != null) {
|
||||
processBuilder.directory(workingDirectory.toFile());
|
||||
}
|
||||
return processBuilder;
|
||||
}
|
||||
|
||||
private void restrictEnvironment(ProcessBuilder processBuilder, Path configDirectory) {
|
||||
Map<String, String> inherited = new HashMap<>();
|
||||
Map<String, String> environment = processBuilder.environment();
|
||||
boolean windowsHome = environment.containsKey("USERPROFILE");
|
||||
for (String key : SAFE_ENVIRONMENT_KEYS) {
|
||||
String value = environment.get(key);
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
inherited.put(key, value);
|
||||
}
|
||||
}
|
||||
environment.clear();
|
||||
environment.putAll(inherited);
|
||||
// DWS_CONFIG_DIR and DWS_KEYCHAIN_DIR are explicit below. Point HOME
|
||||
// at the connection directory as an additional guard against the CLI
|
||||
// reading a global ~/.dws cache or writing credentials there.
|
||||
environment.put("HOME", configDirectory.toString());
|
||||
if (windowsHome) {
|
||||
environment.put("USERPROFILE", configDirectory.toString());
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Object> config(SyncConnection connection) {
|
||||
return parseJson(connection.getConfigJson());
|
||||
}
|
||||
|
||||
private void putEnvironment(ProcessBuilder builder, String key, String value) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
builder.environment().put(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
Path resolveConfigDirectory(SyncConnection connection) {
|
||||
if (connection.getConnectionId() == null) {
|
||||
throw new ServiceException("钉钉连接必须先保存后才能执行 DWS 命令");
|
||||
}
|
||||
if (StringUtils.isBlank(profileRoot)) {
|
||||
throw new ServiceException("sync.dingtalk.profile-root 不能为空");
|
||||
}
|
||||
try {
|
||||
return resolvePrivateDirectory(profileRoot, connection.getConnectionId(), "DWS 配置");
|
||||
} catch (ServiceException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new ServiceException("无法创建该连接专属的 DWS 配置目录", e);
|
||||
}
|
||||
}
|
||||
|
||||
private Path resolveKeychainDirectory(SyncConnection connection) {
|
||||
if (connection.getConnectionId() == null) {
|
||||
throw new ServiceException("钉钉连接必须先保存后才能执行 DWS 命令");
|
||||
}
|
||||
String rootValue = StringUtils.isNotBlank(keychainRoot)
|
||||
? keychainRoot : (StringUtils.isNotBlank(profileRoot) ? profileRoot + "-keychain" : null);
|
||||
if (StringUtils.isBlank(rootValue)) {
|
||||
throw new ServiceException("sync.dingtalk.keychain-root 不能为空");
|
||||
}
|
||||
try {
|
||||
return resolvePrivateDirectory(rootValue, connection.getConnectionId(), "DWS keychain");
|
||||
} catch (ServiceException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new ServiceException("无法创建该连接专属的 DWS keychain 目录", e);
|
||||
}
|
||||
}
|
||||
|
||||
private Path resolvePrivateDirectory(String rootValue, Long connectionId, String label) throws IOException {
|
||||
Path root = Path.of(rootValue).toAbsolutePath().normalize();
|
||||
if (Files.isSymbolicLink(root)) {
|
||||
throw new ServiceException(label + "根目录不能是符号链接");
|
||||
}
|
||||
createPrivateDirectory(root);
|
||||
Path realRoot = root.toRealPath(LinkOption.NOFOLLOW_LINKS);
|
||||
Path directory = realRoot.resolve(String.valueOf(connectionId)).normalize();
|
||||
if (!directory.startsWith(realRoot)) {
|
||||
throw new ServiceException(label + "目录越界");
|
||||
}
|
||||
if (Files.isSymbolicLink(directory)) {
|
||||
throw new ServiceException(label + "目录不能是符号链接");
|
||||
}
|
||||
createPrivateDirectory(directory);
|
||||
Path realDirectory = directory.toRealPath(LinkOption.NOFOLLOW_LINKS);
|
||||
if (!realDirectory.startsWith(realRoot) || Files.isSymbolicLink(directory)) {
|
||||
throw new ServiceException(label + "目录越界或包含符号链接");
|
||||
}
|
||||
return realDirectory;
|
||||
}
|
||||
|
||||
private void createPrivateDirectory(Path directory) throws IOException {
|
||||
if (Files.isSymbolicLink(directory)) {
|
||||
throw new IOException("directory is a symbolic link: " + directory);
|
||||
}
|
||||
Files.createDirectories(directory);
|
||||
if (Files.isSymbolicLink(directory)) {
|
||||
throw new IOException("directory became a symbolic link: " + directory);
|
||||
}
|
||||
try {
|
||||
Set<PosixFilePermission> permissions = EnumSet.of(
|
||||
PosixFilePermission.OWNER_READ,
|
||||
PosixFilePermission.OWNER_WRITE,
|
||||
PosixFilePermission.OWNER_EXECUTE);
|
||||
Files.setPosixFilePermissions(directory, permissions);
|
||||
} catch (UnsupportedOperationException ignored) {
|
||||
// Windows 文件系统没有 POSIX 权限模型,依赖进程账户 ACL。
|
||||
}
|
||||
}
|
||||
|
||||
private CapturedOutput read(java.io.InputStream inputStream) throws IOException {
|
||||
ByteArrayOutputStream captured = new ByteArrayOutputStream();
|
||||
byte[] buffer = new byte[8192];
|
||||
int length;
|
||||
long total = 0;
|
||||
while ((length = inputStream.read(buffer)) >= 0) {
|
||||
total += length;
|
||||
int remaining = MAX_CAPTURE_BYTES - captured.size();
|
||||
if (remaining > 0) {
|
||||
captured.write(buffer, 0, Math.min(length, remaining));
|
||||
}
|
||||
}
|
||||
return new CapturedOutput(captured.toString(StandardCharsets.UTF_8), total > MAX_CAPTURE_BYTES);
|
||||
}
|
||||
|
||||
private Map<String, Object> parseJson(String json) {
|
||||
if (StringUtils.isBlank(json)) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
return JsonUtils.parseObject(json, new TypeReference<>() {
|
||||
});
|
||||
} catch (Exception e) {
|
||||
throw new ServiceException("钉钉连接配置不是合法 JSON", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String value(Map<String, Object> values, String key, String defaultValue) {
|
||||
Object value = values.get(key);
|
||||
return value == null ? defaultValue : String.valueOf(value);
|
||||
}
|
||||
|
||||
private String abbreviate(String value) {
|
||||
if (value == null) {
|
||||
return "未知错误";
|
||||
}
|
||||
String trimmed = value.trim();
|
||||
return trimmed.length() <= 2000 ? trimmed : trimmed.substring(0, 2000);
|
||||
}
|
||||
|
||||
private String sanitizeDiagnostic(String value, String... secrets) {
|
||||
String redacted = value;
|
||||
if (redacted == null) {
|
||||
return null;
|
||||
}
|
||||
for (String secret : secrets) {
|
||||
if (StringUtils.isNotBlank(secret)) {
|
||||
redacted = redacted.replace(secret, "***");
|
||||
}
|
||||
}
|
||||
redacted = redacted.replaceAll("(?i)(access[_-]?token|refresh[_-]?token|client[_-]?secret|app[_-]?secret|authorization)([\\\"'\\s:=]+)[^\\s,;\\\"']+", "$1$2***");
|
||||
return redacted.replaceAll("(?i)https?://[^\\s\\\"']+\\?[^\\s\\\"']+", "[signed-url-redacted]");
|
||||
}
|
||||
|
||||
private void terminate(Process process) {
|
||||
if (process == null || !process.isAlive()) {
|
||||
return;
|
||||
}
|
||||
boolean interrupted = Thread.interrupted();
|
||||
try {
|
||||
process.descendants().forEach(ProcessHandle::destroy);
|
||||
process.destroy();
|
||||
if (!process.waitFor(2, TimeUnit.SECONDS)) {
|
||||
process.descendants().forEach(ProcessHandle::destroyForcibly);
|
||||
process.destroyForcibly();
|
||||
process.waitFor(2, TimeUnit.SECONDS);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
interrupted = true;
|
||||
process.descendants().forEach(ProcessHandle::destroyForcibly);
|
||||
process.destroyForcibly();
|
||||
} finally {
|
||||
if (interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A one-shot owner for a connection read/write lock. */
|
||||
static final class ConnectionLockLease implements AutoCloseable {
|
||||
private final StampedLock lock;
|
||||
private final long stamp;
|
||||
private final boolean exclusive;
|
||||
private boolean closed;
|
||||
|
||||
private ConnectionLockLease(StampedLock lock, long stamp, boolean exclusive) {
|
||||
this.lock = lock;
|
||||
this.stamp = stamp;
|
||||
this.exclusive = exclusive;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void close() {
|
||||
if (!closed) {
|
||||
closed = true;
|
||||
if (exclusive) {
|
||||
lock.unlockWrite(stamp);
|
||||
} else {
|
||||
lock.unlockRead(stamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
record DwsCommandResult(String stdout, String stderr) {
|
||||
}
|
||||
|
||||
private record CapturedOutput(String value, boolean truncated) {
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
package org.dromara.sync.connector.model;
|
||||
|
||||
/**
|
||||
* 连接器连通性测试结果。
|
||||
*
|
||||
* @param success 是否成功
|
||||
* @param message 结果说明
|
||||
*/
|
||||
public record ConnectorTestResult(boolean success, String message) {
|
||||
|
||||
public static ConnectorTestResult success(String message) {
|
||||
return new ConnectorTestResult(true, message);
|
||||
}
|
||||
|
||||
public static ConnectorTestResult failure(String message) {
|
||||
return new ConnectorTestResult(false, message);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package org.dromara.sync.connector.model;
|
||||
|
||||
/**
|
||||
* 源端单页扫描请求。
|
||||
*
|
||||
* @param parentObjectId 父对象标识,为空表示根目录
|
||||
* @param cursor 分页游标
|
||||
* @param scopeId 源端作用域标识,例如钉盘 spaceId
|
||||
*/
|
||||
public record ScanRequest(String parentObjectId, String cursor, String scopeId) {
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
package org.dromara.sync.connector.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 源端单页扫描结果。
|
||||
*
|
||||
* @param objects 当前页对象
|
||||
* @param nextCursor 下一页游标
|
||||
* @param hasMore 是否还有下一页
|
||||
*/
|
||||
public record ScanResult(List<SourceObject> objects, String nextCursor, boolean hasMore) {
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
package org.dromara.sync.connector.model;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
/**
|
||||
* 下载或导出到本地临时文件的源内容。
|
||||
*
|
||||
* @param path 本地路径
|
||||
* @param size 实际大小
|
||||
* @param contentType 内容类型
|
||||
* @param deleteOnClose 关闭后是否清理文件及临时目录
|
||||
*/
|
||||
public record SourceContent(Path path, long size, String contentType, boolean deleteOnClose) implements AutoCloseable {
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (!deleteOnClose) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Files.deleteIfExists(path);
|
||||
Path parent = path.getParent();
|
||||
if (parent != null) {
|
||||
try (var children = Files.list(parent)) {
|
||||
for (Path child : children.toList()) {
|
||||
Files.deleteIfExists(child);
|
||||
}
|
||||
}
|
||||
Files.deleteIfExists(parent);
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
// 临时文件清理失败不能把已完成的远端上传回滚成失败状态。
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
package org.dromara.sync.connector.model;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 不依赖具体源厂商的统一数据对象。
|
||||
*
|
||||
* @param objectId 源端稳定对象标识
|
||||
* @param parentObjectId 父对象标识
|
||||
* @param name 对象名称
|
||||
* @param objectType FILE、FOLDER 或 ONLINE_DOCUMENT
|
||||
* @param extension 文件扩展名
|
||||
* @param contentType 内容类型
|
||||
* @param size 文件大小
|
||||
* @param modifiedTime 修改时间
|
||||
* @param versionToken 源端版本标识
|
||||
* @param hash 源端校验值
|
||||
* @param metadata 源端扩展元数据
|
||||
*/
|
||||
public record SourceObject(
|
||||
String objectId,
|
||||
String parentObjectId,
|
||||
String name,
|
||||
String objectType,
|
||||
String extension,
|
||||
String contentType,
|
||||
long size,
|
||||
LocalDateTime modifiedTime,
|
||||
String versionToken,
|
||||
String hash,
|
||||
Map<String, String> metadata
|
||||
) {
|
||||
|
||||
public boolean isFolder() {
|
||||
return "FOLDER".equals(objectType);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,22 @@
|
||||
package org.dromara.sync.connector.model;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 目标端写入请求。
|
||||
*
|
||||
* @param objectKey 目标对象键
|
||||
* @param path 待上传本地文件
|
||||
* @param size 文件大小
|
||||
* @param contentType 内容类型
|
||||
* @param metadata 对象元数据
|
||||
*/
|
||||
public record TargetWriteRequest(
|
||||
String objectKey,
|
||||
Path path,
|
||||
long size,
|
||||
String contentType,
|
||||
Map<String, String> metadata
|
||||
) {
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package org.dromara.sync.connector.model;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 目标端写入结果。
|
||||
*
|
||||
* @param objectKey 目标对象键
|
||||
* @param versionId 目标版本标识
|
||||
* @param eTag ETag
|
||||
* @param size 已写入大小
|
||||
* @param url 对象访问地址
|
||||
* @param metadata 从目标端回读的对象元数据
|
||||
*/
|
||||
public record TargetWriteResult(
|
||||
String objectKey,
|
||||
String versionId,
|
||||
String eTag,
|
||||
long size,
|
||||
String url,
|
||||
Map<String, String> metadata
|
||||
) {
|
||||
}
|
||||
@ -0,0 +1,178 @@
|
||||
package org.dromara.sync.connector.s3;
|
||||
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.core.utils.StringUtils;
|
||||
import org.dromara.common.json.utils.JsonUtils;
|
||||
import org.dromara.common.oss.client.OssClient;
|
||||
import org.dromara.common.oss.config.OssClientConfig;
|
||||
import org.dromara.common.oss.model.Options;
|
||||
import org.dromara.common.oss.model.PutObjectResult;
|
||||
import org.dromara.sync.connector.TargetConnector;
|
||||
import org.dromara.sync.connector.model.ConnectorTestResult;
|
||||
import org.dromara.sync.connector.model.TargetWriteRequest;
|
||||
import org.dromara.sync.connector.model.TargetWriteResult;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* 基于仓库原生 S3 客户端的目标连接器基础实现。
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class AbstractS3TargetConnector implements TargetConnector {
|
||||
|
||||
private final Map<Long, SyncS3OssClient> clients = new ConcurrentHashMap<>();
|
||||
|
||||
@Override
|
||||
public ConnectorTestResult test(SyncConnection connection) {
|
||||
try {
|
||||
S3SseSetting.from(parseJson(connection.getConfigJson()));
|
||||
client(connection).headBucket(connection.getBucketName());
|
||||
return ConnectorTestResult.success("目标存储桶访问成功");
|
||||
} catch (Exception e) {
|
||||
return ConnectorTestResult.failure(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public TargetWriteResult upload(SyncConnection connection, TargetWriteRequest request) {
|
||||
Options options = Options.builder()
|
||||
.setLength(request.size())
|
||||
.setContentType(request.contentType())
|
||||
.setMetadata(request.metadata());
|
||||
SyncS3OssClient client = client(connection);
|
||||
S3SseSetting sse = S3SseSetting.from(parseJson(connection.getConfigJson()));
|
||||
PutObjectResult result = client.upload(connection.getBucketName(), request.objectKey(),
|
||||
request.path(), options, sse);
|
||||
HeadObjectResponse head = client.headObject(connection.getBucketName(), result.key());
|
||||
verifyEncryption(sse, head, result.key());
|
||||
return new TargetWriteResult(result.key(), head.versionId(), head.eTag(),
|
||||
head.contentLength(), result.url(), head.metadata());
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验服务端实际生效的加密方式与连接配置是否一致。
|
||||
*
|
||||
* <p>不同厂商对加密请求头的支持程度不一致,部分服务端会静默忽略。此处仅告警不中断同步,
|
||||
* 便于在日志中及时发现目标端不支持所选加密方式的情况。</p>
|
||||
*
|
||||
* @param sse 期望的加密设置
|
||||
* @param head 对象元信息
|
||||
* @param objectKey 对象键
|
||||
*/
|
||||
private void verifyEncryption(S3SseSetting sse, HeadObjectResponse head, String objectKey) {
|
||||
if (!sse.enabled()) {
|
||||
return;
|
||||
}
|
||||
String actual = head.serverSideEncryptionAsString();
|
||||
if (!sse.expectedAlgorithm().equals(actual)) {
|
||||
log.warn("目标端未按配置应用服务端加密,期望={},实际={},objectKey={}",
|
||||
sse.expectedAlgorithm(), actual == null ? "未加密" : actual, objectKey);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean exists(SyncConnection connection, String objectKey) {
|
||||
return client(connection).objectExists(connection.getBucketName(), objectKey);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(SyncConnection connection, String objectKey) {
|
||||
return client(connection).delete(objectKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取不同厂商的默认路径访问风格。
|
||||
*/
|
||||
protected boolean defaultPathStyleAccess() {
|
||||
return false;
|
||||
}
|
||||
|
||||
private SyncS3OssClient client(SyncConnection connection) {
|
||||
OssClientConfig expected = buildConfig(connection);
|
||||
return clients.compute(connection.getConnectionId(), (id, current) -> {
|
||||
if (current != null && current.verifyConfig(expected)) {
|
||||
return current;
|
||||
}
|
||||
closeQuietly(current);
|
||||
return new SyncS3OssClient("sync-" + type().toLowerCase() + "-" + id, expected);
|
||||
});
|
||||
}
|
||||
|
||||
private OssClientConfig buildConfig(SyncConnection connection) {
|
||||
if (StringUtils.isBlank(connection.getEndpoint())) {
|
||||
throw new ServiceException("目标连接 endpoint 不能为空");
|
||||
}
|
||||
if (StringUtils.isBlank(connection.getBucketName())) {
|
||||
throw new ServiceException("目标连接 bucketName 不能为空");
|
||||
}
|
||||
Map<String, Object> secret = parseJson(connection.getSecretJson());
|
||||
String accessKey = value(secret, "accessKey");
|
||||
String secretKey = value(secret, "secretKey");
|
||||
if (StringUtils.isBlank(accessKey) || StringUtils.isBlank(secretKey)) {
|
||||
throw new ServiceException("目标连接 accessKey、secretKey 不能为空");
|
||||
}
|
||||
Map<String, Object> config = parseJson(connection.getConfigJson());
|
||||
boolean useHttps = booleanValue(config, "useHttps", true);
|
||||
boolean pathStyle = booleanValue(config, "pathStyleAccess", defaultPathStyleAccess());
|
||||
String domain = value(config, "domain");
|
||||
Region region = StringUtils.isBlank(connection.getRegion())
|
||||
? Region.US_EAST_1 : Region.of(connection.getRegion());
|
||||
return OssClientConfig.builder()
|
||||
.endpoint(connection.getEndpoint())
|
||||
.domain(domain)
|
||||
.useHttps(useHttps)
|
||||
.usePathStyleAccess(pathStyle)
|
||||
.accessKey(accessKey)
|
||||
.secretKey(secretKey)
|
||||
.bucket(connection.getBucketName())
|
||||
.region(region)
|
||||
.prefix(connection.getBasePath())
|
||||
.build();
|
||||
}
|
||||
|
||||
private Map<String, Object> parseJson(String json) {
|
||||
if (StringUtils.isBlank(json)) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
return JsonUtils.parseObject(json, new TypeReference<>() {
|
||||
});
|
||||
} catch (Exception e) {
|
||||
throw new ServiceException("连接配置不是合法 JSON", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String value(Map<String, Object> values, String key) {
|
||||
Object value = values.get(key);
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
|
||||
private boolean booleanValue(Map<String, Object> values, String key, boolean defaultValue) {
|
||||
Object value = values.get(key);
|
||||
return value == null ? defaultValue : Boolean.parseBoolean(String.valueOf(value));
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void destroy() {
|
||||
clients.values().forEach(this::closeQuietly);
|
||||
clients.clear();
|
||||
}
|
||||
|
||||
private void closeQuietly(OssClient client) {
|
||||
if (client == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
client.close();
|
||||
} catch (Exception ignored) {
|
||||
// 客户端替换或容器销毁时不再抛出关闭异常。
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
package org.dromara.sync.connector.s3;
|
||||
|
||||
import org.dromara.sync.constant.SyncConstants;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 阿里云 OSS 目标适配器。
|
||||
*
|
||||
* <p>第一版复用仓库现有的 S3 协议客户端,厂商差异保留在独立连接器中,
|
||||
* 后续可在不影响同步业务层的情况下切换阿里云原生 SDK。</p>
|
||||
*/
|
||||
@Component
|
||||
public class AliyunOssTargetConnector extends AbstractS3TargetConnector {
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return SyncConstants.TYPE_ALIYUN_OSS;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,112 @@
|
||||
package org.dromara.sync.connector.s3;
|
||||
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.core.utils.StringUtils;
|
||||
import org.dromara.sync.constant.SyncConstants;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 目标端服务端加密(SSE)设置。
|
||||
*
|
||||
* <p>加密方式由连接扩展配置的 {@code sseMode} 决定,取值为
|
||||
* {@link SyncConstants#SSE_NONE}、{@link SyncConstants#SSE_S3} 或 {@link SyncConstants#SSE_KMS}。
|
||||
* 默认 {@code NONE} 表示不发送任何加密请求头,完全由存储桶自身的默认加密策略决定,
|
||||
* 以保证对不支持 SSE 的服务端保持兼容。</p>
|
||||
*
|
||||
* @param mode 加密方式
|
||||
* @param kmsKeyId KMS 密钥标识,仅 {@code SSE_KMS} 且需要指定密钥时使用
|
||||
*/
|
||||
public record S3SseSetting(String mode, String kmsKeyId) {
|
||||
|
||||
/**
|
||||
* 支持的加密方式集合。
|
||||
*/
|
||||
private static final List<String> SUPPORTED_MODES =
|
||||
List.of(SyncConstants.SSE_NONE, SyncConstants.SSE_S3, SyncConstants.SSE_KMS);
|
||||
|
||||
/**
|
||||
* 不指定加密方式,跟随存储桶默认策略。
|
||||
*/
|
||||
public static final S3SseSetting NONE = new S3SseSetting(SyncConstants.SSE_NONE, null);
|
||||
|
||||
/**
|
||||
* 从连接扩展配置解析加密设置。
|
||||
*
|
||||
* @param config 连接扩展配置
|
||||
* @return 加密设置,未配置时返回 {@link #NONE}
|
||||
*/
|
||||
public static S3SseSetting from(Map<String, Object> config) {
|
||||
if (config == null || config.isEmpty()) {
|
||||
return NONE;
|
||||
}
|
||||
String mode = normalize(config.get("sseMode"));
|
||||
String kmsKeyId = StringUtils.trimToNull(stringValue(config.get("sseKmsKeyId")));
|
||||
if (StringUtils.isBlank(mode)) {
|
||||
mode = SyncConstants.SSE_NONE;
|
||||
}
|
||||
if (!SUPPORTED_MODES.contains(mode)) {
|
||||
throw new ServiceException("目标连接 sseMode 只支持 {}", String.join("、", SUPPORTED_MODES));
|
||||
}
|
||||
if (kmsKeyId != null && !SyncConstants.SSE_KMS.equals(mode)) {
|
||||
throw new ServiceException("仅 SSE_KMS 加密方式可以指定 sseKmsKeyId");
|
||||
}
|
||||
return new S3SseSetting(mode, kmsKeyId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 是否需要显式发送服务端加密请求头。
|
||||
*
|
||||
* @return 需要发送返回 {@code true}
|
||||
*/
|
||||
public boolean enabled() {
|
||||
return !SyncConstants.SSE_NONE.equals(mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将加密设置应用到上传请求。
|
||||
*
|
||||
* <p>{@code NONE} 不写入任何加密请求头,由服务端按存储桶默认策略处理。</p>
|
||||
*
|
||||
* @param builder PutObject 请求构建器
|
||||
*/
|
||||
public void applyTo(PutObjectRequest.Builder builder) {
|
||||
switch (mode) {
|
||||
case SyncConstants.SSE_S3 -> builder.serverSideEncryption(ServerSideEncryption.AES256);
|
||||
case SyncConstants.SSE_KMS -> {
|
||||
builder.serverSideEncryption(ServerSideEncryption.AWS_KMS);
|
||||
if (StringUtils.isNotBlank(kmsKeyId)) {
|
||||
builder.ssekmsKeyId(kmsKeyId);
|
||||
}
|
||||
}
|
||||
default -> {
|
||||
// NONE 不发送加密请求头,保持对不支持 SSE 的服务端的兼容。
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取期望的服务端加密算法标识,用于回写校验。
|
||||
*
|
||||
* @return 加密算法标识;{@code NONE} 返回 {@code null}
|
||||
*/
|
||||
public String expectedAlgorithm() {
|
||||
return switch (mode) {
|
||||
case SyncConstants.SSE_S3 -> ServerSideEncryption.AES256.toString();
|
||||
case SyncConstants.SSE_KMS -> ServerSideEncryption.AWS_KMS.toString();
|
||||
default -> null;
|
||||
};
|
||||
}
|
||||
|
||||
private static String normalize(Object value) {
|
||||
String text = stringValue(value);
|
||||
return text == null ? null : text.trim().toUpperCase();
|
||||
}
|
||||
|
||||
private static String stringValue(Object value) {
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
package org.dromara.sync.connector.s3;
|
||||
|
||||
import org.dromara.sync.constant.SyncConstants;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* S3 标准兼容目标连接器。
|
||||
*/
|
||||
@Component
|
||||
public class S3TargetConnector extends AbstractS3TargetConnector {
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return SyncConstants.TYPE_S3;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,113 @@
|
||||
package org.dromara.sync.connector.s3;
|
||||
|
||||
import org.dromara.common.oss.client.DefaultOssClientImpl;
|
||||
import org.dromara.common.oss.config.OssClientConfig;
|
||||
import org.dromara.common.oss.exception.S3StorageException;
|
||||
import org.dromara.common.oss.model.HandleAsyncResult;
|
||||
import org.dromara.common.oss.model.Options;
|
||||
import org.dromara.common.oss.model.PutObjectResult;
|
||||
import software.amazon.awssdk.core.async.AsyncRequestBody;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
|
||||
import software.amazon.awssdk.services.s3.model.S3Exception;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
||||
/**
|
||||
* 为同步连接器补充只读的存储桶连通性探测与服务端加密上传。
|
||||
*/
|
||||
class SyncS3OssClient extends DefaultOssClientImpl {
|
||||
|
||||
SyncS3OssClient(String clientId, OssClientConfig config) {
|
||||
super(clientId, config);
|
||||
}
|
||||
|
||||
void headBucket(String bucket) {
|
||||
s3AsyncClient.headBucket(builder -> builder.bucket(bucket)).join();
|
||||
}
|
||||
|
||||
HeadObjectResponse headObject(String bucket, String key) {
|
||||
return s3AsyncClient.headObject(builder -> builder.bucket(bucket).key(key)).join();
|
||||
}
|
||||
|
||||
boolean objectExists(String bucket, String key) {
|
||||
try {
|
||||
headObject(bucket, key);
|
||||
return true;
|
||||
} catch (CompletionException e) {
|
||||
Throwable cause = e.getCause();
|
||||
if (cause instanceof S3Exception s3Exception && s3Exception.statusCode() == 404) {
|
||||
return false;
|
||||
}
|
||||
throw e;
|
||||
} catch (S3Exception e) {
|
||||
if (e.statusCode() == 404) {
|
||||
return false;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 上传本地文件并按连接配置附加服务端加密请求头。
|
||||
*
|
||||
* <p>公共 OSS 模块的 {@link Options} 不支持服务端加密,因此这里直接使用底层
|
||||
* {@code doCustomUpload} 自行构建 PutObject 请求,避免修改公共模块 API。</p>
|
||||
*
|
||||
* @param bucket 存储桶名称
|
||||
* @param key 对象键
|
||||
* @param path 待上传文件
|
||||
* @param options 上传选项
|
||||
* @param sse 服务端加密设置
|
||||
* @return 上传结果
|
||||
*/
|
||||
PutObjectResult upload(String bucket, String key, Path path, Options options, S3SseSetting sse) {
|
||||
AsyncRequestBody body = AsyncRequestBody.fromFile(path);
|
||||
// 与公共模块 bucketUpload 保持一致:请求体自带的长度与类型优先于可选项。
|
||||
Long contentLength = body.contentLength().orElse(options.getLength());
|
||||
String contentType = options.getContentType() == null || options.getContentType().isBlank()
|
||||
? body.contentType() : options.getContentType();
|
||||
HandleAsyncResult<PutObjectResponse> result = doCustomUpload(body, builder -> {
|
||||
builder.bucket(bucket)
|
||||
.key(key)
|
||||
.contentMD5(options.getMd5Digest())
|
||||
.contentType(contentType)
|
||||
.contentLength(contentLength)
|
||||
.metadata(options.getMetadata());
|
||||
sse.applyTo(builder);
|
||||
}, options.getTransferListeners());
|
||||
if (result.isFailure()) {
|
||||
throw toStorageException(result.error());
|
||||
}
|
||||
Optional<PutObjectResponse> opt = result.getResult();
|
||||
if (opt.isEmpty()) {
|
||||
throw S3StorageException.form("response is empty.");
|
||||
}
|
||||
PutObjectResponse response = opt.get();
|
||||
Long size = response.size();
|
||||
if (size == null) {
|
||||
size = contentLength == null ? 0 : contentLength;
|
||||
}
|
||||
return PutObjectResult.form("%s/%s".formatted(config.getBucketUrl(bucket), key), key, response.eTag(), size);
|
||||
}
|
||||
|
||||
/**
|
||||
* 转换为统一的 S3 存储异常。
|
||||
*
|
||||
* @param e 原始异常
|
||||
* @return S3 存储异常
|
||||
*/
|
||||
private S3StorageException toStorageException(Throwable e) {
|
||||
Throwable cause = e;
|
||||
while ((cause instanceof CompletionException || cause instanceof ExecutionException) && cause.getCause() != null) {
|
||||
cause = cause.getCause();
|
||||
}
|
||||
if (cause instanceof S3StorageException ex) {
|
||||
return ex;
|
||||
}
|
||||
return S3StorageException.form(cause);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,50 @@
|
||||
package org.dromara.sync.constant;
|
||||
|
||||
/**
|
||||
* 数据同步模块常量。
|
||||
*/
|
||||
public interface SyncConstants {
|
||||
|
||||
String ROLE_SOURCE = "SOURCE";
|
||||
String ROLE_TARGET = "TARGET";
|
||||
|
||||
String TYPE_DINGTALK = "DINGTALK";
|
||||
String TYPE_S3 = "S3";
|
||||
String TYPE_ALIYUN_OSS = "ALIYUN_OSS";
|
||||
|
||||
String STATUS_NORMAL = "0";
|
||||
String STATUS_DISABLED = "1";
|
||||
|
||||
String JOB_PENDING = "PENDING";
|
||||
String JOB_RUNNING = "RUNNING";
|
||||
String JOB_CANCELING = "CANCELING";
|
||||
String JOB_SUCCESS = "SUCCESS";
|
||||
String JOB_PARTIAL_FAILED = "PARTIAL_FAILED";
|
||||
String JOB_FAILED = "FAILED";
|
||||
String JOB_CANCELED = "CANCELED";
|
||||
|
||||
String ITEM_PENDING = "PENDING";
|
||||
String ITEM_RUNNING = "RUNNING";
|
||||
String ITEM_SUCCESS = "SUCCESS";
|
||||
String ITEM_SKIPPED = "SKIPPED";
|
||||
String ITEM_FAILED = "FAILED";
|
||||
|
||||
String OBJECT_FILE = "FILE";
|
||||
String OBJECT_FOLDER = "FOLDER";
|
||||
String OBJECT_ONLINE_DOCUMENT = "ONLINE_DOCUMENT";
|
||||
|
||||
/**
|
||||
* 目标端服务端加密方式:不指定,由存储桶默认策略决定。
|
||||
*/
|
||||
String SSE_NONE = "NONE";
|
||||
|
||||
/**
|
||||
* 目标端服务端加密方式:SSE-S3,由服务端托管密钥(AES256)。
|
||||
*/
|
||||
String SSE_S3 = "SSE_S3";
|
||||
|
||||
/**
|
||||
* 目标端服务端加密方式:SSE-KMS,由 KMS 托管密钥。
|
||||
*/
|
||||
String SSE_KMS = "SSE_KMS";
|
||||
}
|
||||
@ -0,0 +1,185 @@
|
||||
package org.dromara.sync.controller;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.core.domain.R;
|
||||
import org.dromara.common.core.validate.AddGroup;
|
||||
import org.dromara.common.core.validate.EditGroup;
|
||||
import org.dromara.common.core.validate.QueryGroup;
|
||||
import org.dromara.common.log.annotation.Log;
|
||||
import org.dromara.common.log.enums.BusinessType;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.common.redis.annotation.RepeatSubmit;
|
||||
import org.dromara.common.satoken.utils.LoginHelper;
|
||||
import org.dromara.common.web.core.BaseController;
|
||||
import org.dromara.sync.connector.model.ConnectorTestResult;
|
||||
import org.dromara.sync.domain.bo.SyncConnectionBo;
|
||||
import org.dromara.sync.domain.vo.DwsAuthSessionVo;
|
||||
import org.dromara.sync.domain.vo.SyncConnectionVo;
|
||||
import org.dromara.sync.service.ISyncConnectionService;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 同步连接管理。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
@Validated
|
||||
@RequiredArgsConstructor
|
||||
@RestController
|
||||
@RequestMapping("/sync/connection")
|
||||
public class SyncConnectionController extends BaseController {
|
||||
|
||||
private final ISyncConnectionService connectionService;
|
||||
|
||||
/**
|
||||
* 分页查询同步连接列表。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @param pageQuery 分页参数
|
||||
* @return 同步连接分页列表
|
||||
*/
|
||||
@SaCheckPermission("sync:connection:list")
|
||||
@GetMapping("/list")
|
||||
public R<PageResult<SyncConnectionVo>> list(@Validated(QueryGroup.class) SyncConnectionBo bo,
|
||||
PageQuery pageQuery) {
|
||||
return R.ok(connectionService.queryPageList(bo, pageQuery));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取同步连接详情。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @return 同步连接详情
|
||||
*/
|
||||
@SaCheckPermission("sync:connection:query")
|
||||
@GetMapping("/{connectionId}")
|
||||
public R<SyncConnectionVo> getInfo(@NotNull(message = "连接主键不能为空")
|
||||
@PathVariable Long connectionId) {
|
||||
return R.ok(connectionService.queryById(connectionId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 新增同步连接。
|
||||
*
|
||||
* @param bo 同步连接业务对象
|
||||
* @return 操作结果
|
||||
*/
|
||||
@SaCheckPermission("sync:connection:add")
|
||||
@Log(title = "同步连接", businessType = BusinessType.INSERT, excludeParamNames = "secretJson")
|
||||
@RepeatSubmit()
|
||||
@PostMapping()
|
||||
public R<Void> add(@Validated(AddGroup.class) @RequestBody SyncConnectionBo bo) {
|
||||
return toAjax(connectionService.insertByBo(bo));
|
||||
}
|
||||
|
||||
/**
|
||||
* 修改同步连接。
|
||||
*
|
||||
* @param bo 同步连接业务对象
|
||||
* @return 操作结果
|
||||
*/
|
||||
@SaCheckPermission("sync:connection:edit")
|
||||
@Log(title = "同步连接", businessType = BusinessType.UPDATE, excludeParamNames = "secretJson")
|
||||
@RepeatSubmit()
|
||||
@PutMapping()
|
||||
public R<Void> edit(@Validated(EditGroup.class) @RequestBody SyncConnectionBo bo) {
|
||||
return toAjax(connectionService.updateByBo(bo));
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量删除同步连接。
|
||||
*
|
||||
* @param connectionIds 连接主键数组
|
||||
* @return 操作结果
|
||||
*/
|
||||
@SaCheckPermission("sync:connection:remove")
|
||||
@Log(title = "同步连接", businessType = BusinessType.DELETE)
|
||||
@RepeatSubmit()
|
||||
@DeleteMapping("/{connectionIds}")
|
||||
public R<Void> remove(@NotEmpty(message = "连接主键不能为空")
|
||||
@PathVariable Long[] connectionIds) {
|
||||
return toAjax(connectionService.deleteWithValidByIds(List.of(connectionIds), true));
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验同步连接配置并执行连通性测试。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @return 操作结果
|
||||
*/
|
||||
@SaCheckPermission("sync:connection:test")
|
||||
@Log(title = "同步连接测试", businessType = BusinessType.OTHER)
|
||||
@RepeatSubmit()
|
||||
@PostMapping("/test/{connectionId}")
|
||||
public R<ConnectorTestResult> testConnection(@NotNull(message = "连接主键不能为空")
|
||||
@PathVariable Long connectionId) {
|
||||
return R.ok(connectionService.testConnection(connectionId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 在 Web 端启动钉钉设备授权。服务器负责运行 DWS,浏览器只接收一次性
|
||||
* 授权链接和验证码,不会接触任何 token 或应用密钥。
|
||||
*
|
||||
* @param connectionId 钉钉连接主键
|
||||
* @param expectedCorpId 可选的组织 ID,用于防止授权到错误组织
|
||||
* @return 授权会话及展示信息
|
||||
*/
|
||||
@SaCheckPermission("sync:connection:auth")
|
||||
@Log(title = "钉钉 Web 登录", businessType = BusinessType.OTHER)
|
||||
@RepeatSubmit()
|
||||
@PostMapping("/{connectionId}/auth/login/start")
|
||||
public R<DwsAuthSessionVo> startDingTalkAuth(
|
||||
@NotNull(message = "连接主键不能为空") @PathVariable Long connectionId,
|
||||
@RequestParam(required = false) String expectedCorpId) {
|
||||
return R.ok(connectionService.startDingTalkAuth(connectionId, LoginHelper.getUserId(), expectedCorpId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询钉钉 Web 设备授权状态。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @param sessionId 授权会话 ID
|
||||
* @return 授权状态
|
||||
*/
|
||||
@SaCheckPermission("sync:connection:auth")
|
||||
@GetMapping("/{connectionId}/auth/login/{sessionId}")
|
||||
public R<DwsAuthSessionVo> getDingTalkAuth(
|
||||
@NotNull(message = "连接主键不能为空") @PathVariable Long connectionId,
|
||||
@PathVariable String sessionId) {
|
||||
return R.ok(connectionService.getDingTalkAuth(connectionId, sessionId, LoginHelper.getUserId()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消钉钉 Web 设备授权并终止服务器上的 DWS 进程。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @param sessionId 授权会话 ID
|
||||
* @return 取消后的授权状态
|
||||
*/
|
||||
@SaCheckPermission("sync:connection:auth")
|
||||
@Log(title = "取消钉钉 Web 登录", businessType = BusinessType.UPDATE)
|
||||
@RepeatSubmit()
|
||||
@PostMapping("/{connectionId}/auth/login/{sessionId}/cancel")
|
||||
public R<DwsAuthSessionVo> cancelDingTalkAuth(
|
||||
@NotNull(message = "连接主键不能为空") @PathVariable Long connectionId,
|
||||
@PathVariable String sessionId) {
|
||||
return R.ok(connectionService.cancelDingTalkAuth(connectionId, sessionId, LoginHelper.getUserId()));
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,83 @@
|
||||
package org.dromara.sync.controller;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.core.domain.R;
|
||||
import org.dromara.common.log.annotation.Log;
|
||||
import org.dromara.common.log.enums.BusinessType;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.common.redis.annotation.RepeatSubmit;
|
||||
import org.dromara.common.web.core.BaseController;
|
||||
import org.dromara.sync.domain.bo.SyncJobBo;
|
||||
import org.dromara.sync.domain.bo.SyncJobItemBo;
|
||||
import org.dromara.sync.domain.vo.SyncJobItemVo;
|
||||
import org.dromara.sync.domain.vo.SyncJobVo;
|
||||
import org.dromara.sync.service.ISyncJobService;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 同步任务管理接口。
|
||||
*/
|
||||
@Validated
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/sync/job")
|
||||
public class SyncJobController extends BaseController {
|
||||
|
||||
private final ISyncJobService jobService;
|
||||
|
||||
@SaCheckPermission("sync:job:list")
|
||||
@GetMapping("/list")
|
||||
public R<PageResult<SyncJobVo>> list(SyncJobBo bo, PageQuery pageQuery) {
|
||||
return R.ok(jobService.queryPageList(bo, pageQuery));
|
||||
}
|
||||
|
||||
@SaCheckPermission("sync:job:query")
|
||||
@GetMapping("/{jobId}")
|
||||
public R<SyncJobVo> getInfo(@NotNull(message = "任务ID不能为空") @PathVariable Long jobId) {
|
||||
return R.ok(jobService.queryById(jobId));
|
||||
}
|
||||
|
||||
@SaCheckPermission("sync:job:query")
|
||||
@GetMapping("/{jobId}/items")
|
||||
public R<PageResult<SyncJobItemVo>> items(@PathVariable Long jobId, SyncJobItemBo bo, PageQuery pageQuery) {
|
||||
bo.setJobId(jobId);
|
||||
return R.ok(jobService.queryItemPageList(bo, pageQuery));
|
||||
}
|
||||
|
||||
@SaCheckPermission("sync:job:run")
|
||||
@Log(title = "同步任务", businessType = BusinessType.OTHER)
|
||||
@RepeatSubmit
|
||||
@PostMapping("/run/{planId}")
|
||||
public R<Long> run(@NotNull(message = "计划ID不能为空") @PathVariable Long planId) {
|
||||
return R.ok(jobService.startPlan(planId, "MANUAL", null));
|
||||
}
|
||||
|
||||
@SaCheckPermission("sync:job:retry")
|
||||
@Log(title = "同步任务重试", businessType = BusinessType.OTHER)
|
||||
@RepeatSubmit
|
||||
@PostMapping("/retry/{jobId}")
|
||||
public R<Long> retry(@NotNull(message = "任务ID不能为空") @PathVariable Long jobId) {
|
||||
return R.ok(jobService.retry(jobId));
|
||||
}
|
||||
|
||||
@SaCheckPermission("sync:job:cancel")
|
||||
@Log(title = "取消同步任务", businessType = BusinessType.UPDATE)
|
||||
@PostMapping("/cancel/{jobId}")
|
||||
public R<Void> cancel(@NotNull(message = "任务ID不能为空") @PathVariable Long jobId) {
|
||||
return toAjax(jobService.cancel(jobId));
|
||||
}
|
||||
|
||||
@SaCheckPermission("sync:job:remove")
|
||||
@Log(title = "同步任务", businessType = BusinessType.DELETE)
|
||||
@DeleteMapping("/{jobIds}")
|
||||
public R<Void> remove(@NotEmpty(message = "任务ID不能为空") @PathVariable Long[] jobIds) {
|
||||
return toAjax(jobService.deleteWithValidByIds(List.of(jobIds), true));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
package org.dromara.sync.controller;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.core.domain.R;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.common.web.core.BaseController;
|
||||
import org.dromara.sync.domain.bo.SyncObjectBo;
|
||||
import org.dromara.sync.domain.vo.SyncObjectVo;
|
||||
import org.dromara.sync.service.ISyncObjectService;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
/**
|
||||
* 同步对象清单接口。
|
||||
*/
|
||||
@Validated
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/sync/object")
|
||||
public class SyncObjectController extends BaseController {
|
||||
|
||||
private final ISyncObjectService objectService;
|
||||
|
||||
@SaCheckPermission("sync:object:list")
|
||||
@GetMapping("/list")
|
||||
public R<PageResult<SyncObjectVo>> list(SyncObjectBo bo, PageQuery pageQuery) {
|
||||
return R.ok(objectService.queryPageList(bo, pageQuery));
|
||||
}
|
||||
|
||||
@SaCheckPermission("sync:object:query")
|
||||
@GetMapping("/{objectId}")
|
||||
public R<SyncObjectVo> getInfo(@NotNull(message = "对象ID不能为空") @PathVariable Long objectId) {
|
||||
return R.ok(objectService.queryById(objectId));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,121 @@
|
||||
package org.dromara.sync.controller;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.core.domain.R;
|
||||
import org.dromara.common.core.validate.AddGroup;
|
||||
import org.dromara.common.core.validate.EditGroup;
|
||||
import org.dromara.common.core.validate.QueryGroup;
|
||||
import org.dromara.common.log.annotation.Log;
|
||||
import org.dromara.common.log.enums.BusinessType;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.common.redis.annotation.RepeatSubmit;
|
||||
import org.dromara.common.web.core.BaseController;
|
||||
import org.dromara.sync.domain.bo.SyncPlanBo;
|
||||
import org.dromara.sync.domain.vo.SyncPlanVo;
|
||||
import org.dromara.sync.service.ISyncPlanService;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 同步计划管理。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
@Validated
|
||||
@RequiredArgsConstructor
|
||||
@RestController
|
||||
@RequestMapping("/sync/plan")
|
||||
public class SyncPlanController extends BaseController {
|
||||
|
||||
private final ISyncPlanService planService;
|
||||
|
||||
/**
|
||||
* 分页查询同步计划列表。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @param pageQuery 分页参数
|
||||
* @return 同步计划分页数据
|
||||
*/
|
||||
@SaCheckPermission("sync:plan:list")
|
||||
@GetMapping("/list")
|
||||
public R<PageResult<SyncPlanVo>> list(@Validated(QueryGroup.class) SyncPlanBo bo, PageQuery pageQuery) {
|
||||
return R.ok(planService.queryPageList(bo, pageQuery));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取同步计划详细信息。
|
||||
*
|
||||
* @param planId 计划主键
|
||||
* @return 同步计划详情
|
||||
*/
|
||||
@SaCheckPermission("sync:plan:query")
|
||||
@GetMapping("/{planId}")
|
||||
public R<SyncPlanVo> getInfo(@NotNull(message = "计划主键不能为空")
|
||||
@PathVariable Long planId) {
|
||||
return R.ok(planService.queryById(planId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 新增同步计划。
|
||||
*
|
||||
* @param bo 同步计划
|
||||
* @return 操作结果
|
||||
*/
|
||||
@SaCheckPermission("sync:plan:add")
|
||||
@Log(title = "同步计划", businessType = BusinessType.INSERT)
|
||||
@RepeatSubmit()
|
||||
@PostMapping()
|
||||
public R<Void> add(@Validated(AddGroup.class) @RequestBody SyncPlanBo bo) {
|
||||
if (!planService.checkPlanNameUnique(bo)) {
|
||||
return R.fail("新增同步计划'" + bo.getPlanName() + "'失败,计划名称已存在");
|
||||
}
|
||||
return toAjax(planService.insertByBo(bo));
|
||||
}
|
||||
|
||||
/**
|
||||
* 修改同步计划。
|
||||
*
|
||||
* @param bo 同步计划
|
||||
* @return 操作结果
|
||||
*/
|
||||
@SaCheckPermission("sync:plan:edit")
|
||||
@Log(title = "同步计划", businessType = BusinessType.UPDATE)
|
||||
@RepeatSubmit()
|
||||
@PutMapping()
|
||||
public R<Void> edit(@Validated(EditGroup.class) @RequestBody SyncPlanBo bo) {
|
||||
if (!planService.checkPlanNameUnique(bo)) {
|
||||
return R.fail("修改同步计划'" + bo.getPlanName() + "'失败,计划名称已存在");
|
||||
}
|
||||
return toAjax(planService.updateByBo(bo));
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量删除同步计划。
|
||||
*
|
||||
* @param planIds 计划主键集合
|
||||
* @return 操作结果
|
||||
*/
|
||||
@SaCheckPermission("sync:plan:remove")
|
||||
@Log(title = "同步计划", businessType = BusinessType.DELETE)
|
||||
@RepeatSubmit()
|
||||
@DeleteMapping("/{planIds}")
|
||||
public R<Void> remove(@NotEmpty(message = "计划主键不能为空")
|
||||
@PathVariable Long[] planIds) {
|
||||
return toAjax(planService.deleteWithValidByIds(List.of(planIds), true));
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
package org.dromara.sync.domain;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.baomidou.mybatisplus.annotation.Version;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import org.dromara.common.mybatis.core.domain.BaseEntity;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 同步增量与分页断点 sync_checkpoint。
|
||||
*/
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
@TableName("sync_checkpoint")
|
||||
public class SyncCheckpoint extends BaseEntity {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@TableId("checkpoint_id")
|
||||
private Long checkpointId;
|
||||
private Long planId;
|
||||
private String checkpointType;
|
||||
private String checkpointKey;
|
||||
private String checkpointValue;
|
||||
private LocalDateTime watermarkTime;
|
||||
private Long lastJobId;
|
||||
@Version
|
||||
private Long version;
|
||||
}
|
||||
@ -0,0 +1,96 @@
|
||||
package org.dromara.sync.domain;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableLogic;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import org.dromara.common.encrypt.annotation.EncryptField;
|
||||
import org.dromara.common.encrypt.enums.AlgorithmType;
|
||||
import org.dromara.common.mybatis.core.domain.BaseEntity;
|
||||
|
||||
import java.io.Serial;
|
||||
|
||||
/**
|
||||
* 同步连接对象 sync_connection。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
@TableName("sync_connection")
|
||||
public class SyncConnection extends BaseEntity {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* 连接主键。
|
||||
*/
|
||||
@TableId(value = "connection_id")
|
||||
private Long connectionId;
|
||||
|
||||
/**
|
||||
* 连接名称。
|
||||
*/
|
||||
private String connectionName;
|
||||
|
||||
/**
|
||||
* 连接角色(SOURCE 源端、TARGET 目标端)。
|
||||
*/
|
||||
private String connectionRole;
|
||||
|
||||
/**
|
||||
* 连接类型(DINGTALK、S3、ALIYUN_OSS)。
|
||||
*/
|
||||
private String connectionType;
|
||||
|
||||
/**
|
||||
* 服务端点。
|
||||
*/
|
||||
private String endpoint;
|
||||
|
||||
/**
|
||||
* 服务区域。
|
||||
*/
|
||||
private String region;
|
||||
|
||||
/**
|
||||
* 存储桶名称。
|
||||
*/
|
||||
private String bucketName;
|
||||
|
||||
/**
|
||||
* 目标基础路径。
|
||||
*/
|
||||
private String basePath;
|
||||
|
||||
/**
|
||||
* 非敏感扩展配置 JSON。
|
||||
*/
|
||||
private String configJson;
|
||||
|
||||
/**
|
||||
* 敏感凭证配置 JSON,持久化时由 MyBatis 加密处理器加密。
|
||||
*/
|
||||
@EncryptField(algorithm = AlgorithmType.AES)
|
||||
private String secretJson;
|
||||
|
||||
/**
|
||||
* 状态(0 正常、1 停用)。
|
||||
*/
|
||||
private String status;
|
||||
|
||||
/**
|
||||
* 删除标志(0 存在、1 删除)。
|
||||
*/
|
||||
@TableLogic
|
||||
private String delFlag;
|
||||
|
||||
/**
|
||||
* 备注。
|
||||
*/
|
||||
private String remark;
|
||||
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
package org.dromara.sync.domain;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import org.dromara.common.mybatis.core.domain.BaseEntity;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 同步任务运行实例 sync_job。
|
||||
*/
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
@TableName("sync_job")
|
||||
public class SyncJob extends BaseEntity {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@TableId("job_id")
|
||||
private Long jobId;
|
||||
private Long planId;
|
||||
private String triggerType;
|
||||
private String runType;
|
||||
private String status;
|
||||
private String checkpointBefore;
|
||||
private String checkpointAfter;
|
||||
private Long totalCount;
|
||||
private Long processedCount;
|
||||
private Long successCount;
|
||||
private Long failedCount;
|
||||
private Long skippedCount;
|
||||
private Long deletedCount;
|
||||
private Long totalBytes;
|
||||
private Long transferredBytes;
|
||||
private LocalDateTime startTime;
|
||||
private LocalDateTime finishTime;
|
||||
private String errorMessage;
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
package org.dromara.sync.domain;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import org.dromara.common.mybatis.core.domain.BaseEntity;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 同步任务文件明细 sync_job_item。
|
||||
*/
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
@TableName("sync_job_item")
|
||||
public class SyncJobItem extends BaseEntity {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@TableId("item_id")
|
||||
private Long itemId;
|
||||
private Long jobId;
|
||||
private String sourceObjectId;
|
||||
private String parentObjectId;
|
||||
private String sourcePath;
|
||||
private String targetKey;
|
||||
private String objectType;
|
||||
private String actionType;
|
||||
private String status;
|
||||
private Long size;
|
||||
private Long transferredBytes;
|
||||
private String versionToken;
|
||||
private String sourceEtag;
|
||||
private String sourceSha256;
|
||||
private String targetEtag;
|
||||
private String targetVersionId;
|
||||
private Integer retryCount;
|
||||
private LocalDateTime startTime;
|
||||
private LocalDateTime finishTime;
|
||||
private String errorCode;
|
||||
private String errorMessage;
|
||||
}
|
||||
@ -0,0 +1,51 @@
|
||||
package org.dromara.sync.domain;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.FieldStrategy;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import org.dromara.common.mybatis.core.domain.BaseEntity;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 源对象与目标对象映射清单 sync_object。
|
||||
*/
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
@TableName("sync_object")
|
||||
public class SyncObject extends BaseEntity {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@TableId("object_id")
|
||||
private Long objectId;
|
||||
private Long planId;
|
||||
private String sourceObjectId;
|
||||
private String parentObjectId;
|
||||
private String sourcePath;
|
||||
private String objectName;
|
||||
private String objectType;
|
||||
private Long size;
|
||||
private LocalDateTime modifiedTime;
|
||||
private String versionToken;
|
||||
private String sourceEtag;
|
||||
private String sourceSha256;
|
||||
private String contentType;
|
||||
private String metadataJson;
|
||||
private String targetKey;
|
||||
private String targetVersionId;
|
||||
private String targetEtag;
|
||||
private String syncStatus;
|
||||
private String sourceDeleted;
|
||||
private Long firstSeenJobId;
|
||||
private Long lastSeenJobId;
|
||||
private Long lastSyncJobId;
|
||||
private LocalDateTime lastSyncTime;
|
||||
@TableField(updateStrategy = FieldStrategy.ALWAYS)
|
||||
private String lastErrorMessage;
|
||||
}
|
||||
@ -0,0 +1,132 @@
|
||||
package org.dromara.sync.domain;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.FieldStrategy;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableLogic;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import org.dromara.common.mybatis.core.domain.BaseEntity;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 同步计划对象 sync_plan。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
@TableName("sync_plan")
|
||||
public class SyncPlan extends BaseEntity {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* 计划主键。
|
||||
*/
|
||||
@TableId(value = "plan_id")
|
||||
private Long planId;
|
||||
|
||||
/**
|
||||
* 计划名称。
|
||||
*/
|
||||
private String planName;
|
||||
|
||||
/**
|
||||
* 源端连接主键。
|
||||
*/
|
||||
private Long sourceConnectionId;
|
||||
|
||||
/**
|
||||
* 目标端连接主键。
|
||||
*/
|
||||
private Long targetConnectionId;
|
||||
|
||||
/**
|
||||
* 源端同步根路径或对象标识。
|
||||
*/
|
||||
private String sourceRoot;
|
||||
|
||||
/**
|
||||
* 目标端对象键前缀。
|
||||
*/
|
||||
private String targetPrefix;
|
||||
|
||||
/**
|
||||
* 同步模式(FULL 全量、INCREMENTAL 增量)。
|
||||
*/
|
||||
private String syncMode;
|
||||
|
||||
/**
|
||||
* 调度类型(MANUAL 手动、CRON 定时)。
|
||||
*/
|
||||
private String scheduleType;
|
||||
|
||||
/**
|
||||
* CRON 表达式;切换为手动调度时需要显式清空。
|
||||
*/
|
||||
@TableField(updateStrategy = FieldStrategy.ALWAYS)
|
||||
private String cronExpression;
|
||||
|
||||
/**
|
||||
* 冲突处理策略(OVERWRITE 覆盖、SKIP 跳过、KEEP_BOTH 两者保留)。
|
||||
*/
|
||||
private String conflictStrategy;
|
||||
|
||||
/**
|
||||
* 源端删除处理策略(KEEP 保留、MARK 标记、DELETE 删除)。
|
||||
*/
|
||||
private String deleteStrategy;
|
||||
|
||||
/**
|
||||
* 单次目标删除比例保护阈值,范围 0 至 100。
|
||||
*/
|
||||
private Integer deleteGuardPercent;
|
||||
|
||||
/**
|
||||
* 完整性校验方式(SIZE、ETAG、SHA256)。
|
||||
*/
|
||||
private String verifyMode;
|
||||
|
||||
/**
|
||||
* 最大并发传输数。
|
||||
*/
|
||||
private Integer maxConcurrency;
|
||||
|
||||
/**
|
||||
* 带宽上限,单位 KB/s;0 表示不限速。
|
||||
*/
|
||||
private Long bandwidthLimitKbps;
|
||||
|
||||
/**
|
||||
* 状态(0 正常、1 停用)。
|
||||
*/
|
||||
private String status;
|
||||
|
||||
/**
|
||||
* 最近一次运行时间。
|
||||
*/
|
||||
private LocalDateTime lastRunTime;
|
||||
|
||||
/**
|
||||
* 下一次计划运行时间。
|
||||
*/
|
||||
private LocalDateTime nextRunTime;
|
||||
|
||||
/**
|
||||
* 删除标志(0 存在、1 删除)。
|
||||
*/
|
||||
@TableLogic
|
||||
private String delFlag;
|
||||
|
||||
/**
|
||||
* 备注。
|
||||
*/
|
||||
private String remark;
|
||||
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
package org.dromara.sync.domain;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import org.dromara.common.mybatis.core.domain.BaseEntity;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 大文件分片传输断点 sync_transfer_part。
|
||||
*/
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
@TableName("sync_transfer_part")
|
||||
public class SyncTransferPart extends BaseEntity {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@TableId("part_id")
|
||||
private Long partId;
|
||||
private Long jobItemId;
|
||||
private String uploadId;
|
||||
private Integer partNumber;
|
||||
private Long partOffset;
|
||||
private Long partSize;
|
||||
private Long transferredBytes;
|
||||
private String partEtag;
|
||||
private String checksumSha256;
|
||||
private String status;
|
||||
private Integer retryCount;
|
||||
private LocalDateTime startTime;
|
||||
private LocalDateTime finishTime;
|
||||
private String errorMessage;
|
||||
}
|
||||
@ -0,0 +1,105 @@
|
||||
package org.dromara.sync.domain.bo;
|
||||
|
||||
import io.github.linpeilie.annotations.AutoMapper;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
import lombok.Data;
|
||||
import org.dromara.common.core.validate.AddGroup;
|
||||
import org.dromara.common.core.validate.EditGroup;
|
||||
import org.dromara.common.json.validate.JsonPattern;
|
||||
import org.dromara.common.json.validate.JsonType;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* 同步连接业务对象 sync_connection。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
@Data
|
||||
@AutoMapper(target = SyncConnection.class, reverseConvertGenerate = false)
|
||||
public class SyncConnectionBo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* 连接主键。
|
||||
*/
|
||||
@NotNull(message = "连接主键不能为空", groups = EditGroup.class)
|
||||
private Long connectionId;
|
||||
|
||||
/**
|
||||
* 连接名称。
|
||||
*/
|
||||
@NotBlank(message = "连接名称不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
private String connectionName;
|
||||
|
||||
/**
|
||||
* 连接角色(SOURCE 源端、TARGET 目标端)。
|
||||
*/
|
||||
@NotBlank(message = "连接角色不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Pattern(regexp = "SOURCE|TARGET", message = "连接角色只支持 SOURCE 或 TARGET",
|
||||
groups = {AddGroup.class, EditGroup.class})
|
||||
private String connectionRole;
|
||||
|
||||
/**
|
||||
* 连接类型(DINGTALK、S3、ALIYUN_OSS)。
|
||||
*/
|
||||
@NotBlank(message = "连接类型不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Pattern(regexp = "DINGTALK|S3|ALIYUN_OSS", message = "连接类型不受支持",
|
||||
groups = {AddGroup.class, EditGroup.class})
|
||||
private String connectionType;
|
||||
|
||||
/**
|
||||
* 服务端点。
|
||||
*/
|
||||
private String endpoint;
|
||||
|
||||
/**
|
||||
* 服务区域。
|
||||
*/
|
||||
private String region;
|
||||
|
||||
/**
|
||||
* 存储桶名称。
|
||||
*/
|
||||
private String bucketName;
|
||||
|
||||
/**
|
||||
* 目标基础路径。
|
||||
*/
|
||||
private String basePath;
|
||||
|
||||
/**
|
||||
* 非敏感扩展配置 JSON。
|
||||
*/
|
||||
@JsonPattern(type = JsonType.OBJECT, message = "扩展配置必须是 JSON 对象",
|
||||
groups = {AddGroup.class, EditGroup.class})
|
||||
private String configJson;
|
||||
|
||||
/**
|
||||
* 敏感凭证配置 JSON;修改时留空表示保留原值。
|
||||
*/
|
||||
@JsonPattern(type = JsonType.OBJECT, message = "敏感配置必须是 JSON 对象",
|
||||
groups = {AddGroup.class, EditGroup.class})
|
||||
private String secretJson;
|
||||
|
||||
/**
|
||||
* 状态(0 正常、1 停用)。
|
||||
*/
|
||||
@NotBlank(message = "连接状态不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Pattern(regexp = "[01]", message = "连接状态只支持 0 或 1",
|
||||
groups = {AddGroup.class, EditGroup.class})
|
||||
private String status;
|
||||
|
||||
/**
|
||||
* 备注。
|
||||
*/
|
||||
private String remark;
|
||||
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
package org.dromara.sync.domain.bo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 同步任务查询对象。
|
||||
*/
|
||||
@Data
|
||||
public class SyncJobBo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private Long planId;
|
||||
private String triggerType;
|
||||
private String runType;
|
||||
private String status;
|
||||
private Map<String, Object> params = new HashMap<>();
|
||||
}
|
||||
@ -0,0 +1,22 @@
|
||||
package org.dromara.sync.domain.bo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* 同步任务明细查询对象。
|
||||
*/
|
||||
@Data
|
||||
public class SyncJobItemBo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private Long jobId;
|
||||
private String sourcePath;
|
||||
private String objectType;
|
||||
private String actionType;
|
||||
private String status;
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package org.dromara.sync.domain.bo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* 同步对象清单查询对象。
|
||||
*/
|
||||
@Data
|
||||
public class SyncObjectBo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private Long planId;
|
||||
private String sourceObjectId;
|
||||
private String sourcePath;
|
||||
private String objectType;
|
||||
private String syncStatus;
|
||||
private String sourceDeleted;
|
||||
}
|
||||
@ -0,0 +1,165 @@
|
||||
package org.dromara.sync.domain.bo;
|
||||
|
||||
import io.github.linpeilie.annotations.AutoMapper;
|
||||
import jakarta.validation.constraints.Max;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
import jakarta.validation.constraints.PositiveOrZero;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Data;
|
||||
import org.dromara.common.core.validate.AddGroup;
|
||||
import org.dromara.common.core.validate.EditGroup;
|
||||
import org.dromara.sync.domain.SyncPlan;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 同步计划业务对象 sync_plan。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
@Data
|
||||
@AutoMapper(target = SyncPlan.class, reverseConvertGenerate = false)
|
||||
public class SyncPlanBo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* 计划主键。
|
||||
*/
|
||||
@NotNull(message = "计划主键不能为空", groups = EditGroup.class)
|
||||
private Long planId;
|
||||
|
||||
/**
|
||||
* 计划名称。
|
||||
*/
|
||||
@NotBlank(message = "计划名称不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Size(max = 100, message = "计划名称不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class})
|
||||
private String planName;
|
||||
|
||||
/**
|
||||
* 源端连接主键。
|
||||
*/
|
||||
@NotNull(message = "源端连接不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
private Long sourceConnectionId;
|
||||
|
||||
/**
|
||||
* 目标端连接主键。
|
||||
*/
|
||||
@NotNull(message = "目标端连接不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
private Long targetConnectionId;
|
||||
|
||||
/**
|
||||
* 源端同步根路径或对象标识。
|
||||
*/
|
||||
@Size(max = 1024, message = "源端同步根路径不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class})
|
||||
private String sourceRoot;
|
||||
|
||||
/**
|
||||
* 目标端对象键前缀。
|
||||
*/
|
||||
@Size(max = 1024, message = "目标端对象键前缀不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class})
|
||||
private String targetPrefix;
|
||||
|
||||
/**
|
||||
* 同步模式(FULL、INCREMENTAL)。
|
||||
*/
|
||||
@NotBlank(message = "同步模式不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Pattern(regexp = "FULL|INCREMENTAL", message = "同步模式不正确", groups = {AddGroup.class, EditGroup.class})
|
||||
private String syncMode;
|
||||
|
||||
/**
|
||||
* 调度类型(MANUAL、CRON)。
|
||||
*/
|
||||
@NotBlank(message = "调度类型不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Pattern(regexp = "MANUAL|CRON", message = "调度类型不正确", groups = {AddGroup.class, EditGroup.class})
|
||||
private String scheduleType;
|
||||
|
||||
/**
|
||||
* CRON 表达式。
|
||||
*/
|
||||
@Size(max = 128, message = "CRON表达式不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class})
|
||||
private String cronExpression;
|
||||
|
||||
/**
|
||||
* 冲突处理策略(OVERWRITE、SKIP、KEEP_BOTH)。
|
||||
*/
|
||||
@NotBlank(message = "冲突处理策略不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Pattern(regexp = "OVERWRITE|SKIP|KEEP_BOTH", message = "冲突处理策略不正确",
|
||||
groups = {AddGroup.class, EditGroup.class})
|
||||
private String conflictStrategy;
|
||||
|
||||
/**
|
||||
* 源端删除处理策略(KEEP、MARK、DELETE)。
|
||||
*/
|
||||
@NotBlank(message = "删除处理策略不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Pattern(regexp = "KEEP|MARK|DELETE", message = "删除处理策略不正确", groups = {AddGroup.class, EditGroup.class})
|
||||
private String deleteStrategy;
|
||||
|
||||
/**
|
||||
* 单次目标删除比例保护阈值,100 表示允许一次删除全部对象。
|
||||
*/
|
||||
@NotNull(message = "删除保护阈值不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Min(value = 0, message = "删除保护阈值不能小于{value}", groups = {AddGroup.class, EditGroup.class})
|
||||
@Max(value = 100, message = "删除保护阈值不能大于{value}", groups = {AddGroup.class, EditGroup.class})
|
||||
private Integer deleteGuardPercent;
|
||||
|
||||
/**
|
||||
* 完整性校验方式(SIZE、ETAG、SHA256)。
|
||||
*/
|
||||
@NotBlank(message = "校验方式不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Pattern(regexp = "SIZE|ETAG|SHA256", message = "校验方式不正确", groups = {AddGroup.class, EditGroup.class})
|
||||
private String verifyMode;
|
||||
|
||||
/**
|
||||
* 最大并发传输数,范围为 1 至 100。
|
||||
*/
|
||||
@NotNull(message = "最大并发数不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Min(value = 1, message = "最大并发数不能小于{value}", groups = {AddGroup.class, EditGroup.class})
|
||||
@Max(value = 100, message = "最大并发数不能大于{value}", groups = {AddGroup.class, EditGroup.class})
|
||||
private Integer maxConcurrency;
|
||||
|
||||
/**
|
||||
* 带宽上限,单位 KB/s;0 表示不限速。
|
||||
*/
|
||||
@NotNull(message = "带宽上限不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@PositiveOrZero(message = "带宽上限不能小于0", groups = {AddGroup.class, EditGroup.class})
|
||||
private Long bandwidthLimitKbps;
|
||||
|
||||
/**
|
||||
* 状态(0 正常、1 停用)。
|
||||
*/
|
||||
@NotBlank(message = "状态不能为空", groups = {AddGroup.class, EditGroup.class})
|
||||
@Pattern(regexp = "0|1", message = "状态值不正确", groups = {AddGroup.class, EditGroup.class})
|
||||
private String status;
|
||||
|
||||
/**
|
||||
* 最近一次运行时间。
|
||||
*/
|
||||
private LocalDateTime lastRunTime;
|
||||
|
||||
/**
|
||||
* 下一次计划运行时间。
|
||||
*/
|
||||
private LocalDateTime nextRunTime;
|
||||
|
||||
/**
|
||||
* 备注。
|
||||
*/
|
||||
@Size(max = 500, message = "备注不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class})
|
||||
private String remark;
|
||||
|
||||
/**
|
||||
* 扩展查询参数。
|
||||
*/
|
||||
private Map<String, Object> params = new HashMap<>();
|
||||
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
package org.dromara.sync.domain.vo;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* 钉钉 Web 设备授权会话视图。
|
||||
*
|
||||
* <p>该对象只包含一次性授权链接、验证码和脱敏身份信息,绝不包含
|
||||
* access token、refresh token、device code 或应用密钥。</p>
|
||||
*
|
||||
* @param sessionId 授权会话 ID
|
||||
* @param connectionId 同步连接 ID
|
||||
* @param status 会话状态
|
||||
* @param verificationUri 用户输入验证码的授权地址
|
||||
* @param verificationUriComplete 可直接打开或生成二维码的完整授权地址
|
||||
* @param userCode 一次性用户验证码
|
||||
* @param expiresAt 会话过期时间(Unix 毫秒)
|
||||
* @param remainingSeconds 距离过期的秒数
|
||||
* @param pollInterval 前端建议轮询间隔(秒)
|
||||
* @param corpId 授权后的组织 ID
|
||||
* @param corpName 授权后的组织名称
|
||||
* @param userId 授权后的钉钉用户 ID
|
||||
* @param userName 授权后的用户名称
|
||||
* @param profile DWS 稳定 profile(corpId:userId)
|
||||
* @param message 面向用户的脱敏提示
|
||||
*/
|
||||
public record DwsAuthSessionVo(
|
||||
String sessionId,
|
||||
Long connectionId,
|
||||
String status,
|
||||
String verificationUri,
|
||||
String verificationUriComplete,
|
||||
String userCode,
|
||||
Long expiresAt,
|
||||
long remainingSeconds,
|
||||
int pollInterval,
|
||||
String corpId,
|
||||
String corpName,
|
||||
String userId,
|
||||
String userName,
|
||||
String profile,
|
||||
String message
|
||||
) implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
package org.dromara.sync.domain.vo;
|
||||
|
||||
import io.github.linpeilie.annotations.AutoMapper;
|
||||
import lombok.Data;
|
||||
import org.dromara.sync.domain.SyncCheckpoint;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 同步断点视图对象。
|
||||
*/
|
||||
@Data
|
||||
@AutoMapper(target = SyncCheckpoint.class)
|
||||
public class SyncCheckpointVo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private Long checkpointId;
|
||||
private Long planId;
|
||||
private String checkpointType;
|
||||
private String checkpointKey;
|
||||
private String checkpointValue;
|
||||
private LocalDateTime watermarkTime;
|
||||
private Long lastJobId;
|
||||
private Long version;
|
||||
}
|
||||
@ -0,0 +1,106 @@
|
||||
package org.dromara.sync.domain.vo;
|
||||
|
||||
import io.github.linpeilie.annotations.AutoMapper;
|
||||
import lombok.Data;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 同步连接视图对象 sync_connection。
|
||||
*
|
||||
* <p>敏感凭证字段不在该视图对象中声明,任何查询接口均不会返回凭证内容。</p>
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
@Data
|
||||
@AutoMapper(target = SyncConnection.class)
|
||||
public class SyncConnectionVo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* 连接主键。
|
||||
*/
|
||||
private Long connectionId;
|
||||
|
||||
/**
|
||||
* 连接名称。
|
||||
*/
|
||||
private String connectionName;
|
||||
|
||||
/**
|
||||
* 连接角色(SOURCE 源端、TARGET 目标端)。
|
||||
*/
|
||||
private String connectionRole;
|
||||
|
||||
/**
|
||||
* 连接类型(DINGTALK、S3、ALIYUN_OSS)。
|
||||
*/
|
||||
private String connectionType;
|
||||
|
||||
/**
|
||||
* 服务端点。
|
||||
*/
|
||||
private String endpoint;
|
||||
|
||||
/**
|
||||
* 服务区域。
|
||||
*/
|
||||
private String region;
|
||||
|
||||
/**
|
||||
* 存储桶名称。
|
||||
*/
|
||||
private String bucketName;
|
||||
|
||||
/**
|
||||
* 目标基础路径。
|
||||
*/
|
||||
private String basePath;
|
||||
|
||||
/**
|
||||
* 非敏感扩展配置 JSON。
|
||||
*/
|
||||
private String configJson;
|
||||
|
||||
/**
|
||||
* 状态(0 正常、1 停用)。
|
||||
*/
|
||||
private String status;
|
||||
|
||||
/**
|
||||
* 备注。
|
||||
*/
|
||||
private String remark;
|
||||
|
||||
/**
|
||||
* 创建部门。
|
||||
*/
|
||||
private Long createDept;
|
||||
|
||||
/**
|
||||
* 创建者。
|
||||
*/
|
||||
private Long createBy;
|
||||
|
||||
/**
|
||||
* 创建时间。
|
||||
*/
|
||||
private LocalDateTime createTime;
|
||||
|
||||
/**
|
||||
* 更新者。
|
||||
*/
|
||||
private Long updateBy;
|
||||
|
||||
/**
|
||||
* 更新时间。
|
||||
*/
|
||||
private LocalDateTime updateTime;
|
||||
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
package org.dromara.sync.domain.vo;
|
||||
|
||||
import io.github.linpeilie.annotations.AutoMapper;
|
||||
import lombok.Data;
|
||||
import org.dromara.sync.domain.SyncJobItem;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 同步任务明细视图对象。
|
||||
*/
|
||||
@Data
|
||||
@AutoMapper(target = SyncJobItem.class)
|
||||
public class SyncJobItemVo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private Long itemId;
|
||||
private Long jobId;
|
||||
private String sourceObjectId;
|
||||
private String parentObjectId;
|
||||
private String sourcePath;
|
||||
private String targetKey;
|
||||
private String objectType;
|
||||
private String actionType;
|
||||
private String status;
|
||||
private Long size;
|
||||
private Long transferredBytes;
|
||||
private String versionToken;
|
||||
private String sourceEtag;
|
||||
private String sourceSha256;
|
||||
private String targetEtag;
|
||||
private String targetVersionId;
|
||||
private Integer retryCount;
|
||||
private LocalDateTime startTime;
|
||||
private LocalDateTime finishTime;
|
||||
private String errorCode;
|
||||
private String errorMessage;
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
package org.dromara.sync.domain.vo;
|
||||
|
||||
import io.github.linpeilie.annotations.AutoMapper;
|
||||
import lombok.Data;
|
||||
import org.dromara.sync.domain.SyncJob;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 同步任务视图对象。
|
||||
*/
|
||||
@Data
|
||||
@AutoMapper(target = SyncJob.class)
|
||||
public class SyncJobVo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private Long jobId;
|
||||
private Long planId;
|
||||
private String triggerType;
|
||||
private String runType;
|
||||
private String status;
|
||||
private String checkpointBefore;
|
||||
private String checkpointAfter;
|
||||
private Long totalCount;
|
||||
private Long processedCount;
|
||||
private Long successCount;
|
||||
private Long failedCount;
|
||||
private Long skippedCount;
|
||||
private Long deletedCount;
|
||||
private Long totalBytes;
|
||||
private Long transferredBytes;
|
||||
private LocalDateTime startTime;
|
||||
private LocalDateTime finishTime;
|
||||
private String errorMessage;
|
||||
private LocalDateTime createTime;
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
package org.dromara.sync.domain.vo;
|
||||
|
||||
import io.github.linpeilie.annotations.AutoMapper;
|
||||
import lombok.Data;
|
||||
import org.dromara.sync.domain.SyncObject;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 同步对象清单视图对象。
|
||||
*/
|
||||
@Data
|
||||
@AutoMapper(target = SyncObject.class)
|
||||
public class SyncObjectVo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private Long objectId;
|
||||
private Long planId;
|
||||
private String sourceObjectId;
|
||||
private String parentObjectId;
|
||||
private String sourcePath;
|
||||
private String objectName;
|
||||
private String objectType;
|
||||
private Long size;
|
||||
private LocalDateTime modifiedTime;
|
||||
private String versionToken;
|
||||
private String sourceEtag;
|
||||
private String sourceSha256;
|
||||
private String contentType;
|
||||
private String metadataJson;
|
||||
private String targetKey;
|
||||
private String targetVersionId;
|
||||
private String targetEtag;
|
||||
private String syncStatus;
|
||||
private String sourceDeleted;
|
||||
private Long firstSeenJobId;
|
||||
private Long lastSeenJobId;
|
||||
private Long lastSyncJobId;
|
||||
private LocalDateTime lastSyncTime;
|
||||
private String lastErrorMessage;
|
||||
private LocalDateTime createTime;
|
||||
private LocalDateTime updateTime;
|
||||
}
|
||||
@ -0,0 +1,144 @@
|
||||
package org.dromara.sync.domain.vo;
|
||||
|
||||
import io.github.linpeilie.annotations.AutoMapper;
|
||||
import lombok.Data;
|
||||
import org.dromara.sync.domain.SyncPlan;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 同步计划视图对象 sync_plan。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
@Data
|
||||
@AutoMapper(target = SyncPlan.class)
|
||||
public class SyncPlanVo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* 计划主键。
|
||||
*/
|
||||
private Long planId;
|
||||
|
||||
/**
|
||||
* 计划名称。
|
||||
*/
|
||||
private String planName;
|
||||
|
||||
/**
|
||||
* 源端连接主键。
|
||||
*/
|
||||
private Long sourceConnectionId;
|
||||
|
||||
/**
|
||||
* 目标端连接主键。
|
||||
*/
|
||||
private Long targetConnectionId;
|
||||
|
||||
/**
|
||||
* 源端同步根路径或对象标识。
|
||||
*/
|
||||
private String sourceRoot;
|
||||
|
||||
/**
|
||||
* 目标端对象键前缀。
|
||||
*/
|
||||
private String targetPrefix;
|
||||
|
||||
/**
|
||||
* 同步模式(FULL、INCREMENTAL)。
|
||||
*/
|
||||
private String syncMode;
|
||||
|
||||
/**
|
||||
* 调度类型(MANUAL、CRON)。
|
||||
*/
|
||||
private String scheduleType;
|
||||
|
||||
/**
|
||||
* CRON 表达式。
|
||||
*/
|
||||
private String cronExpression;
|
||||
|
||||
/**
|
||||
* 冲突处理策略(OVERWRITE、SKIP、KEEP_BOTH)。
|
||||
*/
|
||||
private String conflictStrategy;
|
||||
|
||||
/**
|
||||
* 源端删除处理策略(KEEP、MARK、DELETE)。
|
||||
*/
|
||||
private String deleteStrategy;
|
||||
|
||||
/**
|
||||
* 单次目标删除比例保护阈值。
|
||||
*/
|
||||
private Integer deleteGuardPercent;
|
||||
|
||||
/**
|
||||
* 完整性校验方式(SIZE、ETAG、SHA256)。
|
||||
*/
|
||||
private String verifyMode;
|
||||
|
||||
/**
|
||||
* 最大并发传输数。
|
||||
*/
|
||||
private Integer maxConcurrency;
|
||||
|
||||
/**
|
||||
* 带宽上限,单位 KB/s;0 表示不限速。
|
||||
*/
|
||||
private Long bandwidthLimitKbps;
|
||||
|
||||
/**
|
||||
* 状态(0 正常、1 停用)。
|
||||
*/
|
||||
private String status;
|
||||
|
||||
/**
|
||||
* 最近一次运行时间。
|
||||
*/
|
||||
private LocalDateTime lastRunTime;
|
||||
|
||||
/**
|
||||
* 下一次计划运行时间。
|
||||
*/
|
||||
private LocalDateTime nextRunTime;
|
||||
|
||||
/**
|
||||
* 备注。
|
||||
*/
|
||||
private String remark;
|
||||
|
||||
/**
|
||||
* 创建部门。
|
||||
*/
|
||||
private Long createDept;
|
||||
|
||||
/**
|
||||
* 创建者。
|
||||
*/
|
||||
private Long createBy;
|
||||
|
||||
/**
|
||||
* 创建时间。
|
||||
*/
|
||||
private LocalDateTime createTime;
|
||||
|
||||
/**
|
||||
* 更新者。
|
||||
*/
|
||||
private Long updateBy;
|
||||
|
||||
/**
|
||||
* 更新时间。
|
||||
*/
|
||||
private LocalDateTime updateTime;
|
||||
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package org.dromara.sync.domain.vo;
|
||||
|
||||
import io.github.linpeilie.annotations.AutoMapper;
|
||||
import lombok.Data;
|
||||
import org.dromara.sync.domain.SyncTransferPart;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 传输分片视图对象。
|
||||
*/
|
||||
@Data
|
||||
@AutoMapper(target = SyncTransferPart.class)
|
||||
public class SyncTransferPartVo implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private Long partId;
|
||||
private Long jobItemId;
|
||||
private String uploadId;
|
||||
private Integer partNumber;
|
||||
private Long partOffset;
|
||||
private Long partSize;
|
||||
private Long transferredBytes;
|
||||
private String partEtag;
|
||||
private String checksumSha256;
|
||||
private String status;
|
||||
private Integer retryCount;
|
||||
private LocalDateTime startTime;
|
||||
private LocalDateTime finishTime;
|
||||
private String errorMessage;
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
package org.dromara.sync.job;
|
||||
|
||||
import cn.hutool.core.convert.Convert;
|
||||
import com.aizuda.snailjob.client.job.core.annotation.JobExecutor;
|
||||
import com.aizuda.snailjob.client.job.core.dto.JobArgs;
|
||||
import com.aizuda.snailjob.model.dto.ExecuteResult;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.dromara.sync.service.ISyncJobService;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* SnailJob 同步计划执行入口。
|
||||
*
|
||||
* <p>执行器名称为 {@code syncPlanJobExecutor},任务参数填写同步计划 ID。</p>
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@JobExecutor(name = "syncPlanJobExecutor")
|
||||
public class SyncPlanJobExecutor {
|
||||
|
||||
private final ISyncJobService jobService;
|
||||
|
||||
public ExecuteResult jobExecute(JobArgs jobArgs) {
|
||||
Long planId = Convert.toLong(jobArgs.getJobParams());
|
||||
if (planId == null) {
|
||||
return ExecuteResult.failure("任务参数必须是同步计划 ID");
|
||||
}
|
||||
Long jobId = jobService.startPlan(planId, "SCHEDULE", null);
|
||||
return ExecuteResult.success("同步任务已提交,jobId=" + jobId);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,75 @@
|
||||
package org.dromara.sync.job;
|
||||
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.dromara.sync.constant.SyncConstants;
|
||||
import org.dromara.sync.domain.SyncPlan;
|
||||
import org.dromara.sync.mapper.SyncPlanMapper;
|
||||
import org.dromara.sync.service.ISyncJobService;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.scheduling.support.CronExpression;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 从同步计划表领取到期的 Cron 计划。
|
||||
*
|
||||
* <p>通过 next_run_time 条件更新完成多实例抢占,同一触发时刻只有一个应用实例能够提交任务。</p>
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnProperty(prefix = "sync.scheduler", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class SyncPlanScheduler {
|
||||
|
||||
private final SyncPlanMapper planMapper;
|
||||
private final ISyncJobService jobService;
|
||||
|
||||
@Scheduled(fixedDelayString = "${sync.scheduler.poll-interval-ms:30000}")
|
||||
public void dispatchDuePlans() {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
List<SyncPlan> plans = planMapper.selectList(Wrappers.<SyncPlan>lambdaQuery()
|
||||
.eq(SyncPlan::getStatus, SyncConstants.STATUS_NORMAL)
|
||||
.eq(SyncPlan::getScheduleType, "CRON")
|
||||
.isNotNull(SyncPlan::getNextRunTime)
|
||||
.le(SyncPlan::getNextRunTime, now)
|
||||
.orderByAsc(SyncPlan::getNextRunTime)
|
||||
.last("limit 100"));
|
||||
for (SyncPlan plan : plans) {
|
||||
dispatch(plan, now);
|
||||
}
|
||||
}
|
||||
|
||||
private void dispatch(SyncPlan plan, LocalDateTime now) {
|
||||
try {
|
||||
LocalDateTime nextRunTime = nextRunTime(plan.getCronExpression(), now);
|
||||
boolean claimed = planMapper.lambda()
|
||||
.set(SyncPlan::getNextRunTime, nextRunTime)
|
||||
.eq(SyncPlan::getPlanId, plan.getPlanId())
|
||||
.eq(SyncPlan::getNextRunTime, plan.getNextRunTime())
|
||||
.eq(SyncPlan::getCronExpression, plan.getCronExpression())
|
||||
.eq(SyncPlan::getScheduleType, "CRON")
|
||||
.eq(SyncPlan::getStatus, SyncConstants.STATUS_NORMAL)
|
||||
.update();
|
||||
if (!claimed) {
|
||||
return;
|
||||
}
|
||||
jobService.startPlan(plan.getPlanId(), "SCHEDULE", null);
|
||||
} catch (Exception e) {
|
||||
log.warn("到期同步计划提交失败,planId={},原因={}", plan.getPlanId(), e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private LocalDateTime nextRunTime(String cronExpression, LocalDateTime now) {
|
||||
ZonedDateTime next = CronExpression.parse(cronExpression).next(now.atZone(java.time.ZoneId.systemDefault()));
|
||||
if (next == null) {
|
||||
throw new IllegalArgumentException("Cron 表达式没有下一次触发时间");
|
||||
}
|
||||
return next.toLocalDateTime();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package org.dromara.sync.mapper;
|
||||
|
||||
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
|
||||
import org.dromara.sync.domain.SyncCheckpoint;
|
||||
import org.dromara.sync.domain.vo.SyncCheckpointVo;
|
||||
|
||||
/**
|
||||
* 同步断点 Mapper。
|
||||
*/
|
||||
public interface SyncCheckpointMapper extends BaseMapperPlus<SyncCheckpoint, SyncCheckpointVo> {
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
package org.dromara.sync.mapper;
|
||||
|
||||
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
import org.dromara.sync.domain.vo.SyncConnectionVo;
|
||||
|
||||
/**
|
||||
* 同步连接 Mapper 接口。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
public interface SyncConnectionMapper extends BaseMapperPlus<SyncConnection, SyncConnectionVo> {
|
||||
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package org.dromara.sync.mapper;
|
||||
|
||||
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
|
||||
import org.dromara.sync.domain.SyncJobItem;
|
||||
import org.dromara.sync.domain.vo.SyncJobItemVo;
|
||||
|
||||
/**
|
||||
* 同步任务明细 Mapper。
|
||||
*/
|
||||
public interface SyncJobItemMapper extends BaseMapperPlus<SyncJobItem, SyncJobItemVo> {
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package org.dromara.sync.mapper;
|
||||
|
||||
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
|
||||
import org.dromara.sync.domain.SyncJob;
|
||||
import org.dromara.sync.domain.vo.SyncJobVo;
|
||||
|
||||
/**
|
||||
* 同步任务 Mapper。
|
||||
*/
|
||||
public interface SyncJobMapper extends BaseMapperPlus<SyncJob, SyncJobVo> {
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package org.dromara.sync.mapper;
|
||||
|
||||
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
|
||||
import org.dromara.sync.domain.SyncObject;
|
||||
import org.dromara.sync.domain.vo.SyncObjectVo;
|
||||
|
||||
/**
|
||||
* 同步对象清单 Mapper。
|
||||
*/
|
||||
public interface SyncObjectMapper extends BaseMapperPlus<SyncObject, SyncObjectVo> {
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
package org.dromara.sync.mapper;
|
||||
|
||||
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
|
||||
import org.dromara.sync.domain.SyncPlan;
|
||||
import org.dromara.sync.domain.vo.SyncPlanVo;
|
||||
|
||||
/**
|
||||
* 同步计划 Mapper 接口。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
public interface SyncPlanMapper extends BaseMapperPlus<SyncPlan, SyncPlanVo> {
|
||||
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package org.dromara.sync.mapper;
|
||||
|
||||
import org.dromara.common.mybatis.core.mapper.BaseMapperPlus;
|
||||
import org.dromara.sync.domain.SyncTransferPart;
|
||||
import org.dromara.sync.domain.vo.SyncTransferPartVo;
|
||||
|
||||
/**
|
||||
* 同步传输分片 Mapper。
|
||||
*/
|
||||
public interface SyncTransferPartMapper extends BaseMapperPlus<SyncTransferPart, SyncTransferPartVo> {
|
||||
}
|
||||
@ -0,0 +1,109 @@
|
||||
package org.dromara.sync.service;
|
||||
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.sync.connector.model.ConnectorTestResult;
|
||||
import org.dromara.sync.domain.bo.SyncConnectionBo;
|
||||
import org.dromara.sync.domain.vo.DwsAuthSessionVo;
|
||||
import org.dromara.sync.domain.vo.SyncConnectionVo;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 同步连接 Service 接口。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
public interface ISyncConnectionService {
|
||||
|
||||
/**
|
||||
* 根据主键查询同步连接。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @return 同步连接详情
|
||||
*/
|
||||
SyncConnectionVo queryById(Long connectionId);
|
||||
|
||||
/**
|
||||
* 分页查询同步连接列表。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @param pageQuery 分页参数
|
||||
* @return 同步连接分页列表
|
||||
*/
|
||||
PageResult<SyncConnectionVo> queryPageList(SyncConnectionBo bo, PageQuery pageQuery);
|
||||
|
||||
/**
|
||||
* 查询符合条件的同步连接列表。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @return 同步连接列表
|
||||
*/
|
||||
List<SyncConnectionVo> queryList(SyncConnectionBo bo);
|
||||
|
||||
/**
|
||||
* 新增同步连接。
|
||||
*
|
||||
* @param bo 同步连接业务对象
|
||||
* @return 是否新增成功
|
||||
*/
|
||||
Boolean insertByBo(SyncConnectionBo bo);
|
||||
|
||||
/**
|
||||
* 修改同步连接。
|
||||
*
|
||||
* @param bo 同步连接业务对象
|
||||
* @return 是否修改成功
|
||||
*/
|
||||
Boolean updateByBo(SyncConnectionBo bo);
|
||||
|
||||
/**
|
||||
* 校验并批量删除同步连接。
|
||||
*
|
||||
* @param ids 待删除的连接主键集合
|
||||
* @param isValid 是否执行删除前校验
|
||||
* @return 是否删除成功
|
||||
*/
|
||||
Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid);
|
||||
|
||||
/**
|
||||
* 校验配置并调用对应连接器执行连通性测试。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @return 连通性测试结果
|
||||
*/
|
||||
ConnectorTestResult testConnection(Long connectionId);
|
||||
|
||||
/**
|
||||
* 启动钉钉 Web 设备授权。
|
||||
*
|
||||
* @param connectionId 钉钉连接主键
|
||||
* @param ownerId 发起授权的系统用户
|
||||
* @param expectedCorpId 可选的组织 ID,用于防止误授权到其他组织
|
||||
* @return 授权会话快照
|
||||
*/
|
||||
DwsAuthSessionVo startDingTalkAuth(Long connectionId, Long ownerId, String expectedCorpId);
|
||||
|
||||
/**
|
||||
* 查询钉钉 Web 设备授权状态。
|
||||
*
|
||||
* @param connectionId 钉钉连接主键
|
||||
* @param sessionId 授权会话 ID
|
||||
* @param ownerId 发起授权的系统用户
|
||||
* @return 授权会话快照
|
||||
*/
|
||||
DwsAuthSessionVo getDingTalkAuth(Long connectionId, String sessionId, Long ownerId);
|
||||
|
||||
/**
|
||||
* 取消钉钉 Web 设备授权。
|
||||
*
|
||||
* @param connectionId 钉钉连接主键
|
||||
* @param sessionId 授权会话 ID
|
||||
* @param ownerId 发起授权的系统用户
|
||||
* @return 取消后的会话快照
|
||||
*/
|
||||
DwsAuthSessionVo cancelDingTalkAuth(Long connectionId, String sessionId, Long ownerId);
|
||||
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
package org.dromara.sync.service;
|
||||
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.sync.domain.bo.SyncJobBo;
|
||||
import org.dromara.sync.domain.bo.SyncJobItemBo;
|
||||
import org.dromara.sync.domain.vo.SyncJobItemVo;
|
||||
import org.dromara.sync.domain.vo.SyncJobVo;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
/**
|
||||
* 同步任务服务。
|
||||
*/
|
||||
public interface ISyncJobService {
|
||||
|
||||
SyncJobVo queryById(Long jobId);
|
||||
|
||||
PageResult<SyncJobVo> queryPageList(SyncJobBo bo, PageQuery pageQuery);
|
||||
|
||||
PageResult<SyncJobItemVo> queryItemPageList(SyncJobItemBo bo, PageQuery pageQuery);
|
||||
|
||||
Long startPlan(Long planId, String triggerType, String runType);
|
||||
|
||||
Long retry(Long jobId);
|
||||
|
||||
Boolean cancel(Long jobId);
|
||||
|
||||
Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid);
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
package org.dromara.sync.service;
|
||||
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.sync.domain.bo.SyncObjectBo;
|
||||
import org.dromara.sync.domain.vo.SyncObjectVo;
|
||||
|
||||
/**
|
||||
* 同步对象清单服务。
|
||||
*/
|
||||
public interface ISyncObjectService {
|
||||
|
||||
SyncObjectVo queryById(Long objectId);
|
||||
|
||||
PageResult<SyncObjectVo> queryPageList(SyncObjectBo bo, PageQuery pageQuery);
|
||||
}
|
||||
@ -0,0 +1,77 @@
|
||||
package org.dromara.sync.service;
|
||||
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.sync.domain.bo.SyncPlanBo;
|
||||
import org.dromara.sync.domain.vo.SyncPlanVo;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 同步计划 Service 接口。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
public interface ISyncPlanService {
|
||||
|
||||
/**
|
||||
* 根据主键查询同步计划。
|
||||
*
|
||||
* @param planId 计划主键
|
||||
* @return 同步计划详情
|
||||
*/
|
||||
SyncPlanVo queryById(Long planId);
|
||||
|
||||
/**
|
||||
* 分页查询同步计划列表。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @param pageQuery 分页参数
|
||||
* @return 同步计划分页列表
|
||||
*/
|
||||
PageResult<SyncPlanVo> queryPageList(SyncPlanBo bo, PageQuery pageQuery);
|
||||
|
||||
/**
|
||||
* 查询符合条件的同步计划列表。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @return 同步计划列表
|
||||
*/
|
||||
List<SyncPlanVo> queryList(SyncPlanBo bo);
|
||||
|
||||
/**
|
||||
* 校验计划名称是否唯一。
|
||||
*
|
||||
* @param bo 同步计划
|
||||
* @return 名称未被占用返回 {@code true}
|
||||
*/
|
||||
boolean checkPlanNameUnique(SyncPlanBo bo);
|
||||
|
||||
/**
|
||||
* 新增同步计划。
|
||||
*
|
||||
* @param bo 同步计划
|
||||
* @return 是否新增成功
|
||||
*/
|
||||
Boolean insertByBo(SyncPlanBo bo);
|
||||
|
||||
/**
|
||||
* 修改同步计划。
|
||||
*
|
||||
* @param bo 同步计划
|
||||
* @return 是否修改成功
|
||||
*/
|
||||
Boolean updateByBo(SyncPlanBo bo);
|
||||
|
||||
/**
|
||||
* 校验并批量删除同步计划。
|
||||
*
|
||||
* @param ids 计划主键集合
|
||||
* @param isValid 是否执行删除前业务校验
|
||||
* @return 是否删除成功
|
||||
*/
|
||||
Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid);
|
||||
|
||||
}
|
||||
@ -0,0 +1,716 @@
|
||||
package org.dromara.sync.service.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.core.utils.MapstructUtils;
|
||||
import org.dromara.common.core.utils.StringUtils;
|
||||
import org.dromara.common.json.utils.JsonUtils;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.common.mybatis.core.query.QueryBuilder;
|
||||
import org.dromara.common.satoken.utils.LoginHelper;
|
||||
import org.dromara.sync.connector.ConnectorRegistry;
|
||||
import org.dromara.sync.connector.dingtalk.DwsAuthIdentity;
|
||||
import org.dromara.sync.connector.dingtalk.DwsAuthSessionManager;
|
||||
import org.dromara.sync.connector.model.ConnectorTestResult;
|
||||
import org.dromara.sync.connector.s3.S3SseSetting;
|
||||
import org.dromara.sync.constant.SyncConstants;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
import org.dromara.sync.domain.SyncJob;
|
||||
import org.dromara.sync.domain.SyncPlan;
|
||||
import org.dromara.sync.domain.bo.SyncConnectionBo;
|
||||
import org.dromara.sync.domain.vo.DwsAuthSessionVo;
|
||||
import org.dromara.sync.domain.vo.SyncConnectionVo;
|
||||
import org.dromara.sync.mapper.SyncConnectionMapper;
|
||||
import org.dromara.sync.mapper.SyncJobMapper;
|
||||
import org.dromara.sync.mapper.SyncPlanMapper;
|
||||
import org.dromara.sync.service.ISyncConnectionService;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 同步连接 Service 业务层处理。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
@RequiredArgsConstructor
|
||||
@Service
|
||||
public class SyncConnectionServiceImpl implements ISyncConnectionService {
|
||||
|
||||
private static final Pattern DOWNLOAD_PART_SIZE_PATTERN = Pattern.compile("(?i)^([1-9]\\d*)(KB|MB|GB)$");
|
||||
private static final long ONE_MIB = 1024L * 1024L;
|
||||
private static final long ONE_GIB = 1024L * 1024L * 1024L;
|
||||
|
||||
private final SyncConnectionMapper connectionMapper;
|
||||
private final SyncPlanMapper planMapper;
|
||||
private final SyncJobMapper jobMapper;
|
||||
private final ConnectorRegistry connectorRegistry;
|
||||
private final DwsAuthSessionManager dwsAuthSessionManager;
|
||||
|
||||
/**
|
||||
* Serializes connection mutations with the asynchronous DWS profile
|
||||
* callback. Without this small critical section an edit could pass the
|
||||
* active-session check just before login starts and then overwrite the
|
||||
* profile written by the callback (or vice versa).
|
||||
*/
|
||||
private final Object connectionMutationLock = new Object();
|
||||
|
||||
@Value("${mybatis-encryptor.enable:false}")
|
||||
private boolean fieldEncryptionEnabled;
|
||||
|
||||
@Value("${mybatis-encryptor.password:}")
|
||||
private String fieldEncryptionPassword;
|
||||
|
||||
/**
|
||||
* 根据主键查询同步连接详情。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @return 同步连接详情
|
||||
*/
|
||||
@Override
|
||||
public SyncConnectionVo queryById(Long connectionId) {
|
||||
ensureConnectionAccess(connectionId);
|
||||
return connectionMapper.selectVoById(connectionId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页查询同步连接列表。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @param pageQuery 分页参数
|
||||
* @return 同步连接分页列表
|
||||
*/
|
||||
@Override
|
||||
public PageResult<SyncConnectionVo> queryPageList(SyncConnectionBo bo, PageQuery pageQuery) {
|
||||
LambdaQueryWrapper<SyncConnection> lqw = buildQueryWrapper(bo);
|
||||
Page<SyncConnectionVo> result = connectionMapper.selectVoPage(pageQuery.build(), lqw);
|
||||
return PageResult.build(result.getRecords(), result.getTotal());
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询符合条件的同步连接列表。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @return 同步连接列表
|
||||
*/
|
||||
@Override
|
||||
public List<SyncConnectionVo> queryList(SyncConnectionBo bo) {
|
||||
return connectionMapper.selectVoList(buildQueryWrapper(bo));
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造同步连接动态查询条件。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @return 查询条件包装器
|
||||
*/
|
||||
private LambdaQueryWrapper<SyncConnection> buildQueryWrapper(SyncConnectionBo bo) {
|
||||
LambdaQueryWrapper<SyncConnection> wrapper = QueryBuilder.lambda(SyncConnection.class)
|
||||
.eqIfPresent(SyncConnection::getConnectionId, bo.getConnectionId())
|
||||
.likeIfText(SyncConnection::getConnectionName, bo.getConnectionName())
|
||||
.eqIfText(SyncConnection::getConnectionRole, bo.getConnectionRole())
|
||||
.eqIfText(SyncConnection::getConnectionType, bo.getConnectionType())
|
||||
.likeIfText(SyncConnection::getEndpoint, bo.getEndpoint())
|
||||
.likeIfText(SyncConnection::getBucketName, bo.getBucketName())
|
||||
.eqIfText(SyncConnection::getStatus, bo.getStatus())
|
||||
.orderByAsc(SyncConnection::getConnectionId)
|
||||
.build();
|
||||
// Connections contain credentials and are creator-owned resources. Do
|
||||
// not rely on the menu permission alone for row-level isolation.
|
||||
if (!LoginHelper.isSuperAdmin()) {
|
||||
wrapper.eq(SyncConnection::getCreateBy, requireCurrentUserId());
|
||||
}
|
||||
return wrapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 新增同步连接。
|
||||
*
|
||||
* @param bo 同步连接业务对象
|
||||
* @return 是否新增成功
|
||||
*/
|
||||
@Override
|
||||
public Boolean insertByBo(SyncConnectionBo bo) {
|
||||
requireCurrentUserId();
|
||||
SyncConnection add = MapstructUtils.convert(bo, SyncConnection.class);
|
||||
validEntityBeforeSave(add);
|
||||
boolean flag;
|
||||
try {
|
||||
flag = connectionMapper.insert(add) > 0;
|
||||
} catch (DuplicateKeyException e) {
|
||||
throw new ServiceException("连接名称【{}】已存在", add.getConnectionName());
|
||||
}
|
||||
if (flag) {
|
||||
bo.setConnectionId(add.getConnectionId());
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
/**
|
||||
* 修改同步连接,未填写敏感配置时不更新数据库中的原凭证。
|
||||
*
|
||||
* @param bo 同步连接业务对象
|
||||
* @return 是否修改成功
|
||||
*/
|
||||
@Override
|
||||
public Boolean updateByBo(SyncConnectionBo bo) {
|
||||
synchronized (connectionMutationLock) {
|
||||
SyncConnection current = connectionMapper.selectById(bo.getConnectionId());
|
||||
if (current == null) {
|
||||
throw new ServiceException("同步连接不存在");
|
||||
}
|
||||
ensureConnectionAccess(current);
|
||||
if (hasActiveJob(current.getConnectionId())) {
|
||||
throw new ServiceException("连接正在被同步任务使用,任务结束后才能修改");
|
||||
}
|
||||
ensureNoDingTalkAuth(current.getConnectionId());
|
||||
SyncConnection update = MapstructUtils.convert(bo, SyncConnection.class);
|
||||
boolean connectorChanged = !Objects.equals(current.getConnectionRole(), update.getConnectionRole())
|
||||
|| !Objects.equals(current.getConnectionType(), update.getConnectionType());
|
||||
if (connectorChanged && isReferenced(update.getConnectionId())) {
|
||||
throw new ServiceException("连接已被同步计划引用,不能修改连接角色或类型");
|
||||
}
|
||||
if (connectorChanged && StringUtils.isBlank(update.getSecretJson())) {
|
||||
throw new ServiceException("修改连接角色或类型时必须重新填写敏感凭证");
|
||||
}
|
||||
preserveDingTalkProfile(current, update);
|
||||
validEntityBeforeSave(update);
|
||||
if (StringUtils.isBlank(update.getSecretJson())) {
|
||||
update.setSecretJson(null);
|
||||
}
|
||||
try {
|
||||
return connectionMapper.updateById(update) > 0;
|
||||
} catch (DuplicateKeyException e) {
|
||||
throw new ServiceException("连接名称【{}】已存在", update.getConnectionName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行保存前的唯一性、枚举组合与 JSON 格式校验。
|
||||
*
|
||||
* @param entity 待保存的同步连接
|
||||
*/
|
||||
private void validEntityBeforeSave(SyncConnection entity) {
|
||||
if (StringUtils.isNotBlank(entity.getSecretJson())
|
||||
&& (!fieldEncryptionEnabled || StringUtils.isBlank(fieldEncryptionPassword))) {
|
||||
throw new ServiceException("保存敏感凭证前必须启用 mybatis-encryptor 并配置 MYBATIS_ENCRYPTOR_PASSWORD");
|
||||
}
|
||||
entity.setConnectionName(StringUtils.trim(entity.getConnectionName()));
|
||||
entity.setConnectionRole(StringUtils.trim(entity.getConnectionRole()));
|
||||
entity.setConnectionType(StringUtils.trim(entity.getConnectionType()));
|
||||
entity.setStatus(StringUtils.trim(entity.getStatus()));
|
||||
entity.setEndpoint(StringUtils.trim(entity.getEndpoint()));
|
||||
entity.setRegion(StringUtils.trim(entity.getRegion()));
|
||||
entity.setBucketName(StringUtils.trim(entity.getBucketName()));
|
||||
entity.setBasePath(StringUtils.trim(entity.getBasePath()));
|
||||
requireText(entity.getConnectionName(), "连接名称不能为空");
|
||||
validateLength("连接名称", entity.getConnectionName(), 100);
|
||||
validateLength("服务端点", entity.getEndpoint(), 512);
|
||||
validateLength("区域", entity.getRegion(), 64);
|
||||
validateLength("存储桶名称", entity.getBucketName(), 255);
|
||||
validateLength("基础路径", entity.getBasePath(), 1024);
|
||||
validateLength("备注", entity.getRemark(), 500);
|
||||
validateObjectKeyPrefix(entity.getBasePath(), "基础路径");
|
||||
validateRoleAndType(entity);
|
||||
validateStatus(entity.getStatus());
|
||||
validateJsonObjectIfPresent(entity.getConfigJson(), "扩展配置");
|
||||
validateJsonObjectIfPresent(entity.getSecretJson(), "敏感配置");
|
||||
validateRequiredConfig(entity);
|
||||
if (!isConnectionNameUnique(entity)) {
|
||||
throw new ServiceException("连接名称【{}】已存在", entity.getConnectionName());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验连接名称唯一性,编辑时排除当前记录。
|
||||
*
|
||||
* @param entity 同步连接
|
||||
* @return 名称唯一返回 {@code true}
|
||||
*/
|
||||
private boolean isConnectionNameUnique(SyncConnection entity) {
|
||||
if (StringUtils.isBlank(entity.getConnectionName())) {
|
||||
return true;
|
||||
}
|
||||
LambdaQueryWrapper<SyncConnection> lqw = QueryBuilder.lambda(SyncConnection.class)
|
||||
.eq(SyncConnection::getConnectionName, entity.getConnectionName())
|
||||
.neIfPresent(SyncConnection::getConnectionId, entity.getConnectionId())
|
||||
.build();
|
||||
return !connectionMapper.exists(lqw);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验连接角色与连接类型组合。
|
||||
*
|
||||
* @param connection 同步连接
|
||||
*/
|
||||
private void validateRoleAndType(SyncConnection connection) {
|
||||
String role = connection.getConnectionRole();
|
||||
String type = connection.getConnectionType();
|
||||
if (SyncConstants.TYPE_DINGTALK.equals(type)) {
|
||||
if (!SyncConstants.ROLE_SOURCE.equals(role)) {
|
||||
throw new ServiceException("DINGTALK 连接只能配置为 SOURCE 角色");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (SyncConstants.TYPE_S3.equals(type) || SyncConstants.TYPE_ALIYUN_OSS.equals(type)) {
|
||||
if (!SyncConstants.ROLE_TARGET.equals(role)) {
|
||||
throw new ServiceException("{} 连接只能配置为 TARGET 角色", type);
|
||||
}
|
||||
return;
|
||||
}
|
||||
throw new ServiceException("不支持的连接类型:{}", type);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验连接状态取值。
|
||||
*
|
||||
* @param status 连接状态
|
||||
*/
|
||||
private void validateStatus(String status) {
|
||||
if (!SyncConstants.STATUS_NORMAL.equals(status) && !SyncConstants.STATUS_DISABLED.equals(status)) {
|
||||
throw new ServiceException("连接状态只支持 0 或 1");
|
||||
}
|
||||
}
|
||||
|
||||
private void validateRequiredConfig(SyncConnection connection) {
|
||||
validateRequiredConfig(connection, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验连接运行所需配置。
|
||||
*
|
||||
* @param connection 连接
|
||||
* @param requireProfile 是否要求已经完成 DWS profile 配置
|
||||
*/
|
||||
private void validateRequiredConfig(SyncConnection connection, boolean requireProfile) {
|
||||
boolean credentialRequired = connection.getConnectionId() == null
|
||||
|| StringUtils.isNotBlank(connection.getSecretJson());
|
||||
Map<String, Object> secrets = credentialRequired
|
||||
? parseSecretJson(connection.getSecretJson()) : Map.of();
|
||||
if (SyncConstants.TYPE_DINGTALK.equals(connection.getConnectionType())) {
|
||||
if (credentialRequired) {
|
||||
requireSecret(secrets, "clientId", "钉钉连接 clientId 不能为空");
|
||||
requireSecret(secrets, "clientSecret", "钉钉连接 clientSecret 不能为空");
|
||||
}
|
||||
Map<String, Object> config = parseOptionalJson(connection.getConfigJson());
|
||||
String profile = stringValue(config, "profile");
|
||||
if (requireProfile && StringUtils.isBlank(profile)) {
|
||||
throw new ServiceException("钉钉连接尚未完成登录,请先在网页端完成设备授权");
|
||||
}
|
||||
if (StringUtils.isNotBlank(profile) && !profile.matches("[^\\s:]+:[^\\s:]+")) {
|
||||
throw new ServiceException("钉钉 profile 必须使用 corpId:userId 稳定标识");
|
||||
}
|
||||
if (config.containsKey("configDir")) {
|
||||
throw new ServiceException("钉钉 configDir 由服务端按连接隔离分配,不能在连接配置中指定");
|
||||
}
|
||||
Object spaceType = config.get("spaceType");
|
||||
if (spaceType != null && !List.of("orgSpace", "mySpace").contains(String.valueOf(spaceType))) {
|
||||
throw new ServiceException("钉钉 spaceType 只支持 orgSpace 或 mySpace");
|
||||
}
|
||||
validateDownloadOptions(config);
|
||||
return;
|
||||
}
|
||||
requireText(connection.getEndpoint(), "目标连接 endpoint 不能为空");
|
||||
requireText(connection.getBucketName(), "目标连接 bucketName 不能为空");
|
||||
if (credentialRequired) {
|
||||
requireSecret(secrets, "accessKey", "目标连接 accessKey 不能为空");
|
||||
requireSecret(secrets, "secretKey", "目标连接 secretKey 不能为空");
|
||||
}
|
||||
// 加密方式与 KMS 密钥组合的合法性由连接器统一定义,保存时提前拦截非法配置。
|
||||
S3SseSetting.from(parseOptionalJson(connection.getConfigJson()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验可选 JSON 字段格式。
|
||||
*
|
||||
* @param value JSON 字符串
|
||||
* @param fieldName 字段名称
|
||||
*/
|
||||
private void validateJsonObjectIfPresent(String value, String fieldName) {
|
||||
if (StringUtils.isNotBlank(value) && !JsonUtils.isJsonObject(value)) {
|
||||
throw new ServiceException("{}必须是 JSON 对象", fieldName);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> parseOptionalJson(String json) {
|
||||
return StringUtils.isBlank(json) ? Map.of() : JsonUtils.parseMap(json);
|
||||
}
|
||||
|
||||
private void validateLength(String fieldName, String value, int maxLength) {
|
||||
if (value != null && value.length() > maxLength) {
|
||||
throw new ServiceException("{}不能超过{}个字符", fieldName, maxLength);
|
||||
}
|
||||
}
|
||||
|
||||
private void validateObjectKeyPrefix(String value, String fieldName) {
|
||||
if (StringUtils.isBlank(value)) {
|
||||
return;
|
||||
}
|
||||
for (String segment : value.replace('\\', '/').split("/")) {
|
||||
if (".".equals(segment) || "..".equals(segment)) {
|
||||
throw new ServiceException("{}不能包含 . 或 .. 路径段", fieldName);
|
||||
}
|
||||
}
|
||||
if (value.chars().anyMatch(Character::isISOControl)) {
|
||||
throw new ServiceException("{}不能包含控制字符", fieldName);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验并批量删除同步连接。
|
||||
*
|
||||
* @param ids 待删除的连接主键集合
|
||||
* @param isValid 是否执行删除前校验
|
||||
* @return 是否删除成功
|
||||
*/
|
||||
@Override
|
||||
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
|
||||
synchronized (connectionMutationLock) {
|
||||
if (isValid && (ids == null || ids.isEmpty())) {
|
||||
throw new ServiceException("待删除的连接主键不能为空");
|
||||
}
|
||||
if (ids != null) {
|
||||
for (Long connectionId : ids) {
|
||||
ensureConnectionAccess(connectionId);
|
||||
if (isValid) {
|
||||
ensureNoDingTalkAuth(connectionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (isValid && planMapper.exists(com.baomidou.mybatisplus.core.toolkit.Wrappers.<SyncPlan>lambdaQuery()
|
||||
.and(wrapper -> wrapper.in(SyncPlan::getSourceConnectionId, ids)
|
||||
.or().in(SyncPlan::getTargetConnectionId, ids)))) {
|
||||
throw new ServiceException("连接已被同步计划引用,不能删除");
|
||||
}
|
||||
return connectionMapper.deleteByIds(ids) > 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验连接配置并调用对应连接器执行连通性测试。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @return 连通性测试结果
|
||||
*/
|
||||
@Override
|
||||
public ConnectorTestResult testConnection(Long connectionId) {
|
||||
SyncConnection connection = connectionMapper.selectById(connectionId);
|
||||
if (connection == null) {
|
||||
throw new ServiceException("同步连接不存在");
|
||||
}
|
||||
ensureConnectionAccess(connection);
|
||||
requireText(connection.getConnectionName(), "连接名称不能为空");
|
||||
validateRoleAndType(connection);
|
||||
validateStatus(connection.getStatus());
|
||||
validateJsonObjectIfPresent(connection.getConfigJson(), "扩展配置");
|
||||
validateJsonObjectIfPresent(connection.getSecretJson(), "敏感配置");
|
||||
validateRequiredConfig(connection, true);
|
||||
return connectorRegistry.resolve(connection).test(connection);
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动钉钉 Web 设备授权。登录完成后由回调把 DWS 返回的稳定 profile
|
||||
* 合并回连接的非敏感配置,不会覆盖其他配置项或敏感凭证。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @param ownerId 当前系统用户
|
||||
* @param expectedCorpId 可选组织 ID
|
||||
* @return 授权会话快照
|
||||
*/
|
||||
@Override
|
||||
public DwsAuthSessionVo startDingTalkAuth(Long connectionId, Long ownerId, String expectedCorpId) {
|
||||
synchronized (connectionMutationLock) {
|
||||
SyncConnection connection = requireDingTalkConnection(connectionId, ownerId);
|
||||
if (hasActiveJob(connectionId)) {
|
||||
throw new ServiceException("连接正在被同步任务使用,任务结束后才能重新登录");
|
||||
}
|
||||
validateJsonObjectIfPresent(connection.getConfigJson(), "扩展配置");
|
||||
validateJsonObjectIfPresent(connection.getSecretJson(), "敏感配置");
|
||||
// 保存时允许 profile 为空;启动 Web 登录前仍必须有应用凭证。
|
||||
validateRequiredConfig(connection, false);
|
||||
return dwsAuthSessionManager.start(connection, ownerId, expectedCorpId,
|
||||
identity -> saveDingTalkProfile(connectionId, ownerId, identity));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询钉钉 Web 设备授权状态。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @param sessionId 会话 ID
|
||||
* @param ownerId 当前系统用户
|
||||
* @return 授权会话快照
|
||||
*/
|
||||
@Override
|
||||
public DwsAuthSessionVo getDingTalkAuth(Long connectionId, String sessionId, Long ownerId) {
|
||||
requireDingTalkConnection(connectionId, ownerId);
|
||||
return dwsAuthSessionManager.get(connectionId, sessionId, ownerId, LoginHelper.isSuperAdmin(ownerId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消钉钉 Web 设备授权。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @param sessionId 会话 ID
|
||||
* @param ownerId 当前系统用户
|
||||
* @return 取消后的授权会话快照
|
||||
*/
|
||||
@Override
|
||||
public DwsAuthSessionVo cancelDingTalkAuth(Long connectionId, String sessionId, Long ownerId) {
|
||||
requireDingTalkConnection(connectionId, ownerId);
|
||||
return dwsAuthSessionManager.cancel(connectionId, sessionId, ownerId, LoginHelper.isSuperAdmin(ownerId));
|
||||
}
|
||||
|
||||
private void ensureNoDingTalkAuth(Long connectionId) {
|
||||
if (connectionId != null && dwsAuthSessionManager.hasActive(connectionId)) {
|
||||
throw new ServiceException("连接正在进行钉钉 Web 登录,登录结束后才能修改或删除");
|
||||
}
|
||||
}
|
||||
|
||||
private SyncConnection requireDingTalkConnection(Long connectionId) {
|
||||
if (connectionId == null) {
|
||||
throw new ServiceException("连接主键不能为空");
|
||||
}
|
||||
SyncConnection connection = connectionMapper.selectById(connectionId);
|
||||
if (connection == null) {
|
||||
throw new ServiceException("同步连接不存在");
|
||||
}
|
||||
if (!SyncConstants.TYPE_DINGTALK.equals(connection.getConnectionType())
|
||||
|| !SyncConstants.ROLE_SOURCE.equals(connection.getConnectionRole())) {
|
||||
throw new ServiceException("只有 SOURCE 角色的 DINGTALK 连接支持 Web 登录");
|
||||
}
|
||||
validateStatus(connection.getStatus());
|
||||
return connection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures that the current user may access a connection row. The
|
||||
* super-admin is the only account allowed to operate across owners.
|
||||
*
|
||||
* @param connectionId connection id
|
||||
*/
|
||||
private void ensureConnectionAccess(Long connectionId) {
|
||||
if (connectionId == null) {
|
||||
throw new ServiceException("连接主键不能为空");
|
||||
}
|
||||
SyncConnection connection = connectionMapper.selectById(connectionId);
|
||||
if (connection == null) {
|
||||
throw new ServiceException("同步连接不存在");
|
||||
}
|
||||
ensureConnectionAccess(connection);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures that the current user may access a loaded connection row.
|
||||
*
|
||||
* @param connection connection row
|
||||
*/
|
||||
private void ensureConnectionAccess(SyncConnection connection) {
|
||||
Long userId = requireCurrentUserId();
|
||||
if (!LoginHelper.isSuperAdmin(userId) && !Objects.equals(connection.getCreateBy(), userId)) {
|
||||
throw new ServiceException("无权访问该同步连接");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the authenticated system user id for service-layer access checks.
|
||||
*
|
||||
* @return current user id
|
||||
*/
|
||||
private Long requireCurrentUserId() {
|
||||
Long userId = LoginHelper.getUserId();
|
||||
if (userId == null) {
|
||||
throw new ServiceException("当前登录用户无效,请重新登录系统");
|
||||
}
|
||||
return userId;
|
||||
}
|
||||
|
||||
private SyncConnection requireDingTalkConnection(Long connectionId, Long ownerId) {
|
||||
SyncConnection connection = requireDingTalkConnection(connectionId);
|
||||
if (ownerId == null) {
|
||||
throw new ServiceException("当前登录用户无效,请重新登录系统");
|
||||
}
|
||||
if (!LoginHelper.isSuperAdmin(ownerId) && !Objects.equals(connection.getCreateBy(), ownerId)) {
|
||||
throw new ServiceException("只有连接创建者或超级管理员可以发起/查看钉钉 Web 登录");
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
private void saveDingTalkProfile(Long connectionId, Long ownerId, DwsAuthIdentity identity) {
|
||||
synchronized (connectionMutationLock) {
|
||||
SyncConnection current = requireDingTalkConnection(connectionId);
|
||||
if (ownerId == null || (!LoginHelper.isSuperAdmin(ownerId)
|
||||
&& !Objects.equals(current.getCreateBy(), ownerId))) {
|
||||
throw new ServiceException("连接归属已变化,不能保存钉钉登录身份");
|
||||
}
|
||||
Map<String, Object> config = parseOptionalJson(current.getConfigJson());
|
||||
if (config.isEmpty()) {
|
||||
config = new java.util.LinkedHashMap<>();
|
||||
} else {
|
||||
config = new java.util.LinkedHashMap<>(config);
|
||||
}
|
||||
config.put("profile", identity.profile());
|
||||
String configJson = JsonUtils.toJsonString(config);
|
||||
com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper<SyncConnection> wrapper =
|
||||
com.baomidou.mybatisplus.core.toolkit.Wrappers.lambdaUpdate(SyncConnection.class)
|
||||
.eq(SyncConnection::getConnectionId, connectionId)
|
||||
.eq(SyncConnection::getConnectionType, SyncConstants.TYPE_DINGTALK)
|
||||
.eq(SyncConnection::getConnectionRole, SyncConstants.ROLE_SOURCE)
|
||||
.eq(SyncConnection::getStatus, SyncConstants.STATUS_NORMAL)
|
||||
.set(SyncConnection::getConfigJson, configJson)
|
||||
.set(SyncConnection::getUpdateBy, ownerId);
|
||||
if (!LoginHelper.isSuperAdmin(ownerId)) {
|
||||
wrapper.eq(SyncConnection::getCreateBy, ownerId);
|
||||
}
|
||||
if (connectionMapper.update(null, wrapper) <= 0) {
|
||||
throw new ServiceException("保存钉钉登录身份失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The DWS profile is an authentication result, not editable connection
|
||||
* metadata. Keep it server-authoritative so a normal PUT cannot redirect
|
||||
* a connection to another user's/org's profile (or clear a valid profile
|
||||
* by submitting a form that omits the read-only field).
|
||||
*
|
||||
* @param current persisted connection
|
||||
* @param update client update
|
||||
*/
|
||||
private void preserveDingTalkProfile(SyncConnection current, SyncConnection update) {
|
||||
if (!SyncConstants.TYPE_DINGTALK.equals(current.getConnectionType())
|
||||
|| !SyncConstants.ROLE_SOURCE.equals(current.getConnectionRole())
|
||||
|| !SyncConstants.TYPE_DINGTALK.equals(update.getConnectionType())
|
||||
|| !SyncConstants.ROLE_SOURCE.equals(update.getConnectionRole())) {
|
||||
return;
|
||||
}
|
||||
Map<String, Object> currentConfig = parseOptionalJson(current.getConfigJson());
|
||||
String currentProfile = stringValue(currentConfig, "profile");
|
||||
Map<String, Object> requestedConfig = StringUtils.isBlank(update.getConfigJson())
|
||||
? new java.util.LinkedHashMap<>(currentConfig)
|
||||
: new java.util.LinkedHashMap<>(parseOptionalJson(update.getConfigJson()));
|
||||
String requestedProfile = stringValue(requestedConfig, "profile");
|
||||
if (StringUtils.isNotBlank(requestedProfile)
|
||||
&& !Objects.equals(currentProfile, requestedProfile)) {
|
||||
throw new ServiceException("钉钉 profile 只能通过 Web 登录授权变更");
|
||||
}
|
||||
if (StringUtils.isNotBlank(currentProfile)) {
|
||||
requestedConfig.put("profile", currentProfile);
|
||||
} else {
|
||||
requestedConfig.remove("profile");
|
||||
}
|
||||
update.setConfigJson(requestedConfig.isEmpty() ? null : JsonUtils.toJsonString(requestedConfig));
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析敏感配置 JSON。
|
||||
*
|
||||
* @param secretJson 敏感配置 JSON
|
||||
* @return 敏感配置键值
|
||||
*/
|
||||
private Map<String, Object> parseSecretJson(String secretJson) {
|
||||
if (StringUtils.isBlank(secretJson)) {
|
||||
throw new ServiceException("敏感配置不能为空");
|
||||
}
|
||||
return JsonUtils.parseMap(secretJson);
|
||||
}
|
||||
|
||||
private boolean isReferenced(Long connectionId) {
|
||||
return planMapper.exists(com.baomidou.mybatisplus.core.toolkit.Wrappers.<SyncPlan>lambdaQuery()
|
||||
.and(wrapper -> wrapper.eq(SyncPlan::getSourceConnectionId, connectionId)
|
||||
.or().eq(SyncPlan::getTargetConnectionId, connectionId)));
|
||||
}
|
||||
|
||||
private boolean hasActiveJob(Long connectionId) {
|
||||
List<Long> planIds = planMapper.selectList(com.baomidou.mybatisplus.core.toolkit.Wrappers.<SyncPlan>lambdaQuery()
|
||||
.select(SyncPlan::getPlanId)
|
||||
.and(wrapper -> wrapper.eq(SyncPlan::getSourceConnectionId, connectionId)
|
||||
.or().eq(SyncPlan::getTargetConnectionId, connectionId)))
|
||||
.stream().map(SyncPlan::getPlanId).toList();
|
||||
return !planIds.isEmpty() && jobMapper.exists(
|
||||
com.baomidou.mybatisplus.core.toolkit.Wrappers.<SyncJob>lambdaQuery()
|
||||
.in(SyncJob::getPlanId, planIds)
|
||||
.in(SyncJob::getStatus, SyncConstants.JOB_PENDING,
|
||||
SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING));
|
||||
}
|
||||
|
||||
private void validateDownloadOptions(Map<String, Object> config) {
|
||||
String parallel = stringValue(config, "downloadParallel");
|
||||
if (StringUtils.isNotBlank(parallel)) {
|
||||
try {
|
||||
int value = Integer.parseInt(parallel);
|
||||
if (value < 1 || value > 8) {
|
||||
throw new ServiceException("钉钉 downloadParallel 必须在1到8之间");
|
||||
}
|
||||
} catch (NumberFormatException e) {
|
||||
throw new ServiceException("钉钉 downloadParallel 必须是整数");
|
||||
}
|
||||
}
|
||||
String partSize = stringValue(config, "downloadPartSize");
|
||||
if (StringUtils.isNotBlank(partSize)) {
|
||||
Matcher matcher = DOWNLOAD_PART_SIZE_PATTERN.matcher(partSize);
|
||||
if (!matcher.matches()) {
|
||||
throw new ServiceException("钉钉 downloadPartSize 格式不正确,例如 32MB");
|
||||
}
|
||||
try {
|
||||
long multiplier = switch (matcher.group(2).toUpperCase(Locale.ROOT)) {
|
||||
case "KB" -> 1024L;
|
||||
case "MB" -> ONE_MIB;
|
||||
case "GB" -> ONE_GIB;
|
||||
default -> throw new IllegalStateException("无法识别容量单位");
|
||||
};
|
||||
long bytes = Math.multiplyExact(Long.parseLong(matcher.group(1)), multiplier);
|
||||
if (bytes < ONE_MIB || bytes > ONE_GIB) {
|
||||
throw new ServiceException("钉钉 downloadPartSize 必须在1MB到1GB之间");
|
||||
}
|
||||
} catch (NumberFormatException | ArithmeticException e) {
|
||||
throw new ServiceException("钉钉 downloadPartSize 超出有效范围");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String stringValue(Map<String, Object> values, String key) {
|
||||
Object value = values.get(key);
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验敏感配置中的必填项。
|
||||
*
|
||||
* @param secrets 敏感配置
|
||||
* @param key 配置键
|
||||
* @param message 校验失败提示
|
||||
*/
|
||||
private void requireSecret(Map<String, Object> secrets, String key, String message) {
|
||||
Object value = secrets.get(key);
|
||||
if (value == null || StringUtils.isBlank(String.valueOf(value))) {
|
||||
throw new ServiceException(message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验必填文本。
|
||||
*
|
||||
* @param value 文本值
|
||||
* @param message 校验失败提示
|
||||
*/
|
||||
private void requireText(String value, String message) {
|
||||
if (StringUtils.isBlank(value)) {
|
||||
throw new ServiceException(message);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,199 @@
|
||||
package org.dromara.sync.service.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.core.utils.StringUtils;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.sync.constant.SyncConstants;
|
||||
import org.dromara.sync.domain.SyncJob;
|
||||
import org.dromara.sync.domain.SyncJobItem;
|
||||
import org.dromara.sync.domain.SyncPlan;
|
||||
import org.dromara.sync.domain.SyncTransferPart;
|
||||
import org.dromara.sync.domain.bo.SyncJobBo;
|
||||
import org.dromara.sync.domain.bo.SyncJobItemBo;
|
||||
import org.dromara.sync.domain.vo.SyncJobItemVo;
|
||||
import org.dromara.sync.domain.vo.SyncJobVo;
|
||||
import org.dromara.sync.mapper.SyncJobItemMapper;
|
||||
import org.dromara.sync.mapper.SyncJobMapper;
|
||||
import org.dromara.sync.mapper.SyncPlanMapper;
|
||||
import org.dromara.sync.mapper.SyncTransferPartMapper;
|
||||
import org.dromara.sync.service.ISyncJobService;
|
||||
import org.dromara.sync.worker.SyncExecutionWorker;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 同步任务服务实现。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class SyncJobServiceImpl implements ISyncJobService {
|
||||
|
||||
private final SyncJobMapper jobMapper;
|
||||
private final SyncJobItemMapper itemMapper;
|
||||
private final SyncTransferPartMapper transferPartMapper;
|
||||
private final SyncPlanMapper planMapper;
|
||||
private final SyncExecutionWorker executionWorker;
|
||||
|
||||
@Override
|
||||
public SyncJobVo queryById(Long jobId) {
|
||||
return jobMapper.selectVoById(jobId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageResult<SyncJobVo> queryPageList(SyncJobBo bo, PageQuery pageQuery) {
|
||||
LambdaQueryWrapper<SyncJob> lqw = Wrappers.lambdaQuery();
|
||||
lqw.eq(bo.getPlanId() != null, SyncJob::getPlanId, bo.getPlanId());
|
||||
lqw.eq(StringUtils.isNotBlank(bo.getTriggerType()), SyncJob::getTriggerType, bo.getTriggerType());
|
||||
lqw.eq(StringUtils.isNotBlank(bo.getRunType()), SyncJob::getRunType, bo.getRunType());
|
||||
lqw.eq(StringUtils.isNotBlank(bo.getStatus()), SyncJob::getStatus, bo.getStatus());
|
||||
Object beginTime = bo.getParams().get("beginTime");
|
||||
Object endTime = bo.getParams().get("endTime");
|
||||
lqw.ge(beginTime != null, SyncJob::getCreateTime, beginTime);
|
||||
lqw.le(endTime != null, SyncJob::getCreateTime, endTime);
|
||||
lqw.orderByDesc(SyncJob::getJobId);
|
||||
Page<SyncJobVo> page = jobMapper.selectVoPage(pageQuery.build(), lqw);
|
||||
return PageResult.build(page.getRecords(), page.getTotal());
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageResult<SyncJobItemVo> queryItemPageList(SyncJobItemBo bo, PageQuery pageQuery) {
|
||||
LambdaQueryWrapper<SyncJobItem> lqw = Wrappers.lambdaQuery();
|
||||
lqw.eq(bo.getJobId() != null, SyncJobItem::getJobId, bo.getJobId());
|
||||
lqw.like(StringUtils.isNotBlank(bo.getSourcePath()), SyncJobItem::getSourcePath, bo.getSourcePath());
|
||||
lqw.eq(StringUtils.isNotBlank(bo.getObjectType()), SyncJobItem::getObjectType, bo.getObjectType());
|
||||
lqw.eq(StringUtils.isNotBlank(bo.getActionType()), SyncJobItem::getActionType, bo.getActionType());
|
||||
lqw.eq(StringUtils.isNotBlank(bo.getStatus()), SyncJobItem::getStatus, bo.getStatus());
|
||||
lqw.orderByAsc(SyncJobItem::getItemId);
|
||||
Page<SyncJobItemVo> page = itemMapper.selectVoPage(pageQuery.build(), lqw);
|
||||
return PageResult.build(page.getRecords(), page.getTotal());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long startPlan(Long planId, String triggerType, String runType) {
|
||||
SyncPlan plan = planMapper.selectById(planId);
|
||||
if (plan == null) {
|
||||
throw new ServiceException("同步计划不存在");
|
||||
}
|
||||
if (!SyncConstants.STATUS_NORMAL.equals(plan.getStatus())) {
|
||||
throw new ServiceException("同步计划已停用");
|
||||
}
|
||||
boolean running = jobMapper.exists(Wrappers.<SyncJob>lambdaQuery()
|
||||
.eq(SyncJob::getPlanId, planId)
|
||||
.in(SyncJob::getStatus, SyncConstants.JOB_PENDING,
|
||||
SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING));
|
||||
if (running) {
|
||||
throw new ServiceException("该同步计划已有运行中任务");
|
||||
}
|
||||
String actualTriggerType = StringUtils.isBlank(triggerType) ? "MANUAL" : triggerType;
|
||||
String actualRunType = StringUtils.isBlank(runType) ? plan.getSyncMode() : runType;
|
||||
if (!List.of("MANUAL", "SCHEDULE", "RETRY").contains(actualTriggerType)) {
|
||||
throw new ServiceException("不支持的任务触发类型:{}", actualTriggerType);
|
||||
}
|
||||
if (!List.of("FULL", "INCREMENTAL").contains(actualRunType)) {
|
||||
throw new ServiceException("不支持的任务运行类型:{}", actualRunType);
|
||||
}
|
||||
SyncJob job = new SyncJob();
|
||||
job.setPlanId(planId);
|
||||
job.setTriggerType(actualTriggerType);
|
||||
job.setRunType(actualRunType);
|
||||
job.setStatus(SyncConstants.JOB_PENDING);
|
||||
job.setTotalCount(0L);
|
||||
job.setProcessedCount(0L);
|
||||
job.setSuccessCount(0L);
|
||||
job.setFailedCount(0L);
|
||||
job.setSkippedCount(0L);
|
||||
job.setDeletedCount(0L);
|
||||
job.setTotalBytes(0L);
|
||||
job.setTransferredBytes(0L);
|
||||
try {
|
||||
if (jobMapper.insert(job) <= 0) {
|
||||
throw new ServiceException("创建同步任务失败");
|
||||
}
|
||||
} catch (DuplicateKeyException e) {
|
||||
throw new ServiceException("该同步计划已有等待或运行中的任务");
|
||||
}
|
||||
try {
|
||||
executionWorker.executeAsync(job.getJobId());
|
||||
} catch (RuntimeException e) {
|
||||
jobMapper.lambda()
|
||||
.set(SyncJob::getStatus, SyncConstants.JOB_FAILED)
|
||||
.set(SyncJob::getFinishTime, LocalDateTime.now())
|
||||
.set(SyncJob::getErrorMessage, "同步任务未能提交到后台执行器")
|
||||
.eq(SyncJob::getJobId, job.getJobId())
|
||||
.eq(SyncJob::getStatus, SyncConstants.JOB_PENDING)
|
||||
.update();
|
||||
throw new ServiceException("同步任务未能提交到后台执行器", e);
|
||||
}
|
||||
return job.getJobId();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long retry(Long jobId) {
|
||||
SyncJob oldJob = jobMapper.selectById(jobId);
|
||||
if (oldJob == null) {
|
||||
throw new ServiceException("原同步任务不存在");
|
||||
}
|
||||
if (List.of(SyncConstants.JOB_PENDING, SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING)
|
||||
.contains(oldJob.getStatus())) {
|
||||
throw new ServiceException("运行中的任务不能重试");
|
||||
}
|
||||
return startPlan(oldJob.getPlanId(), "RETRY", "INCREMENTAL");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean cancel(Long jobId) {
|
||||
SyncJob job = jobMapper.selectById(jobId);
|
||||
if (job == null) {
|
||||
throw new ServiceException("同步任务不存在");
|
||||
}
|
||||
if (!List.of(SyncConstants.JOB_PENDING, SyncConstants.JOB_RUNNING).contains(job.getStatus())) {
|
||||
throw new ServiceException("只有等待或运行中的任务可以取消");
|
||||
}
|
||||
String targetStatus = SyncConstants.JOB_PENDING.equals(job.getStatus())
|
||||
? SyncConstants.JOB_CANCELED : SyncConstants.JOB_CANCELING;
|
||||
boolean canceled = jobMapper.lambda()
|
||||
.set(SyncJob::getStatus, targetStatus)
|
||||
.set(SyncConstants.JOB_CANCELED.equals(targetStatus), SyncJob::getFinishTime, LocalDateTime.now())
|
||||
.eq(SyncJob::getJobId, jobId)
|
||||
.eq(SyncJob::getStatus, job.getStatus())
|
||||
.update();
|
||||
if (canceled && SyncConstants.JOB_CANCELING.equals(targetStatus)) {
|
||||
executionWorker.cancel(jobId);
|
||||
}
|
||||
return canceled;
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
|
||||
if (isValid) {
|
||||
boolean active = jobMapper.exists(Wrappers.<SyncJob>lambdaQuery()
|
||||
.in(SyncJob::getJobId, ids)
|
||||
.in(SyncJob::getStatus, SyncConstants.JOB_PENDING,
|
||||
SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING));
|
||||
if (active) {
|
||||
throw new ServiceException("等待或运行中的同步任务不能删除");
|
||||
}
|
||||
}
|
||||
List<Long> itemIds = itemMapper.selectList(Wrappers.<SyncJobItem>lambdaQuery()
|
||||
.select(SyncJobItem::getItemId)
|
||||
.in(SyncJobItem::getJobId, ids))
|
||||
.stream().map(SyncJobItem::getItemId).toList();
|
||||
if (!itemIds.isEmpty()) {
|
||||
transferPartMapper.delete(Wrappers.<SyncTransferPart>lambdaQuery()
|
||||
.in(SyncTransferPart::getJobItemId, itemIds));
|
||||
}
|
||||
itemMapper.delete(Wrappers.<SyncJobItem>lambdaQuery().in(SyncJobItem::getJobId, ids));
|
||||
return jobMapper.deleteByIds(ids) > 0;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
package org.dromara.sync.service.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.core.utils.StringUtils;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.sync.domain.SyncObject;
|
||||
import org.dromara.sync.domain.bo.SyncObjectBo;
|
||||
import org.dromara.sync.domain.vo.SyncObjectVo;
|
||||
import org.dromara.sync.mapper.SyncObjectMapper;
|
||||
import org.dromara.sync.service.ISyncObjectService;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 同步对象清单服务实现。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class SyncObjectServiceImpl implements ISyncObjectService {
|
||||
|
||||
private final SyncObjectMapper objectMapper;
|
||||
|
||||
@Override
|
||||
public SyncObjectVo queryById(Long objectId) {
|
||||
return objectMapper.selectVoById(objectId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageResult<SyncObjectVo> queryPageList(SyncObjectBo bo, PageQuery pageQuery) {
|
||||
LambdaQueryWrapper<SyncObject> lqw = Wrappers.lambdaQuery();
|
||||
lqw.eq(bo.getPlanId() != null, SyncObject::getPlanId, bo.getPlanId());
|
||||
lqw.eq(StringUtils.isNotBlank(bo.getSourceObjectId()), SyncObject::getSourceObjectId, bo.getSourceObjectId());
|
||||
lqw.like(StringUtils.isNotBlank(bo.getSourcePath()), SyncObject::getSourcePath, bo.getSourcePath());
|
||||
lqw.eq(StringUtils.isNotBlank(bo.getObjectType()), SyncObject::getObjectType, bo.getObjectType());
|
||||
lqw.eq(StringUtils.isNotBlank(bo.getSyncStatus()), SyncObject::getSyncStatus, bo.getSyncStatus());
|
||||
lqw.eq(StringUtils.isNotBlank(bo.getSourceDeleted()), SyncObject::getSourceDeleted, bo.getSourceDeleted());
|
||||
lqw.orderByAsc(SyncObject::getSourcePath);
|
||||
Page<SyncObjectVo> page = objectMapper.selectVoPage(pageQuery.build(), lqw);
|
||||
return PageResult.build(page.getRecords(), page.getTotal());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,380 @@
|
||||
package org.dromara.sync.service.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.dromara.common.core.domain.PageResult;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.core.utils.MapstructUtils;
|
||||
import org.dromara.common.core.utils.StringUtils;
|
||||
import org.dromara.common.mybatis.core.page.PageQuery;
|
||||
import org.dromara.common.mybatis.core.query.QueryBuilder;
|
||||
import org.dromara.sync.constant.SyncConstants;
|
||||
import org.dromara.sync.domain.SyncConnection;
|
||||
import org.dromara.sync.domain.SyncJob;
|
||||
import org.dromara.sync.domain.SyncPlan;
|
||||
import org.dromara.sync.domain.bo.SyncPlanBo;
|
||||
import org.dromara.sync.domain.vo.SyncPlanVo;
|
||||
import org.dromara.sync.mapper.SyncConnectionMapper;
|
||||
import org.dromara.sync.mapper.SyncJobMapper;
|
||||
import org.dromara.sync.mapper.SyncPlanMapper;
|
||||
import org.dromara.sync.service.ISyncPlanService;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.scheduling.support.CronExpression;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 同步计划 Service 业务层处理。
|
||||
*
|
||||
* @author Codex
|
||||
* @date 2026-09-02
|
||||
*/
|
||||
@RequiredArgsConstructor
|
||||
@Service
|
||||
public class SyncPlanServiceImpl implements ISyncPlanService {
|
||||
|
||||
private static final String SCHEDULE_MANUAL = "MANUAL";
|
||||
private static final String SCHEDULE_CRON = "CRON";
|
||||
|
||||
private static final Set<String> SYNC_MODES = Set.of("FULL", "INCREMENTAL");
|
||||
private static final Set<String> SCHEDULE_TYPES = Set.of(SCHEDULE_MANUAL, SCHEDULE_CRON);
|
||||
private static final Set<String> CONFLICT_STRATEGIES = Set.of("OVERWRITE", "SKIP", "KEEP_BOTH");
|
||||
private static final Set<String> DELETE_STRATEGIES = Set.of("KEEP", "MARK", "DELETE");
|
||||
private static final Set<String> VERIFY_MODES = Set.of("SIZE", "ETAG", "SHA256");
|
||||
private static final Set<String> PLAN_STATUSES = Set.of(SyncConstants.STATUS_NORMAL, SyncConstants.STATUS_DISABLED);
|
||||
|
||||
private final SyncPlanMapper planMapper;
|
||||
private final SyncConnectionMapper connectionMapper;
|
||||
private final SyncJobMapper jobMapper;
|
||||
|
||||
/**
|
||||
* 根据主键查询同步计划。
|
||||
*
|
||||
* @param planId 计划主键
|
||||
* @return 同步计划详情
|
||||
*/
|
||||
@Override
|
||||
public SyncPlanVo queryById(Long planId) {
|
||||
return planMapper.selectVoById(planId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页查询同步计划列表。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @param pageQuery 分页参数
|
||||
* @return 同步计划分页列表
|
||||
*/
|
||||
@Override
|
||||
public PageResult<SyncPlanVo> queryPageList(SyncPlanBo bo, PageQuery pageQuery) {
|
||||
LambdaQueryWrapper<SyncPlan> lqw = buildQueryWrapper(bo);
|
||||
Page<SyncPlanVo> result = planMapper.selectVoPage(pageQuery.build(), lqw);
|
||||
return PageResult.build(result.getRecords(), result.getTotal());
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询符合条件的同步计划列表。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @return 同步计划列表
|
||||
*/
|
||||
@Override
|
||||
public List<SyncPlanVo> queryList(SyncPlanBo bo) {
|
||||
return planMapper.selectVoList(buildQueryWrapper(bo));
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建同步计划动态查询条件。
|
||||
*
|
||||
* @param bo 查询条件
|
||||
* @return 查询条件包装器
|
||||
*/
|
||||
private LambdaQueryWrapper<SyncPlan> buildQueryWrapper(SyncPlanBo bo) {
|
||||
Map<String, Object> params = bo.getParams();
|
||||
return QueryBuilder.lambda(SyncPlan.class)
|
||||
.eqIfPresent(SyncPlan::getPlanId, bo.getPlanId())
|
||||
.likeIfText(SyncPlan::getPlanName, bo.getPlanName())
|
||||
.eqIfPresent(SyncPlan::getSourceConnectionId, bo.getSourceConnectionId())
|
||||
.eqIfPresent(SyncPlan::getTargetConnectionId, bo.getTargetConnectionId())
|
||||
.likeIfText(SyncPlan::getSourceRoot, bo.getSourceRoot())
|
||||
.likeIfText(SyncPlan::getTargetPrefix, bo.getTargetPrefix())
|
||||
.eqIfText(SyncPlan::getSyncMode, bo.getSyncMode())
|
||||
.eqIfText(SyncPlan::getScheduleType, bo.getScheduleType())
|
||||
.eqIfText(SyncPlan::getConflictStrategy, bo.getConflictStrategy())
|
||||
.eqIfText(SyncPlan::getDeleteStrategy, bo.getDeleteStrategy())
|
||||
.eqIfPresent(SyncPlan::getDeleteGuardPercent, bo.getDeleteGuardPercent())
|
||||
.eqIfText(SyncPlan::getVerifyMode, bo.getVerifyMode())
|
||||
.eqIfPresent(SyncPlan::getMaxConcurrency, bo.getMaxConcurrency())
|
||||
.eqIfPresent(SyncPlan::getBandwidthLimitKbps, bo.getBandwidthLimitKbps())
|
||||
.eqIfText(SyncPlan::getStatus, bo.getStatus())
|
||||
.betweenParams(SyncPlan::getLastRunTime, params, "beginLastRunTime", "endLastRunTime")
|
||||
.betweenParams(SyncPlan::getNextRunTime, params, "beginNextRunTime", "endNextRunTime")
|
||||
.orderByAsc(SyncPlan::getPlanId)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验计划名称是否唯一。
|
||||
*
|
||||
* @param bo 同步计划
|
||||
* @return 名称未被占用返回 {@code true}
|
||||
*/
|
||||
@Override
|
||||
public boolean checkPlanNameUnique(SyncPlanBo bo) {
|
||||
String planName = StringUtils.trim(bo.getPlanName());
|
||||
if (StringUtils.isBlank(planName)) {
|
||||
return true;
|
||||
}
|
||||
return !planMapper.lambda()
|
||||
.eq(SyncPlan::getPlanName, planName)
|
||||
.neIfPresent(SyncPlan::getPlanId, bo.getPlanId())
|
||||
.exists();
|
||||
}
|
||||
|
||||
/**
|
||||
* 新增同步计划。
|
||||
*
|
||||
* @param bo 同步计划
|
||||
* @return 是否新增成功
|
||||
*/
|
||||
@Override
|
||||
public Boolean insertByBo(SyncPlanBo bo) {
|
||||
SyncPlan add = MapstructUtils.convert(bo, SyncPlan.class);
|
||||
validEntityBeforeSave(add);
|
||||
boolean flag;
|
||||
try {
|
||||
flag = planMapper.insert(add) > 0;
|
||||
} catch (DuplicateKeyException e) {
|
||||
throw new ServiceException("计划名称'{}'已存在", add.getPlanName());
|
||||
}
|
||||
if (flag) {
|
||||
bo.setPlanId(add.getPlanId());
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
/**
|
||||
* 修改同步计划。
|
||||
*
|
||||
* @param bo 同步计划
|
||||
* @return 是否修改成功
|
||||
*/
|
||||
@Override
|
||||
public Boolean updateByBo(SyncPlanBo bo) {
|
||||
SyncPlan current = planMapper.selectById(bo.getPlanId());
|
||||
if (current == null) {
|
||||
throw new ServiceException("同步计划不存在");
|
||||
}
|
||||
boolean hasActiveJob = jobMapper.lambda()
|
||||
.eq(SyncJob::getPlanId, bo.getPlanId())
|
||||
.in(SyncJob::getStatus, SyncConstants.JOB_PENDING,
|
||||
SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING)
|
||||
.exists();
|
||||
if (hasActiveJob) {
|
||||
throw new ServiceException("计划存在等待或运行中的任务,任务结束后才能修改");
|
||||
}
|
||||
SyncPlan update = MapstructUtils.convert(bo, SyncPlan.class);
|
||||
validEntityBeforeSave(update);
|
||||
try {
|
||||
return planMapper.updateById(update) > 0;
|
||||
} catch (DuplicateKeyException e) {
|
||||
throw new ServiceException("计划名称'{}'已存在", update.getPlanName());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行同步计划保存前的业务校验和数据归一化。
|
||||
*
|
||||
* @param entity 同步计划实体
|
||||
*/
|
||||
private void validEntityBeforeSave(SyncPlan entity) {
|
||||
entity.setPlanName(StringUtils.trim(entity.getPlanName()));
|
||||
entity.setSourceRoot(StringUtils.trim(entity.getSourceRoot()));
|
||||
entity.setTargetPrefix(StringUtils.trim(entity.getTargetPrefix()));
|
||||
if (StringUtils.isBlank(entity.getPlanName())) {
|
||||
throw new ServiceException("计划名称不能为空");
|
||||
}
|
||||
validateLength("计划名称", entity.getPlanName(), 100);
|
||||
validateLength("源端同步根路径", entity.getSourceRoot(), 1024);
|
||||
validateLength("目标端对象键前缀", entity.getTargetPrefix(), 1024);
|
||||
validateLength("备注", entity.getRemark(), 500);
|
||||
validateObjectKeyPrefix(entity.getTargetPrefix(), "目标端对象键前缀");
|
||||
|
||||
boolean nameExists = planMapper.lambda()
|
||||
.eq(SyncPlan::getPlanName, entity.getPlanName())
|
||||
.neIfPresent(SyncPlan::getPlanId, entity.getPlanId())
|
||||
.exists();
|
||||
if (nameExists) {
|
||||
throw new ServiceException("计划名称'{}'已存在", entity.getPlanName());
|
||||
}
|
||||
|
||||
validateConnection(entity.getSourceConnectionId(), SyncConstants.ROLE_SOURCE, "源端");
|
||||
validateConnection(entity.getTargetConnectionId(), SyncConstants.ROLE_TARGET, "目标端");
|
||||
|
||||
entity.setSyncMode(normalizeOption(entity.getSyncMode()));
|
||||
entity.setScheduleType(normalizeOption(entity.getScheduleType()));
|
||||
entity.setConflictStrategy(normalizeOption(entity.getConflictStrategy()));
|
||||
entity.setDeleteStrategy(normalizeOption(entity.getDeleteStrategy()));
|
||||
entity.setVerifyMode(normalizeOption(entity.getVerifyMode()));
|
||||
entity.setStatus(normalizeOption(entity.getStatus()));
|
||||
|
||||
validateOption("同步模式", entity.getSyncMode(), SYNC_MODES);
|
||||
validateOption("调度类型", entity.getScheduleType(), SCHEDULE_TYPES);
|
||||
validateOption("冲突处理策略", entity.getConflictStrategy(), CONFLICT_STRATEGIES);
|
||||
validateOption("删除处理策略", entity.getDeleteStrategy(), DELETE_STRATEGIES);
|
||||
validateOption("校验方式", entity.getVerifyMode(), VERIFY_MODES);
|
||||
validateOption("状态", entity.getStatus(), PLAN_STATUSES);
|
||||
|
||||
if (entity.getDeleteGuardPercent() == null
|
||||
|| entity.getDeleteGuardPercent() < 0
|
||||
|| entity.getDeleteGuardPercent() > 100) {
|
||||
throw new ServiceException("删除保护阈值必须在0到100之间");
|
||||
}
|
||||
|
||||
if (entity.getMaxConcurrency() == null
|
||||
|| entity.getMaxConcurrency() < 1
|
||||
|| entity.getMaxConcurrency() > 100) {
|
||||
throw new ServiceException("最大并发数必须在1到100之间");
|
||||
}
|
||||
if (entity.getBandwidthLimitKbps() == null || entity.getBandwidthLimitKbps() < 0) {
|
||||
throw new ServiceException("带宽上限不能小于0");
|
||||
}
|
||||
if (entity.getBandwidthLimitKbps() > 0) {
|
||||
throw new ServiceException("当前版本尚未启用带宽限速,请将带宽上限设置为0");
|
||||
}
|
||||
|
||||
if (SCHEDULE_CRON.equals(entity.getScheduleType())) {
|
||||
String expression = StringUtils.trim(entity.getCronExpression());
|
||||
if (StringUtils.isBlank(expression)) {
|
||||
throw new ServiceException("CRON调度的表达式不能为空");
|
||||
}
|
||||
validateLength("CRON表达式", expression, 128);
|
||||
try {
|
||||
CronExpression.parse(expression);
|
||||
} catch (IllegalArgumentException ex) {
|
||||
throw new ServiceException("CRON表达式格式不正确");
|
||||
}
|
||||
entity.setCronExpression(expression);
|
||||
ZonedDateTime next = CronExpression.parse(expression).next(ZonedDateTime.now());
|
||||
entity.setNextRunTime(next == null ? null : next.toLocalDateTime());
|
||||
} else {
|
||||
entity.setCronExpression(null);
|
||||
entity.setNextRunTime(null);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验连接存在、启用且角色与计划端点一致。
|
||||
*
|
||||
* @param connectionId 连接主键
|
||||
* @param expectedRole 期望角色
|
||||
* @param endpointName 端点名称
|
||||
*/
|
||||
private void validateConnection(Long connectionId, String expectedRole, String endpointName) {
|
||||
if (connectionId == null) {
|
||||
throw new ServiceException("{}连接不能为空", endpointName);
|
||||
}
|
||||
SyncConnection connection = connectionMapper.selectById(connectionId);
|
||||
if (connection == null) {
|
||||
throw new ServiceException("{}连接不存在或已删除", endpointName);
|
||||
}
|
||||
if (!SyncConstants.STATUS_NORMAL.equals(connection.getStatus())) {
|
||||
throw new ServiceException("{}连接'{}'已停用", endpointName, connection.getConnectionName());
|
||||
}
|
||||
if (!expectedRole.equals(connection.getConnectionRole())) {
|
||||
throw new ServiceException("{}连接'{}'的角色必须为{}", endpointName,
|
||||
connection.getConnectionName(), expectedRole);
|
||||
}
|
||||
if (SyncConstants.ROLE_SOURCE.equals(expectedRole)
|
||||
&& !SyncConstants.TYPE_DINGTALK.equals(connection.getConnectionType())) {
|
||||
throw new ServiceException("首版源端连接只支持 DINGTALK");
|
||||
}
|
||||
if (SyncConstants.ROLE_TARGET.equals(expectedRole)
|
||||
&& !Set.of(SyncConstants.TYPE_S3, SyncConstants.TYPE_ALIYUN_OSS)
|
||||
.contains(connection.getConnectionType())) {
|
||||
throw new ServiceException("首版目标端连接只支持 S3 或 ALIYUN_OSS");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 去除枚举型文本首尾空白。
|
||||
*
|
||||
* @param value 原始值
|
||||
* @return 归一化后的值
|
||||
*/
|
||||
private String normalizeOption(String value) {
|
||||
return StringUtils.trim(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验枚举型配置值。
|
||||
*
|
||||
* @param fieldName 字段名称
|
||||
* @param value 字段值
|
||||
* @param options 允许值
|
||||
*/
|
||||
private void validateOption(String fieldName, String value, Set<String> options) {
|
||||
if (!options.contains(value)) {
|
||||
throw new ServiceException("{}不正确,可选值为{}", fieldName, String.join(",", options));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验文本长度不超过数据库字段上限。
|
||||
*
|
||||
* @param fieldName 字段名称
|
||||
* @param value 字段值
|
||||
* @param maxLength 最大长度
|
||||
*/
|
||||
private void validateLength(String fieldName, String value, int maxLength) {
|
||||
if (value != null && value.length() > maxLength) {
|
||||
throw new ServiceException("{}不能超过{}个字符", fieldName, maxLength);
|
||||
}
|
||||
}
|
||||
|
||||
private void validateObjectKeyPrefix(String value, String fieldName) {
|
||||
if (StringUtils.isBlank(value)) {
|
||||
return;
|
||||
}
|
||||
for (String segment : value.replace('\\', '/').split("/")) {
|
||||
if (".".equals(segment) || "..".equals(segment)) {
|
||||
throw new ServiceException("{}不能包含 . 或 .. 路径段", fieldName);
|
||||
}
|
||||
}
|
||||
if (value.chars().anyMatch(Character::isISOControl)) {
|
||||
throw new ServiceException("{}不能包含控制字符", fieldName);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验并批量删除同步计划。
|
||||
*
|
||||
* @param ids 计划主键集合
|
||||
* @param isValid 是否执行删除前业务校验
|
||||
* @return 是否删除成功
|
||||
*/
|
||||
@Override
|
||||
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
|
||||
if (isValid && (ids == null || ids.isEmpty())) {
|
||||
throw new ServiceException("待删除的计划主键不能为空");
|
||||
}
|
||||
if (isValid) {
|
||||
boolean hasRunningJob = jobMapper.lambda()
|
||||
.in(SyncJob::getPlanId, ids)
|
||||
.in(SyncJob::getStatus, List.of(SyncConstants.JOB_PENDING,
|
||||
SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING))
|
||||
.exists();
|
||||
if (hasRunningJob) {
|
||||
throw new ServiceException("存在等待中或运行中的同步任务,不能删除计划");
|
||||
}
|
||||
}
|
||||
return planMapper.deleteByIds(ids) > 0;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,966 @@
|
||||
package org.dromara.sync.worker;
|
||||
|
||||
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.dromara.common.core.utils.StringUtils;
|
||||
import org.dromara.common.json.utils.JsonUtils;
|
||||
import org.dromara.sync.connector.ConnectorRegistry;
|
||||
import org.dromara.sync.connector.SourceConnector;
|
||||
import org.dromara.sync.connector.TargetConnector;
|
||||
import org.dromara.sync.connector.model.*;
|
||||
import org.dromara.sync.constant.SyncConstants;
|
||||
import org.dromara.sync.domain.*;
|
||||
import org.dromara.sync.mapper.*;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.scheduling.support.CronExpression;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CompletionService;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorCompletionService;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 同步任务后台执行器。
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class SyncExecutionWorker {
|
||||
|
||||
private static final String CHECKPOINT_TYPE = "RUN_WATERMARK";
|
||||
private static final String CHECKPOINT_KEY = "default";
|
||||
private static final int MAX_PAGES_PER_FOLDER = 100_000;
|
||||
private static final long MAX_OBJECTS_PER_RUN = 10_000_000L;
|
||||
|
||||
private final SyncJobMapper jobMapper;
|
||||
private final SyncJobItemMapper itemMapper;
|
||||
private final SyncPlanMapper planMapper;
|
||||
private final SyncConnectionMapper connectionMapper;
|
||||
private final SyncObjectMapper objectMapper;
|
||||
private final SyncCheckpointMapper checkpointMapper;
|
||||
private final ConnectorRegistry connectorRegistry;
|
||||
private final Map<Long, Thread> activeExecutions = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 异步执行已创建的同步任务。
|
||||
*
|
||||
* @param jobId 任务ID
|
||||
*/
|
||||
@Async
|
||||
public void executeAsync(Long jobId) {
|
||||
execute(jobId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 同步执行任务,供测试和调度适配器复用。
|
||||
*
|
||||
* @param jobId 任务ID
|
||||
*/
|
||||
public void execute(Long jobId) {
|
||||
if (!markRunning(jobId)) {
|
||||
return;
|
||||
}
|
||||
Thread executionThread = Thread.currentThread();
|
||||
activeExecutions.put(jobId, executionThread);
|
||||
Stats stats = new Stats();
|
||||
try {
|
||||
SyncJob job = requireJob(jobId);
|
||||
SyncPlan plan = requirePlan(job.getPlanId());
|
||||
SyncConnection sourceConnection = requireConnection(plan.getSourceConnectionId(), SyncConstants.ROLE_SOURCE);
|
||||
SyncConnection targetConnection = requireConnection(plan.getTargetConnectionId(), SyncConstants.ROLE_TARGET);
|
||||
SourceConnector sourceConnector = connectorRegistry.source(sourceConnection.getConnectionType());
|
||||
TargetConnector targetConnector = connectorRegistry.target(targetConnection.getConnectionType());
|
||||
String checkpointBefore = loadCheckpoint(plan.getPlanId());
|
||||
jobMapper.lambda().set(SyncJob::getCheckpointBefore, checkpointBefore)
|
||||
.eq(SyncJob::getJobId, jobId).update();
|
||||
scanAndTransfer(job, plan, sourceConnection, targetConnection, sourceConnector, targetConnector, stats);
|
||||
checkCanceled(jobId);
|
||||
if (hasFailures(stats)) {
|
||||
log.warn("本次同步存在文件失败项,跳过源端删除处理,jobId={}", jobId);
|
||||
} else {
|
||||
processDeletedObjects(job, plan, targetConnection, targetConnector, stats);
|
||||
}
|
||||
checkCanceled(jobId);
|
||||
String checkpointAfter = LocalDateTime.now() + ":" + jobId;
|
||||
saveCheckpoint(plan.getPlanId(), jobId, checkpointAfter);
|
||||
if (!finish(jobId, stats, checkpointAfter)) {
|
||||
checkCanceled(jobId);
|
||||
throw new ServiceException("同步任务状态已发生变化,不能提交完成状态");
|
||||
}
|
||||
planMapper.lambda()
|
||||
.set(SyncPlan::getLastRunTime, LocalDateTime.now())
|
||||
.set(SyncPlan::getNextRunTime, calculateNextRunSafely(plan))
|
||||
.eq(SyncPlan::getPlanId, plan.getPlanId()).update();
|
||||
} catch (JobCanceledException ignored) {
|
||||
markCanceled(jobId);
|
||||
log.info("同步任务已取消,jobId={}", jobId);
|
||||
} catch (Exception e) {
|
||||
if (isCancellationRequested(jobId)) {
|
||||
markCanceled(jobId);
|
||||
log.info("同步任务已取消,jobId={}", jobId);
|
||||
} else {
|
||||
log.error("同步任务执行失败,jobId={}", jobId, e);
|
||||
fail(jobId, stats, e.getMessage());
|
||||
}
|
||||
} finally {
|
||||
activeExecutions.remove(jobId, executionThread);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 中断当前节点上正在执行的任务。数据库状态仍是跨节点取消的最终依据。
|
||||
*
|
||||
* @param jobId 任务ID
|
||||
*/
|
||||
public void cancel(Long jobId) {
|
||||
Thread executionThread = activeExecutions.get(jobId);
|
||||
if (executionThread != null) {
|
||||
executionThread.interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
private void scanAndTransfer(SyncJob job, SyncPlan plan, SyncConnection sourceConnection,
|
||||
SyncConnection targetConnection, SourceConnector sourceConnector,
|
||||
TargetConnector targetConnector, Stats stats) {
|
||||
int configuredConcurrency = plan.getMaxConcurrency() == null ? 1 : plan.getMaxConcurrency();
|
||||
int concurrency = Math.max(1, Math.min(configuredConcurrency, 100));
|
||||
ExecutorService executor = Executors.newFixedThreadPool(concurrency,
|
||||
Thread.ofVirtual().name("sync-transfer-" + job.getJobId() + "-", 0).factory());
|
||||
CompletionService<Void> transfers = new ExecutorCompletionService<>(executor);
|
||||
int outstanding = 0;
|
||||
try {
|
||||
Deque<FolderNode> folders = new ArrayDeque<>();
|
||||
FolderNode root = new FolderNode(normalizeSourceRoot(plan.getSourceRoot()), "", null);
|
||||
folders.add(root);
|
||||
Set<FolderKey> discoveredFolders = new HashSet<>();
|
||||
discoveredFolders.add(new FolderKey(root.scopeId(), root.objectId()));
|
||||
Set<String> discoveredObjects = new HashSet<>();
|
||||
long objectCount = 0;
|
||||
while (!folders.isEmpty()) {
|
||||
FolderNode folder = folders.removeFirst();
|
||||
String cursor = null;
|
||||
Set<String> visitedCursors = new HashSet<>();
|
||||
int pageCount = 0;
|
||||
do {
|
||||
checkCanceled(job.getJobId());
|
||||
if (++pageCount > MAX_PAGES_PER_FOLDER) {
|
||||
throw new ServiceException("单个源目录分页超过安全上限,已停止同步");
|
||||
}
|
||||
String cursorKey = StringUtils.isBlank(cursor) ? "<FIRST_PAGE>" : cursor;
|
||||
if (!visitedCursors.add(cursorKey)) {
|
||||
throw new ServiceException("源端返回了重复分页游标,已停止同步");
|
||||
}
|
||||
ScanResult page = sourceConnector.scan(sourceConnection,
|
||||
new ScanRequest(folder.objectId(), cursor, folder.scopeId()));
|
||||
if (page == null || page.objects() == null) {
|
||||
throw new ServiceException("源端返回了不完整的分页结果,已停止同步");
|
||||
}
|
||||
for (SourceObject sourceObject : page.objects()) {
|
||||
checkCanceled(job.getJobId());
|
||||
if (sourceObject == null || StringUtils.isBlank(sourceObject.objectId())) {
|
||||
throw new ServiceException("源端返回了缺少对象ID的条目,已停止同步");
|
||||
}
|
||||
if (!discoveredObjects.add(sourceObject.objectId())) {
|
||||
throw new ServiceException("源端返回了重复对象ID:{}", sourceObject.objectId());
|
||||
}
|
||||
if (++objectCount > MAX_OBJECTS_PER_RUN) {
|
||||
throw new ServiceException("本次扫描对象数超过安全上限,已停止同步");
|
||||
}
|
||||
String sourcePath = joinPath(folder.path(), sourceObject.name());
|
||||
if (sourcePath.length() > 2048) {
|
||||
throw new ServiceException("源端路径超过数据库长度上限:{}", sourcePath.substring(0, 256));
|
||||
}
|
||||
SyncObject existing = findObject(plan.getPlanId(), sourceObject.objectId());
|
||||
if (existing != null && !Objects.equals(existing.getObjectType(), sourceObject.objectType())) {
|
||||
throw new ServiceException("源对象类型发生变化,需人工确认后再同步:{}", sourcePath);
|
||||
}
|
||||
boolean existed = existing != null;
|
||||
boolean changed = isChanged(existing, sourcePath, sourceObject);
|
||||
String previousTargetKey = existing == null ? null : existing.getTargetKey();
|
||||
SyncObject current = saveSeenObject(existing, plan.getPlanId(), job.getJobId(),
|
||||
sourcePath, sourceObject);
|
||||
if (sourceObject.isFolder()) {
|
||||
FolderNode child = new FolderNode(sourceObject.objectId(), sourcePath,
|
||||
metadataValue(sourceObject, "spaceId"));
|
||||
if (!discoveredFolders.add(new FolderKey(child.scopeId(), child.objectId()))) {
|
||||
throw new ServiceException("源端目录结构存在重复或循环节点:{}", sourcePath);
|
||||
}
|
||||
folders.addLast(child);
|
||||
continue;
|
||||
}
|
||||
while (outstanding >= concurrency * 2) {
|
||||
awaitTransfer(transfers);
|
||||
outstanding--;
|
||||
}
|
||||
transfers.submit(() -> {
|
||||
checkCanceled(job.getJobId());
|
||||
transferObject(job, plan, targetConnection, sourceConnector, targetConnector,
|
||||
sourceConnection, sourceObject, existed, changed, previousTargetKey,
|
||||
current, sourcePath, stats);
|
||||
return null;
|
||||
});
|
||||
outstanding++;
|
||||
}
|
||||
cursor = page.hasMore() ? page.nextCursor() : null;
|
||||
if (page.hasMore() && StringUtils.isBlank(cursor)) {
|
||||
throw new ServiceException("源端返回 hasMore=true 但没有分页游标,已停止同步");
|
||||
}
|
||||
} while (StringUtils.isNotBlank(cursor));
|
||||
}
|
||||
while (outstanding > 0) {
|
||||
awaitTransfer(transfers);
|
||||
outstanding--;
|
||||
}
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
boolean interrupted = Thread.interrupted();
|
||||
long nextWarningAt = System.nanoTime() + TimeUnit.MINUTES.toNanos(1);
|
||||
try {
|
||||
while (!executor.isTerminated()) {
|
||||
try {
|
||||
if (executor.awaitTermination(10, TimeUnit.SECONDS)) {
|
||||
break;
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
interrupted = true;
|
||||
executor.shutdownNow();
|
||||
}
|
||||
if (System.nanoTime() >= nextWarningAt) {
|
||||
log.warn("仍在等待同步传输线程安全退出,jobId={}", job.getJobId());
|
||||
nextWarningAt = System.nanoTime() + TimeUnit.MINUTES.toNanos(1);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void awaitTransfer(CompletionService<Void> transfers) {
|
||||
try {
|
||||
transfers.take().get();
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new ServiceException("等待文件传输时线程被中断", e);
|
||||
} catch (ExecutionException e) {
|
||||
Throwable cause = e.getCause();
|
||||
if (cause instanceof JobCanceledException canceledException) {
|
||||
throw canceledException;
|
||||
}
|
||||
if (cause instanceof RuntimeException runtimeException) {
|
||||
throw runtimeException;
|
||||
}
|
||||
throw new ServiceException("文件传输线程执行失败", cause);
|
||||
}
|
||||
}
|
||||
|
||||
private void transferObject(SyncJob job, SyncPlan plan, SyncConnection targetConnection,
|
||||
SourceConnector sourceConnector, TargetConnector targetConnector,
|
||||
SyncConnection sourceConnection, SourceObject sourceObject,
|
||||
boolean existed, boolean changed, String previousTargetKey, SyncObject current,
|
||||
String sourcePath, Stats stats) {
|
||||
boolean full = "FULL".equalsIgnoreCase(job.getRunType());
|
||||
boolean shouldTransfer = full || !existed || changed;
|
||||
String action = !existed ? "CREATE" : changed || full ? "UPDATE" : "SKIP";
|
||||
if (!shouldTransfer) {
|
||||
action = "SKIP";
|
||||
}
|
||||
String targetKey = buildTargetKey(targetConnection, plan, sourcePath, sourceObject);
|
||||
SyncJobItem item = createItem(job.getJobId(), sourceObject, sourcePath, targetKey, action);
|
||||
synchronized (stats) {
|
||||
stats.totalCount++;
|
||||
}
|
||||
String unsupportedMessage = unsupportedSourceMessage(sourceObject);
|
||||
if (unsupportedMessage != null) {
|
||||
completeUnsupported(item, current, unsupportedMessage);
|
||||
synchronized (stats) {
|
||||
stats.processedCount++;
|
||||
stats.skippedCount++;
|
||||
}
|
||||
updateProgress(job.getJobId(), stats);
|
||||
return;
|
||||
}
|
||||
if (!shouldTransfer) {
|
||||
completeSkipped(item, current, false);
|
||||
synchronized (stats) {
|
||||
stats.processedCount++;
|
||||
stats.skippedCount++;
|
||||
}
|
||||
updateProgress(job.getJobId(), stats);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if ("SKIP".equals(plan.getConflictStrategy()) || "KEEP_BOTH".equals(plan.getConflictStrategy())) {
|
||||
checkCanceled(job.getJobId());
|
||||
boolean targetExists = targetConnector.exists(targetConnection, targetKey);
|
||||
checkCanceled(job.getJobId());
|
||||
if ("SKIP".equals(plan.getConflictStrategy()) && targetExists) {
|
||||
completeSkipped(item, current, true);
|
||||
synchronized (stats) {
|
||||
stats.skippedCount++;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if ("KEEP_BOTH".equals(plan.getConflictStrategy()) && targetExists) {
|
||||
targetKey = addVersionSuffix(targetKey, sourceVersionIdentity(sourceObject));
|
||||
item.setTargetKey(targetKey);
|
||||
itemMapper.updateById(item);
|
||||
}
|
||||
}
|
||||
synchronized (stats) {
|
||||
stats.totalBytes += Math.max(sourceObject.size(), 0L);
|
||||
}
|
||||
item.setStatus(SyncConstants.ITEM_RUNNING);
|
||||
item.setStartTime(LocalDateTime.now());
|
||||
itemMapper.updateById(item);
|
||||
try (SourceContent content = sourceConnector.download(sourceConnection, sourceObject)) {
|
||||
checkCanceled(job.getJobId());
|
||||
if (content.size() != sourceObject.size()) {
|
||||
synchronized (stats) {
|
||||
stats.totalBytes += content.size() - Math.max(sourceObject.size(), 0L);
|
||||
}
|
||||
item.setSize(content.size());
|
||||
}
|
||||
String sha256 = "SHA256".equals(plan.getVerifyMode())
|
||||
? sha256(content.path(), job.getJobId()) : sourceSha256(sourceObject.hash());
|
||||
checkCanceled(job.getJobId());
|
||||
Map<String, String> metadata = new HashMap<>();
|
||||
metadata.put("sync-source-id", sourceObject.objectId());
|
||||
if (StringUtils.isNotBlank(sha256)) {
|
||||
metadata.put("sync-sha256", sha256);
|
||||
}
|
||||
TargetWriteResult result = targetConnector.upload(targetConnection,
|
||||
new TargetWriteRequest(targetKey, content.path(), content.size(), content.contentType(), metadata));
|
||||
checkCanceled(job.getJobId());
|
||||
if (result.size() != content.size()) {
|
||||
throw new ServiceException("上传后大小校验失败,源大小 {},目标大小 {}", content.size(), result.size());
|
||||
}
|
||||
if ("SHA256".equals(plan.getVerifyMode())
|
||||
&& !Objects.equals(sha256, metadataValue(result.metadata(), "sync-sha256"))) {
|
||||
throw new ServiceException("上传后 SHA-256 元数据回读校验失败");
|
||||
}
|
||||
if ("ETAG".equals(plan.getVerifyMode()) && StringUtils.isBlank(result.eTag())) {
|
||||
throw new ServiceException("上传后未能读取目标 ETag");
|
||||
}
|
||||
item.setTransferredBytes(result.size());
|
||||
if (StringUtils.isNotBlank(previousTargetKey)
|
||||
&& !previousTargetKey.equals(result.objectKey())
|
||||
&& "DELETE".equals(plan.getDeleteStrategy())
|
||||
&& !"KEEP_BOTH".equals(plan.getConflictStrategy())) {
|
||||
checkCanceled(job.getJobId());
|
||||
targetConnector.delete(targetConnection, previousTargetKey);
|
||||
}
|
||||
item.setStatus(SyncConstants.ITEM_SUCCESS);
|
||||
item.setSourceSha256(sha256);
|
||||
item.setTargetEtag(result.eTag());
|
||||
item.setTargetVersionId(result.versionId());
|
||||
item.setFinishTime(LocalDateTime.now());
|
||||
itemMapper.updateById(item);
|
||||
current.setTargetKey(result.objectKey());
|
||||
current.setTargetVersionId(result.versionId());
|
||||
current.setTargetEtag(result.eTag());
|
||||
current.setSourceSha256(sha256);
|
||||
current.setSyncStatus("SYNCED");
|
||||
current.setLastSyncJobId(job.getJobId());
|
||||
current.setLastSyncTime(LocalDateTime.now());
|
||||
current.setLastErrorMessage(null);
|
||||
objectMapper.updateById(current);
|
||||
synchronized (stats) {
|
||||
stats.successCount++;
|
||||
stats.transferredBytes += result.size();
|
||||
}
|
||||
}
|
||||
} catch (JobCanceledException e) {
|
||||
item.setStatus(SyncConstants.ITEM_FAILED);
|
||||
item.setFinishTime(LocalDateTime.now());
|
||||
item.setErrorCode("CANCELED");
|
||||
item.setErrorMessage("任务已取消");
|
||||
itemMapper.updateById(item);
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
if (isAITableLink(sourceObject)) {
|
||||
String message = "钉钉多维表链接(.dlink)导出失败:" + abbreviate(e.getMessage());
|
||||
item.setStatus(SyncConstants.ITEM_SKIPPED);
|
||||
item.setFinishTime(LocalDateTime.now());
|
||||
item.setErrorCode("UNSUPPORTED");
|
||||
item.setErrorMessage(message);
|
||||
itemMapper.updateById(item);
|
||||
current.setSyncStatus("SKIPPED");
|
||||
current.setLastErrorMessage(message);
|
||||
objectMapper.updateById(current);
|
||||
synchronized (stats) {
|
||||
stats.skippedCount++;
|
||||
}
|
||||
} else {
|
||||
item.setStatus(SyncConstants.ITEM_FAILED);
|
||||
item.setFinishTime(LocalDateTime.now());
|
||||
item.setErrorCode(e.getClass().getSimpleName());
|
||||
item.setErrorMessage(abbreviate(e.getMessage()));
|
||||
itemMapper.updateById(item);
|
||||
current.setSyncStatus("FAILED");
|
||||
current.setLastErrorMessage(abbreviate(e.getMessage()));
|
||||
objectMapper.updateById(current);
|
||||
synchronized (stats) {
|
||||
stats.failedCount++;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
synchronized (stats) {
|
||||
stats.processedCount++;
|
||||
}
|
||||
updateProgress(job.getJobId(), stats);
|
||||
}
|
||||
}
|
||||
|
||||
private void processDeletedObjects(SyncJob job, SyncPlan plan, SyncConnection targetConnection,
|
||||
TargetConnector targetConnector, Stats stats) {
|
||||
List<SyncObject> deleted = objectMapper.selectList(Wrappers.<SyncObject>lambdaQuery()
|
||||
.eq(SyncObject::getPlanId, plan.getPlanId())
|
||||
.eq(SyncObject::getSourceDeleted, "0")
|
||||
.and(wrapper -> wrapper.isNull(SyncObject::getLastSeenJobId)
|
||||
.or().ne(SyncObject::getLastSeenJobId, job.getJobId())));
|
||||
validateDeleteGuard(plan, deleted);
|
||||
for (SyncObject object : deleted) {
|
||||
checkCanceled(job.getJobId());
|
||||
if (SyncConstants.OBJECT_FOLDER.equals(object.getObjectType())) {
|
||||
markSourceDeleted(object, job.getJobId());
|
||||
synchronized (stats) {
|
||||
stats.deletedCount++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
SyncJobItem item = createDeleteItem(job.getJobId(), object);
|
||||
synchronized (stats) {
|
||||
stats.totalCount++;
|
||||
}
|
||||
try {
|
||||
checkCanceled(job.getJobId());
|
||||
if ("DELETE".equals(plan.getDeleteStrategy()) && StringUtils.isNotBlank(object.getTargetKey())) {
|
||||
targetConnector.delete(targetConnection, object.getTargetKey());
|
||||
}
|
||||
checkCanceled(job.getJobId());
|
||||
item.setStatus(SyncConstants.ITEM_SUCCESS);
|
||||
item.setFinishTime(LocalDateTime.now());
|
||||
itemMapper.updateById(item);
|
||||
markSourceDeleted(object, job.getJobId());
|
||||
synchronized (stats) {
|
||||
stats.successCount++;
|
||||
stats.deletedCount++;
|
||||
}
|
||||
} catch (JobCanceledException e) {
|
||||
item.setStatus(SyncConstants.ITEM_FAILED);
|
||||
item.setFinishTime(LocalDateTime.now());
|
||||
item.setErrorCode("CANCELED");
|
||||
item.setErrorMessage("任务已取消");
|
||||
itemMapper.updateById(item);
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
item.setStatus(SyncConstants.ITEM_FAILED);
|
||||
item.setFinishTime(LocalDateTime.now());
|
||||
item.setErrorCode(e.getClass().getSimpleName());
|
||||
item.setErrorMessage(abbreviate(e.getMessage()));
|
||||
itemMapper.updateById(item);
|
||||
object.setSyncStatus("FAILED");
|
||||
object.setLastErrorMessage(abbreviate(e.getMessage()));
|
||||
objectMapper.updateById(object);
|
||||
synchronized (stats) {
|
||||
stats.failedCount++;
|
||||
}
|
||||
}
|
||||
synchronized (stats) {
|
||||
stats.processedCount++;
|
||||
}
|
||||
updateProgress(job.getJobId(), stats);
|
||||
}
|
||||
}
|
||||
|
||||
private void validateDeleteGuard(SyncPlan plan, List<SyncObject> deleted) {
|
||||
if (!"DELETE".equals(plan.getDeleteStrategy())) {
|
||||
return;
|
||||
}
|
||||
long destructiveCount = deleted.stream()
|
||||
.filter(object -> !SyncConstants.OBJECT_FOLDER.equals(object.getObjectType()))
|
||||
.filter(object -> StringUtils.isNotBlank(object.getTargetKey()))
|
||||
.count();
|
||||
if (destructiveCount == 0) {
|
||||
return;
|
||||
}
|
||||
long activeFileCount = objectMapper.selectCount(Wrappers.<SyncObject>lambdaQuery()
|
||||
.eq(SyncObject::getPlanId, plan.getPlanId())
|
||||
.eq(SyncObject::getSourceDeleted, "0")
|
||||
.ne(SyncObject::getObjectType, SyncConstants.OBJECT_FOLDER));
|
||||
int allowedPercent = plan.getDeleteGuardPercent() == null ? 50 : plan.getDeleteGuardPercent();
|
||||
if (activeFileCount > 0 && destructiveCount * 100L > activeFileCount * allowedPercent) {
|
||||
throw new ServiceException(
|
||||
"本次拟删除目标对象 {} 个,占现有文件 {} 个的比例超过保护阈值 {}%,已拒绝删除",
|
||||
destructiveCount, activeFileCount, allowedPercent);
|
||||
}
|
||||
}
|
||||
|
||||
private void markSourceDeleted(SyncObject object, Long jobId) {
|
||||
object.setSourceDeleted("1");
|
||||
object.setSyncStatus("DELETED");
|
||||
object.setLastSyncJobId(jobId);
|
||||
object.setLastSyncTime(LocalDateTime.now());
|
||||
object.setLastErrorMessage(null);
|
||||
objectMapper.updateById(object);
|
||||
}
|
||||
|
||||
private SyncObject saveSeenObject(SyncObject existing, Long planId, Long jobId,
|
||||
String sourcePath, SourceObject sourceObject) {
|
||||
SyncObject entity = existing == null ? new SyncObject() : existing;
|
||||
entity.setPlanId(planId);
|
||||
entity.setSourceObjectId(sourceObject.objectId());
|
||||
entity.setParentObjectId(sourceObject.parentObjectId());
|
||||
entity.setSourcePath(sourcePath);
|
||||
entity.setObjectName(sourceObject.name());
|
||||
entity.setObjectType(sourceObject.objectType());
|
||||
entity.setSize(sourceObject.size());
|
||||
entity.setModifiedTime(sourceObject.modifiedTime());
|
||||
entity.setVersionToken(sourceObject.versionToken());
|
||||
entity.setSourceEtag(sourceObject.hash());
|
||||
entity.setContentType(sourceObject.contentType());
|
||||
entity.setMetadataJson(JsonUtils.toJsonString(sourceObject.metadata()));
|
||||
entity.setSourceDeleted("0");
|
||||
entity.setLastSeenJobId(jobId);
|
||||
if (sourceObject.isFolder()) {
|
||||
entity.setSyncStatus("SYNCED");
|
||||
} else if (existing == null) {
|
||||
entity.setSyncStatus("PENDING");
|
||||
}
|
||||
if (existing == null) {
|
||||
entity.setFirstSeenJobId(jobId);
|
||||
objectMapper.insert(entity);
|
||||
} else {
|
||||
objectMapper.updateById(entity);
|
||||
}
|
||||
return entity;
|
||||
}
|
||||
|
||||
private SyncJobItem createItem(Long jobId, SourceObject object, String sourcePath,
|
||||
String targetKey, String action) {
|
||||
SyncJobItem item = new SyncJobItem();
|
||||
item.setJobId(jobId);
|
||||
item.setSourceObjectId(object.objectId());
|
||||
item.setParentObjectId(object.parentObjectId());
|
||||
item.setSourcePath(sourcePath);
|
||||
item.setTargetKey(targetKey);
|
||||
item.setObjectType(object.objectType());
|
||||
item.setActionType(action);
|
||||
item.setStatus(SyncConstants.ITEM_PENDING);
|
||||
item.setSize(object.size());
|
||||
item.setTransferredBytes(0L);
|
||||
item.setVersionToken(object.versionToken());
|
||||
item.setSourceEtag(object.hash());
|
||||
item.setRetryCount(0);
|
||||
itemMapper.insert(item);
|
||||
return item;
|
||||
}
|
||||
|
||||
private SyncJobItem createDeleteItem(Long jobId, SyncObject object) {
|
||||
SourceObject sourceObject = new SourceObject(object.getSourceObjectId(), object.getParentObjectId(),
|
||||
object.getObjectName(), object.getObjectType(), null, null, object.getSize(),
|
||||
object.getModifiedTime(), object.getVersionToken(), object.getSourceEtag(), Map.of());
|
||||
SyncJobItem item = createItem(jobId, sourceObject, object.getSourcePath(), object.getTargetKey(), "DELETE");
|
||||
item.setStartTime(LocalDateTime.now());
|
||||
itemMapper.updateById(item);
|
||||
return item;
|
||||
}
|
||||
|
||||
private void completeSkipped(SyncJobItem item, SyncObject object, boolean conflictSkipped) {
|
||||
item.setActionType("SKIP");
|
||||
item.setStatus(SyncConstants.ITEM_SKIPPED);
|
||||
item.setFinishTime(LocalDateTime.now());
|
||||
itemMapper.updateById(item);
|
||||
object.setSyncStatus(conflictSkipped ? "SKIPPED" : "SYNCED");
|
||||
object.setLastErrorMessage(null);
|
||||
objectMapper.updateById(object);
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记源端当前版本暂不支持的对象,避免把预期能力缺失计为任务失败。
|
||||
*/
|
||||
private void completeUnsupported(SyncJobItem item, SyncObject object, String message) {
|
||||
item.setActionType("SKIP");
|
||||
item.setStatus(SyncConstants.ITEM_SKIPPED);
|
||||
item.setFinishTime(LocalDateTime.now());
|
||||
item.setErrorCode("UNSUPPORTED");
|
||||
item.setErrorMessage(message);
|
||||
itemMapper.updateById(item);
|
||||
object.setSyncStatus("SKIPPED");
|
||||
object.setLastErrorMessage(message);
|
||||
objectMapper.updateById(object);
|
||||
}
|
||||
|
||||
private String unsupportedSourceMessage(SourceObject sourceObject) {
|
||||
String extension = sourceObject.extension();
|
||||
if (extension == null) {
|
||||
return null;
|
||||
}
|
||||
extension = extension.startsWith(".") ? extension.substring(1) : extension;
|
||||
extension = extension.toLowerCase(Locale.ROOT);
|
||||
if ("amind".equals(extension) || "adraw".equals(extension)) {
|
||||
return "钉钉官方 DWS 暂不支持自动导出:" + extension;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean isAITableLink(SourceObject sourceObject) {
|
||||
String extension = sourceObject.extension();
|
||||
if (extension == null) return false;
|
||||
return "dlink".equalsIgnoreCase(extension.startsWith(".") ? extension.substring(1) : extension);
|
||||
}
|
||||
|
||||
private boolean isChanged(SyncObject existing, String sourcePath, SourceObject sourceObject) {
|
||||
if (existing == null) {
|
||||
return true;
|
||||
}
|
||||
if (!"SYNCED".equals(existing.getSyncStatus())
|
||||
|| "1".equals(existing.getSourceDeleted())
|
||||
|| StringUtils.isBlank(existing.getTargetKey())
|
||||
|| !Objects.equals(existing.getSourcePath(), sourcePath)) {
|
||||
return true;
|
||||
}
|
||||
if (StringUtils.isNotBlank(sourceObject.versionToken()) || StringUtils.isNotBlank(existing.getVersionToken())) {
|
||||
return !Objects.equals(existing.getVersionToken(), sourceObject.versionToken());
|
||||
}
|
||||
if (StringUtils.isNotBlank(sourceObject.hash()) || StringUtils.isNotBlank(existing.getSourceEtag())) {
|
||||
return !Objects.equals(existing.getSourceEtag(), sourceObject.hash())
|
||||
|| !Objects.equals(existing.getSize(), sourceObject.size());
|
||||
}
|
||||
if (sourceObject.modifiedTime() == null || existing.getModifiedTime() == null) {
|
||||
return true;
|
||||
}
|
||||
return !Objects.equals(existing.getSize(), sourceObject.size())
|
||||
|| !Objects.equals(existing.getModifiedTime(), sourceObject.modifiedTime());
|
||||
}
|
||||
|
||||
private String buildTargetKey(SyncConnection targetConnection, SyncPlan plan, String sourcePath,
|
||||
SourceObject sourceObject) {
|
||||
String relativePath = sourcePath;
|
||||
if ("dlink".equalsIgnoreCase(normalizeExtension(sourceObject.extension()))) {
|
||||
relativePath = replaceExtension(relativePath, "xlsx");
|
||||
} else if (SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(sourceObject.objectType())) {
|
||||
if ("adoc".equalsIgnoreCase(sourceObject.extension())) {
|
||||
relativePath = replaceExtension(relativePath, "docx");
|
||||
} else if ("axls".equalsIgnoreCase(sourceObject.extension())) {
|
||||
relativePath = replaceExtension(relativePath, "xlsx");
|
||||
} else if ("amind".equalsIgnoreCase(sourceObject.extension())
|
||||
|| "adraw".equalsIgnoreCase(sourceObject.extension())) {
|
||||
relativePath = replaceExtension(relativePath, "pdf");
|
||||
}
|
||||
}
|
||||
return joinObjectKey(targetConnection.getBasePath(), plan.getTargetPrefix(), relativePath);
|
||||
}
|
||||
|
||||
private String joinObjectKey(String... values) {
|
||||
List<String> segments = new ArrayList<>();
|
||||
for (String value : values) {
|
||||
if (StringUtils.isBlank(value)) {
|
||||
continue;
|
||||
}
|
||||
for (String segment : value.replace('\\', '/').split("/")) {
|
||||
if (StringUtils.isBlank(segment)) {
|
||||
continue;
|
||||
}
|
||||
if (".".equals(segment) || "..".equals(segment)) {
|
||||
throw new ServiceException("对象键路径不能包含 . 或 .. 段");
|
||||
}
|
||||
segments.add(segment);
|
||||
}
|
||||
}
|
||||
if (segments.isEmpty()) {
|
||||
throw new ServiceException("目标对象键不能为空");
|
||||
}
|
||||
String key = String.join("/", segments);
|
||||
if (key.length() > 2048) {
|
||||
throw new ServiceException("目标对象键超过数据库长度上限");
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
private String joinPath(String parent, String name) {
|
||||
return StringUtils.isBlank(parent) ? name : parent + '/' + name;
|
||||
}
|
||||
|
||||
private String normalizeSourceRoot(String sourceRoot) {
|
||||
String value = StringUtils.trim(sourceRoot);
|
||||
return StringUtils.isBlank(value) || "/".equals(value) ? null : value;
|
||||
}
|
||||
|
||||
private LocalDateTime calculateNextRunSafely(SyncPlan plan) {
|
||||
if (!"CRON".equals(plan.getScheduleType()) || StringUtils.isBlank(plan.getCronExpression())) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
ZonedDateTime next = CronExpression.parse(plan.getCronExpression()).next(ZonedDateTime.now());
|
||||
return next == null ? null : next.toLocalDateTime();
|
||||
} catch (Exception e) {
|
||||
log.warn("同步计划 Cron 表达式已失效,planId={},原因={}", plan.getPlanId(), e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String metadataValue(SourceObject sourceObject, String key) {
|
||||
return sourceObject.metadata() == null ? null : sourceObject.metadata().get(key);
|
||||
}
|
||||
|
||||
private String metadataValue(Map<String, String> metadata, String key) {
|
||||
return metadata == null ? null : metadata.get(key);
|
||||
}
|
||||
|
||||
private String sourceSha256(String hash) {
|
||||
if (StringUtils.isBlank(hash) || !hash.matches("(?i)[0-9a-f]{64}")) {
|
||||
return null;
|
||||
}
|
||||
return hash.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String normalizeExtension(String extension) {
|
||||
if (StringUtils.isBlank(extension)) return "";
|
||||
String value = extension.startsWith(".") ? extension.substring(1) : extension;
|
||||
return value.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String replaceExtension(String path, String extension) {
|
||||
int slash = path.lastIndexOf('/');
|
||||
int dot = path.lastIndexOf('.');
|
||||
return dot > slash ? path.substring(0, dot + 1) + extension : path + '.' + extension;
|
||||
}
|
||||
|
||||
private String addVersionSuffix(String key, String versionToken) {
|
||||
String suffix = StringUtils.isBlank(versionToken)
|
||||
? String.valueOf(System.currentTimeMillis()) : versionToken.replaceAll("[^a-zA-Z0-9_-]", "_");
|
||||
if (suffix.length() > 48) {
|
||||
suffix = suffix.substring(0, 48);
|
||||
}
|
||||
int slash = key.lastIndexOf('/');
|
||||
int dot = key.lastIndexOf('.');
|
||||
String versionedKey = dot > slash
|
||||
? key.substring(0, dot) + "__" + suffix + key.substring(dot) : key + "__" + suffix;
|
||||
if (versionedKey.length() > 2048) {
|
||||
throw new ServiceException("保留多版本后的目标对象键超过数据库长度上限");
|
||||
}
|
||||
return versionedKey;
|
||||
}
|
||||
|
||||
private String sourceVersionIdentity(SourceObject sourceObject) {
|
||||
if (StringUtils.isNotBlank(sourceObject.versionToken())) {
|
||||
return sourceObject.versionToken();
|
||||
}
|
||||
if (StringUtils.isNotBlank(sourceObject.hash())) {
|
||||
return sourceObject.hash();
|
||||
}
|
||||
if (sourceObject.modifiedTime() != null) {
|
||||
return sourceObject.modifiedTime() + "_" + sourceObject.size();
|
||||
}
|
||||
return "size_" + sourceObject.size();
|
||||
}
|
||||
|
||||
private String sha256(java.nio.file.Path path, Long jobId) throws Exception {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
try (InputStream input = Files.newInputStream(path)) {
|
||||
byte[] buffer = new byte[1024 * 1024];
|
||||
int length;
|
||||
while ((length = input.read(buffer)) >= 0) {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
checkCanceled(jobId);
|
||||
throw new ServiceException("计算文件摘要时线程被中断");
|
||||
}
|
||||
if (length > 0) {
|
||||
digest.update(buffer, 0, length);
|
||||
}
|
||||
}
|
||||
}
|
||||
checkCanceled(jobId);
|
||||
return HexFormat.of().formatHex(digest.digest());
|
||||
}
|
||||
|
||||
private SyncObject findObject(Long planId, String sourceObjectId) {
|
||||
return objectMapper.selectOne(Wrappers.<SyncObject>lambdaQuery()
|
||||
.eq(SyncObject::getPlanId, planId)
|
||||
.eq(SyncObject::getSourceObjectId, sourceObjectId));
|
||||
}
|
||||
|
||||
private String loadCheckpoint(Long planId) {
|
||||
SyncCheckpoint checkpoint = checkpointMapper.selectOne(Wrappers.<SyncCheckpoint>lambdaQuery()
|
||||
.eq(SyncCheckpoint::getPlanId, planId)
|
||||
.eq(SyncCheckpoint::getCheckpointType, CHECKPOINT_TYPE)
|
||||
.eq(SyncCheckpoint::getCheckpointKey, CHECKPOINT_KEY));
|
||||
return checkpoint == null ? null : checkpoint.getCheckpointValue();
|
||||
}
|
||||
|
||||
private void saveCheckpoint(Long planId, Long jobId, String value) {
|
||||
SyncCheckpoint checkpoint = checkpointMapper.selectOne(Wrappers.<SyncCheckpoint>lambdaQuery()
|
||||
.eq(SyncCheckpoint::getPlanId, planId)
|
||||
.eq(SyncCheckpoint::getCheckpointType, CHECKPOINT_TYPE)
|
||||
.eq(SyncCheckpoint::getCheckpointKey, CHECKPOINT_KEY));
|
||||
if (checkpoint == null) {
|
||||
checkpoint = new SyncCheckpoint();
|
||||
checkpoint.setPlanId(planId);
|
||||
checkpoint.setCheckpointType(CHECKPOINT_TYPE);
|
||||
checkpoint.setCheckpointKey(CHECKPOINT_KEY);
|
||||
checkpoint.setCheckpointValue(value);
|
||||
checkpoint.setWatermarkTime(LocalDateTime.now());
|
||||
checkpoint.setLastJobId(jobId);
|
||||
checkpoint.setVersion(0L);
|
||||
checkpointMapper.insert(checkpoint);
|
||||
} else {
|
||||
checkpoint.setCheckpointValue(value);
|
||||
checkpoint.setWatermarkTime(LocalDateTime.now());
|
||||
checkpoint.setLastJobId(jobId);
|
||||
checkpointMapper.updateById(checkpoint);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean markRunning(Long jobId) {
|
||||
return jobMapper.lambda()
|
||||
.set(SyncJob::getStatus, SyncConstants.JOB_RUNNING)
|
||||
.set(SyncJob::getStartTime, LocalDateTime.now())
|
||||
.eq(SyncJob::getJobId, jobId)
|
||||
.eq(SyncJob::getStatus, SyncConstants.JOB_PENDING)
|
||||
.update();
|
||||
}
|
||||
|
||||
private void updateProgress(Long jobId, Stats stats) {
|
||||
synchronized (stats) {
|
||||
jobMapper.lambda()
|
||||
.set(SyncJob::getTotalCount, stats.totalCount)
|
||||
.set(SyncJob::getProcessedCount, stats.processedCount)
|
||||
.set(SyncJob::getSuccessCount, stats.successCount)
|
||||
.set(SyncJob::getFailedCount, stats.failedCount)
|
||||
.set(SyncJob::getSkippedCount, stats.skippedCount)
|
||||
.set(SyncJob::getDeletedCount, stats.deletedCount)
|
||||
.set(SyncJob::getTotalBytes, stats.totalBytes)
|
||||
.set(SyncJob::getTransferredBytes, stats.transferredBytes)
|
||||
.eq(SyncJob::getJobId, jobId)
|
||||
.update();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean hasFailures(Stats stats) {
|
||||
synchronized (stats) {
|
||||
return stats.failedCount > 0;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean finish(Long jobId, Stats stats, String checkpointAfter) {
|
||||
String status;
|
||||
synchronized (stats) {
|
||||
status = stats.failedCount == 0 ? SyncConstants.JOB_SUCCESS
|
||||
: stats.successCount > 0 || stats.skippedCount > 0
|
||||
? SyncConstants.JOB_PARTIAL_FAILED : SyncConstants.JOB_FAILED;
|
||||
}
|
||||
updateProgress(jobId, stats);
|
||||
return jobMapper.lambda()
|
||||
.set(SyncJob::getStatus, status)
|
||||
.set(SyncJob::getCheckpointAfter, checkpointAfter)
|
||||
.set(SyncJob::getFinishTime, LocalDateTime.now())
|
||||
.eq(SyncJob::getJobId, jobId)
|
||||
.eq(SyncJob::getStatus, SyncConstants.JOB_RUNNING)
|
||||
.update();
|
||||
}
|
||||
|
||||
private void fail(Long jobId, Stats stats, String message) {
|
||||
updateProgress(jobId, stats);
|
||||
jobMapper.lambda()
|
||||
.set(SyncJob::getStatus, SyncConstants.JOB_FAILED)
|
||||
.set(SyncJob::getFinishTime, LocalDateTime.now())
|
||||
.set(SyncJob::getErrorMessage, abbreviate(message))
|
||||
.eq(SyncJob::getJobId, jobId)
|
||||
.eq(SyncJob::getStatus, SyncConstants.JOB_RUNNING)
|
||||
.update();
|
||||
}
|
||||
|
||||
private boolean isCancellationRequested(Long jobId) {
|
||||
SyncJob job = jobMapper.selectById(jobId);
|
||||
return job != null && (SyncConstants.JOB_CANCELING.equals(job.getStatus())
|
||||
|| SyncConstants.JOB_CANCELED.equals(job.getStatus()));
|
||||
}
|
||||
|
||||
private void markCanceled(Long jobId) {
|
||||
jobMapper.lambda()
|
||||
.set(SyncJob::getStatus, SyncConstants.JOB_CANCELED)
|
||||
.set(SyncJob::getFinishTime, LocalDateTime.now())
|
||||
.eq(SyncJob::getJobId, jobId)
|
||||
.eq(SyncJob::getStatus, SyncConstants.JOB_CANCELING)
|
||||
.update();
|
||||
}
|
||||
|
||||
private void checkCanceled(Long jobId) {
|
||||
SyncJob job = requireJob(jobId);
|
||||
if (SyncConstants.JOB_CANCELING.equals(job.getStatus())
|
||||
|| SyncConstants.JOB_CANCELED.equals(job.getStatus())) {
|
||||
throw new JobCanceledException();
|
||||
}
|
||||
}
|
||||
|
||||
private SyncJob requireJob(Long jobId) {
|
||||
SyncJob job = jobMapper.selectById(jobId);
|
||||
if (job == null) {
|
||||
throw new ServiceException("同步任务不存在:{}", jobId);
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
private SyncPlan requirePlan(Long planId) {
|
||||
SyncPlan plan = planMapper.selectById(planId);
|
||||
if (plan == null) {
|
||||
throw new ServiceException("同步计划不存在:{}", planId);
|
||||
}
|
||||
return plan;
|
||||
}
|
||||
|
||||
private SyncConnection requireConnection(Long connectionId, String expectedRole) {
|
||||
SyncConnection connection = connectionMapper.selectById(connectionId);
|
||||
if (connection == null || !expectedRole.equals(connection.getConnectionRole())) {
|
||||
throw new ServiceException("同步连接不存在或角色不正确:{}", connectionId);
|
||||
}
|
||||
if (!SyncConstants.STATUS_NORMAL.equals(connection.getStatus())) {
|
||||
throw new ServiceException("同步连接已停用:{}", connection.getConnectionName());
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
private String abbreviate(String message) {
|
||||
if (message == null) {
|
||||
return "未知错误";
|
||||
}
|
||||
return message.length() <= 2000 ? message : message.substring(0, 2000);
|
||||
}
|
||||
|
||||
private record FolderNode(String objectId, String path, String scopeId) {
|
||||
}
|
||||
|
||||
private record FolderKey(String scopeId, String objectId) {
|
||||
}
|
||||
|
||||
private static final class Stats {
|
||||
private long totalCount;
|
||||
private long processedCount;
|
||||
private long successCount;
|
||||
private long failedCount;
|
||||
private long skippedCount;
|
||||
private long deletedCount;
|
||||
private long totalBytes;
|
||||
private long transferredBytes;
|
||||
}
|
||||
|
||||
private static final class JobCanceledException extends RuntimeException {
|
||||
private static final long serialVersionUID = 1L;
|
||||
}
|
||||
}
|
||||
359
script/sql/ry_sync.sql
Normal file
359
script/sql/ry_sync.sql
Normal file
@ -0,0 +1,359 @@
|
||||
-- ----------------------------
|
||||
-- 数据同步模块(MySQL)
|
||||
-- 依赖 ry_vue.sql 中的 sys_menu、sys_dict_type、sys_dict_data 表
|
||||
-- ----------------------------
|
||||
|
||||
-- ----------------------------
|
||||
-- 1、连接配置表
|
||||
-- ----------------------------
|
||||
create table sync_connection
|
||||
(
|
||||
connection_id bigint(20) not null comment '连接ID',
|
||||
connection_name varchar(100) not null comment '连接名称',
|
||||
connection_role varchar(16) not null comment '连接角色(SOURCE源端 TARGET目标端)',
|
||||
connection_type varchar(32) not null comment '连接类型(DINGTALK钉钉 S3标准S3 ALIYUN_OSS阿里云OSS)',
|
||||
endpoint varchar(512) default null comment '服务端点',
|
||||
region varchar(64) default null comment '区域',
|
||||
bucket_name varchar(255) default null comment '存储桶名称',
|
||||
base_path varchar(1024) default '' comment '连接默认根路径',
|
||||
config_json text comment '非敏感扩展配置JSON',
|
||||
secret_json text comment '加密后的敏感配置JSON(禁止明文存储)',
|
||||
status char(1) default '0' comment '状态(0正常 1停用)',
|
||||
del_flag char(1) default '0' comment '删除标志(0代表存在 1代表删除)',
|
||||
active_name varchar(100) generated always as
|
||||
(case when del_flag = '0' then connection_name else null end) stored
|
||||
comment '有效连接名称(用于唯一约束)',
|
||||
create_dept bigint(20) default null comment '创建部门',
|
||||
create_by bigint(20) default null comment '创建者',
|
||||
create_time datetime default null comment '创建时间',
|
||||
update_by bigint(20) default null comment '更新者',
|
||||
update_time datetime default null comment '更新时间',
|
||||
remark varchar(500) default null comment '备注',
|
||||
primary key (connection_id),
|
||||
unique key uk_sync_connection_active_name (active_name),
|
||||
key idx_sync_connection_name_del (connection_name, del_flag),
|
||||
key idx_sync_connection_role_type (connection_role, connection_type),
|
||||
key idx_sync_connection_status_del (status, del_flag)
|
||||
) engine = innodb comment = '数据同步连接配置表';
|
||||
|
||||
-- ----------------------------
|
||||
-- 2、同步计划表
|
||||
-- ----------------------------
|
||||
create table sync_plan
|
||||
(
|
||||
plan_id bigint(20) not null comment '同步计划ID',
|
||||
plan_name varchar(100) not null comment '计划名称',
|
||||
source_connection_id bigint(20) not null comment '源端连接ID',
|
||||
target_connection_id bigint(20) not null comment '目标端连接ID',
|
||||
source_root varchar(1024) default '/' comment '源端同步根路径或空间标识',
|
||||
target_prefix varchar(1024) default '' comment '目标端对象键前缀',
|
||||
sync_mode varchar(16) default 'INCREMENTAL' comment '同步模式(FULL全量 INCREMENTAL增量)',
|
||||
schedule_type varchar(16) default 'MANUAL' comment '调度类型(MANUAL手动 CRON定时)',
|
||||
cron_expression varchar(128) default null comment 'Cron表达式',
|
||||
conflict_strategy varchar(16) default 'OVERWRITE' comment '冲突策略(OVERWRITE覆盖 SKIP跳过 KEEP_BOTH两者保留)',
|
||||
delete_strategy varchar(16) default 'KEEP' comment '删除策略(KEEP保留 MARK标记 DELETE删除目标)',
|
||||
delete_guard_percent int(11) default 50 comment '单次目标删除比例保护阈值(0-100,100表示允许全删)',
|
||||
verify_mode varchar(16) default 'SIZE' comment '校验方式(SIZE大小 ETAG标识 SHA256摘要)',
|
||||
max_concurrency int(11) default 4 comment '最大并发传输数',
|
||||
bandwidth_limit_kbps bigint(20) default 0 comment '带宽上限KB/s(0表示不限速)',
|
||||
status char(1) default '0' comment '状态(0正常 1停用)',
|
||||
last_run_time datetime default null comment '最近运行时间',
|
||||
next_run_time datetime default null comment '下次运行时间',
|
||||
del_flag char(1) default '0' comment '删除标志(0代表存在 1代表删除)',
|
||||
active_name varchar(100) generated always as
|
||||
(case when del_flag = '0' then plan_name else null end) stored
|
||||
comment '有效计划名称(用于唯一约束)',
|
||||
create_dept bigint(20) default null comment '创建部门',
|
||||
create_by bigint(20) default null comment '创建者',
|
||||
create_time datetime default null comment '创建时间',
|
||||
update_by bigint(20) default null comment '更新者',
|
||||
update_time datetime default null comment '更新时间',
|
||||
remark varchar(500) default null comment '备注',
|
||||
primary key (plan_id),
|
||||
unique key uk_sync_plan_active_name (active_name),
|
||||
key idx_sync_plan_name_del (plan_name, del_flag),
|
||||
key idx_sync_plan_source_connection (source_connection_id),
|
||||
key idx_sync_plan_target_connection (target_connection_id),
|
||||
key idx_sync_plan_schedule (status, del_flag, schedule_type, next_run_time)
|
||||
) engine = innodb comment = '数据同步计划表';
|
||||
|
||||
-- ----------------------------
|
||||
-- 3、同步任务表
|
||||
-- ----------------------------
|
||||
create table sync_job
|
||||
(
|
||||
job_id bigint(20) not null comment '同步任务ID',
|
||||
plan_id bigint(20) not null comment '同步计划ID',
|
||||
trigger_type varchar(16) not null comment '触发类型(MANUAL手动 SCHEDULE调度 RETRY重试)',
|
||||
run_type varchar(16) not null comment '运行类型(FULL全量 INCREMENTAL增量)',
|
||||
status varchar(20) default 'PENDING' comment '任务状态(PENDING待执行 RUNNING执行中 CANCELING取消中 SUCCESS成功 PARTIAL_FAILED部分失败 FAILED失败 CANCELED已取消)',
|
||||
active_plan_id bigint(20) generated always as
|
||||
(case when status in ('PENDING', 'RUNNING', 'CANCELING') then plan_id else null end) stored
|
||||
comment '活动任务计划ID(用于保证单计划仅一个运行任务)',
|
||||
checkpoint_before text comment '运行前检查点快照',
|
||||
checkpoint_after text comment '运行后检查点快照',
|
||||
total_count bigint(20) default 0 comment '待处理对象总数',
|
||||
processed_count bigint(20) default 0 comment '已处理对象数',
|
||||
success_count bigint(20) default 0 comment '成功对象数',
|
||||
failed_count bigint(20) default 0 comment '失败对象数',
|
||||
skipped_count bigint(20) default 0 comment '跳过对象数',
|
||||
deleted_count bigint(20) default 0 comment '删除或标记对象数',
|
||||
total_bytes bigint(20) default 0 comment '待传输总字节数',
|
||||
transferred_bytes bigint(20) default 0 comment '已传输字节数',
|
||||
start_time datetime default null comment '开始时间',
|
||||
finish_time datetime default null comment '结束时间',
|
||||
error_message text comment '任务错误信息',
|
||||
create_dept bigint(20) default null comment '创建部门',
|
||||
create_by bigint(20) default null comment '创建者',
|
||||
create_time datetime default null comment '创建时间',
|
||||
update_by bigint(20) default null comment '更新者',
|
||||
update_time datetime default null comment '更新时间',
|
||||
primary key (job_id),
|
||||
unique key uk_sync_job_active_plan (active_plan_id),
|
||||
key idx_sync_job_plan_time (plan_id, create_time),
|
||||
key idx_sync_job_status_time (status, create_time)
|
||||
) engine = innodb comment = '数据同步任务表';
|
||||
|
||||
-- ----------------------------
|
||||
-- 4、同步任务明细表
|
||||
-- ----------------------------
|
||||
create table sync_job_item
|
||||
(
|
||||
item_id bigint(20) not null comment '任务明细ID',
|
||||
job_id bigint(20) not null comment '同步任务ID',
|
||||
source_object_id varchar(255) not null comment '源端对象唯一标识',
|
||||
parent_object_id varchar(255) default null comment '源端父对象标识',
|
||||
source_path varchar(2048) not null comment '源端相对路径',
|
||||
target_key varchar(2048) default null comment '目标端对象键',
|
||||
object_type varchar(16) not null comment '对象类型(FILE文件 FOLDER目录 ONLINE_DOCUMENT在线文档)',
|
||||
action_type varchar(16) not null comment '动作类型(CREATE新增上传 UPDATE更新上传 SKIP跳过 DELETE删除)',
|
||||
status varchar(20) default 'PENDING' comment '明细状态(PENDING待执行 RUNNING执行中 SUCCESS成功 SKIPPED已跳过 FAILED失败)',
|
||||
size bigint(20) default 0 comment '源对象字节数',
|
||||
transferred_bytes bigint(20) default 0 comment '已传输字节数',
|
||||
version_token varchar(512) default null comment '源端版本标识',
|
||||
source_etag varchar(255) default null comment '源端ETag',
|
||||
source_sha256 char(64) default null comment '源端SHA-256摘要',
|
||||
target_etag varchar(255) default null comment '目标端ETag',
|
||||
target_version_id varchar(512) default null comment '目标端版本ID',
|
||||
retry_count int(11) default 0 comment '已重试次数',
|
||||
start_time datetime default null comment '开始时间',
|
||||
finish_time datetime default null comment '结束时间',
|
||||
error_code varchar(64) default null comment '错误码',
|
||||
error_message text comment '错误信息',
|
||||
create_dept bigint(20) default null comment '创建部门',
|
||||
create_by bigint(20) default null comment '创建者',
|
||||
create_time datetime default null comment '创建时间',
|
||||
update_by bigint(20) default null comment '更新者',
|
||||
update_time datetime default null comment '更新时间',
|
||||
primary key (item_id),
|
||||
unique key uk_sync_job_item_job_object (job_id, source_object_id),
|
||||
key idx_sync_job_item_job_status (job_id, status),
|
||||
key idx_sync_job_item_retry (status, retry_count),
|
||||
key idx_sync_job_item_parent (job_id, parent_object_id)
|
||||
) engine = innodb comment = '数据同步任务明细表';
|
||||
|
||||
-- ----------------------------
|
||||
-- 5、源对象状态表
|
||||
-- ----------------------------
|
||||
create table sync_object
|
||||
(
|
||||
object_id bigint(20) not null comment '对象记录ID',
|
||||
plan_id bigint(20) not null comment '同步计划ID',
|
||||
source_object_id varchar(255) not null comment '源端对象唯一标识',
|
||||
parent_object_id varchar(255) default null comment '源端父对象标识',
|
||||
object_name varchar(1024) not null comment '对象名称',
|
||||
source_path varchar(2048) not null comment '相对同步根目录的源端路径',
|
||||
object_type varchar(16) not null comment '对象类型(FILE文件 FOLDER目录 ONLINE_DOCUMENT在线文档)',
|
||||
size bigint(20) default 0 comment '对象字节数',
|
||||
modified_time datetime default null comment '源端最后修改时间',
|
||||
version_token varchar(512) default null comment '源端版本标识',
|
||||
source_etag varchar(255) default null comment '源端ETag',
|
||||
source_sha256 char(64) default null comment '源端SHA-256摘要',
|
||||
content_type varchar(255) default null comment '内容类型',
|
||||
metadata_json text comment '源端扩展元数据JSON',
|
||||
target_key varchar(2048) default null comment '最近成功同步的目标对象键',
|
||||
target_version_id varchar(512) default null comment '最近成功同步的目标版本ID',
|
||||
target_etag varchar(255) default null comment '最近成功同步的目标端ETag',
|
||||
sync_status varchar(20) default 'DISCOVERED' comment '同步状态(DISCOVERED已发现 PENDING待同步 SYNCED已同步 SKIPPED已跳过 FAILED失败 DELETED源端已删除)',
|
||||
source_deleted char(1) default '0' comment '源端删除标志(0未删除 1已删除)',
|
||||
first_seen_job_id bigint(20) default null comment '首次发现任务ID',
|
||||
last_seen_job_id bigint(20) default null comment '最近扫描发现任务ID',
|
||||
last_sync_job_id bigint(20) default null comment '最近成功同步任务ID',
|
||||
last_sync_time datetime default null comment '最近成功同步时间',
|
||||
last_error_message text comment '最近同步错误信息',
|
||||
create_dept bigint(20) default null comment '创建部门',
|
||||
create_by bigint(20) default null comment '创建者',
|
||||
create_time datetime default null comment '创建时间',
|
||||
update_by bigint(20) default null comment '更新者',
|
||||
update_time datetime default null comment '更新时间',
|
||||
primary key (object_id),
|
||||
unique key uk_sync_object_plan_source (plan_id, source_object_id),
|
||||
key idx_sync_object_plan_status (plan_id, sync_status, source_deleted),
|
||||
key idx_sync_object_last_seen (plan_id, last_seen_job_id),
|
||||
key idx_sync_object_target_key (plan_id, target_key(191))
|
||||
) engine = innodb comment = '数据同步源对象状态表';
|
||||
|
||||
-- ----------------------------
|
||||
-- 6、增量检查点表
|
||||
-- ----------------------------
|
||||
create table sync_checkpoint
|
||||
(
|
||||
checkpoint_id bigint(20) not null comment '检查点ID',
|
||||
plan_id bigint(20) not null comment '同步计划ID',
|
||||
checkpoint_type varchar(32) not null comment '检查点类型(PAGE_CURSOR分页游标 INCREMENTAL_WATERMARK增量水位 RUN_WATERMARK运行水位)',
|
||||
checkpoint_key varchar(255) not null comment '检查点作用域键(空间、目录或分区标识)',
|
||||
checkpoint_value longtext comment '检查点值或JSON快照',
|
||||
watermark_time datetime default null comment '增量水位时间',
|
||||
last_job_id bigint(20) default null comment '最近提交检查点的任务ID',
|
||||
version bigint(20) default 0 comment '乐观锁版本号',
|
||||
create_dept bigint(20) default null comment '创建部门',
|
||||
create_by bigint(20) default null comment '创建者',
|
||||
create_time datetime default null comment '创建时间',
|
||||
update_by bigint(20) default null comment '更新者',
|
||||
update_time datetime default null comment '更新时间',
|
||||
primary key (checkpoint_id),
|
||||
unique key uk_sync_checkpoint_plan_type_key (plan_id, checkpoint_type, checkpoint_key),
|
||||
key idx_sync_checkpoint_last_job (last_job_id)
|
||||
) engine = innodb comment = '数据同步增量检查点表';
|
||||
|
||||
-- ----------------------------
|
||||
-- 7、分片传输记录表
|
||||
-- ----------------------------
|
||||
create table sync_transfer_part
|
||||
(
|
||||
part_id bigint(20) not null comment '分片记录ID',
|
||||
job_item_id bigint(20) not null comment '任务明细ID',
|
||||
upload_id varchar(512) not null comment '目标端分片上传会话ID',
|
||||
part_number int(11) not null comment '分片序号(从1开始)',
|
||||
part_offset bigint(20) default 0 comment '分片起始字节偏移',
|
||||
part_size bigint(20) default 0 comment '分片字节数',
|
||||
transferred_bytes bigint(20) default 0 comment '分片已传输字节数',
|
||||
part_etag varchar(255) default null comment '目标端分片ETag',
|
||||
checksum_sha256 char(64) default null comment '分片SHA-256摘要',
|
||||
status varchar(20) default 'PENDING' comment '分片状态(PENDING待上传 UPLOADING上传中 SUCCESS成功 FAILED失败)',
|
||||
retry_count int(11) default 0 comment '已重试次数',
|
||||
start_time datetime default null comment '开始时间',
|
||||
finish_time datetime default null comment '结束时间',
|
||||
error_message text comment '错误信息',
|
||||
create_dept bigint(20) default null comment '创建部门',
|
||||
create_by bigint(20) default null comment '创建者',
|
||||
create_time datetime default null comment '创建时间',
|
||||
update_by bigint(20) default null comment '更新者',
|
||||
update_time datetime default null comment '更新时间',
|
||||
primary key (part_id),
|
||||
unique key uk_sync_transfer_part_item_number (job_item_id, part_number),
|
||||
key idx_sync_transfer_part_upload_status (upload_id(191), status),
|
||||
key idx_sync_transfer_part_status_retry (status, retry_count)
|
||||
) engine = innodb comment = '数据同步分片传输记录表';
|
||||
|
||||
-- ----------------------------
|
||||
-- 8、数据同步菜单
|
||||
-- ----------------------------
|
||||
insert into sys_menu
|
||||
(menu_id, menu_name, parent_id, order_num, path, component, query_param, is_frame, is_cache, menu_type,
|
||||
visible, status, perms, icon, active_menu, ext, create_dept, create_by, create_time, update_by, update_time, remark)
|
||||
values
|
||||
(1901400000000000001, '数据同步', 0, 2, 'sync', null, null, 'N', 'Y', 'M', '0', '0', null, 'sync', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步目录'),
|
||||
(1901400000000000100, '连接管理', 1901400000000000001, 1, 'connection', 'sync/connection/index', null, 'N', 'Y', 'C', '0', '0', 'sync:connection:list', 'link', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步连接管理菜单'),
|
||||
(1901400000000000101, '同步计划', 1901400000000000001, 2, 'plan', 'sync/plan/index', null, 'N', 'Y', 'C', '0', '0', 'sync:plan:list', 'calendar', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步计划菜单'),
|
||||
(1901400000000000102, '同步任务', 1901400000000000001, 3, 'job', 'sync/job/index', null, 'N', 'Y', 'C', '0', '0', 'sync:job:list', 'job', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步任务菜单'),
|
||||
(1901400000000000103, '对象清单', 1901400000000000001, 4, 'object', 'sync/object/index', null, 'N', 'Y', 'C', '0', '0', 'sync:object:list', 'list', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步对象清单菜单'),
|
||||
(1901400000000001001, '连接查询', 1901400000000000100, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001002, '连接新增', 1901400000000000100, 2, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:add', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001003, '连接修改', 1901400000000000100, 3, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:edit', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001004, '连接删除', 1901400000000000100, 4, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:remove', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001006, '连接测试', 1901400000000000100, 5, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:test', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001007, '钉钉 Web 登录', 1901400000000000100, 6, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:auth', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001011, '计划查询', 1901400000000000101, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001012, '计划新增', 1901400000000000101, 2, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:add', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001013, '计划修改', 1901400000000000101, 3, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:edit', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001014, '计划删除', 1901400000000000101, 4, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:remove', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001016, '立即运行', 1901400000000000101, 5, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:run', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001021, '任务查询', 1901400000000000102, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001022, '任务删除', 1901400000000000102, 2, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:remove', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001024, '取消任务', 1901400000000000102, 3, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:cancel', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001025, '重试任务', 1901400000000000102, 4, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:retry', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''),
|
||||
(1901400000000001031, '对象查询', 1901400000000000103, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:object:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '');
|
||||
|
||||
-- ----------------------------
|
||||
-- 9、数据同步字典类型
|
||||
-- ----------------------------
|
||||
insert into sys_dict_type
|
||||
(dict_id, dict_name, dict_type, create_dept, create_by, create_time, update_by, update_time, remark)
|
||||
values
|
||||
(1901500000000000001, '同步连接角色', 'sync_connection_role', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步连接角色'),
|
||||
(1901500000000000002, '同步连接类型', 'sync_connection_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步连接类型'),
|
||||
(1901500000000000003, '同步模式', 'sync_mode', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '全量或增量同步模式'),
|
||||
(1901500000000000004, '同步调度类型', 'sync_schedule_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '手动或Cron调度'),
|
||||
(1901500000000000005, '同步冲突策略', 'sync_conflict_strategy', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '目标对象冲突处理策略'),
|
||||
(1901500000000000006, '同步删除策略', 'sync_delete_strategy', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端删除处理策略'),
|
||||
(1901500000000000007, '同步校验方式', 'sync_verify_mode', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '传输完整性校验方式'),
|
||||
(1901500000000000008, '同步触发类型', 'sync_trigger_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务触发类型'),
|
||||
(1901500000000000009, '同步任务状态', 'sync_job_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务运行状态'),
|
||||
(1901500000000000010, '同步明细状态', 'sync_job_item_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务明细状态'),
|
||||
(1901500000000000011, '同步对象类型', 'sync_object_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步对象类型'),
|
||||
(1901500000000000012, '同步动作类型', 'sync_action_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务明细动作类型'),
|
||||
(1901500000000000013, '同步对象状态', 'sync_object_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源对象同步状态'),
|
||||
(1901500000000000014, '同步分片状态', 'sync_transfer_part_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片上传状态'),
|
||||
(1901500000000000015, '同步检查点类型', 'sync_checkpoint_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '增量检查点类型');
|
||||
|
||||
-- ----------------------------
|
||||
-- 10、数据同步字典数据
|
||||
-- ----------------------------
|
||||
insert into sys_dict_data
|
||||
(dict_code, dict_sort, dict_label, dict_value, dict_type, css_class, list_class, is_default,
|
||||
create_dept, create_by, create_time, update_by, update_time, remark)
|
||||
values
|
||||
(1901600000000000001, 1, '源端', 'SOURCE', 'sync_connection_role', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据来源连接'),
|
||||
(1901600000000000002, 2, '目标端', 'TARGET', 'sync_connection_role', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据目标连接'),
|
||||
(1901600000000000003, 1, '钉钉', 'DINGTALK', 'sync_connection_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '钉钉企业网盘'),
|
||||
(1901600000000000004, 2, '标准S3', 'S3', 'sync_connection_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, 'S3标准兼容存储'),
|
||||
(1901600000000000005, 3, '阿里云OSS', 'ALIYUN_OSS', 'sync_connection_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '阿里云对象存储OSS'),
|
||||
(1901600000000000006, 1, '全量同步', 'FULL', 'sync_mode', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '每次扫描全部对象'),
|
||||
(1901600000000000007, 2, '增量同步', 'INCREMENTAL', 'sync_mode', '', 'success', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '完整枚举后按持久清单差异传输'),
|
||||
(1901600000000000008, 1, '手动执行', 'MANUAL', 'sync_schedule_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '仅手动触发'),
|
||||
(1901600000000000009, 2, 'Cron调度', 'CRON', 'sync_schedule_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按Cron表达式触发'),
|
||||
(1901600000000000010, 1, '覆盖', 'OVERWRITE', 'sync_conflict_strategy', '', 'warning', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '覆盖目标对象'),
|
||||
(1901600000000000011, 2, '跳过', 'SKIP', 'sync_conflict_strategy', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '保留目标对象并跳过'),
|
||||
(1901600000000000012, 3, '两者保留', 'KEEP_BOTH', 'sync_conflict_strategy', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '重命名后保留两个对象'),
|
||||
(1901600000000000013, 1, '保留目标', 'KEEP', 'sync_delete_strategy', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端删除时保留目标对象'),
|
||||
(1901600000000000014, 2, '仅标记', 'MARK', 'sync_delete_strategy', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '仅记录源端删除状态'),
|
||||
(1901600000000000015, 3, '删除目标', 'DELETE', 'sync_delete_strategy', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步删除目标对象'),
|
||||
(1901600000000000016, 1, '文件大小', 'SIZE', 'sync_verify_mode', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按对象大小校验'),
|
||||
(1901600000000000017, 2, 'ETag', 'ETAG', 'sync_verify_mode', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按ETag校验'),
|
||||
(1901600000000000018, 3, 'SHA-256', 'SHA256', 'sync_verify_mode', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按SHA-256摘要校验'),
|
||||
(1901600000000000019, 1, '手动触发', 'MANUAL', 'sync_trigger_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '用户手动触发任务'),
|
||||
(1901600000000000020, 2, '调度触发', 'SCHEDULE', 'sync_trigger_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '计划调度触发任务'),
|
||||
(1901600000000000021, 3, '重试触发', 'RETRY', 'sync_trigger_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '失败任务重试触发'),
|
||||
(1901600000000000022, 1, '待执行', 'PENDING', 'sync_job_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '等待执行'),
|
||||
(1901600000000000023, 2, '执行中', 'RUNNING', 'sync_job_status', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '正在执行'),
|
||||
(1901600000000000024, 3, '成功', 'SUCCESS', 'sync_job_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '全部成功'),
|
||||
(1901600000000000025, 4, '部分失败', 'PARTIAL_FAILED', 'sync_job_status', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '部分对象失败'),
|
||||
(1901600000000000026, 5, '失败', 'FAILED', 'sync_job_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '任务失败'),
|
||||
(1901600000000000027, 6, '已取消', 'CANCELED', 'sync_job_status', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '任务已取消'),
|
||||
(1901600000000000053, 7, '取消中', 'CANCELING', 'sync_job_status', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '正在终止本节点传输'),
|
||||
(1901600000000000028, 1, '待执行', 'PENDING', 'sync_job_item_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '等待执行'),
|
||||
(1901600000000000029, 2, '执行中', 'RUNNING', 'sync_job_item_status', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '正在执行'),
|
||||
(1901600000000000030, 3, '成功', 'SUCCESS', 'sync_job_item_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '对象同步成功'),
|
||||
(1901600000000000031, 4, '已跳过', 'SKIPPED', 'sync_job_item_status', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '对象已跳过'),
|
||||
(1901600000000000032, 5, '失败', 'FAILED', 'sync_job_item_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '对象同步失败'),
|
||||
(1901600000000000033, 1, '文件', 'FILE', 'sync_object_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '普通文件'),
|
||||
(1901600000000000034, 2, '目录', 'FOLDER', 'sync_object_type', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '目录对象'),
|
||||
(1901600000000000035, 3, '在线文档', 'ONLINE_DOCUMENT', 'sync_object_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '需导出的在线文档'),
|
||||
(1901600000000000036, 1, '新增上传', 'CREATE', 'sync_action_type', '', 'success', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '首次上传目标对象'),
|
||||
(1901600000000000037, 2, '更新上传', 'UPDATE', 'sync_action_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '更新或覆盖目标对象'),
|
||||
(1901600000000000038, 3, '跳过', 'SKIP', 'sync_action_type', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '跳过目标操作'),
|
||||
(1901600000000000051, 4, '删除', 'DELETE', 'sync_action_type', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '删除目标对象'),
|
||||
(1901600000000000039, 1, '已发现', 'DISCOVERED', 'sync_object_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '扫描已发现'),
|
||||
(1901600000000000040, 2, '待同步', 'PENDING', 'sync_object_status', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '等待同步'),
|
||||
(1901600000000000041, 3, '已同步', 'SYNCED', 'sync_object_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '最近同步成功'),
|
||||
(1901600000000000042, 4, '已跳过', 'SKIPPED', 'sync_object_status', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按策略跳过'),
|
||||
(1901600000000000043, 5, '失败', 'FAILED', 'sync_object_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '最近同步失败'),
|
||||
(1901600000000000044, 6, '源端已删除', 'DELETED', 'sync_object_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端对象已删除'),
|
||||
(1901600000000000045, 1, '待上传', 'PENDING', 'sync_transfer_part_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片等待上传'),
|
||||
(1901600000000000046, 2, '上传中', 'UPLOADING', 'sync_transfer_part_status', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片正在上传'),
|
||||
(1901600000000000047, 3, '成功', 'SUCCESS', 'sync_transfer_part_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片上传成功'),
|
||||
(1901600000000000048, 4, '失败', 'FAILED', 'sync_transfer_part_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片上传失败'),
|
||||
(1901600000000000049, 1, '分页游标', 'PAGE_CURSOR', 'sync_checkpoint_type', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端分页扫描游标'),
|
||||
(1901600000000000050, 2, '增量水位', 'INCREMENTAL_WATERMARK', 'sync_checkpoint_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端增量同步水位'),
|
||||
(1901600000000000052, 3, '运行水位', 'RUN_WATERMARK', 'sync_checkpoint_type', '', 'warning', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '当前版本任务运行水位');
|
||||
@ -82,7 +82,7 @@ create table sys_user (
|
||||
user_id bigint(20) not null comment '用户ID',
|
||||
dept_id bigint(20) default null comment '部门ID',
|
||||
user_name varchar(30) not null comment '用户账号',
|
||||
nick_name varchar(30) not null comment '用户昵称',
|
||||
nick_name varchar(64) not null comment '用户昵称',
|
||||
user_type varchar(10) default 'sys_user' comment '用户类型(sys_user系统用户)',
|
||||
email varchar(50) default '' comment '用户邮箱',
|
||||
phone_number varchar(11) default '' comment '手机号码',
|
||||
@ -931,4 +931,3 @@ INSERT INTO test_tree VALUES (1762200000000000010, 1762200000000000007, 17610000
|
||||
INSERT INTO test_tree VALUES (1762200000000000011, 1762200000000000007, 1761000000000000108, 1761100000000000003, '子节点77', 0, 1761000000000000103, sysdate(), 1761100000000000001, NULL, NULL, 0);
|
||||
INSERT INTO test_tree VALUES (1762200000000000012, 1762200000000000010, 1761000000000000108, 1761100000000000003, '子节点88', 0, 1761000000000000103, sysdate(), 1761100000000000001, NULL, NULL, 0);
|
||||
INSERT INTO test_tree VALUES (1762200000000000013, 1762200000000000010, 1761000000000000108, 1761100000000000003, '子节点99', 0, 1761000000000000103, sysdate(), 1761100000000000001, NULL, NULL, 0);
|
||||
|
||||
|
||||
1
ui/data-sync-s3-vue
Submodule
1
ui/data-sync-s3-vue
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit cfab6cebb66815bdf0e27ee8c1aa8b36593597e5
|
||||
Loading…
Reference in New Issue
Block a user