From 6a3b4ed4c3701e73a8c652d43e79fa10bf6745e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=A1=E9=92=B1?= Date: Thu, 3 Sep 2026 14:14:40 +0800 Subject: [PATCH 1/5] init --- .DS_Store | Bin 0 -> 10244 bytes .dockerignore | 9 + .../projects/8f418c8c0283cb0e672809522b013ebf | 1 + deploy/Dockerfile.backend | 29 + deploy/Dockerfile.web | 15 + deploy/docker-compose.prod.yml | 103 ++ deploy/nginx.conf | 55 + pom.xml | 7 + ruoyi-admin/pom.xml | 6 + .../web/controller/AuthController.java | 23 - .../src/main/resources/application.yml | 30 +- ruoyi-modules/pom.xml | 1 + ruoyi-modules/ruoyi-sync/README.md | 280 +++++ ruoyi-modules/ruoyi-sync/pom.xml | 59 ++ .../sync/config/SyncSchedulingConfig.java | 16 + .../sync/connector/ConnectorRegistry.java | 44 + .../sync/connector/SourceConnector.java | 17 + .../dromara/sync/connector/SyncConnector.java | 25 + .../sync/connector/TargetConnector.java | 31 + .../dingtalk/DingTalkSourceConnector.java | 737 +++++++++++++ .../connector/dingtalk/DwsAuthIdentity.java | 16 + .../dingtalk/DwsAuthSessionManager.java | 912 +++++++++++++++++ .../dingtalk/DwsAuthSessionState.java | 15 + .../connector/dingtalk/DwsCommandRunner.java | 459 +++++++++ .../connector/model/ConnectorTestResult.java | 18 + .../sync/connector/model/ScanRequest.java | 11 + .../sync/connector/model/ScanResult.java | 13 + .../sync/connector/model/SourceContent.java | 36 + .../sync/connector/model/SourceObject.java | 38 + .../connector/model/TargetWriteRequest.java | 22 + .../connector/model/TargetWriteResult.java | 23 + .../s3/AbstractS3TargetConnector.java | 151 +++ .../s3/AliyunOssTargetConnector.java | 19 + .../sync/connector/s3/S3TargetConnector.java | 16 + .../sync/connector/s3/SyncS3OssClient.java | 44 + .../dromara/sync/constant/SyncConstants.java | 35 + .../controller/SyncConnectionController.java | 185 ++++ .../sync/controller/SyncJobController.java | 83 ++ .../sync/controller/SyncObjectController.java | 38 + .../sync/controller/SyncPlanController.java | 121 +++ .../dromara/sync/domain/SyncCheckpoint.java | 34 + .../dromara/sync/domain/SyncConnection.java | 96 ++ .../java/org/dromara/sync/domain/SyncJob.java | 42 + .../org/dromara/sync/domain/SyncJobItem.java | 45 + .../org/dromara/sync/domain/SyncObject.java | 51 + .../org/dromara/sync/domain/SyncPlan.java | 132 +++ .../dromara/sync/domain/SyncTransferPart.java | 38 + .../sync/domain/bo/SyncConnectionBo.java | 105 ++ .../org/dromara/sync/domain/bo/SyncJobBo.java | 24 + .../dromara/sync/domain/bo/SyncJobItemBo.java | 22 + .../dromara/sync/domain/bo/SyncObjectBo.java | 23 + .../dromara/sync/domain/bo/SyncPlanBo.java | 165 +++ .../sync/domain/vo/DwsAuthSessionVo.java | 48 + .../sync/domain/vo/SyncCheckpointVo.java | 29 + .../sync/domain/vo/SyncConnectionVo.java | 106 ++ .../dromara/sync/domain/vo/SyncJobItemVo.java | 42 + .../org/dromara/sync/domain/vo/SyncJobVo.java | 40 + .../dromara/sync/domain/vo/SyncObjectVo.java | 47 + .../dromara/sync/domain/vo/SyncPlanVo.java | 144 +++ .../sync/domain/vo/SyncTransferPartVo.java | 35 + .../dromara/sync/job/SyncPlanJobExecutor.java | 31 + .../dromara/sync/job/SyncPlanScheduler.java | 75 ++ .../sync/mapper/SyncCheckpointMapper.java | 11 + .../sync/mapper/SyncConnectionMapper.java | 15 + .../sync/mapper/SyncJobItemMapper.java | 11 + .../dromara/sync/mapper/SyncJobMapper.java | 11 + .../dromara/sync/mapper/SyncObjectMapper.java | 11 + .../dromara/sync/mapper/SyncPlanMapper.java | 15 + .../sync/mapper/SyncTransferPartMapper.java | 11 + .../sync/service/ISyncConnectionService.java | 109 ++ .../dromara/sync/service/ISyncJobService.java | 30 + .../sync/service/ISyncObjectService.java | 16 + .../sync/service/ISyncPlanService.java | 77 ++ .../impl/SyncConnectionServiceImpl.java | 713 +++++++++++++ .../sync/service/impl/SyncJobServiceImpl.java | 199 ++++ .../service/impl/SyncObjectServiceImpl.java | 44 + .../service/impl/SyncPlanServiceImpl.java | 380 +++++++ .../sync/worker/SyncExecutionWorker.java | 966 ++++++++++++++++++ script/sql/ry_sync.sql | 359 +++++++ script/sql/ry_vue.sql | 3 +- ui/data-sync-s3-vue | 1 + 81 files changed, 8072 insertions(+), 27 deletions(-) create mode 100644 .DS_Store create mode 100644 .dockerignore create mode 120000 .pnpm-store/v10/projects/8f418c8c0283cb0e672809522b013ebf create mode 100644 deploy/Dockerfile.backend create mode 100644 deploy/Dockerfile.web create mode 100644 deploy/docker-compose.prod.yml create mode 100644 deploy/nginx.conf create mode 100644 ruoyi-modules/ruoyi-sync/README.md create mode 100644 ruoyi-modules/ruoyi-sync/pom.xml create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/config/SyncSchedulingConfig.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/ConnectorRegistry.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/SourceConnector.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/SyncConnector.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/TargetConnector.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DingTalkSourceConnector.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthIdentity.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthSessionManager.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthSessionState.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsCommandRunner.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ConnectorTestResult.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ScanRequest.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ScanResult.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/SourceContent.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/SourceObject.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/TargetWriteRequest.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/TargetWriteResult.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AbstractS3TargetConnector.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AliyunOssTargetConnector.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/S3TargetConnector.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/SyncS3OssClient.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/constant/SyncConstants.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncConnectionController.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncJobController.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncObjectController.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncPlanController.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncCheckpoint.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncConnection.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncJob.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncJobItem.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncObject.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncPlan.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncTransferPart.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncConnectionBo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncJobBo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncJobItemBo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncObjectBo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncPlanBo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/DwsAuthSessionVo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncCheckpointVo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncConnectionVo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncJobItemVo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncJobVo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncObjectVo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncPlanVo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncTransferPartVo.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/job/SyncPlanJobExecutor.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/job/SyncPlanScheduler.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncCheckpointMapper.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncConnectionMapper.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncJobItemMapper.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncJobMapper.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncObjectMapper.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncPlanMapper.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncTransferPartMapper.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncConnectionService.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncJobService.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncObjectService.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncPlanService.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncConnectionServiceImpl.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncJobServiceImpl.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncObjectServiceImpl.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncPlanServiceImpl.java create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/worker/SyncExecutionWorker.java create mode 100644 script/sql/ry_sync.sql create mode 160000 ui/data-sync-s3-vue diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..7f96dc19ccc1100a115f609d53c75988a715ffe4 GIT binary patch literal 10244 zcmeHM&u<$=6n^7|+Dj6$X%lEbNUNFyQcLQlRiz4{xQ-*RN=X!(kkA%)e@xs});p`+ zb=)+HFs2y_xsEnVoOv&3eWF zKx#ebDu4k1CCpsvGgypC+|KhzDkLpSC<)>LK7b~yH#}xr-iWtqPz)#r6a$I@#eibq z-@pKVvw12_OQ}l5fMP%~aKZrJ9}LV~RuegpQfwVqh$8^XDQp%Abvy?MjFHG{A_r0m zDE28&4?(zsPxOu!YZi3+iwOtxoGli$;&kzT^C9auV>9kWH8Z@+ zYc0k+E7eVF+ok=szOcsp*mQjw3V`cSbh-DD>pQGpV?E!A1hy#+npV=<`h}yTl^d1I z#}64Yo*2VmAALr`(D>)pW?hRkz{*?Sy?|RKSOQI%v}2*nHl*$ zo!PItp3JSJy*NEH`_h~~fA0MJ!u;am;>)jGSbFufrM7xwHB?M*3*(WsEv8Q2hGs;nBPGwN6uuT!4~n_#WEK~!1sdTn8@NNMtBSq_mFXW!Z2<& zGaetw!5H=s8)c)A@cm&Jc@`qvbwZ5r0OG8#*cndsosd|~%wI+61Vw4)&F12>fYFir zK9Lqi#P^utJjm?QKjEkfBLcT#*Henk|6f@S%TNFS literal 0 HcmV?d00001 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..0a325740d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.idea +.vscode +.pnpm-store +**/.DS_Store +**/node_modules +**/dist +**/target +**/*.log diff --git a/.pnpm-store/v10/projects/8f418c8c0283cb0e672809522b013ebf b/.pnpm-store/v10/projects/8f418c8c0283cb0e672809522b013ebf new file mode 120000 index 000000000..23e53fa04 --- /dev/null +++ b/.pnpm-store/v10/projects/8f418c8c0283cb0e672809522b013ebf @@ -0,0 +1 @@ +../../../ui/data-sync-s3-vue \ No newline at end of file diff --git a/deploy/Dockerfile.backend b/deploy/Dockerfile.backend new file mode 100644 index 000000000..7d635df6f --- /dev/null +++ b/deploy/Dockerfile.backend @@ -0,0 +1,29 @@ +FROM docker.m.daocloud.io/library/maven:3.9.11-eclipse-temurin-21 AS builder + +WORKDIR /build +COPY . . +RUN mvn -Pprod -DskipTests -pl ruoyi-admin -am clean package + +FROM docker.m.daocloud.io/library/eclipse-temurin:21-jre-jammy + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + && curl -fsSL --connect-timeout 10 --max-time 30 https://ghfast.top/https://raw.githubusercontent.com/DingTalk-Real-AI/dingtalk-workspace-cli/main/scripts/install.sh -o /tmp/install-dws.sh \ + && env DWS_GITEE_REPO=DingTalk-Real-AI/dingtalk-workspace-cli DWS_VERSION=v1.0.60 DWS_NO_SKILLS=1 DWS_INSTALL_DIR=/usr/local/bin sh /tmp/install-dws.sh \ + && dws version \ + && rm -rf /var/lib/apt/lists/* /tmp/install-dws.sh /root/.cache + +WORKDIR /app +RUN mkdir -p /app/logs /app/temp /var/lib/ruoyi-sync/dws +COPY --from=builder /build/ruoyi-admin/target/ruoyi-admin.jar /app/app.jar + +ENV TZ=Asia/Shanghai \ + JAVA_OPTS="-XX:+UseZGC -XX:+HeapDumpOnOutOfMemoryError" \ + DWS_EXECUTABLE=/usr/local/bin/dws \ + DWS_PROFILE_ROOT=/var/lib/ruoyi-sync/dws + +EXPOSE 8080 +HEALTHCHECK --interval=15s --timeout=5s --start-period=60s --retries=10 \ + CMD curl -fsS http://127.0.0.1:8080/ >/dev/null || exit 1 + +ENTRYPOINT ["sh", "-c", "exec java $JAVA_OPTS -jar /app/app.jar"] diff --git a/deploy/Dockerfile.web b/deploy/Dockerfile.web new file mode 100644 index 000000000..02fb73c7a --- /dev/null +++ b/deploy/Dockerfile.web @@ -0,0 +1,15 @@ +FROM docker.m.daocloud.io/library/node:22-bookworm-slim AS builder + +WORKDIR /build +RUN corepack enable && corepack prepare pnpm@10.34.5 --activate +COPY ui/data-sync-s3-vue/package.json ui/data-sync-s3-vue/pnpm-lock.yaml ui/data-sync-s3-vue/pnpm-workspace.yaml ./ +RUN pnpm install --frozen-lockfile +COPY ui/data-sync-s3-vue/ ./ +RUN pnpm build:prod + +FROM docker.m.daocloud.io/library/nginx:1.31.1-alpine +COPY deploy/nginx.conf /etc/nginx/nginx.conf +COPY --from=builder /build/dist/ /usr/share/nginx/html/ +EXPOSE 80 +HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=5 \ + CMD wget -qO- http://127.0.0.1/ >/dev/null || exit 1 diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml new file mode 100644 index 000000000..4f8fb7125 --- /dev/null +++ b/deploy/docker-compose.prod.yml @@ -0,0 +1,103 @@ +name: data-sync-s3 + +services: + mysql: + image: docker.m.daocloud.io/library/mysql:8.4.9 + restart: unless-stopped + environment: + TZ: Asia/Shanghai + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD} + MYSQL_DATABASE: ${MYSQL_DATABASE:-ry-vue} + command: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_general_ci + - --explicit_defaults_for_timestamp=true + - --lower_case_table_names=1 + volumes: + - mysql-data:/var/lib/mysql + - ../script/sql/ry_vue.sql:/docker-entrypoint-initdb.d/01-ry-vue.sql:ro + - ../script/sql/ry_job.sql:/docker-entrypoint-initdb.d/02-ry-job.sql:ro + - ../script/sql/ry_workflow.sql:/docker-entrypoint-initdb.d/03-ry-workflow.sql:ro + - ../script/sql/ry_ai.sql:/docker-entrypoint-initdb.d/04-ry-ai.sql:ro + - ../script/sql/ry_sync.sql:/docker-entrypoint-initdb.d/05-ry-sync.sql:ro + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -p$$MYSQL_ROOT_PASSWORD --silent"] + interval: 10s + timeout: 5s + retries: 15 + start_period: 30s + networks: [app] + + redis: + image: docker.m.daocloud.io/library/redis:8.2.1-alpine + restart: unless-stopped + command: ["sh", "-c", "exec redis-server --appendonly yes --requirepass \"$$REDIS_PASSWORD\""] + environment: + TZ: Asia/Shanghai + REDIS_PASSWORD: ${REDIS_PASSWORD} + volumes: + - redis-data:/data + healthcheck: + test: ["CMD-SHELL", "redis-cli -a \"$$REDIS_PASSWORD\" ping | grep PONG"] + interval: 10s + timeout: 5s + retries: 10 + networks: [app] + + backend: + build: + context: .. + dockerfile: deploy/Dockerfile.backend + image: data-sync-s3/backend:latest + restart: unless-stopped + environment: + TZ: Asia/Shanghai + SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_URL: jdbc:mysql://mysql:3306/${MYSQL_DATABASE:-ry-vue}?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=Asia/Shanghai&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true + SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_USERNAME: root + SPRING_DATASOURCE_DYNAMIC_DATASOURCE_MASTER_PASSWORD: ${MYSQL_ROOT_PASSWORD} + SPRING_DATA_REDIS_HOST: redis + SPRING_DATA_REDIS_PORT: 6379 + SPRING_DATA_REDIS_PASSWORD: ${REDIS_PASSWORD} + MYBATIS_ENCRYPTOR_ENABLE: "true" + MYBATIS_ENCRYPTOR_PASSWORD: ${MYBATIS_ENCRYPTOR_PASSWORD} + DWS_EXECUTABLE: /usr/local/bin/dws + DWS_PROFILE_ROOT: /var/lib/ruoyi-sync/dws + DWS_KEYCHAIN_ROOT: /var/lib/ruoyi-sync/dws-keychain + SYNC_SCHEDULER_ENABLED: "true" + JAVA_OPTS: -Xms256m -Xmx1024m -XX:+UseZGC -XX:+HeapDumpOnOutOfMemoryError + volumes: + - backend-logs:/app/logs + - dws-profiles:/var/lib/ruoyi-sync/dws + - dws-keychains:/var/lib/ruoyi-sync/dws-keychain + depends_on: + mysql: + condition: service_healthy + redis: + condition: service_healthy + networks: [app] + + web: + build: + context: .. + dockerfile: deploy/Dockerfile.web + image: data-sync-s3/web:latest + restart: unless-stopped + ports: + # The host already runs another Nginx on port 80. Expose this stack on + # 8080 and let the existing reverse proxy forward traffic here. + - "8080:80" + depends_on: + backend: + condition: service_healthy + networks: [app] + +networks: + app: + driver: bridge + +volumes: + mysql-data: + redis-data: + backend-logs: + dws-profiles: + dws-keychains: diff --git a/deploy/nginx.conf b/deploy/nginx.conf new file mode 100644 index 000000000..2e7bc316e --- /dev/null +++ b/deploy/nginx.conf @@ -0,0 +1,55 @@ +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 2048; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + client_max_body_size 100m; + gzip_static on; + server_tokens off; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + upstream backend { + server backend:8080; + } + + server { + listen 80; + server_name _; + + location / { + root /usr/share/nginx/html; + try_files $uri $uri/ /index.html; + index index.html; + } + + location /prod-api/ { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_cache off; + proxy_pass http://backend/; + } + + location ~ ^(/[^/]*)?/actuator.*(/.*)?$ { + return 403; + } + } +} diff --git a/pom.xml b/pom.xml index 8494ba12b..ec2c6add6 100644 --- a/pom.xml +++ b/pom.xml @@ -429,6 +429,13 @@ ${revision} + + + org.dromara + ruoyi-sync + ${revision} + + org.dromara diff --git a/ruoyi-admin/pom.xml b/ruoyi-admin/pom.xml index 8f2d51cdf..2dce98035 100644 --- a/ruoyi-admin/pom.xml +++ b/ruoyi-admin/pom.xml @@ -92,6 +92,12 @@ ruoyi-ai + + + org.dromara + ruoyi-sync + + org.dromara diff --git a/ruoyi-admin/src/main/java/org/dromara/web/controller/AuthController.java b/ruoyi-admin/src/main/java/org/dromara/web/controller/AuthController.java index 3a98ec153..a711a570d 100644 --- a/ruoyi-admin/src/main/java/org/dromara/web/controller/AuthController.java +++ b/ruoyi-admin/src/main/java/org/dromara/web/controller/AuthController.java @@ -12,9 +12,6 @@ import me.zhyd.oauth.utils.AuthStateUtils; import org.dromara.common.core.constant.SystemConstants; import org.dromara.common.core.domain.R; import org.dromara.common.core.domain.model.LoginBody; -import org.dromara.common.core.enums.PushSourceEnum; -import org.dromara.common.core.enums.PushTypeEnum; -import org.dromara.common.core.utils.DateUtils; import org.dromara.common.core.utils.MessageUtils; import org.dromara.common.core.utils.StringUtils; import org.dromara.common.core.utils.ValidatorUtils; @@ -24,8 +21,6 @@ import org.dromara.common.satoken.utils.LoginHelper; import org.dromara.common.social.config.properties.SocialLoginConfigProperties; import org.dromara.common.social.config.properties.SocialProperties; import org.dromara.common.social.utils.SocialUtils; -import org.dromara.system.api.MessageService; -import org.dromara.system.api.domain.PushPayloadDTO; import org.dromara.system.api.model.RegisterBody; import org.dromara.system.api.model.SocialLoginBody; import org.dromara.system.domain.vo.SysClientVo; @@ -39,10 +34,6 @@ import org.dromara.web.service.SysRegisterService; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; -import java.util.Date; -import java.util.List; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; /** * 认证控制器，提供登录、注册、社交绑定和退出能力。 @@ -62,8 +53,6 @@ public class AuthController { private final ISysConfigService configService; private final ISysSocialService socialUserService; private final ISysClientService clientService; - private final ScheduledExecutorService scheduledExecutorService; - private final MessageService messageService; /** @@ -91,18 +80,6 @@ public class AuthController { // 登录 LoginVo loginVo = IAuthStrategy.login(body, client, grantType); - Long userId = LoginHelper.getUserId(); - scheduledExecutorService.schedule(() -> { - messageService.publishMessage( - List.of(userId), - PushPayloadDTO.of( - PushTypeEnum.MESSAGE, - PushSourceEnum.BACKEND, - DateUtils.getTodayHour(new Date()) + "好，欢迎登录 RuoYi-Vue-Plus 后台管理系统", - null - ) - ); - }, 5, TimeUnit.SECONDS); return R.ok(loginVo); } diff --git a/ruoyi-admin/src/main/resources/application.yml b/ruoyi-admin/src/main/resources/application.yml index ec1e1d1a9..052e058fd 100644 --- a/ruoyi-admin/src/main/resources/application.yml +++ b/ruoyi-admin/src/main/resources/application.yml @@ -42,6 +42,32 @@ user: # 密码锁定时间（默认10分钟） lockTime: 10 +# 数据同步配置 +sync: + dingtalk: + # DingTalk Workspace CLI 可执行文件；生产环境可通过环境变量指定绝对路径 + dws-executable: ${DWS_EXECUTABLE:dws} + # 每个连接使用独立 DWS 登录态目录，实际目录为该路径下的连接 ID + profile-root: ${DWS_PROFILE_ROOT:${user.home}/.ruoyi-sync/dws} + # DWS 加密 keychain 也按连接隔离；生产环境必须和 profile 一起持久化 + keychain-root: ${DWS_KEYCHAIN_ROOT:${user.home}/.ruoyi-sync/dws-keychain} + auth: + # 是否允许通过管理后台发起钉钉设备授权 + enabled: ${DWS_WEB_AUTH_ENABLED:true} + # 设备流最长等待时间（DWS 服务端默认约 10 分钟） + timeout-seconds: ${DWS_WEB_AUTH_TIMEOUT_SECONDS:600} + # 启动接口等待 DWS 打印授权链接的最长时间；超时后前端继续轮询 STARTING + metadata-wait-seconds: ${DWS_WEB_AUTH_METADATA_WAIT_SECONDS:15} + # 单节点同时进行的设备授权数量上限 + max-concurrent: ${DWS_WEB_AUTH_MAX_CONCURRENT:4} + # 终态会话保留时间，便于前端读取最终结果 + retention-seconds: ${DWS_WEB_AUTH_RETENTION_SECONDS:900} + scheduler: + # 使用 SnailJob 等外部调度时应关闭内置调度器，避免重复触发 + enabled: ${SYNC_SCHEDULER_ENABLED:true} + # Cron 同步计划扫描间隔（毫秒） + poll-interval-ms: ${SYNC_SCHEDULER_POLL_INTERVAL_MS:30000} + # Spring配置 spring: application: @@ -133,13 +159,13 @@ mybatis-plus: # 数据加密 mybatis-encryptor: # 是否开启加密 - enable: false + enable: ${MYBATIS_ENCRYPTOR_ENABLE:false} # 默认加密算法 algorithm: BASE64 # 编码方式 BASE64/HEX。默认BASE64 encode: BASE64 # 安全秘钥 对称算法的秘钥 如：AES，SM4 - password: + password: ${MYBATIS_ENCRYPTOR_PASSWORD:} # 公私钥 非对称算法的公私钥 如：SM2，RSA publicKey: privateKey: diff --git a/ruoyi-modules/pom.xml b/ruoyi-modules/pom.xml index 4a79d2500..d2831a799 100644 --- a/ruoyi-modules/pom.xml +++ b/ruoyi-modules/pom.xml @@ -22,6 +22,7 @@ ruoyi-system ruoyi-workflow ruoyi-ai + ruoyi-sync diff --git a/ruoyi-modules/ruoyi-sync/README.md b/ruoyi-modules/ruoyi-sync/README.md new file mode 100644 index 000000000..bae21bc42 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/README.md @@ -0,0 +1,280 @@ +# 数据同步后端（V1） + +本模块把原来的“钉钉盘备份到 OSS”收敛为可扩展的数据同步内核。V1 支持： + +- 源端：钉钉企业网盘 / 我的文件（DingTalk Workspace CLI，简称 DWS） +- 目标端：标准 S3 兼容存储、阿里云 OSS（复用项目原生 S3 客户端） +- 执行：手动运行、内置 Cron 调度、可选 SnailJob 入口 +- 能力：全量或清单差异同步、在线文档导出、冲突策略、源端删除策略、连接测试、任务明细、对象清单和检查点 + +## 1. 架构与扩展点 + +```text +DingTalkSourceConnector + │ ScanResult / SourceContent + ▼ +SyncExecutionWorker ── sync_job / sync_job_item / sync_object / sync_checkpoint + │ TargetWriteRequest + ├──────────────► S3TargetConnector + └──────────────► AliyunOssTargetConnector +``` + +源、目标均通过接口隔离： + +- `SourceConnector`：分页扫描、下载或导出源对象 +- `TargetConnector`：检查对象、上传、删除目标对象 +- `ConnectorRegistry`：按连接角色和类型选择实现 + +后续增加其他源或目标时，不需要修改计划、任务和对象清单的数据模型。 + +## 2. 数据库 + +先导入项目基础 SQL，再导入 [`script/sql/ry_sync.sql`](../../script/sql/ry_sync.sql)。该脚本面向 MySQL，包含菜单、权限和字典数据。 + +| 表 | 用途 | +|---|---| +| `sync_connection` | 源/目标连接及加密凭证 | +| `sync_plan` | 路径、策略、并发、Cron 和删除保护配置 | +| `sync_job` | 一次运行及汇总；数据库唯一键保证同一计划只有一个活动任务 | +| `sync_job_item` | 每个文件的上传、跳过或删除结果 | +| `sync_object` | 源对象与目标对象的持久清单，用于差异判断和删除发现 | +| `sync_checkpoint` | 已完成任务水位；V1 不把未完成分页游标当作可提交检查点 | +| `sync_transfer_part` | 应用级分片续传的预留表，V1 尚未写入 | + +`delete_guard_percent` 默认是 50。若一次运行拟删除的目标文件比例超过阈值，任务会失败且不会执行任何删除；只有显式设为 100 才允许一次删除全部目标文件。 + +## 3. 必需的服务配置 + +生产环境必须启用字段加密，否则服务会拒绝保存 `secretJson`： + +```bash +MYBATIS_ENCRYPTOR_ENABLE=true +MYBATIS_ENCRYPTOR_PASSWORD=<由密钥管理系统注入的 AES 密钥> +DWS_EXECUTABLE=/opt/dws/bin/dws +DWS_PROFILE_ROOT=/var/lib/ruoyi-sync/dws +DWS_KEYCHAIN_ROOT=/var/lib/ruoyi-sync/dws-keychain +``` + +请不要把真实密钥写进 YAML、启动脚本或 Git。`SyncConnectionVo` 不含 `secretJson`，新增/修改接口的操作日志也排除了该字段。 + +后端运行时需要 JDK 21（项目使用虚拟线程）。DWS 建议固定到已验收版本；仓库提供的生产镜像默认使用 `v1.0.60`，升级 DWS 后应重新做一次设备登录和真实空间访问验收。 + +内置调度器默认开启。如改由 SnailJob 触发，应设置： + +```bash +SYNC_SCHEDULER_ENABLED=false +``` + +SnailJob 执行器名为 `syncPlanJobExecutor`，任务参数是同步计划 ID。内置调度和 SnailJob 二选一。 + +## 4. 钉钉连接 + +### 4.1 DWS 运行契约 + +运行节点需要安装与验收 DWS，并至少确认以下命令存在： + +```bash +dws drive list --help +dws drive download --help +dws doc export --help +dws wiki space list --help +``` + +本模块使用原子 `drive list` 保留版本、修改时间、扩展名等增量证据；普通文件使用原子 `drive download` 的分片、并发和断点能力；`adoc` 在线文档使用 `doc export`（自动提交、轮询并下载）导出为 `docx`。 + +每个同步连接都有独立的 DWS 配置目录： + +```text +${DWS_PROFILE_ROOT}/${connectionId} +``` + +登录 token 的加密 keychain 也按连接隔离，目录为： + +```text +${DWS_KEYCHAIN_ROOT}/${connectionId}/dws-cli +``` + +所有业务命令还会显式传入该连接配置中的 `profile=corpId:userId`，不会依赖服务进程当前选中的账号。 + +### 4.2 登录态初始化 + +应用密钥不能代替用户 OAuth 登录态。现在首次登录可以完全从管理后台完成，不需要 SSH 到服务器： + +1. 创建一个 `DINGTALK/SOURCE` 连接。`secretJson` 填写 AppKey/AppSecret；`configJson` 可以暂时不填 `profile`。 +2. 网页调用 `POST /sync/connection/{connectionId}/auth/login/start`。 +3. 页面展示返回的 `verificationUriComplete`（可生成二维码），同时展示 `verificationUri` 和 `userCode` 作为备用方式。 +4. 用户在自己的浏览器中打开链接并在钉钉完成确认；页面每 2 秒左右调用状态接口，直到 `SUCCESS`、`FAILED`、`EXPIRED` 或 `CANCELLED`。 +5. 授权成功后，服务器自动把 DWS 返回的稳定 `corpId:userId` 写入该连接的 `configJson.profile`，之后即可执行连接测试和同步。 + +启动接口返回的对象只含一次性授权链接、验证码和脱敏身份信息，绝不返回 access token、refresh token、device code 或 AppSecret。DWS 进程和加密登录态始终留在后端节点；浏览器只负责完成钉钉授权。 + +后端只接受 DWS 最终 JSON 中明确返回的 `corp_id`/`corpId` 与 `user_id`/`userId`（或明确的 `profile` 字段），缺少任一 ID 会安全失败，不会根据普通日志文本猜测登录人。连接凭证属于创建者，普通用户只能查看和操作自己创建的连接，超级管理员可跨创建者管理；登录操作也仅允许连接创建人或超级管理员发起、查询和取消。登录进行中禁止修改或删除该连接，Profile 只能由 Web 授权回调变更，编辑旧表单不会清掉已完成的登录态。 + +接口契约： + +| 方法 | 地址 | 说明 | +|---|---|---| +| `POST` | `/sync/connection/{id}/auth/login/start` | 启动设备流；可选查询参数 `expectedCorpId` 用于组织校验 | +| `GET` | `/sync/connection/{id}/auth/login/{sessionId}` | 查询会话状态和展示信息 | +| `POST` | `/sync/connection/{id}/auth/login/{sessionId}/cancel` | 取消会话并终止服务器上的 DWS 进程 | + +DWS 设备流有服务端有效期（通常约 10 分钟），过期后重新点击“登录”即可。DWS 会自动刷新已保存的登录态，日常同步不需要重复登录。组织必须允许 DWS/CLI 数据访问，且授权用户必须拥有目标空间的访问权限；否则会话会返回失败提示。 + +会话状态接口返回 `STARTING`、`WAITING_USER`、`FINALIZING`、`SUCCESS`、`FAILED`、`EXPIRED` 或 `CANCELLED`；进入 `FINALIZING` 后正在写回连接身份，页面应继续轮询并禁止重复发起登录。 + +生产部署必须持久化下面两个目录，否则重启后会丢失 DWS 登录态： + +```text +${DWS_PROFILE_ROOT} +${DWS_KEYCHAIN_ROOT} +``` + +当前会话管理器保存在单节点内存中，因此 V1 要求登录页面的三次请求落到同一个后端实例（单实例部署天然满足；多实例请配置会话亲和或单独的认证节点）。后续需要无亲和的集群部署时，再把会话状态和进程协调迁移到 Redis/专用认证服务。 + +同一节点内，Web 授权会独占该连接的 DWS 配置/keychain 锁；正在执行的扫描或下载会让登录请求稍后重试，登录期间新发起的 DWS 业务命令也会被拒绝。若同步 worker 和 Web/API 分布在多个后端节点，除会话亲和外还需要把这把连接锁迁移到 Redis 或数据库分布式锁。 + +如需运维兜底，仍可在服务器上执行等价的设备流命令；这不是正常使用路径： + +```bash +DWS_CONFIG_DIR=/var/lib/ruoyi-sync/dws/ \ +DWS_KEYCHAIN_DIR=/var/lib/ruoyi-sync/dws-keychain/ \ +DWS_CLIENT_ID=<与该连接 secretJson 一致的 AppKey> \ +DWS_CLIENT_SECRET=<与该连接 secretJson 一致的 AppSecret> \ +dws auth login --device --no-browser +``` + +### 4.3 连接示例 + +请求中的 JSON 字符串为展示方便做了格式化；实际字段仍是字符串。 + +```json +{ + "connectionName": "钉钉企业盘", + "connectionRole": "SOURCE", + "connectionType": "DINGTALK", + "configJson": "{\"profile\":\"corpId:userId\",\"spaceType\":\"orgSpace\",\"spaceId\":\"可选\",\"downloadPartSize\":\"32MB\",\"downloadParallel\":\"4\"}", + "secretJson": "{\"clientId\":\"AppKey\",\"clientSecret\":\"AppSecret\"}", + "status": "0" +} +``` + +- `spaceId` 可选：填写时只同步该空间；不填时按 `spaceType` 发现空间。 +- `spaceType`：`orgSpace` 或 `mySpace`，默认 `orgSpace`。 +- `sourceRoot`：`/` 表示空间根；指定子目录时必须填写该目录的 `dentryUuid`，且连接必须指定 `spaceId`。 +- `downloadParallel`：1 至 8，默认 4。 +- `downloadPartSize`：DWS 支持的容量字符串，例如 `32MB`。 +- `configDir` 不允许由连接指定，避免一个连接读取另一个连接的登录态。 + +V1 只导出 `adoc -> docx`。`axls`、`able`、`appt`、`amind`、`adraw` 会被识别为在线对象并明确失败，不会错误地按普通文件下载；相应导出适配器留到后续版本。 + +## 5. S3 / 阿里云 OSS 连接 + +标准 S3 示例： + +```json +{ + "connectionName": "备份 S3", + "connectionRole": "TARGET", + "connectionType": "S3", + "endpoint": "s3.example.com", + "region": "us-east-1", + "bucketName": "company-backup", + "basePath": "dingtalk", + "configJson": "{\"useHttps\":true,\"pathStyleAccess\":true}", + "secretJson": "{\"accessKey\":\"...\",\"secretKey\":\"...\"}", + "status": "0" +} +``` + +阿里云 OSS 示例： + +```json +{ + "connectionName": "阿里云 OSS", + "connectionRole": "TARGET", + "connectionType": "ALIYUN_OSS", + "endpoint": "oss-cn-hangzhou.aliyuncs.com", + "region": "cn-hangzhou", + "bucketName": "company-backup", + "basePath": "dingtalk", + "configJson": "{\"useHttps\":true,\"pathStyleAccess\":false}", + "secretJson": "{\"accessKey\":\"...\",\"secretKey\":\"...\"}", + "status": "0" +} +``` + +`region` 为可选配置。留空时与 ruoyi-vue-plus 原生 OSS 配置保持一致，客户端使用 `us-east-1` 作为默认 Region；如已知存储桶所在地域，仍建议填写实际值（例如 `cn-hangzhou`）。 + +连接测试会真实执行 `HeadBucket`。上传完成后再执行 `HeadObject`，以远端实际大小、ETag、版本 ID 和元数据作为任务结果。 + +## 6. 同步语义 + +计划请求示例： + +```json +{ + "planName": "钉钉企业盘每日备份", + "sourceConnectionId": 1, + "targetConnectionId": 2, + "sourceRoot": "/", + "targetPrefix": "daily", + "syncMode": "INCREMENTAL", + "scheduleType": "CRON", + "cronExpression": "0 0 2 * * *", + "conflictStrategy": "OVERWRITE", + "deleteStrategy": "KEEP", + "deleteGuardPercent": 50, + "verifyMode": "SHA256", + "maxConcurrency": 4, + "bandwidthLimitKbps": 0, + "status": "0" +} +``` + +- `FULL`：扫描并重新处理全部文件。 +- `INCREMENTAL`：仍完整枚举权威目录树，然后用 `versionToken -> hash -> modifiedTime + size` 依次判断变化；证据不足时保守地重新传输，不会只凭相同大小跳过。 +- `OVERWRITE`：写入目标键。 +- `SKIP`：目标键已存在时跳过。 +- `KEEP_BOTH`：目标键已存在时附加稳定版本后缀。 +- `KEEP` / `MARK`：源端消失时保留目标，仅更新清单和任务记录。 +- `DELETE`：通过删除比例保护后删除目标；删除失败会保留待重试状态，不会误记为已完成。 +- `SIZE`：以 `HeadObject.contentLength` 校验。 +- `ETAG`：要求目标端回读到 ETag，不能把不同厂商/不同分片策略的 ETag 当作源端内容哈希。 +- `SHA256`：本地计算摘要、写入对象元数据并通过 `HeadObject` 回读校验。 + +目录页缺少集合、条目缺少关键字段、部分错误、重复游标、重复对象 ID 或目录环路都会使任务失败。失败任务不会进入源删除发现阶段。 + +## 7. 后端接口 + +| 方法 | 地址 | 权限 | 用途 | +|---|---|---|---| +| `GET` | `/sync/connection/list` | `sync:connection:list` | 连接分页 | +| `GET` | `/sync/connection/{id}` | `sync:connection:query` | 连接详情（不返回凭证） | +| `POST` | `/sync/connection` | `sync:connection:add` | 新增连接 | +| `PUT` | `/sync/connection` | `sync:connection:edit` | 修改连接；空 `secretJson` 保留原凭证 | +| `DELETE` | `/sync/connection/{ids}` | `sync:connection:remove` | 删除未被计划引用的连接 | +| `POST` | `/sync/connection/test/{id}` | `sync:connection:test` | 真实连通性测试 | +| `POST` | `/sync/connection/{id}/auth/login/start` | `sync:connection:auth` | Web 端启动钉钉设备授权 | +| `GET` | `/sync/connection/{id}/auth/login/{sessionId}` | `sync:connection:auth` | 查询 Web 授权状态 | +| `POST` | `/sync/connection/{id}/auth/login/{sessionId}/cancel` | `sync:connection:auth` | 取消 Web 授权 | +| `GET` | `/sync/plan/list` | `sync:plan:list` | 计划分页 | +| `GET` | `/sync/plan/{id}` | `sync:plan:query` | 计划详情 | +| `POST` / `PUT` / `DELETE` | `/sync/plan` | 对应 add/edit/remove | 计划维护 | +| `POST` | `/sync/job/run/{planId}` | `sync:job:run` | 立即运行 | +| `POST` | `/sync/job/retry/{jobId}` | `sync:job:retry` | 重新枚举并重试失败对象 | +| `POST` | `/sync/job/cancel/{jobId}` | `sync:job:cancel` | 取消并中断本节点任务 | +| `GET` | `/sync/job/list` | `sync:job:list` | 任务分页 | +| `GET` | `/sync/job/{id}` | `sync:job:query` | 任务汇总 | +| `GET` | `/sync/job/{id}/items` | `sync:job:query` | 文件明细 | +| `DELETE` | `/sync/job/{ids}` | `sync:job:remove` | 删除非活动任务及明细 | +| `GET` | `/sync/object/list` | `sync:object:list` | 对象清单分页 | +| `GET` | `/sync/object/{id}` | `sync:object:query` | 对象详情 | + +## 8. 当前版本边界 + +- Web 登录会话是单节点内存态；多实例部署需要网关会话亲和或专用认证节点。不会把 token 或 device code 写入数据库或返回给浏览器。 +- `sync_transfer_part` 为应用级、跨任务分片续传预留。当前 DWS 下载和项目原生 S3 客户端各自使用其内部大文件机制，但应用数据库尚不接管 uploadId/part 状态。 +- 带宽字段已预留，但 V1 会拒绝非 0 值，避免界面显示“已限速”而运行时未生效。 +- 应用进程异常退出可能遗留 `PENDING/RUNNING/CANCELING` 任务。为避免多实例误接管仍在其他节点运行的任务，V1 不做激进的启动自动恢复；运维确认原节点已停止后，需要由 DBA 将遗留任务核销为 `CANCELED`，再点重试。 +- 取消会中断本节点 DWS 命令并在上传、校验、删除前复查数据库状态；对象存储 SDK 已经提交到远端的请求只能尽力取消，取消后重试仍按幂等对象键收敛。 +- 未接真实钉钉账号、S3 或 OSS 凭证时只能完成编译和静态契约验证；上线前必须做真实空间枚举、空文件、大文件、在线文档、权限变化、分页和删除保护验收。 diff --git a/ruoyi-modules/ruoyi-sync/pom.xml b/ruoyi-modules/ruoyi-sync/pom.xml new file mode 100644 index 000000000..e8fc12f59 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/pom.xml @@ -0,0 +1,59 @@ + + + + org.dromara + ruoyi-modules + ${revision} + + 4.0.0 + + ruoyi-sync + + 可扩展数据同步与备份模块 + + + + org.dromara + ruoyi-common-core + + + org.dromara + ruoyi-common-json + + + org.dromara + ruoyi-common-redis + + + org.dromara + ruoyi-common-mybatis + + + org.dromara + ruoyi-common-oss + + + org.dromara + ruoyi-common-log + + + org.dromara + ruoyi-common-security + + + org.dromara + ruoyi-common-web + + + org.dromara + ruoyi-common-encrypt + + + org.dromara + ruoyi-common-job + + + + diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/config/SyncSchedulingConfig.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/config/SyncSchedulingConfig.java new file mode 100644 index 000000000..663227343 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/config/SyncSchedulingConfig.java @@ -0,0 +1,16 @@ +package org.dromara.sync.config; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; + +/** + * 数据同步内置调度配置。 + * + *

该配置不依赖 SnailJob，确保仅启用内置调度器时 {@code @Scheduled} 仍然生效。

+ */ +@Configuration(proxyBeanMethods = false) +@EnableScheduling +@ConditionalOnProperty(prefix = "sync.scheduler", name = "enabled", havingValue = "true", matchIfMissing = true) +public class SyncSchedulingConfig { +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/ConnectorRegistry.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/ConnectorRegistry.java new file mode 100644 index 000000000..37d232d54 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/ConnectorRegistry.java @@ -0,0 +1,44 @@ +package org.dromara.sync.connector; + +import lombok.RequiredArgsConstructor; +import org.dromara.common.core.exception.ServiceException; +import org.dromara.sync.constant.SyncConstants; +import org.dromara.sync.domain.SyncConnection; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 按连接角色和类型解析连接器。 + */ +@Component +@RequiredArgsConstructor +public class ConnectorRegistry { + + private final List sourceConnectors; + private final List targetConnectors; + + public SyncConnector resolve(SyncConnection connection) { + if (SyncConstants.ROLE_SOURCE.equals(connection.getConnectionRole())) { + return source(connection.getConnectionType()); + } + if (SyncConstants.ROLE_TARGET.equals(connection.getConnectionRole())) { + return target(connection.getConnectionType()); + } + throw new ServiceException("不支持的连接角色：{}", connection.getConnectionRole()); + } + + public SourceConnector source(String type) { + return sourceConnectors.stream() + .filter(connector -> connector.type().equals(type)) + .findFirst() + .orElseThrow(() -> new ServiceException("未找到源端连接器：{}", type)); + } + + public TargetConnector target(String type) { + return targetConnectors.stream() + .filter(connector -> connector.type().equals(type)) + .findFirst() + .orElseThrow(() -> new ServiceException("未找到目标端连接器：{}", type)); + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/SourceConnector.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/SourceConnector.java new file mode 100644 index 000000000..1070d0503 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/SourceConnector.java @@ -0,0 +1,17 @@ +package org.dromara.sync.connector; + +import org.dromara.sync.connector.model.ScanRequest; +import org.dromara.sync.connector.model.ScanResult; +import org.dromara.sync.connector.model.SourceContent; +import org.dromara.sync.connector.model.SourceObject; +import org.dromara.sync.domain.SyncConnection; + +/** + * 源端连接器契约。 + */ +public interface SourceConnector extends SyncConnector { + + ScanResult scan(SyncConnection connection, ScanRequest request); + + SourceContent download(SyncConnection connection, SourceObject object); +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/SyncConnector.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/SyncConnector.java new file mode 100644 index 000000000..70d0aa0f5 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/SyncConnector.java @@ -0,0 +1,25 @@ +package org.dromara.sync.connector; + +import org.dromara.sync.connector.model.ConnectorTestResult; +import org.dromara.sync.domain.SyncConnection; + +/** + * 数据同步连接器基础契约。 + */ +public interface SyncConnector { + + /** + * 获取连接器类型编码。 + * + * @return 连接类型 + */ + String type(); + + /** + * 测试连接。 + * + * @param connection 连接配置 + * @return 测试结果 + */ + ConnectorTestResult test(SyncConnection connection); +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/TargetConnector.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/TargetConnector.java new file mode 100644 index 000000000..b1b9560cd --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/TargetConnector.java @@ -0,0 +1,31 @@ +package org.dromara.sync.connector; + +import org.dromara.sync.connector.model.TargetWriteRequest; +import org.dromara.sync.connector.model.TargetWriteResult; +import org.dromara.sync.domain.SyncConnection; + +/** + * 目标端连接器契约。 + */ +public interface TargetConnector extends SyncConnector { + + TargetWriteResult upload(SyncConnection connection, TargetWriteRequest request); + + /** + * 判断目标对象是否存在，供冲突策略使用。 + * + * @param connection 目标连接 + * @param objectKey 对象键 + * @return 对象存在返回 {@code true} + */ + boolean exists(SyncConnection connection, String objectKey); + + /** + * 删除目标对象。只有同步计划明确配置删除策略时才允许调用。 + * + * @param connection 目标连接 + * @param objectKey 对象键 + * @return 是否删除成功 + */ + boolean delete(SyncConnection connection, String objectKey); +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DingTalkSourceConnector.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DingTalkSourceConnector.java new file mode 100644 index 000000000..2f8d652db --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DingTalkSourceConnector.java @@ -0,0 +1,737 @@ +package org.dromara.sync.connector.dingtalk; + +import lombok.RequiredArgsConstructor; +import org.dromara.common.core.exception.ServiceException; +import org.dromara.common.core.utils.StringUtils; +import org.dromara.common.json.utils.JsonUtils; +import org.dromara.sync.connector.SourceConnector; +import org.dromara.sync.connector.model.*; +import org.dromara.sync.constant.SyncConstants; +import org.dromara.sync.domain.SyncConnection; +import org.springframework.stereotype.Component; +import tools.jackson.databind.JsonNode; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.charset.StandardCharsets; +import java.io.IOException; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeParseException; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import java.util.zip.ZipInputStream; + +/** + * 基于 DingTalk Workspace CLI 的钉钉网盘源连接器。 + */ +@Component +@RequiredArgsConstructor +public class DingTalkSourceConnector implements SourceConnector { + + private static final int PAGE_SIZE = 50; + private static final Set ONLINE_EXTENSIONS = Set.of("adoc", "axls", "able", "appt", "amind", "adraw"); + private static final String XLSX_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + + private final DwsCommandRunner commandRunner; + + @Override + public String type() { + return SyncConstants.TYPE_DINGTALK; + } + + @Override + public ConnectorTestResult test(SyncConnection connection) { + try { + Map config = commandRunner.config(connection); + String spaceId = value(config, "spaceId"); + if (StringUtils.isNotBlank(spaceId)) { + DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, List.of( + "drive", "list", "--space-id", spaceId, "--limit", "1", "--format", "json" + ), Duration.ofSeconds(60)); + parseScanResult(result.stdout(), spaceId); + return ConnectorTestResult.success("钉钉空间访问成功"); + } + List arguments = new ArrayList<>(List.of( + "wiki", "space", "list", "--type", spaceType(config), "--format", "json" + )); + if ("orgSpace".equals(spaceType(config))) { + arguments.add("--limit"); + arguments.add("1"); + } + DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, arguments, Duration.ofSeconds(60)); + parseSpaceResult(result.stdout()); + return ConnectorTestResult.success("钉钉空间发现成功"); + } catch (Exception e) { + return ConnectorTestResult.failure(e.getMessage()); + } + } + + @Override + public ScanResult scan(SyncConnection connection, ScanRequest request) { + Map config = commandRunner.config(connection); + String spaceId = StringUtils.firstNonBlank(request.scopeId(), value(config, "spaceId")); + if (StringUtils.isBlank(spaceId)) { + if (StringUtils.isNotBlank(request.parentObjectId())) { + throw new ServiceException("指定钉盘子目录时必须在连接配置中提供 spaceId"); + } + return scanSpaces(connection, config, request.cursor()); + } + List arguments = new ArrayList<>(List.of( + "drive", "list", "--limit", String.valueOf(PAGE_SIZE), "--format", "json" + )); + addOption(arguments, "--space-id", spaceId); + addOption(arguments, "--folder", request.parentObjectId()); + addOption(arguments, "--cursor", request.cursor()); + DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, arguments, Duration.ofMinutes(5)); + return parseScanResult(result.stdout(), spaceId); + } + + @Override + public SourceContent download(SyncConnection connection, SourceObject object) { + if (object.isFolder()) { + throw new ServiceException("文件夹不能下载：{}", object.name()); + } + String extension = normalizeExtension(object.extension()); + if ("dlink".equalsIgnoreCase(extension)) { + throw new ServiceException("钉钉多维表链接（.dlink）不支持按普通文件下载"); + } + if ("amind".equalsIgnoreCase(extension) || "adraw".equalsIgnoreCase(extension)) { + throw new ServiceException("钉钉官方 DWS 暂不支持自动导出：{}", extension); + } + Path directory = null; + try { + directory = Files.createTempDirectory("ruoyi-sync-dingtalk-"); + Path output = directory.resolve(resolveOutputName(object)); + if ("dlink".equalsIgnoreCase(extension)) { + exportAITable(connection, object, output); + return new SourceContent(output, Files.size(output), XLSX_CONTENT_TYPE, true); + } + List arguments = buildDownloadArguments(connection, object, output); + try { + commandRunner.run(connection, arguments, Duration.ofHours(2), directory); + } catch (ServiceException exportFailure) { + // DWS v1.0.60 cannot export an empty ALIDOC and reports the + // asynchronous submit as doc_write_commit_unknown. Confirm + // the document is truly empty before applying the fallback; + // all non-empty documents and unrelated failures must retain + // the original error. + if (!SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType()) + || !"adoc".equalsIgnoreCase(normalizeExtension(object.extension())) + || !isEmptyOnlineDocument(connection, object)) { + throw exportFailure; + } + writeEmptyDocx(output); + } + if (!Files.isRegularFile(output)) { + throw new ServiceException("DWS 执行成功但未生成文件：{}", object.name()); + } + String contentType = SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType()) + ? onlineDocumentContentType(object) + : object.contentType(); + return new SourceContent(output, Files.size(output), contentType, true); + } catch (ServiceException e) { + cleanDirectory(directory); + throw e; + } catch (Exception e) { + cleanDirectory(directory); + throw new ServiceException("下载钉钉对象失败：{}", object.name(), e); + } + } + + private boolean isEmptyOnlineDocument(SyncConnection connection, SourceObject object) { + try { + DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, List.of( + "doc", "+fetch", "--node", object.objectId(), "--format", "json", "--detail", "full" + ), Duration.ofMinutes(2)); + JsonNode root = JsonUtils.getJsonMapper().readTree(result.stdout()); + JsonNode content = root.path("content.jsonml"); + return !containsNonBlankText(content); + } catch (Exception ignored) { + return false; + } + } + + private boolean containsNonBlankText(JsonNode node) { + if (node == null || node.isNull()) { + return false; + } + if (node.isTextual()) { + return StringUtils.isNotBlank(node.asString()); + } + if (node.isArray() || node.isObject()) { + for (JsonNode child : node) { + if (containsNonBlankText(child)) { + return true; + } + } + } + return false; + } + + private void writeEmptyDocx(Path output) throws Exception { + Files.deleteIfExists(output); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(output), StandardCharsets.UTF_8)) { + putZipEntry(zip, "[Content_Types].xml", """ + + + + + + + + """); + putZipEntry(zip, "_rels/.rels", """ + + + + + """); + putZipEntry(zip, "word/document.xml", """ + + + + + """); + putZipEntry(zip, "word/styles.xml", """ + + + """); + } + } + + private void putZipEntry(ZipOutputStream zip, String name, String content) throws IOException { + zip.putNextEntry(new ZipEntry(name)); + zip.write(content.getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + + private List buildDownloadArguments(SyncConnection connection, SourceObject object, Path output) { + if (SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType())) { + String extension = normalizeExtension(object.extension()); + if ("axls".equalsIgnoreCase(extension)) { + return List.of( + "sheet", "export", "--node", object.objectId(), "--output", output.getFileName().toString(), "--yes" + ); + } + if ("amind".equalsIgnoreCase(extension) || "adraw".equalsIgnoreCase(extension)) { + throw new ServiceException("钉钉官方 DWS 暂不支持自动导出：{}", extension); + } + if (!"adoc".equalsIgnoreCase(extension)) { + throw new ServiceException("当前版本暂不支持导出钉钉在线文档类型：{}", object.extension()); + } + // Use the consolidated `doc export` command. The `+export` + // shortcut maps to the low-level submit operation in some DWS + // releases (including v1.0.60), which can return + // `doc_write_commit_unknown` before polling/downloading the + // completed export job. `doc export` performs submit, polling + // and download atomically and only returns after the output file + // has been written. + return List.of( + "doc", "export", "--node", object.objectId(), "--output", output.getFileName().toString(), + "--export-format", "docx", "--yes" + ); + } + Map config = commandRunner.config(connection); + List arguments = new ArrayList<>(List.of( + "drive", "download", "--node", object.objectId(), + "--output", output.getFileName().toString(), "--format", "json" + )); + addOption(arguments, "--space-id", + StringUtils.firstNonBlank(metadataValue(object, "spaceId"), value(config, "spaceId"))); + String partSize = value(config, "downloadPartSize"); + addOption(arguments, "--part-size", StringUtils.isBlank(partSize) ? "32MB" : partSize); + String parallel = value(config, "downloadParallel"); + addOption(arguments, "--parallel", StringUtils.isBlank(parallel) ? "4" : parallel); + return arguments; + } + + private void exportAITable(SyncConnection connection, SourceObject object, Path output) throws Exception { + String url = StringUtils.firstNonBlank(metadataValue(object, "sourceUrl"), + object.name() != null && object.name().startsWith("http") ? object.name() : null, + object.objectId() != null && object.objectId().startsWith("http") ? object.objectId() : null); + if (StringUtils.isBlank(url) || !url.startsWith("http")) { + throw new ServiceException("无法解析钉钉多维表链接（.dlink）：缺少完整 alidocs/dlink URL"); + } + DwsCommandRunner.DwsCommandResult resolved = commandRunner.run(connection, List.of( + "aitable", "+url-resolve", "--url", url, "--format", "json" + ), Duration.ofMinutes(2)); + JsonNode ids = JsonUtils.getJsonMapper().readTree(resolved.stdout()); + String baseId = findText(ids, "baseId", "baseID"); + String tableId = findText(ids, "tableId", "datasheetId", "sheetId"); + if (StringUtils.isAnyBlank(baseId, tableId)) { + throw new ServiceException("无法解析钉钉多维表链接（.dlink）：未找到 baseId/tableId"); + } + JsonNode result; + try { + DwsCommandRunner.DwsCommandResult exported = commandRunner.run(connection, List.of( + "aitable", "+export-data", "--base-id", baseId, "--scope", "table", "--format", "excel", + "--table-id", tableId, "--timeout-ms", "120000" + ), Duration.ofMinutes(5)); + String exportOutput = exported.stdout(); + result = JsonUtils.getJsonMapper().readTree(exportOutput); + if (result == null && exportOutput != null && exportOutput.trim().startsWith("http")) { + result = JsonUtils.getJsonMapper().createObjectNode().put("downloadUrl", exportOutput.trim()); + } + } catch (Exception exportFailure) { + result = null; + } + String downloadUrl = findText(result, "downloadUrl", "url", "fileUrl", "downloadURL"); + if (StringUtils.isBlank(downloadUrl)) { + DwsCommandRunner.DwsCommandResult records = commandRunner.run(connection, List.of( + "aitable", "record", "query", "--base-id", baseId, "--table-id", tableId, + "--all", "--page-limit", "100" + ), Duration.ofMinutes(5)); + writeAITableWorkbook(output, minimalWorkbook(JsonUtils.getJsonMapper().readTree(records.stdout())), + url, object, baseId, tableId); + return; + } + HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build(); + HttpRequest request = HttpRequest.newBuilder(URI.create(downloadUrl)).GET().build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() < 200 || response.statusCode() >= 300 || response.body().length == 0) { + throw new ServiceException("下载钉钉多维表导出文件失败，HTTP {}", response.statusCode()); + } + writeAITableWorkbook(output, response.body(), url, object, baseId, tableId); + } + + private byte[] minimalWorkbook(JsonNode data) throws IOException { + StringBuilder rows = new StringBuilder(); + int rowNumber = 1; + JsonNode records = data == null ? null : (data.isArray() ? data : data.path("records")); + if (records != null && records.isArray()) { + for (JsonNode record : records) { + rows.append(row(rowNumber++, "record", xmlEscape(record.toString()))); + } + } + if (rows.isEmpty()) rows.append(row(1, "data", "")); + Map entries = new LinkedHashMap<>(); + entries.put("[Content_Types].xml", "".getBytes(StandardCharsets.UTF_8)); + entries.put("_rels/.rels", "".getBytes(StandardCharsets.UTF_8)); + entries.put("xl/workbook.xml", "".getBytes(StandardCharsets.UTF_8)); + entries.put("xl/_rels/workbook.xml.rels", "".getBytes(StandardCharsets.UTF_8)); + entries.put("xl/worksheets/sheet1.xml", ("" + rows + "").getBytes(StandardCharsets.UTF_8)); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (ZipOutputStream zip = new ZipOutputStream(out)) { + for (Map.Entry entry : entries.entrySet()) { zip.putNextEntry(new ZipEntry(entry.getKey())); zip.write(entry.getValue()); zip.closeEntry(); } + } + return out.toByteArray(); + } + + private String findText(JsonNode node, String... names) { + if (node == null || node.isNull()) return null; + for (String name : names) { + JsonNode value = node.path(name); + if (value.isTextual() && StringUtils.isNotBlank(value.asString())) return value.asString(); + } + if (node.isArray() || node.isObject()) { + for (JsonNode child : node) { + String found = findText(child, names); + if (StringUtils.isNotBlank(found)) return found; + } + } + return null; + } + + private void writeAITableWorkbook(Path output, byte[] workbook, String url, SourceObject object, + String baseId, String tableId) throws IOException { + // Keep the official exported workbook as Sheet1 and add a metadata Sheet2. + Map entries = new LinkedHashMap<>(); + try (ZipInputStream zip = new ZipInputStream(new ByteArrayInputStream(workbook))) { + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + entries.put(entry.getName(), zip.readAllBytes()); + } + } + byte[] workbookXml = entries.get("xl/workbook.xml"); + byte[] rels = entries.get("xl/_rels/workbook.xml.rels"); + if (workbookXml == null || rels == null) { + throw new IOException("导出的文件不是有效 XLSX"); + } + String sheetName = "Sheet2"; + String relId = "rId" + (entries.size() + 1); + String xml = new String(workbookXml, StandardCharsets.UTF_8); + int sheetsEnd = xml.indexOf(""); + if (sheetsEnd < 0) throw new IOException("XLSX 缺少 sheets 节点"); + xml = xml.substring(0, sheetsEnd) + "" + xml.substring(sheetsEnd); + String relXml = new String(rels, StandardCharsets.UTF_8); + int relEnd = relXml.indexOf(""); + relXml = relXml.substring(0, relEnd) + "" + relXml.substring(relEnd); + entries.put("xl/workbook.xml", xml.getBytes(StandardCharsets.UTF_8)); + entries.put("xl/_rels/workbook.xml.rels", relXml.getBytes(StandardCharsets.UTF_8)); + String contentTypes = new String(entries.get("[Content_Types].xml"), StandardCharsets.UTF_8); + int ctEnd = contentTypes.indexOf(""); + contentTypes = contentTypes.substring(0, ctEnd) + "" + contentTypes.substring(ctEnd); + entries.put("[Content_Types].xml", contentTypes.getBytes(StandardCharsets.UTF_8)); + String escapedUrl = xmlEscape(url); + String escapedName = xmlEscape(object.name()); + String sheet2 = "" + + row(1, "Field", "Value") + row(2, "sourceUrl", escapedUrl) + row(3, "sourceObjectId", xmlEscape(object.objectId())) + + row(4, "name", escapedName) + row(5, "baseId", xmlEscape(baseId)) + row(6, "tableId", xmlEscape(tableId)) + + row(7, "exportedAt", xmlEscape(Instant.now().toString())) + ""; + entries.put("xl/worksheets/sheet2.xml", sheet2.getBytes(StandardCharsets.UTF_8)); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(output))) { + for (Map.Entry entry : entries.entrySet()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); zip.write(entry.getValue()); zip.closeEntry(); + } + } + } + + private String row(int row, String key, String value) { + return "" + key + "" + value + ""; + } + + private String xmlEscape(String value) { + return value == null ? "" : value.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """); + } + + private String onlineDocumentContentType(SourceObject object) { + String extension = normalizeExtension(object.extension()); + if ("axls".equalsIgnoreCase(extension)) { + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + } + if ("amind".equalsIgnoreCase(extension) || "adraw".equalsIgnoreCase(extension)) { + return "application/pdf"; + } + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + } + + private ScanResult scanSpaces(SyncConnection connection, Map config, String cursor) { + String spaceType = spaceType(config); + List arguments = new ArrayList<>(List.of( + "wiki", "space", "list", "--type", spaceType, "--format", "json" + )); + if ("orgSpace".equals(spaceType)) { + arguments.add("--limit"); + arguments.add(String.valueOf(PAGE_SIZE)); + addOption(arguments, "--cursor", cursor); + } + DwsCommandRunner.DwsCommandResult result = commandRunner.run(connection, arguments, Duration.ofMinutes(2)); + return parseSpaceResult(result.stdout()); + } + + private ScanResult parseScanResult(String json, String spaceId) { + JsonNode root = JsonUtils.getJsonMapper().readTree(json); + validateEnvelope(root); + JsonNode payload = root.has("body") ? root.path("body") : root; + JsonNode array = findObjectArray(payload); + if (array == null) { + throw new ServiceException("无法识别 DWS 目录列表结果，已停止同步以避免误判删除"); + } + List objects = new ArrayList<>(); + for (JsonNode item : array) { + objects.add(mapObject(item, spaceId)); + } + String cursor = nestedText(payload, "nextCursor", "nextToken"); + boolean hasMore = nestedBoolean(payload, "hasMore") || StringUtils.isNotBlank(cursor); + return new ScanResult(objects, cursor, hasMore); + } + + private ScanResult parseSpaceResult(String json) { + JsonNode root = JsonUtils.getJsonMapper().readTree(json); + validateEnvelope(root); + JsonNode payload = root.has("body") ? root.path("body") : root; + JsonNode array = findObjectArray(payload); + if (array == null) { + throw new ServiceException("无法识别 DWS 空间列表结果，已停止同步"); + } + List spaces = new ArrayList<>(); + for (JsonNode item : array) { + String spaceId = text(item, "spaceId", "id"); + String rootFolderId = text(item, "rootFolderId", "rootDentryUuid", "rootNodeId"); + String name = text(item, "spaceName", "name", "title"); + if (StringUtils.isAnyBlank(spaceId, rootFolderId, name)) { + throw new ServiceException("DWS 空间列表包含缺少 spaceId、rootFolderId 或名称的条目，已停止同步"); + } + validateObjectIdentity(rootFolderId, name); + validateLength("空间ID", spaceId, 255); + Map metadata = new HashMap<>(); + metadata.put("spaceId", spaceId); + putIfPresent(metadata, "spaceType", text(item, "spaceType", "type")); + spaces.add(new SourceObject(rootFolderId, null, name, SyncConstants.OBJECT_FOLDER, + "", null, 0L, null, spaceId, null, metadata)); + } + String cursor = nestedText(payload, "nextToken", "nextCursor"); + return new ScanResult(spaces, cursor, StringUtils.isNotBlank(cursor)); + } + + private void validateEnvelope(JsonNode root) { + if (root == null || root.isNull()) { + throw new ServiceException("DWS 返回了空结果"); + } + if (root.path("success").isBoolean() && !root.path("success").asBoolean()) { + throw new ServiceException("DWS 返回失败：{}", root.path("message").asString("未知错误")); + } + JsonNode errors = findField(root, "errors"); + if (errors != null && errors.isArray() && !errors.isEmpty()) { + throw new ServiceException("DWS 返回了 {} 项目录读取错误，已停止同步", errors.size()); + } + } + + private JsonNode findObjectArray(JsonNode payload) { + if (payload == null) { + return null; + } + if (payload.isArray()) { + return payload; + } + for (String name : List.of("items", "files", "dentries", "entries", "fileList", "spaces", + "list", "data", "result", "content", "body")) { + JsonNode candidate = payload.path(name); + if (candidate.isArray()) { + return candidate; + } + if (candidate.isObject()) { + JsonNode nested = findObjectArray(candidate); + if (nested != null) { + return nested; + } + } + } + return null; + } + + private SourceObject mapObject(JsonNode item, String scanSpaceId) { + String objectId = text(item, "fileId", "dentryUuid", "nodeId", "id"); + String name = text(item, "name", "fileName", "title"); + if (StringUtils.isBlank(objectId) || StringUtils.isBlank(name)) { + throw new ServiceException("DWS 目录列表包含缺少对象ID或名称的条目，已停止同步"); + } + validateObjectIdentity(objectId, name); + String extension = normalizeExtension(text(item, "extension", "fileExtension", "suffix")); + String dentryType = text(item, "dentryType", "nodeType", "type"); + String contentType = text(item, "contentType", "mimeType"); + String objectType = resolveObjectType(dentryType, contentType, extension); + Long listedSize = longValue(item, "size", "fileSize", "contentSize"); + if (SyncConstants.OBJECT_FILE.equals(objectType) && listedSize == null) { + throw new ServiceException("DWS 普通文件缺少大小字段：{}", name); + } + long size = listedSize == null ? 0L : listedSize; + if (size < 0) { + throw new ServiceException("DWS 对象大小不能为负数：{}", name); + } + LocalDateTime modifiedTime = timeValue(item, "modifyTime", "modifiedTime", "updateTime", "updatedAt"); + String versionToken = text(item, "versionToken", "version", "versionNumber", "etag", "eTag"); + if (StringUtils.isBlank(versionToken) && modifiedTime != null) { + versionToken = modifiedTime.toString() + ':' + size; + } + Map metadata = new HashMap<>(); + putIfPresent(metadata, "spaceId", StringUtils.firstNonBlank(text(item, "spaceId"), scanSpaceId)); + putIfPresent(metadata, "dentryId", text(item, "dentryId")); + putIfPresent(metadata, "extension", extension); + putIfPresent(metadata, "sourceUrl", text(item, "url", "link", "dlink", "dlinkUrl", "alidocsUrl", "shareUrl", "sourceUrl")); + if (!metadata.containsKey("sourceUrl") && "dlink".equalsIgnoreCase(extension) + && name.startsWith("http")) { + metadata.put("sourceUrl", name); + } + String parentObjectId = text(item, "parentId", "parentFileId", "parentDentryUuid"); + String hash = text(item, "sha256", "hash", "etag", "eTag"); + validateLength("父对象ID", parentObjectId, 255); + validateLength("版本标识", versionToken, 512); + validateLength("源端ETag", hash, 255); + validateLength("内容类型", contentType, 255); + return new SourceObject( + objectId, + parentObjectId, + name, + objectType, + extension, + contentType, + size, + modifiedTime, + versionToken, + hash, + metadata + ); + } + + private void validateObjectIdentity(String objectId, String name) { + validateLength("对象ID", objectId, 255); + validateLength("对象名称", name, 1024); + if (".".equals(name) || "..".equals(name) + || name.indexOf('/') >= 0 || name.indexOf('\\') >= 0 || name.indexOf('\0') >= 0) { + throw new ServiceException("DWS 返回了不安全的对象名称，已停止同步"); + } + for (int index = 0; index < name.length(); index++) { + if (Character.isISOControl(name.charAt(index))) { + throw new ServiceException("DWS 对象名称包含控制字符，已停止同步"); + } + } + } + + private void validateLength(String fieldName, String value, int maxLength) { + if (value != null && value.length() > maxLength) { + throw new ServiceException("DWS {}超过数据库长度上限{}", fieldName, maxLength); + } + } + + private String resolveObjectType(String dentryType, String contentType, String extension) { + if ("folder".equalsIgnoreCase(dentryType) || "FOLDER".equalsIgnoreCase(dentryType)) { + return SyncConstants.OBJECT_FOLDER; + } + if ("ALIDOC".equalsIgnoreCase(contentType) || ONLINE_EXTENSIONS.contains(extension)) { + return SyncConstants.OBJECT_ONLINE_DOCUMENT; + } + if ("file".equalsIgnoreCase(dentryType) || "FILE".equalsIgnoreCase(dentryType)) { + return SyncConstants.OBJECT_FILE; + } + throw new ServiceException("无法识别 DWS 对象类型：{}", StringUtils.defaultIfBlank(dentryType, "空")); + } + + private String resolveOutputName(SourceObject object) { + if ("dlink".equalsIgnoreCase(normalizeExtension(object.extension()))) { + return "payload-" + UUID.randomUUID() + ".xlsx"; + } + if (SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(object.objectType())) { + String sourceExtension = normalizeExtension(object.extension()); + String extension = "axls".equalsIgnoreCase(sourceExtension) ? "xlsx" + : ("amind".equalsIgnoreCase(sourceExtension) || "adraw".equalsIgnoreCase(sourceExtension) + ? "pdf" : "docx"); + return "payload-" + UUID.randomUUID() + "." + extension; + } + String extension = normalizeExtension(object.extension()); + if (!extension.matches("[a-z0-9]{1,16}")) { + extension = ""; + } + return "payload-" + UUID.randomUUID() + (extension.isEmpty() ? "" : "." + extension); + } + + private LocalDateTime timeValue(JsonNode node, String... names) { + String value = text(node, names); + if (StringUtils.isBlank(value)) { + return null; + } + try { + long timestamp = Long.parseLong(value); + if (timestamp < 10_000_000_000L) { + timestamp *= 1000; + } + return LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.systemDefault()); + } catch (NumberFormatException ignored) { + try { + return LocalDateTime.ofInstant(Instant.parse(value), ZoneId.systemDefault()); + } catch (DateTimeParseException ignoredAgain) { + try { + return LocalDateTime.parse(value); + } catch (DateTimeParseException ignoredLast) { + throw new ServiceException("DWS 返回了无法解析的时间字段：{}", value); + } + } + } + } + + private Long longValue(JsonNode node, String... names) { + String value = text(node, names); + if (StringUtils.isBlank(value)) { + return null; + } + try { + return Long.parseLong(value); + } catch (NumberFormatException ignored) { + throw new ServiceException("DWS 返回了无法解析的大小字段：{}", value); + } + } + + private String nestedText(JsonNode node, String... names) { + JsonNode value = findField(node, names); + return value == null || value.isObject() || value.isArray() ? null : value.asString(); + } + + private boolean nestedBoolean(JsonNode node, String name) { + JsonNode value = findField(node, name); + return value != null && (value.isBoolean() ? value.asBoolean() + : Boolean.parseBoolean(value.asString("false"))); + } + + private JsonNode findField(JsonNode node, String... names) { + if (node == null || node.isNull() || node.isArray()) { + return null; + } + for (String name : names) { + JsonNode value = node.get(name); + if (value != null && !value.isNull()) { + return value; + } + } + for (JsonNode child : node) { + if (child.isObject()) { + JsonNode value = findField(child, names); + if (value != null) { + return value; + } + } + } + return null; + } + + private String text(JsonNode node, String... names) { + for (String name : names) { + JsonNode value = node.path(name); + if (!value.isMissingNode() && !value.isNull()) { + String text = value.asString(); + if (StringUtils.isNotBlank(text)) { + return text; + } + } + } + return null; + } + + private String value(Map values, String key) { + Object value = values.get(key); + return value == null ? null : String.valueOf(value); + } + + private String metadataValue(SourceObject object, String key) { + return object.metadata() == null ? null : object.metadata().get(key); + } + + private void addOption(List arguments, String name, String value) { + if (StringUtils.isNotBlank(value)) { + arguments.add(name); + arguments.add(value); + } + } + + private String normalizeExtension(String extension) { + if (StringUtils.isBlank(extension)) { + return ""; + } + return extension.startsWith(".") ? extension.substring(1).toLowerCase(Locale.ROOT) + : extension.toLowerCase(Locale.ROOT); + } + + private String spaceType(Map config) { + String type = value(config, "spaceType"); + return "mySpace".equals(type) ? "mySpace" : "orgSpace"; + } + + private void cleanDirectory(Path directory) { + if (directory == null) { + return; + } + try (var children = Files.list(directory)) { + for (Path child : children.toList()) { + Files.deleteIfExists(child); + } + Files.deleteIfExists(directory); + } catch (Exception ignored) { + // 下载失败后的临时目录采用尽力清理。 + } + } + + private void putIfPresent(Map target, String key, String value) { + if (StringUtils.isNotBlank(value)) { + target.put(key, value); + } + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthIdentity.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthIdentity.java new file mode 100644 index 000000000..2ca496bca --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthIdentity.java @@ -0,0 +1,16 @@ +package org.dromara.sync.connector.dingtalk; + +/** + * DWS 登录完成后返回的最小身份信息。 + * + * @param corpId 组织 ID + * @param corpName 组织名称 + * @param userId 钉钉用户 ID + * @param userName 用户名称 + */ +public record DwsAuthIdentity(String corpId, String corpName, String userId, String userName) { + + public String profile() { + return corpId + ":" + userId; + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthSessionManager.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthSessionManager.java new file mode 100644 index 000000000..af1a35d1b --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthSessionManager.java @@ -0,0 +1,912 @@ +package org.dromara.sync.connector.dingtalk; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.dromara.common.core.exception.ServiceException; +import org.dromara.common.core.utils.StringUtils; +import org.dromara.common.json.utils.JsonUtils; +import org.dromara.sync.domain.SyncConnection; +import org.dromara.sync.domain.vo.DwsAuthSessionVo; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import tools.jackson.databind.JsonNode; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 管理 DWS Web 设备授权会话。 + * + *

DWS 本身是本地 CLI，且设备流需要进程持续轮询。此组件把该进程 + * 放在后端节点运行，只把一次性授权链接和验证码返回给 Web 页面。会话 + * 采用内存保存进程句柄，适用于单节点部署；多节点部署时应通过网关会话 + * 亲和或专用认证节点保证后续状态请求落到同一节点。

+ */ +@Slf4j +@Component +@RequiredArgsConstructor +public class DwsAuthSessionManager { + + private static final Pattern URL_PATTERN = Pattern.compile("https://[^\\s<>\\\"']+"); + private static final Pattern USER_CODE_PATTERN = Pattern.compile( + "(?i)(?:授权码|user\\s*code|authorization\\s*code|verification\\s*code|code)\\s*[:：=]\\s*" + + "([A-Za-z0-9][A-Za-z0-9_-]{2,63})"); + private static final Pattern POLL_INTERVAL_PATTERN = Pattern.compile( + "(?i)(?:每|every)\\s*([1-9]\\d*)\\s*(?:秒|seconds?)"); + private static final Pattern PROFILE_PATTERN = Pattern.compile("^([^\\s:]+):([^\\s:]+)$"); + private static final Pattern ANSI_PATTERN = Pattern.compile("\\u001B\\[[;\\d]*m"); + private static final int MAX_OUTPUT_BYTES = 512 * 1024; + private static final int MAX_OUTPUT_LINE_CHARS = 16 * 1024; + private static final int MAX_ERROR_LENGTH = 2000; + private static final long DEFAULT_TIMEOUT_SECONDS = 600; + private static final long DEFAULT_METADATA_WAIT_SECONDS = 15; + private static final int DEFAULT_MAX_CONCURRENT = 4; + + private final DwsCommandRunner commandRunner; + + @Value("${sync.dingtalk.auth.enabled:true}") + private boolean enabled; + + @Value("${sync.dingtalk.auth.timeout-seconds:600}") + private long timeoutSeconds; + + @Value("${sync.dingtalk.auth.metadata-wait-seconds:15}") + private long metadataWaitSeconds; + + @Value("${sync.dingtalk.auth.max-concurrent:4}") + private int maxConcurrent; + + @Value("${sync.dingtalk.auth.retention-seconds:900}") + private long retentionSeconds; + + private final ConcurrentMap sessions = new ConcurrentHashMap<>(); + private final ConcurrentMap activeByConnection = new ConcurrentHashMap<>(); + private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor(); + private final ScheduledExecutorService cleanupExecutor = Executors.newSingleThreadScheduledExecutor(); + private volatile Semaphore permits; + + /** + * 初始化并启动过期会话清理任务。 + */ + @PostConstruct + public void init() { + // Keep operator-provided values useful while preventing an accidental + // overflow/indefinite HTTP request from turning a login slot into a + // permanent resource leak. + timeoutSeconds = boundedPositive(timeoutSeconds, DEFAULT_TIMEOUT_SECONDS, 1800); + metadataWaitSeconds = Math.min(timeoutSeconds, + boundedPositive(metadataWaitSeconds, DEFAULT_METADATA_WAIT_SECONDS, 30)); + maxConcurrent = (int) Math.max(1, Math.min(maxConcurrent, 32)); + retentionSeconds = boundedPositive(retentionSeconds, 900, 86400); + permits = new Semaphore(maxConcurrent); + cleanupExecutor.scheduleAtFixedRate(this::cleanupFinishedSessions, 1, 1, TimeUnit.MINUTES); + } + + /** + * 停止所有活动会话并释放执行器。 + */ + @PreDestroy + public void destroy() { + sessions.values().stream().filter(session -> !session.isTerminal()).forEach(session -> + cancelProcess(session, DwsAuthSessionState.CANCELLED, "服务正在关闭，授权会话已取消")); + cleanupExecutor.shutdownNow(); + // Let the watcher finish its finally block and let a just-started + // profile write complete before interrupting the virtual-thread pool. + // This avoids clearing the connection reference while FINALIZING is + // still invoking the persistence callback. + executor.shutdown(); + try { + if (!executor.awaitTermination(10, TimeUnit.SECONDS)) { + executor.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + executor.shutdownNow(); + } + sessions.values().stream().filter(Session::isTerminal).forEach(Session::clearSensitiveReferences); + } + + /** + * 启动一个 Web 设备授权会话。 + * + * @param connection 钉钉连接 + * @param ownerId 发起用户 ID + * @param expectedCorpId 可选的组织 ID，用于防止授权到错误组织 + * @param successHandler 授权成功后的身份持久化回调 + * @return 会话快照 + */ + public DwsAuthSessionVo start(SyncConnection connection, Long ownerId, String expectedCorpId, + Consumer successHandler) { + if (!enabled) { + throw new ServiceException("Web 钉钉登录功能已被管理员关闭"); + } + if (connection == null || connection.getConnectionId() == null) { + throw new ServiceException("钉钉连接必须先保存后才能登录"); + } + String normalizedExpectedCorpId = normalize(expectedCorpId); + if (normalizedExpectedCorpId != null && !isSafeIdentityPart(normalizedExpectedCorpId)) { + throw new ServiceException("expectedCorpId 格式不正确"); + } + Semaphore currentPermits = permits; + if (currentPermits == null || !currentPermits.tryAcquire()) { + throw new ServiceException("当前登录请求较多，请稍后重试"); + } + String sessionId = newSessionId(); + Session session = new Session(sessionId, connection.getConnectionId(), ownerId, + normalizedExpectedCorpId, System.currentTimeMillis() + timeoutSeconds * 1000L, successHandler); + session.connection = connection; + // Put the session and its per-connection index under one lock. A + // putIfAbsent followed by sessions.put leaves a small window in which + // a concurrent status request can observe an index with no session and + // accidentally remove a newly-started login. + synchronized (activeByConnection) { + String previous = activeByConnection.get(connection.getConnectionId()); + if (previous != null) { + Session existing = sessions.get(previous); + if (existing != null && !existing.isTerminal()) { + currentPermits.release(); + throw new ServiceException("该钉钉连接已有一个正在进行的登录会话"); + } + activeByConnection.remove(connection.getConnectionId(), previous); + } + sessions.put(sessionId, session); + activeByConnection.put(connection.getConnectionId(), sessionId); + } + Process process = null; + Future stdoutReader = null; + Future stderrReader = null; + DwsCommandRunner.ConnectionLockLease authLock = null; + boolean handedOff = false; + try { + authLock = commandRunner.tryAcquireAuthLock(connection); + if (authLock == null) { + throw new ServiceException("该钉钉连接正在执行同步命令，请等待任务结束后再登录"); + } + // Cancellation can arrive while the short lock acquisition is + // waiting for an in-flight download/scan. Publish the lease only + // after rechecking the session state; process publication below is + // serialized on the same session monitor. + synchronized (session) { + if (session.isTerminal()) { + authLock.close(); + authLock = null; + throw new IllegalStateException("授权会话在取得连接锁前已结束"); + } + session.authLock = authLock; + } + ProcessBuilder builder = commandRunner.newProcessBuilderWithoutProfile(connection, List.of( + "auth", "login", "--device", "--no-browser", "--yes", "--format", "json")); + /* + * Serialize process publication with cancel/expire/destroy. The + * session is visible before the CLI is started so a duplicate + * login can be rejected, but that also means a concurrent cancel + * can otherwise observe a null process and let an orphan process + * escape. Holding this lock through publication closes that + * window: cancellation either happens before start, or sees and + * stops the fully-published process. + */ + synchronized (session) { + if (session.isTerminal()) { + throw new IllegalStateException("授权会话在 DWS 进程启动前已结束"); + } + process = builder.start(); + process.getOutputStream().close(); + session.process = process; + Process runningProcess = process; + Future startedStdoutReader = executor.submit( + () -> readStream(session, runningProcess.getInputStream(), false)); + Future startedStderrReader = executor.submit( + () -> readStream(session, runningProcess.getErrorStream(), true)); + stdoutReader = startedStdoutReader; + stderrReader = startedStderrReader; + Future task = executor.submit( + () -> awaitProcess(session, runningProcess, startedStdoutReader, startedStderrReader)); + session.task = task; + // From this point awaitProcess owns the process and semaphore. + // A later HTTP response error must not release the same permit + // twice. The lock also guarantees cancel cannot run between + // process publication and this hand-off marker. + handedOff = true; + } + + // 通常 DWS 会在首次请求完成后立即打印二维码/链接；短暂等待可以让 + // 前端首个响应直接得到可展示的数据，网络慢时则返回 STARTING。 + try { + session.metadataReady.await(metadataWaitSeconds, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return snapshot(session); + } catch (Exception e) { + if (!handedOff) { + cancelFuture(stdoutReader); + cancelFuture(stderrReader); + commandRunner.stop(process); + session.process = null; + sessions.remove(sessionId, session); + activeByConnection.remove(connection.getConnectionId(), sessionId); + session.clearOutputs(); + session.clearSensitiveReferences(); + DwsCommandRunner.ConnectionLockLease publishedLock = session.authLock; + session.authLock = null; + if (publishedLock != null) { + publishedLock.close(); + } else if (authLock != null) { + authLock.close(); + } + currentPermits.release(); + } + if (e instanceof ServiceException serviceException) { + throw serviceException; + } + throw new ServiceException("无法启动 DWS Web 登录，请确认服务器已安装可执行的 DWS", e); + } + } + + /** + * 读取指定用户的授权会话状态。 + * + * @param connectionId 连接 ID + * @param sessionId 会话 ID + * @param ownerId 当前用户 ID + * @return 会话快照 + */ + public DwsAuthSessionVo get(Long connectionId, String sessionId, Long ownerId) { + return get(connectionId, sessionId, ownerId, false); + } + + /** + * 查询会话状态，可由超级管理员接管查看其他用户发起的会话。 + * + * @param connectionId 连接 ID + * @param sessionId 会话 ID + * @param ownerId 当前用户 ID + * @param privileged 是否为超级管理员 + * @return 会话快照 + */ + public DwsAuthSessionVo get(Long connectionId, String sessionId, Long ownerId, boolean privileged) { + Session session = requireSession(connectionId, sessionId, ownerId, privileged); + if (!session.isTerminal() && System.currentTimeMillis() >= session.expiresAt) { + expire(session); + } + return snapshot(session); + } + + /** + * 取消指定用户的授权会话。 + * + * @param connectionId 连接 ID + * @param sessionId 会话 ID + * @param ownerId 当前用户 ID + * @return 会话快照 + */ + public DwsAuthSessionVo cancel(Long connectionId, String sessionId, Long ownerId) { + return cancel(connectionId, sessionId, ownerId, false); + } + + /** + * 取消会话，可由超级管理员接管取消其他用户发起的会话。 + * + * @param connectionId 连接 ID + * @param sessionId 会话 ID + * @param ownerId 当前用户 ID + * @param privileged 是否为超级管理员 + * @return 取消后的会话快照 + */ + public DwsAuthSessionVo cancel(Long connectionId, String sessionId, Long ownerId, boolean privileged) { + Session session = requireSession(connectionId, sessionId, ownerId, privileged); + if (!session.isTerminal()) { + cancelProcess(session, DwsAuthSessionState.CANCELLED, "已取消钉钉登录"); + } + return snapshot(session); + } + + /** + * 判断连接是否仍有一个 Web 登录会话占用登录态目录。 + * + *

编辑或删除连接前调用此方法可以避免回调把 profile 写回已经 + * 变更/删除的记录。

+ */ + public boolean hasActive(Long connectionId) { + if (connectionId == null) { + return false; + } + String sessionId = activeByConnection.get(connectionId); + if (sessionId == null) { + return false; + } + Session session = sessions.get(sessionId); + if (session == null || session.isTerminal()) { + activeByConnection.remove(connectionId, sessionId); + return false; + } + return true; + } + + private void awaitProcess(Session session, Process process, Future stdoutReader, Future stderrReader) { + try { + if (!process.waitFor(Math.max(1, timeoutSeconds), TimeUnit.SECONDS)) { + expire(session); + return; + } + waitReader(stdoutReader); + waitReader(stderrReader); + if (session.state == DwsAuthSessionState.CANCELLED + || session.state == DwsAuthSessionState.EXPIRED + || session.state == DwsAuthSessionState.FINALIZING + || session.state == DwsAuthSessionState.SUCCESS) { + return; + } + if (process.exitValue() != 0) { + fail(session, diagnostic(session.errorOutput(), session.stdoutOutput(), + "DWS 登录命令执行失败")); + return; + } + complete(session); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + if (!session.isTerminal()) { + fail(session, "DWS 登录进程被中断"); + } + } catch (Exception e) { + if (!session.isTerminal()) { + fail(session, diagnostic(e.getMessage(), null, "DWS 登录失败")); + } + } finally { + // A shutdown/interruption can happen while waitFor is blocked. + // Always terminate the native process before dropping the handle; + // otherwise the watcher could leave an orphan DWS process behind. + commandRunner.stop(process); + Semaphore currentPermits = permits; + if (currentPermits != null) { + currentPermits.release(); + } + activeByConnection.remove(session.connectionId, session.id); + session.process = null; + session.terminalAt = System.currentTimeMillis(); + // CLI output can contain the one-time user code and verification + // URL. Do not retain it for the whole terminal-session TTL. + session.clearOutputs(); + session.clearSensitiveReferences(); + cancelFuture(stdoutReader); + cancelFuture(stderrReader); + DwsCommandRunner.ConnectionLockLease authLock = session.authLock; + session.authLock = null; + if (authLock != null) { + authLock.close(); + } + } + } + + private void complete(Session session) { + synchronized (session) { + if (session.isTerminal() || session.state == DwsAuthSessionState.FINALIZING) { + return; + } + // Once DWS has returned success, make cancellation deterministic: + // the session is finalizing its database write and cannot be + // changed back to CANCELLED halfway through that write. + session.state = DwsAuthSessionState.FINALIZING; + session.clearDevicePrompt(); + session.metadataReady.countDown(); + } + + String stdout = session.stdoutOutput(); + DwsAuthIdentity identity = parseIdentity(stdout); + session.clearOutputs(); + if (identity == null) { + fail(session, "DWS 登录成功但未返回完整 corpId:userId，请固定已验收版本后重试"); + return; + } + if (StringUtils.isNotBlank(session.expectedCorpId) + && !Objects.equals(session.expectedCorpId, identity.corpId())) { + fail(session, "授权组织与填写的组织 ID 不一致，请确认后重试"); + return; + } + try { + if (session.successHandler != null) { + session.successHandler.accept(identity); + } + synchronized (session) { + if (session.state != DwsAuthSessionState.FINALIZING) { + return; + } + session.corpId = identity.corpId(); + session.corpName = identity.corpName(); + session.userId = identity.userId(); + session.userName = identity.userName(); + session.profile = identity.profile(); + session.message = "钉钉登录成功，已保存登录态"; + session.state = DwsAuthSessionState.SUCCESS; + } + } catch (Exception e) { + fail(session, diagnostic(e.getMessage(), null, "登录成功，但保存连接身份失败")); + } + } + + private void readStream(Session session, InputStream inputStream, boolean errorStream) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + char[] buffer = new char[4096]; + StringBuilder line = new StringBuilder(Math.min(MAX_OUTPUT_LINE_CHARS, 4096)); + int length; + while ((length = reader.read(buffer)) >= 0) { + for (int index = 0; index < length; index++) { + char character = buffer[index]; + if (character == '\n') { + appendOutputLine(session, line.toString(), errorStream); + line.setLength(0); + } else if (character != '\r') { + if (line.length() < MAX_OUTPUT_LINE_CHARS) { + line.append(character); + } else { + // Flush bounded chunks from a pathological line so + // read() never accumulates unbounded attacker/CLI + // output in memory. Prompt lines are far shorter. + appendOutputLine(session, line.toString(), errorStream); + line.setLength(0); + } + } + } + } + if (line.length() > 0) { + appendOutputLine(session, line.toString(), errorStream); + } + } catch (IOException e) { + if (!session.isTerminal()) { + session.followUpError = diagnostic(e.getMessage(), session.followUpError, "读取 DWS 登录输出失败"); + } + } + } + + private void appendOutputLine(Session session, String line, boolean errorStream) { + String clean = stripAnsi(line); + if (errorStream) { + session.appendError(clean); + } else { + session.appendStdout(clean); + } + parseDevicePrompt(session, clean); + } + + private void parseDevicePrompt(Session session, String line) { + if (StringUtils.isBlank(line)) { + return; + } + synchronized (session) { + if (session.isTerminal() || session.state == DwsAuthSessionState.FINALIZING) { + return; + } + Matcher urlMatcher = URL_PATTERN.matcher(line); + while (urlMatcher.find()) { + String url = trimUrl(urlMatcher.group()); + if (!isAllowedVerificationUrl(url)) { + continue; + } + if (url.contains("?") || url.contains("&")) { + session.verificationUriComplete = url; + } else if (StringUtils.isBlank(session.verificationUri)) { + session.verificationUri = url; + } + } + Matcher codeMatcher = USER_CODE_PATTERN.matcher(line); + if (codeMatcher.find()) { + session.userCode = codeMatcher.group(1); + } + Matcher intervalMatcher = POLL_INTERVAL_PATTERN.matcher(line); + if (intervalMatcher.find()) { + try { + int interval = Integer.parseInt(intervalMatcher.group(1)); + session.pollInterval = Math.max(1, Math.min(interval, 30)); + } catch (NumberFormatException ignored) { + // Keep the safe default when a localized line is malformed. + } + } + if (StringUtils.isNotBlank(session.verificationUriComplete) + || (StringUtils.isNotBlank(session.verificationUri) && StringUtils.isNotBlank(session.userCode))) { + if (StringUtils.isBlank(session.verificationUri)) { + session.verificationUri = session.verificationUriComplete; + } + session.state = DwsAuthSessionState.WAITING_USER; + session.metadataReady.countDown(); + } + } + } + + private DwsAuthIdentity parseIdentity(String output) { + if (StringUtils.isBlank(output)) { + return null; + } + String json = extractJson(stripAnsi(output)); + if (json == null) { + return null; + } + try { + JsonNode root = JsonUtils.getJsonMapper().readTree(json); + return findIdentity(root); + } catch (Exception ignored) { + return null; + } + } + + private DwsAuthIdentity findIdentity(JsonNode node) { + if (node == null || node.isNull()) { + return null; + } + if (node.isObject()) { + // Prefer explicit IDs. A profile/display name may contain a + // colon too (for example "企业名:张三") and must not outrank the + // authoritative corp_id/user_id fields returned by DWS. + String corpId = text(node, "corpId", "corp_id", "organizationId", "orgId", "org_id"); + String userId = text(node, "userId", "user_id", "userid"); + if (StringUtils.isNotBlank(corpId) && StringUtils.isNotBlank(userId)) { + DwsAuthIdentity identity = validIdentity(corpId, text(node, "corpName", "corp_name", "organizationName"), + userId, text(node, "userName", "user_name", "name", "nick")); + if (identity != null) { + return identity; + } + } + String profile = text(node, "profile", "account", "accountProfile"); + DwsAuthIdentity byProfile = identityFromProfile(profile, node); + if (byProfile != null) { + return byProfile; + } + for (var field : node.properties()) { + DwsAuthIdentity nested = findIdentity(field.getValue()); + if (nested != null) { + return nested; + } + } + } else if (node.isArray()) { + for (JsonNode child : node) { + DwsAuthIdentity nested = findIdentity(child); + if (nested != null) { + return nested; + } + } + } + return null; + } + + private DwsAuthIdentity identityFromProfile(String profile, JsonNode source) { + if (StringUtils.isBlank(profile)) { + return null; + } + Matcher matcher = PROFILE_PATTERN.matcher(profile.trim()); + if (!matcher.matches()) { + return null; + } + String corpId = matcher.group(1); + String userId = matcher.group(2); + return validIdentity(corpId, + source == null ? null : text(source, "corpName", "corp_name", "organizationName"), + userId, + source == null ? null : text(source, "userName", "user_name", "name", "nick")); + } + + private DwsAuthIdentity validIdentity(String corpId, String corpName, String userId, String userName) { + if (!isSafeIdentityPart(corpId) || !isSafeIdentityPart(userId)) { + return null; + } + return new DwsAuthIdentity(corpId, normalize(corpName), userId, normalize(userName)); + } + + private boolean isSafeIdentityPart(String value) { + return StringUtils.isNotBlank(value) + && value.length() <= 255 + && value.chars().noneMatch(character -> Character.isWhitespace(character) + || Character.isISOControl(character)) + && value.indexOf(':') < 0 + && value.indexOf('/') < 0 + && value.indexOf('\\') < 0 + && value.indexOf('?') < 0 + && value.indexOf('&') < 0 + && value.indexOf('"') < 0 + && value.indexOf('\'') < 0; + } + + private Session requireSession(Long connectionId, String sessionId, Long ownerId, boolean privileged) { + if (connectionId == null || StringUtils.isBlank(sessionId)) { + throw new ServiceException("授权会话参数不能为空"); + } + Session session = sessions.get(sessionId); + if (session == null || !Objects.equals(session.connectionId, connectionId) + || (!privileged && !Objects.equals(session.ownerId, ownerId))) { + throw new ServiceException("授权会话不存在或已失效"); + } + return session; + } + + private void expire(Session session) { + if (session.isTerminal()) { + return; + } + cancelProcess(session, DwsAuthSessionState.EXPIRED, "授权链接已过期，请重新发起登录"); + } + + private void fail(Session session, String message) { + synchronized (session) { + if (session.isTerminal()) { + return; + } + session.state = DwsAuthSessionState.FAILED; + session.message = diagnostic(message, null, "DWS 登录失败，请重试"); + session.clearDevicePrompt(); + session.metadataReady.countDown(); + session.clearSensitiveReferences(); + } + } + + private void cancelProcess(Session session, DwsAuthSessionState state, String message) { + synchronized (session) { + if (session.isTerminal() || session.state == DwsAuthSessionState.FINALIZING) { + return; + } + session.state = state; + session.message = message; + session.clearDevicePrompt(); + Process process = session.process; + if (process != null) { + commandRunner.stop(process); + } + session.metadataReady.countDown(); + session.clearOutputs(); + session.clearSensitiveReferences(); + } + } + + private DwsAuthSessionVo snapshot(Session session) { + long remaining = Math.max(0, (session.expiresAt - System.currentTimeMillis()) / 1000L); + if (session.isTerminal() && session.state != DwsAuthSessionState.SUCCESS) { + remaining = 0; + } + boolean exposeDevicePrompt = session.state == DwsAuthSessionState.STARTING + || session.state == DwsAuthSessionState.WAITING_USER; + return new DwsAuthSessionVo( + session.id, + session.connectionId, + session.state.name(), + exposeDevicePrompt ? session.verificationUri : null, + exposeDevicePrompt ? session.verificationUriComplete : null, + exposeDevicePrompt ? session.userCode : null, + session.expiresAt, + remaining, + session.pollInterval, + session.corpId, + session.corpName, + session.userId, + session.userName, + session.profile, + session.message); + } + + private void cleanupFinishedSessions() { + long threshold = System.currentTimeMillis() - retentionSeconds * 1000L; + sessions.entrySet().removeIf(entry -> { + Session session = entry.getValue(); + return session.isTerminal() && session.terminalAt > 0 && session.terminalAt < threshold; + }); + } + + private void waitReader(Future reader) throws Exception { + if (reader != null) { + reader.get(15, TimeUnit.SECONDS); + } + } + + private void cancelFuture(Future future) { + if (future != null && !future.isDone()) { + future.cancel(true); + } + } + + private String diagnostic(String primary, String secondary, String fallback) { + String value = StringUtils.firstNonBlank(primary, secondary, fallback); + if (value == null) { + return fallback; + } + String sanitized = value + .replaceAll("(?i)(access[_-]?token|refresh[_-]?token|device[_-]?code|client[_-]?secret|app[_-]?secret)" + + "([\\\"'\\s:=]+)[^\\s,;\\\"']+", "$1$2***") + .replaceAll("(?i)(授权码|user\\s*code|authorization\\s*code|verification\\s*code|code)" + + "\\s*[:：=]\\s*[A-Za-z0-9][A-Za-z0-9_-]{2,63}", "$1:***") + .replaceAll("(?i)https?://[^\\s\\\"']+", "[授权链接已隐藏]") + .trim(); + return sanitized.length() <= MAX_ERROR_LENGTH ? sanitized : sanitized.substring(0, MAX_ERROR_LENGTH); + } + + private String extractJson(String output) { + int start = output.indexOf('{'); + int end = output.lastIndexOf('}'); + if (start < 0 || end <= start) { + start = output.indexOf('['); + end = output.lastIndexOf(']'); + } + return start >= 0 && end > start ? output.substring(start, end + 1) : null; + } + + private String text(JsonNode node, String... names) { + for (String name : names) { + JsonNode value = node.path(name); + if ((value.isTextual() || value.isNumber()) && StringUtils.isNotBlank(value.asString())) { + return value.asString().trim(); + } + } + return null; + } + + private String trimUrl(String value) { + return value.replaceAll("[\\.,;:!?\\)\\]\\}，。；：！？）】》]+$", ""); + } + + private boolean isAllowedVerificationUrl(String value) { + try { + URI uri = URI.create(value); + String host = uri.getHost(); + if (!"https".equalsIgnoreCase(uri.getScheme()) || StringUtils.isBlank(host)) { + return false; + } + String normalizedHost = host.toLowerCase(Locale.ROOT); + return "dingtalk.com".equals(normalizedHost) + || normalizedHost.endsWith(".dingtalk.com"); + } catch (Exception e) { + return false; + } + } + + private String stripAnsi(String value) { + return value == null ? null : ANSI_PATTERN.matcher(value).replaceAll(""); + } + + private String normalize(String value) { + return StringUtils.isBlank(value) ? null : value.trim(); + } + + private long positive(long value, long fallback) { + return value > 0 ? value : fallback; + } + + private long boundedPositive(long value, long fallback, long maximum) { + return Math.min(positive(value, fallback), maximum); + } + + private String newSessionId() { + byte[] bytes = new byte[24]; + SecureRandomHolder.INSTANCE.nextBytes(bytes); + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + private static final class SecureRandomHolder { + private static final java.security.SecureRandom INSTANCE = new java.security.SecureRandom(); + } + + private static final class Session { + private final String id; + private final Long connectionId; + private final Long ownerId; + private final String expectedCorpId; + private final long expiresAt; + private final CountDownLatch metadataReady = new CountDownLatch(1); + private final LimitedOutput stdout = new LimitedOutput(); + private final LimitedOutput stderr = new LimitedOutput(); + private volatile SyncConnection connection; + private volatile Process process; + private volatile Future task; + private volatile DwsCommandRunner.ConnectionLockLease authLock; + private volatile Consumer successHandler; + private volatile DwsAuthSessionState state = DwsAuthSessionState.STARTING; + private volatile String verificationUri; + private volatile String verificationUriComplete; + private volatile String userCode; + private volatile int pollInterval = 2; + private volatile String corpId; + private volatile String corpName; + private volatile String userId; + private volatile String userName; + private volatile String profile; + private volatile String message; + private volatile String followUpError; + private volatile long terminalAt; + + private Session(String id, Long connectionId, Long ownerId, String expectedCorpId, + long expiresAt, Consumer successHandler) { + this.id = id; + this.connectionId = connectionId; + this.ownerId = ownerId; + this.expectedCorpId = expectedCorpId; + this.expiresAt = expiresAt; + this.successHandler = successHandler; + } + + private boolean isTerminal() { + return switch (state) { + case SUCCESS, FAILED, EXPIRED, CANCELLED -> true; + default -> false; + }; + } + + private void appendStdout(String line) { + stdout.append(line); + } + + private void appendError(String line) { + stderr.append(line); + } + + private String stdoutOutput() { + return stdout.value(); + } + + private String errorOutput() { + return stderr.value(); + } + + private void clearOutputs() { + stdout.clear(); + stderr.clear(); + } + + private void clearDevicePrompt() { + verificationUri = null; + verificationUriComplete = null; + userCode = null; + } + + private void clearSensitiveReferences() { + connection = null; + successHandler = null; + task = null; + } + } + + private static final class LimitedOutput { + private final StringBuilder value = new StringBuilder(); + private int bytes; + + private synchronized void append(String line) { + if (bytes >= MAX_OUTPUT_BYTES) { + return; + } + String text = line + System.lineSeparator(); + byte[] encoded = text.getBytes(StandardCharsets.UTF_8); + int remaining = MAX_OUTPUT_BYTES - bytes; + if (encoded.length <= remaining) { + value.append(text); + bytes += encoded.length; + } else { + value.append(new String(encoded, 0, remaining, StandardCharsets.UTF_8)); + bytes = MAX_OUTPUT_BYTES; + } + } + + private synchronized String value() { + return value.toString(); + } + + private synchronized void clear() { + value.setLength(0); + bytes = 0; + } + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthSessionState.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthSessionState.java new file mode 100644 index 000000000..deeab6af8 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsAuthSessionState.java @@ -0,0 +1,15 @@ +package org.dromara.sync.connector.dingtalk; + +/** + * DWS Web 设备授权会话状态。 + */ +enum DwsAuthSessionState { + STARTING, + WAITING_USER, + /** DWS 已完成授权，后端正在把身份写回连接配置。 */ + FINALIZING, + SUCCESS, + FAILED, + EXPIRED, + CANCELLED +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsCommandRunner.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsCommandRunner.java new file mode 100644 index 000000000..1f1635caa --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/dingtalk/DwsCommandRunner.java @@ -0,0 +1,459 @@ +package org.dromara.sync.connector.dingtalk; + +import org.dromara.common.core.exception.ServiceException; +import org.dromara.common.core.utils.StringUtils; +import org.dromara.common.json.utils.JsonUtils; +import org.dromara.sync.domain.SyncConnection; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import tools.jackson.core.type.TypeReference; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.LinkOption; +import java.nio.file.attribute.PosixFilePermission; +import java.time.Duration; +import java.util.ArrayList; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.StampedLock; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +/** + * 安全执行 DingTalk Workspace CLI，不通过命令行参数传递凭证。 + */ +@Component +class DwsCommandRunner { + + private static final int MAX_CAPTURE_BYTES = 8 * 1024 * 1024; + private static final long COMMAND_LOCK_WAIT_MILLIS = 2000; + private static final long AUTH_LOCK_WAIT_MILLIS = 1000; + /** + * Environment variables needed by a native CLI/network stack. In + * particular, do not inherit the whole Spring process environment: it may + * contain database, object-storage, or encryption secrets. + */ + private static final List SAFE_ENVIRONMENT_KEYS = List.of( + "PATH", "USER", "LOGNAME", "LANG", "LC_ALL", "TZ", "TMPDIR", "TMP", "TEMP", + "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "all_proxy", "no_proxy", + "SSL_CERT_FILE", "SSL_CERT_DIR", "DWS_LANG", "DWS_TRUSTED_DOMAINS", + "DWS_ALLOW_HTTP_ENDPOINTS", "DWS_CHANNEL", "DWS_AGENT_PRODUCT"); + + @Value("${sync.dingtalk.dws-executable:dws}") + private String executable; + + @Value("${sync.dingtalk.profile-root:}") + private String profileRoot; + + /** + * DWS 加密 keychain 的根目录。DWS_CONFIG_DIR 只隔离配置文件，不能保证 + * 不同连接之间的 token/key 不串用，因此这里也按连接隔离 keychain。 + */ + @Value("${sync.dingtalk.keychain-root:}") + private String keychainRoot; + + /** + * A DWS config/keychain directory is mutable state. Keep business + * commands as readers so normal parallel downloads remain possible, while + * a Web login takes the exclusive writer lock for the whole device flow. + * This is intentionally JVM-local; a multi-node deployment still needs + * sticky routing or a distributed lock around DWS operations. + */ + private final ConcurrentMap connectionLocks = new ConcurrentHashMap<>(); + + DwsCommandResult run(SyncConnection connection, List arguments, Duration timeout) { + return run(connection, arguments, timeout, null); + } + + DwsCommandResult run(SyncConnection connection, List arguments, Duration timeout, Path workingDirectory) { + return runInternal(connection, arguments, timeout, workingDirectory, true); + } + + /** + * 执行不需要 profile 的 DWS 命令（例如 auth status/profile list）。 + * + * @param connection 连接 + * @param arguments 命令参数 + * @param timeout 超时时间 + * @return 命令结果 + */ + DwsCommandResult runWithoutProfile(SyncConnection connection, List arguments, Duration timeout) { + return runInternal(connection, arguments, timeout, null, false); + } + + /** + * 尝试为 Web 登录取得连接级独占锁。 + * + * @param connection 连接 + * @return 成功时的锁租约，忙时返回 {@code null} + */ + ConnectionLockLease tryAcquireAuthLock(SyncConnection connection) { + return tryAcquire(connection, true, AUTH_LOCK_WAIT_MILLIS); + } + + /** + * 为 Web 设备登录创建进程构造器。该进程不会带 profile，登录完成后 + * 才能由 DWS 返回实际的 corpId:userId。 + * + * @param connection 连接 + * @param arguments 命令参数 + * @return 已设置安全环境变量的进程构造器 + */ + ProcessBuilder newProcessBuilderWithoutProfile(SyncConnection connection, List arguments) { + return buildProcessBuilder(connection, arguments, null, false); + } + + /** + * 终止正在运行的 DWS 子进程及其子孙进程。 + * + * @param process 进程 + */ + void stop(Process process) { + terminate(process); + } + + private DwsCommandResult runInternal(SyncConnection connection, List arguments, Duration timeout, + Path workingDirectory, boolean includeProfile) { + Map secret = parseJson(connection.getSecretJson()); + String clientId = value(secret, "clientId", null); + String clientSecret = value(secret, "clientSecret", null); + if (StringUtils.isAnyBlank(clientId, clientSecret)) { + throw new ServiceException("钉钉连接 clientId、clientSecret 不能为空"); + } + if (StringUtils.isBlank(executable)) { + throw new ServiceException("sync.dingtalk.dws-executable 不能为空"); + } + ExecutorService executor = null; + ConnectionLockLease commandLock = null; + Process process = null; + try { + commandLock = tryAcquire(connection, false, COMMAND_LOCK_WAIT_MILLIS); + if (commandLock == null) { + throw new ServiceException("钉钉连接正在进行 Web 登录，请登录结束后重试"); + } + executor = java.util.concurrent.Executors.newVirtualThreadPerTaskExecutor(); + ProcessBuilder processBuilder = buildProcessBuilder(connection, arguments, workingDirectory, includeProfile); + process = processBuilder.start(); + process.getOutputStream().close(); + Process startedProcess = process; + Future stdoutFuture = executor.submit(() -> read(startedProcess.getInputStream())); + Future stderrFuture = executor.submit(() -> read(startedProcess.getErrorStream())); + if (!process.waitFor(timeout.toMillis(), TimeUnit.MILLISECONDS)) { + terminate(process); + throw new ServiceException("DWS 命令执行超时（{} 秒）", timeout.toSeconds()); + } + CapturedOutput stdoutResult = stdoutFuture.get(30, TimeUnit.SECONDS); + CapturedOutput stderrResult = stderrFuture.get(30, TimeUnit.SECONDS); + if (stdoutResult.truncated() || stderrResult.truncated()) { + throw new ServiceException("DWS 命令输出超过 {} MiB 安全上限", MAX_CAPTURE_BYTES / 1024 / 1024); + } + String stdout = stdoutResult.value(); + String stderr = stderrResult.value(); + if (process.exitValue() != 0) { + String message = StringUtils.isNotBlank(stderr) ? stderr : stdout; + throw new ServiceException("DWS 命令执行失败：{}", + abbreviate(sanitizeDiagnostic(message, clientId, clientSecret))); + } + return new DwsCommandResult(stdout, stderr); + } catch (ServiceException e) { + throw e; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new ServiceException("DWS 命令执行被中断", e); + } catch (Exception e) { + throw new ServiceException("无法执行 DWS，请确认 DingTalk Workspace CLI 已安装并可用", e); + } finally { + terminate(process); + if (executor != null) { + executor.shutdownNow(); + } + if (commandLock != null) { + commandLock.close(); + } + } + } + + private ConnectionLockLease tryAcquire(SyncConnection connection, boolean exclusive, long waitMillis) { + if (connection == null || connection.getConnectionId() == null) { + throw new ServiceException("钉钉连接必须先保存后才能执行 DWS 命令"); + } + StampedLock lock = connectionLocks.computeIfAbsent(connection.getConnectionId(), + ignored -> new StampedLock()); + try { + long stamp = exclusive + ? lock.tryWriteLock(waitMillis, TimeUnit.MILLISECONDS) + : lock.tryReadLock(waitMillis, TimeUnit.MILLISECONDS); + if (stamp == 0L) { + return null; + } + return new ConnectionLockLease(lock, stamp, exclusive); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new ServiceException("等待钉钉连接执行锁时被中断", e); + } + } + + private ProcessBuilder buildProcessBuilder(SyncConnection connection, List arguments, + Path workingDirectory, boolean includeProfile) { + Map config = parseJson(connection.getConfigJson()); + Map secret = parseJson(connection.getSecretJson()); + String clientId = value(secret, "clientId", null); + String clientSecret = value(secret, "clientSecret", null); + if (StringUtils.isAnyBlank(clientId, clientSecret)) { + throw new ServiceException("钉钉连接 clientId、clientSecret 不能为空"); + } + if (StringUtils.isBlank(executable)) { + throw new ServiceException("sync.dingtalk.dws-executable 不能为空"); + } + List command = new ArrayList<>(arguments.size() + (includeProfile ? 3 : 1)); + command.add(executable); + if (includeProfile) { + String profile = value(config, "profile", null); + if (StringUtils.isBlank(profile)) { + throw new ServiceException("钉钉连接尚未完成登录，请先在网页端完成设备授权"); + } + command.add("--profile"); + command.add(profile); + } + command.addAll(arguments); + Path configDirectory = resolveConfigDirectory(connection); + Path keychainDirectory = resolveKeychainDirectory(connection); + ProcessBuilder processBuilder = new ProcessBuilder(command); + restrictEnvironment(processBuilder, configDirectory); + putEnvironment(processBuilder, "DWS_CLIENT_ID", clientId); + putEnvironment(processBuilder, "DWS_CLIENT_SECRET", clientSecret); + putEnvironment(processBuilder, "DWS_CONFIG_DIR", configDirectory.toString()); + putEnvironment(processBuilder, "DWS_KEYCHAIN_DIR", keychainDirectory.toString()); + if (workingDirectory != null) { + processBuilder.directory(workingDirectory.toFile()); + } + return processBuilder; + } + + private void restrictEnvironment(ProcessBuilder processBuilder, Path configDirectory) { + Map inherited = new HashMap<>(); + Map environment = processBuilder.environment(); + boolean windowsHome = environment.containsKey("USERPROFILE"); + for (String key : SAFE_ENVIRONMENT_KEYS) { + String value = environment.get(key); + if (StringUtils.isNotBlank(value)) { + inherited.put(key, value); + } + } + environment.clear(); + environment.putAll(inherited); + // DWS_CONFIG_DIR and DWS_KEYCHAIN_DIR are explicit below. Point HOME + // at the connection directory as an additional guard against the CLI + // reading a global ~/.dws cache or writing credentials there. + environment.put("HOME", configDirectory.toString()); + if (windowsHome) { + environment.put("USERPROFILE", configDirectory.toString()); + } + } + + Map config(SyncConnection connection) { + return parseJson(connection.getConfigJson()); + } + + private void putEnvironment(ProcessBuilder builder, String key, String value) { + if (StringUtils.isNotBlank(value)) { + builder.environment().put(key, value); + } + } + + Path resolveConfigDirectory(SyncConnection connection) { + if (connection.getConnectionId() == null) { + throw new ServiceException("钉钉连接必须先保存后才能执行 DWS 命令"); + } + if (StringUtils.isBlank(profileRoot)) { + throw new ServiceException("sync.dingtalk.profile-root 不能为空"); + } + try { + return resolvePrivateDirectory(profileRoot, connection.getConnectionId(), "DWS 配置"); + } catch (ServiceException e) { + throw e; + } catch (Exception e) { + throw new ServiceException("无法创建该连接专属的 DWS 配置目录", e); + } + } + + private Path resolveKeychainDirectory(SyncConnection connection) { + if (connection.getConnectionId() == null) { + throw new ServiceException("钉钉连接必须先保存后才能执行 DWS 命令"); + } + String rootValue = StringUtils.isNotBlank(keychainRoot) + ? keychainRoot : (StringUtils.isNotBlank(profileRoot) ? profileRoot + "-keychain" : null); + if (StringUtils.isBlank(rootValue)) { + throw new ServiceException("sync.dingtalk.keychain-root 不能为空"); + } + try { + return resolvePrivateDirectory(rootValue, connection.getConnectionId(), "DWS keychain"); + } catch (ServiceException e) { + throw e; + } catch (Exception e) { + throw new ServiceException("无法创建该连接专属的 DWS keychain 目录", e); + } + } + + private Path resolvePrivateDirectory(String rootValue, Long connectionId, String label) throws IOException { + Path root = Path.of(rootValue).toAbsolutePath().normalize(); + if (Files.isSymbolicLink(root)) { + throw new ServiceException(label + "根目录不能是符号链接"); + } + createPrivateDirectory(root); + Path realRoot = root.toRealPath(LinkOption.NOFOLLOW_LINKS); + Path directory = realRoot.resolve(String.valueOf(connectionId)).normalize(); + if (!directory.startsWith(realRoot)) { + throw new ServiceException(label + "目录越界"); + } + if (Files.isSymbolicLink(directory)) { + throw new ServiceException(label + "目录不能是符号链接"); + } + createPrivateDirectory(directory); + Path realDirectory = directory.toRealPath(LinkOption.NOFOLLOW_LINKS); + if (!realDirectory.startsWith(realRoot) || Files.isSymbolicLink(directory)) { + throw new ServiceException(label + "目录越界或包含符号链接"); + } + return realDirectory; + } + + private void createPrivateDirectory(Path directory) throws IOException { + if (Files.isSymbolicLink(directory)) { + throw new IOException("directory is a symbolic link: " + directory); + } + Files.createDirectories(directory); + if (Files.isSymbolicLink(directory)) { + throw new IOException("directory became a symbolic link: " + directory); + } + try { + Set permissions = EnumSet.of( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE); + Files.setPosixFilePermissions(directory, permissions); + } catch (UnsupportedOperationException ignored) { + // Windows 文件系统没有 POSIX 权限模型，依赖进程账户 ACL。 + } + } + + private CapturedOutput read(java.io.InputStream inputStream) throws IOException { + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int length; + long total = 0; + while ((length = inputStream.read(buffer)) >= 0) { + total += length; + int remaining = MAX_CAPTURE_BYTES - captured.size(); + if (remaining > 0) { + captured.write(buffer, 0, Math.min(length, remaining)); + } + } + return new CapturedOutput(captured.toString(StandardCharsets.UTF_8), total > MAX_CAPTURE_BYTES); + } + + private Map parseJson(String json) { + if (StringUtils.isBlank(json)) { + return Map.of(); + } + try { + return JsonUtils.parseObject(json, new TypeReference<>() { + }); + } catch (Exception e) { + throw new ServiceException("钉钉连接配置不是合法 JSON", e); + } + } + + private String value(Map values, String key, String defaultValue) { + Object value = values.get(key); + return value == null ? defaultValue : String.valueOf(value); + } + + private String abbreviate(String value) { + if (value == null) { + return "未知错误"; + } + String trimmed = value.trim(); + return trimmed.length() <= 2000 ? trimmed : trimmed.substring(0, 2000); + } + + private String sanitizeDiagnostic(String value, String... secrets) { + String redacted = value; + if (redacted == null) { + return null; + } + for (String secret : secrets) { + if (StringUtils.isNotBlank(secret)) { + redacted = redacted.replace(secret, "***"); + } + } + redacted = redacted.replaceAll("(?i)(access[_-]?token|refresh[_-]?token|client[_-]?secret|app[_-]?secret|authorization)([\\\"'\\s:=]+)[^\\s,;\\\"']+", "$1$2***"); + return redacted.replaceAll("(?i)https?://[^\\s\\\"']+\\?[^\\s\\\"']+", "[signed-url-redacted]"); + } + + private void terminate(Process process) { + if (process == null || !process.isAlive()) { + return; + } + boolean interrupted = Thread.interrupted(); + try { + process.descendants().forEach(ProcessHandle::destroy); + process.destroy(); + if (!process.waitFor(2, TimeUnit.SECONDS)) { + process.descendants().forEach(ProcessHandle::destroyForcibly); + process.destroyForcibly(); + process.waitFor(2, TimeUnit.SECONDS); + } + } catch (InterruptedException e) { + interrupted = true; + process.descendants().forEach(ProcessHandle::destroyForcibly); + process.destroyForcibly(); + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + /** A one-shot owner for a connection read/write lock. */ + static final class ConnectionLockLease implements AutoCloseable { + private final StampedLock lock; + private final long stamp; + private final boolean exclusive; + private boolean closed; + + private ConnectionLockLease(StampedLock lock, long stamp, boolean exclusive) { + this.lock = lock; + this.stamp = stamp; + this.exclusive = exclusive; + } + + @Override + public synchronized void close() { + if (!closed) { + closed = true; + if (exclusive) { + lock.unlockWrite(stamp); + } else { + lock.unlockRead(stamp); + } + } + } + } + + record DwsCommandResult(String stdout, String stderr) { + } + + private record CapturedOutput(String value, boolean truncated) { + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ConnectorTestResult.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ConnectorTestResult.java new file mode 100644 index 000000000..35b743ae2 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ConnectorTestResult.java @@ -0,0 +1,18 @@ +package org.dromara.sync.connector.model; + +/** + * 连接器连通性测试结果。 + * + * @param success 是否成功 + * @param message 结果说明 + */ +public record ConnectorTestResult(boolean success, String message) { + + public static ConnectorTestResult success(String message) { + return new ConnectorTestResult(true, message); + } + + public static ConnectorTestResult failure(String message) { + return new ConnectorTestResult(false, message); + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ScanRequest.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ScanRequest.java new file mode 100644 index 000000000..0c1f531a5 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ScanRequest.java @@ -0,0 +1,11 @@ +package org.dromara.sync.connector.model; + +/** + * 源端单页扫描请求。 + * + * @param parentObjectId 父对象标识，为空表示根目录 + * @param cursor 分页游标 + * @param scopeId 源端作用域标识，例如钉盘 spaceId + */ +public record ScanRequest(String parentObjectId, String cursor, String scopeId) { +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ScanResult.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ScanResult.java new file mode 100644 index 000000000..ac3a9c880 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/ScanResult.java @@ -0,0 +1,13 @@ +package org.dromara.sync.connector.model; + +import java.util.List; + +/** + * 源端单页扫描结果。 + * + * @param objects 当前页对象 + * @param nextCursor 下一页游标 + * @param hasMore 是否还有下一页 + */ +public record ScanResult(List objects, String nextCursor, boolean hasMore) { +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/SourceContent.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/SourceContent.java new file mode 100644 index 000000000..1920d9808 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/SourceContent.java @@ -0,0 +1,36 @@ +package org.dromara.sync.connector.model; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * 下载或导出到本地临时文件的源内容。 + * + * @param path 本地路径 + * @param size 实际大小 + * @param contentType 内容类型 + * @param deleteOnClose 关闭后是否清理文件及临时目录 + */ +public record SourceContent(Path path, long size, String contentType, boolean deleteOnClose) implements AutoCloseable { + + @Override + public void close() { + if (!deleteOnClose) { + return; + } + try { + Files.deleteIfExists(path); + Path parent = path.getParent(); + if (parent != null) { + try (var children = Files.list(parent)) { + for (Path child : children.toList()) { + Files.deleteIfExists(child); + } + } + Files.deleteIfExists(parent); + } + } catch (Exception ignored) { + // 临时文件清理失败不能把已完成的远端上传回滚成失败状态。 + } + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/SourceObject.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/SourceObject.java new file mode 100644 index 000000000..cc4668976 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/SourceObject.java @@ -0,0 +1,38 @@ +package org.dromara.sync.connector.model; + +import java.time.LocalDateTime; +import java.util.Map; + +/** + * 不依赖具体源厂商的统一数据对象。 + * + * @param objectId 源端稳定对象标识 + * @param parentObjectId 父对象标识 + * @param name 对象名称 + * @param objectType FILE、FOLDER 或 ONLINE_DOCUMENT + * @param extension 文件扩展名 + * @param contentType 内容类型 + * @param size 文件大小 + * @param modifiedTime 修改时间 + * @param versionToken 源端版本标识 + * @param hash 源端校验值 + * @param metadata 源端扩展元数据 + */ +public record SourceObject( + String objectId, + String parentObjectId, + String name, + String objectType, + String extension, + String contentType, + long size, + LocalDateTime modifiedTime, + String versionToken, + String hash, + Map metadata +) { + + public boolean isFolder() { + return "FOLDER".equals(objectType); + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/TargetWriteRequest.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/TargetWriteRequest.java new file mode 100644 index 000000000..fd1bf861e --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/TargetWriteRequest.java @@ -0,0 +1,22 @@ +package org.dromara.sync.connector.model; + +import java.nio.file.Path; +import java.util.Map; + +/** + * 目标端写入请求。 + * + * @param objectKey 目标对象键 + * @param path 待上传本地文件 + * @param size 文件大小 + * @param contentType 内容类型 + * @param metadata 对象元数据 + */ +public record TargetWriteRequest( + String objectKey, + Path path, + long size, + String contentType, + Map metadata +) { +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/TargetWriteResult.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/TargetWriteResult.java new file mode 100644 index 000000000..197aae68a --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/model/TargetWriteResult.java @@ -0,0 +1,23 @@ +package org.dromara.sync.connector.model; + +import java.util.Map; + +/** + * 目标端写入结果。 + * + * @param objectKey 目标对象键 + * @param versionId 目标版本标识 + * @param eTag ETag + * @param size 已写入大小 + * @param url 对象访问地址 + * @param metadata 从目标端回读的对象元数据 + */ +public record TargetWriteResult( + String objectKey, + String versionId, + String eTag, + long size, + String url, + Map metadata +) { +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AbstractS3TargetConnector.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AbstractS3TargetConnector.java new file mode 100644 index 000000000..d75f32c5d --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AbstractS3TargetConnector.java @@ -0,0 +1,151 @@ +package org.dromara.sync.connector.s3; + +import jakarta.annotation.PreDestroy; +import org.dromara.common.core.exception.ServiceException; +import org.dromara.common.core.utils.StringUtils; +import org.dromara.common.json.utils.JsonUtils; +import org.dromara.common.oss.client.OssClient; +import org.dromara.common.oss.config.OssClientConfig; +import org.dromara.common.oss.model.Options; +import org.dromara.common.oss.model.PutObjectResult; +import org.dromara.sync.connector.TargetConnector; +import org.dromara.sync.connector.model.ConnectorTestResult; +import org.dromara.sync.connector.model.TargetWriteRequest; +import org.dromara.sync.connector.model.TargetWriteResult; +import org.dromara.sync.domain.SyncConnection; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import tools.jackson.core.type.TypeReference; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 基于仓库原生 S3 客户端的目标连接器基础实现。 + */ +public abstract class AbstractS3TargetConnector implements TargetConnector { + + private final Map clients = new ConcurrentHashMap<>(); + + @Override + public ConnectorTestResult test(SyncConnection connection) { + try { + client(connection).headBucket(connection.getBucketName()); + return ConnectorTestResult.success("目标存储桶访问成功"); + } catch (Exception e) { + return ConnectorTestResult.failure(e.getMessage()); + } + } + + @Override + public TargetWriteResult upload(SyncConnection connection, TargetWriteRequest request) { + Options options = Options.builder() + .setLength(request.size()) + .setContentType(request.contentType()) + .setMetadata(request.metadata()); + SyncS3OssClient client = client(connection); + PutObjectResult result = client.upload(request.objectKey(), request.path(), options); + HeadObjectResponse head = client.headObject(connection.getBucketName(), result.key()); + return new TargetWriteResult(result.key(), head.versionId(), head.eTag(), + head.contentLength(), result.url(), head.metadata()); + } + + @Override + public boolean exists(SyncConnection connection, String objectKey) { + return client(connection).objectExists(connection.getBucketName(), objectKey); + } + + @Override + public boolean delete(SyncConnection connection, String objectKey) { + return client(connection).delete(objectKey); + } + + /** + * 获取不同厂商的默认路径访问风格。 + */ + protected boolean defaultPathStyleAccess() { + return false; + } + + private SyncS3OssClient client(SyncConnection connection) { + OssClientConfig expected = buildConfig(connection); + return clients.compute(connection.getConnectionId(), (id, current) -> { + if (current != null && current.verifyConfig(expected)) { + return current; + } + closeQuietly(current); + return new SyncS3OssClient("sync-" + type().toLowerCase() + "-" + id, expected); + }); + } + + private OssClientConfig buildConfig(SyncConnection connection) { + if (StringUtils.isBlank(connection.getEndpoint())) { + throw new ServiceException("目标连接 endpoint 不能为空"); + } + if (StringUtils.isBlank(connection.getBucketName())) { + throw new ServiceException("目标连接 bucketName 不能为空"); + } + Map secret = parseJson(connection.getSecretJson()); + String accessKey = value(secret, "accessKey"); + String secretKey = value(secret, "secretKey"); + if (StringUtils.isBlank(accessKey) || StringUtils.isBlank(secretKey)) { + throw new ServiceException("目标连接 accessKey、secretKey 不能为空"); + } + Map config = parseJson(connection.getConfigJson()); + boolean useHttps = booleanValue(config, "useHttps", true); + boolean pathStyle = booleanValue(config, "pathStyleAccess", defaultPathStyleAccess()); + String domain = value(config, "domain"); + Region region = StringUtils.isBlank(connection.getRegion()) + ? Region.US_EAST_1 : Region.of(connection.getRegion()); + return OssClientConfig.builder() + .endpoint(connection.getEndpoint()) + .domain(domain) + .useHttps(useHttps) + .usePathStyleAccess(pathStyle) + .accessKey(accessKey) + .secretKey(secretKey) + .bucket(connection.getBucketName()) + .region(region) + .prefix(connection.getBasePath()) + .build(); + } + + private Map parseJson(String json) { + if (StringUtils.isBlank(json)) { + return Map.of(); + } + try { + return JsonUtils.parseObject(json, new TypeReference<>() { + }); + } catch (Exception e) { + throw new ServiceException("连接配置不是合法 JSON", e); + } + } + + private String value(Map values, String key) { + Object value = values.get(key); + return value == null ? null : String.valueOf(value); + } + + private boolean booleanValue(Map values, String key, boolean defaultValue) { + Object value = values.get(key); + return value == null ? defaultValue : Boolean.parseBoolean(String.valueOf(value)); + } + + @PreDestroy + public void destroy() { + clients.values().forEach(this::closeQuietly); + clients.clear(); + } + + private void closeQuietly(OssClient client) { + if (client == null) { + return; + } + try { + client.close(); + } catch (Exception ignored) { + // 客户端替换或容器销毁时不再抛出关闭异常。 + } + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AliyunOssTargetConnector.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AliyunOssTargetConnector.java new file mode 100644 index 000000000..b3ddbe38b --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AliyunOssTargetConnector.java @@ -0,0 +1,19 @@ +package org.dromara.sync.connector.s3; + +import org.dromara.sync.constant.SyncConstants; +import org.springframework.stereotype.Component; + +/** + * 阿里云 OSS 目标适配器。 + * + *

第一版复用仓库现有的 S3 协议客户端，厂商差异保留在独立连接器中， + * 后续可在不影响同步业务层的情况下切换阿里云原生 SDK。

+ */ +@Component +public class AliyunOssTargetConnector extends AbstractS3TargetConnector { + + @Override + public String type() { + return SyncConstants.TYPE_ALIYUN_OSS; + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/S3TargetConnector.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/S3TargetConnector.java new file mode 100644 index 000000000..0eae93f29 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/S3TargetConnector.java @@ -0,0 +1,16 @@ +package org.dromara.sync.connector.s3; + +import org.dromara.sync.constant.SyncConstants; +import org.springframework.stereotype.Component; + +/** + * S3 标准兼容目标连接器。 + */ +@Component +public class S3TargetConnector extends AbstractS3TargetConnector { + + @Override + public String type() { + return SyncConstants.TYPE_S3; + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/SyncS3OssClient.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/SyncS3OssClient.java new file mode 100644 index 000000000..f023a0169 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/SyncS3OssClient.java @@ -0,0 +1,44 @@ +package org.dromara.sync.connector.s3; + +import org.dromara.common.oss.client.DefaultOssClientImpl; +import org.dromara.common.oss.config.OssClientConfig; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; + +import java.util.concurrent.CompletionException; + +/** + * 为同步连接器补充只读的存储桶连通性探测。 + */ +class SyncS3OssClient extends DefaultOssClientImpl { + + SyncS3OssClient(String clientId, OssClientConfig config) { + super(clientId, config); + } + + void headBucket(String bucket) { + s3AsyncClient.headBucket(builder -> builder.bucket(bucket)).join(); + } + + HeadObjectResponse headObject(String bucket, String key) { + return s3AsyncClient.headObject(builder -> builder.bucket(bucket).key(key)).join(); + } + + boolean objectExists(String bucket, String key) { + try { + headObject(bucket, key); + return true; + } catch (CompletionException e) { + Throwable cause = e.getCause(); + if (cause instanceof S3Exception s3Exception && s3Exception.statusCode() == 404) { + return false; + } + throw e; + } catch (S3Exception e) { + if (e.statusCode() == 404) { + return false; + } + throw e; + } + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/constant/SyncConstants.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/constant/SyncConstants.java new file mode 100644 index 000000000..cd91a8294 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/constant/SyncConstants.java @@ -0,0 +1,35 @@ +package org.dromara.sync.constant; + +/** + * 数据同步模块常量。 + */ +public interface SyncConstants { + + String ROLE_SOURCE = "SOURCE"; + String ROLE_TARGET = "TARGET"; + + String TYPE_DINGTALK = "DINGTALK"; + String TYPE_S3 = "S3"; + String TYPE_ALIYUN_OSS = "ALIYUN_OSS"; + + String STATUS_NORMAL = "0"; + String STATUS_DISABLED = "1"; + + String JOB_PENDING = "PENDING"; + String JOB_RUNNING = "RUNNING"; + String JOB_CANCELING = "CANCELING"; + String JOB_SUCCESS = "SUCCESS"; + String JOB_PARTIAL_FAILED = "PARTIAL_FAILED"; + String JOB_FAILED = "FAILED"; + String JOB_CANCELED = "CANCELED"; + + String ITEM_PENDING = "PENDING"; + String ITEM_RUNNING = "RUNNING"; + String ITEM_SUCCESS = "SUCCESS"; + String ITEM_SKIPPED = "SKIPPED"; + String ITEM_FAILED = "FAILED"; + + String OBJECT_FILE = "FILE"; + String OBJECT_FOLDER = "FOLDER"; + String OBJECT_ONLINE_DOCUMENT = "ONLINE_DOCUMENT"; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncConnectionController.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncConnectionController.java new file mode 100644 index 000000000..be0cf1cd9 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncConnectionController.java @@ -0,0 +1,185 @@ +package org.dromara.sync.controller; + +import cn.dev33.satoken.annotation.SaCheckPermission; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.core.domain.R; +import org.dromara.common.core.validate.AddGroup; +import org.dromara.common.core.validate.EditGroup; +import org.dromara.common.core.validate.QueryGroup; +import org.dromara.common.log.annotation.Log; +import org.dromara.common.log.enums.BusinessType; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.common.redis.annotation.RepeatSubmit; +import org.dromara.common.satoken.utils.LoginHelper; +import org.dromara.common.web.core.BaseController; +import org.dromara.sync.connector.model.ConnectorTestResult; +import org.dromara.sync.domain.bo.SyncConnectionBo; +import org.dromara.sync.domain.vo.DwsAuthSessionVo; +import org.dromara.sync.domain.vo.SyncConnectionVo; +import org.dromara.sync.service.ISyncConnectionService; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +/** + * 同步连接管理。 + * + * @author Codex + * @date 2026-09-02 + */ +@Validated +@RequiredArgsConstructor +@RestController +@RequestMapping("/sync/connection") +public class SyncConnectionController extends BaseController { + + private final ISyncConnectionService connectionService; + + /** + * 分页查询同步连接列表。 + * + * @param bo 查询条件 + * @param pageQuery 分页参数 + * @return 同步连接分页列表 + */ + @SaCheckPermission("sync:connection:list") + @GetMapping("/list") + public R> list(@Validated(QueryGroup.class) SyncConnectionBo bo, + PageQuery pageQuery) { + return R.ok(connectionService.queryPageList(bo, pageQuery)); + } + + /** + * 获取同步连接详情。 + * + * @param connectionId 连接主键 + * @return 同步连接详情 + */ + @SaCheckPermission("sync:connection:query") + @GetMapping("/{connectionId}") + public R getInfo(@NotNull(message = "连接主键不能为空") + @PathVariable Long connectionId) { + return R.ok(connectionService.queryById(connectionId)); + } + + /** + * 新增同步连接。 + * + * @param bo 同步连接业务对象 + * @return 操作结果 + */ + @SaCheckPermission("sync:connection:add") + @Log(title = "同步连接", businessType = BusinessType.INSERT, excludeParamNames = "secretJson") + @RepeatSubmit() + @PostMapping() + public R add(@Validated(AddGroup.class) @RequestBody SyncConnectionBo bo) { + return toAjax(connectionService.insertByBo(bo)); + } + + /** + * 修改同步连接。 + * + * @param bo 同步连接业务对象 + * @return 操作结果 + */ + @SaCheckPermission("sync:connection:edit") + @Log(title = "同步连接", businessType = BusinessType.UPDATE, excludeParamNames = "secretJson") + @RepeatSubmit() + @PutMapping() + public R edit(@Validated(EditGroup.class) @RequestBody SyncConnectionBo bo) { + return toAjax(connectionService.updateByBo(bo)); + } + + /** + * 批量删除同步连接。 + * + * @param connectionIds 连接主键数组 + * @return 操作结果 + */ + @SaCheckPermission("sync:connection:remove") + @Log(title = "同步连接", businessType = BusinessType.DELETE) + @RepeatSubmit() + @DeleteMapping("/{connectionIds}") + public R remove(@NotEmpty(message = "连接主键不能为空") + @PathVariable Long[] connectionIds) { + return toAjax(connectionService.deleteWithValidByIds(List.of(connectionIds), true)); + } + + /** + * 校验同步连接配置并执行连通性测试。 + * + * @param connectionId 连接主键 + * @return 操作结果 + */ + @SaCheckPermission("sync:connection:test") + @Log(title = "同步连接测试", businessType = BusinessType.OTHER) + @RepeatSubmit() + @PostMapping("/test/{connectionId}") + public R testConnection(@NotNull(message = "连接主键不能为空") + @PathVariable Long connectionId) { + return R.ok(connectionService.testConnection(connectionId)); + } + + /** + * 在 Web 端启动钉钉设备授权。服务器负责运行 DWS，浏览器只接收一次性 + * 授权链接和验证码，不会接触任何 token 或应用密钥。 + * + * @param connectionId 钉钉连接主键 + * @param expectedCorpId 可选的组织 ID，用于防止授权到错误组织 + * @return 授权会话及展示信息 + */ + @SaCheckPermission("sync:connection:auth") + @Log(title = "钉钉 Web 登录", businessType = BusinessType.OTHER) + @RepeatSubmit() + @PostMapping("/{connectionId}/auth/login/start") + public R startDingTalkAuth( + @NotNull(message = "连接主键不能为空") @PathVariable Long connectionId, + @RequestParam(required = false) String expectedCorpId) { + return R.ok(connectionService.startDingTalkAuth(connectionId, LoginHelper.getUserId(), expectedCorpId)); + } + + /** + * 查询钉钉 Web 设备授权状态。 + * + * @param connectionId 连接主键 + * @param sessionId 授权会话 ID + * @return 授权状态 + */ + @SaCheckPermission("sync:connection:auth") + @GetMapping("/{connectionId}/auth/login/{sessionId}") + public R getDingTalkAuth( + @NotNull(message = "连接主键不能为空") @PathVariable Long connectionId, + @PathVariable String sessionId) { + return R.ok(connectionService.getDingTalkAuth(connectionId, sessionId, LoginHelper.getUserId())); + } + + /** + * 取消钉钉 Web 设备授权并终止服务器上的 DWS 进程。 + * + * @param connectionId 连接主键 + * @param sessionId 授权会话 ID + * @return 取消后的授权状态 + */ + @SaCheckPermission("sync:connection:auth") + @Log(title = "取消钉钉 Web 登录", businessType = BusinessType.UPDATE) + @RepeatSubmit() + @PostMapping("/{connectionId}/auth/login/{sessionId}/cancel") + public R cancelDingTalkAuth( + @NotNull(message = "连接主键不能为空") @PathVariable Long connectionId, + @PathVariable String sessionId) { + return R.ok(connectionService.cancelDingTalkAuth(connectionId, sessionId, LoginHelper.getUserId())); + } + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncJobController.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncJobController.java new file mode 100644 index 000000000..43a983345 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncJobController.java @@ -0,0 +1,83 @@ +package org.dromara.sync.controller; + +import cn.dev33.satoken.annotation.SaCheckPermission; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.core.domain.R; +import org.dromara.common.log.annotation.Log; +import org.dromara.common.log.enums.BusinessType; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.common.redis.annotation.RepeatSubmit; +import org.dromara.common.web.core.BaseController; +import org.dromara.sync.domain.bo.SyncJobBo; +import org.dromara.sync.domain.bo.SyncJobItemBo; +import org.dromara.sync.domain.vo.SyncJobItemVo; +import org.dromara.sync.domain.vo.SyncJobVo; +import org.dromara.sync.service.ISyncJobService; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +/** + * 同步任务管理接口。 + */ +@Validated +@RestController +@RequiredArgsConstructor +@RequestMapping("/sync/job") +public class SyncJobController extends BaseController { + + private final ISyncJobService jobService; + + @SaCheckPermission("sync:job:list") + @GetMapping("/list") + public R> list(SyncJobBo bo, PageQuery pageQuery) { + return R.ok(jobService.queryPageList(bo, pageQuery)); + } + + @SaCheckPermission("sync:job:query") + @GetMapping("/{jobId}") + public R getInfo(@NotNull(message = "任务ID不能为空") @PathVariable Long jobId) { + return R.ok(jobService.queryById(jobId)); + } + + @SaCheckPermission("sync:job:query") + @GetMapping("/{jobId}/items") + public R> items(@PathVariable Long jobId, SyncJobItemBo bo, PageQuery pageQuery) { + bo.setJobId(jobId); + return R.ok(jobService.queryItemPageList(bo, pageQuery)); + } + + @SaCheckPermission("sync:job:run") + @Log(title = "同步任务", businessType = BusinessType.OTHER) + @RepeatSubmit + @PostMapping("/run/{planId}") + public R run(@NotNull(message = "计划ID不能为空") @PathVariable Long planId) { + return R.ok(jobService.startPlan(planId, "MANUAL", null)); + } + + @SaCheckPermission("sync:job:retry") + @Log(title = "同步任务重试", businessType = BusinessType.OTHER) + @RepeatSubmit + @PostMapping("/retry/{jobId}") + public R retry(@NotNull(message = "任务ID不能为空") @PathVariable Long jobId) { + return R.ok(jobService.retry(jobId)); + } + + @SaCheckPermission("sync:job:cancel") + @Log(title = "取消同步任务", businessType = BusinessType.UPDATE) + @PostMapping("/cancel/{jobId}") + public R cancel(@NotNull(message = "任务ID不能为空") @PathVariable Long jobId) { + return toAjax(jobService.cancel(jobId)); + } + + @SaCheckPermission("sync:job:remove") + @Log(title = "同步任务", businessType = BusinessType.DELETE) + @DeleteMapping("/{jobIds}") + public R remove(@NotEmpty(message = "任务ID不能为空") @PathVariable Long[] jobIds) { + return toAjax(jobService.deleteWithValidByIds(List.of(jobIds), true)); + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncObjectController.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncObjectController.java new file mode 100644 index 000000000..0241c7a47 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncObjectController.java @@ -0,0 +1,38 @@ +package org.dromara.sync.controller; + +import cn.dev33.satoken.annotation.SaCheckPermission; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.core.domain.R; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.common.web.core.BaseController; +import org.dromara.sync.domain.bo.SyncObjectBo; +import org.dromara.sync.domain.vo.SyncObjectVo; +import org.dromara.sync.service.ISyncObjectService; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +/** + * 同步对象清单接口。 + */ +@Validated +@RestController +@RequiredArgsConstructor +@RequestMapping("/sync/object") +public class SyncObjectController extends BaseController { + + private final ISyncObjectService objectService; + + @SaCheckPermission("sync:object:list") + @GetMapping("/list") + public R> list(SyncObjectBo bo, PageQuery pageQuery) { + return R.ok(objectService.queryPageList(bo, pageQuery)); + } + + @SaCheckPermission("sync:object:query") + @GetMapping("/{objectId}") + public R getInfo(@NotNull(message = "对象ID不能为空") @PathVariable Long objectId) { + return R.ok(objectService.queryById(objectId)); + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncPlanController.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncPlanController.java new file mode 100644 index 000000000..5fa6e24dd --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/controller/SyncPlanController.java @@ -0,0 +1,121 @@ +package org.dromara.sync.controller; + +import cn.dev33.satoken.annotation.SaCheckPermission; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.core.domain.R; +import org.dromara.common.core.validate.AddGroup; +import org.dromara.common.core.validate.EditGroup; +import org.dromara.common.core.validate.QueryGroup; +import org.dromara.common.log.annotation.Log; +import org.dromara.common.log.enums.BusinessType; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.common.redis.annotation.RepeatSubmit; +import org.dromara.common.web.core.BaseController; +import org.dromara.sync.domain.bo.SyncPlanBo; +import org.dromara.sync.domain.vo.SyncPlanVo; +import org.dromara.sync.service.ISyncPlanService; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +/** + * 同步计划管理。 + * + * @author Codex + * @date 2026-09-02 + */ +@Validated +@RequiredArgsConstructor +@RestController +@RequestMapping("/sync/plan") +public class SyncPlanController extends BaseController { + + private final ISyncPlanService planService; + + /** + * 分页查询同步计划列表。 + * + * @param bo 查询条件 + * @param pageQuery 分页参数 + * @return 同步计划分页数据 + */ + @SaCheckPermission("sync:plan:list") + @GetMapping("/list") + public R> list(@Validated(QueryGroup.class) SyncPlanBo bo, PageQuery pageQuery) { + return R.ok(planService.queryPageList(bo, pageQuery)); + } + + /** + * 获取同步计划详细信息。 + * + * @param planId 计划主键 + * @return 同步计划详情 + */ + @SaCheckPermission("sync:plan:query") + @GetMapping("/{planId}") + public R getInfo(@NotNull(message = "计划主键不能为空") + @PathVariable Long planId) { + return R.ok(planService.queryById(planId)); + } + + /** + * 新增同步计划。 + * + * @param bo 同步计划 + * @return 操作结果 + */ + @SaCheckPermission("sync:plan:add") + @Log(title = "同步计划", businessType = BusinessType.INSERT) + @RepeatSubmit() + @PostMapping() + public R add(@Validated(AddGroup.class) @RequestBody SyncPlanBo bo) { + if (!planService.checkPlanNameUnique(bo)) { + return R.fail("新增同步计划'" + bo.getPlanName() + "'失败，计划名称已存在"); + } + return toAjax(planService.insertByBo(bo)); + } + + /** + * 修改同步计划。 + * + * @param bo 同步计划 + * @return 操作结果 + */ + @SaCheckPermission("sync:plan:edit") + @Log(title = "同步计划", businessType = BusinessType.UPDATE) + @RepeatSubmit() + @PutMapping() + public R edit(@Validated(EditGroup.class) @RequestBody SyncPlanBo bo) { + if (!planService.checkPlanNameUnique(bo)) { + return R.fail("修改同步计划'" + bo.getPlanName() + "'失败，计划名称已存在"); + } + return toAjax(planService.updateByBo(bo)); + } + + /** + * 批量删除同步计划。 + * + * @param planIds 计划主键集合 + * @return 操作结果 + */ + @SaCheckPermission("sync:plan:remove") + @Log(title = "同步计划", businessType = BusinessType.DELETE) + @RepeatSubmit() + @DeleteMapping("/{planIds}") + public R remove(@NotEmpty(message = "计划主键不能为空") + @PathVariable Long[] planIds) { + return toAjax(planService.deleteWithValidByIds(List.of(planIds), true)); + } + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncCheckpoint.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncCheckpoint.java new file mode 100644 index 000000000..5d7786a75 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncCheckpoint.java @@ -0,0 +1,34 @@ +package org.dromara.sync.domain; + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.annotation.Version; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.dromara.common.mybatis.core.domain.BaseEntity; + +import java.io.Serial; +import java.time.LocalDateTime; + +/** + * 同步增量与分页断点 sync_checkpoint。 + */ +@Data +@EqualsAndHashCode(callSuper = true) +@TableName("sync_checkpoint") +public class SyncCheckpoint extends BaseEntity { + + @Serial + private static final long serialVersionUID = 1L; + + @TableId("checkpoint_id") + private Long checkpointId; + private Long planId; + private String checkpointType; + private String checkpointKey; + private String checkpointValue; + private LocalDateTime watermarkTime; + private Long lastJobId; + @Version + private Long version; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncConnection.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncConnection.java new file mode 100644 index 000000000..f0f63e71d --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncConnection.java @@ -0,0 +1,96 @@ +package org.dromara.sync.domain; + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.dromara.common.encrypt.annotation.EncryptField; +import org.dromara.common.encrypt.enums.AlgorithmType; +import org.dromara.common.mybatis.core.domain.BaseEntity; + +import java.io.Serial; + +/** + * 同步连接对象 sync_connection。 + * + * @author Codex + * @date 2026-09-02 + */ +@Data +@EqualsAndHashCode(callSuper = true) +@TableName("sync_connection") +public class SyncConnection extends BaseEntity { + + @Serial + private static final long serialVersionUID = 1L; + + /** + * 连接主键。 + */ + @TableId(value = "connection_id") + private Long connectionId; + + /** + * 连接名称。 + */ + private String connectionName; + + /** + * 连接角色（SOURCE 源端、TARGET 目标端）。 + */ + private String connectionRole; + + /** + * 连接类型（DINGTALK、S3、ALIYUN_OSS）。 + */ + private String connectionType; + + /** + * 服务端点。 + */ + private String endpoint; + + /** + * 服务区域。 + */ + private String region; + + /** + * 存储桶名称。 + */ + private String bucketName; + + /** + * 目标基础路径。 + */ + private String basePath; + + /** + * 非敏感扩展配置 JSON。 + */ + private String configJson; + + /** + * 敏感凭证配置 JSON，持久化时由 MyBatis 加密处理器加密。 + */ + @EncryptField(algorithm = AlgorithmType.AES) + private String secretJson; + + /** + * 状态（0 正常、1 停用）。 + */ + private String status; + + /** + * 删除标志（0 存在、1 删除）。 + */ + @TableLogic + private String delFlag; + + /** + * 备注。 + */ + private String remark; + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncJob.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncJob.java new file mode 100644 index 000000000..71dc0be0a --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncJob.java @@ -0,0 +1,42 @@ +package org.dromara.sync.domain; + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.dromara.common.mybatis.core.domain.BaseEntity; + +import java.io.Serial; +import java.time.LocalDateTime; + +/** + * 同步任务运行实例 sync_job。 + */ +@Data +@EqualsAndHashCode(callSuper = true) +@TableName("sync_job") +public class SyncJob extends BaseEntity { + + @Serial + private static final long serialVersionUID = 1L; + + @TableId("job_id") + private Long jobId; + private Long planId; + private String triggerType; + private String runType; + private String status; + private String checkpointBefore; + private String checkpointAfter; + private Long totalCount; + private Long processedCount; + private Long successCount; + private Long failedCount; + private Long skippedCount; + private Long deletedCount; + private Long totalBytes; + private Long transferredBytes; + private LocalDateTime startTime; + private LocalDateTime finishTime; + private String errorMessage; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncJobItem.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncJobItem.java new file mode 100644 index 000000000..c718744b9 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncJobItem.java @@ -0,0 +1,45 @@ +package org.dromara.sync.domain; + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.dromara.common.mybatis.core.domain.BaseEntity; + +import java.io.Serial; +import java.time.LocalDateTime; + +/** + * 同步任务文件明细 sync_job_item。 + */ +@Data +@EqualsAndHashCode(callSuper = true) +@TableName("sync_job_item") +public class SyncJobItem extends BaseEntity { + + @Serial + private static final long serialVersionUID = 1L; + + @TableId("item_id") + private Long itemId; + private Long jobId; + private String sourceObjectId; + private String parentObjectId; + private String sourcePath; + private String targetKey; + private String objectType; + private String actionType; + private String status; + private Long size; + private Long transferredBytes; + private String versionToken; + private String sourceEtag; + private String sourceSha256; + private String targetEtag; + private String targetVersionId; + private Integer retryCount; + private LocalDateTime startTime; + private LocalDateTime finishTime; + private String errorCode; + private String errorMessage; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncObject.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncObject.java new file mode 100644 index 000000000..72f442802 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncObject.java @@ -0,0 +1,51 @@ +package org.dromara.sync.domain; + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.FieldStrategy; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.dromara.common.mybatis.core.domain.BaseEntity; + +import java.io.Serial; +import java.time.LocalDateTime; + +/** + * 源对象与目标对象映射清单 sync_object。 + */ +@Data +@EqualsAndHashCode(callSuper = true) +@TableName("sync_object") +public class SyncObject extends BaseEntity { + + @Serial + private static final long serialVersionUID = 1L; + + @TableId("object_id") + private Long objectId; + private Long planId; + private String sourceObjectId; + private String parentObjectId; + private String sourcePath; + private String objectName; + private String objectType; + private Long size; + private LocalDateTime modifiedTime; + private String versionToken; + private String sourceEtag; + private String sourceSha256; + private String contentType; + private String metadataJson; + private String targetKey; + private String targetVersionId; + private String targetEtag; + private String syncStatus; + private String sourceDeleted; + private Long firstSeenJobId; + private Long lastSeenJobId; + private Long lastSyncJobId; + private LocalDateTime lastSyncTime; + @TableField(updateStrategy = FieldStrategy.ALWAYS) + private String lastErrorMessage; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncPlan.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncPlan.java new file mode 100644 index 000000000..8c2098868 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncPlan.java @@ -0,0 +1,132 @@ +package org.dromara.sync.domain; + +import com.baomidou.mybatisplus.annotation.FieldStrategy; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.dromara.common.mybatis.core.domain.BaseEntity; + +import java.io.Serial; +import java.time.LocalDateTime; + +/** + * 同步计划对象 sync_plan。 + * + * @author Codex + * @date 2026-09-02 + */ +@Data +@EqualsAndHashCode(callSuper = true) +@TableName("sync_plan") +public class SyncPlan extends BaseEntity { + + @Serial + private static final long serialVersionUID = 1L; + + /** + * 计划主键。 + */ + @TableId(value = "plan_id") + private Long planId; + + /** + * 计划名称。 + */ + private String planName; + + /** + * 源端连接主键。 + */ + private Long sourceConnectionId; + + /** + * 目标端连接主键。 + */ + private Long targetConnectionId; + + /** + * 源端同步根路径或对象标识。 + */ + private String sourceRoot; + + /** + * 目标端对象键前缀。 + */ + private String targetPrefix; + + /** + * 同步模式（FULL 全量、INCREMENTAL 增量）。 + */ + private String syncMode; + + /** + * 调度类型（MANUAL 手动、CRON 定时）。 + */ + private String scheduleType; + + /** + * CRON 表达式；切换为手动调度时需要显式清空。 + */ + @TableField(updateStrategy = FieldStrategy.ALWAYS) + private String cronExpression; + + /** + * 冲突处理策略（OVERWRITE 覆盖、SKIP 跳过、KEEP_BOTH 两者保留）。 + */ + private String conflictStrategy; + + /** + * 源端删除处理策略（KEEP 保留、MARK 标记、DELETE 删除）。 + */ + private String deleteStrategy; + + /** + * 单次目标删除比例保护阈值，范围 0 至 100。 + */ + private Integer deleteGuardPercent; + + /** + * 完整性校验方式（SIZE、ETAG、SHA256）。 + */ + private String verifyMode; + + /** + * 最大并发传输数。 + */ + private Integer maxConcurrency; + + /** + * 带宽上限，单位 KB/s；0 表示不限速。 + */ + private Long bandwidthLimitKbps; + + /** + * 状态（0 正常、1 停用）。 + */ + private String status; + + /** + * 最近一次运行时间。 + */ + private LocalDateTime lastRunTime; + + /** + * 下一次计划运行时间。 + */ + private LocalDateTime nextRunTime; + + /** + * 删除标志（0 存在、1 删除）。 + */ + @TableLogic + private String delFlag; + + /** + * 备注。 + */ + private String remark; + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncTransferPart.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncTransferPart.java new file mode 100644 index 000000000..8a3ef0e3c --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/SyncTransferPart.java @@ -0,0 +1,38 @@ +package org.dromara.sync.domain; + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.dromara.common.mybatis.core.domain.BaseEntity; + +import java.io.Serial; +import java.time.LocalDateTime; + +/** + * 大文件分片传输断点 sync_transfer_part。 + */ +@Data +@EqualsAndHashCode(callSuper = true) +@TableName("sync_transfer_part") +public class SyncTransferPart extends BaseEntity { + + @Serial + private static final long serialVersionUID = 1L; + + @TableId("part_id") + private Long partId; + private Long jobItemId; + private String uploadId; + private Integer partNumber; + private Long partOffset; + private Long partSize; + private Long transferredBytes; + private String partEtag; + private String checksumSha256; + private String status; + private Integer retryCount; + private LocalDateTime startTime; + private LocalDateTime finishTime; + private String errorMessage; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncConnectionBo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncConnectionBo.java new file mode 100644 index 000000000..9e87cea18 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncConnectionBo.java @@ -0,0 +1,105 @@ +package org.dromara.sync.domain.bo; + +import io.github.linpeilie.annotations.AutoMapper; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; +import lombok.Data; +import org.dromara.common.core.validate.AddGroup; +import org.dromara.common.core.validate.EditGroup; +import org.dromara.common.json.validate.JsonPattern; +import org.dromara.common.json.validate.JsonType; +import org.dromara.sync.domain.SyncConnection; + +import java.io.Serial; +import java.io.Serializable; + +/** + * 同步连接业务对象 sync_connection。 + * + * @author Codex + * @date 2026-09-02 + */ +@Data +@AutoMapper(target = SyncConnection.class, reverseConvertGenerate = false) +public class SyncConnectionBo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + /** + * 连接主键。 + */ + @NotNull(message = "连接主键不能为空", groups = EditGroup.class) + private Long connectionId; + + /** + * 连接名称。 + */ + @NotBlank(message = "连接名称不能为空", groups = {AddGroup.class, EditGroup.class}) + private String connectionName; + + /** + * 连接角色（SOURCE 源端、TARGET 目标端）。 + */ + @NotBlank(message = "连接角色不能为空", groups = {AddGroup.class, EditGroup.class}) + @Pattern(regexp = "SOURCE|TARGET", message = "连接角色只支持 SOURCE 或 TARGET", + groups = {AddGroup.class, EditGroup.class}) + private String connectionRole; + + /** + * 连接类型（DINGTALK、S3、ALIYUN_OSS）。 + */ + @NotBlank(message = "连接类型不能为空", groups = {AddGroup.class, EditGroup.class}) + @Pattern(regexp = "DINGTALK|S3|ALIYUN_OSS", message = "连接类型不受支持", + groups = {AddGroup.class, EditGroup.class}) + private String connectionType; + + /** + * 服务端点。 + */ + private String endpoint; + + /** + * 服务区域。 + */ + private String region; + + /** + * 存储桶名称。 + */ + private String bucketName; + + /** + * 目标基础路径。 + */ + private String basePath; + + /** + * 非敏感扩展配置 JSON。 + */ + @JsonPattern(type = JsonType.OBJECT, message = "扩展配置必须是 JSON 对象", + groups = {AddGroup.class, EditGroup.class}) + private String configJson; + + /** + * 敏感凭证配置 JSON；修改时留空表示保留原值。 + */ + @JsonPattern(type = JsonType.OBJECT, message = "敏感配置必须是 JSON 对象", + groups = {AddGroup.class, EditGroup.class}) + private String secretJson; + + /** + * 状态（0 正常、1 停用）。 + */ + @NotBlank(message = "连接状态不能为空", groups = {AddGroup.class, EditGroup.class}) + @Pattern(regexp = "[01]", message = "连接状态只支持 0 或 1", + groups = {AddGroup.class, EditGroup.class}) + private String status; + + /** + * 备注。 + */ + private String remark; + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncJobBo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncJobBo.java new file mode 100644 index 000000000..ac84071af --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncJobBo.java @@ -0,0 +1,24 @@ +package org.dromara.sync.domain.bo; + +import lombok.Data; + +import java.io.Serial; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +/** + * 同步任务查询对象。 + */ +@Data +public class SyncJobBo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private Long planId; + private String triggerType; + private String runType; + private String status; + private Map params = new HashMap<>(); +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncJobItemBo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncJobItemBo.java new file mode 100644 index 000000000..062bf8ce0 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncJobItemBo.java @@ -0,0 +1,22 @@ +package org.dromara.sync.domain.bo; + +import lombok.Data; + +import java.io.Serial; +import java.io.Serializable; + +/** + * 同步任务明细查询对象。 + */ +@Data +public class SyncJobItemBo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private Long jobId; + private String sourcePath; + private String objectType; + private String actionType; + private String status; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncObjectBo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncObjectBo.java new file mode 100644 index 000000000..29f133ceb --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncObjectBo.java @@ -0,0 +1,23 @@ +package org.dromara.sync.domain.bo; + +import lombok.Data; + +import java.io.Serial; +import java.io.Serializable; + +/** + * 同步对象清单查询对象。 + */ +@Data +public class SyncObjectBo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private Long planId; + private String sourceObjectId; + private String sourcePath; + private String objectType; + private String syncStatus; + private String sourceDeleted; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncPlanBo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncPlanBo.java new file mode 100644 index 000000000..ca79d9d89 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/bo/SyncPlanBo.java @@ -0,0 +1,165 @@ +package org.dromara.sync.domain.bo; + +import io.github.linpeilie.annotations.AutoMapper; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; +import jakarta.validation.constraints.PositiveOrZero; +import jakarta.validation.constraints.Size; +import lombok.Data; +import org.dromara.common.core.validate.AddGroup; +import org.dromara.common.core.validate.EditGroup; +import org.dromara.sync.domain.SyncPlan; + +import java.io.Serial; +import java.io.Serializable; +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; + +/** + * 同步计划业务对象 sync_plan。 + * + * @author Codex + * @date 2026-09-02 + */ +@Data +@AutoMapper(target = SyncPlan.class, reverseConvertGenerate = false) +public class SyncPlanBo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + /** + * 计划主键。 + */ + @NotNull(message = "计划主键不能为空", groups = EditGroup.class) + private Long planId; + + /** + * 计划名称。 + */ + @NotBlank(message = "计划名称不能为空", groups = {AddGroup.class, EditGroup.class}) + @Size(max = 100, message = "计划名称不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class}) + private String planName; + + /** + * 源端连接主键。 + */ + @NotNull(message = "源端连接不能为空", groups = {AddGroup.class, EditGroup.class}) + private Long sourceConnectionId; + + /** + * 目标端连接主键。 + */ + @NotNull(message = "目标端连接不能为空", groups = {AddGroup.class, EditGroup.class}) + private Long targetConnectionId; + + /** + * 源端同步根路径或对象标识。 + */ + @Size(max = 1024, message = "源端同步根路径不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class}) + private String sourceRoot; + + /** + * 目标端对象键前缀。 + */ + @Size(max = 1024, message = "目标端对象键前缀不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class}) + private String targetPrefix; + + /** + * 同步模式（FULL、INCREMENTAL）。 + */ + @NotBlank(message = "同步模式不能为空", groups = {AddGroup.class, EditGroup.class}) + @Pattern(regexp = "FULL|INCREMENTAL", message = "同步模式不正确", groups = {AddGroup.class, EditGroup.class}) + private String syncMode; + + /** + * 调度类型（MANUAL、CRON）。 + */ + @NotBlank(message = "调度类型不能为空", groups = {AddGroup.class, EditGroup.class}) + @Pattern(regexp = "MANUAL|CRON", message = "调度类型不正确", groups = {AddGroup.class, EditGroup.class}) + private String scheduleType; + + /** + * CRON 表达式。 + */ + @Size(max = 128, message = "CRON表达式不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class}) + private String cronExpression; + + /** + * 冲突处理策略（OVERWRITE、SKIP、KEEP_BOTH）。 + */ + @NotBlank(message = "冲突处理策略不能为空", groups = {AddGroup.class, EditGroup.class}) + @Pattern(regexp = "OVERWRITE|SKIP|KEEP_BOTH", message = "冲突处理策略不正确", + groups = {AddGroup.class, EditGroup.class}) + private String conflictStrategy; + + /** + * 源端删除处理策略（KEEP、MARK、DELETE）。 + */ + @NotBlank(message = "删除处理策略不能为空", groups = {AddGroup.class, EditGroup.class}) + @Pattern(regexp = "KEEP|MARK|DELETE", message = "删除处理策略不正确", groups = {AddGroup.class, EditGroup.class}) + private String deleteStrategy; + + /** + * 单次目标删除比例保护阈值，100 表示允许一次删除全部对象。 + */ + @NotNull(message = "删除保护阈值不能为空", groups = {AddGroup.class, EditGroup.class}) + @Min(value = 0, message = "删除保护阈值不能小于{value}", groups = {AddGroup.class, EditGroup.class}) + @Max(value = 100, message = "删除保护阈值不能大于{value}", groups = {AddGroup.class, EditGroup.class}) + private Integer deleteGuardPercent; + + /** + * 完整性校验方式（SIZE、ETAG、SHA256）。 + */ + @NotBlank(message = "校验方式不能为空", groups = {AddGroup.class, EditGroup.class}) + @Pattern(regexp = "SIZE|ETAG|SHA256", message = "校验方式不正确", groups = {AddGroup.class, EditGroup.class}) + private String verifyMode; + + /** + * 最大并发传输数，范围为 1 至 100。 + */ + @NotNull(message = "最大并发数不能为空", groups = {AddGroup.class, EditGroup.class}) + @Min(value = 1, message = "最大并发数不能小于{value}", groups = {AddGroup.class, EditGroup.class}) + @Max(value = 100, message = "最大并发数不能大于{value}", groups = {AddGroup.class, EditGroup.class}) + private Integer maxConcurrency; + + /** + * 带宽上限，单位 KB/s；0 表示不限速。 + */ + @NotNull(message = "带宽上限不能为空", groups = {AddGroup.class, EditGroup.class}) + @PositiveOrZero(message = "带宽上限不能小于0", groups = {AddGroup.class, EditGroup.class}) + private Long bandwidthLimitKbps; + + /** + * 状态（0 正常、1 停用）。 + */ + @NotBlank(message = "状态不能为空", groups = {AddGroup.class, EditGroup.class}) + @Pattern(regexp = "0|1", message = "状态值不正确", groups = {AddGroup.class, EditGroup.class}) + private String status; + + /** + * 最近一次运行时间。 + */ + private LocalDateTime lastRunTime; + + /** + * 下一次计划运行时间。 + */ + private LocalDateTime nextRunTime; + + /** + * 备注。 + */ + @Size(max = 500, message = "备注不能超过{max}个字符", groups = {AddGroup.class, EditGroup.class}) + private String remark; + + /** + * 扩展查询参数。 + */ + private Map params = new HashMap<>(); + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/DwsAuthSessionVo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/DwsAuthSessionVo.java new file mode 100644 index 000000000..2efbd2bfe --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/DwsAuthSessionVo.java @@ -0,0 +1,48 @@ +package org.dromara.sync.domain.vo; + +import java.io.Serial; +import java.io.Serializable; + +/** + * 钉钉 Web 设备授权会话视图。 + * + *

该对象只包含一次性授权链接、验证码和脱敏身份信息，绝不包含 + * access token、refresh token、device code 或应用密钥。

+ * + * @param sessionId 授权会话 ID + * @param connectionId 同步连接 ID + * @param status 会话状态 + * @param verificationUri 用户输入验证码的授权地址 + * @param verificationUriComplete 可直接打开或生成二维码的完整授权地址 + * @param userCode 一次性用户验证码 + * @param expiresAt 会话过期时间（Unix 毫秒） + * @param remainingSeconds 距离过期的秒数 + * @param pollInterval 前端建议轮询间隔（秒） + * @param corpId 授权后的组织 ID + * @param corpName 授权后的组织名称 + * @param userId 授权后的钉钉用户 ID + * @param userName 授权后的用户名称 + * @param profile DWS 稳定 profile（corpId:userId） + * @param message 面向用户的脱敏提示 + */ +public record DwsAuthSessionVo( + String sessionId, + Long connectionId, + String status, + String verificationUri, + String verificationUriComplete, + String userCode, + Long expiresAt, + long remainingSeconds, + int pollInterval, + String corpId, + String corpName, + String userId, + String userName, + String profile, + String message +) implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncCheckpointVo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncCheckpointVo.java new file mode 100644 index 000000000..955444120 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncCheckpointVo.java @@ -0,0 +1,29 @@ +package org.dromara.sync.domain.vo; + +import io.github.linpeilie.annotations.AutoMapper; +import lombok.Data; +import org.dromara.sync.domain.SyncCheckpoint; + +import java.io.Serial; +import java.io.Serializable; +import java.time.LocalDateTime; + +/** + * 同步断点视图对象。 + */ +@Data +@AutoMapper(target = SyncCheckpoint.class) +public class SyncCheckpointVo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private Long checkpointId; + private Long planId; + private String checkpointType; + private String checkpointKey; + private String checkpointValue; + private LocalDateTime watermarkTime; + private Long lastJobId; + private Long version; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncConnectionVo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncConnectionVo.java new file mode 100644 index 000000000..07790e8f0 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncConnectionVo.java @@ -0,0 +1,106 @@ +package org.dromara.sync.domain.vo; + +import io.github.linpeilie.annotations.AutoMapper; +import lombok.Data; +import org.dromara.sync.domain.SyncConnection; + +import java.io.Serial; +import java.io.Serializable; +import java.time.LocalDateTime; + +/** + * 同步连接视图对象 sync_connection。 + * + *

敏感凭证字段不在该视图对象中声明，任何查询接口均不会返回凭证内容。

+ * + * @author Codex + * @date 2026-09-02 + */ +@Data +@AutoMapper(target = SyncConnection.class) +public class SyncConnectionVo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + /** + * 连接主键。 + */ + private Long connectionId; + + /** + * 连接名称。 + */ + private String connectionName; + + /** + * 连接角色（SOURCE 源端、TARGET 目标端）。 + */ + private String connectionRole; + + /** + * 连接类型（DINGTALK、S3、ALIYUN_OSS）。 + */ + private String connectionType; + + /** + * 服务端点。 + */ + private String endpoint; + + /** + * 服务区域。 + */ + private String region; + + /** + * 存储桶名称。 + */ + private String bucketName; + + /** + * 目标基础路径。 + */ + private String basePath; + + /** + * 非敏感扩展配置 JSON。 + */ + private String configJson; + + /** + * 状态（0 正常、1 停用）。 + */ + private String status; + + /** + * 备注。 + */ + private String remark; + + /** + * 创建部门。 + */ + private Long createDept; + + /** + * 创建者。 + */ + private Long createBy; + + /** + * 创建时间。 + */ + private LocalDateTime createTime; + + /** + * 更新者。 + */ + private Long updateBy; + + /** + * 更新时间。 + */ + private LocalDateTime updateTime; + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncJobItemVo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncJobItemVo.java new file mode 100644 index 000000000..1059f62fa --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncJobItemVo.java @@ -0,0 +1,42 @@ +package org.dromara.sync.domain.vo; + +import io.github.linpeilie.annotations.AutoMapper; +import lombok.Data; +import org.dromara.sync.domain.SyncJobItem; + +import java.io.Serial; +import java.io.Serializable; +import java.time.LocalDateTime; + +/** + * 同步任务明细视图对象。 + */ +@Data +@AutoMapper(target = SyncJobItem.class) +public class SyncJobItemVo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private Long itemId; + private Long jobId; + private String sourceObjectId; + private String parentObjectId; + private String sourcePath; + private String targetKey; + private String objectType; + private String actionType; + private String status; + private Long size; + private Long transferredBytes; + private String versionToken; + private String sourceEtag; + private String sourceSha256; + private String targetEtag; + private String targetVersionId; + private Integer retryCount; + private LocalDateTime startTime; + private LocalDateTime finishTime; + private String errorCode; + private String errorMessage; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncJobVo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncJobVo.java new file mode 100644 index 000000000..c2e206fc0 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncJobVo.java @@ -0,0 +1,40 @@ +package org.dromara.sync.domain.vo; + +import io.github.linpeilie.annotations.AutoMapper; +import lombok.Data; +import org.dromara.sync.domain.SyncJob; + +import java.io.Serial; +import java.io.Serializable; +import java.time.LocalDateTime; + +/** + * 同步任务视图对象。 + */ +@Data +@AutoMapper(target = SyncJob.class) +public class SyncJobVo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private Long jobId; + private Long planId; + private String triggerType; + private String runType; + private String status; + private String checkpointBefore; + private String checkpointAfter; + private Long totalCount; + private Long processedCount; + private Long successCount; + private Long failedCount; + private Long skippedCount; + private Long deletedCount; + private Long totalBytes; + private Long transferredBytes; + private LocalDateTime startTime; + private LocalDateTime finishTime; + private String errorMessage; + private LocalDateTime createTime; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncObjectVo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncObjectVo.java new file mode 100644 index 000000000..00c06724d --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncObjectVo.java @@ -0,0 +1,47 @@ +package org.dromara.sync.domain.vo; + +import io.github.linpeilie.annotations.AutoMapper; +import lombok.Data; +import org.dromara.sync.domain.SyncObject; + +import java.io.Serial; +import java.io.Serializable; +import java.time.LocalDateTime; + +/** + * 同步对象清单视图对象。 + */ +@Data +@AutoMapper(target = SyncObject.class) +public class SyncObjectVo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private Long objectId; + private Long planId; + private String sourceObjectId; + private String parentObjectId; + private String sourcePath; + private String objectName; + private String objectType; + private Long size; + private LocalDateTime modifiedTime; + private String versionToken; + private String sourceEtag; + private String sourceSha256; + private String contentType; + private String metadataJson; + private String targetKey; + private String targetVersionId; + private String targetEtag; + private String syncStatus; + private String sourceDeleted; + private Long firstSeenJobId; + private Long lastSeenJobId; + private Long lastSyncJobId; + private LocalDateTime lastSyncTime; + private String lastErrorMessage; + private LocalDateTime createTime; + private LocalDateTime updateTime; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncPlanVo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncPlanVo.java new file mode 100644 index 000000000..7d13e85e4 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncPlanVo.java @@ -0,0 +1,144 @@ +package org.dromara.sync.domain.vo; + +import io.github.linpeilie.annotations.AutoMapper; +import lombok.Data; +import org.dromara.sync.domain.SyncPlan; + +import java.io.Serial; +import java.io.Serializable; +import java.time.LocalDateTime; + +/** + * 同步计划视图对象 sync_plan。 + * + * @author Codex + * @date 2026-09-02 + */ +@Data +@AutoMapper(target = SyncPlan.class) +public class SyncPlanVo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + /** + * 计划主键。 + */ + private Long planId; + + /** + * 计划名称。 + */ + private String planName; + + /** + * 源端连接主键。 + */ + private Long sourceConnectionId; + + /** + * 目标端连接主键。 + */ + private Long targetConnectionId; + + /** + * 源端同步根路径或对象标识。 + */ + private String sourceRoot; + + /** + * 目标端对象键前缀。 + */ + private String targetPrefix; + + /** + * 同步模式（FULL、INCREMENTAL）。 + */ + private String syncMode; + + /** + * 调度类型（MANUAL、CRON）。 + */ + private String scheduleType; + + /** + * CRON 表达式。 + */ + private String cronExpression; + + /** + * 冲突处理策略（OVERWRITE、SKIP、KEEP_BOTH）。 + */ + private String conflictStrategy; + + /** + * 源端删除处理策略（KEEP、MARK、DELETE）。 + */ + private String deleteStrategy; + + /** + * 单次目标删除比例保护阈值。 + */ + private Integer deleteGuardPercent; + + /** + * 完整性校验方式（SIZE、ETAG、SHA256）。 + */ + private String verifyMode; + + /** + * 最大并发传输数。 + */ + private Integer maxConcurrency; + + /** + * 带宽上限，单位 KB/s；0 表示不限速。 + */ + private Long bandwidthLimitKbps; + + /** + * 状态（0 正常、1 停用）。 + */ + private String status; + + /** + * 最近一次运行时间。 + */ + private LocalDateTime lastRunTime; + + /** + * 下一次计划运行时间。 + */ + private LocalDateTime nextRunTime; + + /** + * 备注。 + */ + private String remark; + + /** + * 创建部门。 + */ + private Long createDept; + + /** + * 创建者。 + */ + private Long createBy; + + /** + * 创建时间。 + */ + private LocalDateTime createTime; + + /** + * 更新者。 + */ + private Long updateBy; + + /** + * 更新时间。 + */ + private LocalDateTime updateTime; + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncTransferPartVo.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncTransferPartVo.java new file mode 100644 index 000000000..0cddf5380 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/domain/vo/SyncTransferPartVo.java @@ -0,0 +1,35 @@ +package org.dromara.sync.domain.vo; + +import io.github.linpeilie.annotations.AutoMapper; +import lombok.Data; +import org.dromara.sync.domain.SyncTransferPart; + +import java.io.Serial; +import java.io.Serializable; +import java.time.LocalDateTime; + +/** + * 传输分片视图对象。 + */ +@Data +@AutoMapper(target = SyncTransferPart.class) +public class SyncTransferPartVo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private Long partId; + private Long jobItemId; + private String uploadId; + private Integer partNumber; + private Long partOffset; + private Long partSize; + private Long transferredBytes; + private String partEtag; + private String checksumSha256; + private String status; + private Integer retryCount; + private LocalDateTime startTime; + private LocalDateTime finishTime; + private String errorMessage; +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/job/SyncPlanJobExecutor.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/job/SyncPlanJobExecutor.java new file mode 100644 index 000000000..2ebb4c217 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/job/SyncPlanJobExecutor.java @@ -0,0 +1,31 @@ +package org.dromara.sync.job; + +import cn.hutool.core.convert.Convert; +import com.aizuda.snailjob.client.job.core.annotation.JobExecutor; +import com.aizuda.snailjob.client.job.core.dto.JobArgs; +import com.aizuda.snailjob.model.dto.ExecuteResult; +import lombok.RequiredArgsConstructor; +import org.dromara.sync.service.ISyncJobService; +import org.springframework.stereotype.Component; + +/** + * SnailJob 同步计划执行入口。 + * + *

执行器名称为 {@code syncPlanJobExecutor}，任务参数填写同步计划 ID。

+ */ +@Component +@RequiredArgsConstructor +@JobExecutor(name = "syncPlanJobExecutor") +public class SyncPlanJobExecutor { + + private final ISyncJobService jobService; + + public ExecuteResult jobExecute(JobArgs jobArgs) { + Long planId = Convert.toLong(jobArgs.getJobParams()); + if (planId == null) { + return ExecuteResult.failure("任务参数必须是同步计划 ID"); + } + Long jobId = jobService.startPlan(planId, "SCHEDULE", null); + return ExecuteResult.success("同步任务已提交，jobId=" + jobId); + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/job/SyncPlanScheduler.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/job/SyncPlanScheduler.java new file mode 100644 index 000000000..ad715cda5 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/job/SyncPlanScheduler.java @@ -0,0 +1,75 @@ +package org.dromara.sync.job; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.dromara.sync.constant.SyncConstants; +import org.dromara.sync.domain.SyncPlan; +import org.dromara.sync.mapper.SyncPlanMapper; +import org.dromara.sync.service.ISyncJobService; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.scheduling.support.CronExpression; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.List; + +/** + * 从同步计划表领取到期的 Cron 计划。 + * + *

通过 next_run_time 条件更新完成多实例抢占，同一触发时刻只有一个应用实例能够提交任务。

+ */ +@Slf4j +@Component +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "sync.scheduler", name = "enabled", havingValue = "true", matchIfMissing = true) +public class SyncPlanScheduler { + + private final SyncPlanMapper planMapper; + private final ISyncJobService jobService; + + @Scheduled(fixedDelayString = "${sync.scheduler.poll-interval-ms:30000}") + public void dispatchDuePlans() { + LocalDateTime now = LocalDateTime.now(); + List plans = planMapper.selectList(Wrappers.lambdaQuery() + .eq(SyncPlan::getStatus, SyncConstants.STATUS_NORMAL) + .eq(SyncPlan::getScheduleType, "CRON") + .isNotNull(SyncPlan::getNextRunTime) + .le(SyncPlan::getNextRunTime, now) + .orderByAsc(SyncPlan::getNextRunTime) + .last("limit 100")); + for (SyncPlan plan : plans) { + dispatch(plan, now); + } + } + + private void dispatch(SyncPlan plan, LocalDateTime now) { + try { + LocalDateTime nextRunTime = nextRunTime(plan.getCronExpression(), now); + boolean claimed = planMapper.lambda() + .set(SyncPlan::getNextRunTime, nextRunTime) + .eq(SyncPlan::getPlanId, plan.getPlanId()) + .eq(SyncPlan::getNextRunTime, plan.getNextRunTime()) + .eq(SyncPlan::getCronExpression, plan.getCronExpression()) + .eq(SyncPlan::getScheduleType, "CRON") + .eq(SyncPlan::getStatus, SyncConstants.STATUS_NORMAL) + .update(); + if (!claimed) { + return; + } + jobService.startPlan(plan.getPlanId(), "SCHEDULE", null); + } catch (Exception e) { + log.warn("到期同步计划提交失败，planId={}，原因={}", plan.getPlanId(), e.getMessage()); + } + } + + private LocalDateTime nextRunTime(String cronExpression, LocalDateTime now) { + ZonedDateTime next = CronExpression.parse(cronExpression).next(now.atZone(java.time.ZoneId.systemDefault())); + if (next == null) { + throw new IllegalArgumentException("Cron 表达式没有下一次触发时间"); + } + return next.toLocalDateTime(); + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncCheckpointMapper.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncCheckpointMapper.java new file mode 100644 index 000000000..3f0e36537 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncCheckpointMapper.java @@ -0,0 +1,11 @@ +package org.dromara.sync.mapper; + +import org.dromara.common.mybatis.core.mapper.BaseMapperPlus; +import org.dromara.sync.domain.SyncCheckpoint; +import org.dromara.sync.domain.vo.SyncCheckpointVo; + +/** + * 同步断点 Mapper。 + */ +public interface SyncCheckpointMapper extends BaseMapperPlus { +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncConnectionMapper.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncConnectionMapper.java new file mode 100644 index 000000000..8b1a3365b --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncConnectionMapper.java @@ -0,0 +1,15 @@ +package org.dromara.sync.mapper; + +import org.dromara.common.mybatis.core.mapper.BaseMapperPlus; +import org.dromara.sync.domain.SyncConnection; +import org.dromara.sync.domain.vo.SyncConnectionVo; + +/** + * 同步连接 Mapper 接口。 + * + * @author Codex + * @date 2026-09-02 + */ +public interface SyncConnectionMapper extends BaseMapperPlus { + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncJobItemMapper.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncJobItemMapper.java new file mode 100644 index 000000000..2184eb3e3 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncJobItemMapper.java @@ -0,0 +1,11 @@ +package org.dromara.sync.mapper; + +import org.dromara.common.mybatis.core.mapper.BaseMapperPlus; +import org.dromara.sync.domain.SyncJobItem; +import org.dromara.sync.domain.vo.SyncJobItemVo; + +/** + * 同步任务明细 Mapper。 + */ +public interface SyncJobItemMapper extends BaseMapperPlus { +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncJobMapper.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncJobMapper.java new file mode 100644 index 000000000..1b5b36ae6 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncJobMapper.java @@ -0,0 +1,11 @@ +package org.dromara.sync.mapper; + +import org.dromara.common.mybatis.core.mapper.BaseMapperPlus; +import org.dromara.sync.domain.SyncJob; +import org.dromara.sync.domain.vo.SyncJobVo; + +/** + * 同步任务 Mapper。 + */ +public interface SyncJobMapper extends BaseMapperPlus { +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncObjectMapper.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncObjectMapper.java new file mode 100644 index 000000000..49e97c171 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncObjectMapper.java @@ -0,0 +1,11 @@ +package org.dromara.sync.mapper; + +import org.dromara.common.mybatis.core.mapper.BaseMapperPlus; +import org.dromara.sync.domain.SyncObject; +import org.dromara.sync.domain.vo.SyncObjectVo; + +/** + * 同步对象清单 Mapper。 + */ +public interface SyncObjectMapper extends BaseMapperPlus { +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncPlanMapper.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncPlanMapper.java new file mode 100644 index 000000000..dc7d9c85c --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncPlanMapper.java @@ -0,0 +1,15 @@ +package org.dromara.sync.mapper; + +import org.dromara.common.mybatis.core.mapper.BaseMapperPlus; +import org.dromara.sync.domain.SyncPlan; +import org.dromara.sync.domain.vo.SyncPlanVo; + +/** + * 同步计划 Mapper 接口。 + * + * @author Codex + * @date 2026-09-02 + */ +public interface SyncPlanMapper extends BaseMapperPlus { + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncTransferPartMapper.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncTransferPartMapper.java new file mode 100644 index 000000000..e97649076 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/mapper/SyncTransferPartMapper.java @@ -0,0 +1,11 @@ +package org.dromara.sync.mapper; + +import org.dromara.common.mybatis.core.mapper.BaseMapperPlus; +import org.dromara.sync.domain.SyncTransferPart; +import org.dromara.sync.domain.vo.SyncTransferPartVo; + +/** + * 同步传输分片 Mapper。 + */ +public interface SyncTransferPartMapper extends BaseMapperPlus { +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncConnectionService.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncConnectionService.java new file mode 100644 index 000000000..c7ab116b9 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncConnectionService.java @@ -0,0 +1,109 @@ +package org.dromara.sync.service; + +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.sync.connector.model.ConnectorTestResult; +import org.dromara.sync.domain.bo.SyncConnectionBo; +import org.dromara.sync.domain.vo.DwsAuthSessionVo; +import org.dromara.sync.domain.vo.SyncConnectionVo; + +import java.util.Collection; +import java.util.List; + +/** + * 同步连接 Service 接口。 + * + * @author Codex + * @date 2026-09-02 + */ +public interface ISyncConnectionService { + + /** + * 根据主键查询同步连接。 + * + * @param connectionId 连接主键 + * @return 同步连接详情 + */ + SyncConnectionVo queryById(Long connectionId); + + /** + * 分页查询同步连接列表。 + * + * @param bo 查询条件 + * @param pageQuery 分页参数 + * @return 同步连接分页列表 + */ + PageResult queryPageList(SyncConnectionBo bo, PageQuery pageQuery); + + /** + * 查询符合条件的同步连接列表。 + * + * @param bo 查询条件 + * @return 同步连接列表 + */ + List queryList(SyncConnectionBo bo); + + /** + * 新增同步连接。 + * + * @param bo 同步连接业务对象 + * @return 是否新增成功 + */ + Boolean insertByBo(SyncConnectionBo bo); + + /** + * 修改同步连接。 + * + * @param bo 同步连接业务对象 + * @return 是否修改成功 + */ + Boolean updateByBo(SyncConnectionBo bo); + + /** + * 校验并批量删除同步连接。 + * + * @param ids 待删除的连接主键集合 + * @param isValid 是否执行删除前校验 + * @return 是否删除成功 + */ + Boolean deleteWithValidByIds(Collection ids, Boolean isValid); + + /** + * 校验配置并调用对应连接器执行连通性测试。 + * + * @param connectionId 连接主键 + * @return 连通性测试结果 + */ + ConnectorTestResult testConnection(Long connectionId); + + /** + * 启动钉钉 Web 设备授权。 + * + * @param connectionId 钉钉连接主键 + * @param ownerId 发起授权的系统用户 + * @param expectedCorpId 可选的组织 ID，用于防止误授权到其他组织 + * @return 授权会话快照 + */ + DwsAuthSessionVo startDingTalkAuth(Long connectionId, Long ownerId, String expectedCorpId); + + /** + * 查询钉钉 Web 设备授权状态。 + * + * @param connectionId 钉钉连接主键 + * @param sessionId 授权会话 ID + * @param ownerId 发起授权的系统用户 + * @return 授权会话快照 + */ + DwsAuthSessionVo getDingTalkAuth(Long connectionId, String sessionId, Long ownerId); + + /** + * 取消钉钉 Web 设备授权。 + * + * @param connectionId 钉钉连接主键 + * @param sessionId 授权会话 ID + * @param ownerId 发起授权的系统用户 + * @return 取消后的会话快照 + */ + DwsAuthSessionVo cancelDingTalkAuth(Long connectionId, String sessionId, Long ownerId); + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncJobService.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncJobService.java new file mode 100644 index 000000000..2b62e2123 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncJobService.java @@ -0,0 +1,30 @@ +package org.dromara.sync.service; + +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.sync.domain.bo.SyncJobBo; +import org.dromara.sync.domain.bo.SyncJobItemBo; +import org.dromara.sync.domain.vo.SyncJobItemVo; +import org.dromara.sync.domain.vo.SyncJobVo; + +import java.util.Collection; + +/** + * 同步任务服务。 + */ +public interface ISyncJobService { + + SyncJobVo queryById(Long jobId); + + PageResult queryPageList(SyncJobBo bo, PageQuery pageQuery); + + PageResult queryItemPageList(SyncJobItemBo bo, PageQuery pageQuery); + + Long startPlan(Long planId, String triggerType, String runType); + + Long retry(Long jobId); + + Boolean cancel(Long jobId); + + Boolean deleteWithValidByIds(Collection ids, Boolean isValid); +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncObjectService.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncObjectService.java new file mode 100644 index 000000000..68999022e --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncObjectService.java @@ -0,0 +1,16 @@ +package org.dromara.sync.service; + +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.sync.domain.bo.SyncObjectBo; +import org.dromara.sync.domain.vo.SyncObjectVo; + +/** + * 同步对象清单服务。 + */ +public interface ISyncObjectService { + + SyncObjectVo queryById(Long objectId); + + PageResult queryPageList(SyncObjectBo bo, PageQuery pageQuery); +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncPlanService.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncPlanService.java new file mode 100644 index 000000000..f00f0d806 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/ISyncPlanService.java @@ -0,0 +1,77 @@ +package org.dromara.sync.service; + +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.sync.domain.bo.SyncPlanBo; +import org.dromara.sync.domain.vo.SyncPlanVo; + +import java.util.Collection; +import java.util.List; + +/** + * 同步计划 Service 接口。 + * + * @author Codex + * @date 2026-09-02 + */ +public interface ISyncPlanService { + + /** + * 根据主键查询同步计划。 + * + * @param planId 计划主键 + * @return 同步计划详情 + */ + SyncPlanVo queryById(Long planId); + + /** + * 分页查询同步计划列表。 + * + * @param bo 查询条件 + * @param pageQuery 分页参数 + * @return 同步计划分页列表 + */ + PageResult queryPageList(SyncPlanBo bo, PageQuery pageQuery); + + /** + * 查询符合条件的同步计划列表。 + * + * @param bo 查询条件 + * @return 同步计划列表 + */ + List queryList(SyncPlanBo bo); + + /** + * 校验计划名称是否唯一。 + * + * @param bo 同步计划 + * @return 名称未被占用返回 {@code true} + */ + boolean checkPlanNameUnique(SyncPlanBo bo); + + /** + * 新增同步计划。 + * + * @param bo 同步计划 + * @return 是否新增成功 + */ + Boolean insertByBo(SyncPlanBo bo); + + /** + * 修改同步计划。 + * + * @param bo 同步计划 + * @return 是否修改成功 + */ + Boolean updateByBo(SyncPlanBo bo); + + /** + * 校验并批量删除同步计划。 + * + * @param ids 计划主键集合 + * @param isValid 是否执行删除前业务校验 + * @return 是否删除成功 + */ + Boolean deleteWithValidByIds(Collection ids, Boolean isValid); + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncConnectionServiceImpl.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncConnectionServiceImpl.java new file mode 100644 index 000000000..b8b7cc753 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncConnectionServiceImpl.java @@ -0,0 +1,713 @@ +package org.dromara.sync.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import lombok.RequiredArgsConstructor; +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.core.exception.ServiceException; +import org.dromara.common.core.utils.MapstructUtils; +import org.dromara.common.core.utils.StringUtils; +import org.dromara.common.json.utils.JsonUtils; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.common.mybatis.core.query.QueryBuilder; +import org.dromara.common.satoken.utils.LoginHelper; +import org.dromara.sync.connector.ConnectorRegistry; +import org.dromara.sync.connector.dingtalk.DwsAuthIdentity; +import org.dromara.sync.connector.dingtalk.DwsAuthSessionManager; +import org.dromara.sync.connector.model.ConnectorTestResult; +import org.dromara.sync.constant.SyncConstants; +import org.dromara.sync.domain.SyncConnection; +import org.dromara.sync.domain.SyncJob; +import org.dromara.sync.domain.SyncPlan; +import org.dromara.sync.domain.bo.SyncConnectionBo; +import org.dromara.sync.domain.vo.DwsAuthSessionVo; +import org.dromara.sync.domain.vo.SyncConnectionVo; +import org.dromara.sync.mapper.SyncConnectionMapper; +import org.dromara.sync.mapper.SyncJobMapper; +import org.dromara.sync.mapper.SyncPlanMapper; +import org.dromara.sync.service.ISyncConnectionService; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; + +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 同步连接 Service 业务层处理。 + * + * @author Codex + * @date 2026-09-02 + */ +@RequiredArgsConstructor +@Service +public class SyncConnectionServiceImpl implements ISyncConnectionService { + + private static final Pattern DOWNLOAD_PART_SIZE_PATTERN = Pattern.compile("(?i)^([1-9]\\d*)(KB|MB|GB)$"); + private static final long ONE_MIB = 1024L * 1024L; + private static final long ONE_GIB = 1024L * 1024L * 1024L; + + private final SyncConnectionMapper connectionMapper; + private final SyncPlanMapper planMapper; + private final SyncJobMapper jobMapper; + private final ConnectorRegistry connectorRegistry; + private final DwsAuthSessionManager dwsAuthSessionManager; + + /** + * Serializes connection mutations with the asynchronous DWS profile + * callback. Without this small critical section an edit could pass the + * active-session check just before login starts and then overwrite the + * profile written by the callback (or vice versa). + */ + private final Object connectionMutationLock = new Object(); + + @Value("${mybatis-encryptor.enable:false}") + private boolean fieldEncryptionEnabled; + + @Value("${mybatis-encryptor.password:}") + private String fieldEncryptionPassword; + + /** + * 根据主键查询同步连接详情。 + * + * @param connectionId 连接主键 + * @return 同步连接详情 + */ + @Override + public SyncConnectionVo queryById(Long connectionId) { + ensureConnectionAccess(connectionId); + return connectionMapper.selectVoById(connectionId); + } + + /** + * 分页查询同步连接列表。 + * + * @param bo 查询条件 + * @param pageQuery 分页参数 + * @return 同步连接分页列表 + */ + @Override + public PageResult queryPageList(SyncConnectionBo bo, PageQuery pageQuery) { + LambdaQueryWrapper lqw = buildQueryWrapper(bo); + Page result = connectionMapper.selectVoPage(pageQuery.build(), lqw); + return PageResult.build(result.getRecords(), result.getTotal()); + } + + /** + * 查询符合条件的同步连接列表。 + * + * @param bo 查询条件 + * @return 同步连接列表 + */ + @Override + public List queryList(SyncConnectionBo bo) { + return connectionMapper.selectVoList(buildQueryWrapper(bo)); + } + + /** + * 构造同步连接动态查询条件。 + * + * @param bo 查询条件 + * @return 查询条件包装器 + */ + private LambdaQueryWrapper buildQueryWrapper(SyncConnectionBo bo) { + LambdaQueryWrapper wrapper = QueryBuilder.lambda(SyncConnection.class) + .eqIfPresent(SyncConnection::getConnectionId, bo.getConnectionId()) + .likeIfText(SyncConnection::getConnectionName, bo.getConnectionName()) + .eqIfText(SyncConnection::getConnectionRole, bo.getConnectionRole()) + .eqIfText(SyncConnection::getConnectionType, bo.getConnectionType()) + .likeIfText(SyncConnection::getEndpoint, bo.getEndpoint()) + .likeIfText(SyncConnection::getBucketName, bo.getBucketName()) + .eqIfText(SyncConnection::getStatus, bo.getStatus()) + .orderByAsc(SyncConnection::getConnectionId) + .build(); + // Connections contain credentials and are creator-owned resources. Do + // not rely on the menu permission alone for row-level isolation. + if (!LoginHelper.isSuperAdmin()) { + wrapper.eq(SyncConnection::getCreateBy, requireCurrentUserId()); + } + return wrapper; + } + + /** + * 新增同步连接。 + * + * @param bo 同步连接业务对象 + * @return 是否新增成功 + */ + @Override + public Boolean insertByBo(SyncConnectionBo bo) { + requireCurrentUserId(); + SyncConnection add = MapstructUtils.convert(bo, SyncConnection.class); + validEntityBeforeSave(add); + boolean flag; + try { + flag = connectionMapper.insert(add) > 0; + } catch (DuplicateKeyException e) { + throw new ServiceException("连接名称【{}】已存在", add.getConnectionName()); + } + if (flag) { + bo.setConnectionId(add.getConnectionId()); + } + return flag; + } + + /** + * 修改同步连接，未填写敏感配置时不更新数据库中的原凭证。 + * + * @param bo 同步连接业务对象 + * @return 是否修改成功 + */ + @Override + public Boolean updateByBo(SyncConnectionBo bo) { + synchronized (connectionMutationLock) { + SyncConnection current = connectionMapper.selectById(bo.getConnectionId()); + if (current == null) { + throw new ServiceException("同步连接不存在"); + } + ensureConnectionAccess(current); + if (hasActiveJob(current.getConnectionId())) { + throw new ServiceException("连接正在被同步任务使用，任务结束后才能修改"); + } + ensureNoDingTalkAuth(current.getConnectionId()); + SyncConnection update = MapstructUtils.convert(bo, SyncConnection.class); + boolean connectorChanged = !Objects.equals(current.getConnectionRole(), update.getConnectionRole()) + || !Objects.equals(current.getConnectionType(), update.getConnectionType()); + if (connectorChanged && isReferenced(update.getConnectionId())) { + throw new ServiceException("连接已被同步计划引用，不能修改连接角色或类型"); + } + if (connectorChanged && StringUtils.isBlank(update.getSecretJson())) { + throw new ServiceException("修改连接角色或类型时必须重新填写敏感凭证"); + } + preserveDingTalkProfile(current, update); + validEntityBeforeSave(update); + if (StringUtils.isBlank(update.getSecretJson())) { + update.setSecretJson(null); + } + try { + return connectionMapper.updateById(update) > 0; + } catch (DuplicateKeyException e) { + throw new ServiceException("连接名称【{}】已存在", update.getConnectionName()); + } + } + } + + /** + * 执行保存前的唯一性、枚举组合与 JSON 格式校验。 + * + * @param entity 待保存的同步连接 + */ + private void validEntityBeforeSave(SyncConnection entity) { + if (StringUtils.isNotBlank(entity.getSecretJson()) + && (!fieldEncryptionEnabled || StringUtils.isBlank(fieldEncryptionPassword))) { + throw new ServiceException("保存敏感凭证前必须启用 mybatis-encryptor 并配置 MYBATIS_ENCRYPTOR_PASSWORD"); + } + entity.setConnectionName(StringUtils.trim(entity.getConnectionName())); + entity.setConnectionRole(StringUtils.trim(entity.getConnectionRole())); + entity.setConnectionType(StringUtils.trim(entity.getConnectionType())); + entity.setStatus(StringUtils.trim(entity.getStatus())); + entity.setEndpoint(StringUtils.trim(entity.getEndpoint())); + entity.setRegion(StringUtils.trim(entity.getRegion())); + entity.setBucketName(StringUtils.trim(entity.getBucketName())); + entity.setBasePath(StringUtils.trim(entity.getBasePath())); + requireText(entity.getConnectionName(), "连接名称不能为空"); + validateLength("连接名称", entity.getConnectionName(), 100); + validateLength("服务端点", entity.getEndpoint(), 512); + validateLength("区域", entity.getRegion(), 64); + validateLength("存储桶名称", entity.getBucketName(), 255); + validateLength("基础路径", entity.getBasePath(), 1024); + validateLength("备注", entity.getRemark(), 500); + validateObjectKeyPrefix(entity.getBasePath(), "基础路径"); + validateRoleAndType(entity); + validateStatus(entity.getStatus()); + validateJsonObjectIfPresent(entity.getConfigJson(), "扩展配置"); + validateJsonObjectIfPresent(entity.getSecretJson(), "敏感配置"); + validateRequiredConfig(entity); + if (!isConnectionNameUnique(entity)) { + throw new ServiceException("连接名称【{}】已存在", entity.getConnectionName()); + } + } + + /** + * 校验连接名称唯一性，编辑时排除当前记录。 + * + * @param entity 同步连接 + * @return 名称唯一返回 {@code true} + */ + private boolean isConnectionNameUnique(SyncConnection entity) { + if (StringUtils.isBlank(entity.getConnectionName())) { + return true; + } + LambdaQueryWrapper lqw = QueryBuilder.lambda(SyncConnection.class) + .eq(SyncConnection::getConnectionName, entity.getConnectionName()) + .neIfPresent(SyncConnection::getConnectionId, entity.getConnectionId()) + .build(); + return !connectionMapper.exists(lqw); + } + + /** + * 校验连接角色与连接类型组合。 + * + * @param connection 同步连接 + */ + private void validateRoleAndType(SyncConnection connection) { + String role = connection.getConnectionRole(); + String type = connection.getConnectionType(); + if (SyncConstants.TYPE_DINGTALK.equals(type)) { + if (!SyncConstants.ROLE_SOURCE.equals(role)) { + throw new ServiceException("DINGTALK 连接只能配置为 SOURCE 角色"); + } + return; + } + if (SyncConstants.TYPE_S3.equals(type) || SyncConstants.TYPE_ALIYUN_OSS.equals(type)) { + if (!SyncConstants.ROLE_TARGET.equals(role)) { + throw new ServiceException("{} 连接只能配置为 TARGET 角色", type); + } + return; + } + throw new ServiceException("不支持的连接类型：{}", type); + } + + /** + * 校验连接状态取值。 + * + * @param status 连接状态 + */ + private void validateStatus(String status) { + if (!SyncConstants.STATUS_NORMAL.equals(status) && !SyncConstants.STATUS_DISABLED.equals(status)) { + throw new ServiceException("连接状态只支持 0 或 1"); + } + } + + private void validateRequiredConfig(SyncConnection connection) { + validateRequiredConfig(connection, false); + } + + /** + * 校验连接运行所需配置。 + * + * @param connection 连接 + * @param requireProfile 是否要求已经完成 DWS profile 配置 + */ + private void validateRequiredConfig(SyncConnection connection, boolean requireProfile) { + boolean credentialRequired = connection.getConnectionId() == null + || StringUtils.isNotBlank(connection.getSecretJson()); + Map secrets = credentialRequired + ? parseSecretJson(connection.getSecretJson()) : Map.of(); + if (SyncConstants.TYPE_DINGTALK.equals(connection.getConnectionType())) { + if (credentialRequired) { + requireSecret(secrets, "clientId", "钉钉连接 clientId 不能为空"); + requireSecret(secrets, "clientSecret", "钉钉连接 clientSecret 不能为空"); + } + Map config = parseOptionalJson(connection.getConfigJson()); + String profile = stringValue(config, "profile"); + if (requireProfile && StringUtils.isBlank(profile)) { + throw new ServiceException("钉钉连接尚未完成登录，请先在网页端完成设备授权"); + } + if (StringUtils.isNotBlank(profile) && !profile.matches("[^\\s:]+:[^\\s:]+")) { + throw new ServiceException("钉钉 profile 必须使用 corpId:userId 稳定标识"); + } + if (config.containsKey("configDir")) { + throw new ServiceException("钉钉 configDir 由服务端按连接隔离分配，不能在连接配置中指定"); + } + Object spaceType = config.get("spaceType"); + if (spaceType != null && !List.of("orgSpace", "mySpace").contains(String.valueOf(spaceType))) { + throw new ServiceException("钉钉 spaceType 只支持 orgSpace 或 mySpace"); + } + validateDownloadOptions(config); + return; + } + requireText(connection.getEndpoint(), "目标连接 endpoint 不能为空"); + requireText(connection.getBucketName(), "目标连接 bucketName 不能为空"); + if (credentialRequired) { + requireSecret(secrets, "accessKey", "目标连接 accessKey 不能为空"); + requireSecret(secrets, "secretKey", "目标连接 secretKey 不能为空"); + } + } + + /** + * 校验可选 JSON 字段格式。 + * + * @param value JSON 字符串 + * @param fieldName 字段名称 + */ + private void validateJsonObjectIfPresent(String value, String fieldName) { + if (StringUtils.isNotBlank(value) && !JsonUtils.isJsonObject(value)) { + throw new ServiceException("{}必须是 JSON 对象", fieldName); + } + } + + private Map parseOptionalJson(String json) { + return StringUtils.isBlank(json) ? Map.of() : JsonUtils.parseMap(json); + } + + private void validateLength(String fieldName, String value, int maxLength) { + if (value != null && value.length() > maxLength) { + throw new ServiceException("{}不能超过{}个字符", fieldName, maxLength); + } + } + + private void validateObjectKeyPrefix(String value, String fieldName) { + if (StringUtils.isBlank(value)) { + return; + } + for (String segment : value.replace('\\', '/').split("/")) { + if (".".equals(segment) || "..".equals(segment)) { + throw new ServiceException("{}不能包含 . 或 .. 路径段", fieldName); + } + } + if (value.chars().anyMatch(Character::isISOControl)) { + throw new ServiceException("{}不能包含控制字符", fieldName); + } + } + + /** + * 校验并批量删除同步连接。 + * + * @param ids 待删除的连接主键集合 + * @param isValid 是否执行删除前校验 + * @return 是否删除成功 + */ + @Override + public Boolean deleteWithValidByIds(Collection ids, Boolean isValid) { + synchronized (connectionMutationLock) { + if (isValid && (ids == null || ids.isEmpty())) { + throw new ServiceException("待删除的连接主键不能为空"); + } + if (ids != null) { + for (Long connectionId : ids) { + ensureConnectionAccess(connectionId); + if (isValid) { + ensureNoDingTalkAuth(connectionId); + } + } + } + if (isValid && planMapper.exists(com.baomidou.mybatisplus.core.toolkit.Wrappers.lambdaQuery() + .and(wrapper -> wrapper.in(SyncPlan::getSourceConnectionId, ids) + .or().in(SyncPlan::getTargetConnectionId, ids)))) { + throw new ServiceException("连接已被同步计划引用，不能删除"); + } + return connectionMapper.deleteByIds(ids) > 0; + } + } + + /** + * 校验连接配置并调用对应连接器执行连通性测试。 + * + * @param connectionId 连接主键 + * @return 连通性测试结果 + */ + @Override + public ConnectorTestResult testConnection(Long connectionId) { + SyncConnection connection = connectionMapper.selectById(connectionId); + if (connection == null) { + throw new ServiceException("同步连接不存在"); + } + ensureConnectionAccess(connection); + requireText(connection.getConnectionName(), "连接名称不能为空"); + validateRoleAndType(connection); + validateStatus(connection.getStatus()); + validateJsonObjectIfPresent(connection.getConfigJson(), "扩展配置"); + validateJsonObjectIfPresent(connection.getSecretJson(), "敏感配置"); + validateRequiredConfig(connection, true); + return connectorRegistry.resolve(connection).test(connection); + } + + /** + * 启动钉钉 Web 设备授权。登录完成后由回调把 DWS 返回的稳定 profile + * 合并回连接的非敏感配置，不会覆盖其他配置项或敏感凭证。 + * + * @param connectionId 连接主键 + * @param ownerId 当前系统用户 + * @param expectedCorpId 可选组织 ID + * @return 授权会话快照 + */ + @Override + public DwsAuthSessionVo startDingTalkAuth(Long connectionId, Long ownerId, String expectedCorpId) { + synchronized (connectionMutationLock) { + SyncConnection connection = requireDingTalkConnection(connectionId, ownerId); + if (hasActiveJob(connectionId)) { + throw new ServiceException("连接正在被同步任务使用，任务结束后才能重新登录"); + } + validateJsonObjectIfPresent(connection.getConfigJson(), "扩展配置"); + validateJsonObjectIfPresent(connection.getSecretJson(), "敏感配置"); + // 保存时允许 profile 为空；启动 Web 登录前仍必须有应用凭证。 + validateRequiredConfig(connection, false); + return dwsAuthSessionManager.start(connection, ownerId, expectedCorpId, + identity -> saveDingTalkProfile(connectionId, ownerId, identity)); + } + } + + /** + * 查询钉钉 Web 设备授权状态。 + * + * @param connectionId 连接主键 + * @param sessionId 会话 ID + * @param ownerId 当前系统用户 + * @return 授权会话快照 + */ + @Override + public DwsAuthSessionVo getDingTalkAuth(Long connectionId, String sessionId, Long ownerId) { + requireDingTalkConnection(connectionId, ownerId); + return dwsAuthSessionManager.get(connectionId, sessionId, ownerId, LoginHelper.isSuperAdmin(ownerId)); + } + + /** + * 取消钉钉 Web 设备授权。 + * + * @param connectionId 连接主键 + * @param sessionId 会话 ID + * @param ownerId 当前系统用户 + * @return 取消后的授权会话快照 + */ + @Override + public DwsAuthSessionVo cancelDingTalkAuth(Long connectionId, String sessionId, Long ownerId) { + requireDingTalkConnection(connectionId, ownerId); + return dwsAuthSessionManager.cancel(connectionId, sessionId, ownerId, LoginHelper.isSuperAdmin(ownerId)); + } + + private void ensureNoDingTalkAuth(Long connectionId) { + if (connectionId != null && dwsAuthSessionManager.hasActive(connectionId)) { + throw new ServiceException("连接正在进行钉钉 Web 登录，登录结束后才能修改或删除"); + } + } + + private SyncConnection requireDingTalkConnection(Long connectionId) { + if (connectionId == null) { + throw new ServiceException("连接主键不能为空"); + } + SyncConnection connection = connectionMapper.selectById(connectionId); + if (connection == null) { + throw new ServiceException("同步连接不存在"); + } + if (!SyncConstants.TYPE_DINGTALK.equals(connection.getConnectionType()) + || !SyncConstants.ROLE_SOURCE.equals(connection.getConnectionRole())) { + throw new ServiceException("只有 SOURCE 角色的 DINGTALK 连接支持 Web 登录"); + } + validateStatus(connection.getStatus()); + return connection; + } + + /** + * Ensures that the current user may access a connection row. The + * super-admin is the only account allowed to operate across owners. + * + * @param connectionId connection id + */ + private void ensureConnectionAccess(Long connectionId) { + if (connectionId == null) { + throw new ServiceException("连接主键不能为空"); + } + SyncConnection connection = connectionMapper.selectById(connectionId); + if (connection == null) { + throw new ServiceException("同步连接不存在"); + } + ensureConnectionAccess(connection); + } + + /** + * Ensures that the current user may access a loaded connection row. + * + * @param connection connection row + */ + private void ensureConnectionAccess(SyncConnection connection) { + Long userId = requireCurrentUserId(); + if (!LoginHelper.isSuperAdmin(userId) && !Objects.equals(connection.getCreateBy(), userId)) { + throw new ServiceException("无权访问该同步连接"); + } + } + + /** + * Gets the authenticated system user id for service-layer access checks. + * + * @return current user id + */ + private Long requireCurrentUserId() { + Long userId = LoginHelper.getUserId(); + if (userId == null) { + throw new ServiceException("当前登录用户无效，请重新登录系统"); + } + return userId; + } + + private SyncConnection requireDingTalkConnection(Long connectionId, Long ownerId) { + SyncConnection connection = requireDingTalkConnection(connectionId); + if (ownerId == null) { + throw new ServiceException("当前登录用户无效，请重新登录系统"); + } + if (!LoginHelper.isSuperAdmin(ownerId) && !Objects.equals(connection.getCreateBy(), ownerId)) { + throw new ServiceException("只有连接创建者或超级管理员可以发起/查看钉钉 Web 登录"); + } + return connection; + } + + private void saveDingTalkProfile(Long connectionId, Long ownerId, DwsAuthIdentity identity) { + synchronized (connectionMutationLock) { + SyncConnection current = requireDingTalkConnection(connectionId); + if (ownerId == null || (!LoginHelper.isSuperAdmin(ownerId) + && !Objects.equals(current.getCreateBy(), ownerId))) { + throw new ServiceException("连接归属已变化，不能保存钉钉登录身份"); + } + Map config = parseOptionalJson(current.getConfigJson()); + if (config.isEmpty()) { + config = new java.util.LinkedHashMap<>(); + } else { + config = new java.util.LinkedHashMap<>(config); + } + config.put("profile", identity.profile()); + String configJson = JsonUtils.toJsonString(config); + com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper wrapper = + com.baomidou.mybatisplus.core.toolkit.Wrappers.lambdaUpdate(SyncConnection.class) + .eq(SyncConnection::getConnectionId, connectionId) + .eq(SyncConnection::getConnectionType, SyncConstants.TYPE_DINGTALK) + .eq(SyncConnection::getConnectionRole, SyncConstants.ROLE_SOURCE) + .eq(SyncConnection::getStatus, SyncConstants.STATUS_NORMAL) + .set(SyncConnection::getConfigJson, configJson) + .set(SyncConnection::getUpdateBy, ownerId); + if (!LoginHelper.isSuperAdmin(ownerId)) { + wrapper.eq(SyncConnection::getCreateBy, ownerId); + } + if (connectionMapper.update(null, wrapper) <= 0) { + throw new ServiceException("保存钉钉登录身份失败"); + } + } + } + + /** + * The DWS profile is an authentication result, not editable connection + * metadata. Keep it server-authoritative so a normal PUT cannot redirect + * a connection to another user's/org's profile (or clear a valid profile + * by submitting a form that omits the read-only field). + * + * @param current persisted connection + * @param update client update + */ + private void preserveDingTalkProfile(SyncConnection current, SyncConnection update) { + if (!SyncConstants.TYPE_DINGTALK.equals(current.getConnectionType()) + || !SyncConstants.ROLE_SOURCE.equals(current.getConnectionRole()) + || !SyncConstants.TYPE_DINGTALK.equals(update.getConnectionType()) + || !SyncConstants.ROLE_SOURCE.equals(update.getConnectionRole())) { + return; + } + Map currentConfig = parseOptionalJson(current.getConfigJson()); + String currentProfile = stringValue(currentConfig, "profile"); + Map requestedConfig = StringUtils.isBlank(update.getConfigJson()) + ? new java.util.LinkedHashMap<>(currentConfig) + : new java.util.LinkedHashMap<>(parseOptionalJson(update.getConfigJson())); + String requestedProfile = stringValue(requestedConfig, "profile"); + if (StringUtils.isNotBlank(requestedProfile) + && !Objects.equals(currentProfile, requestedProfile)) { + throw new ServiceException("钉钉 profile 只能通过 Web 登录授权变更"); + } + if (StringUtils.isNotBlank(currentProfile)) { + requestedConfig.put("profile", currentProfile); + } else { + requestedConfig.remove("profile"); + } + update.setConfigJson(requestedConfig.isEmpty() ? null : JsonUtils.toJsonString(requestedConfig)); + } + + /** + * 解析敏感配置 JSON。 + * + * @param secretJson 敏感配置 JSON + * @return 敏感配置键值 + */ + private Map parseSecretJson(String secretJson) { + if (StringUtils.isBlank(secretJson)) { + throw new ServiceException("敏感配置不能为空"); + } + return JsonUtils.parseMap(secretJson); + } + + private boolean isReferenced(Long connectionId) { + return planMapper.exists(com.baomidou.mybatisplus.core.toolkit.Wrappers.lambdaQuery() + .and(wrapper -> wrapper.eq(SyncPlan::getSourceConnectionId, connectionId) + .or().eq(SyncPlan::getTargetConnectionId, connectionId))); + } + + private boolean hasActiveJob(Long connectionId) { + List planIds = planMapper.selectList(com.baomidou.mybatisplus.core.toolkit.Wrappers.lambdaQuery() + .select(SyncPlan::getPlanId) + .and(wrapper -> wrapper.eq(SyncPlan::getSourceConnectionId, connectionId) + .or().eq(SyncPlan::getTargetConnectionId, connectionId))) + .stream().map(SyncPlan::getPlanId).toList(); + return !planIds.isEmpty() && jobMapper.exists( + com.baomidou.mybatisplus.core.toolkit.Wrappers.lambdaQuery() + .in(SyncJob::getPlanId, planIds) + .in(SyncJob::getStatus, SyncConstants.JOB_PENDING, + SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING)); + } + + private void validateDownloadOptions(Map config) { + String parallel = stringValue(config, "downloadParallel"); + if (StringUtils.isNotBlank(parallel)) { + try { + int value = Integer.parseInt(parallel); + if (value < 1 || value > 8) { + throw new ServiceException("钉钉 downloadParallel 必须在1到8之间"); + } + } catch (NumberFormatException e) { + throw new ServiceException("钉钉 downloadParallel 必须是整数"); + } + } + String partSize = stringValue(config, "downloadPartSize"); + if (StringUtils.isNotBlank(partSize)) { + Matcher matcher = DOWNLOAD_PART_SIZE_PATTERN.matcher(partSize); + if (!matcher.matches()) { + throw new ServiceException("钉钉 downloadPartSize 格式不正确，例如 32MB"); + } + try { + long multiplier = switch (matcher.group(2).toUpperCase(Locale.ROOT)) { + case "KB" -> 1024L; + case "MB" -> ONE_MIB; + case "GB" -> ONE_GIB; + default -> throw new IllegalStateException("无法识别容量单位"); + }; + long bytes = Math.multiplyExact(Long.parseLong(matcher.group(1)), multiplier); + if (bytes < ONE_MIB || bytes > ONE_GIB) { + throw new ServiceException("钉钉 downloadPartSize 必须在1MB到1GB之间"); + } + } catch (NumberFormatException | ArithmeticException e) { + throw new ServiceException("钉钉 downloadPartSize 超出有效范围"); + } + } + } + + private String stringValue(Map values, String key) { + Object value = values.get(key); + return value == null ? null : String.valueOf(value); + } + + /** + * 校验敏感配置中的必填项。 + * + * @param secrets 敏感配置 + * @param key 配置键 + * @param message 校验失败提示 + */ + private void requireSecret(Map secrets, String key, String message) { + Object value = secrets.get(key); + if (value == null || StringUtils.isBlank(String.valueOf(value))) { + throw new ServiceException(message); + } + } + + /** + * 校验必填文本。 + * + * @param value 文本值 + * @param message 校验失败提示 + */ + private void requireText(String value, String message) { + if (StringUtils.isBlank(value)) { + throw new ServiceException(message); + } + } + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncJobServiceImpl.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncJobServiceImpl.java new file mode 100644 index 000000000..94175d3cf --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncJobServiceImpl.java @@ -0,0 +1,199 @@ +package org.dromara.sync.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import lombok.RequiredArgsConstructor; +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.core.exception.ServiceException; +import org.dromara.common.core.utils.StringUtils; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.sync.constant.SyncConstants; +import org.dromara.sync.domain.SyncJob; +import org.dromara.sync.domain.SyncJobItem; +import org.dromara.sync.domain.SyncPlan; +import org.dromara.sync.domain.SyncTransferPart; +import org.dromara.sync.domain.bo.SyncJobBo; +import org.dromara.sync.domain.bo.SyncJobItemBo; +import org.dromara.sync.domain.vo.SyncJobItemVo; +import org.dromara.sync.domain.vo.SyncJobVo; +import org.dromara.sync.mapper.SyncJobItemMapper; +import org.dromara.sync.mapper.SyncJobMapper; +import org.dromara.sync.mapper.SyncPlanMapper; +import org.dromara.sync.mapper.SyncTransferPartMapper; +import org.dromara.sync.service.ISyncJobService; +import org.dromara.sync.worker.SyncExecutionWorker; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.util.Collection; +import java.util.List; + +/** + * 同步任务服务实现。 + */ +@Service +@RequiredArgsConstructor +public class SyncJobServiceImpl implements ISyncJobService { + + private final SyncJobMapper jobMapper; + private final SyncJobItemMapper itemMapper; + private final SyncTransferPartMapper transferPartMapper; + private final SyncPlanMapper planMapper; + private final SyncExecutionWorker executionWorker; + + @Override + public SyncJobVo queryById(Long jobId) { + return jobMapper.selectVoById(jobId); + } + + @Override + public PageResult queryPageList(SyncJobBo bo, PageQuery pageQuery) { + LambdaQueryWrapper lqw = Wrappers.lambdaQuery(); + lqw.eq(bo.getPlanId() != null, SyncJob::getPlanId, bo.getPlanId()); + lqw.eq(StringUtils.isNotBlank(bo.getTriggerType()), SyncJob::getTriggerType, bo.getTriggerType()); + lqw.eq(StringUtils.isNotBlank(bo.getRunType()), SyncJob::getRunType, bo.getRunType()); + lqw.eq(StringUtils.isNotBlank(bo.getStatus()), SyncJob::getStatus, bo.getStatus()); + Object beginTime = bo.getParams().get("beginTime"); + Object endTime = bo.getParams().get("endTime"); + lqw.ge(beginTime != null, SyncJob::getCreateTime, beginTime); + lqw.le(endTime != null, SyncJob::getCreateTime, endTime); + lqw.orderByDesc(SyncJob::getJobId); + Page page = jobMapper.selectVoPage(pageQuery.build(), lqw); + return PageResult.build(page.getRecords(), page.getTotal()); + } + + @Override + public PageResult queryItemPageList(SyncJobItemBo bo, PageQuery pageQuery) { + LambdaQueryWrapper lqw = Wrappers.lambdaQuery(); + lqw.eq(bo.getJobId() != null, SyncJobItem::getJobId, bo.getJobId()); + lqw.like(StringUtils.isNotBlank(bo.getSourcePath()), SyncJobItem::getSourcePath, bo.getSourcePath()); + lqw.eq(StringUtils.isNotBlank(bo.getObjectType()), SyncJobItem::getObjectType, bo.getObjectType()); + lqw.eq(StringUtils.isNotBlank(bo.getActionType()), SyncJobItem::getActionType, bo.getActionType()); + lqw.eq(StringUtils.isNotBlank(bo.getStatus()), SyncJobItem::getStatus, bo.getStatus()); + lqw.orderByAsc(SyncJobItem::getItemId); + Page page = itemMapper.selectVoPage(pageQuery.build(), lqw); + return PageResult.build(page.getRecords(), page.getTotal()); + } + + @Override + public Long startPlan(Long planId, String triggerType, String runType) { + SyncPlan plan = planMapper.selectById(planId); + if (plan == null) { + throw new ServiceException("同步计划不存在"); + } + if (!SyncConstants.STATUS_NORMAL.equals(plan.getStatus())) { + throw new ServiceException("同步计划已停用"); + } + boolean running = jobMapper.exists(Wrappers.lambdaQuery() + .eq(SyncJob::getPlanId, planId) + .in(SyncJob::getStatus, SyncConstants.JOB_PENDING, + SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING)); + if (running) { + throw new ServiceException("该同步计划已有运行中任务"); + } + String actualTriggerType = StringUtils.isBlank(triggerType) ? "MANUAL" : triggerType; + String actualRunType = StringUtils.isBlank(runType) ? plan.getSyncMode() : runType; + if (!List.of("MANUAL", "SCHEDULE", "RETRY").contains(actualTriggerType)) { + throw new ServiceException("不支持的任务触发类型：{}", actualTriggerType); + } + if (!List.of("FULL", "INCREMENTAL").contains(actualRunType)) { + throw new ServiceException("不支持的任务运行类型：{}", actualRunType); + } + SyncJob job = new SyncJob(); + job.setPlanId(planId); + job.setTriggerType(actualTriggerType); + job.setRunType(actualRunType); + job.setStatus(SyncConstants.JOB_PENDING); + job.setTotalCount(0L); + job.setProcessedCount(0L); + job.setSuccessCount(0L); + job.setFailedCount(0L); + job.setSkippedCount(0L); + job.setDeletedCount(0L); + job.setTotalBytes(0L); + job.setTransferredBytes(0L); + try { + if (jobMapper.insert(job) <= 0) { + throw new ServiceException("创建同步任务失败"); + } + } catch (DuplicateKeyException e) { + throw new ServiceException("该同步计划已有等待或运行中的任务"); + } + try { + executionWorker.executeAsync(job.getJobId()); + } catch (RuntimeException e) { + jobMapper.lambda() + .set(SyncJob::getStatus, SyncConstants.JOB_FAILED) + .set(SyncJob::getFinishTime, LocalDateTime.now()) + .set(SyncJob::getErrorMessage, "同步任务未能提交到后台执行器") + .eq(SyncJob::getJobId, job.getJobId()) + .eq(SyncJob::getStatus, SyncConstants.JOB_PENDING) + .update(); + throw new ServiceException("同步任务未能提交到后台执行器", e); + } + return job.getJobId(); + } + + @Override + public Long retry(Long jobId) { + SyncJob oldJob = jobMapper.selectById(jobId); + if (oldJob == null) { + throw new ServiceException("原同步任务不存在"); + } + if (List.of(SyncConstants.JOB_PENDING, SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING) + .contains(oldJob.getStatus())) { + throw new ServiceException("运行中的任务不能重试"); + } + return startPlan(oldJob.getPlanId(), "RETRY", "INCREMENTAL"); + } + + @Override + public Boolean cancel(Long jobId) { + SyncJob job = jobMapper.selectById(jobId); + if (job == null) { + throw new ServiceException("同步任务不存在"); + } + if (!List.of(SyncConstants.JOB_PENDING, SyncConstants.JOB_RUNNING).contains(job.getStatus())) { + throw new ServiceException("只有等待或运行中的任务可以取消"); + } + String targetStatus = SyncConstants.JOB_PENDING.equals(job.getStatus()) + ? SyncConstants.JOB_CANCELED : SyncConstants.JOB_CANCELING; + boolean canceled = jobMapper.lambda() + .set(SyncJob::getStatus, targetStatus) + .set(SyncConstants.JOB_CANCELED.equals(targetStatus), SyncJob::getFinishTime, LocalDateTime.now()) + .eq(SyncJob::getJobId, jobId) + .eq(SyncJob::getStatus, job.getStatus()) + .update(); + if (canceled && SyncConstants.JOB_CANCELING.equals(targetStatus)) { + executionWorker.cancel(jobId); + } + return canceled; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public Boolean deleteWithValidByIds(Collection ids, Boolean isValid) { + if (isValid) { + boolean active = jobMapper.exists(Wrappers.lambdaQuery() + .in(SyncJob::getJobId, ids) + .in(SyncJob::getStatus, SyncConstants.JOB_PENDING, + SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING)); + if (active) { + throw new ServiceException("等待或运行中的同步任务不能删除"); + } + } + List itemIds = itemMapper.selectList(Wrappers.lambdaQuery() + .select(SyncJobItem::getItemId) + .in(SyncJobItem::getJobId, ids)) + .stream().map(SyncJobItem::getItemId).toList(); + if (!itemIds.isEmpty()) { + transferPartMapper.delete(Wrappers.lambdaQuery() + .in(SyncTransferPart::getJobItemId, itemIds)); + } + itemMapper.delete(Wrappers.lambdaQuery().in(SyncJobItem::getJobId, ids)); + return jobMapper.deleteByIds(ids) > 0; + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncObjectServiceImpl.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncObjectServiceImpl.java new file mode 100644 index 000000000..162ccdb2a --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncObjectServiceImpl.java @@ -0,0 +1,44 @@ +package org.dromara.sync.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import lombok.RequiredArgsConstructor; +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.core.utils.StringUtils; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.sync.domain.SyncObject; +import org.dromara.sync.domain.bo.SyncObjectBo; +import org.dromara.sync.domain.vo.SyncObjectVo; +import org.dromara.sync.mapper.SyncObjectMapper; +import org.dromara.sync.service.ISyncObjectService; +import org.springframework.stereotype.Service; + +/** + * 同步对象清单服务实现。 + */ +@Service +@RequiredArgsConstructor +public class SyncObjectServiceImpl implements ISyncObjectService { + + private final SyncObjectMapper objectMapper; + + @Override + public SyncObjectVo queryById(Long objectId) { + return objectMapper.selectVoById(objectId); + } + + @Override + public PageResult queryPageList(SyncObjectBo bo, PageQuery pageQuery) { + LambdaQueryWrapper lqw = Wrappers.lambdaQuery(); + lqw.eq(bo.getPlanId() != null, SyncObject::getPlanId, bo.getPlanId()); + lqw.eq(StringUtils.isNotBlank(bo.getSourceObjectId()), SyncObject::getSourceObjectId, bo.getSourceObjectId()); + lqw.like(StringUtils.isNotBlank(bo.getSourcePath()), SyncObject::getSourcePath, bo.getSourcePath()); + lqw.eq(StringUtils.isNotBlank(bo.getObjectType()), SyncObject::getObjectType, bo.getObjectType()); + lqw.eq(StringUtils.isNotBlank(bo.getSyncStatus()), SyncObject::getSyncStatus, bo.getSyncStatus()); + lqw.eq(StringUtils.isNotBlank(bo.getSourceDeleted()), SyncObject::getSourceDeleted, bo.getSourceDeleted()); + lqw.orderByAsc(SyncObject::getSourcePath); + Page page = objectMapper.selectVoPage(pageQuery.build(), lqw); + return PageResult.build(page.getRecords(), page.getTotal()); + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncPlanServiceImpl.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncPlanServiceImpl.java new file mode 100644 index 000000000..eb8c44f84 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncPlanServiceImpl.java @@ -0,0 +1,380 @@ +package org.dromara.sync.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import lombok.RequiredArgsConstructor; +import org.dromara.common.core.domain.PageResult; +import org.dromara.common.core.exception.ServiceException; +import org.dromara.common.core.utils.MapstructUtils; +import org.dromara.common.core.utils.StringUtils; +import org.dromara.common.mybatis.core.page.PageQuery; +import org.dromara.common.mybatis.core.query.QueryBuilder; +import org.dromara.sync.constant.SyncConstants; +import org.dromara.sync.domain.SyncConnection; +import org.dromara.sync.domain.SyncJob; +import org.dromara.sync.domain.SyncPlan; +import org.dromara.sync.domain.bo.SyncPlanBo; +import org.dromara.sync.domain.vo.SyncPlanVo; +import org.dromara.sync.mapper.SyncConnectionMapper; +import org.dromara.sync.mapper.SyncJobMapper; +import org.dromara.sync.mapper.SyncPlanMapper; +import org.dromara.sync.service.ISyncPlanService; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.scheduling.support.CronExpression; +import org.springframework.stereotype.Service; + +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 同步计划 Service 业务层处理。 + * + * @author Codex + * @date 2026-09-02 + */ +@RequiredArgsConstructor +@Service +public class SyncPlanServiceImpl implements ISyncPlanService { + + private static final String SCHEDULE_MANUAL = "MANUAL"; + private static final String SCHEDULE_CRON = "CRON"; + + private static final Set SYNC_MODES = Set.of("FULL", "INCREMENTAL"); + private static final Set SCHEDULE_TYPES = Set.of(SCHEDULE_MANUAL, SCHEDULE_CRON); + private static final Set CONFLICT_STRATEGIES = Set.of("OVERWRITE", "SKIP", "KEEP_BOTH"); + private static final Set DELETE_STRATEGIES = Set.of("KEEP", "MARK", "DELETE"); + private static final Set VERIFY_MODES = Set.of("SIZE", "ETAG", "SHA256"); + private static final Set PLAN_STATUSES = Set.of(SyncConstants.STATUS_NORMAL, SyncConstants.STATUS_DISABLED); + + private final SyncPlanMapper planMapper; + private final SyncConnectionMapper connectionMapper; + private final SyncJobMapper jobMapper; + + /** + * 根据主键查询同步计划。 + * + * @param planId 计划主键 + * @return 同步计划详情 + */ + @Override + public SyncPlanVo queryById(Long planId) { + return planMapper.selectVoById(planId); + } + + /** + * 分页查询同步计划列表。 + * + * @param bo 查询条件 + * @param pageQuery 分页参数 + * @return 同步计划分页列表 + */ + @Override + public PageResult queryPageList(SyncPlanBo bo, PageQuery pageQuery) { + LambdaQueryWrapper lqw = buildQueryWrapper(bo); + Page result = planMapper.selectVoPage(pageQuery.build(), lqw); + return PageResult.build(result.getRecords(), result.getTotal()); + } + + /** + * 查询符合条件的同步计划列表。 + * + * @param bo 查询条件 + * @return 同步计划列表 + */ + @Override + public List queryList(SyncPlanBo bo) { + return planMapper.selectVoList(buildQueryWrapper(bo)); + } + + /** + * 构建同步计划动态查询条件。 + * + * @param bo 查询条件 + * @return 查询条件包装器 + */ + private LambdaQueryWrapper buildQueryWrapper(SyncPlanBo bo) { + Map params = bo.getParams(); + return QueryBuilder.lambda(SyncPlan.class) + .eqIfPresent(SyncPlan::getPlanId, bo.getPlanId()) + .likeIfText(SyncPlan::getPlanName, bo.getPlanName()) + .eqIfPresent(SyncPlan::getSourceConnectionId, bo.getSourceConnectionId()) + .eqIfPresent(SyncPlan::getTargetConnectionId, bo.getTargetConnectionId()) + .likeIfText(SyncPlan::getSourceRoot, bo.getSourceRoot()) + .likeIfText(SyncPlan::getTargetPrefix, bo.getTargetPrefix()) + .eqIfText(SyncPlan::getSyncMode, bo.getSyncMode()) + .eqIfText(SyncPlan::getScheduleType, bo.getScheduleType()) + .eqIfText(SyncPlan::getConflictStrategy, bo.getConflictStrategy()) + .eqIfText(SyncPlan::getDeleteStrategy, bo.getDeleteStrategy()) + .eqIfPresent(SyncPlan::getDeleteGuardPercent, bo.getDeleteGuardPercent()) + .eqIfText(SyncPlan::getVerifyMode, bo.getVerifyMode()) + .eqIfPresent(SyncPlan::getMaxConcurrency, bo.getMaxConcurrency()) + .eqIfPresent(SyncPlan::getBandwidthLimitKbps, bo.getBandwidthLimitKbps()) + .eqIfText(SyncPlan::getStatus, bo.getStatus()) + .betweenParams(SyncPlan::getLastRunTime, params, "beginLastRunTime", "endLastRunTime") + .betweenParams(SyncPlan::getNextRunTime, params, "beginNextRunTime", "endNextRunTime") + .orderByAsc(SyncPlan::getPlanId) + .build(); + } + + /** + * 校验计划名称是否唯一。 + * + * @param bo 同步计划 + * @return 名称未被占用返回 {@code true} + */ + @Override + public boolean checkPlanNameUnique(SyncPlanBo bo) { + String planName = StringUtils.trim(bo.getPlanName()); + if (StringUtils.isBlank(planName)) { + return true; + } + return !planMapper.lambda() + .eq(SyncPlan::getPlanName, planName) + .neIfPresent(SyncPlan::getPlanId, bo.getPlanId()) + .exists(); + } + + /** + * 新增同步计划。 + * + * @param bo 同步计划 + * @return 是否新增成功 + */ + @Override + public Boolean insertByBo(SyncPlanBo bo) { + SyncPlan add = MapstructUtils.convert(bo, SyncPlan.class); + validEntityBeforeSave(add); + boolean flag; + try { + flag = planMapper.insert(add) > 0; + } catch (DuplicateKeyException e) { + throw new ServiceException("计划名称'{}'已存在", add.getPlanName()); + } + if (flag) { + bo.setPlanId(add.getPlanId()); + } + return flag; + } + + /** + * 修改同步计划。 + * + * @param bo 同步计划 + * @return 是否修改成功 + */ + @Override + public Boolean updateByBo(SyncPlanBo bo) { + SyncPlan current = planMapper.selectById(bo.getPlanId()); + if (current == null) { + throw new ServiceException("同步计划不存在"); + } + boolean hasActiveJob = jobMapper.lambda() + .eq(SyncJob::getPlanId, bo.getPlanId()) + .in(SyncJob::getStatus, SyncConstants.JOB_PENDING, + SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING) + .exists(); + if (hasActiveJob) { + throw new ServiceException("计划存在等待或运行中的任务，任务结束后才能修改"); + } + SyncPlan update = MapstructUtils.convert(bo, SyncPlan.class); + validEntityBeforeSave(update); + try { + return planMapper.updateById(update) > 0; + } catch (DuplicateKeyException e) { + throw new ServiceException("计划名称'{}'已存在", update.getPlanName()); + } + } + + /** + * 执行同步计划保存前的业务校验和数据归一化。 + * + * @param entity 同步计划实体 + */ + private void validEntityBeforeSave(SyncPlan entity) { + entity.setPlanName(StringUtils.trim(entity.getPlanName())); + entity.setSourceRoot(StringUtils.trim(entity.getSourceRoot())); + entity.setTargetPrefix(StringUtils.trim(entity.getTargetPrefix())); + if (StringUtils.isBlank(entity.getPlanName())) { + throw new ServiceException("计划名称不能为空"); + } + validateLength("计划名称", entity.getPlanName(), 100); + validateLength("源端同步根路径", entity.getSourceRoot(), 1024); + validateLength("目标端对象键前缀", entity.getTargetPrefix(), 1024); + validateLength("备注", entity.getRemark(), 500); + validateObjectKeyPrefix(entity.getTargetPrefix(), "目标端对象键前缀"); + + boolean nameExists = planMapper.lambda() + .eq(SyncPlan::getPlanName, entity.getPlanName()) + .neIfPresent(SyncPlan::getPlanId, entity.getPlanId()) + .exists(); + if (nameExists) { + throw new ServiceException("计划名称'{}'已存在", entity.getPlanName()); + } + + validateConnection(entity.getSourceConnectionId(), SyncConstants.ROLE_SOURCE, "源端"); + validateConnection(entity.getTargetConnectionId(), SyncConstants.ROLE_TARGET, "目标端"); + + entity.setSyncMode(normalizeOption(entity.getSyncMode())); + entity.setScheduleType(normalizeOption(entity.getScheduleType())); + entity.setConflictStrategy(normalizeOption(entity.getConflictStrategy())); + entity.setDeleteStrategy(normalizeOption(entity.getDeleteStrategy())); + entity.setVerifyMode(normalizeOption(entity.getVerifyMode())); + entity.setStatus(normalizeOption(entity.getStatus())); + + validateOption("同步模式", entity.getSyncMode(), SYNC_MODES); + validateOption("调度类型", entity.getScheduleType(), SCHEDULE_TYPES); + validateOption("冲突处理策略", entity.getConflictStrategy(), CONFLICT_STRATEGIES); + validateOption("删除处理策略", entity.getDeleteStrategy(), DELETE_STRATEGIES); + validateOption("校验方式", entity.getVerifyMode(), VERIFY_MODES); + validateOption("状态", entity.getStatus(), PLAN_STATUSES); + + if (entity.getDeleteGuardPercent() == null + || entity.getDeleteGuardPercent() < 0 + || entity.getDeleteGuardPercent() > 100) { + throw new ServiceException("删除保护阈值必须在0到100之间"); + } + + if (entity.getMaxConcurrency() == null + || entity.getMaxConcurrency() < 1 + || entity.getMaxConcurrency() > 100) { + throw new ServiceException("最大并发数必须在1到100之间"); + } + if (entity.getBandwidthLimitKbps() == null || entity.getBandwidthLimitKbps() < 0) { + throw new ServiceException("带宽上限不能小于0"); + } + if (entity.getBandwidthLimitKbps() > 0) { + throw new ServiceException("当前版本尚未启用带宽限速，请将带宽上限设置为0"); + } + + if (SCHEDULE_CRON.equals(entity.getScheduleType())) { + String expression = StringUtils.trim(entity.getCronExpression()); + if (StringUtils.isBlank(expression)) { + throw new ServiceException("CRON调度的表达式不能为空"); + } + validateLength("CRON表达式", expression, 128); + try { + CronExpression.parse(expression); + } catch (IllegalArgumentException ex) { + throw new ServiceException("CRON表达式格式不正确"); + } + entity.setCronExpression(expression); + ZonedDateTime next = CronExpression.parse(expression).next(ZonedDateTime.now()); + entity.setNextRunTime(next == null ? null : next.toLocalDateTime()); + } else { + entity.setCronExpression(null); + entity.setNextRunTime(null); + } + } + + /** + * 校验连接存在、启用且角色与计划端点一致。 + * + * @param connectionId 连接主键 + * @param expectedRole 期望角色 + * @param endpointName 端点名称 + */ + private void validateConnection(Long connectionId, String expectedRole, String endpointName) { + if (connectionId == null) { + throw new ServiceException("{}连接不能为空", endpointName); + } + SyncConnection connection = connectionMapper.selectById(connectionId); + if (connection == null) { + throw new ServiceException("{}连接不存在或已删除", endpointName); + } + if (!SyncConstants.STATUS_NORMAL.equals(connection.getStatus())) { + throw new ServiceException("{}连接'{}'已停用", endpointName, connection.getConnectionName()); + } + if (!expectedRole.equals(connection.getConnectionRole())) { + throw new ServiceException("{}连接'{}'的角色必须为{}", endpointName, + connection.getConnectionName(), expectedRole); + } + if (SyncConstants.ROLE_SOURCE.equals(expectedRole) + && !SyncConstants.TYPE_DINGTALK.equals(connection.getConnectionType())) { + throw new ServiceException("首版源端连接只支持 DINGTALK"); + } + if (SyncConstants.ROLE_TARGET.equals(expectedRole) + && !Set.of(SyncConstants.TYPE_S3, SyncConstants.TYPE_ALIYUN_OSS) + .contains(connection.getConnectionType())) { + throw new ServiceException("首版目标端连接只支持 S3 或 ALIYUN_OSS"); + } + } + + /** + * 去除枚举型文本首尾空白。 + * + * @param value 原始值 + * @return 归一化后的值 + */ + private String normalizeOption(String value) { + return StringUtils.trim(value); + } + + /** + * 校验枚举型配置值。 + * + * @param fieldName 字段名称 + * @param value 字段值 + * @param options 允许值 + */ + private void validateOption(String fieldName, String value, Set options) { + if (!options.contains(value)) { + throw new ServiceException("{}不正确，可选值为{}", fieldName, String.join(",", options)); + } + } + + /** + * 校验文本长度不超过数据库字段上限。 + * + * @param fieldName 字段名称 + * @param value 字段值 + * @param maxLength 最大长度 + */ + private void validateLength(String fieldName, String value, int maxLength) { + if (value != null && value.length() > maxLength) { + throw new ServiceException("{}不能超过{}个字符", fieldName, maxLength); + } + } + + private void validateObjectKeyPrefix(String value, String fieldName) { + if (StringUtils.isBlank(value)) { + return; + } + for (String segment : value.replace('\\', '/').split("/")) { + if (".".equals(segment) || "..".equals(segment)) { + throw new ServiceException("{}不能包含 . 或 .. 路径段", fieldName); + } + } + if (value.chars().anyMatch(Character::isISOControl)) { + throw new ServiceException("{}不能包含控制字符", fieldName); + } + } + + /** + * 校验并批量删除同步计划。 + * + * @param ids 计划主键集合 + * @param isValid 是否执行删除前业务校验 + * @return 是否删除成功 + */ + @Override + public Boolean deleteWithValidByIds(Collection ids, Boolean isValid) { + if (isValid && (ids == null || ids.isEmpty())) { + throw new ServiceException("待删除的计划主键不能为空"); + } + if (isValid) { + boolean hasRunningJob = jobMapper.lambda() + .in(SyncJob::getPlanId, ids) + .in(SyncJob::getStatus, List.of(SyncConstants.JOB_PENDING, + SyncConstants.JOB_RUNNING, SyncConstants.JOB_CANCELING)) + .exists(); + if (hasRunningJob) { + throw new ServiceException("存在等待中或运行中的同步任务，不能删除计划"); + } + } + return planMapper.deleteByIds(ids) > 0; + } + +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/worker/SyncExecutionWorker.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/worker/SyncExecutionWorker.java new file mode 100644 index 000000000..3ddd873e7 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/worker/SyncExecutionWorker.java @@ -0,0 +1,966 @@ +package org.dromara.sync.worker; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.dromara.common.core.exception.ServiceException; +import org.dromara.common.core.utils.StringUtils; +import org.dromara.common.json.utils.JsonUtils; +import org.dromara.sync.connector.ConnectorRegistry; +import org.dromara.sync.connector.SourceConnector; +import org.dromara.sync.connector.TargetConnector; +import org.dromara.sync.connector.model.*; +import org.dromara.sync.constant.SyncConstants; +import org.dromara.sync.domain.*; +import org.dromara.sync.mapper.*; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.support.CronExpression; +import org.springframework.stereotype.Component; + +import java.io.InputStream; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.*; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * 同步任务后台执行器。 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class SyncExecutionWorker { + + private static final String CHECKPOINT_TYPE = "RUN_WATERMARK"; + private static final String CHECKPOINT_KEY = "default"; + private static final int MAX_PAGES_PER_FOLDER = 100_000; + private static final long MAX_OBJECTS_PER_RUN = 10_000_000L; + + private final SyncJobMapper jobMapper; + private final SyncJobItemMapper itemMapper; + private final SyncPlanMapper planMapper; + private final SyncConnectionMapper connectionMapper; + private final SyncObjectMapper objectMapper; + private final SyncCheckpointMapper checkpointMapper; + private final ConnectorRegistry connectorRegistry; + private final Map activeExecutions = new ConcurrentHashMap<>(); + + /** + * 异步执行已创建的同步任务。 + * + * @param jobId 任务ID + */ + @Async + public void executeAsync(Long jobId) { + execute(jobId); + } + + /** + * 同步执行任务，供测试和调度适配器复用。 + * + * @param jobId 任务ID + */ + public void execute(Long jobId) { + if (!markRunning(jobId)) { + return; + } + Thread executionThread = Thread.currentThread(); + activeExecutions.put(jobId, executionThread); + Stats stats = new Stats(); + try { + SyncJob job = requireJob(jobId); + SyncPlan plan = requirePlan(job.getPlanId()); + SyncConnection sourceConnection = requireConnection(plan.getSourceConnectionId(), SyncConstants.ROLE_SOURCE); + SyncConnection targetConnection = requireConnection(plan.getTargetConnectionId(), SyncConstants.ROLE_TARGET); + SourceConnector sourceConnector = connectorRegistry.source(sourceConnection.getConnectionType()); + TargetConnector targetConnector = connectorRegistry.target(targetConnection.getConnectionType()); + String checkpointBefore = loadCheckpoint(plan.getPlanId()); + jobMapper.lambda().set(SyncJob::getCheckpointBefore, checkpointBefore) + .eq(SyncJob::getJobId, jobId).update(); + scanAndTransfer(job, plan, sourceConnection, targetConnection, sourceConnector, targetConnector, stats); + checkCanceled(jobId); + if (hasFailures(stats)) { + log.warn("本次同步存在文件失败项，跳过源端删除处理，jobId={}", jobId); + } else { + processDeletedObjects(job, plan, targetConnection, targetConnector, stats); + } + checkCanceled(jobId); + String checkpointAfter = LocalDateTime.now() + ":" + jobId; + saveCheckpoint(plan.getPlanId(), jobId, checkpointAfter); + if (!finish(jobId, stats, checkpointAfter)) { + checkCanceled(jobId); + throw new ServiceException("同步任务状态已发生变化，不能提交完成状态"); + } + planMapper.lambda() + .set(SyncPlan::getLastRunTime, LocalDateTime.now()) + .set(SyncPlan::getNextRunTime, calculateNextRunSafely(plan)) + .eq(SyncPlan::getPlanId, plan.getPlanId()).update(); + } catch (JobCanceledException ignored) { + markCanceled(jobId); + log.info("同步任务已取消，jobId={}", jobId); + } catch (Exception e) { + if (isCancellationRequested(jobId)) { + markCanceled(jobId); + log.info("同步任务已取消，jobId={}", jobId); + } else { + log.error("同步任务执行失败，jobId={}", jobId, e); + fail(jobId, stats, e.getMessage()); + } + } finally { + activeExecutions.remove(jobId, executionThread); + } + } + + /** + * 中断当前节点上正在执行的任务。数据库状态仍是跨节点取消的最终依据。 + * + * @param jobId 任务ID + */ + public void cancel(Long jobId) { + Thread executionThread = activeExecutions.get(jobId); + if (executionThread != null) { + executionThread.interrupt(); + } + } + + private void scanAndTransfer(SyncJob job, SyncPlan plan, SyncConnection sourceConnection, + SyncConnection targetConnection, SourceConnector sourceConnector, + TargetConnector targetConnector, Stats stats) { + int configuredConcurrency = plan.getMaxConcurrency() == null ? 1 : plan.getMaxConcurrency(); + int concurrency = Math.max(1, Math.min(configuredConcurrency, 100)); + ExecutorService executor = Executors.newFixedThreadPool(concurrency, + Thread.ofVirtual().name("sync-transfer-" + job.getJobId() + "-", 0).factory()); + CompletionService transfers = new ExecutorCompletionService<>(executor); + int outstanding = 0; + try { + Deque folders = new ArrayDeque<>(); + FolderNode root = new FolderNode(normalizeSourceRoot(plan.getSourceRoot()), "", null); + folders.add(root); + Set discoveredFolders = new HashSet<>(); + discoveredFolders.add(new FolderKey(root.scopeId(), root.objectId())); + Set discoveredObjects = new HashSet<>(); + long objectCount = 0; + while (!folders.isEmpty()) { + FolderNode folder = folders.removeFirst(); + String cursor = null; + Set visitedCursors = new HashSet<>(); + int pageCount = 0; + do { + checkCanceled(job.getJobId()); + if (++pageCount > MAX_PAGES_PER_FOLDER) { + throw new ServiceException("单个源目录分页超过安全上限，已停止同步"); + } + String cursorKey = StringUtils.isBlank(cursor) ? "" : cursor; + if (!visitedCursors.add(cursorKey)) { + throw new ServiceException("源端返回了重复分页游标，已停止同步"); + } + ScanResult page = sourceConnector.scan(sourceConnection, + new ScanRequest(folder.objectId(), cursor, folder.scopeId())); + if (page == null || page.objects() == null) { + throw new ServiceException("源端返回了不完整的分页结果，已停止同步"); + } + for (SourceObject sourceObject : page.objects()) { + checkCanceled(job.getJobId()); + if (sourceObject == null || StringUtils.isBlank(sourceObject.objectId())) { + throw new ServiceException("源端返回了缺少对象ID的条目，已停止同步"); + } + if (!discoveredObjects.add(sourceObject.objectId())) { + throw new ServiceException("源端返回了重复对象ID：{}", sourceObject.objectId()); + } + if (++objectCount > MAX_OBJECTS_PER_RUN) { + throw new ServiceException("本次扫描对象数超过安全上限，已停止同步"); + } + String sourcePath = joinPath(folder.path(), sourceObject.name()); + if (sourcePath.length() > 2048) { + throw new ServiceException("源端路径超过数据库长度上限：{}", sourcePath.substring(0, 256)); + } + SyncObject existing = findObject(plan.getPlanId(), sourceObject.objectId()); + if (existing != null && !Objects.equals(existing.getObjectType(), sourceObject.objectType())) { + throw new ServiceException("源对象类型发生变化，需人工确认后再同步：{}", sourcePath); + } + boolean existed = existing != null; + boolean changed = isChanged(existing, sourcePath, sourceObject); + String previousTargetKey = existing == null ? null : existing.getTargetKey(); + SyncObject current = saveSeenObject(existing, plan.getPlanId(), job.getJobId(), + sourcePath, sourceObject); + if (sourceObject.isFolder()) { + FolderNode child = new FolderNode(sourceObject.objectId(), sourcePath, + metadataValue(sourceObject, "spaceId")); + if (!discoveredFolders.add(new FolderKey(child.scopeId(), child.objectId()))) { + throw new ServiceException("源端目录结构存在重复或循环节点：{}", sourcePath); + } + folders.addLast(child); + continue; + } + while (outstanding >= concurrency * 2) { + awaitTransfer(transfers); + outstanding--; + } + transfers.submit(() -> { + checkCanceled(job.getJobId()); + transferObject(job, plan, targetConnection, sourceConnector, targetConnector, + sourceConnection, sourceObject, existed, changed, previousTargetKey, + current, sourcePath, stats); + return null; + }); + outstanding++; + } + cursor = page.hasMore() ? page.nextCursor() : null; + if (page.hasMore() && StringUtils.isBlank(cursor)) { + throw new ServiceException("源端返回 hasMore=true 但没有分页游标，已停止同步"); + } + } while (StringUtils.isNotBlank(cursor)); + } + while (outstanding > 0) { + awaitTransfer(transfers); + outstanding--; + } + } finally { + executor.shutdownNow(); + boolean interrupted = Thread.interrupted(); + long nextWarningAt = System.nanoTime() + TimeUnit.MINUTES.toNanos(1); + try { + while (!executor.isTerminated()) { + try { + if (executor.awaitTermination(10, TimeUnit.SECONDS)) { + break; + } + } catch (InterruptedException e) { + interrupted = true; + executor.shutdownNow(); + } + if (System.nanoTime() >= nextWarningAt) { + log.warn("仍在等待同步传输线程安全退出，jobId={}", job.getJobId()); + nextWarningAt = System.nanoTime() + TimeUnit.MINUTES.toNanos(1); + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + } + + private void awaitTransfer(CompletionService transfers) { + try { + transfers.take().get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new ServiceException("等待文件传输时线程被中断", e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof JobCanceledException canceledException) { + throw canceledException; + } + if (cause instanceof RuntimeException runtimeException) { + throw runtimeException; + } + throw new ServiceException("文件传输线程执行失败", cause); + } + } + + private void transferObject(SyncJob job, SyncPlan plan, SyncConnection targetConnection, + SourceConnector sourceConnector, TargetConnector targetConnector, + SyncConnection sourceConnection, SourceObject sourceObject, + boolean existed, boolean changed, String previousTargetKey, SyncObject current, + String sourcePath, Stats stats) { + boolean full = "FULL".equalsIgnoreCase(job.getRunType()); + boolean shouldTransfer = full || !existed || changed; + String action = !existed ? "CREATE" : changed || full ? "UPDATE" : "SKIP"; + if (!shouldTransfer) { + action = "SKIP"; + } + String targetKey = buildTargetKey(targetConnection, plan, sourcePath, sourceObject); + SyncJobItem item = createItem(job.getJobId(), sourceObject, sourcePath, targetKey, action); + synchronized (stats) { + stats.totalCount++; + } + String unsupportedMessage = unsupportedSourceMessage(sourceObject); + if (unsupportedMessage != null) { + completeUnsupported(item, current, unsupportedMessage); + synchronized (stats) { + stats.processedCount++; + stats.skippedCount++; + } + updateProgress(job.getJobId(), stats); + return; + } + if (!shouldTransfer) { + completeSkipped(item, current, false); + synchronized (stats) { + stats.processedCount++; + stats.skippedCount++; + } + updateProgress(job.getJobId(), stats); + return; + } + try { + if ("SKIP".equals(plan.getConflictStrategy()) || "KEEP_BOTH".equals(plan.getConflictStrategy())) { + checkCanceled(job.getJobId()); + boolean targetExists = targetConnector.exists(targetConnection, targetKey); + checkCanceled(job.getJobId()); + if ("SKIP".equals(plan.getConflictStrategy()) && targetExists) { + completeSkipped(item, current, true); + synchronized (stats) { + stats.skippedCount++; + } + return; + } + if ("KEEP_BOTH".equals(plan.getConflictStrategy()) && targetExists) { + targetKey = addVersionSuffix(targetKey, sourceVersionIdentity(sourceObject)); + item.setTargetKey(targetKey); + itemMapper.updateById(item); + } + } + synchronized (stats) { + stats.totalBytes += Math.max(sourceObject.size(), 0L); + } + item.setStatus(SyncConstants.ITEM_RUNNING); + item.setStartTime(LocalDateTime.now()); + itemMapper.updateById(item); + try (SourceContent content = sourceConnector.download(sourceConnection, sourceObject)) { + checkCanceled(job.getJobId()); + if (content.size() != sourceObject.size()) { + synchronized (stats) { + stats.totalBytes += content.size() - Math.max(sourceObject.size(), 0L); + } + item.setSize(content.size()); + } + String sha256 = "SHA256".equals(plan.getVerifyMode()) + ? sha256(content.path(), job.getJobId()) : sourceSha256(sourceObject.hash()); + checkCanceled(job.getJobId()); + Map metadata = new HashMap<>(); + metadata.put("sync-source-id", sourceObject.objectId()); + if (StringUtils.isNotBlank(sha256)) { + metadata.put("sync-sha256", sha256); + } + TargetWriteResult result = targetConnector.upload(targetConnection, + new TargetWriteRequest(targetKey, content.path(), content.size(), content.contentType(), metadata)); + checkCanceled(job.getJobId()); + if (result.size() != content.size()) { + throw new ServiceException("上传后大小校验失败，源大小 {}，目标大小 {}", content.size(), result.size()); + } + if ("SHA256".equals(plan.getVerifyMode()) + && !Objects.equals(sha256, metadataValue(result.metadata(), "sync-sha256"))) { + throw new ServiceException("上传后 SHA-256 元数据回读校验失败"); + } + if ("ETAG".equals(plan.getVerifyMode()) && StringUtils.isBlank(result.eTag())) { + throw new ServiceException("上传后未能读取目标 ETag"); + } + item.setTransferredBytes(result.size()); + if (StringUtils.isNotBlank(previousTargetKey) + && !previousTargetKey.equals(result.objectKey()) + && "DELETE".equals(plan.getDeleteStrategy()) + && !"KEEP_BOTH".equals(plan.getConflictStrategy())) { + checkCanceled(job.getJobId()); + targetConnector.delete(targetConnection, previousTargetKey); + } + item.setStatus(SyncConstants.ITEM_SUCCESS); + item.setSourceSha256(sha256); + item.setTargetEtag(result.eTag()); + item.setTargetVersionId(result.versionId()); + item.setFinishTime(LocalDateTime.now()); + itemMapper.updateById(item); + current.setTargetKey(result.objectKey()); + current.setTargetVersionId(result.versionId()); + current.setTargetEtag(result.eTag()); + current.setSourceSha256(sha256); + current.setSyncStatus("SYNCED"); + current.setLastSyncJobId(job.getJobId()); + current.setLastSyncTime(LocalDateTime.now()); + current.setLastErrorMessage(null); + objectMapper.updateById(current); + synchronized (stats) { + stats.successCount++; + stats.transferredBytes += result.size(); + } + } + } catch (JobCanceledException e) { + item.setStatus(SyncConstants.ITEM_FAILED); + item.setFinishTime(LocalDateTime.now()); + item.setErrorCode("CANCELED"); + item.setErrorMessage("任务已取消"); + itemMapper.updateById(item); + throw e; + } catch (Exception e) { + if (isAITableLink(sourceObject)) { + String message = "钉钉多维表链接（.dlink）导出失败：" + abbreviate(e.getMessage()); + item.setStatus(SyncConstants.ITEM_SKIPPED); + item.setFinishTime(LocalDateTime.now()); + item.setErrorCode("UNSUPPORTED"); + item.setErrorMessage(message); + itemMapper.updateById(item); + current.setSyncStatus("SKIPPED"); + current.setLastErrorMessage(message); + objectMapper.updateById(current); + synchronized (stats) { + stats.skippedCount++; + } + } else { + item.setStatus(SyncConstants.ITEM_FAILED); + item.setFinishTime(LocalDateTime.now()); + item.setErrorCode(e.getClass().getSimpleName()); + item.setErrorMessage(abbreviate(e.getMessage())); + itemMapper.updateById(item); + current.setSyncStatus("FAILED"); + current.setLastErrorMessage(abbreviate(e.getMessage())); + objectMapper.updateById(current); + synchronized (stats) { + stats.failedCount++; + } + } + } finally { + synchronized (stats) { + stats.processedCount++; + } + updateProgress(job.getJobId(), stats); + } + } + + private void processDeletedObjects(SyncJob job, SyncPlan plan, SyncConnection targetConnection, + TargetConnector targetConnector, Stats stats) { + List deleted = objectMapper.selectList(Wrappers.lambdaQuery() + .eq(SyncObject::getPlanId, plan.getPlanId()) + .eq(SyncObject::getSourceDeleted, "0") + .and(wrapper -> wrapper.isNull(SyncObject::getLastSeenJobId) + .or().ne(SyncObject::getLastSeenJobId, job.getJobId()))); + validateDeleteGuard(plan, deleted); + for (SyncObject object : deleted) { + checkCanceled(job.getJobId()); + if (SyncConstants.OBJECT_FOLDER.equals(object.getObjectType())) { + markSourceDeleted(object, job.getJobId()); + synchronized (stats) { + stats.deletedCount++; + } + continue; + } + SyncJobItem item = createDeleteItem(job.getJobId(), object); + synchronized (stats) { + stats.totalCount++; + } + try { + checkCanceled(job.getJobId()); + if ("DELETE".equals(plan.getDeleteStrategy()) && StringUtils.isNotBlank(object.getTargetKey())) { + targetConnector.delete(targetConnection, object.getTargetKey()); + } + checkCanceled(job.getJobId()); + item.setStatus(SyncConstants.ITEM_SUCCESS); + item.setFinishTime(LocalDateTime.now()); + itemMapper.updateById(item); + markSourceDeleted(object, job.getJobId()); + synchronized (stats) { + stats.successCount++; + stats.deletedCount++; + } + } catch (JobCanceledException e) { + item.setStatus(SyncConstants.ITEM_FAILED); + item.setFinishTime(LocalDateTime.now()); + item.setErrorCode("CANCELED"); + item.setErrorMessage("任务已取消"); + itemMapper.updateById(item); + throw e; + } catch (Exception e) { + item.setStatus(SyncConstants.ITEM_FAILED); + item.setFinishTime(LocalDateTime.now()); + item.setErrorCode(e.getClass().getSimpleName()); + item.setErrorMessage(abbreviate(e.getMessage())); + itemMapper.updateById(item); + object.setSyncStatus("FAILED"); + object.setLastErrorMessage(abbreviate(e.getMessage())); + objectMapper.updateById(object); + synchronized (stats) { + stats.failedCount++; + } + } + synchronized (stats) { + stats.processedCount++; + } + updateProgress(job.getJobId(), stats); + } + } + + private void validateDeleteGuard(SyncPlan plan, List deleted) { + if (!"DELETE".equals(plan.getDeleteStrategy())) { + return; + } + long destructiveCount = deleted.stream() + .filter(object -> !SyncConstants.OBJECT_FOLDER.equals(object.getObjectType())) + .filter(object -> StringUtils.isNotBlank(object.getTargetKey())) + .count(); + if (destructiveCount == 0) { + return; + } + long activeFileCount = objectMapper.selectCount(Wrappers.lambdaQuery() + .eq(SyncObject::getPlanId, plan.getPlanId()) + .eq(SyncObject::getSourceDeleted, "0") + .ne(SyncObject::getObjectType, SyncConstants.OBJECT_FOLDER)); + int allowedPercent = plan.getDeleteGuardPercent() == null ? 50 : plan.getDeleteGuardPercent(); + if (activeFileCount > 0 && destructiveCount * 100L > activeFileCount * allowedPercent) { + throw new ServiceException( + "本次拟删除目标对象 {} 个，占现有文件 {} 个的比例超过保护阈值 {}%，已拒绝删除", + destructiveCount, activeFileCount, allowedPercent); + } + } + + private void markSourceDeleted(SyncObject object, Long jobId) { + object.setSourceDeleted("1"); + object.setSyncStatus("DELETED"); + object.setLastSyncJobId(jobId); + object.setLastSyncTime(LocalDateTime.now()); + object.setLastErrorMessage(null); + objectMapper.updateById(object); + } + + private SyncObject saveSeenObject(SyncObject existing, Long planId, Long jobId, + String sourcePath, SourceObject sourceObject) { + SyncObject entity = existing == null ? new SyncObject() : existing; + entity.setPlanId(planId); + entity.setSourceObjectId(sourceObject.objectId()); + entity.setParentObjectId(sourceObject.parentObjectId()); + entity.setSourcePath(sourcePath); + entity.setObjectName(sourceObject.name()); + entity.setObjectType(sourceObject.objectType()); + entity.setSize(sourceObject.size()); + entity.setModifiedTime(sourceObject.modifiedTime()); + entity.setVersionToken(sourceObject.versionToken()); + entity.setSourceEtag(sourceObject.hash()); + entity.setContentType(sourceObject.contentType()); + entity.setMetadataJson(JsonUtils.toJsonString(sourceObject.metadata())); + entity.setSourceDeleted("0"); + entity.setLastSeenJobId(jobId); + if (sourceObject.isFolder()) { + entity.setSyncStatus("SYNCED"); + } else if (existing == null) { + entity.setSyncStatus("PENDING"); + } + if (existing == null) { + entity.setFirstSeenJobId(jobId); + objectMapper.insert(entity); + } else { + objectMapper.updateById(entity); + } + return entity; + } + + private SyncJobItem createItem(Long jobId, SourceObject object, String sourcePath, + String targetKey, String action) { + SyncJobItem item = new SyncJobItem(); + item.setJobId(jobId); + item.setSourceObjectId(object.objectId()); + item.setParentObjectId(object.parentObjectId()); + item.setSourcePath(sourcePath); + item.setTargetKey(targetKey); + item.setObjectType(object.objectType()); + item.setActionType(action); + item.setStatus(SyncConstants.ITEM_PENDING); + item.setSize(object.size()); + item.setTransferredBytes(0L); + item.setVersionToken(object.versionToken()); + item.setSourceEtag(object.hash()); + item.setRetryCount(0); + itemMapper.insert(item); + return item; + } + + private SyncJobItem createDeleteItem(Long jobId, SyncObject object) { + SourceObject sourceObject = new SourceObject(object.getSourceObjectId(), object.getParentObjectId(), + object.getObjectName(), object.getObjectType(), null, null, object.getSize(), + object.getModifiedTime(), object.getVersionToken(), object.getSourceEtag(), Map.of()); + SyncJobItem item = createItem(jobId, sourceObject, object.getSourcePath(), object.getTargetKey(), "DELETE"); + item.setStartTime(LocalDateTime.now()); + itemMapper.updateById(item); + return item; + } + + private void completeSkipped(SyncJobItem item, SyncObject object, boolean conflictSkipped) { + item.setActionType("SKIP"); + item.setStatus(SyncConstants.ITEM_SKIPPED); + item.setFinishTime(LocalDateTime.now()); + itemMapper.updateById(item); + object.setSyncStatus(conflictSkipped ? "SKIPPED" : "SYNCED"); + object.setLastErrorMessage(null); + objectMapper.updateById(object); + } + + /** + * 标记源端当前版本暂不支持的对象，避免把预期能力缺失计为任务失败。 + */ + private void completeUnsupported(SyncJobItem item, SyncObject object, String message) { + item.setActionType("SKIP"); + item.setStatus(SyncConstants.ITEM_SKIPPED); + item.setFinishTime(LocalDateTime.now()); + item.setErrorCode("UNSUPPORTED"); + item.setErrorMessage(message); + itemMapper.updateById(item); + object.setSyncStatus("SKIPPED"); + object.setLastErrorMessage(message); + objectMapper.updateById(object); + } + + private String unsupportedSourceMessage(SourceObject sourceObject) { + String extension = sourceObject.extension(); + if (extension == null) { + return null; + } + extension = extension.startsWith(".") ? extension.substring(1) : extension; + extension = extension.toLowerCase(Locale.ROOT); + if ("amind".equals(extension) || "adraw".equals(extension)) { + return "钉钉官方 DWS 暂不支持自动导出：" + extension; + } + return null; + } + + private boolean isAITableLink(SourceObject sourceObject) { + String extension = sourceObject.extension(); + if (extension == null) return false; + return "dlink".equalsIgnoreCase(extension.startsWith(".") ? extension.substring(1) : extension); + } + + private boolean isChanged(SyncObject existing, String sourcePath, SourceObject sourceObject) { + if (existing == null) { + return true; + } + if (!"SYNCED".equals(existing.getSyncStatus()) + || "1".equals(existing.getSourceDeleted()) + || StringUtils.isBlank(existing.getTargetKey()) + || !Objects.equals(existing.getSourcePath(), sourcePath)) { + return true; + } + if (StringUtils.isNotBlank(sourceObject.versionToken()) || StringUtils.isNotBlank(existing.getVersionToken())) { + return !Objects.equals(existing.getVersionToken(), sourceObject.versionToken()); + } + if (StringUtils.isNotBlank(sourceObject.hash()) || StringUtils.isNotBlank(existing.getSourceEtag())) { + return !Objects.equals(existing.getSourceEtag(), sourceObject.hash()) + || !Objects.equals(existing.getSize(), sourceObject.size()); + } + if (sourceObject.modifiedTime() == null || existing.getModifiedTime() == null) { + return true; + } + return !Objects.equals(existing.getSize(), sourceObject.size()) + || !Objects.equals(existing.getModifiedTime(), sourceObject.modifiedTime()); + } + + private String buildTargetKey(SyncConnection targetConnection, SyncPlan plan, String sourcePath, + SourceObject sourceObject) { + String relativePath = sourcePath; + if ("dlink".equalsIgnoreCase(normalizeExtension(sourceObject.extension()))) { + relativePath = replaceExtension(relativePath, "xlsx"); + } else if (SyncConstants.OBJECT_ONLINE_DOCUMENT.equals(sourceObject.objectType())) { + if ("adoc".equalsIgnoreCase(sourceObject.extension())) { + relativePath = replaceExtension(relativePath, "docx"); + } else if ("axls".equalsIgnoreCase(sourceObject.extension())) { + relativePath = replaceExtension(relativePath, "xlsx"); + } else if ("amind".equalsIgnoreCase(sourceObject.extension()) + || "adraw".equalsIgnoreCase(sourceObject.extension())) { + relativePath = replaceExtension(relativePath, "pdf"); + } + } + return joinObjectKey(targetConnection.getBasePath(), plan.getTargetPrefix(), relativePath); + } + + private String joinObjectKey(String... values) { + List segments = new ArrayList<>(); + for (String value : values) { + if (StringUtils.isBlank(value)) { + continue; + } + for (String segment : value.replace('\\', '/').split("/")) { + if (StringUtils.isBlank(segment)) { + continue; + } + if (".".equals(segment) || "..".equals(segment)) { + throw new ServiceException("对象键路径不能包含 . 或 .. 段"); + } + segments.add(segment); + } + } + if (segments.isEmpty()) { + throw new ServiceException("目标对象键不能为空"); + } + String key = String.join("/", segments); + if (key.length() > 2048) { + throw new ServiceException("目标对象键超过数据库长度上限"); + } + return key; + } + + private String joinPath(String parent, String name) { + return StringUtils.isBlank(parent) ? name : parent + '/' + name; + } + + private String normalizeSourceRoot(String sourceRoot) { + String value = StringUtils.trim(sourceRoot); + return StringUtils.isBlank(value) || "/".equals(value) ? null : value; + } + + private LocalDateTime calculateNextRunSafely(SyncPlan plan) { + if (!"CRON".equals(plan.getScheduleType()) || StringUtils.isBlank(plan.getCronExpression())) { + return null; + } + try { + ZonedDateTime next = CronExpression.parse(plan.getCronExpression()).next(ZonedDateTime.now()); + return next == null ? null : next.toLocalDateTime(); + } catch (Exception e) { + log.warn("同步计划 Cron 表达式已失效，planId={}，原因={}", plan.getPlanId(), e.getMessage()); + return null; + } + } + + private String metadataValue(SourceObject sourceObject, String key) { + return sourceObject.metadata() == null ? null : sourceObject.metadata().get(key); + } + + private String metadataValue(Map metadata, String key) { + return metadata == null ? null : metadata.get(key); + } + + private String sourceSha256(String hash) { + if (StringUtils.isBlank(hash) || !hash.matches("(?i)[0-9a-f]{64}")) { + return null; + } + return hash.toLowerCase(Locale.ROOT); + } + + private String normalizeExtension(String extension) { + if (StringUtils.isBlank(extension)) return ""; + String value = extension.startsWith(".") ? extension.substring(1) : extension; + return value.toLowerCase(Locale.ROOT); + } + + private String replaceExtension(String path, String extension) { + int slash = path.lastIndexOf('/'); + int dot = path.lastIndexOf('.'); + return dot > slash ? path.substring(0, dot + 1) + extension : path + '.' + extension; + } + + private String addVersionSuffix(String key, String versionToken) { + String suffix = StringUtils.isBlank(versionToken) + ? String.valueOf(System.currentTimeMillis()) : versionToken.replaceAll("[^a-zA-Z0-9_-]", "_"); + if (suffix.length() > 48) { + suffix = suffix.substring(0, 48); + } + int slash = key.lastIndexOf('/'); + int dot = key.lastIndexOf('.'); + String versionedKey = dot > slash + ? key.substring(0, dot) + "__" + suffix + key.substring(dot) : key + "__" + suffix; + if (versionedKey.length() > 2048) { + throw new ServiceException("保留多版本后的目标对象键超过数据库长度上限"); + } + return versionedKey; + } + + private String sourceVersionIdentity(SourceObject sourceObject) { + if (StringUtils.isNotBlank(sourceObject.versionToken())) { + return sourceObject.versionToken(); + } + if (StringUtils.isNotBlank(sourceObject.hash())) { + return sourceObject.hash(); + } + if (sourceObject.modifiedTime() != null) { + return sourceObject.modifiedTime() + "_" + sourceObject.size(); + } + return "size_" + sourceObject.size(); + } + + private String sha256(java.nio.file.Path path, Long jobId) throws Exception { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + try (InputStream input = Files.newInputStream(path)) { + byte[] buffer = new byte[1024 * 1024]; + int length; + while ((length = input.read(buffer)) >= 0) { + if (Thread.currentThread().isInterrupted()) { + checkCanceled(jobId); + throw new ServiceException("计算文件摘要时线程被中断"); + } + if (length > 0) { + digest.update(buffer, 0, length); + } + } + } + checkCanceled(jobId); + return HexFormat.of().formatHex(digest.digest()); + } + + private SyncObject findObject(Long planId, String sourceObjectId) { + return objectMapper.selectOne(Wrappers.lambdaQuery() + .eq(SyncObject::getPlanId, planId) + .eq(SyncObject::getSourceObjectId, sourceObjectId)); + } + + private String loadCheckpoint(Long planId) { + SyncCheckpoint checkpoint = checkpointMapper.selectOne(Wrappers.lambdaQuery() + .eq(SyncCheckpoint::getPlanId, planId) + .eq(SyncCheckpoint::getCheckpointType, CHECKPOINT_TYPE) + .eq(SyncCheckpoint::getCheckpointKey, CHECKPOINT_KEY)); + return checkpoint == null ? null : checkpoint.getCheckpointValue(); + } + + private void saveCheckpoint(Long planId, Long jobId, String value) { + SyncCheckpoint checkpoint = checkpointMapper.selectOne(Wrappers.lambdaQuery() + .eq(SyncCheckpoint::getPlanId, planId) + .eq(SyncCheckpoint::getCheckpointType, CHECKPOINT_TYPE) + .eq(SyncCheckpoint::getCheckpointKey, CHECKPOINT_KEY)); + if (checkpoint == null) { + checkpoint = new SyncCheckpoint(); + checkpoint.setPlanId(planId); + checkpoint.setCheckpointType(CHECKPOINT_TYPE); + checkpoint.setCheckpointKey(CHECKPOINT_KEY); + checkpoint.setCheckpointValue(value); + checkpoint.setWatermarkTime(LocalDateTime.now()); + checkpoint.setLastJobId(jobId); + checkpoint.setVersion(0L); + checkpointMapper.insert(checkpoint); + } else { + checkpoint.setCheckpointValue(value); + checkpoint.setWatermarkTime(LocalDateTime.now()); + checkpoint.setLastJobId(jobId); + checkpointMapper.updateById(checkpoint); + } + } + + private boolean markRunning(Long jobId) { + return jobMapper.lambda() + .set(SyncJob::getStatus, SyncConstants.JOB_RUNNING) + .set(SyncJob::getStartTime, LocalDateTime.now()) + .eq(SyncJob::getJobId, jobId) + .eq(SyncJob::getStatus, SyncConstants.JOB_PENDING) + .update(); + } + + private void updateProgress(Long jobId, Stats stats) { + synchronized (stats) { + jobMapper.lambda() + .set(SyncJob::getTotalCount, stats.totalCount) + .set(SyncJob::getProcessedCount, stats.processedCount) + .set(SyncJob::getSuccessCount, stats.successCount) + .set(SyncJob::getFailedCount, stats.failedCount) + .set(SyncJob::getSkippedCount, stats.skippedCount) + .set(SyncJob::getDeletedCount, stats.deletedCount) + .set(SyncJob::getTotalBytes, stats.totalBytes) + .set(SyncJob::getTransferredBytes, stats.transferredBytes) + .eq(SyncJob::getJobId, jobId) + .update(); + } + } + + private boolean hasFailures(Stats stats) { + synchronized (stats) { + return stats.failedCount > 0; + } + } + + private boolean finish(Long jobId, Stats stats, String checkpointAfter) { + String status; + synchronized (stats) { + status = stats.failedCount == 0 ? SyncConstants.JOB_SUCCESS + : stats.successCount > 0 || stats.skippedCount > 0 + ? SyncConstants.JOB_PARTIAL_FAILED : SyncConstants.JOB_FAILED; + } + updateProgress(jobId, stats); + return jobMapper.lambda() + .set(SyncJob::getStatus, status) + .set(SyncJob::getCheckpointAfter, checkpointAfter) + .set(SyncJob::getFinishTime, LocalDateTime.now()) + .eq(SyncJob::getJobId, jobId) + .eq(SyncJob::getStatus, SyncConstants.JOB_RUNNING) + .update(); + } + + private void fail(Long jobId, Stats stats, String message) { + updateProgress(jobId, stats); + jobMapper.lambda() + .set(SyncJob::getStatus, SyncConstants.JOB_FAILED) + .set(SyncJob::getFinishTime, LocalDateTime.now()) + .set(SyncJob::getErrorMessage, abbreviate(message)) + .eq(SyncJob::getJobId, jobId) + .eq(SyncJob::getStatus, SyncConstants.JOB_RUNNING) + .update(); + } + + private boolean isCancellationRequested(Long jobId) { + SyncJob job = jobMapper.selectById(jobId); + return job != null && (SyncConstants.JOB_CANCELING.equals(job.getStatus()) + || SyncConstants.JOB_CANCELED.equals(job.getStatus())); + } + + private void markCanceled(Long jobId) { + jobMapper.lambda() + .set(SyncJob::getStatus, SyncConstants.JOB_CANCELED) + .set(SyncJob::getFinishTime, LocalDateTime.now()) + .eq(SyncJob::getJobId, jobId) + .eq(SyncJob::getStatus, SyncConstants.JOB_CANCELING) + .update(); + } + + private void checkCanceled(Long jobId) { + SyncJob job = requireJob(jobId); + if (SyncConstants.JOB_CANCELING.equals(job.getStatus()) + || SyncConstants.JOB_CANCELED.equals(job.getStatus())) { + throw new JobCanceledException(); + } + } + + private SyncJob requireJob(Long jobId) { + SyncJob job = jobMapper.selectById(jobId); + if (job == null) { + throw new ServiceException("同步任务不存在：{}", jobId); + } + return job; + } + + private SyncPlan requirePlan(Long planId) { + SyncPlan plan = planMapper.selectById(planId); + if (plan == null) { + throw new ServiceException("同步计划不存在：{}", planId); + } + return plan; + } + + private SyncConnection requireConnection(Long connectionId, String expectedRole) { + SyncConnection connection = connectionMapper.selectById(connectionId); + if (connection == null || !expectedRole.equals(connection.getConnectionRole())) { + throw new ServiceException("同步连接不存在或角色不正确：{}", connectionId); + } + if (!SyncConstants.STATUS_NORMAL.equals(connection.getStatus())) { + throw new ServiceException("同步连接已停用：{}", connection.getConnectionName()); + } + return connection; + } + + private String abbreviate(String message) { + if (message == null) { + return "未知错误"; + } + return message.length() <= 2000 ? message : message.substring(0, 2000); + } + + private record FolderNode(String objectId, String path, String scopeId) { + } + + private record FolderKey(String scopeId, String objectId) { + } + + private static final class Stats { + private long totalCount; + private long processedCount; + private long successCount; + private long failedCount; + private long skippedCount; + private long deletedCount; + private long totalBytes; + private long transferredBytes; + } + + private static final class JobCanceledException extends RuntimeException { + private static final long serialVersionUID = 1L; + } +} diff --git a/script/sql/ry_sync.sql b/script/sql/ry_sync.sql new file mode 100644 index 000000000..2048b3571 --- /dev/null +++ b/script/sql/ry_sync.sql @@ -0,0 +1,359 @@ +-- ---------------------------- +-- 数据同步模块（MySQL） +-- 依赖 ry_vue.sql 中的 sys_menu、sys_dict_type、sys_dict_data 表 +-- ---------------------------- + +-- ---------------------------- +-- 1、连接配置表 +-- ---------------------------- +create table sync_connection +( + connection_id bigint(20) not null comment '连接ID', + connection_name varchar(100) not null comment '连接名称', + connection_role varchar(16) not null comment '连接角色（SOURCE源端 TARGET目标端）', + connection_type varchar(32) not null comment '连接类型（DINGTALK钉钉 S3标准S3 ALIYUN_OSS阿里云OSS）', + endpoint varchar(512) default null comment '服务端点', + region varchar(64) default null comment '区域', + bucket_name varchar(255) default null comment '存储桶名称', + base_path varchar(1024) default '' comment '连接默认根路径', + config_json text comment '非敏感扩展配置JSON', + secret_json text comment '加密后的敏感配置JSON（禁止明文存储）', + status char(1) default '0' comment '状态（0正常 1停用）', + del_flag char(1) default '0' comment '删除标志（0代表存在 1代表删除）', + active_name varchar(100) generated always as + (case when del_flag = '0' then connection_name else null end) stored + comment '有效连接名称（用于唯一约束）', + create_dept bigint(20) default null comment '创建部门', + create_by bigint(20) default null comment '创建者', + create_time datetime default null comment '创建时间', + update_by bigint(20) default null comment '更新者', + update_time datetime default null comment '更新时间', + remark varchar(500) default null comment '备注', + primary key (connection_id), + unique key uk_sync_connection_active_name (active_name), + key idx_sync_connection_name_del (connection_name, del_flag), + key idx_sync_connection_role_type (connection_role, connection_type), + key idx_sync_connection_status_del (status, del_flag) +) engine = innodb comment = '数据同步连接配置表'; + +-- ---------------------------- +-- 2、同步计划表 +-- ---------------------------- +create table sync_plan +( + plan_id bigint(20) not null comment '同步计划ID', + plan_name varchar(100) not null comment '计划名称', + source_connection_id bigint(20) not null comment '源端连接ID', + target_connection_id bigint(20) not null comment '目标端连接ID', + source_root varchar(1024) default '/' comment '源端同步根路径或空间标识', + target_prefix varchar(1024) default '' comment '目标端对象键前缀', + sync_mode varchar(16) default 'INCREMENTAL' comment '同步模式（FULL全量 INCREMENTAL增量）', + schedule_type varchar(16) default 'MANUAL' comment '调度类型（MANUAL手动 CRON定时）', + cron_expression varchar(128) default null comment 'Cron表达式', + conflict_strategy varchar(16) default 'OVERWRITE' comment '冲突策略（OVERWRITE覆盖 SKIP跳过 KEEP_BOTH两者保留）', + delete_strategy varchar(16) default 'KEEP' comment '删除策略（KEEP保留 MARK标记 DELETE删除目标）', + delete_guard_percent int(11) default 50 comment '单次目标删除比例保护阈值（0-100，100表示允许全删）', + verify_mode varchar(16) default 'SIZE' comment '校验方式（SIZE大小 ETAG标识 SHA256摘要）', + max_concurrency int(11) default 4 comment '最大并发传输数', + bandwidth_limit_kbps bigint(20) default 0 comment '带宽上限KB/s（0表示不限速）', + status char(1) default '0' comment '状态（0正常 1停用）', + last_run_time datetime default null comment '最近运行时间', + next_run_time datetime default null comment '下次运行时间', + del_flag char(1) default '0' comment '删除标志（0代表存在 1代表删除）', + active_name varchar(100) generated always as + (case when del_flag = '0' then plan_name else null end) stored + comment '有效计划名称（用于唯一约束）', + create_dept bigint(20) default null comment '创建部门', + create_by bigint(20) default null comment '创建者', + create_time datetime default null comment '创建时间', + update_by bigint(20) default null comment '更新者', + update_time datetime default null comment '更新时间', + remark varchar(500) default null comment '备注', + primary key (plan_id), + unique key uk_sync_plan_active_name (active_name), + key idx_sync_plan_name_del (plan_name, del_flag), + key idx_sync_plan_source_connection (source_connection_id), + key idx_sync_plan_target_connection (target_connection_id), + key idx_sync_plan_schedule (status, del_flag, schedule_type, next_run_time) +) engine = innodb comment = '数据同步计划表'; + +-- ---------------------------- +-- 3、同步任务表 +-- ---------------------------- +create table sync_job +( + job_id bigint(20) not null comment '同步任务ID', + plan_id bigint(20) not null comment '同步计划ID', + trigger_type varchar(16) not null comment '触发类型（MANUAL手动 SCHEDULE调度 RETRY重试）', + run_type varchar(16) not null comment '运行类型（FULL全量 INCREMENTAL增量）', + status varchar(20) default 'PENDING' comment '任务状态（PENDING待执行 RUNNING执行中 CANCELING取消中 SUCCESS成功 PARTIAL_FAILED部分失败 FAILED失败 CANCELED已取消）', + active_plan_id bigint(20) generated always as + (case when status in ('PENDING', 'RUNNING', 'CANCELING') then plan_id else null end) stored + comment '活动任务计划ID（用于保证单计划仅一个运行任务）', + checkpoint_before text comment '运行前检查点快照', + checkpoint_after text comment '运行后检查点快照', + total_count bigint(20) default 0 comment '待处理对象总数', + processed_count bigint(20) default 0 comment '已处理对象数', + success_count bigint(20) default 0 comment '成功对象数', + failed_count bigint(20) default 0 comment '失败对象数', + skipped_count bigint(20) default 0 comment '跳过对象数', + deleted_count bigint(20) default 0 comment '删除或标记对象数', + total_bytes bigint(20) default 0 comment '待传输总字节数', + transferred_bytes bigint(20) default 0 comment '已传输字节数', + start_time datetime default null comment '开始时间', + finish_time datetime default null comment '结束时间', + error_message text comment '任务错误信息', + create_dept bigint(20) default null comment '创建部门', + create_by bigint(20) default null comment '创建者', + create_time datetime default null comment '创建时间', + update_by bigint(20) default null comment '更新者', + update_time datetime default null comment '更新时间', + primary key (job_id), + unique key uk_sync_job_active_plan (active_plan_id), + key idx_sync_job_plan_time (plan_id, create_time), + key idx_sync_job_status_time (status, create_time) +) engine = innodb comment = '数据同步任务表'; + +-- ---------------------------- +-- 4、同步任务明细表 +-- ---------------------------- +create table sync_job_item +( + item_id bigint(20) not null comment '任务明细ID', + job_id bigint(20) not null comment '同步任务ID', + source_object_id varchar(255) not null comment '源端对象唯一标识', + parent_object_id varchar(255) default null comment '源端父对象标识', + source_path varchar(2048) not null comment '源端相对路径', + target_key varchar(2048) default null comment '目标端对象键', + object_type varchar(16) not null comment '对象类型（FILE文件 FOLDER目录 ONLINE_DOCUMENT在线文档）', + action_type varchar(16) not null comment '动作类型（CREATE新增上传 UPDATE更新上传 SKIP跳过 DELETE删除）', + status varchar(20) default 'PENDING' comment '明细状态（PENDING待执行 RUNNING执行中 SUCCESS成功 SKIPPED已跳过 FAILED失败）', + size bigint(20) default 0 comment '源对象字节数', + transferred_bytes bigint(20) default 0 comment '已传输字节数', + version_token varchar(512) default null comment '源端版本标识', + source_etag varchar(255) default null comment '源端ETag', + source_sha256 char(64) default null comment '源端SHA-256摘要', + target_etag varchar(255) default null comment '目标端ETag', + target_version_id varchar(512) default null comment '目标端版本ID', + retry_count int(11) default 0 comment '已重试次数', + start_time datetime default null comment '开始时间', + finish_time datetime default null comment '结束时间', + error_code varchar(64) default null comment '错误码', + error_message text comment '错误信息', + create_dept bigint(20) default null comment '创建部门', + create_by bigint(20) default null comment '创建者', + create_time datetime default null comment '创建时间', + update_by bigint(20) default null comment '更新者', + update_time datetime default null comment '更新时间', + primary key (item_id), + unique key uk_sync_job_item_job_object (job_id, source_object_id), + key idx_sync_job_item_job_status (job_id, status), + key idx_sync_job_item_retry (status, retry_count), + key idx_sync_job_item_parent (job_id, parent_object_id) +) engine = innodb comment = '数据同步任务明细表'; + +-- ---------------------------- +-- 5、源对象状态表 +-- ---------------------------- +create table sync_object +( + object_id bigint(20) not null comment '对象记录ID', + plan_id bigint(20) not null comment '同步计划ID', + source_object_id varchar(255) not null comment '源端对象唯一标识', + parent_object_id varchar(255) default null comment '源端父对象标识', + object_name varchar(1024) not null comment '对象名称', + source_path varchar(2048) not null comment '相对同步根目录的源端路径', + object_type varchar(16) not null comment '对象类型（FILE文件 FOLDER目录 ONLINE_DOCUMENT在线文档）', + size bigint(20) default 0 comment '对象字节数', + modified_time datetime default null comment '源端最后修改时间', + version_token varchar(512) default null comment '源端版本标识', + source_etag varchar(255) default null comment '源端ETag', + source_sha256 char(64) default null comment '源端SHA-256摘要', + content_type varchar(255) default null comment '内容类型', + metadata_json text comment '源端扩展元数据JSON', + target_key varchar(2048) default null comment '最近成功同步的目标对象键', + target_version_id varchar(512) default null comment '最近成功同步的目标版本ID', + target_etag varchar(255) default null comment '最近成功同步的目标端ETag', + sync_status varchar(20) default 'DISCOVERED' comment '同步状态（DISCOVERED已发现 PENDING待同步 SYNCED已同步 SKIPPED已跳过 FAILED失败 DELETED源端已删除）', + source_deleted char(1) default '0' comment '源端删除标志（0未删除 1已删除）', + first_seen_job_id bigint(20) default null comment '首次发现任务ID', + last_seen_job_id bigint(20) default null comment '最近扫描发现任务ID', + last_sync_job_id bigint(20) default null comment '最近成功同步任务ID', + last_sync_time datetime default null comment '最近成功同步时间', + last_error_message text comment '最近同步错误信息', + create_dept bigint(20) default null comment '创建部门', + create_by bigint(20) default null comment '创建者', + create_time datetime default null comment '创建时间', + update_by bigint(20) default null comment '更新者', + update_time datetime default null comment '更新时间', + primary key (object_id), + unique key uk_sync_object_plan_source (plan_id, source_object_id), + key idx_sync_object_plan_status (plan_id, sync_status, source_deleted), + key idx_sync_object_last_seen (plan_id, last_seen_job_id), + key idx_sync_object_target_key (plan_id, target_key(191)) +) engine = innodb comment = '数据同步源对象状态表'; + +-- ---------------------------- +-- 6、增量检查点表 +-- ---------------------------- +create table sync_checkpoint +( + checkpoint_id bigint(20) not null comment '检查点ID', + plan_id bigint(20) not null comment '同步计划ID', + checkpoint_type varchar(32) not null comment '检查点类型（PAGE_CURSOR分页游标 INCREMENTAL_WATERMARK增量水位 RUN_WATERMARK运行水位）', + checkpoint_key varchar(255) not null comment '检查点作用域键（空间、目录或分区标识）', + checkpoint_value longtext comment '检查点值或JSON快照', + watermark_time datetime default null comment '增量水位时间', + last_job_id bigint(20) default null comment '最近提交检查点的任务ID', + version bigint(20) default 0 comment '乐观锁版本号', + create_dept bigint(20) default null comment '创建部门', + create_by bigint(20) default null comment '创建者', + create_time datetime default null comment '创建时间', + update_by bigint(20) default null comment '更新者', + update_time datetime default null comment '更新时间', + primary key (checkpoint_id), + unique key uk_sync_checkpoint_plan_type_key (plan_id, checkpoint_type, checkpoint_key), + key idx_sync_checkpoint_last_job (last_job_id) +) engine = innodb comment = '数据同步增量检查点表'; + +-- ---------------------------- +-- 7、分片传输记录表 +-- ---------------------------- +create table sync_transfer_part +( + part_id bigint(20) not null comment '分片记录ID', + job_item_id bigint(20) not null comment '任务明细ID', + upload_id varchar(512) not null comment '目标端分片上传会话ID', + part_number int(11) not null comment '分片序号（从1开始）', + part_offset bigint(20) default 0 comment '分片起始字节偏移', + part_size bigint(20) default 0 comment '分片字节数', + transferred_bytes bigint(20) default 0 comment '分片已传输字节数', + part_etag varchar(255) default null comment '目标端分片ETag', + checksum_sha256 char(64) default null comment '分片SHA-256摘要', + status varchar(20) default 'PENDING' comment '分片状态（PENDING待上传 UPLOADING上传中 SUCCESS成功 FAILED失败）', + retry_count int(11) default 0 comment '已重试次数', + start_time datetime default null comment '开始时间', + finish_time datetime default null comment '结束时间', + error_message text comment '错误信息', + create_dept bigint(20) default null comment '创建部门', + create_by bigint(20) default null comment '创建者', + create_time datetime default null comment '创建时间', + update_by bigint(20) default null comment '更新者', + update_time datetime default null comment '更新时间', + primary key (part_id), + unique key uk_sync_transfer_part_item_number (job_item_id, part_number), + key idx_sync_transfer_part_upload_status (upload_id(191), status), + key idx_sync_transfer_part_status_retry (status, retry_count) +) engine = innodb comment = '数据同步分片传输记录表'; + +-- ---------------------------- +-- 8、数据同步菜单 +-- ---------------------------- +insert into sys_menu + (menu_id, menu_name, parent_id, order_num, path, component, query_param, is_frame, is_cache, menu_type, + visible, status, perms, icon, active_menu, ext, create_dept, create_by, create_time, update_by, update_time, remark) +values + (1901400000000000001, '数据同步', 0, 2, 'sync', null, null, 'N', 'Y', 'M', '0', '0', null, 'sync', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步目录'), + (1901400000000000100, '连接管理', 1901400000000000001, 1, 'connection', 'sync/connection/index', null, 'N', 'Y', 'C', '0', '0', 'sync:connection:list', 'link', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步连接管理菜单'), + (1901400000000000101, '同步计划', 1901400000000000001, 2, 'plan', 'sync/plan/index', null, 'N', 'Y', 'C', '0', '0', 'sync:plan:list', 'calendar', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步计划菜单'), + (1901400000000000102, '同步任务', 1901400000000000001, 3, 'job', 'sync/job/index', null, 'N', 'Y', 'C', '0', '0', 'sync:job:list', 'job', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步任务菜单'), + (1901400000000000103, '对象清单', 1901400000000000001, 4, 'object', 'sync/object/index', null, 'N', 'Y', 'C', '0', '0', 'sync:object:list', 'list', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步对象清单菜单'), + (1901400000000001001, '连接查询', 1901400000000000100, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001002, '连接新增', 1901400000000000100, 2, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:add', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001003, '连接修改', 1901400000000000100, 3, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:edit', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001004, '连接删除', 1901400000000000100, 4, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:remove', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001006, '连接测试', 1901400000000000100, 5, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:test', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001007, '钉钉 Web 登录', 1901400000000000100, 6, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:connection:auth', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001011, '计划查询', 1901400000000000101, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001012, '计划新增', 1901400000000000101, 2, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:add', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001013, '计划修改', 1901400000000000101, 3, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:edit', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001014, '计划删除', 1901400000000000101, 4, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:plan:remove', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001016, '立即运行', 1901400000000000101, 5, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:run', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001021, '任务查询', 1901400000000000102, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001022, '任务删除', 1901400000000000102, 2, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:remove', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001024, '取消任务', 1901400000000000102, 3, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:cancel', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001025, '重试任务', 1901400000000000102, 4, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:job:retry', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''), + (1901400000000001031, '对象查询', 1901400000000000103, 1, '#', '', null, 'N', 'Y', 'F', '0', '0', 'sync:object:query', '#', '', '', 1761000000000000103, 1761100000000000001, sysdate(), null, null, ''); + +-- ---------------------------- +-- 9、数据同步字典类型 +-- ---------------------------- +insert into sys_dict_type + (dict_id, dict_name, dict_type, create_dept, create_by, create_time, update_by, update_time, remark) +values + (1901500000000000001, '同步连接角色', 'sync_connection_role', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步连接角色'), + (1901500000000000002, '同步连接类型', 'sync_connection_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据同步连接类型'), + (1901500000000000003, '同步模式', 'sync_mode', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '全量或增量同步模式'), + (1901500000000000004, '同步调度类型', 'sync_schedule_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '手动或Cron调度'), + (1901500000000000005, '同步冲突策略', 'sync_conflict_strategy', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '目标对象冲突处理策略'), + (1901500000000000006, '同步删除策略', 'sync_delete_strategy', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端删除处理策略'), + (1901500000000000007, '同步校验方式', 'sync_verify_mode', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '传输完整性校验方式'), + (1901500000000000008, '同步触发类型', 'sync_trigger_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务触发类型'), + (1901500000000000009, '同步任务状态', 'sync_job_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务运行状态'), + (1901500000000000010, '同步明细状态', 'sync_job_item_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务明细状态'), + (1901500000000000011, '同步对象类型', 'sync_object_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步对象类型'), + (1901500000000000012, '同步动作类型', 'sync_action_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步任务明细动作类型'), + (1901500000000000013, '同步对象状态', 'sync_object_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源对象同步状态'), + (1901500000000000014, '同步分片状态', 'sync_transfer_part_status', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片上传状态'), + (1901500000000000015, '同步检查点类型', 'sync_checkpoint_type', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '增量检查点类型'); + +-- ---------------------------- +-- 10、数据同步字典数据 +-- ---------------------------- +insert into sys_dict_data + (dict_code, dict_sort, dict_label, dict_value, dict_type, css_class, list_class, is_default, + create_dept, create_by, create_time, update_by, update_time, remark) +values + (1901600000000000001, 1, '源端', 'SOURCE', 'sync_connection_role', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据来源连接'), + (1901600000000000002, 2, '目标端', 'TARGET', 'sync_connection_role', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '数据目标连接'), + (1901600000000000003, 1, '钉钉', 'DINGTALK', 'sync_connection_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '钉钉企业网盘'), + (1901600000000000004, 2, '标准S3', 'S3', 'sync_connection_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, 'S3标准兼容存储'), + (1901600000000000005, 3, '阿里云OSS', 'ALIYUN_OSS', 'sync_connection_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '阿里云对象存储OSS'), + (1901600000000000006, 1, '全量同步', 'FULL', 'sync_mode', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '每次扫描全部对象'), + (1901600000000000007, 2, '增量同步', 'INCREMENTAL', 'sync_mode', '', 'success', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '完整枚举后按持久清单差异传输'), + (1901600000000000008, 1, '手动执行', 'MANUAL', 'sync_schedule_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '仅手动触发'), + (1901600000000000009, 2, 'Cron调度', 'CRON', 'sync_schedule_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按Cron表达式触发'), + (1901600000000000010, 1, '覆盖', 'OVERWRITE', 'sync_conflict_strategy', '', 'warning', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '覆盖目标对象'), + (1901600000000000011, 2, '跳过', 'SKIP', 'sync_conflict_strategy', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '保留目标对象并跳过'), + (1901600000000000012, 3, '两者保留', 'KEEP_BOTH', 'sync_conflict_strategy', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '重命名后保留两个对象'), + (1901600000000000013, 1, '保留目标', 'KEEP', 'sync_delete_strategy', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端删除时保留目标对象'), + (1901600000000000014, 2, '仅标记', 'MARK', 'sync_delete_strategy', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '仅记录源端删除状态'), + (1901600000000000015, 3, '删除目标', 'DELETE', 'sync_delete_strategy', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '同步删除目标对象'), + (1901600000000000016, 1, '文件大小', 'SIZE', 'sync_verify_mode', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按对象大小校验'), + (1901600000000000017, 2, 'ETag', 'ETAG', 'sync_verify_mode', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按ETag校验'), + (1901600000000000018, 3, 'SHA-256', 'SHA256', 'sync_verify_mode', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按SHA-256摘要校验'), + (1901600000000000019, 1, '手动触发', 'MANUAL', 'sync_trigger_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '用户手动触发任务'), + (1901600000000000020, 2, '调度触发', 'SCHEDULE', 'sync_trigger_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '计划调度触发任务'), + (1901600000000000021, 3, '重试触发', 'RETRY', 'sync_trigger_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '失败任务重试触发'), + (1901600000000000022, 1, '待执行', 'PENDING', 'sync_job_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '等待执行'), + (1901600000000000023, 2, '执行中', 'RUNNING', 'sync_job_status', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '正在执行'), + (1901600000000000024, 3, '成功', 'SUCCESS', 'sync_job_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '全部成功'), + (1901600000000000025, 4, '部分失败', 'PARTIAL_FAILED', 'sync_job_status', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '部分对象失败'), + (1901600000000000026, 5, '失败', 'FAILED', 'sync_job_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '任务失败'), + (1901600000000000027, 6, '已取消', 'CANCELED', 'sync_job_status', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '任务已取消'), + (1901600000000000053, 7, '取消中', 'CANCELING', 'sync_job_status', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '正在终止本节点传输'), + (1901600000000000028, 1, '待执行', 'PENDING', 'sync_job_item_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '等待执行'), + (1901600000000000029, 2, '执行中', 'RUNNING', 'sync_job_item_status', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '正在执行'), + (1901600000000000030, 3, '成功', 'SUCCESS', 'sync_job_item_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '对象同步成功'), + (1901600000000000031, 4, '已跳过', 'SKIPPED', 'sync_job_item_status', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '对象已跳过'), + (1901600000000000032, 5, '失败', 'FAILED', 'sync_job_item_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '对象同步失败'), + (1901600000000000033, 1, '文件', 'FILE', 'sync_object_type', '', 'primary', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '普通文件'), + (1901600000000000034, 2, '目录', 'FOLDER', 'sync_object_type', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '目录对象'), + (1901600000000000035, 3, '在线文档', 'ONLINE_DOCUMENT', 'sync_object_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '需导出的在线文档'), + (1901600000000000036, 1, '新增上传', 'CREATE', 'sync_action_type', '', 'success', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '首次上传目标对象'), + (1901600000000000037, 2, '更新上传', 'UPDATE', 'sync_action_type', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '更新或覆盖目标对象'), + (1901600000000000038, 3, '跳过', 'SKIP', 'sync_action_type', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '跳过目标操作'), + (1901600000000000051, 4, '删除', 'DELETE', 'sync_action_type', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '删除目标对象'), + (1901600000000000039, 1, '已发现', 'DISCOVERED', 'sync_object_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '扫描已发现'), + (1901600000000000040, 2, '待同步', 'PENDING', 'sync_object_status', '', 'warning', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '等待同步'), + (1901600000000000041, 3, '已同步', 'SYNCED', 'sync_object_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '最近同步成功'), + (1901600000000000042, 4, '已跳过', 'SKIPPED', 'sync_object_status', '', 'info', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '按策略跳过'), + (1901600000000000043, 5, '失败', 'FAILED', 'sync_object_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '最近同步失败'), + (1901600000000000044, 6, '源端已删除', 'DELETED', 'sync_object_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端对象已删除'), + (1901600000000000045, 1, '待上传', 'PENDING', 'sync_transfer_part_status', '', 'info', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片等待上传'), + (1901600000000000046, 2, '上传中', 'UPLOADING', 'sync_transfer_part_status', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片正在上传'), + (1901600000000000047, 3, '成功', 'SUCCESS', 'sync_transfer_part_status', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片上传成功'), + (1901600000000000048, 4, '失败', 'FAILED', 'sync_transfer_part_status', '', 'danger', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '分片上传失败'), + (1901600000000000049, 1, '分页游标', 'PAGE_CURSOR', 'sync_checkpoint_type', '', 'primary', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端分页扫描游标'), + (1901600000000000050, 2, '增量水位', 'INCREMENTAL_WATERMARK', 'sync_checkpoint_type', '', 'success', 'N', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '源端增量同步水位'), + (1901600000000000052, 3, '运行水位', 'RUN_WATERMARK', 'sync_checkpoint_type', '', 'warning', 'Y', 1761000000000000103, 1761100000000000001, sysdate(), null, null, '当前版本任务运行水位'); diff --git a/script/sql/ry_vue.sql b/script/sql/ry_vue.sql index 9841276a6..c87fd8e14 100644 --- a/script/sql/ry_vue.sql +++ b/script/sql/ry_vue.sql @@ -82,7 +82,7 @@ create table sys_user ( user_id bigint(20) not null comment '用户ID', dept_id bigint(20) default null comment '部门ID', user_name varchar(30) not null comment '用户账号', - nick_name varchar(30) not null comment '用户昵称', + nick_name varchar(64) not null comment '用户昵称', user_type varchar(10) default 'sys_user' comment '用户类型（sys_user系统用户）', email varchar(50) default '' comment '用户邮箱', phone_number varchar(11) default '' comment '手机号码', @@ -931,4 +931,3 @@ INSERT INTO test_tree VALUES (1762200000000000010, 1762200000000000007, 17610000 INSERT INTO test_tree VALUES (1762200000000000011, 1762200000000000007, 1761000000000000108, 1761100000000000003, '子节点77', 0, 1761000000000000103, sysdate(), 1761100000000000001, NULL, NULL, 0); INSERT INTO test_tree VALUES (1762200000000000012, 1762200000000000010, 1761000000000000108, 1761100000000000003, '子节点88', 0, 1761000000000000103, sysdate(), 1761100000000000001, NULL, NULL, 0); INSERT INTO test_tree VALUES (1762200000000000013, 1762200000000000010, 1761000000000000108, 1761100000000000003, '子节点99', 0, 1761000000000000103, sysdate(), 1761100000000000001, NULL, NULL, 0); - diff --git a/ui/data-sync-s3-vue b/ui/data-sync-s3-vue new file mode 160000 index 000000000..a85fa0aee --- /dev/null +++ b/ui/data-sync-s3-vue @@ -0,0 +1 @@ +Subproject commit a85fa0aee44f6f12dc35198126914ce722ee8622 From d857b4b74be8bcf14d7b29c42c1be94ffae60101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=A1=E9=92=B1?= Date: Thu, 3 Sep 2026 18:18:59 +0800 Subject: [PATCH 2/5] =?UTF-8?q?add=20=E6=96=B0=E5=A2=9E=20=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E7=9B=AE=E6=A0=87=E7=AB=AF=E6=9C=8D=E5=8A=A1=E7=AB=AF?= =?UTF-8?q?=E5=8A=A0=E5=AF=86(SSE-S3/SSE-KMS)=E5=8F=AF=E9=80=89=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 目标连接此前不发送任何服务端加密请求头，实际加密方式完全由存储桶默认策略 决定，换到默认策略不同或不支持 KMS 的服务端时行为不可控。 新增连接扩展配置 sseMode(NONE/SSE_S3/SSE_KMS)与 sseKmsKeyId，显式指定加密 方式。默认 NONE 保持不发送加密请求头的原有行为，对不支持 SSE 的服务端无影响。 - S3SseSetting 统一负责加密配置的解析、校验与请求头应用 - SyncS3OssClient 增加带 SSE 的上传，走底层 doCustomUpload 自行构建 PutObject 请求，不修改公共 oss 模块 API - 上传后比对 headObject 返回的实际加密方式，不一致仅告警不中断同步， 便于发现服务端静默忽略加密请求头的情况 - 连接保存与连通性测试阶段前置校验非法的加密配置组合 --- .../s3/AbstractS3TargetConnector.java | 29 ++++- .../sync/connector/s3/S3SseSetting.java | 112 ++++++++++++++++++ .../sync/connector/s3/SyncS3OssClient.java | 71 ++++++++++- .../dromara/sync/constant/SyncConstants.java | 15 +++ .../impl/SyncConnectionServiceImpl.java | 3 + 5 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/S3SseSetting.java diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AbstractS3TargetConnector.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AbstractS3TargetConnector.java index d75f32c5d..53feb3117 100644 --- a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AbstractS3TargetConnector.java +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/AbstractS3TargetConnector.java @@ -1,6 +1,7 @@ package org.dromara.sync.connector.s3; import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; import org.dromara.common.core.exception.ServiceException; import org.dromara.common.core.utils.StringUtils; import org.dromara.common.json.utils.JsonUtils; @@ -23,6 +24,7 @@ import java.util.concurrent.ConcurrentHashMap; /** * 基于仓库原生 S3 客户端的目标连接器基础实现。 */ +@Slf4j public abstract class AbstractS3TargetConnector implements TargetConnector { private final Map clients = new ConcurrentHashMap<>(); @@ -30,6 +32,7 @@ public abstract class AbstractS3TargetConnector implements TargetConnector { @Override public ConnectorTestResult test(SyncConnection connection) { try { + S3SseSetting.from(parseJson(connection.getConfigJson())); client(connection).headBucket(connection.getBucketName()); return ConnectorTestResult.success("目标存储桶访问成功"); } catch (Exception e) { @@ -44,12 +47,36 @@ public abstract class AbstractS3TargetConnector implements TargetConnector { .setContentType(request.contentType()) .setMetadata(request.metadata()); SyncS3OssClient client = client(connection); - PutObjectResult result = client.upload(request.objectKey(), request.path(), options); + S3SseSetting sse = S3SseSetting.from(parseJson(connection.getConfigJson())); + PutObjectResult result = client.upload(connection.getBucketName(), request.objectKey(), + request.path(), options, sse); HeadObjectResponse head = client.headObject(connection.getBucketName(), result.key()); + verifyEncryption(sse, head, result.key()); return new TargetWriteResult(result.key(), head.versionId(), head.eTag(), head.contentLength(), result.url(), head.metadata()); } + /** + * 校验服务端实际生效的加密方式与连接配置是否一致。 + * + *

不同厂商对加密请求头的支持程度不一致，部分服务端会静默忽略。此处仅告警不中断同步， + * 便于在日志中及时发现目标端不支持所选加密方式的情况。

+ * + * @param sse 期望的加密设置 + * @param head 对象元信息 + * @param objectKey 对象键 + */ + private void verifyEncryption(S3SseSetting sse, HeadObjectResponse head, String objectKey) { + if (!sse.enabled()) { + return; + } + String actual = head.serverSideEncryptionAsString(); + if (!sse.expectedAlgorithm().equals(actual)) { + log.warn("目标端未按配置应用服务端加密，期望={}，实际={}，objectKey={}", + sse.expectedAlgorithm(), actual == null ? "未加密" : actual, objectKey); + } + } + @Override public boolean exists(SyncConnection connection, String objectKey) { return client(connection).objectExists(connection.getBucketName(), objectKey); diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/S3SseSetting.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/S3SseSetting.java new file mode 100644 index 000000000..08df58ae5 --- /dev/null +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/S3SseSetting.java @@ -0,0 +1,112 @@ +package org.dromara.sync.connector.s3; + +import org.dromara.common.core.exception.ServiceException; +import org.dromara.common.core.utils.StringUtils; +import org.dromara.sync.constant.SyncConstants; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.ServerSideEncryption; + +import java.util.List; +import java.util.Map; + +/** + * 目标端服务端加密（SSE）设置。 + * + *

加密方式由连接扩展配置的 {@code sseMode} 决定，取值为 + * {@link SyncConstants#SSE_NONE}、{@link SyncConstants#SSE_S3} 或 {@link SyncConstants#SSE_KMS}。 + * 默认 {@code NONE} 表示不发送任何加密请求头，完全由存储桶自身的默认加密策略决定， + * 以保证对不支持 SSE 的服务端保持兼容。

+ * + * @param mode 加密方式 + * @param kmsKeyId KMS 密钥标识，仅 {@code SSE_KMS} 且需要指定密钥时使用 + */ +public record S3SseSetting(String mode, String kmsKeyId) { + + /** + * 支持的加密方式集合。 + */ + private static final List SUPPORTED_MODES = + List.of(SyncConstants.SSE_NONE, SyncConstants.SSE_S3, SyncConstants.SSE_KMS); + + /** + * 不指定加密方式，跟随存储桶默认策略。 + */ + public static final S3SseSetting NONE = new S3SseSetting(SyncConstants.SSE_NONE, null); + + /** + * 从连接扩展配置解析加密设置。 + * + * @param config 连接扩展配置 + * @return 加密设置，未配置时返回 {@link #NONE} + */ + public static S3SseSetting from(Map config) { + if (config == null || config.isEmpty()) { + return NONE; + } + String mode = normalize(config.get("sseMode")); + String kmsKeyId = StringUtils.trimToNull(stringValue(config.get("sseKmsKeyId"))); + if (StringUtils.isBlank(mode)) { + mode = SyncConstants.SSE_NONE; + } + if (!SUPPORTED_MODES.contains(mode)) { + throw new ServiceException("目标连接 sseMode 只支持 {}", String.join("、", SUPPORTED_MODES)); + } + if (kmsKeyId != null && !SyncConstants.SSE_KMS.equals(mode)) { + throw new ServiceException("仅 SSE_KMS 加密方式可以指定 sseKmsKeyId"); + } + return new S3SseSetting(mode, kmsKeyId); + } + + /** + * 是否需要显式发送服务端加密请求头。 + * + * @return 需要发送返回 {@code true} + */ + public boolean enabled() { + return !SyncConstants.SSE_NONE.equals(mode); + } + + /** + * 将加密设置应用到上传请求。 + * + *

{@code NONE} 不写入任何加密请求头，由服务端按存储桶默认策略处理。

+ * + * @param builder PutObject 请求构建器 + */ + public void applyTo(PutObjectRequest.Builder builder) { + switch (mode) { + case SyncConstants.SSE_S3 -> builder.serverSideEncryption(ServerSideEncryption.AES256); + case SyncConstants.SSE_KMS -> { + builder.serverSideEncryption(ServerSideEncryption.AWS_KMS); + if (StringUtils.isNotBlank(kmsKeyId)) { + builder.ssekmsKeyId(kmsKeyId); + } + } + default -> { + // NONE 不发送加密请求头，保持对不支持 SSE 的服务端的兼容。 + } + } + } + + /** + * 获取期望的服务端加密算法标识，用于回写校验。 + * + * @return 加密算法标识；{@code NONE} 返回 {@code null} + */ + public String expectedAlgorithm() { + return switch (mode) { + case SyncConstants.SSE_S3 -> ServerSideEncryption.AES256.toString(); + case SyncConstants.SSE_KMS -> ServerSideEncryption.AWS_KMS.toString(); + default -> null; + }; + } + + private static String normalize(Object value) { + String text = stringValue(value); + return text == null ? null : text.trim().toUpperCase(); + } + + private static String stringValue(Object value) { + return value == null ? null : String.valueOf(value); + } +} diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/SyncS3OssClient.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/SyncS3OssClient.java index f023a0169..6005d7cb1 100644 --- a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/SyncS3OssClient.java +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/connector/s3/SyncS3OssClient.java @@ -2,13 +2,22 @@ package org.dromara.sync.connector.s3; import org.dromara.common.oss.client.DefaultOssClientImpl; import org.dromara.common.oss.config.OssClientConfig; +import org.dromara.common.oss.exception.S3StorageException; +import org.dromara.common.oss.model.HandleAsyncResult; +import org.dromara.common.oss.model.Options; +import org.dromara.common.oss.model.PutObjectResult; +import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; +import java.nio.file.Path; +import java.util.Optional; import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; /** - * 为同步连接器补充只读的存储桶连通性探测。 + * 为同步连接器补充只读的存储桶连通性探测与服务端加密上传。 */ class SyncS3OssClient extends DefaultOssClientImpl { @@ -41,4 +50,64 @@ class SyncS3OssClient extends DefaultOssClientImpl { throw e; } } + + /** + * 上传本地文件并按连接配置附加服务端加密请求头。 + * + *

公共 OSS 模块的 {@link Options} 不支持服务端加密，因此这里直接使用底层 + * {@code doCustomUpload} 自行构建 PutObject 请求，避免修改公共模块 API。

+ * + * @param bucket 存储桶名称 + * @param key 对象键 + * @param path 待上传文件 + * @param options 上传选项 + * @param sse 服务端加密设置 + * @return 上传结果 + */ + PutObjectResult upload(String bucket, String key, Path path, Options options, S3SseSetting sse) { + AsyncRequestBody body = AsyncRequestBody.fromFile(path); + // 与公共模块 bucketUpload 保持一致：请求体自带的长度与类型优先于可选项。 + Long contentLength = body.contentLength().orElse(options.getLength()); + String contentType = options.getContentType() == null || options.getContentType().isBlank() + ? body.contentType() : options.getContentType(); + HandleAsyncResult result = doCustomUpload(body, builder -> { + builder.bucket(bucket) + .key(key) + .contentMD5(options.getMd5Digest()) + .contentType(contentType) + .contentLength(contentLength) + .metadata(options.getMetadata()); + sse.applyTo(builder); + }, options.getTransferListeners()); + if (result.isFailure()) { + throw toStorageException(result.error()); + } + Optional opt = result.getResult(); + if (opt.isEmpty()) { + throw S3StorageException.form("response is empty."); + } + PutObjectResponse response = opt.get(); + Long size = response.size(); + if (size == null) { + size = contentLength == null ? 0 : contentLength; + } + return PutObjectResult.form("%s/%s".formatted(config.getBucketUrl(bucket), key), key, response.eTag(), size); + } + + /** + * 转换为统一的 S3 存储异常。 + * + * @param e 原始异常 + * @return S3 存储异常 + */ + private S3StorageException toStorageException(Throwable e) { + Throwable cause = e; + while ((cause instanceof CompletionException || cause instanceof ExecutionException) && cause.getCause() != null) { + cause = cause.getCause(); + } + if (cause instanceof S3StorageException ex) { + return ex; + } + return S3StorageException.form(cause); + } } diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/constant/SyncConstants.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/constant/SyncConstants.java index cd91a8294..562e6e456 100644 --- a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/constant/SyncConstants.java +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/constant/SyncConstants.java @@ -32,4 +32,19 @@ public interface SyncConstants { String OBJECT_FILE = "FILE"; String OBJECT_FOLDER = "FOLDER"; String OBJECT_ONLINE_DOCUMENT = "ONLINE_DOCUMENT"; + + /** + * 目标端服务端加密方式：不指定，由存储桶默认策略决定。 + */ + String SSE_NONE = "NONE"; + + /** + * 目标端服务端加密方式：SSE-S3，由服务端托管密钥（AES256）。 + */ + String SSE_S3 = "SSE_S3"; + + /** + * 目标端服务端加密方式：SSE-KMS，由 KMS 托管密钥。 + */ + String SSE_KMS = "SSE_KMS"; } diff --git a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncConnectionServiceImpl.java b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncConnectionServiceImpl.java index b8b7cc753..c275156a1 100644 --- a/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncConnectionServiceImpl.java +++ b/ruoyi-modules/ruoyi-sync/src/main/java/org/dromara/sync/service/impl/SyncConnectionServiceImpl.java @@ -15,6 +15,7 @@ import org.dromara.sync.connector.ConnectorRegistry; import org.dromara.sync.connector.dingtalk.DwsAuthIdentity; import org.dromara.sync.connector.dingtalk.DwsAuthSessionManager; import org.dromara.sync.connector.model.ConnectorTestResult; +import org.dromara.sync.connector.s3.S3SseSetting; import org.dromara.sync.constant.SyncConstants; import org.dromara.sync.domain.SyncConnection; import org.dromara.sync.domain.SyncJob; @@ -328,6 +329,8 @@ public class SyncConnectionServiceImpl implements ISyncConnectionService { requireSecret(secrets, "accessKey", "目标连接 accessKey 不能为空"); requireSecret(secrets, "secretKey", "目标连接 secretKey 不能为空"); } + // 加密方式与 KMS 密钥组合的合法性由连接器统一定义，保存时提前拦截非法配置。 + S3SseSetting.from(parseOptionalJson(connection.getConfigJson())); } /** From 350ef62c85cacd536d757c798625805382bd441a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=A1=E9=92=B1?= Date: Thu, 3 Sep 2026 18:20:42 +0800 Subject: [PATCH 3/5] =?UTF-8?q?update=20=E4=BC=98=E5=8C=96=20=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E5=89=8D=E7=AB=AF=E5=AD=90=E9=A1=B9=E7=9B=AE=E6=8C=87?= =?UTF-8?q?=E9=92=88=E8=87=B3=E6=95=B0=E6=8D=AE=E5=90=8C=E6=AD=A5=E6=A8=A1?= =?UTF-8?q?=E5=9D=97=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ui/data-sync-s3-vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/data-sync-s3-vue b/ui/data-sync-s3-vue index a85fa0aee..cfab6cebb 160000 --- a/ui/data-sync-s3-vue +++ b/ui/data-sync-s3-vue @@ -1 +1 @@ -Subproject commit a85fa0aee44f6f12dc35198126914ce722ee8622 +Subproject commit cfab6cebb66815bdf0e27ee8c1aa8b36593597e5 From 8b174407cc7a40b2f32c41d5ff6f04e871f5e22d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=A1=E9=92=B1?= Date: Fri, 11 Sep 2026 11:03:19 +0800 Subject: [PATCH 4/5] init --- deploy/docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 4f8fb7125..554c0645e 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -85,7 +85,7 @@ services: ports: # The host already runs another Nginx on port 80. Expose this stack on # 8080 and let the existing reverse proxy forward traffic here. - - "8080:80" + - "38080:80" depends_on: backend: condition: service_healthy From a42ce86ae4f4a1efa31100d874951d3c642e90a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=A1=E9=92=B1?= Date: Fri, 11 Sep 2026 12:05:52 +0800 Subject: [PATCH 5/5] init --- .DS_Store | Bin 10244 -> 12292 bytes deploy/docker-compose.backend.yml | 44 ++++++++++++++++++ ruoyi-admin/.DS_Store | Bin 0 -> 6148 bytes .../src/main/resources/application-prod.yml | 12 ++--- .../src/main/resources/application.yml | 4 +- 5 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 deploy/docker-compose.backend.yml create mode 100644 ruoyi-admin/.DS_Store diff --git a/.DS_Store b/.DS_Store index 7f96dc19ccc1100a115f609d53c75988a715ffe4..73724f2a443eb0e576cffe66e1a6b1ec7cb01d85 100644 GIT binary patch delta 495 zcmZn(Xh~3DU|?W$DortDV9)?EIe-{M3-ADifgA>Ml{jDnj@gf8=LY|vq{2b#%L1f=Kv2Lm7rP3`0Y!3GgyQ*#{!BSZ6A9ffL3 zBLf`;6JxW<8$?aHkqq7ZLG(PMIFdkFa8X`PeqK7zM8?fL0`iQLzY2)sGtO9?o1uuI z6v>k*47ot><^hd9%czV)64{!`8iGcX6C}?`F|dMt!l1zoq+OB2bz|Xs=E?jjmJ%QZ bz_4S2(kxJ#V{(SdZ0T+w4~KoEI_3-jZy#ze delta 160 zcmZokXbF&DU|?W$DortDU{C-uIe-{M3-C-V6q~3gIoUvmMH0wo01=EpaRvrIhF}I) z&z$_^q@2wkC8XFV3ovzVX6InxV3gd)AT|i z&-$eA{s3Qn5(M$lU*MbQKhQrQdgc;q67{jD?17o@%(>0Xx6IBi0FYFzI1QiyKolFt za1VCB5bo!#Lqf_DK_sFE1s6(Sfeo}HIxqwb1PuIb4Dhqt153cbgl#;vKhAI3W@f== z1@!ZmLkipEHzBA2M4*Z}6_|uw?CrMbmu=Bq;9+I6HHwV2MRkJ@2F8_f9e!Kvvp>Za z>Rwf?^;My2Tb@5P^`3;f!Y8^TkyxZRvXyzn>KWhgYhKE5H~1*am|bY|{qOrU-a&RzvX?{YVnVvtK&=0Q6Oik!> zvzHGKW6^lx;?+C(ZM)*I$GCnJg6~NoeF~fK1eKQkTjq}16M56z(??BHyNbqodQbM9 z>emJahX#fR;_>+DGb5vC&yD7_p@Ly3c7sGs{uU(aj(p6N2%US*!;Tg+MC^pNYc zPb+Ds#BjzPLlCoMkdcuDoBA#o?+; zdCA1W7s+HIx!hDPNJWE$RPYwlNaAhrlNiRb4s&QCxhkD8yyC{;I^2deJO}&m6rRHi zcn$C11AK73W z_GV5q_;^vx9_>BeX@l7U1_B2D0S5T{LC3~1sjwkYygIN8;gI{o9J!#*KNmRa3X=*O z648S~WGbReCF&Q0$aJ*pDlVz8AyK9S5i?^QH8W9vC`8PTc3p%6NlFa*fPsL4RtCD{ zP~-Rijo<(Ow+&1-vJ&3`=$T@ literal 0 HcmV?d00001 diff --git a/ruoyi-admin/src/main/resources/application-prod.yml b/ruoyi-admin/src/main/resources/application-prod.yml index e581b182c..20761afdc 100644 --- a/ruoyi-admin/src/main/resources/application-prod.yml +++ b/ruoyi-admin/src/main/resources/application-prod.yml @@ -103,9 +103,9 @@ spring: driverClassName: com.mysql.cj.jdbc.Driver # jdbc 所有参数配置参考 https://lionli.blog.csdn.net/article/details/122018562 # rewriteBatchedStatements=true 批处理优化 大幅提升批量插入更新删除性能(对数据库有性能损耗 使用批量操作应考虑性能问题) - url: jdbc:mysql://localhost:3306/ry-vue?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true - username: root - password: root + url: jdbc:mysql://kfn-data-sync-s3-mysql:33306/data-sync-s3?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true + username: data-sync-s3 + password: PSGrMaQdZ8z7AfiJ # # 从库数据源 # slave: # lazy: true @@ -152,13 +152,13 @@ spring: spring.data: redis: # 地址 - host: localhost + host: kfn-data-sync-s3 # 端口，默认为6379 - port: 6379 + port: 36379 # 数据库索引 database: 0 # redis 密码必须配置 - password: ruoyi123 + password: redis_RpMMYK # 连接超时时间 timeout: 10s # 是否开启ssl diff --git a/ruoyi-admin/src/main/resources/application.yml b/ruoyi-admin/src/main/resources/application.yml index 052e058fd..cc2cb15dd 100644 --- a/ruoyi-admin/src/main/resources/application.yml +++ b/ruoyi-admin/src/main/resources/application.yml @@ -93,9 +93,9 @@ spring: servlet: multipart: # 单个文件大小 - max-file-size: 10MB + max-file-size: 1000MB # 设置总上传的文件大小 - max-request-size: 20MB + max-request-size: 2000MB mvc: # 设置静态资源路径 防止所有请求都去查静态资源 static-path-pattern: /static/**