",
+ "

",
+ ]
+
+ for name in invalid_names:
+ with pytest.raises(ValueError) as exc_info:
+ CreateAppPayload(name=name, mode="chat")
+ assert "invalid characters or patterns" in str(exc_info.value).lower()
From 27be89c9848d39b2f53c2e5bebc1e4e870db8489 Mon Sep 17 00:00:00 2001
From: Stephen Zhou <38493346+hyoban@users.noreply.github.com>
Date: Wed, 31 Dec 2025 15:31:11 +0800
Subject: [PATCH 15/25] chore: lint for react compiler (#30417)
---
web/eslint.config.mjs | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/web/eslint.config.mjs b/web/eslint.config.mjs
index 89f6d292cd..574dbb091e 100644
--- a/web/eslint.config.mjs
+++ b/web/eslint.config.mjs
@@ -13,6 +13,24 @@ export default antfu(
'react/no-forward-ref': 'off',
'react/no-use-context': 'off',
'react/prefer-namespace-import': 'error',
+
+ // React Compiler rules
+ // Set to warn for gradual adoption
+ 'react-hooks/config': 'warn',
+ 'react-hooks/error-boundaries': 'warn',
+ 'react-hooks/component-hook-factories': 'warn',
+ 'react-hooks/gating': 'warn',
+ 'react-hooks/globals': 'warn',
+ 'react-hooks/immutability': 'warn',
+ 'react-hooks/preserve-manual-memoization': 'warn',
+ 'react-hooks/purity': 'warn',
+ 'react-hooks/refs': 'warn',
+ 'react-hooks/set-state-in-effect': 'warn',
+ 'react-hooks/set-state-in-render': 'warn',
+ 'react-hooks/static-components': 'warn',
+ 'react-hooks/unsupported-syntax': 'warn',
+ 'react-hooks/use-memo': 'warn',
+ 'react-hooks/incompatible-library': 'warn',
},
},
nextjs: true,
From e856287b65fc479a250ef2e2a9afd06b155f90e0 Mon Sep 17 00:00:00 2001
From: Stephen Zhou <38493346+hyoban@users.noreply.github.com>
Date: Wed, 31 Dec 2025 15:38:07 +0800
Subject: [PATCH 16/25] chore: update knip config and include in CI (#30410)
---
.github/workflows/style.yml | 5 +
web/eslint-rules/rules/no-as-any-in-t.js | 5 -
.../rules/no-legacy-namespace-prefix.js | 25 --
web/eslint-rules/rules/require-ns-option.js | 5 -
web/knip.config.ts | 266 +-----------------
web/package.json | 6 +-
web/pnpm-lock.yaml | 29 +-
.../script.mjs => scripts/gen-icons.mjs} | 13 +-
8 files changed, 30 insertions(+), 324 deletions(-)
rename web/{app/components/base/icons/script.mjs => scripts/gen-icons.mjs} (91%)
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
index d463349686..39b559f4ca 100644
--- a/.github/workflows/style.yml
+++ b/.github/workflows/style.yml
@@ -110,6 +110,11 @@ jobs:
working-directory: ./web
run: pnpm run type-check:tsgo
+ - name: Web dead code check
+ if: steps.changed-files.outputs.any_changed == 'true'
+ working-directory: ./web
+ run: pnpm run knip
+
superlinter:
name: SuperLinter
runs-on: ubuntu-latest
diff --git a/web/eslint-rules/rules/no-as-any-in-t.js b/web/eslint-rules/rules/no-as-any-in-t.js
index 0eb134a3cf..5e4ffc8c1c 100644
--- a/web/eslint-rules/rules/no-as-any-in-t.js
+++ b/web/eslint-rules/rules/no-as-any-in-t.js
@@ -29,11 +29,6 @@ export default {
const options = context.options[0] || {}
const mode = options.mode || 'any'
- /**
- * Check if this is a t() function call
- * @param {import('estree').CallExpression} node
- * @returns {boolean}
- */
function isTCall(node) {
// Direct t() call
if (node.callee.type === 'Identifier' && node.callee.name === 't')
diff --git a/web/eslint-rules/rules/no-legacy-namespace-prefix.js b/web/eslint-rules/rules/no-legacy-namespace-prefix.js
index dc268c9b65..023e6b73d3 100644
--- a/web/eslint-rules/rules/no-legacy-namespace-prefix.js
+++ b/web/eslint-rules/rules/no-legacy-namespace-prefix.js
@@ -19,26 +19,11 @@ export default {
create(context) {
const sourceCode = context.sourceCode
- // Track all t() calls to fix
- /** @type {Array<{ node: import('estree').CallExpression }>} */
const tCallsToFix = []
-
- // Track variables with namespace prefix
- /** @type {Map
} */
const variablesToFix = new Map()
-
- // Track all namespaces used in the file (from legacy prefix detection)
- /** @type {Set} */
const namespacesUsed = new Set()
-
- // Track variable values for template literal analysis
- /** @type {Map} */
const variableValues = new Map()
- /**
- * Analyze a template literal and extract namespace info
- * @param {import('estree').TemplateLiteral} node
- */
function analyzeTemplateLiteral(node) {
const quasis = node.quasis
const expressions = node.expressions
@@ -78,11 +63,6 @@ export default {
return { ns: null, canFix: false, fixedQuasis: null, variableToUpdate: null }
}
- /**
- * Build a fixed template literal string
- * @param {string[]} quasis
- * @param {import('estree').Expression[]} expressions
- */
function buildTemplateLiteral(quasis, expressions) {
let result = '`'
for (let i = 0; i < quasis.length; i++) {
@@ -95,11 +75,6 @@ export default {
return result
}
- /**
- * Check if a t() call already has ns in its second argument
- * @param {import('estree').CallExpression} node
- * @returns {boolean}
- */
function hasNsArgument(node) {
if (node.arguments.length < 2)
return false
diff --git a/web/eslint-rules/rules/require-ns-option.js b/web/eslint-rules/rules/require-ns-option.js
index df8f7ec2e8..74621596fd 100644
--- a/web/eslint-rules/rules/require-ns-option.js
+++ b/web/eslint-rules/rules/require-ns-option.js
@@ -12,11 +12,6 @@ export default {
},
},
create(context) {
- /**
- * Check if a t() call has ns in its second argument
- * @param {import('estree').CallExpression} node
- * @returns {boolean}
- */
function hasNsOption(node) {
if (node.arguments.length < 2)
return false
diff --git a/web/knip.config.ts b/web/knip.config.ts
index 975a85b997..414b00fb7f 100644
--- a/web/knip.config.ts
+++ b/web/knip.config.ts
@@ -1,277 +1,33 @@
import type { KnipConfig } from 'knip'
/**
- * Knip Configuration for Dead Code Detection
- *
- * This configuration helps identify unused files, exports, and dependencies
- * in the Dify web application (Next.js 15 + TypeScript + React 19).
- *
- * ⚠️ SAFETY FIRST: This configuration is designed to be conservative and
- * avoid false positives that could lead to deleting actively used code.
- *
* @see https://knip.dev/reference/configuration
*/
const config: KnipConfig = {
- // ============================================================================
- // Next.js Framework Configuration
- // ============================================================================
- // Configure entry points specific to Next.js application structure.
- // These files are automatically treated as entry points by the framework.
- next: {
- entry: [
- // Next.js App Router pages (must exist for routing)
- 'app/**/page.tsx',
- 'app/**/layout.tsx',
- 'app/**/loading.tsx',
- 'app/**/error.tsx',
- 'app/**/not-found.tsx',
- 'app/**/template.tsx',
- 'app/**/default.tsx',
-
- // Middleware (runs before every route)
- 'middleware.ts',
-
- // Configuration files
- 'next.config.js',
- 'tailwind.config.js',
- 'tailwind-common-config.ts',
- 'postcss.config.js',
-
- // Linting configuration
- 'eslint.config.mjs',
- ],
- },
-
- // ============================================================================
- // Global Entry Points
- // ============================================================================
- // Files that serve as entry points for the application.
- // The '!' suffix means these patterns take precedence and are always included.
entry: [
- // Next.js App Router patterns (high priority)
- 'app/**/page.tsx!',
- 'app/**/layout.tsx!',
- 'app/**/loading.tsx!',
- 'app/**/error.tsx!',
- 'app/**/not-found.tsx!',
- 'app/**/template.tsx!',
- 'app/**/default.tsx!',
-
- // Core configuration files
- 'middleware.ts!',
- 'next.config.js!',
- 'tailwind.config.js!',
- 'tailwind-common-config.ts!',
- 'postcss.config.js!',
-
- // Linting setup
- 'eslint.config.mjs!',
-
- // ========================================================================
- // 🔒 CRITICAL: Global Initializers and Providers
- // ========================================================================
- // These files are imported by root layout.tsx and provide global functionality.
- // Even if not directly imported elsewhere, they are essential for app initialization.
-
- // Browser initialization (runs on client startup)
- 'app/components/browser-initializer.tsx!',
- 'app/components/sentry-initializer.tsx!',
- 'app/components/app-initializer.tsx!',
-
- // i18n initialization (server and client)
- 'app/components/i18n.tsx!',
- 'app/components/i18n-server.tsx!',
-
- // Route prefix handling (used in root layout)
- 'app/routePrefixHandle.tsx!',
-
- // ========================================================================
- // 🔒 CRITICAL: Context Providers
- // ========================================================================
- // Context providers might be used via React Context API and imported dynamically.
- // Protecting all context files to prevent breaking the provider chain.
- 'context/**/*.ts?(x)!',
-
- // Component-level contexts (also used via React.useContext)
- 'app/components/**/*.context.ts?(x)!',
-
- // ========================================================================
- // 🔒 CRITICAL: State Management Stores
- // ========================================================================
- // Zustand stores might be imported dynamically or via hooks.
- // These are often imported at module level, so they should be protected.
- 'app/components/**/*.store.ts?(x)!',
- 'context/**/*.store.ts?(x)!',
-
- // ========================================================================
- // 🔒 CRITICAL: Provider Components
- // ========================================================================
- // Provider components wrap the app and provide global state/functionality
- 'app/components/**/*.provider.ts?(x)!',
- 'context/**/*.provider.ts?(x)!',
-
- // ========================================================================
- // Development tools
- // ========================================================================
- // Storybook configuration
- '.storybook/**/*',
+ 'scripts/**/*.{js,ts,mjs}',
+ 'bin/**/*.{js,ts,mjs}',
],
-
- // ============================================================================
- // Project Files to Analyze
- // ============================================================================
- // Glob patterns for files that should be analyzed for unused code.
- // Excludes test files to avoid false positives.
- project: [
- '**/*.{js,jsx,ts,tsx,mjs,cjs}',
- ],
-
- // ============================================================================
- // Ignored Files and Directories
- // ============================================================================
- // Files and directories that should be completely excluded from analysis.
- // These typically contain:
- // - Test files
- // - Internationalization files (loaded dynamically)
- // - Static assets
- // - Build outputs
- // - External libraries
ignore: [
- // Test files and directories
- '**/__tests__/**',
- '**/*.spec.{ts,tsx}',
- '**/*.test.{ts,tsx}',
-
- // ========================================================================
- // 🔒 CRITICAL: i18n Files (Dynamically Loaded)
- // ========================================================================
- // Internationalization files are loaded dynamically at runtime via i18next.
- // Pattern: import(`@/i18n/${locale}/messages`)
- // These will NEVER show up in static analysis but are essential!
'i18n/**',
-
- // ========================================================================
- // 🔒 CRITICAL: Static Assets
- // ========================================================================
- // Static assets are referenced by URL in the browser, not via imports.
- // Examples: /logo.png, /icons/*, /embed.js
'public/**',
-
- // Build outputs and caches
- 'node_modules/**',
- '.next/**',
- 'coverage/**',
-
- // Development tools
- '**/*.stories.{ts,tsx}',
-
- // ========================================================================
- // 🔒 Utility scripts (not part of application runtime)
- // ========================================================================
- // These scripts are run manually (e.g., pnpm gen-icons, pnpm i18n:check)
- // and are not imported by the application code.
- 'scripts/**',
- 'bin/**',
- 'i18n-config/**',
-
- // Icon generation script (generates components, not used in runtime)
- 'app/components/base/icons/script.mjs',
],
-
- // ============================================================================
- // Ignored Dependencies
- // ============================================================================
- // Dependencies that are used but not directly imported in code.
- // These are typically:
- // - Build tools
- // - Plugins loaded by configuration files
- // - CLI tools
- ignoreDependencies: [
- // ========================================================================
- // Next.js plugins (loaded by next.config.js)
- // ========================================================================
- 'next-pwa',
- '@next/bundle-analyzer',
- '@next/mdx',
-
- // ========================================================================
- // Build tools (used by webpack/next.js build process)
- // ========================================================================
- 'code-inspector-plugin',
-
- // ========================================================================
- // Development and translation tools (used by scripts)
- // ========================================================================
- 'bing-translate-api',
- 'uglify-js',
- ],
-
- // ============================================================================
- // Export Analysis Configuration
- // ============================================================================
- // Configure how exports are analyzed
-
- // Ignore exports that are only used within the same file
- // (e.g., helper functions used internally in the same module)
- ignoreExportsUsedInFile: true,
-
- // ⚠️ SAFETY: Include exports from entry files in the analysis
- // This helps find unused public APIs, but be careful with:
- // - Context exports (useContext hooks)
- // - Store exports (useStore hooks)
- // - Type exports (might be used in other files)
- includeEntryExports: true,
-
- // ============================================================================
- // Ignored Binaries
- // ============================================================================
- // Binary executables that are used but not listed in package.json
ignoreBinaries: [
- 'only-allow', // Used in preinstall script to enforce pnpm usage
+ 'only-allow',
],
-
- // ============================================================================
- // Reporting Rules
- // ============================================================================
- // Configure what types of issues to report and at what severity level
rules: {
- // ========================================================================
- // Unused files are ERRORS
- // ========================================================================
- // These should definitely be removed or used.
- // However, always manually verify before deleting!
- files: 'error',
-
- // ========================================================================
- // Unused dependencies are WARNINGS
- // ========================================================================
- // Dependencies might be:
- // - Used in production builds but not in dev
- // - Peer dependencies
- // - Used by other tools
+ files: 'warn',
dependencies: 'warn',
devDependencies: 'warn',
-
- // ========================================================================
- // Unlisted imports are ERRORS
- // ========================================================================
- // Missing from package.json - will break in production!
- unlisted: 'error',
-
- // ========================================================================
- // Unused exports are WARNINGS (not errors!)
- // ========================================================================
- // Exports might be:
- // - Part of public API for future use
- // - Used by external tools
- // - Exported for type inference
- // ⚠️ ALWAYS manually verify before removing exports!
+ optionalPeerDependencies: 'warn',
+ unlisted: 'warn',
+ unresolved: 'warn',
exports: 'warn',
-
- // Unused types are warnings (might be part of type definitions)
+ nsExports: 'warn',
+ classMembers: 'warn',
types: 'warn',
-
- // Duplicate exports are warnings (could cause confusion but not breaking)
+ nsTypes: 'warn',
+ enumMembers: 'warn',
duplicates: 'warn',
},
}
diff --git a/web/package.json b/web/package.json
index a78575304c..0c6821ce86 100644
--- a/web/package.json
+++ b/web/package.json
@@ -31,7 +31,7 @@
"type-check": "tsc --noEmit",
"type-check:tsgo": "tsgo --noEmit",
"prepare": "cd ../ && node -e \"if (process.env.NODE_ENV !== 'production'){process.exit(1)} \" || husky ./web/.husky",
- "gen-icons": "node ./app/components/base/icons/script.mjs",
+ "gen-icons": "node ./scripts/gen-icons.mjs && eslint --fix app/components/base/icons/src/",
"uglify-embed": "node ./bin/uglify-embed",
"i18n:check": "tsx ./scripts/check-i18n.js",
"i18n:gen": "tsx ./scripts/auto-gen-i18n.js",
@@ -190,7 +190,6 @@
"@vitejs/plugin-react": "^5.1.2",
"@vitest/coverage-v8": "4.0.16",
"autoprefixer": "^10.4.21",
- "babel-loader": "^10.0.0",
"bing-translate-api": "^4.1.0",
"code-inspector-plugin": "1.2.9",
"cross-env": "^10.1.0",
@@ -201,10 +200,9 @@
"eslint-plugin-storybook": "^10.1.10",
"eslint-plugin-tailwindcss": "^3.18.2",
"husky": "^9.1.7",
- "istanbul-lib-coverage": "^3.2.2",
"jsdom": "^27.3.0",
"jsdom-testing-mocks": "^1.16.0",
- "knip": "^5.66.1",
+ "knip": "^5.78.0",
"lint-staged": "^15.5.2",
"nock": "^14.0.10",
"postcss": "^8.5.6",
diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml
index fe9032d248..373e2e4020 100644
--- a/web/pnpm-lock.yaml
+++ b/web/pnpm-lock.yaml
@@ -481,9 +481,6 @@ importers:
autoprefixer:
specifier: ^10.4.21
version: 10.4.22(postcss@8.5.6)
- babel-loader:
- specifier: ^10.0.0
- version: 10.0.0(@babel/core@7.28.5)(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3))
bing-translate-api:
specifier: ^4.1.0
version: 4.2.0
@@ -514,9 +511,6 @@ importers:
husky:
specifier: ^9.1.7
version: 9.1.7
- istanbul-lib-coverage:
- specifier: ^3.2.2
- version: 3.2.2
jsdom:
specifier: ^27.3.0
version: 27.3.0(canvas@3.2.0)
@@ -524,8 +518,8 @@ importers:
specifier: ^1.16.0
version: 1.16.0
knip:
- specifier: ^5.66.1
- version: 5.72.0(@types/node@18.15.0)(typescript@5.9.3)
+ specifier: ^5.78.0
+ version: 5.78.0(@types/node@18.15.0)(typescript@5.9.3)
lint-staged:
specifier: ^15.5.2
version: 15.5.2
@@ -4271,13 +4265,6 @@ packages:
peerDependencies:
postcss: ^8.1.0
- babel-loader@10.0.0:
- resolution: {integrity: sha512-z8jt+EdS61AMw22nSfoNJAZ0vrtmhPRVi6ghL3rCeRZI8cdNYFiV5xeV3HbE7rlZZNmGH8BVccwWt8/ED0QOHA==}
- engines: {node: ^18.20.0 || ^20.10.0 || >=22.0.0}
- peerDependencies:
- '@babel/core': ^7.12.0
- webpack: '>=5.61.0'
-
babel-loader@8.4.1:
resolution: {integrity: sha512-nXzRChX+Z1GoE6yWavBQg6jDslyFF3SDjl2paADuoQtQW10JqShJt62R6eJQ5m/pjJFDT8xgKIWSP85OY8eXeA==}
engines: {node: '>= 8.9'}
@@ -6336,8 +6323,8 @@ packages:
resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==}
engines: {node: '>=6'}
- knip@5.72.0:
- resolution: {integrity: sha512-rlyoXI8FcggNtM/QXd/GW0sbsYvNuA/zPXt7bsuVi6kVQogY2PDCr81bPpzNnl0CP8AkFm2Z2plVeL5QQSis2w==}
+ knip@5.78.0:
+ resolution: {integrity: sha512-nB7i/fgiJl7WVxdv5lX4ZPfDt9/zrw/lOgZtyioy988xtFhKuFJCRdHWT1Zg9Avc0yaojvnmEuAXU8SeMblKww==}
engines: {node: '>=18.18.0'}
hasBin: true
peerDependencies:
@@ -13093,12 +13080,6 @@ snapshots:
postcss: 8.5.6
postcss-value-parser: 4.2.0
- babel-loader@10.0.0(@babel/core@7.28.5)(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3)):
- dependencies:
- '@babel/core': 7.28.5
- find-up: 5.0.0
- webpack: 5.103.0(esbuild@0.25.0)(uglify-js@3.19.3)
-
babel-loader@8.4.1(@babel/core@7.28.5)(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3)):
dependencies:
'@babel/core': 7.28.5
@@ -15468,7 +15449,7 @@ snapshots:
kleur@4.1.5: {}
- knip@5.72.0(@types/node@18.15.0)(typescript@5.9.3):
+ knip@5.78.0(@types/node@18.15.0)(typescript@5.9.3):
dependencies:
'@nodelib/fs.walk': 1.2.8
'@types/node': 18.15.0
diff --git a/web/app/components/base/icons/script.mjs b/web/scripts/gen-icons.mjs
similarity index 91%
rename from web/app/components/base/icons/script.mjs
rename to web/scripts/gen-icons.mjs
index 81566cc4cf..f681d65759 100644
--- a/web/app/components/base/icons/script.mjs
+++ b/web/scripts/gen-icons.mjs
@@ -5,6 +5,7 @@ import { parseXml } from '@rgrove/parse-xml'
import { camelCase, template } from 'es-toolkit/compat'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const iconsDir = path.resolve(__dirname, '../app/components/base/icons')
const generateDir = async (currentPath) => {
try {
@@ -32,7 +33,7 @@ const processSvgStructure = (svgStructure, replaceFillOrStrokeColor) => {
}
}
const generateSvgComponent = async (fileHandle, entry, pathList, replaceFillOrStrokeColor) => {
- const currentPath = path.resolve(__dirname, 'src', ...pathList.slice(2))
+ const currentPath = path.resolve(iconsDir, 'src', ...pathList.slice(2))
try {
await access(currentPath)
@@ -86,7 +87,7 @@ export { default as <%= svgName %> } from './<%= svgName %>'
}
const generateImageComponent = async (entry, pathList) => {
- const currentPath = path.resolve(__dirname, 'src', ...pathList.slice(2))
+ const currentPath = path.resolve(iconsDir, 'src', ...pathList.slice(2))
try {
await access(currentPath)
@@ -167,8 +168,8 @@ const walk = async (entry, pathList, replaceFillOrStrokeColor) => {
}
(async () => {
- await rm(path.resolve(__dirname, 'src'), { recursive: true, force: true })
- await walk('public', [__dirname, 'assets'])
- await walk('vender', [__dirname, 'assets'], true)
- await walk('image', [__dirname, 'assets'])
+ await rm(path.resolve(iconsDir, 'src'), { recursive: true, force: true })
+ await walk('public', [iconsDir, 'assets'])
+ await walk('vender', [iconsDir, 'assets'], true)
+ await walk('image', [iconsDir, 'assets'])
})()
From cad7101534c251f5d0c12218d6d000bab39365d1 Mon Sep 17 00:00:00 2001
From: Zhiqiang Yang
Date: Wed, 31 Dec 2025 15:49:06 +0800
Subject: [PATCH 17/25] feat: support image extraction in PDF RAG extractor
(#30399)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
---
api/core/rag/extractor/extract_processor.py | 4 +-
api/core/rag/extractor/pdf_extractor.py | 122 +++++++++++-
.../core/rag/extractor/test_pdf_extractor.py | 186 ++++++++++++++++++
3 files changed, 304 insertions(+), 8 deletions(-)
create mode 100644 api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.py
diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py
index 013c287248..6d28ce25bc 100644
--- a/api/core/rag/extractor/extract_processor.py
+++ b/api/core/rag/extractor/extract_processor.py
@@ -112,7 +112,7 @@ class ExtractProcessor:
if file_extension in {".xlsx", ".xls"}:
extractor = ExcelExtractor(file_path)
elif file_extension == ".pdf":
- extractor = PdfExtractor(file_path)
+ extractor = PdfExtractor(file_path, upload_file.tenant_id, upload_file.created_by)
elif file_extension in {".md", ".markdown", ".mdx"}:
extractor = (
UnstructuredMarkdownExtractor(file_path, unstructured_api_url, unstructured_api_key)
@@ -148,7 +148,7 @@ class ExtractProcessor:
if file_extension in {".xlsx", ".xls"}:
extractor = ExcelExtractor(file_path)
elif file_extension == ".pdf":
- extractor = PdfExtractor(file_path)
+ extractor = PdfExtractor(file_path, upload_file.tenant_id, upload_file.created_by)
elif file_extension in {".md", ".markdown", ".mdx"}:
extractor = MarkdownExtractor(file_path, autodetect_encoding=True)
elif file_extension in {".htm", ".html"}:
diff --git a/api/core/rag/extractor/pdf_extractor.py b/api/core/rag/extractor/pdf_extractor.py
index 80530d99a6..6aabcac704 100644
--- a/api/core/rag/extractor/pdf_extractor.py
+++ b/api/core/rag/extractor/pdf_extractor.py
@@ -1,25 +1,57 @@
"""Abstract interface for document loader implementations."""
import contextlib
+import io
+import logging
+import uuid
from collections.abc import Iterator
+import pypdfium2
+import pypdfium2.raw as pdfium_c
+
+from configs import dify_config
from core.rag.extractor.blob.blob import Blob
from core.rag.extractor.extractor_base import BaseExtractor
from core.rag.models.document import Document
+from extensions.ext_database import db
from extensions.ext_storage import storage
+from libs.datetime_utils import naive_utc_now
+from models.enums import CreatorUserRole
+from models.model import UploadFile
+
+logger = logging.getLogger(__name__)
class PdfExtractor(BaseExtractor):
- """Load pdf files.
-
+ """
+ PdfExtractor is used to extract text and images from PDF files.
Args:
- file_path: Path to the file to load.
+ file_path: Path to the PDF file.
+ tenant_id: Workspace ID.
+ user_id: ID of the user performing the extraction.
+ file_cache_key: Optional cache key for the extracted text.
"""
- def __init__(self, file_path: str, file_cache_key: str | None = None):
- """Initialize with file path."""
+ # Magic bytes for image format detection: (magic_bytes, extension, mime_type)
+ IMAGE_FORMATS = [
+ (b"\xff\xd8\xff", "jpg", "image/jpeg"),
+ (b"\x89PNG\r\n\x1a\n", "png", "image/png"),
+ (b"\x00\x00\x00\x0c\x6a\x50\x20\x20\x0d\x0a\x87\x0a", "jp2", "image/jp2"),
+ (b"GIF8", "gif", "image/gif"),
+ (b"BM", "bmp", "image/bmp"),
+ (b"II*\x00", "tiff", "image/tiff"),
+ (b"MM\x00*", "tiff", "image/tiff"),
+ (b"II+\x00", "tiff", "image/tiff"),
+ (b"MM\x00+", "tiff", "image/tiff"),
+ ]
+ MAX_MAGIC_LEN = max(len(m) for m, _, _ in IMAGE_FORMATS)
+
+ def __init__(self, file_path: str, tenant_id: str, user_id: str, file_cache_key: str | None = None):
+ """Initialize PdfExtractor."""
self._file_path = file_path
+ self._tenant_id = tenant_id
+ self._user_id = user_id
self._file_cache_key = file_cache_key
def extract(self) -> list[Document]:
@@ -50,7 +82,6 @@ class PdfExtractor(BaseExtractor):
def parse(self, blob: Blob) -> Iterator[Document]:
"""Lazily parse the blob."""
- import pypdfium2 # type: ignore
with blob.as_bytes_io() as file_path:
pdf_reader = pypdfium2.PdfDocument(file_path, autoclose=True)
@@ -59,8 +90,87 @@ class PdfExtractor(BaseExtractor):
text_page = page.get_textpage()
content = text_page.get_text_range()
text_page.close()
+
+ image_content = self._extract_images(page)
+ if image_content:
+ content += "\n" + image_content
+
page.close()
metadata = {"source": blob.source, "page": page_number}
yield Document(page_content=content, metadata=metadata)
finally:
pdf_reader.close()
+
+ def _extract_images(self, page) -> str:
+ """
+ Extract images from a PDF page, save them to storage and database,
+ and return markdown image links.
+
+ Args:
+ page: pypdfium2 page object.
+
+ Returns:
+ Markdown string containing links to the extracted images.
+ """
+ image_content = []
+ upload_files = []
+ base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL
+
+ try:
+ image_objects = page.get_objects(filter=(pdfium_c.FPDF_PAGEOBJ_IMAGE,))
+ for obj in image_objects:
+ try:
+ # Extract image bytes
+ img_byte_arr = io.BytesIO()
+ # Extract DCTDecode (JPEG) and JPXDecode (JPEG 2000) images directly
+ # Fallback to png for other formats
+ obj.extract(img_byte_arr, fb_format="png")
+ img_bytes = img_byte_arr.getvalue()
+
+ if not img_bytes:
+ continue
+
+ header = img_bytes[: self.MAX_MAGIC_LEN]
+ image_ext = None
+ mime_type = None
+ for magic, ext, mime in self.IMAGE_FORMATS:
+ if header.startswith(magic):
+ image_ext = ext
+ mime_type = mime
+ break
+
+ if not image_ext or not mime_type:
+ continue
+
+ file_uuid = str(uuid.uuid4())
+ file_key = "image_files/" + self._tenant_id + "/" + file_uuid + "." + image_ext
+
+ storage.save(file_key, img_bytes)
+
+ # save file to db
+ upload_file = UploadFile(
+ tenant_id=self._tenant_id,
+ storage_type=dify_config.STORAGE_TYPE,
+ key=file_key,
+ name=file_key,
+ size=len(img_bytes),
+ extension=image_ext,
+ mime_type=mime_type,
+ created_by=self._user_id,
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_at=naive_utc_now(),
+ used=True,
+ used_by=self._user_id,
+ used_at=naive_utc_now(),
+ )
+ upload_files.append(upload_file)
+ image_content.append(f"")
+ except Exception as e:
+ logger.warning("Failed to extract image from PDF: %s", e)
+ continue
+ except Exception as e:
+ logger.warning("Failed to get objects from PDF page: %s", e)
+ if upload_files:
+ db.session.add_all(upload_files)
+ db.session.commit()
+ return "\n".join(image_content)
diff --git a/api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.py
new file mode 100644
index 0000000000..3167a9a301
--- /dev/null
+++ b/api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.py
@@ -0,0 +1,186 @@
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+import core.rag.extractor.pdf_extractor as pe
+
+
+@pytest.fixture
+def mock_dependencies(monkeypatch):
+ # Mock storage
+ saves = []
+
+ def save(key, data):
+ saves.append((key, data))
+
+ monkeypatch.setattr(pe, "storage", SimpleNamespace(save=save))
+
+ # Mock db
+ class DummySession:
+ def __init__(self):
+ self.added = []
+ self.committed = False
+
+ def add(self, obj):
+ self.added.append(obj)
+
+ def add_all(self, objs):
+ self.added.extend(objs)
+
+ def commit(self):
+ self.committed = True
+
+ db_stub = SimpleNamespace(session=DummySession())
+ monkeypatch.setattr(pe, "db", db_stub)
+
+ # Mock UploadFile
+ class FakeUploadFile:
+ DEFAULT_ID = "test_file_id"
+
+ def __init__(self, **kwargs):
+ # Assign id from DEFAULT_ID, allow override via kwargs if needed
+ self.id = self.DEFAULT_ID
+ for k, v in kwargs.items():
+ setattr(self, k, v)
+
+ monkeypatch.setattr(pe, "UploadFile", FakeUploadFile)
+
+ # Mock config
+ monkeypatch.setattr(pe.dify_config, "FILES_URL", "http://files.local")
+ monkeypatch.setattr(pe.dify_config, "INTERNAL_FILES_URL", None)
+ monkeypatch.setattr(pe.dify_config, "STORAGE_TYPE", "local")
+
+ return SimpleNamespace(saves=saves, db=db_stub, UploadFile=FakeUploadFile)
+
+
+@pytest.mark.parametrize(
+ ("image_bytes", "expected_mime", "expected_ext", "file_id"),
+ [
+ (b"\xff\xd8\xff some jpeg", "image/jpeg", "jpg", "test_file_id_jpeg"),
+ (b"\x89PNG\r\n\x1a\n some png", "image/png", "png", "test_file_id_png"),
+ ],
+)
+def test_extract_images_formats(mock_dependencies, monkeypatch, image_bytes, expected_mime, expected_ext, file_id):
+ saves = mock_dependencies.saves
+ db_stub = mock_dependencies.db
+
+ # Customize FakeUploadFile id for this test case.
+ # Using monkeypatch ensures the class attribute is reset between parameter sets.
+ monkeypatch.setattr(mock_dependencies.UploadFile, "DEFAULT_ID", file_id)
+
+ # Mock page and image objects
+ mock_page = MagicMock()
+ mock_image_obj = MagicMock()
+
+ def mock_extract(buf, fb_format=None):
+ buf.write(image_bytes)
+
+ mock_image_obj.extract.side_effect = mock_extract
+
+ mock_page.get_objects.return_value = [mock_image_obj]
+
+ extractor = pe.PdfExtractor(file_path="test.pdf", tenant_id="t1", user_id="u1")
+
+ # We need to handle the import inside _extract_images
+ with patch("pypdfium2.raw") as mock_raw:
+ mock_raw.FPDF_PAGEOBJ_IMAGE = 1
+ result = extractor._extract_images(mock_page)
+
+ assert f"" in result
+ assert len(saves) == 1
+ assert saves[0][1] == image_bytes
+ assert len(db_stub.session.added) == 1
+ assert db_stub.session.added[0].tenant_id == "t1"
+ assert db_stub.session.added[0].size == len(image_bytes)
+ assert db_stub.session.added[0].mime_type == expected_mime
+ assert db_stub.session.added[0].extension == expected_ext
+ assert db_stub.session.committed is True
+
+
+@pytest.mark.parametrize(
+ ("get_objects_side_effect", "get_objects_return_value"),
+ [
+ (None, []), # Empty list
+ (None, None), # None returned
+ (Exception("Failed to get objects"), None), # Exception raised
+ ],
+)
+def test_extract_images_get_objects_scenarios(mock_dependencies, get_objects_side_effect, get_objects_return_value):
+ mock_page = MagicMock()
+ if get_objects_side_effect:
+ mock_page.get_objects.side_effect = get_objects_side_effect
+ else:
+ mock_page.get_objects.return_value = get_objects_return_value
+
+ extractor = pe.PdfExtractor(file_path="test.pdf", tenant_id="t1", user_id="u1")
+
+ with patch("pypdfium2.raw") as mock_raw:
+ mock_raw.FPDF_PAGEOBJ_IMAGE = 1
+ result = extractor._extract_images(mock_page)
+
+ assert result == ""
+
+
+def test_extract_calls_extract_images(mock_dependencies, monkeypatch):
+ # Mock pypdfium2
+ mock_pdf_doc = MagicMock()
+ mock_page = MagicMock()
+ mock_pdf_doc.__iter__.return_value = [mock_page]
+
+ # Mock text extraction
+ mock_text_page = MagicMock()
+ mock_text_page.get_text_range.return_value = "Page text content"
+ mock_page.get_textpage.return_value = mock_text_page
+
+ with patch("pypdfium2.PdfDocument", return_value=mock_pdf_doc):
+ # Mock Blob
+ mock_blob = MagicMock()
+ mock_blob.source = "test.pdf"
+ with patch("core.rag.extractor.pdf_extractor.Blob.from_path", return_value=mock_blob):
+ extractor = pe.PdfExtractor(file_path="test.pdf", tenant_id="t1", user_id="u1")
+
+ # Mock _extract_images to return a known string
+ monkeypatch.setattr(extractor, "_extract_images", lambda p: "")
+
+ documents = list(extractor.extract())
+
+ assert len(documents) == 1
+ assert "Page text content" in documents[0].page_content
+ assert "" in documents[0].page_content
+ assert documents[0].metadata["page"] == 0
+
+
+def test_extract_images_failures(mock_dependencies):
+ saves = mock_dependencies.saves
+ db_stub = mock_dependencies.db
+
+ # Mock page and image objects
+ mock_page = MagicMock()
+ mock_image_obj_fail = MagicMock()
+ mock_image_obj_ok = MagicMock()
+
+ # First image raises exception
+ mock_image_obj_fail.extract.side_effect = Exception("Extraction failure")
+
+ # Second image is OK (JPEG)
+ jpeg_bytes = b"\xff\xd8\xff some image data"
+
+ def mock_extract(buf, fb_format=None):
+ buf.write(jpeg_bytes)
+
+ mock_image_obj_ok.extract.side_effect = mock_extract
+
+ mock_page.get_objects.return_value = [mock_image_obj_fail, mock_image_obj_ok]
+
+ extractor = pe.PdfExtractor(file_path="test.pdf", tenant_id="t1", user_id="u1")
+
+ with patch("pypdfium2.raw") as mock_raw:
+ mock_raw.FPDF_PAGEOBJ_IMAGE = 1
+ result = extractor._extract_images(mock_page)
+
+ # Should have one success
+ assert "" in result
+ assert len(saves) == 1
+ assert saves[0][1] == jpeg_bytes
+ assert db_stub.session.committed is True
From 2bb1e24fb4593220f7278605ab0154aa43f4e28e Mon Sep 17 00:00:00 2001
From: Stephen Zhou <38493346+hyoban@users.noreply.github.com>
Date: Wed, 31 Dec 2025 15:53:33 +0800
Subject: [PATCH 18/25] test: unify i18next mocks into centralized helpers
(#30376)
Co-authored-by: Claude Opus 4.5
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: yyh
---
.../assets/component-test.template.tsx | 17 ++--
.../frontend-testing/references/mocking.md | 28 ++++---
.../config/agent-setting-button.spec.tsx | 9 ---
.../config/config-audio.spec.tsx | 9 ---
.../base/inline-delete-confirm/index.spec.tsx | 26 ++----
.../base/input-with-copy/index.spec.tsx | 23 ++----
web/app/components/base/input/index.spec.tsx | 19 ++---
.../billing/pricing/footer.spec.tsx | 18 -----
.../components/datasets/create/index.spec.tsx | 10 ---
.../processing/index.spec.tsx | 10 ---
.../components/plugins/card/index.spec.tsx | 27 -------
.../steps/install.spec.tsx | 34 ++++----
.../steps/uploading.spec.tsx | 15 ----
.../plugins/marketplace/index.spec.tsx | 46 +++++------
.../create/common-modal.spec.tsx | 11 ---
.../create/oauth-client.spec.tsx | 11 ---
.../subscription-list/edit/index.spec.tsx | 10 ---
.../plugin-mutation-model/index.spec.tsx | 22 ------
web/test/i18n-mock.ts | 79 +++++++++++++++++++
web/testing/testing.md | 27 ++++---
web/vitest.setup.ts | 20 +----
21 files changed, 178 insertions(+), 293 deletions(-)
create mode 100644 web/test/i18n-mock.ts
diff --git a/.claude/skills/frontend-testing/assets/component-test.template.tsx b/.claude/skills/frontend-testing/assets/component-test.template.tsx
index c39baff916..6b7803bd4b 100644
--- a/.claude/skills/frontend-testing/assets/component-test.template.tsx
+++ b/.claude/skills/frontend-testing/assets/component-test.template.tsx
@@ -28,17 +28,14 @@ import userEvent from '@testing-library/user-event'
// i18n (automatically mocked)
// WHY: Global mock in web/vitest.setup.ts is auto-loaded by Vitest setup
-// No explicit mock needed - it returns translation keys as-is
+// The global mock provides: useTranslation, Trans, useMixedTranslation, useGetLanguage
+// No explicit mock needed for most tests
+//
// Override only if custom translations are required:
-// vi.mock('react-i18next', () => ({
-// useTranslation: () => ({
-// t: (key: string) => {
-// const customTranslations: Record = {
-// 'my.custom.key': 'Custom Translation',
-// }
-// return customTranslations[key] || key
-// },
-// }),
+// import { createReactI18nextMock } from '@/test/i18n-mock'
+// vi.mock('react-i18next', () => createReactI18nextMock({
+// 'my.custom.key': 'Custom Translation',
+// 'button.save': 'Save',
// }))
// Router (if component uses useRouter, usePathname, useSearchParams)
diff --git a/.claude/skills/frontend-testing/references/mocking.md b/.claude/skills/frontend-testing/references/mocking.md
index 23889c8d3d..c70bcf0ae5 100644
--- a/.claude/skills/frontend-testing/references/mocking.md
+++ b/.claude/skills/frontend-testing/references/mocking.md
@@ -52,23 +52,29 @@ Modules are not mocked automatically. Use `vi.mock` in test files, or add global
### 1. i18n (Auto-loaded via Global Mock)
A global mock is defined in `web/vitest.setup.ts` and is auto-loaded by Vitest setup.
-**No explicit mock needed** for most tests - it returns translation keys as-is.
-For tests requiring custom translations, override the mock:
+The global mock provides:
+
+- `useTranslation` - returns translation keys with namespace prefix
+- `Trans` component - renders i18nKey and components
+- `useMixedTranslation` (from `@/app/components/plugins/marketplace/hooks`)
+- `useGetLanguage` (from `@/context/i18n`) - returns `'en-US'`
+
+**Default behavior**: Most tests should use the global mock (no local override needed).
+
+**For custom translations**: Use the helper function from `@/test/i18n-mock`:
```typescript
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string) => {
- const translations: Record = {
- 'my.custom.key': 'Custom translation',
- }
- return translations[key] || key
- },
- }),
+import { createReactI18nextMock } from '@/test/i18n-mock'
+
+vi.mock('react-i18next', () => createReactI18nextMock({
+ 'my.custom.key': 'Custom translation',
+ 'button.save': 'Save',
}))
```
+**Avoid**: Manually defining `useTranslation` mocks that just return the key - the global mock already does this.
+
### 2. Next.js Router
```typescript
diff --git a/web/app/components/app/configuration/config/agent-setting-button.spec.tsx b/web/app/components/app/configuration/config/agent-setting-button.spec.tsx
index 1874a3cccf..492b3b104c 100644
--- a/web/app/components/app/configuration/config/agent-setting-button.spec.tsx
+++ b/web/app/components/app/configuration/config/agent-setting-button.spec.tsx
@@ -5,15 +5,6 @@ import * as React from 'react'
import { AgentStrategy } from '@/types/app'
import AgentSettingButton from './agent-setting-button'
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string, options?: { ns?: string }) => {
- const prefix = options?.ns ? `${options.ns}.` : ''
- return `${prefix}${key}`
- },
- }),
-}))
-
let latestAgentSettingProps: any
vi.mock('./agent/agent-setting', () => ({
default: (props: any) => {
diff --git a/web/app/components/app/configuration/config/config-audio.spec.tsx b/web/app/components/app/configuration/config/config-audio.spec.tsx
index 41219fd1fa..a3e5c7c149 100644
--- a/web/app/components/app/configuration/config/config-audio.spec.tsx
+++ b/web/app/components/app/configuration/config/config-audio.spec.tsx
@@ -15,15 +15,6 @@ vi.mock('use-context-selector', async (importOriginal) => {
}
})
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string, options?: { ns?: string }) => {
- const prefix = options?.ns ? `${options.ns}.` : ''
- return `${prefix}${key}`
- },
- }),
-}))
-
const mockUseFeatures = vi.fn()
const mockUseFeaturesStore = vi.fn()
vi.mock('@/app/components/base/features/hooks', () => ({
diff --git a/web/app/components/base/inline-delete-confirm/index.spec.tsx b/web/app/components/base/inline-delete-confirm/index.spec.tsx
index 0de8c0844f..b770fccc88 100644
--- a/web/app/components/base/inline-delete-confirm/index.spec.tsx
+++ b/web/app/components/base/inline-delete-confirm/index.spec.tsx
@@ -1,26 +1,14 @@
import { cleanup, fireEvent, render } from '@testing-library/react'
import * as React from 'react'
+import { createReactI18nextMock } from '@/test/i18n-mock'
import InlineDeleteConfirm from './index'
-// Mock react-i18next
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string, defaultValueOrOptions?: string | { ns?: string }) => {
- const translations: Record = {
- 'operation.deleteConfirmTitle': 'Delete?',
- 'operation.yes': 'Yes',
- 'operation.no': 'No',
- 'operation.confirmAction': 'Please confirm your action.',
- }
- if (translations[key])
- return translations[key]
- // Handle case where second arg is default value string
- if (typeof defaultValueOrOptions === 'string')
- return defaultValueOrOptions
- const prefix = defaultValueOrOptions?.ns ? `${defaultValueOrOptions.ns}.` : ''
- return `${prefix}${key}`
- },
- }),
+// Mock react-i18next with custom translations for test assertions
+vi.mock('react-i18next', () => createReactI18nextMock({
+ 'operation.deleteConfirmTitle': 'Delete?',
+ 'operation.yes': 'Yes',
+ 'operation.no': 'No',
+ 'operation.confirmAction': 'Please confirm your action.',
}))
afterEach(cleanup)
diff --git a/web/app/components/base/input-with-copy/index.spec.tsx b/web/app/components/base/input-with-copy/index.spec.tsx
index 5a4ca7c97e..438e72d142 100644
--- a/web/app/components/base/input-with-copy/index.spec.tsx
+++ b/web/app/components/base/input-with-copy/index.spec.tsx
@@ -1,5 +1,6 @@
import { fireEvent, render, screen, waitFor } from '@testing-library/react'
import * as React from 'react'
+import { createReactI18nextMock } from '@/test/i18n-mock'
import InputWithCopy from './index'
// Create a mock function that we can track using vi.hoisted
@@ -10,22 +11,12 @@ vi.mock('copy-to-clipboard', () => ({
default: mockCopyToClipboard,
}))
-// Mock the i18n hook
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string, options?: { ns?: string }) => {
- const translations: Record = {
- 'operation.copy': 'Copy',
- 'operation.copied': 'Copied',
- 'overview.appInfo.embedded.copy': 'Copy',
- 'overview.appInfo.embedded.copied': 'Copied',
- }
- if (translations[key])
- return translations[key]
- const prefix = options?.ns ? `${options.ns}.` : ''
- return `${prefix}${key}`
- },
- }),
+// Mock the i18n hook with custom translations for test assertions
+vi.mock('react-i18next', () => createReactI18nextMock({
+ 'operation.copy': 'Copy',
+ 'operation.copied': 'Copied',
+ 'overview.appInfo.embedded.copy': 'Copy',
+ 'overview.appInfo.embedded.copied': 'Copied',
}))
// Mock es-toolkit/compat debounce
diff --git a/web/app/components/base/input/index.spec.tsx b/web/app/components/base/input/index.spec.tsx
index a0de3c4ca4..65589ddcdf 100644
--- a/web/app/components/base/input/index.spec.tsx
+++ b/web/app/components/base/input/index.spec.tsx
@@ -1,21 +1,12 @@
import { fireEvent, render, screen } from '@testing-library/react'
import * as React from 'react'
+import { createReactI18nextMock } from '@/test/i18n-mock'
import Input, { inputVariants } from './index'
-// Mock the i18n hook
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string, options?: { ns?: string }) => {
- const translations: Record = {
- 'operation.search': 'Search',
- 'placeholder.input': 'Please input',
- }
- if (translations[key])
- return translations[key]
- const prefix = options?.ns ? `${options.ns}.` : ''
- return `${prefix}${key}`
- },
- }),
+// Mock the i18n hook with custom translations for test assertions
+vi.mock('react-i18next', () => createReactI18nextMock({
+ 'operation.search': 'Search',
+ 'placeholder.input': 'Please input',
}))
describe('Input component', () => {
diff --git a/web/app/components/billing/pricing/footer.spec.tsx b/web/app/components/billing/pricing/footer.spec.tsx
index 0bbc38224e..85bd72c247 100644
--- a/web/app/components/billing/pricing/footer.spec.tsx
+++ b/web/app/components/billing/pricing/footer.spec.tsx
@@ -3,8 +3,6 @@ import * as React from 'react'
import { CategoryEnum } from '.'
import Footer from './footer'
-let mockTranslations: Record = {}
-
vi.mock('next/link', () => ({
default: ({ children, href, className, target }: { children: React.ReactNode, href: string, className?: string, target?: string }) => (
@@ -13,25 +11,9 @@ vi.mock('next/link', () => ({
),
}))
-vi.mock('react-i18next', async (importOriginal) => {
- const actual = await importOriginal()
- return {
- ...actual,
- useTranslation: () => ({
- t: (key: string, options?: { ns?: string }) => {
- if (mockTranslations[key])
- return mockTranslations[key]
- const prefix = options?.ns ? `${options.ns}.` : ''
- return `${prefix}${key}`
- },
- }),
- }
-})
-
describe('Footer', () => {
beforeEach(() => {
vi.clearAllMocks()
- mockTranslations = {}
})
// Rendering behavior
diff --git a/web/app/components/datasets/create/index.spec.tsx b/web/app/components/datasets/create/index.spec.tsx
index 1cf24e6f21..7e20e4bc1c 100644
--- a/web/app/components/datasets/create/index.spec.tsx
+++ b/web/app/components/datasets/create/index.spec.tsx
@@ -18,16 +18,6 @@ const IndexingTypeValues = {
// Mock External Dependencies
// ==========================================
-// Mock react-i18next (handled by global mock in web/vitest.setup.ts but we override for custom messages)
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string, options?: { ns?: string }) => {
- const prefix = options?.ns ? `${options.ns}.` : ''
- return `${prefix}${key}`
- },
- }),
-}))
-
// Mock next/link
vi.mock('next/link', () => {
return function MockLink({ children, href }: { children: React.ReactNode, href: string }) {
diff --git a/web/app/components/datasets/documents/create-from-pipeline/processing/index.spec.tsx b/web/app/components/datasets/documents/create-from-pipeline/processing/index.spec.tsx
index 875adb2779..d9fea93446 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/processing/index.spec.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/processing/index.spec.tsx
@@ -9,16 +9,6 @@ import Processing from './index'
// Mock External Dependencies
// ==========================================
-// Mock react-i18next (handled by global mock in web/vitest.setup.ts but we override for custom messages)
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string, options?: { ns?: string }) => {
- const prefix = options?.ns ? `${options.ns}.` : ''
- return `${prefix}${key}`
- },
- }),
-}))
-
// Mock useDocLink - returns a function that generates doc URLs
// Strips leading slash from path to match actual implementation behavior
vi.mock('@/context/i18n', () => ({
diff --git a/web/app/components/plugins/card/index.spec.tsx b/web/app/components/plugins/card/index.spec.tsx
index d32aafff57..4a3e5a587b 100644
--- a/web/app/components/plugins/card/index.spec.tsx
+++ b/web/app/components/plugins/card/index.spec.tsx
@@ -21,33 +21,6 @@ import Card from './index'
// Mock External Dependencies Only
// ================================
-// Mock react-i18next (translation hook)
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string) => key,
- }),
-}))
-
-// Mock useMixedTranslation hook
-vi.mock('../marketplace/hooks', () => ({
- useMixedTranslation: (_locale?: string) => ({
- t: (key: string, options?: { ns?: string }) => {
- const fullKey = options?.ns ? `${options.ns}.${key}` : key
- const translations: Record = {
- 'plugin.marketplace.partnerTip': 'Partner plugin',
- 'plugin.marketplace.verifiedTip': 'Verified plugin',
- 'plugin.installModal.installWarning': 'Install warning message',
- }
- return translations[fullKey] || key
- },
- }),
-}))
-
-// Mock useGetLanguage context
-vi.mock('@/context/i18n', () => ({
- useGetLanguage: () => 'en-US',
-}))
-
// Mock useTheme hook
vi.mock('@/hooks/use-theme', () => ({
default: () => ({ theme: 'light' }),
diff --git a/web/app/components/plugins/install-plugin/install-from-local-package/steps/install.spec.tsx b/web/app/components/plugins/install-plugin/install-from-local-package/steps/install.spec.tsx
index 4e3a3307df..7f95eb0b35 100644
--- a/web/app/components/plugins/install-plugin/install-from-local-package/steps/install.spec.tsx
+++ b/web/app/components/plugins/install-plugin/install-from-local-package/steps/install.spec.tsx
@@ -64,26 +64,20 @@ vi.mock('@/context/app-context', () => ({
}),
}))
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string, options?: { ns?: string } & Record) => {
- // Build full key with namespace prefix if provided
- const fullKey = options?.ns ? `${options.ns}.${key}` : key
- // Handle interpolation params (excluding ns)
- const { ns: _ns, ...params } = options || {}
- if (Object.keys(params).length > 0) {
- return `${fullKey}:${JSON.stringify(params)}`
- }
- return fullKey
- },
- }),
- Trans: ({ i18nKey, components }: { i18nKey: string, components?: Record }) => (
-
- {i18nKey}
- {components?.trustSource}
-
- ),
-}))
+vi.mock('react-i18next', async (importOriginal) => {
+ const actual = await importOriginal()
+ const { createReactI18nextMock } = await import('@/test/i18n-mock')
+ return {
+ ...actual,
+ ...createReactI18nextMock(),
+ Trans: ({ i18nKey, components }: { i18nKey: string, components?: Record }) => (
+
+ {i18nKey}
+ {components?.trustSource}
+
+ ),
+ }
+})
vi.mock('../../../card', () => ({
default: ({ payload, titleLeft }: {
diff --git a/web/app/components/plugins/install-plugin/install-from-local-package/steps/uploading.spec.tsx b/web/app/components/plugins/install-plugin/install-from-local-package/steps/uploading.spec.tsx
index c1d7e8cefe..35256b6633 100644
--- a/web/app/components/plugins/install-plugin/install-from-local-package/steps/uploading.spec.tsx
+++ b/web/app/components/plugins/install-plugin/install-from-local-package/steps/uploading.spec.tsx
@@ -48,21 +48,6 @@ vi.mock('@/service/plugins', () => ({
uploadFile: (...args: unknown[]) => mockUploadFile(...args),
}))
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string, options?: { ns?: string } & Record) => {
- // Build full key with namespace prefix if provided
- const fullKey = options?.ns ? `${options.ns}.${key}` : key
- // Handle interpolation params (excluding ns)
- const { ns: _ns, ...params } = options || {}
- if (Object.keys(params).length > 0) {
- return `${fullKey}:${JSON.stringify(params)}`
- }
- return fullKey
- },
- }),
-}))
-
vi.mock('../../../card', () => ({
default: ({ payload, isLoading, loadingFileName }: {
payload: { name: string }
diff --git a/web/app/components/plugins/marketplace/index.spec.tsx b/web/app/components/plugins/marketplace/index.spec.tsx
index 6047afe950..3073897ba1 100644
--- a/web/app/components/plugins/marketplace/index.spec.tsx
+++ b/web/app/components/plugins/marketplace/index.spec.tsx
@@ -27,17 +27,17 @@ import {
// Mock External Dependencies Only
// ================================
-// Mock react-i18next
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string) => key,
- }),
-}))
-
// Mock i18next-config
vi.mock('@/i18n-config/i18next-config', () => ({
default: {
- getFixedT: (_locale: string) => (key: string) => key,
+ getFixedT: (_locale: string) => (key: string, options?: Record) => {
+ if (options && options.ns) {
+ return `${options.ns}.${key}`
+ }
+ else {
+ return key
+ }
+ },
},
}))
@@ -617,8 +617,8 @@ describe('hooks', () => {
it('should return translation key when no translation found', () => {
const { result } = renderHook(() => useMixedTranslation())
- // The mock returns key as-is
- expect(result.current.t('category.all', { ns: 'plugin' })).toBe('category.all')
+ // The global mock returns key with namespace prefix
+ expect(result.current.t('category.all', { ns: 'plugin' })).toBe('plugin.category.all')
})
it('should use locale from outer when provided', () => {
@@ -638,8 +638,8 @@ describe('hooks', () => {
it('should use getFixedT when localeFromOuter is provided', () => {
const { result } = renderHook(() => useMixedTranslation('fr-FR'))
- // Should still return a function
- expect(result.current.t('search', { ns: 'plugin' })).toBe('search')
+ // The global mock returns key with namespace prefix
+ expect(result.current.t('search', { ns: 'plugin' })).toBe('plugin.search')
})
})
})
@@ -2756,15 +2756,15 @@ describe('PluginTypeSwitch Component', () => {
,
)
- // Note: The mock returns the key without namespace prefix
- expect(screen.getByText('category.all')).toBeInTheDocument()
- expect(screen.getByText('category.models')).toBeInTheDocument()
- expect(screen.getByText('category.tools')).toBeInTheDocument()
- expect(screen.getByText('category.datasources')).toBeInTheDocument()
- expect(screen.getByText('category.triggers')).toBeInTheDocument()
- expect(screen.getByText('category.agents')).toBeInTheDocument()
- expect(screen.getByText('category.extensions')).toBeInTheDocument()
- expect(screen.getByText('category.bundles')).toBeInTheDocument()
+ // Note: The global mock returns the key with namespace prefix (plugin.)
+ expect(screen.getByText('plugin.category.all')).toBeInTheDocument()
+ expect(screen.getByText('plugin.category.models')).toBeInTheDocument()
+ expect(screen.getByText('plugin.category.tools')).toBeInTheDocument()
+ expect(screen.getByText('plugin.category.datasources')).toBeInTheDocument()
+ expect(screen.getByText('plugin.category.triggers')).toBeInTheDocument()
+ expect(screen.getByText('plugin.category.agents')).toBeInTheDocument()
+ expect(screen.getByText('plugin.category.extensions')).toBeInTheDocument()
+ expect(screen.getByText('plugin.category.bundles')).toBeInTheDocument()
})
it('should apply className prop', () => {
@@ -2794,7 +2794,7 @@ describe('PluginTypeSwitch Component', () => {
,
)
- fireEvent.click(screen.getByText('category.tools'))
+ fireEvent.click(screen.getByText('plugin.category.tools'))
expect(screen.getByTestId('active-type-display')).toHaveTextContent('tool')
})
@@ -2816,7 +2816,7 @@ describe('PluginTypeSwitch Component', () => {
)
fireEvent.click(screen.getByTestId('set-model'))
- const modelOption = screen.getByText('category.models').closest('div')
+ const modelOption = screen.getByText('plugin.category.models').closest('div')
expect(modelOption).toHaveClass('shadow-xs')
})
})
diff --git a/web/app/components/plugins/plugin-detail-panel/subscription-list/create/common-modal.spec.tsx b/web/app/components/plugins/plugin-detail-panel/subscription-list/create/common-modal.spec.tsx
index 33cb93013d..c87fc1e4da 100644
--- a/web/app/components/plugins/plugin-detail-panel/subscription-list/create/common-modal.spec.tsx
+++ b/web/app/components/plugins/plugin-detail-panel/subscription-list/create/common-modal.spec.tsx
@@ -78,17 +78,6 @@ function createMockLogData(logs: TriggerLogEntity[] = []): { logs: TriggerLogEnt
// Mock Setup
// ============================================================================
-const mockTranslate = vi.fn((key: string, options?: { ns?: string }) => {
- // Build full key with namespace prefix if provided
- const fullKey = options?.ns ? `${options.ns}.${key}` : key
- return fullKey
-})
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: mockTranslate,
- }),
-}))
-
// Mock plugin store
const mockPluginDetail = createMockPluginDetail()
const mockUsePluginStore = vi.fn(() => mockPluginDetail)
diff --git a/web/app/components/plugins/plugin-detail-panel/subscription-list/create/oauth-client.spec.tsx b/web/app/components/plugins/plugin-detail-panel/subscription-list/create/oauth-client.spec.tsx
index 74599a13c5..f1cb7a65ae 100644
--- a/web/app/components/plugins/plugin-detail-panel/subscription-list/create/oauth-client.spec.tsx
+++ b/web/app/components/plugins/plugin-detail-panel/subscription-list/create/oauth-client.spec.tsx
@@ -68,17 +68,6 @@ function createMockSubscriptionBuilder(overrides: Partial {
- // Build full key with namespace prefix if provided
- const fullKey = options?.ns ? `${options.ns}.${key}` : key
- return fullKey
-})
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: mockTranslate,
- }),
-}))
-
// Mock plugin store
const mockPluginDetail = createMockPluginDetail()
const mockUsePluginStore = vi.fn(() => mockPluginDetail)
diff --git a/web/app/components/plugins/plugin-detail-panel/subscription-list/edit/index.spec.tsx b/web/app/components/plugins/plugin-detail-panel/subscription-list/edit/index.spec.tsx
index 4ce1841b05..b7988c916b 100644
--- a/web/app/components/plugins/plugin-detail-panel/subscription-list/edit/index.spec.tsx
+++ b/web/app/components/plugins/plugin-detail-panel/subscription-list/edit/index.spec.tsx
@@ -12,16 +12,6 @@ import { OAuthEditModal } from './oauth-edit-modal'
// ==================== Mock Setup ====================
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string, options?: { ns?: string }) => {
- // Build full key with namespace prefix if provided
- const fullKey = options?.ns ? `${options.ns}.${key}` : key
- return fullKey
- },
- }),
-}))
-
const mockToastNotify = vi.fn()
vi.mock('@/app/components/base/toast', () => ({
default: { notify: (params: unknown) => mockToastNotify(params) },
diff --git a/web/app/components/plugins/plugin-mutation-model/index.spec.tsx b/web/app/components/plugins/plugin-mutation-model/index.spec.tsx
index f007c32ef1..95c9db3c97 100644
--- a/web/app/components/plugins/plugin-mutation-model/index.spec.tsx
+++ b/web/app/components/plugins/plugin-mutation-model/index.spec.tsx
@@ -9,28 +9,6 @@ import PluginMutationModal from './index'
// Mock External Dependencies Only
// ================================
-// Mock react-i18next (translation hook)
-vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string) => key,
- }),
-}))
-
-// Mock useMixedTranslation hook
-vi.mock('../marketplace/hooks', () => ({
- useMixedTranslation: (_locale?: string) => ({
- t: (key: string, options?: { ns?: string }) => {
- const fullKey = options?.ns ? `${options.ns}.${key}` : key
- return fullKey
- },
- }),
-}))
-
-// Mock useGetLanguage context
-vi.mock('@/context/i18n', () => ({
- useGetLanguage: () => 'en-US',
-}))
-
// Mock useTheme hook
vi.mock('@/hooks/use-theme', () => ({
default: () => ({ theme: 'light' }),
diff --git a/web/test/i18n-mock.ts b/web/test/i18n-mock.ts
new file mode 100644
index 0000000000..20e7a22eef
--- /dev/null
+++ b/web/test/i18n-mock.ts
@@ -0,0 +1,79 @@
+import * as React from 'react'
+import { vi } from 'vitest'
+
+type TranslationMap = Record
+
+/**
+ * Create a t function with optional custom translations
+ * Checks translations[key] first, then translations[ns.key], then returns ns.key as fallback
+ */
+export function createTFunction(translations: TranslationMap, defaultNs?: string) {
+ return (key: string, options?: Record) => {
+ // Check custom translations first (without namespace)
+ if (translations[key] !== undefined)
+ return translations[key]
+
+ const ns = (options?.ns as string | undefined) ?? defaultNs
+ const fullKey = ns ? `${ns}.${key}` : key
+
+ // Check custom translations with namespace
+ if (translations[fullKey] !== undefined)
+ return translations[fullKey]
+
+ // Serialize params (excluding ns) for test assertions
+ const params = { ...options }
+ delete params.ns
+ const suffix = Object.keys(params).length > 0 ? `:${JSON.stringify(params)}` : ''
+ return `${fullKey}${suffix}`
+ }
+}
+
+/**
+ * Create useTranslation mock with optional custom translations
+ *
+ * @example
+ * vi.mock('react-i18next', () => createUseTranslationMock({
+ * 'operation.confirm': 'Confirm',
+ * }))
+ */
+export function createUseTranslationMock(translations: TranslationMap = {}) {
+ return {
+ useTranslation: (defaultNs?: string) => ({
+ t: createTFunction(translations, defaultNs),
+ i18n: {
+ language: 'en',
+ changeLanguage: vi.fn(),
+ },
+ }),
+ }
+}
+
+/**
+ * Create Trans component mock with optional custom translations
+ */
+export function createTransMock(translations: TranslationMap = {}) {
+ return {
+ Trans: ({ i18nKey, children }: {
+ i18nKey: string
+ children?: React.ReactNode
+ }) => {
+ const text = translations[i18nKey] ?? i18nKey
+ return React.createElement('span', { 'data-i18n-key': i18nKey }, children ?? text)
+ },
+ }
+}
+
+/**
+ * Create complete react-i18next mock (useTranslation + Trans)
+ *
+ * @example
+ * vi.mock('react-i18next', () => createReactI18nextMock({
+ * 'modal.title': 'My Modal',
+ * }))
+ */
+export function createReactI18nextMock(translations: TranslationMap = {}) {
+ return {
+ ...createUseTranslationMock(translations),
+ ...createTransMock(translations),
+ }
+}
diff --git a/web/testing/testing.md b/web/testing/testing.md
index 1d578ae634..47341e445e 100644
--- a/web/testing/testing.md
+++ b/web/testing/testing.md
@@ -329,21 +329,28 @@ describe('ComponentName', () => {
1. **i18n**: Uses global mock in `web/vitest.setup.ts` (auto-loaded by Vitest setup)
- The global mock returns translation keys as-is. For custom translations, override:
+ The global mock provides:
+
+ - `useTranslation` - returns translation keys with namespace prefix
+ - `Trans` component - renders i18nKey and components
+ - `useMixedTranslation` (from `@/app/components/plugins/marketplace/hooks`)
+ - `useGetLanguage` (from `@/context/i18n`) - returns `'en-US'`
+
+ **Default behavior**: Most tests should use the global mock (no local override needed).
+
+ **For custom translations**: Use the helper function from `@/test/i18n-mock`:
```typescript
- vi.mock('react-i18next', () => ({
- useTranslation: () => ({
- t: (key: string) => {
- const translations: Record = {
- 'my.custom.key': 'Custom translation',
- }
- return translations[key] || key
- },
- }),
+ import { createReactI18nextMock } from '@/test/i18n-mock'
+
+ vi.mock('react-i18next', () => createReactI18nextMock({
+ 'my.custom.key': 'Custom translation',
+ 'button.save': 'Save',
}))
```
+ **Avoid**: Manually defining `useTranslation` mocks that just return the key - the global mock already does this.
+
1. **Forms**: Test validation logic thoroughly
1. **Example - Correct mock with conditional rendering**:
diff --git a/web/vitest.setup.ts b/web/vitest.setup.ts
index 551a22475b..26dc25bbcf 100644
--- a/web/vitest.setup.ts
+++ b/web/vitest.setup.ts
@@ -88,26 +88,10 @@ vi.mock('next/image')
// mock react-i18next
vi.mock('react-i18next', async () => {
const actual = await vi.importActual('react-i18next')
+ const { createReactI18nextMock } = await import('./test/i18n-mock')
return {
...actual,
- useTranslation: (defaultNs?: string) => ({
- t: (key: string, options?: Record) => {
- if (options?.returnObjects)
- return [`${key}-feature-1`, `${key}-feature-2`]
- const ns = options?.ns ?? defaultNs
- if (options || ns) {
- const { ns: _ns, ...rest } = options ?? {}
- const prefix = ns ? `${ns}.` : ''
- const suffix = Object.keys(rest).length > 0 ? `:${JSON.stringify(rest)}` : ''
- return `${prefix}${key}${suffix}`
- }
- return key
- },
- i18n: {
- language: 'en',
- changeLanguage: vi.fn(),
- },
- }),
+ ...createReactI18nextMock(),
}
})
From 3015e9be73e1781d860699cedf6fe2210fb3c8ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=9D=9E=E6=B3=95=E6=93=8D=E4=BD=9C?=
Date: Wed, 31 Dec 2025 16:14:46 +0800
Subject: [PATCH 19/25] feat: add archive storage client and env config
(#30422)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
---
api/.env.example | 9 +
api/configs/extra/__init__.py | 2 +
api/configs/extra/archive_config.py | 43 +++
api/libs/archive_storage.py | 347 ++++++++++++++++++
.../unit_tests/libs/test_archive_storage.py | 272 ++++++++++++++
docker/.env.example | 9 +
docker/docker-compose.yaml | 7 +
7 files changed, 689 insertions(+)
create mode 100644 api/configs/extra/archive_config.py
create mode 100644 api/libs/archive_storage.py
create mode 100644 api/tests/unit_tests/libs/test_archive_storage.py
diff --git a/api/.env.example b/api/.env.example
index 99cd2ba558..5f8d369ec4 100644
--- a/api/.env.example
+++ b/api/.env.example
@@ -101,6 +101,15 @@ S3_ACCESS_KEY=your-access-key
S3_SECRET_KEY=your-secret-key
S3_REGION=your-region
+# Workflow run and Conversation archive storage (S3-compatible)
+ARCHIVE_STORAGE_ENABLED=false
+ARCHIVE_STORAGE_ENDPOINT=
+ARCHIVE_STORAGE_ARCHIVE_BUCKET=
+ARCHIVE_STORAGE_EXPORT_BUCKET=
+ARCHIVE_STORAGE_ACCESS_KEY=
+ARCHIVE_STORAGE_SECRET_KEY=
+ARCHIVE_STORAGE_REGION=auto
+
# Azure Blob Storage configuration
AZURE_BLOB_ACCOUNT_NAME=your-account-name
AZURE_BLOB_ACCOUNT_KEY=your-account-key
diff --git a/api/configs/extra/__init__.py b/api/configs/extra/__init__.py
index 4543b5389d..de97adfc0e 100644
--- a/api/configs/extra/__init__.py
+++ b/api/configs/extra/__init__.py
@@ -1,9 +1,11 @@
+from configs.extra.archive_config import ArchiveStorageConfig
from configs.extra.notion_config import NotionConfig
from configs.extra.sentry_config import SentryConfig
class ExtraServiceConfig(
# place the configs in alphabet order
+ ArchiveStorageConfig,
NotionConfig,
SentryConfig,
):
diff --git a/api/configs/extra/archive_config.py b/api/configs/extra/archive_config.py
new file mode 100644
index 0000000000..a85628fa61
--- /dev/null
+++ b/api/configs/extra/archive_config.py
@@ -0,0 +1,43 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class ArchiveStorageConfig(BaseSettings):
+ """
+ Configuration settings for workflow run logs archiving storage.
+ """
+
+ ARCHIVE_STORAGE_ENABLED: bool = Field(
+ description="Enable workflow run logs archiving to S3-compatible storage",
+ default=False,
+ )
+
+ ARCHIVE_STORAGE_ENDPOINT: str | None = Field(
+ description="URL of the S3-compatible storage endpoint (e.g., 'https://storage.example.com')",
+ default=None,
+ )
+
+ ARCHIVE_STORAGE_ARCHIVE_BUCKET: str | None = Field(
+ description="Name of the bucket to store archived workflow logs",
+ default=None,
+ )
+
+ ARCHIVE_STORAGE_EXPORT_BUCKET: str | None = Field(
+ description="Name of the bucket to store exported workflow runs",
+ default=None,
+ )
+
+ ARCHIVE_STORAGE_ACCESS_KEY: str | None = Field(
+ description="Access key ID for authenticating with storage",
+ default=None,
+ )
+
+ ARCHIVE_STORAGE_SECRET_KEY: str | None = Field(
+ description="Secret access key for authenticating with storage",
+ default=None,
+ )
+
+ ARCHIVE_STORAGE_REGION: str = Field(
+ description="Region for storage (use 'auto' if the provider supports it)",
+ default="auto",
+ )
diff --git a/api/libs/archive_storage.py b/api/libs/archive_storage.py
new file mode 100644
index 0000000000..f84d226447
--- /dev/null
+++ b/api/libs/archive_storage.py
@@ -0,0 +1,347 @@
+"""
+Archive Storage Client for S3-compatible storage.
+
+This module provides a dedicated storage client for archiving or exporting logs
+to S3-compatible object storage.
+"""
+
+import base64
+import datetime
+import gzip
+import hashlib
+import logging
+from collections.abc import Generator
+from typing import Any, cast
+
+import boto3
+import orjson
+from botocore.client import Config
+from botocore.exceptions import ClientError
+
+from configs import dify_config
+
+logger = logging.getLogger(__name__)
+
+
+class ArchiveStorageError(Exception):
+ """Base exception for archive storage operations."""
+
+ pass
+
+
+class ArchiveStorageNotConfiguredError(ArchiveStorageError):
+ """Raised when archive storage is not properly configured."""
+
+ pass
+
+
+class ArchiveStorage:
+ """
+ S3-compatible storage client for archiving or exporting.
+
+ This client provides methods for storing and retrieving archived data in JSONL+gzip format.
+ """
+
+ def __init__(self, bucket: str):
+ if not dify_config.ARCHIVE_STORAGE_ENABLED:
+ raise ArchiveStorageNotConfiguredError("Archive storage is not enabled")
+
+ if not bucket:
+ raise ArchiveStorageNotConfiguredError("Archive storage bucket is not configured")
+ if not all(
+ [
+ dify_config.ARCHIVE_STORAGE_ENDPOINT,
+ bucket,
+ dify_config.ARCHIVE_STORAGE_ACCESS_KEY,
+ dify_config.ARCHIVE_STORAGE_SECRET_KEY,
+ ]
+ ):
+ raise ArchiveStorageNotConfiguredError(
+ "Archive storage configuration is incomplete. "
+ "Required: ARCHIVE_STORAGE_ENDPOINT, ARCHIVE_STORAGE_ACCESS_KEY, "
+ "ARCHIVE_STORAGE_SECRET_KEY, and a bucket name"
+ )
+
+ self.bucket = bucket
+ self.client = boto3.client(
+ "s3",
+ endpoint_url=dify_config.ARCHIVE_STORAGE_ENDPOINT,
+ aws_access_key_id=dify_config.ARCHIVE_STORAGE_ACCESS_KEY,
+ aws_secret_access_key=dify_config.ARCHIVE_STORAGE_SECRET_KEY,
+ region_name=dify_config.ARCHIVE_STORAGE_REGION,
+ config=Config(s3={"addressing_style": "path"}),
+ )
+
+ # Verify bucket accessibility
+ try:
+ self.client.head_bucket(Bucket=self.bucket)
+ except ClientError as e:
+ error_code = e.response.get("Error", {}).get("Code")
+ if error_code == "404":
+ raise ArchiveStorageNotConfiguredError(f"Archive bucket '{self.bucket}' does not exist")
+ elif error_code == "403":
+ raise ArchiveStorageNotConfiguredError(f"Access denied to archive bucket '{self.bucket}'")
+ else:
+ raise ArchiveStorageError(f"Failed to access archive bucket: {e}")
+
+ def put_object(self, key: str, data: bytes) -> str:
+ """
+ Upload an object to the archive storage.
+
+ Args:
+ key: Object key (path) within the bucket
+ data: Binary data to upload
+
+ Returns:
+ MD5 checksum of the uploaded data
+
+ Raises:
+ ArchiveStorageError: If upload fails
+ """
+ checksum = hashlib.md5(data).hexdigest()
+ try:
+ self.client.put_object(
+ Bucket=self.bucket,
+ Key=key,
+ Body=data,
+ ContentMD5=self._content_md5(data),
+ )
+ logger.debug("Uploaded object: %s (size=%d, checksum=%s)", key, len(data), checksum)
+ return checksum
+ except ClientError as e:
+ raise ArchiveStorageError(f"Failed to upload object '{key}': {e}")
+
+ def get_object(self, key: str) -> bytes:
+ """
+ Download an object from the archive storage.
+
+ Args:
+ key: Object key (path) within the bucket
+
+ Returns:
+ Binary data of the object
+
+ Raises:
+ ArchiveStorageError: If download fails
+ FileNotFoundError: If object does not exist
+ """
+ try:
+ response = self.client.get_object(Bucket=self.bucket, Key=key)
+ return response["Body"].read()
+ except ClientError as e:
+ error_code = e.response.get("Error", {}).get("Code")
+ if error_code == "NoSuchKey":
+ raise FileNotFoundError(f"Archive object not found: {key}")
+ raise ArchiveStorageError(f"Failed to download object '{key}': {e}")
+
+ def get_object_stream(self, key: str) -> Generator[bytes, None, None]:
+ """
+ Stream an object from the archive storage.
+
+ Args:
+ key: Object key (path) within the bucket
+
+ Yields:
+ Chunks of binary data
+
+ Raises:
+ ArchiveStorageError: If download fails
+ FileNotFoundError: If object does not exist
+ """
+ try:
+ response = self.client.get_object(Bucket=self.bucket, Key=key)
+ yield from response["Body"].iter_chunks()
+ except ClientError as e:
+ error_code = e.response.get("Error", {}).get("Code")
+ if error_code == "NoSuchKey":
+ raise FileNotFoundError(f"Archive object not found: {key}")
+ raise ArchiveStorageError(f"Failed to stream object '{key}': {e}")
+
+ def object_exists(self, key: str) -> bool:
+ """
+ Check if an object exists in the archive storage.
+
+ Args:
+ key: Object key (path) within the bucket
+
+ Returns:
+ True if object exists, False otherwise
+ """
+ try:
+ self.client.head_object(Bucket=self.bucket, Key=key)
+ return True
+ except ClientError:
+ return False
+
+ def delete_object(self, key: str) -> None:
+ """
+ Delete an object from the archive storage.
+
+ Args:
+ key: Object key (path) within the bucket
+
+ Raises:
+ ArchiveStorageError: If deletion fails
+ """
+ try:
+ self.client.delete_object(Bucket=self.bucket, Key=key)
+ logger.debug("Deleted object: %s", key)
+ except ClientError as e:
+ raise ArchiveStorageError(f"Failed to delete object '{key}': {e}")
+
+ def generate_presigned_url(self, key: str, expires_in: int = 3600) -> str:
+ """
+ Generate a pre-signed URL for downloading an object.
+
+ Args:
+ key: Object key (path) within the bucket
+ expires_in: URL validity duration in seconds (default: 1 hour)
+
+ Returns:
+ Pre-signed URL string.
+
+ Raises:
+ ArchiveStorageError: If generation fails
+ """
+ try:
+ return self.client.generate_presigned_url(
+ ClientMethod="get_object",
+ Params={"Bucket": self.bucket, "Key": key},
+ ExpiresIn=expires_in,
+ )
+ except ClientError as e:
+ raise ArchiveStorageError(f"Failed to generate pre-signed URL for '{key}': {e}")
+
+ def list_objects(self, prefix: str) -> list[str]:
+ """
+ List objects under a given prefix.
+
+ Args:
+ prefix: Object key prefix to filter by
+
+ Returns:
+ List of object keys matching the prefix
+ """
+ keys = []
+ paginator = self.client.get_paginator("list_objects_v2")
+
+ try:
+ for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
+ for obj in page.get("Contents", []):
+ keys.append(obj["Key"])
+ except ClientError as e:
+ raise ArchiveStorageError(f"Failed to list objects with prefix '{prefix}': {e}")
+
+ return keys
+
+ @staticmethod
+ def _content_md5(data: bytes) -> str:
+ """Calculate base64-encoded MD5 for Content-MD5 header."""
+ return base64.b64encode(hashlib.md5(data).digest()).decode()
+
+ @staticmethod
+ def serialize_to_jsonl_gz(records: list[dict[str, Any]]) -> bytes:
+ """
+ Serialize records to gzipped JSONL format.
+
+ Args:
+ records: List of dictionaries to serialize
+
+ Returns:
+ Gzipped JSONL bytes
+ """
+ lines = []
+ for record in records:
+ # Convert datetime objects to ISO format strings
+ serialized = ArchiveStorage._serialize_record(record)
+ lines.append(orjson.dumps(serialized))
+
+ jsonl_content = b"\n".join(lines)
+ if jsonl_content:
+ jsonl_content += b"\n"
+
+ return gzip.compress(jsonl_content)
+
+ @staticmethod
+ def deserialize_from_jsonl_gz(data: bytes) -> list[dict[str, Any]]:
+ """
+ Deserialize gzipped JSONL data to records.
+
+ Args:
+ data: Gzipped JSONL bytes
+
+ Returns:
+ List of dictionaries
+ """
+ jsonl_content = gzip.decompress(data)
+ records = []
+
+ for line in jsonl_content.splitlines():
+ if line:
+ records.append(orjson.loads(line))
+
+ return records
+
+ @staticmethod
+ def _serialize_record(record: dict[str, Any]) -> dict[str, Any]:
+ """Serialize a single record, converting special types."""
+
+ def _serialize(item: Any) -> Any:
+ if isinstance(item, datetime.datetime):
+ return item.isoformat()
+ if isinstance(item, dict):
+ return {key: _serialize(value) for key, value in item.items()}
+ if isinstance(item, list):
+ return [_serialize(value) for value in item]
+ return item
+
+ return cast(dict[str, Any], _serialize(record))
+
+ @staticmethod
+ def compute_checksum(data: bytes) -> str:
+ """Compute MD5 checksum of data."""
+ return hashlib.md5(data).hexdigest()
+
+
+# Singleton instance (lazy initialization)
+_archive_storage: ArchiveStorage | None = None
+_export_storage: ArchiveStorage | None = None
+
+
+def get_archive_storage() -> ArchiveStorage:
+ """
+ Get the archive storage singleton instance.
+
+ Returns:
+ ArchiveStorage instance
+
+ Raises:
+ ArchiveStorageNotConfiguredError: If archive storage is not configured
+ """
+ global _archive_storage
+ if _archive_storage is None:
+ archive_bucket = dify_config.ARCHIVE_STORAGE_ARCHIVE_BUCKET
+ if not archive_bucket:
+ raise ArchiveStorageNotConfiguredError(
+ "Archive storage bucket is not configured. Required: ARCHIVE_STORAGE_ARCHIVE_BUCKET"
+ )
+ _archive_storage = ArchiveStorage(bucket=archive_bucket)
+ return _archive_storage
+
+
+def get_export_storage() -> ArchiveStorage:
+ """
+ Get the export storage singleton instance.
+
+ Returns:
+ ArchiveStorage instance
+ """
+ global _export_storage
+ if _export_storage is None:
+ export_bucket = dify_config.ARCHIVE_STORAGE_EXPORT_BUCKET
+ if not export_bucket:
+ raise ArchiveStorageNotConfiguredError(
+ "Archive export bucket is not configured. Required: ARCHIVE_STORAGE_EXPORT_BUCKET"
+ )
+ _export_storage = ArchiveStorage(bucket=export_bucket)
+ return _export_storage
diff --git a/api/tests/unit_tests/libs/test_archive_storage.py b/api/tests/unit_tests/libs/test_archive_storage.py
new file mode 100644
index 0000000000..697760e33a
--- /dev/null
+++ b/api/tests/unit_tests/libs/test_archive_storage.py
@@ -0,0 +1,272 @@
+import base64
+import hashlib
+from datetime import datetime
+from unittest.mock import ANY, MagicMock
+
+import pytest
+from botocore.exceptions import ClientError
+
+from libs import archive_storage as storage_module
+from libs.archive_storage import (
+ ArchiveStorage,
+ ArchiveStorageError,
+ ArchiveStorageNotConfiguredError,
+)
+
+BUCKET_NAME = "archive-bucket"
+
+
+def _configure_storage(monkeypatch, **overrides):
+ defaults = {
+ "ARCHIVE_STORAGE_ENABLED": True,
+ "ARCHIVE_STORAGE_ENDPOINT": "https://storage.example.com",
+ "ARCHIVE_STORAGE_ARCHIVE_BUCKET": BUCKET_NAME,
+ "ARCHIVE_STORAGE_ACCESS_KEY": "access",
+ "ARCHIVE_STORAGE_SECRET_KEY": "secret",
+ "ARCHIVE_STORAGE_REGION": "auto",
+ }
+ defaults.update(overrides)
+ for key, value in defaults.items():
+ monkeypatch.setattr(storage_module.dify_config, key, value, raising=False)
+
+
+def _client_error(code: str) -> ClientError:
+ return ClientError({"Error": {"Code": code}}, "Operation")
+
+
+def _mock_client(monkeypatch):
+ client = MagicMock()
+ client.head_bucket.return_value = None
+ boto_client = MagicMock(return_value=client)
+ monkeypatch.setattr(storage_module.boto3, "client", boto_client)
+ return client, boto_client
+
+
+def test_init_disabled(monkeypatch):
+ _configure_storage(monkeypatch, ARCHIVE_STORAGE_ENABLED=False)
+ with pytest.raises(ArchiveStorageNotConfiguredError, match="not enabled"):
+ ArchiveStorage(bucket=BUCKET_NAME)
+
+
+def test_init_missing_config(monkeypatch):
+ _configure_storage(monkeypatch, ARCHIVE_STORAGE_ENDPOINT=None)
+ with pytest.raises(ArchiveStorageNotConfiguredError, match="incomplete"):
+ ArchiveStorage(bucket=BUCKET_NAME)
+
+
+def test_init_bucket_not_found(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ client.head_bucket.side_effect = _client_error("404")
+
+ with pytest.raises(ArchiveStorageNotConfiguredError, match="does not exist"):
+ ArchiveStorage(bucket=BUCKET_NAME)
+
+
+def test_init_bucket_access_denied(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ client.head_bucket.side_effect = _client_error("403")
+
+ with pytest.raises(ArchiveStorageNotConfiguredError, match="Access denied"):
+ ArchiveStorage(bucket=BUCKET_NAME)
+
+
+def test_init_bucket_other_error(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ client.head_bucket.side_effect = _client_error("500")
+
+ with pytest.raises(ArchiveStorageError, match="Failed to access archive bucket"):
+ ArchiveStorage(bucket=BUCKET_NAME)
+
+
+def test_init_sets_client(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, boto_client = _mock_client(monkeypatch)
+
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ boto_client.assert_called_once_with(
+ "s3",
+ endpoint_url="https://storage.example.com",
+ aws_access_key_id="access",
+ aws_secret_access_key="secret",
+ region_name="auto",
+ config=ANY,
+ )
+ assert storage.client is client
+ assert storage.bucket == BUCKET_NAME
+
+
+def test_put_object_returns_checksum(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ data = b"hello"
+ checksum = storage.put_object("key", data)
+
+ expected_md5 = hashlib.md5(data).hexdigest()
+ expected_content_md5 = base64.b64encode(hashlib.md5(data).digest()).decode()
+ client.put_object.assert_called_once_with(
+ Bucket="archive-bucket",
+ Key="key",
+ Body=data,
+ ContentMD5=expected_content_md5,
+ )
+ assert checksum == expected_md5
+
+
+def test_put_object_raises_on_error(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+ client.put_object.side_effect = _client_error("500")
+
+ with pytest.raises(ArchiveStorageError, match="Failed to upload object"):
+ storage.put_object("key", b"data")
+
+
+def test_get_object_returns_bytes(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ body = MagicMock()
+ body.read.return_value = b"payload"
+ client.get_object.return_value = {"Body": body}
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ assert storage.get_object("key") == b"payload"
+
+
+def test_get_object_missing(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ client.get_object.side_effect = _client_error("NoSuchKey")
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ with pytest.raises(FileNotFoundError, match="Archive object not found"):
+ storage.get_object("missing")
+
+
+def test_get_object_stream(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ body = MagicMock()
+ body.iter_chunks.return_value = [b"a", b"b"]
+ client.get_object.return_value = {"Body": body}
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ assert list(storage.get_object_stream("key")) == [b"a", b"b"]
+
+
+def test_get_object_stream_missing(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ client.get_object.side_effect = _client_error("NoSuchKey")
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ with pytest.raises(FileNotFoundError, match="Archive object not found"):
+ list(storage.get_object_stream("missing"))
+
+
+def test_object_exists(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ assert storage.object_exists("key") is True
+ client.head_object.side_effect = _client_error("404")
+ assert storage.object_exists("missing") is False
+
+
+def test_delete_object_error(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ client.delete_object.side_effect = _client_error("500")
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ with pytest.raises(ArchiveStorageError, match="Failed to delete object"):
+ storage.delete_object("key")
+
+
+def test_list_objects(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ paginator = MagicMock()
+ paginator.paginate.return_value = [
+ {"Contents": [{"Key": "a"}, {"Key": "b"}]},
+ {"Contents": [{"Key": "c"}]},
+ ]
+ client.get_paginator.return_value = paginator
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ assert storage.list_objects("prefix") == ["a", "b", "c"]
+ paginator.paginate.assert_called_once_with(Bucket="archive-bucket", Prefix="prefix")
+
+
+def test_list_objects_error(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ paginator = MagicMock()
+ paginator.paginate.side_effect = _client_error("500")
+ client.get_paginator.return_value = paginator
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ with pytest.raises(ArchiveStorageError, match="Failed to list objects"):
+ storage.list_objects("prefix")
+
+
+def test_generate_presigned_url(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ client.generate_presigned_url.return_value = "http://signed-url"
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ url = storage.generate_presigned_url("key", expires_in=123)
+
+ client.generate_presigned_url.assert_called_once_with(
+ ClientMethod="get_object",
+ Params={"Bucket": "archive-bucket", "Key": "key"},
+ ExpiresIn=123,
+ )
+ assert url == "http://signed-url"
+
+
+def test_generate_presigned_url_error(monkeypatch):
+ _configure_storage(monkeypatch)
+ client, _ = _mock_client(monkeypatch)
+ client.generate_presigned_url.side_effect = _client_error("500")
+ storage = ArchiveStorage(bucket=BUCKET_NAME)
+
+ with pytest.raises(ArchiveStorageError, match="Failed to generate pre-signed URL"):
+ storage.generate_presigned_url("key")
+
+
+def test_serialization_roundtrip():
+ records = [
+ {
+ "id": "1",
+ "created_at": datetime(2024, 1, 1, 12, 0, 0),
+ "payload": {"nested": "value"},
+ "items": [{"name": "a"}],
+ },
+ {"id": "2", "value": 123},
+ ]
+
+ data = ArchiveStorage.serialize_to_jsonl_gz(records)
+ decoded = ArchiveStorage.deserialize_from_jsonl_gz(data)
+
+ assert decoded[0]["id"] == "1"
+ assert decoded[0]["payload"]["nested"] == "value"
+ assert decoded[0]["items"][0]["name"] == "a"
+ assert "2024-01-01T12:00:00" in decoded[0]["created_at"]
+ assert decoded[1]["value"] == 123
+
+
+def test_content_md5_matches_checksum():
+ data = b"checksum"
+ expected = base64.b64encode(hashlib.md5(data).digest()).decode()
+
+ assert ArchiveStorage._content_md5(data) == expected
+ assert ArchiveStorage.compute_checksum(data) == hashlib.md5(data).hexdigest()
diff --git a/docker/.env.example b/docker/.env.example
index 0e09d6869d..5c1089408c 100644
--- a/docker/.env.example
+++ b/docker/.env.example
@@ -447,6 +447,15 @@ S3_SECRET_KEY=
# If set to false, the access key and secret key must be provided.
S3_USE_AWS_MANAGED_IAM=false
+# Workflow run and Conversation archive storage (S3-compatible)
+ARCHIVE_STORAGE_ENABLED=false
+ARCHIVE_STORAGE_ENDPOINT=
+ARCHIVE_STORAGE_ARCHIVE_BUCKET=
+ARCHIVE_STORAGE_EXPORT_BUCKET=
+ARCHIVE_STORAGE_ACCESS_KEY=
+ARCHIVE_STORAGE_SECRET_KEY=
+ARCHIVE_STORAGE_REGION=auto
+
# Azure Blob Configuration
#
AZURE_BLOB_ACCOUNT_NAME=difyai
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index 1c8d8d03e3..9910c95a84 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -122,6 +122,13 @@ x-shared-env: &shared-api-worker-env
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-}
S3_SECRET_KEY: ${S3_SECRET_KEY:-}
S3_USE_AWS_MANAGED_IAM: ${S3_USE_AWS_MANAGED_IAM:-false}
+ ARCHIVE_STORAGE_ENABLED: ${ARCHIVE_STORAGE_ENABLED:-false}
+ ARCHIVE_STORAGE_ENDPOINT: ${ARCHIVE_STORAGE_ENDPOINT:-}
+ ARCHIVE_STORAGE_ARCHIVE_BUCKET: ${ARCHIVE_STORAGE_ARCHIVE_BUCKET:-}
+ ARCHIVE_STORAGE_EXPORT_BUCKET: ${ARCHIVE_STORAGE_EXPORT_BUCKET:-}
+ ARCHIVE_STORAGE_ACCESS_KEY: ${ARCHIVE_STORAGE_ACCESS_KEY:-}
+ ARCHIVE_STORAGE_SECRET_KEY: ${ARCHIVE_STORAGE_SECRET_KEY:-}
+ ARCHIVE_STORAGE_REGION: ${ARCHIVE_STORAGE_REGION:-auto}
AZURE_BLOB_ACCOUNT_NAME: ${AZURE_BLOB_ACCOUNT_NAME:-difyai}
AZURE_BLOB_ACCOUNT_KEY: ${AZURE_BLOB_ACCOUNT_KEY:-difyai}
AZURE_BLOB_CONTAINER_NAME: ${AZURE_BLOB_CONTAINER_NAME:-difyai-container}
From 184077c37c7ddbaff4c16f0d9bb417ea1493d5c1 Mon Sep 17 00:00:00 2001
From: Stephen Zhou <38493346+hyoban@users.noreply.github.com>
Date: Wed, 31 Dec 2025 16:41:43 +0800
Subject: [PATCH 20/25] build: bring back babel-loader, add build check
(#30427)
---
.github/workflows/style.yml | 5 +++++
web/knip.config.ts | 4 ++++
web/package.json | 1 +
web/pnpm-lock.yaml | 16 ++++++++++++++++
4 files changed, 26 insertions(+)
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
index 39b559f4ca..462ece303e 100644
--- a/.github/workflows/style.yml
+++ b/.github/workflows/style.yml
@@ -115,6 +115,11 @@ jobs:
working-directory: ./web
run: pnpm run knip
+ - name: Web build check
+ if: steps.changed-files.outputs.any_changed == 'true'
+ working-directory: ./web
+ run: pnpm run build
+
superlinter:
name: SuperLinter
runs-on: ubuntu-latest
diff --git a/web/knip.config.ts b/web/knip.config.ts
index 414b00fb7f..6ffda0316a 100644
--- a/web/knip.config.ts
+++ b/web/knip.config.ts
@@ -15,6 +15,10 @@ const config: KnipConfig = {
ignoreBinaries: [
'only-allow',
],
+ ignoreDependencies: [
+ // required by next-pwa
+ 'babel-loader',
+ ],
rules: {
files: 'warn',
dependencies: 'warn',
diff --git a/web/package.json b/web/package.json
index 0c6821ce86..7ee2325dbc 100644
--- a/web/package.json
+++ b/web/package.json
@@ -190,6 +190,7 @@
"@vitejs/plugin-react": "^5.1.2",
"@vitest/coverage-v8": "4.0.16",
"autoprefixer": "^10.4.21",
+ "babel-loader": "^10.0.0",
"bing-translate-api": "^4.1.0",
"code-inspector-plugin": "1.2.9",
"cross-env": "^10.1.0",
diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml
index 373e2e4020..cdd194da37 100644
--- a/web/pnpm-lock.yaml
+++ b/web/pnpm-lock.yaml
@@ -481,6 +481,9 @@ importers:
autoprefixer:
specifier: ^10.4.21
version: 10.4.22(postcss@8.5.6)
+ babel-loader:
+ specifier: ^10.0.0
+ version: 10.0.0(@babel/core@7.28.5)(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3))
bing-translate-api:
specifier: ^4.1.0
version: 4.2.0
@@ -4265,6 +4268,13 @@ packages:
peerDependencies:
postcss: ^8.1.0
+ babel-loader@10.0.0:
+ resolution: {integrity: sha512-z8jt+EdS61AMw22nSfoNJAZ0vrtmhPRVi6ghL3rCeRZI8cdNYFiV5xeV3HbE7rlZZNmGH8BVccwWt8/ED0QOHA==}
+ engines: {node: ^18.20.0 || ^20.10.0 || >=22.0.0}
+ peerDependencies:
+ '@babel/core': ^7.12.0
+ webpack: '>=5.61.0'
+
babel-loader@8.4.1:
resolution: {integrity: sha512-nXzRChX+Z1GoE6yWavBQg6jDslyFF3SDjl2paADuoQtQW10JqShJt62R6eJQ5m/pjJFDT8xgKIWSP85OY8eXeA==}
engines: {node: '>= 8.9'}
@@ -13080,6 +13090,12 @@ snapshots:
postcss: 8.5.6
postcss-value-parser: 4.2.0
+ babel-loader@10.0.0(@babel/core@7.28.5)(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3)):
+ dependencies:
+ '@babel/core': 7.28.5
+ find-up: 5.0.0
+ webpack: 5.103.0(esbuild@0.25.0)(uglify-js@3.19.3)
+
babel-loader@8.4.1(@babel/core@7.28.5)(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3)):
dependencies:
'@babel/core': 7.28.5
From ee1d0df927b8b64baf835b9df1e7aaba62c2cb2c Mon Sep 17 00:00:00 2001
From: Stephen Zhou <38493346+hyoban@users.noreply.github.com>
Date: Wed, 31 Dec 2025 17:55:25 +0800
Subject: [PATCH 21/25] chore: add jotai store (#30432)
Signed-off-by: yyh
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: yyh
---
web/app/layout.tsx | 45 ++++++++++++++++++++++++---------------------
1 file changed, 24 insertions(+), 21 deletions(-)
diff --git a/web/app/layout.tsx b/web/app/layout.tsx
index fa1f7d48b5..8fc5f8abcc 100644
--- a/web/app/layout.tsx
+++ b/web/app/layout.tsx
@@ -1,4 +1,5 @@
import type { Viewport } from 'next'
+import { Provider as JotaiProvider } from 'jotai'
import { ThemeProvider } from 'next-themes'
import { Instrument_Serif } from 'next/font/google'
import { NuqsAdapter } from 'nuqs/adapters/next/app'
@@ -91,27 +92,29 @@ const LocaleLayout = async ({
{...datasetMap}
>
-
-
-
-
-
-
-
- {children}
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+ {children}
+
+
+
+
+
+
+
+