diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000000..7d42234cae --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,8 @@ +{ + "enabledPlugins": { + "feature-dev@claude-plugins-official": true, + "context7@claude-plugins-official": true, + "typescript-lsp@claude-plugins-official": true, + "pyright-lsp@claude-plugins-official": true + } +} diff --git a/.claude/settings.json.example b/.claude/settings.json.example deleted file mode 100644 index 1149895340..0000000000 --- a/.claude/settings.json.example +++ /dev/null @@ -1,19 +0,0 @@ -{ - "permissions": { - "allow": [], - "deny": [] - }, - "env": { - "__comment": "Environment variables for MCP servers. Override in .claude/settings.local.json with actual values.", - "GITHUB_PERSONAL_ACCESS_TOKEN": "ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - "enabledMcpjsonServers": [ - "context7", - "sequential-thinking", - "github", - "fetch", - "playwright", - "ide" - ], - "enableAllProjectMcpServers": true - } \ No newline at end of file diff --git a/.claude/skills/component-refactoring/SKILL.md b/.claude/skills/component-refactoring/SKILL.md new file mode 100644 index 0000000000..7006c382c8 --- /dev/null +++ b/.claude/skills/component-refactoring/SKILL.md @@ -0,0 +1,483 @@ +--- +name: component-refactoring +description: Refactor high-complexity React components in Dify frontend. Use when `pnpm analyze-component --json` shows complexity > 50 or lineCount > 300, when the user asks for code splitting, hook extraction, or complexity reduction, or when `pnpm analyze-component` warns to refactor before testing; avoid for simple/well-structured components, third-party wrappers, or when the user explicitly wants testing without refactoring. +--- + +# Dify Component Refactoring Skill + +Refactor high-complexity React components in the Dify frontend codebase with the patterns and workflow below. + +> **Complexity Threshold**: Components with complexity > 50 (measured by `pnpm analyze-component`) should be refactored before testing. + +## Quick Reference + +### Commands (run from `web/`) + +Use paths relative to `web/` (e.g., `app/components/...`). +Use `refactor-component` for refactoring prompts and `analyze-component` for testing prompts and metrics. + +```bash +cd web + +# Generate refactoring prompt +pnpm refactor-component + +# Output refactoring analysis as JSON +pnpm refactor-component --json + +# Generate testing prompt (after refactoring) +pnpm analyze-component + +# Output testing analysis as JSON +pnpm analyze-component --json +``` + +### Complexity Analysis + +```bash +# Analyze component complexity +pnpm analyze-component --json + +# Key metrics to check: +# - complexity: normalized score 0-100 (target < 50) +# - maxComplexity: highest single function complexity +# - lineCount: total lines (target < 300) +``` + +### Complexity Score Interpretation + +| Score | Level | Action | +|-------|-------|--------| +| 0-25 | 🟢 Simple | Ready for testing | +| 26-50 | 🟡 Medium | Consider minor refactoring | +| 51-75 | 🟠 Complex | **Refactor before testing** | +| 76-100 | 🔴 Very Complex | **Must refactor** | + +## Core Refactoring Patterns + +### Pattern 1: Extract Custom Hooks + +**When**: Component has complex state management, multiple `useState`/`useEffect`, or business logic mixed with UI. + +**Dify Convention**: Place hooks in a `hooks/` subdirectory or alongside the component as `use-.ts`. + +```typescript +// ❌ Before: Complex state logic in component +const Configuration: FC = () => { + const [modelConfig, setModelConfig] = useState(...) + const [datasetConfigs, setDatasetConfigs] = useState(...) + const [completionParams, setCompletionParams] = useState({}) + + // 50+ lines of state management logic... + + return
...
+} + +// ✅ After: Extract to custom hook +// hooks/use-model-config.ts +export const useModelConfig = (appId: string) => { + const [modelConfig, setModelConfig] = useState(...) + const [completionParams, setCompletionParams] = useState({}) + + // Related state management logic here + + return { modelConfig, setModelConfig, completionParams, setCompletionParams } +} + +// Component becomes cleaner +const Configuration: FC = () => { + const { modelConfig, setModelConfig } = useModelConfig(appId) + return
...
+} +``` + +**Dify Examples**: +- `web/app/components/app/configuration/hooks/use-advanced-prompt-config.ts` +- `web/app/components/app/configuration/debug/hooks.tsx` +- `web/app/components/workflow/hooks/use-workflow.ts` + +### Pattern 2: Extract Sub-Components + +**When**: Single component has multiple UI sections, conditional rendering blocks, or repeated patterns. + +**Dify Convention**: Place sub-components in subdirectories or as separate files in the same directory. + +```typescript +// ❌ Before: Monolithic JSX with multiple sections +const AppInfo = () => { + return ( +
+ {/* 100 lines of header UI */} + {/* 100 lines of operations UI */} + {/* 100 lines of modals */} +
+ ) +} + +// ✅ After: Split into focused components +// app-info/ +// ├── index.tsx (orchestration only) +// ├── app-header.tsx (header UI) +// ├── app-operations.tsx (operations UI) +// └── app-modals.tsx (modal management) + +const AppInfo = () => { + const { showModal, setShowModal } = useAppInfoModals() + + return ( +
+ + + setShowModal(null)} /> +
+ ) +} +``` + +**Dify Examples**: +- `web/app/components/app/configuration/` directory structure +- `web/app/components/workflow/nodes/` per-node organization + +### Pattern 3: Simplify Conditional Logic + +**When**: Deep nesting (> 3 levels), complex ternaries, or multiple `if/else` chains. + +```typescript +// ❌ Before: Deeply nested conditionals +const Template = useMemo(() => { + if (appDetail?.mode === AppModeEnum.CHAT) { + switch (locale) { + case LanguagesSupported[1]: + return + case LanguagesSupported[7]: + return + default: + return + } + } + if (appDetail?.mode === AppModeEnum.ADVANCED_CHAT) { + // Another 15 lines... + } + // More conditions... +}, [appDetail, locale]) + +// ✅ After: Use lookup tables + early returns +const TEMPLATE_MAP = { + [AppModeEnum.CHAT]: { + [LanguagesSupported[1]]: TemplateChatZh, + [LanguagesSupported[7]]: TemplateChatJa, + default: TemplateChatEn, + }, + [AppModeEnum.ADVANCED_CHAT]: { + [LanguagesSupported[1]]: TemplateAdvancedChatZh, + // ... + }, +} + +const Template = useMemo(() => { + const modeTemplates = TEMPLATE_MAP[appDetail?.mode] + if (!modeTemplates) return null + + const TemplateComponent = modeTemplates[locale] || modeTemplates.default + return +}, [appDetail, locale]) +``` + +### Pattern 4: Extract API/Data Logic + +**When**: Component directly handles API calls, data transformation, or complex async operations. + +**Dify Convention**: Use `@tanstack/react-query` hooks from `web/service/use-*.ts` or create custom data hooks. + +```typescript +// ❌ Before: API logic in component +const MCPServiceCard = () => { + const [basicAppConfig, setBasicAppConfig] = useState({}) + + useEffect(() => { + if (isBasicApp && appId) { + (async () => { + const res = await fetchAppDetail({ url: '/apps', id: appId }) + setBasicAppConfig(res?.model_config || {}) + })() + } + }, [appId, isBasicApp]) + + // More API-related logic... +} + +// ✅ After: Extract to data hook using React Query +// use-app-config.ts +import { useQuery } from '@tanstack/react-query' +import { get } from '@/service/base' + +const NAME_SPACE = 'appConfig' + +export const useAppConfig = (appId: string, isBasicApp: boolean) => { + return useQuery({ + enabled: isBasicApp && !!appId, + queryKey: [NAME_SPACE, 'detail', appId], + queryFn: () => get(`/apps/${appId}`), + select: data => data?.model_config || {}, + }) +} + +// Component becomes cleaner +const MCPServiceCard = () => { + const { data: config, isLoading } = useAppConfig(appId, isBasicApp) + // UI only +} +``` + +**React Query Best Practices in Dify**: +- Define `NAME_SPACE` for query key organization +- Use `enabled` option for conditional fetching +- Use `select` for data transformation +- Export invalidation hooks: `useInvalidXxx` + +**Dify Examples**: +- `web/service/use-workflow.ts` +- `web/service/use-common.ts` +- `web/service/knowledge/use-dataset.ts` +- `web/service/knowledge/use-document.ts` + +### Pattern 5: Extract Modal/Dialog Management + +**When**: Component manages multiple modals with complex open/close states. + +**Dify Convention**: Modals should be extracted with their state management. + +```typescript +// ❌ Before: Multiple modal states in component +const AppInfo = () => { + const [showEditModal, setShowEditModal] = useState(false) + const [showDuplicateModal, setShowDuplicateModal] = useState(false) + const [showConfirmDelete, setShowConfirmDelete] = useState(false) + const [showSwitchModal, setShowSwitchModal] = useState(false) + const [showImportDSLModal, setShowImportDSLModal] = useState(false) + // 5+ more modal states... +} + +// ✅ After: Extract to modal management hook +type ModalType = 'edit' | 'duplicate' | 'delete' | 'switch' | 'import' | null + +const useAppInfoModals = () => { + const [activeModal, setActiveModal] = useState(null) + + const openModal = useCallback((type: ModalType) => setActiveModal(type), []) + const closeModal = useCallback(() => setActiveModal(null), []) + + return { + activeModal, + openModal, + closeModal, + isOpen: (type: ModalType) => activeModal === type, + } +} +``` + +### Pattern 6: Extract Form Logic + +**When**: Complex form validation, submission handling, or field transformation. + +**Dify Convention**: Use `@tanstack/react-form` patterns from `web/app/components/base/form/`. + +```typescript +// ✅ Use existing form infrastructure +import { useAppForm } from '@/app/components/base/form' + +const ConfigForm = () => { + const form = useAppForm({ + defaultValues: { name: '', description: '' }, + onSubmit: handleSubmit, + }) + + return ... +} +``` + +## Dify-Specific Refactoring Guidelines + +### 1. Context Provider Extraction + +**When**: Component provides complex context values with multiple states. + +```typescript +// ❌ Before: Large context value object +const value = { + appId, isAPIKeySet, isTrailFinished, mode, modelModeType, + promptMode, isAdvancedMode, isAgent, isOpenAI, isFunctionCall, + // 50+ more properties... +} +return ... + +// ✅ After: Split into domain-specific contexts + + + + {children} + + + +``` + +**Dify Reference**: `web/context/` directory structure + +### 2. Workflow Node Components + +**When**: Refactoring workflow node components (`web/app/components/workflow/nodes/`). + +**Conventions**: +- Keep node logic in `use-interactions.ts` +- Extract panel UI to separate files +- Use `_base` components for common patterns + +``` +nodes// + ├── index.tsx # Node registration + ├── node.tsx # Node visual component + ├── panel.tsx # Configuration panel + ├── use-interactions.ts # Node-specific hooks + └── types.ts # Type definitions +``` + +### 3. Configuration Components + +**When**: Refactoring app configuration components. + +**Conventions**: +- Separate config sections into subdirectories +- Use existing patterns from `web/app/components/app/configuration/` +- Keep feature toggles in dedicated components + +### 4. Tool/Plugin Components + +**When**: Refactoring tool-related components (`web/app/components/tools/`). + +**Conventions**: +- Follow existing modal patterns +- Use service hooks from `web/service/use-tools.ts` +- Keep provider-specific logic isolated + +## Refactoring Workflow + +### Step 1: Generate Refactoring Prompt + +```bash +pnpm refactor-component +``` + +This command will: +- Analyze component complexity and features +- Identify specific refactoring actions needed +- Generate a prompt for AI assistant (auto-copied to clipboard on macOS) +- Provide detailed requirements based on detected patterns + +### Step 2: Analyze Details + +```bash +pnpm analyze-component --json +``` + +Identify: +- Total complexity score +- Max function complexity +- Line count +- Features detected (state, effects, API, etc.) + +### Step 3: Plan + +Create a refactoring plan based on detected features: + +| Detected Feature | Refactoring Action | +|------------------|-------------------| +| `hasState: true` + `hasEffects: true` | Extract custom hook | +| `hasAPI: true` | Extract data/service hook | +| `hasEvents: true` (many) | Extract event handlers | +| `lineCount > 300` | Split into sub-components | +| `maxComplexity > 50` | Simplify conditional logic | + +### Step 4: Execute Incrementally + +1. **Extract one piece at a time** +2. **Run lint, type-check, and tests after each extraction** +3. **Verify functionality before next step** + +``` +For each extraction: + ┌────────────────────────────────────────┐ + │ 1. Extract code │ + │ 2. Run: pnpm lint:fix │ + │ 3. Run: pnpm type-check:tsgo │ + │ 4. Run: pnpm test │ + │ 5. Test functionality manually │ + │ 6. PASS? → Next extraction │ + │ FAIL? → Fix before continuing │ + └────────────────────────────────────────┘ +``` + +### Step 5: Verify + +After refactoring: + +```bash +# Re-run refactor command to verify improvements +pnpm refactor-component + +# If complexity < 25 and lines < 200, you'll see: +# ✅ COMPONENT IS WELL-STRUCTURED + +# For detailed metrics: +pnpm analyze-component --json + +# Target metrics: +# - complexity < 50 +# - lineCount < 300 +# - maxComplexity < 30 +``` + +## Common Mistakes to Avoid + +### ❌ Over-Engineering + +```typescript +// ❌ Too many tiny hooks +const useButtonText = () => useState('Click') +const useButtonDisabled = () => useState(false) +const useButtonLoading = () => useState(false) + +// ✅ Cohesive hook with related state +const useButtonState = () => { + const [text, setText] = useState('Click') + const [disabled, setDisabled] = useState(false) + const [loading, setLoading] = useState(false) + return { text, setText, disabled, setDisabled, loading, setLoading } +} +``` + +### ❌ Breaking Existing Patterns + +- Follow existing directory structures +- Maintain naming conventions +- Preserve export patterns for compatibility + +### ❌ Premature Abstraction + +- Only extract when there's clear complexity benefit +- Don't create abstractions for single-use code +- Keep refactored code in the same domain area + +## References + +### Dify Codebase Examples + +- **Hook extraction**: `web/app/components/app/configuration/hooks/` +- **Component splitting**: `web/app/components/app/configuration/` +- **Service hooks**: `web/service/use-*.ts` +- **Workflow patterns**: `web/app/components/workflow/hooks/` +- **Form patterns**: `web/app/components/base/form/` + +### Related Skills + +- `frontend-testing` - For testing refactored components +- `web/testing/testing.md` - Testing specification diff --git a/.claude/skills/component-refactoring/references/complexity-patterns.md b/.claude/skills/component-refactoring/references/complexity-patterns.md new file mode 100644 index 0000000000..5a0a268f38 --- /dev/null +++ b/.claude/skills/component-refactoring/references/complexity-patterns.md @@ -0,0 +1,493 @@ +# Complexity Reduction Patterns + +This document provides patterns for reducing cognitive complexity in Dify React components. + +## Understanding Complexity + +### SonarJS Cognitive Complexity + +The `pnpm analyze-component` tool uses SonarJS cognitive complexity metrics: + +- **Total Complexity**: Sum of all functions' complexity in the file +- **Max Complexity**: Highest single function complexity + +### What Increases Complexity + +| Pattern | Complexity Impact | +|---------|-------------------| +| `if/else` | +1 per branch | +| Nested conditions | +1 per nesting level | +| `switch/case` | +1 per case | +| `for/while/do` | +1 per loop | +| `&&`/`||` chains | +1 per operator | +| Nested callbacks | +1 per nesting level | +| `try/catch` | +1 per catch | +| Ternary expressions | +1 per nesting | + +## Pattern 1: Replace Conditionals with Lookup Tables + +**Before** (complexity: ~15): + +```typescript +const Template = useMemo(() => { + if (appDetail?.mode === AppModeEnum.CHAT) { + switch (locale) { + case LanguagesSupported[1]: + return + case LanguagesSupported[7]: + return + default: + return + } + } + if (appDetail?.mode === AppModeEnum.ADVANCED_CHAT) { + switch (locale) { + case LanguagesSupported[1]: + return + case LanguagesSupported[7]: + return + default: + return + } + } + if (appDetail?.mode === AppModeEnum.WORKFLOW) { + // Similar pattern... + } + return null +}, [appDetail, locale]) +``` + +**After** (complexity: ~3): + +```typescript +// Define lookup table outside component +const TEMPLATE_MAP: Record>> = { + [AppModeEnum.CHAT]: { + [LanguagesSupported[1]]: TemplateChatZh, + [LanguagesSupported[7]]: TemplateChatJa, + default: TemplateChatEn, + }, + [AppModeEnum.ADVANCED_CHAT]: { + [LanguagesSupported[1]]: TemplateAdvancedChatZh, + [LanguagesSupported[7]]: TemplateAdvancedChatJa, + default: TemplateAdvancedChatEn, + }, + [AppModeEnum.WORKFLOW]: { + [LanguagesSupported[1]]: TemplateWorkflowZh, + [LanguagesSupported[7]]: TemplateWorkflowJa, + default: TemplateWorkflowEn, + }, + // ... +} + +// Clean component logic +const Template = useMemo(() => { + if (!appDetail?.mode) return null + + const templates = TEMPLATE_MAP[appDetail.mode] + if (!templates) return null + + const TemplateComponent = templates[locale] ?? templates.default + return +}, [appDetail, locale]) +``` + +## Pattern 2: Use Early Returns + +**Before** (complexity: ~10): + +```typescript +const handleSubmit = () => { + if (isValid) { + if (hasChanges) { + if (isConnected) { + submitData() + } else { + showConnectionError() + } + } else { + showNoChangesMessage() + } + } else { + showValidationError() + } +} +``` + +**After** (complexity: ~4): + +```typescript +const handleSubmit = () => { + if (!isValid) { + showValidationError() + return + } + + if (!hasChanges) { + showNoChangesMessage() + return + } + + if (!isConnected) { + showConnectionError() + return + } + + submitData() +} +``` + +## Pattern 3: Extract Complex Conditions + +**Before** (complexity: high): + +```typescript +const canPublish = (() => { + if (mode !== AppModeEnum.COMPLETION) { + if (!isAdvancedMode) + return true + + if (modelModeType === ModelModeType.completion) { + if (!hasSetBlockStatus.history || !hasSetBlockStatus.query) + return false + return true + } + return true + } + return !promptEmpty +})() +``` + +**After** (complexity: lower): + +```typescript +// Extract to named functions +const canPublishInCompletionMode = () => !promptEmpty + +const canPublishInChatMode = () => { + if (!isAdvancedMode) return true + if (modelModeType !== ModelModeType.completion) return true + return hasSetBlockStatus.history && hasSetBlockStatus.query +} + +// Clean main logic +const canPublish = mode === AppModeEnum.COMPLETION + ? canPublishInCompletionMode() + : canPublishInChatMode() +``` + +## Pattern 4: Replace Chained Ternaries + +**Before** (complexity: ~5): + +```typescript +const statusText = serverActivated + ? t('status.running') + : serverPublished + ? t('status.inactive') + : appUnpublished + ? t('status.unpublished') + : t('status.notConfigured') +``` + +**After** (complexity: ~2): + +```typescript +const getStatusText = () => { + if (serverActivated) return t('status.running') + if (serverPublished) return t('status.inactive') + if (appUnpublished) return t('status.unpublished') + return t('status.notConfigured') +} + +const statusText = getStatusText() +``` + +Or use lookup: + +```typescript +const STATUS_TEXT_MAP = { + running: 'status.running', + inactive: 'status.inactive', + unpublished: 'status.unpublished', + notConfigured: 'status.notConfigured', +} as const + +const getStatusKey = (): keyof typeof STATUS_TEXT_MAP => { + if (serverActivated) return 'running' + if (serverPublished) return 'inactive' + if (appUnpublished) return 'unpublished' + return 'notConfigured' +} + +const statusText = t(STATUS_TEXT_MAP[getStatusKey()]) +``` + +## Pattern 5: Flatten Nested Loops + +**Before** (complexity: high): + +```typescript +const processData = (items: Item[]) => { + const results: ProcessedItem[] = [] + + for (const item of items) { + if (item.isValid) { + for (const child of item.children) { + if (child.isActive) { + for (const prop of child.properties) { + if (prop.value !== null) { + results.push({ + itemId: item.id, + childId: child.id, + propValue: prop.value, + }) + } + } + } + } + } + } + + return results +} +``` + +**After** (complexity: lower): + +```typescript +// Use functional approach +const processData = (items: Item[]) => { + return items + .filter(item => item.isValid) + .flatMap(item => + item.children + .filter(child => child.isActive) + .flatMap(child => + child.properties + .filter(prop => prop.value !== null) + .map(prop => ({ + itemId: item.id, + childId: child.id, + propValue: prop.value, + })) + ) + ) +} +``` + +## Pattern 6: Extract Event Handler Logic + +**Before** (complexity: high in component): + +```typescript +const Component = () => { + const handleSelect = (data: DataSet[]) => { + if (isEqual(data.map(item => item.id), dataSets.map(item => item.id))) { + hideSelectDataSet() + return + } + + formattingChangedDispatcher() + let newDatasets = data + if (data.find(item => !item.name)) { + const newSelected = produce(data, (draft) => { + data.forEach((item, index) => { + if (!item.name) { + const newItem = dataSets.find(i => i.id === item.id) + if (newItem) + draft[index] = newItem + } + }) + }) + setDataSets(newSelected) + newDatasets = newSelected + } + else { + setDataSets(data) + } + hideSelectDataSet() + + // 40 more lines of logic... + } + + return
...
+} +``` + +**After** (complexity: lower): + +```typescript +// Extract to hook or utility +const useDatasetSelection = (dataSets: DataSet[], setDataSets: SetState) => { + const normalizeSelection = (data: DataSet[]) => { + const hasUnloadedItem = data.some(item => !item.name) + if (!hasUnloadedItem) return data + + return produce(data, (draft) => { + data.forEach((item, index) => { + if (!item.name) { + const existing = dataSets.find(i => i.id === item.id) + if (existing) draft[index] = existing + } + }) + }) + } + + const hasSelectionChanged = (newData: DataSet[]) => { + return !isEqual( + newData.map(item => item.id), + dataSets.map(item => item.id) + ) + } + + return { normalizeSelection, hasSelectionChanged } +} + +// Component becomes cleaner +const Component = () => { + const { normalizeSelection, hasSelectionChanged } = useDatasetSelection(dataSets, setDataSets) + + const handleSelect = (data: DataSet[]) => { + if (!hasSelectionChanged(data)) { + hideSelectDataSet() + return + } + + formattingChangedDispatcher() + const normalized = normalizeSelection(data) + setDataSets(normalized) + hideSelectDataSet() + } + + return
...
+} +``` + +## Pattern 7: Reduce Boolean Logic Complexity + +**Before** (complexity: ~8): + +```typescript +const toggleDisabled = hasInsufficientPermissions + || appUnpublished + || missingStartNode + || triggerModeDisabled + || (isAdvancedApp && !currentWorkflow?.graph) + || (isBasicApp && !basicAppConfig.updated_at) +``` + +**After** (complexity: ~3): + +```typescript +// Extract meaningful boolean functions +const isAppReady = () => { + if (isAdvancedApp) return !!currentWorkflow?.graph + return !!basicAppConfig.updated_at +} + +const hasRequiredPermissions = () => { + return isCurrentWorkspaceEditor && !hasInsufficientPermissions +} + +const canToggle = () => { + if (!hasRequiredPermissions()) return false + if (!isAppReady()) return false + if (missingStartNode) return false + if (triggerModeDisabled) return false + return true +} + +const toggleDisabled = !canToggle() +``` + +## Pattern 8: Simplify useMemo/useCallback Dependencies + +**Before** (complexity: multiple recalculations): + +```typescript +const payload = useMemo(() => { + let parameters: Parameter[] = [] + let outputParameters: OutputParameter[] = [] + + if (!published) { + parameters = (inputs || []).map((item) => ({ + name: item.variable, + description: '', + form: 'llm', + required: item.required, + type: item.type, + })) + outputParameters = (outputs || []).map((item) => ({ + name: item.variable, + description: '', + type: item.value_type, + })) + } + else if (detail && detail.tool) { + parameters = (inputs || []).map((item) => ({ + // Complex transformation... + })) + outputParameters = (outputs || []).map((item) => ({ + // Complex transformation... + })) + } + + return { + icon: detail?.icon || icon, + label: detail?.label || name, + // ...more fields + } +}, [detail, published, workflowAppId, icon, name, description, inputs, outputs]) +``` + +**After** (complexity: separated concerns): + +```typescript +// Separate transformations +const useParameterTransform = (inputs: InputVar[], detail?: ToolDetail, published?: boolean) => { + return useMemo(() => { + if (!published) { + return inputs.map(item => ({ + name: item.variable, + description: '', + form: 'llm', + required: item.required, + type: item.type, + })) + } + + if (!detail?.tool) return [] + + return inputs.map(item => ({ + name: item.variable, + required: item.required, + type: item.type === 'paragraph' ? 'string' : item.type, + description: detail.tool.parameters.find(p => p.name === item.variable)?.llm_description || '', + form: detail.tool.parameters.find(p => p.name === item.variable)?.form || 'llm', + })) + }, [inputs, detail, published]) +} + +// Component uses hook +const parameters = useParameterTransform(inputs, detail, published) +const outputParameters = useOutputTransform(outputs, detail, published) + +const payload = useMemo(() => ({ + icon: detail?.icon || icon, + label: detail?.label || name, + parameters, + outputParameters, + // ... +}), [detail, icon, name, parameters, outputParameters]) +``` + +## Target Metrics After Refactoring + +| Metric | Target | +|--------|--------| +| Total Complexity | < 50 | +| Max Function Complexity | < 30 | +| Function Length | < 30 lines | +| Nesting Depth | ≤ 3 levels | +| Conditional Chains | ≤ 3 conditions | diff --git a/.claude/skills/component-refactoring/references/component-splitting.md b/.claude/skills/component-refactoring/references/component-splitting.md new file mode 100644 index 0000000000..78a3389100 --- /dev/null +++ b/.claude/skills/component-refactoring/references/component-splitting.md @@ -0,0 +1,477 @@ +# Component Splitting Patterns + +This document provides detailed guidance on splitting large components into smaller, focused components in Dify. + +## When to Split Components + +Split a component when you identify: + +1. **Multiple UI sections** - Distinct visual areas with minimal coupling that can be composed independently +1. **Conditional rendering blocks** - Large `{condition && }` blocks +1. **Repeated patterns** - Similar UI structures used multiple times +1. **300+ lines** - Component exceeds manageable size +1. **Modal clusters** - Multiple modals rendered in one component + +## Splitting Strategies + +### Strategy 1: Section-Based Splitting + +Identify visual sections and extract each as a component. + +```typescript +// ❌ Before: Monolithic component (500+ lines) +const ConfigurationPage = () => { + return ( +
+ {/* Header Section - 50 lines */} +
+

{t('configuration.title')}

+
+ {isAdvancedMode && Advanced} + + +
+
+ + {/* Config Section - 200 lines */} +
+ +
+ + {/* Debug Section - 150 lines */} +
+ +
+ + {/* Modals Section - 100 lines */} + {showSelectDataSet && } + {showHistoryModal && } + {showUseGPT4Confirm && } +
+ ) +} + +// ✅ After: Split into focused components +// configuration/ +// ├── index.tsx (orchestration) +// ├── configuration-header.tsx +// ├── configuration-content.tsx +// ├── configuration-debug.tsx +// └── configuration-modals.tsx + +// configuration-header.tsx +interface ConfigurationHeaderProps { + isAdvancedMode: boolean + onPublish: () => void +} + +const ConfigurationHeader: FC = ({ + isAdvancedMode, + onPublish, +}) => { + const { t } = useTranslation() + + return ( +
+

{t('configuration.title')}

+
+ {isAdvancedMode && Advanced} + + +
+
+ ) +} + +// index.tsx (orchestration only) +const ConfigurationPage = () => { + const { modelConfig, setModelConfig } = useModelConfig() + const { activeModal, openModal, closeModal } = useModalState() + + return ( +
+ + + {!isMobile && ( + + )} + +
+ ) +} +``` + +### Strategy 2: Conditional Block Extraction + +Extract large conditional rendering blocks. + +```typescript +// ❌ Before: Large conditional blocks +const AppInfo = () => { + return ( +
+ {expand ? ( +
+ {/* 100 lines of expanded view */} +
+ ) : ( +
+ {/* 50 lines of collapsed view */} +
+ )} +
+ ) +} + +// ✅ After: Separate view components +const AppInfoExpanded: FC = ({ appDetail, onAction }) => { + return ( +
+ {/* Clean, focused expanded view */} +
+ ) +} + +const AppInfoCollapsed: FC = ({ appDetail, onAction }) => { + return ( +
+ {/* Clean, focused collapsed view */} +
+ ) +} + +const AppInfo = () => { + return ( +
+ {expand + ? + : + } +
+ ) +} +``` + +### Strategy 3: Modal Extraction + +Extract modals with their trigger logic. + +```typescript +// ❌ Before: Multiple modals in one component +const AppInfo = () => { + const [showEdit, setShowEdit] = useState(false) + const [showDuplicate, setShowDuplicate] = useState(false) + const [showDelete, setShowDelete] = useState(false) + const [showSwitch, setShowSwitch] = useState(false) + + const onEdit = async (data) => { /* 20 lines */ } + const onDuplicate = async (data) => { /* 20 lines */ } + const onDelete = async () => { /* 15 lines */ } + + return ( +
+ {/* Main content */} + + {showEdit && setShowEdit(false)} />} + {showDuplicate && setShowDuplicate(false)} />} + {showDelete && setShowDelete(false)} />} + {showSwitch && } +
+ ) +} + +// ✅ After: Modal manager component +// app-info-modals.tsx +type ModalType = 'edit' | 'duplicate' | 'delete' | 'switch' | null + +interface AppInfoModalsProps { + appDetail: AppDetail + activeModal: ModalType + onClose: () => void + onSuccess: () => void +} + +const AppInfoModals: FC = ({ + appDetail, + activeModal, + onClose, + onSuccess, +}) => { + const handleEdit = async (data) => { /* logic */ } + const handleDuplicate = async (data) => { /* logic */ } + const handleDelete = async () => { /* logic */ } + + return ( + <> + {activeModal === 'edit' && ( + + )} + {activeModal === 'duplicate' && ( + + )} + {activeModal === 'delete' && ( + + )} + {activeModal === 'switch' && ( + + )} + + ) +} + +// Parent component +const AppInfo = () => { + const { activeModal, openModal, closeModal } = useModalState() + + return ( +
+ {/* Main content with openModal triggers */} + + + +
+ ) +} +``` + +### Strategy 4: List Item Extraction + +Extract repeated item rendering. + +```typescript +// ❌ Before: Inline item rendering +const OperationsList = () => { + return ( +
+ {operations.map(op => ( +
+ {op.icon} + {op.title} + {op.description} + + {op.badge && {op.badge}} + {/* More complex rendering... */} +
+ ))} +
+ ) +} + +// ✅ After: Extracted item component +interface OperationItemProps { + operation: Operation + onAction: (id: string) => void +} + +const OperationItem: FC = ({ operation, onAction }) => { + return ( +
+ {operation.icon} + {operation.title} + {operation.description} + + {operation.badge && {operation.badge}} +
+ ) +} + +const OperationsList = () => { + const handleAction = useCallback((id: string) => { + const op = operations.find(o => o.id === id) + op?.onClick() + }, [operations]) + + return ( +
+ {operations.map(op => ( + + ))} +
+ ) +} +``` + +## Directory Structure Patterns + +### Pattern A: Flat Structure (Simple Components) + +For components with 2-3 sub-components: + +``` +component-name/ + ├── index.tsx # Main component + ├── sub-component-a.tsx + ├── sub-component-b.tsx + └── types.ts # Shared types +``` + +### Pattern B: Nested Structure (Complex Components) + +For components with many sub-components: + +``` +component-name/ + ├── index.tsx # Main orchestration + ├── types.ts # Shared types + ├── hooks/ + │ ├── use-feature-a.ts + │ └── use-feature-b.ts + ├── components/ + │ ├── header/ + │ │ └── index.tsx + │ ├── content/ + │ │ └── index.tsx + │ └── modals/ + │ └── index.tsx + └── utils/ + └── helpers.ts +``` + +### Pattern C: Feature-Based Structure (Dify Standard) + +Following Dify's existing patterns: + +``` +configuration/ + ├── index.tsx # Main page component + ├── base/ # Base/shared components + │ ├── feature-panel/ + │ ├── group-name/ + │ └── operation-btn/ + ├── config/ # Config section + │ ├── index.tsx + │ ├── agent/ + │ └── automatic/ + ├── dataset-config/ # Dataset section + │ ├── index.tsx + │ ├── card-item/ + │ └── params-config/ + ├── debug/ # Debug section + │ ├── index.tsx + │ └── hooks.tsx + └── hooks/ # Shared hooks + └── use-advanced-prompt-config.ts +``` + +## Props Design + +### Minimal Props Principle + +Pass only what's needed: + +```typescript +// ❌ Bad: Passing entire objects when only some fields needed + + +// ✅ Good: Destructure to minimum required + +``` + +### Callback Props Pattern + +Use callbacks for child-to-parent communication: + +```typescript +// Parent +const Parent = () => { + const [value, setValue] = useState('') + + return ( + + ) +} + +// Child +interface ChildProps { + value: string + onChange: (value: string) => void + onSubmit: () => void +} + +const Child: FC = ({ value, onChange, onSubmit }) => { + return ( +
+ onChange(e.target.value)} /> + +
+ ) +} +``` + +### Render Props for Flexibility + +When sub-components need parent context: + +```typescript +interface ListProps { + items: T[] + renderItem: (item: T, index: number) => React.ReactNode + renderEmpty?: () => React.ReactNode +} + +function List({ items, renderItem, renderEmpty }: ListProps) { + if (items.length === 0 && renderEmpty) { + return <>{renderEmpty()} + } + + return ( +
+ {items.map((item, index) => renderItem(item, index))} +
+ ) +} + +// Usage + } + renderEmpty={() => } +/> +``` diff --git a/.claude/skills/component-refactoring/references/hook-extraction.md b/.claude/skills/component-refactoring/references/hook-extraction.md new file mode 100644 index 0000000000..a8d75deffd --- /dev/null +++ b/.claude/skills/component-refactoring/references/hook-extraction.md @@ -0,0 +1,317 @@ +# Hook Extraction Patterns + +This document provides detailed guidance on extracting custom hooks from complex components in Dify. + +## When to Extract Hooks + +Extract a custom hook when you identify: + +1. **Coupled state groups** - Multiple `useState` hooks that are always used together +1. **Complex effects** - `useEffect` with multiple dependencies or cleanup logic +1. **Business logic** - Data transformations, validations, or calculations +1. **Reusable patterns** - Logic that appears in multiple components + +## Extraction Process + +### Step 1: Identify State Groups + +Look for state variables that are logically related: + +```typescript +// ❌ These belong together - extract to hook +const [modelConfig, setModelConfig] = useState(...) +const [completionParams, setCompletionParams] = useState({}) +const [modelModeType, setModelModeType] = useState(...) + +// These are model-related state that should be in useModelConfig() +``` + +### Step 2: Identify Related Effects + +Find effects that modify the grouped state: + +```typescript +// ❌ These effects belong with the state above +useEffect(() => { + if (hasFetchedDetail && !modelModeType) { + const mode = currModel?.model_properties.mode + if (mode) { + const newModelConfig = produce(modelConfig, (draft) => { + draft.mode = mode + }) + setModelConfig(newModelConfig) + } + } +}, [textGenerationModelList, hasFetchedDetail, modelModeType, currModel]) +``` + +### Step 3: Create the Hook + +```typescript +// hooks/use-model-config.ts +import type { FormValue } from '@/app/components/header/account-setting/model-provider-page/declarations' +import type { ModelConfig } from '@/models/debug' +import { produce } from 'immer' +import { useEffect, useState } from 'react' +import { ModelModeType } from '@/types/app' + +interface UseModelConfigParams { + initialConfig?: Partial + currModel?: { model_properties?: { mode?: ModelModeType } } + hasFetchedDetail: boolean +} + +interface UseModelConfigReturn { + modelConfig: ModelConfig + setModelConfig: (config: ModelConfig) => void + completionParams: FormValue + setCompletionParams: (params: FormValue) => void + modelModeType: ModelModeType +} + +export const useModelConfig = ({ + initialConfig, + currModel, + hasFetchedDetail, +}: UseModelConfigParams): UseModelConfigReturn => { + const [modelConfig, setModelConfig] = useState({ + provider: 'langgenius/openai/openai', + model_id: 'gpt-3.5-turbo', + mode: ModelModeType.unset, + // ... default values + ...initialConfig, + }) + + const [completionParams, setCompletionParams] = useState({}) + + const modelModeType = modelConfig.mode + + // Fill old app data missing model mode + useEffect(() => { + if (hasFetchedDetail && !modelModeType) { + const mode = currModel?.model_properties?.mode + if (mode) { + setModelConfig(produce(modelConfig, (draft) => { + draft.mode = mode + })) + } + } + }, [hasFetchedDetail, modelModeType, currModel]) + + return { + modelConfig, + setModelConfig, + completionParams, + setCompletionParams, + modelModeType, + } +} +``` + +### Step 4: Update Component + +```typescript +// Before: 50+ lines of state management +const Configuration: FC = () => { + const [modelConfig, setModelConfig] = useState(...) + // ... lots of related state and effects +} + +// After: Clean component +const Configuration: FC = () => { + const { + modelConfig, + setModelConfig, + completionParams, + setCompletionParams, + modelModeType, + } = useModelConfig({ + currModel, + hasFetchedDetail, + }) + + // Component now focuses on UI +} +``` + +## Naming Conventions + +### Hook Names + +- Use `use` prefix: `useModelConfig`, `useDatasetConfig` +- Be specific: `useAdvancedPromptConfig` not `usePrompt` +- Include domain: `useWorkflowVariables`, `useMCPServer` + +### File Names + +- Kebab-case: `use-model-config.ts` +- Place in `hooks/` subdirectory when multiple hooks exist +- Place alongside component for single-use hooks + +### Return Type Names + +- Suffix with `Return`: `UseModelConfigReturn` +- Suffix params with `Params`: `UseModelConfigParams` + +## Common Hook Patterns in Dify + +### 1. Data Fetching Hook (React Query) + +```typescript +// Pattern: Use @tanstack/react-query for data fetching +import { useQuery, useQueryClient } from '@tanstack/react-query' +import { get } from '@/service/base' +import { useInvalid } from '@/service/use-base' + +const NAME_SPACE = 'appConfig' + +// Query keys for cache management +export const appConfigQueryKeys = { + detail: (appId: string) => [NAME_SPACE, 'detail', appId] as const, +} + +// Main data hook +export const useAppConfig = (appId: string) => { + return useQuery({ + enabled: !!appId, + queryKey: appConfigQueryKeys.detail(appId), + queryFn: () => get(`/apps/${appId}`), + select: data => data?.model_config || null, + }) +} + +// Invalidation hook for refreshing data +export const useInvalidAppConfig = () => { + return useInvalid([NAME_SPACE]) +} + +// Usage in component +const Component = () => { + const { data: config, isLoading, error, refetch } = useAppConfig(appId) + const invalidAppConfig = useInvalidAppConfig() + + const handleRefresh = () => { + invalidAppConfig() // Invalidates cache and triggers refetch + } + + return
...
+} +``` + +### 2. Form State Hook + +```typescript +// Pattern: Form state + validation + submission +export const useConfigForm = (initialValues: ConfigFormValues) => { + const [values, setValues] = useState(initialValues) + const [errors, setErrors] = useState>({}) + const [isSubmitting, setIsSubmitting] = useState(false) + + const validate = useCallback(() => { + const newErrors: Record = {} + if (!values.name) newErrors.name = 'Name is required' + setErrors(newErrors) + return Object.keys(newErrors).length === 0 + }, [values]) + + const handleChange = useCallback((field: string, value: any) => { + setValues(prev => ({ ...prev, [field]: value })) + }, []) + + const handleSubmit = useCallback(async (onSubmit: (values: ConfigFormValues) => Promise) => { + if (!validate()) return + setIsSubmitting(true) + try { + await onSubmit(values) + } finally { + setIsSubmitting(false) + } + }, [values, validate]) + + return { values, errors, isSubmitting, handleChange, handleSubmit } +} +``` + +### 3. Modal State Hook + +```typescript +// Pattern: Multiple modal management +type ModalType = 'edit' | 'delete' | 'duplicate' | null + +export const useModalState = () => { + const [activeModal, setActiveModal] = useState(null) + const [modalData, setModalData] = useState(null) + + const openModal = useCallback((type: ModalType, data?: any) => { + setActiveModal(type) + setModalData(data) + }, []) + + const closeModal = useCallback(() => { + setActiveModal(null) + setModalData(null) + }, []) + + return { + activeModal, + modalData, + openModal, + closeModal, + isOpen: useCallback((type: ModalType) => activeModal === type, [activeModal]), + } +} +``` + +### 4. Toggle/Boolean Hook + +```typescript +// Pattern: Boolean state with convenience methods +export const useToggle = (initialValue = false) => { + const [value, setValue] = useState(initialValue) + + const toggle = useCallback(() => setValue(v => !v), []) + const setTrue = useCallback(() => setValue(true), []) + const setFalse = useCallback(() => setValue(false), []) + + return [value, { toggle, setTrue, setFalse, set: setValue }] as const +} + +// Usage +const [isExpanded, { toggle, setTrue: expand, setFalse: collapse }] = useToggle() +``` + +## Testing Extracted Hooks + +After extraction, test hooks in isolation: + +```typescript +// use-model-config.spec.ts +import { renderHook, act } from '@testing-library/react' +import { useModelConfig } from './use-model-config' + +describe('useModelConfig', () => { + it('should initialize with default values', () => { + const { result } = renderHook(() => useModelConfig({ + hasFetchedDetail: false, + })) + + expect(result.current.modelConfig.provider).toBe('langgenius/openai/openai') + expect(result.current.modelModeType).toBe(ModelModeType.unset) + }) + + it('should update model config', () => { + const { result } = renderHook(() => useModelConfig({ + hasFetchedDetail: true, + })) + + act(() => { + result.current.setModelConfig({ + ...result.current.modelConfig, + model_id: 'gpt-4', + }) + }) + + expect(result.current.modelConfig.model_id).toBe('gpt-4') + }) +}) +``` diff --git a/.claude/skills/frontend-testing/SKILL.md b/.claude/skills/frontend-testing/SKILL.md index 06cb672141..dd9677a78e 100644 --- a/.claude/skills/frontend-testing/SKILL.md +++ b/.claude/skills/frontend-testing/SKILL.md @@ -1,13 +1,13 @@ --- -name: Dify Frontend Testing -description: Generate Jest + React Testing Library tests for Dify frontend components, hooks, and utilities. Triggers on testing, spec files, coverage, Jest, RTL, unit tests, integration tests, or write/review test requests. +name: frontend-testing +description: Generate Vitest + React Testing Library tests for Dify frontend components, hooks, and utilities. Triggers on testing, spec files, coverage, Vitest, RTL, unit tests, integration tests, or write/review test requests. --- # Dify Frontend Testing Skill This skill enables Claude to generate high-quality, comprehensive frontend tests for the Dify project following established conventions and best practices. -> **⚠️ Authoritative Source**: This skill is derived from `web/testing/testing.md`. When in doubt, always refer to that document as the canonical specification. +> **⚠️ Authoritative Source**: This skill is derived from `web/testing/testing.md`. Use Vitest mock/timer APIs (`vi.*`). ## When to Apply This Skill @@ -15,7 +15,7 @@ Apply this skill when the user: - Asks to **write tests** for a component, hook, or utility - Asks to **review existing tests** for completeness -- Mentions **Jest**, **React Testing Library**, **RTL**, or **spec files** +- Mentions **Vitest**, **React Testing Library**, **RTL**, or **spec files** - Requests **test coverage** improvement - Uses `pnpm analyze-component` output as context - Mentions **testing**, **unit tests**, or **integration tests** for frontend code @@ -33,9 +33,9 @@ Apply this skill when the user: | Tool | Version | Purpose | |------|---------|---------| -| Jest | 29.7 | Test runner | +| Vitest | 4.0.16 | Test runner | | React Testing Library | 16.0 | Component testing | -| happy-dom | - | Test environment | +| jsdom | - | Test environment | | nock | 14.0 | HTTP mocking | | TypeScript | 5.x | Type safety | @@ -46,13 +46,13 @@ Apply this skill when the user: pnpm test # Watch mode -pnpm test -- --watch +pnpm test:watch # Run specific file -pnpm test -- path/to/file.spec.tsx +pnpm test path/to/file.spec.tsx # Generate coverage report -pnpm test -- --coverage +pnpm test:coverage # Analyze component complexity pnpm analyze-component @@ -77,9 +77,9 @@ import Component from './index' // import { ChildComponent } from './child-component' // ✅ Mock external dependencies only -jest.mock('@/service/api') -jest.mock('next/navigation', () => ({ - useRouter: () => ({ push: jest.fn() }), +vi.mock('@/service/api') +vi.mock('next/navigation', () => ({ + useRouter: () => ({ push: vi.fn() }), usePathname: () => '/test', })) @@ -88,7 +88,7 @@ let mockSharedState = false describe('ComponentName', () => { beforeEach(() => { - jest.clearAllMocks() // ✅ Reset mocks BEFORE each test + vi.clearAllMocks() // ✅ Reset mocks BEFORE each test mockSharedState = false // ✅ Reset shared state }) @@ -117,7 +117,7 @@ describe('ComponentName', () => { // User Interactions describe('User Interactions', () => { it('should handle click events', () => { - const handleClick = jest.fn() + const handleClick = vi.fn() render() fireEvent.click(screen.getByRole('button')) @@ -155,7 +155,7 @@ describe('ComponentName', () => { For each file: ┌────────────────────────────────────────┐ │ 1. Write test │ - │ 2. Run: pnpm test -- .spec.tsx │ + │ 2. Run: pnpm test .spec.tsx │ │ 3. PASS? → Mark complete, next file │ │ FAIL? → Fix first, then continue │ └────────────────────────────────────────┘ @@ -178,7 +178,7 @@ Process in this order for multi-file testing: - **500+ lines**: Consider splitting before testing - **Many dependencies**: Extract logic into hooks first -> 📖 See `guides/workflow.md` for complete workflow details and todo list format. +> 📖 See `references/workflow.md` for complete workflow details and todo list format. ## Testing Strategy @@ -289,17 +289,18 @@ For each test file generated, aim for: - ✅ **>95%** branch coverage - ✅ **>95%** line coverage -> **Note**: For multi-file directories, process one file at a time with full coverage each. See `guides/workflow.md`. +> **Note**: For multi-file directories, process one file at a time with full coverage each. See `references/workflow.md`. ## Detailed Guides For more detailed information, refer to: -- `guides/workflow.md` - **Incremental testing workflow** (MUST READ for multi-file testing) -- `guides/mocking.md` - Mock patterns and best practices -- `guides/async-testing.md` - Async operations and API calls -- `guides/domain-components.md` - Workflow, Dataset, Configuration testing -- `guides/common-patterns.md` - Frequently used testing patterns +- `references/workflow.md` - **Incremental testing workflow** (MUST READ for multi-file testing) +- `references/mocking.md` - Mock patterns and best practices +- `references/async-testing.md` - Async operations and API calls +- `references/domain-components.md` - Workflow, Dataset, Configuration testing +- `references/common-patterns.md` - Frequently used testing patterns +- `references/checklist.md` - Test generation checklist and validation steps ## Authoritative References @@ -315,7 +316,7 @@ For more detailed information, refer to: ### Project Configuration -- `web/jest.config.ts` - Jest configuration -- `web/jest.setup.ts` - Test environment setup -- `web/testing/analyze-component.js` - Component analysis tool -- `web/__mocks__/react-i18next.ts` - Shared i18n mock (auto-loaded by Jest, no explicit mock needed; override locally only for custom translations) +- `web/vitest.config.ts` - Vitest configuration +- `web/vitest.setup.ts` - Test environment setup +- `web/scripts/analyze-component.js` - Component analysis tool +- Modules are not mocked automatically. Global mocks live in `web/vitest.setup.ts` (for example `react-i18next`, `next/image`); mock other modules like `ky` or `mime` locally in test files. diff --git a/.claude/skills/frontend-testing/templates/component-test.template.tsx b/.claude/skills/frontend-testing/assets/component-test.template.tsx similarity index 95% rename from .claude/skills/frontend-testing/templates/component-test.template.tsx rename to .claude/skills/frontend-testing/assets/component-test.template.tsx index f1ea71a3fd..c39baff916 100644 --- a/.claude/skills/frontend-testing/templates/component-test.template.tsx +++ b/.claude/skills/frontend-testing/assets/component-test.template.tsx @@ -23,14 +23,14 @@ import userEvent from '@testing-library/user-event' // ============================================================================ // Mocks // ============================================================================ -// WHY: Mocks must be hoisted to top of file (Jest requirement). +// WHY: Mocks must be hoisted to top of file (Vitest requirement). // They run BEFORE imports, so keep them before component imports. // i18n (automatically mocked) -// WHY: Shared mock at web/__mocks__/react-i18next.ts is auto-loaded by Jest +// WHY: Global mock in web/vitest.setup.ts is auto-loaded by Vitest setup // No explicit mock needed - it returns translation keys as-is // Override only if custom translations are required: -// jest.mock('react-i18next', () => ({ +// vi.mock('react-i18next', () => ({ // useTranslation: () => ({ // t: (key: string) => { // const customTranslations: Record = { @@ -43,17 +43,17 @@ import userEvent from '@testing-library/user-event' // Router (if component uses useRouter, usePathname, useSearchParams) // WHY: Isolates tests from Next.js routing, enables testing navigation behavior -// const mockPush = jest.fn() -// jest.mock('next/navigation', () => ({ +// const mockPush = vi.fn() +// vi.mock('next/navigation', () => ({ // useRouter: () => ({ push: mockPush }), // usePathname: () => '/test-path', // })) // API services (if component fetches data) // WHY: Prevents real network calls, enables testing all states (loading/success/error) -// jest.mock('@/service/api') +// vi.mock('@/service/api') // import * as api from '@/service/api' -// const mockedApi = api as jest.Mocked +// const mockedApi = vi.mocked(api) // Shared mock state (for portal/dropdown components) // WHY: Portal components like PortalToFollowElem need shared state between @@ -98,7 +98,7 @@ describe('ComponentName', () => { // - Prevents mock call history from leaking between tests // - MUST be beforeEach (not afterEach) to reset BEFORE assertions like toHaveBeenCalledTimes beforeEach(() => { - jest.clearAllMocks() + vi.clearAllMocks() // Reset shared mock state if used (CRITICAL for portal/dropdown tests) // mockOpenState = false }) @@ -155,7 +155,7 @@ describe('ComponentName', () => { // - userEvent simulates real user behavior (focus, hover, then click) // - fireEvent is lower-level, doesn't trigger all browser events // const user = userEvent.setup() - // const handleClick = jest.fn() + // const handleClick = vi.fn() // render() // // await user.click(screen.getByRole('button')) @@ -165,7 +165,7 @@ describe('ComponentName', () => { it('should call onChange when value changes', async () => { // const user = userEvent.setup() - // const handleChange = jest.fn() + // const handleChange = vi.fn() // render() // // await user.type(screen.getByRole('textbox'), 'new value') @@ -198,7 +198,7 @@ describe('ComponentName', () => { }) // -------------------------------------------------------------------------- - // Async Operations (if component fetches data - useSWR, useQuery, fetch) + // Async Operations (if component fetches data - useQuery, fetch) // -------------------------------------------------------------------------- // WHY: Async operations have 3 states users experience: loading, success, error describe('Async Operations', () => { diff --git a/.claude/skills/frontend-testing/templates/hook-test.template.ts b/.claude/skills/frontend-testing/assets/hook-test.template.ts similarity index 95% rename from .claude/skills/frontend-testing/templates/hook-test.template.ts rename to .claude/skills/frontend-testing/assets/hook-test.template.ts index 4fb7fd21ec..99161848a4 100644 --- a/.claude/skills/frontend-testing/templates/hook-test.template.ts +++ b/.claude/skills/frontend-testing/assets/hook-test.template.ts @@ -15,9 +15,9 @@ import { renderHook, act, waitFor } from '@testing-library/react' // ============================================================================ // API services (if hook fetches data) -// jest.mock('@/service/api') +// vi.mock('@/service/api') // import * as api from '@/service/api' -// const mockedApi = api as jest.Mocked +// const mockedApi = vi.mocked(api) // ============================================================================ // Test Helpers @@ -38,7 +38,7 @@ import { renderHook, act, waitFor } from '@testing-library/react' describe('useHookName', () => { beforeEach(() => { - jest.clearAllMocks() + vi.clearAllMocks() }) // -------------------------------------------------------------------------- @@ -145,7 +145,7 @@ describe('useHookName', () => { // -------------------------------------------------------------------------- describe('Side Effects', () => { it('should call callback when value changes', () => { - // const callback = jest.fn() + // const callback = vi.fn() // const { result } = renderHook(() => useHookName({ onChange: callback })) // // act(() => { @@ -156,9 +156,9 @@ describe('useHookName', () => { }) it('should cleanup on unmount', () => { - // const cleanup = jest.fn() - // jest.spyOn(window, 'addEventListener') - // jest.spyOn(window, 'removeEventListener') + // const cleanup = vi.fn() + // vi.spyOn(window, 'addEventListener') + // vi.spyOn(window, 'removeEventListener') // // const { unmount } = renderHook(() => useHookName()) // diff --git a/.claude/skills/frontend-testing/templates/utility-test.template.ts b/.claude/skills/frontend-testing/assets/utility-test.template.ts similarity index 100% rename from .claude/skills/frontend-testing/templates/utility-test.template.ts rename to .claude/skills/frontend-testing/assets/utility-test.template.ts diff --git a/.claude/skills/frontend-testing/guides/async-testing.md b/.claude/skills/frontend-testing/references/async-testing.md similarity index 92% rename from .claude/skills/frontend-testing/guides/async-testing.md rename to .claude/skills/frontend-testing/references/async-testing.md index f9912debbf..ae775a87a9 100644 --- a/.claude/skills/frontend-testing/guides/async-testing.md +++ b/.claude/skills/frontend-testing/references/async-testing.md @@ -49,7 +49,7 @@ import userEvent from '@testing-library/user-event' it('should submit form', async () => { const user = userEvent.setup() - const onSubmit = jest.fn() + const onSubmit = vi.fn() render(
) @@ -77,15 +77,15 @@ it('should submit form', async () => { ```typescript describe('Debounced Search', () => { beforeEach(() => { - jest.useFakeTimers() + vi.useFakeTimers() }) afterEach(() => { - jest.useRealTimers() + vi.useRealTimers() }) it('should debounce search input', async () => { - const onSearch = jest.fn() + const onSearch = vi.fn() render() // Type in the input @@ -95,7 +95,7 @@ describe('Debounced Search', () => { expect(onSearch).not.toHaveBeenCalled() // Advance timers - jest.advanceTimersByTime(300) + vi.advanceTimersByTime(300) // Now search is called expect(onSearch).toHaveBeenCalledWith('query') @@ -107,8 +107,8 @@ describe('Debounced Search', () => { ```typescript it('should retry on failure', async () => { - jest.useFakeTimers() - const fetchData = jest.fn() + vi.useFakeTimers() + const fetchData = vi.fn() .mockRejectedValueOnce(new Error('Network error')) .mockResolvedValueOnce({ data: 'success' }) @@ -120,7 +120,7 @@ it('should retry on failure', async () => { }) // Advance timer for retry - jest.advanceTimersByTime(1000) + vi.advanceTimersByTime(1000) // Second call succeeds await waitFor(() => { @@ -128,7 +128,7 @@ it('should retry on failure', async () => { expect(screen.getByText('success')).toBeInTheDocument() }) - jest.useRealTimers() + vi.useRealTimers() }) ``` @@ -136,19 +136,19 @@ it('should retry on failure', async () => { ```typescript // Run all pending timers -jest.runAllTimers() +vi.runAllTimers() // Run only pending timers (not new ones created during execution) -jest.runOnlyPendingTimers() +vi.runOnlyPendingTimers() // Advance by specific time -jest.advanceTimersByTime(1000) +vi.advanceTimersByTime(1000) // Get current fake time -jest.now() +Date.now() // Clear all timers -jest.clearAllTimers() +vi.clearAllTimers() ``` ## API Testing Patterns @@ -158,7 +158,7 @@ jest.clearAllTimers() ```typescript describe('DataFetcher', () => { beforeEach(() => { - jest.clearAllMocks() + vi.clearAllMocks() }) it('should show loading state', () => { @@ -241,7 +241,7 @@ it('should submit form and show success', async () => { ```typescript it('should fetch data on mount', async () => { - const fetchData = jest.fn().mockResolvedValue({ data: 'test' }) + const fetchData = vi.fn().mockResolvedValue({ data: 'test' }) render() @@ -255,7 +255,7 @@ it('should fetch data on mount', async () => { ```typescript it('should refetch when id changes', async () => { - const fetchData = jest.fn().mockResolvedValue({ data: 'test' }) + const fetchData = vi.fn().mockResolvedValue({ data: 'test' }) const { rerender } = render() @@ -276,8 +276,8 @@ it('should refetch when id changes', async () => { ```typescript it('should cleanup subscription on unmount', () => { - const subscribe = jest.fn() - const unsubscribe = jest.fn() + const subscribe = vi.fn() + const unsubscribe = vi.fn() subscribe.mockReturnValue(unsubscribe) const { unmount } = render() @@ -332,14 +332,14 @@ expect(description).toBeInTheDocument() ```typescript // Bad - fake timers don't work well with real Promises -jest.useFakeTimers() +vi.useFakeTimers() await waitFor(() => { expect(screen.getByText('Data')).toBeInTheDocument() }) // May timeout! // Good - use runAllTimers or advanceTimersByTime -jest.useFakeTimers() +vi.useFakeTimers() render() -jest.runAllTimers() +vi.runAllTimers() expect(screen.getByText('Data')).toBeInTheDocument() ``` diff --git a/.claude/skills/frontend-testing/CHECKLIST.md b/.claude/skills/frontend-testing/references/checklist.md similarity index 88% rename from .claude/skills/frontend-testing/CHECKLIST.md rename to .claude/skills/frontend-testing/references/checklist.md index b960067264..1ff2b27bbb 100644 --- a/.claude/skills/frontend-testing/CHECKLIST.md +++ b/.claude/skills/frontend-testing/references/checklist.md @@ -74,9 +74,9 @@ Use this checklist when generating or reviewing tests for Dify frontend componen ### Mocks - [ ] **DO NOT mock base components** (`@/app/components/base/*`) -- [ ] `jest.clearAllMocks()` in `beforeEach` (not `afterEach`) +- [ ] `vi.clearAllMocks()` in `beforeEach` (not `afterEach`) - [ ] Shared mock state reset in `beforeEach` -- [ ] i18n uses shared mock (auto-loaded); only override locally for custom translations +- [ ] i18n uses global mock (auto-loaded in `web/vitest.setup.ts`); only override locally for custom translations - [ ] Router mocks match actual Next.js API - [ ] Mocks reflect actual component conditional behavior - [ ] Only mock: API services, complex context providers, third-party libs @@ -114,15 +114,15 @@ For the current file being tested: **Run these checks after EACH test file, not just at the end:** -- [ ] Run `pnpm test -- path/to/file.spec.tsx` - **MUST PASS before next file** +- [ ] Run `pnpm test path/to/file.spec.tsx` - **MUST PASS before next file** - [ ] Fix any failures immediately - [ ] Mark file as complete in todo list - [ ] Only then proceed to next file ### After All Files Complete -- [ ] Run full directory test: `pnpm test -- path/to/directory/` -- [ ] Check coverage report: `pnpm test -- --coverage` +- [ ] Run full directory test: `pnpm test path/to/directory/` +- [ ] Check coverage report: `pnpm test:coverage` - [ ] Run `pnpm lint:fix` on all test files - [ ] Run `pnpm type-check:tsgo` @@ -132,10 +132,10 @@ For the current file being tested: ```typescript // ❌ Mock doesn't match actual behavior -jest.mock('./Component', () => () =>
Mocked
) +vi.mock('./Component', () => () =>
Mocked
) // ✅ Mock matches actual conditional logic -jest.mock('./Component', () => ({ isOpen }: any) => +vi.mock('./Component', () => ({ isOpen }: any) => isOpen ?
Content
: null ) ``` @@ -145,7 +145,7 @@ jest.mock('./Component', () => ({ isOpen }: any) => ```typescript // ❌ Shared state not reset let mockState = false -jest.mock('./useHook', () => () => mockState) +vi.mock('./useHook', () => () => mockState) // ✅ Reset in beforeEach beforeEach(() => { @@ -186,16 +186,16 @@ Always test these scenarios: ```bash # Run specific test -pnpm test -- path/to/file.spec.tsx +pnpm test path/to/file.spec.tsx # Run with coverage -pnpm test -- --coverage path/to/file.spec.tsx +pnpm test:coverage path/to/file.spec.tsx # Watch mode -pnpm test -- --watch path/to/file.spec.tsx +pnpm test:watch path/to/file.spec.tsx # Update snapshots (use sparingly) -pnpm test -- -u path/to/file.spec.tsx +pnpm test -u path/to/file.spec.tsx # Analyze component pnpm analyze-component path/to/component.tsx diff --git a/.claude/skills/frontend-testing/guides/common-patterns.md b/.claude/skills/frontend-testing/references/common-patterns.md similarity index 94% rename from .claude/skills/frontend-testing/guides/common-patterns.md rename to .claude/skills/frontend-testing/references/common-patterns.md index 84a6045b04..6eded5ceba 100644 --- a/.claude/skills/frontend-testing/guides/common-patterns.md +++ b/.claude/skills/frontend-testing/references/common-patterns.md @@ -126,7 +126,7 @@ describe('Counter', () => { describe('ControlledInput', () => { it('should call onChange with new value', async () => { const user = userEvent.setup() - const handleChange = jest.fn() + const handleChange = vi.fn() render() @@ -136,7 +136,7 @@ describe('ControlledInput', () => { }) it('should display controlled value', () => { - render() + render() expect(screen.getByRole('textbox')).toHaveValue('controlled') }) @@ -195,7 +195,7 @@ describe('ItemList', () => { it('should handle item selection', async () => { const user = userEvent.setup() - const onSelect = jest.fn() + const onSelect = vi.fn() render() @@ -217,20 +217,20 @@ describe('ItemList', () => { ```typescript describe('Modal', () => { it('should not render when closed', () => { - render() + render() expect(screen.queryByRole('dialog')).not.toBeInTheDocument() }) it('should render when open', () => { - render() + render() expect(screen.getByRole('dialog')).toBeInTheDocument() }) it('should call onClose when clicking overlay', async () => { const user = userEvent.setup() - const handleClose = jest.fn() + const handleClose = vi.fn() render() @@ -241,7 +241,7 @@ describe('Modal', () => { it('should call onClose when pressing Escape', async () => { const user = userEvent.setup() - const handleClose = jest.fn() + const handleClose = vi.fn() render() @@ -254,7 +254,7 @@ describe('Modal', () => { const user = userEvent.setup() render( - + @@ -279,7 +279,7 @@ describe('Modal', () => { describe('LoginForm', () => { it('should submit valid form', async () => { const user = userEvent.setup() - const onSubmit = jest.fn() + const onSubmit = vi.fn() render() @@ -296,7 +296,7 @@ describe('LoginForm', () => { it('should show validation errors', async () => { const user = userEvent.setup() - render() + render() // Submit empty form await user.click(screen.getByRole('button', { name: /sign in/i })) @@ -308,7 +308,7 @@ describe('LoginForm', () => { it('should validate email format', async () => { const user = userEvent.setup() - render() + render() await user.type(screen.getByLabelText(/email/i), 'invalid-email') await user.click(screen.getByRole('button', { name: /sign in/i })) @@ -318,7 +318,7 @@ describe('LoginForm', () => { it('should disable submit button while submitting', async () => { const user = userEvent.setup() - const onSubmit = jest.fn(() => new Promise(resolve => setTimeout(resolve, 100))) + const onSubmit = vi.fn(() => new Promise(resolve => setTimeout(resolve, 100))) render() @@ -407,7 +407,7 @@ it('test 1', () => { // Good - cleanup is automatic with RTL, but reset mocks beforeEach(() => { - jest.clearAllMocks() + vi.clearAllMocks() }) ``` diff --git a/.claude/skills/frontend-testing/guides/domain-components.md b/.claude/skills/frontend-testing/references/domain-components.md similarity index 95% rename from .claude/skills/frontend-testing/guides/domain-components.md rename to .claude/skills/frontend-testing/references/domain-components.md index ed2cc6eb8a..5535d28f3d 100644 --- a/.claude/skills/frontend-testing/guides/domain-components.md +++ b/.claude/skills/frontend-testing/references/domain-components.md @@ -23,7 +23,7 @@ import NodeConfigPanel from './node-config-panel' import { createMockNode, createMockWorkflowContext } from '@/__mocks__/workflow' // Mock workflow context -jest.mock('@/app/components/workflow/hooks', () => ({ +vi.mock('@/app/components/workflow/hooks', () => ({ useWorkflowStore: () => mockWorkflowStore, useNodesInteractions: () => mockNodesInteractions, })) @@ -31,21 +31,21 @@ jest.mock('@/app/components/workflow/hooks', () => ({ let mockWorkflowStore = { nodes: [], edges: [], - updateNode: jest.fn(), + updateNode: vi.fn(), } let mockNodesInteractions = { - handleNodeSelect: jest.fn(), - handleNodeDelete: jest.fn(), + handleNodeSelect: vi.fn(), + handleNodeDelete: vi.fn(), } describe('NodeConfigPanel', () => { beforeEach(() => { - jest.clearAllMocks() + vi.clearAllMocks() mockWorkflowStore = { nodes: [], edges: [], - updateNode: jest.fn(), + updateNode: vi.fn(), } }) @@ -161,23 +161,23 @@ import { render, screen, fireEvent, waitFor } from '@testing-library/react' import userEvent from '@testing-library/user-event' import DocumentUploader from './document-uploader' -jest.mock('@/service/datasets', () => ({ - uploadDocument: jest.fn(), - parseDocument: jest.fn(), +vi.mock('@/service/datasets', () => ({ + uploadDocument: vi.fn(), + parseDocument: vi.fn(), })) import * as datasetService from '@/service/datasets' -const mockedService = datasetService as jest.Mocked +const mockedService = vi.mocked(datasetService) describe('DocumentUploader', () => { beforeEach(() => { - jest.clearAllMocks() + vi.clearAllMocks() }) describe('File Upload', () => { it('should accept valid file types', async () => { const user = userEvent.setup() - const onUpload = jest.fn() + const onUpload = vi.fn() mockedService.uploadDocument.mockResolvedValue({ id: 'doc-1' }) render() @@ -326,14 +326,14 @@ describe('DocumentList', () => { describe('Search & Filtering', () => { it('should filter by search query', async () => { const user = userEvent.setup() - jest.useFakeTimers() + vi.useFakeTimers() render() await user.type(screen.getByPlaceholderText(/search/i), 'test query') // Debounce - jest.advanceTimersByTime(300) + vi.advanceTimersByTime(300) await waitFor(() => { expect(mockedService.getDocuments).toHaveBeenCalledWith( @@ -342,7 +342,7 @@ describe('DocumentList', () => { ) }) - jest.useRealTimers() + vi.useRealTimers() }) }) }) @@ -367,13 +367,13 @@ import { render, screen, fireEvent, waitFor } from '@testing-library/react' import userEvent from '@testing-library/user-event' import AppConfigForm from './app-config-form' -jest.mock('@/service/apps', () => ({ - updateAppConfig: jest.fn(), - getAppConfig: jest.fn(), +vi.mock('@/service/apps', () => ({ + updateAppConfig: vi.fn(), + getAppConfig: vi.fn(), })) import * as appService from '@/service/apps' -const mockedService = appService as jest.Mocked +const mockedService = vi.mocked(appService) describe('AppConfigForm', () => { const defaultConfig = { @@ -384,7 +384,7 @@ describe('AppConfigForm', () => { } beforeEach(() => { - jest.clearAllMocks() + vi.clearAllMocks() mockedService.getAppConfig.mockResolvedValue(defaultConfig) }) diff --git a/.claude/skills/frontend-testing/guides/mocking.md b/.claude/skills/frontend-testing/references/mocking.md similarity index 85% rename from .claude/skills/frontend-testing/guides/mocking.md rename to .claude/skills/frontend-testing/references/mocking.md index bf0bd79690..23889c8d3d 100644 --- a/.claude/skills/frontend-testing/guides/mocking.md +++ b/.claude/skills/frontend-testing/references/mocking.md @@ -19,8 +19,8 @@ ```typescript // ❌ WRONG: Don't mock base components -jest.mock('@/app/components/base/loading', () => () =>
Loading
) -jest.mock('@/app/components/base/button', () => ({ children }: any) => ) +vi.mock('@/app/components/base/loading', () => () =>
Loading
) +vi.mock('@/app/components/base/button', () => ({ children }: any) => ) // ✅ CORRECT: Import and use real base components import Loading from '@/app/components/base/loading' @@ -41,20 +41,23 @@ Only mock these categories: | Location | Purpose | |----------|---------| -| `web/__mocks__/` | Reusable mocks shared across multiple test files | -| Test file | Test-specific mocks, inline with `jest.mock()` | +| `web/vitest.setup.ts` | Global mocks shared by all tests (for example `react-i18next`, `next/image`) | +| `web/__mocks__/` | Reusable mock factories shared across multiple test files | +| Test file | Test-specific mocks, inline with `vi.mock()` | + +Modules are not mocked automatically. Use `vi.mock` in test files, or add global mocks in `web/vitest.setup.ts`. ## Essential Mocks -### 1. i18n (Auto-loaded via Shared Mock) +### 1. i18n (Auto-loaded via Global Mock) -A shared mock is available at `web/__mocks__/react-i18next.ts` and is auto-loaded by Jest. +A global mock is defined in `web/vitest.setup.ts` and is auto-loaded by Vitest setup. **No explicit mock needed** for most tests - it returns translation keys as-is. For tests requiring custom translations, override the mock: ```typescript -jest.mock('react-i18next', () => ({ +vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => { const translations: Record = { @@ -69,15 +72,15 @@ jest.mock('react-i18next', () => ({ ### 2. Next.js Router ```typescript -const mockPush = jest.fn() -const mockReplace = jest.fn() +const mockPush = vi.fn() +const mockReplace = vi.fn() -jest.mock('next/navigation', () => ({ +vi.mock('next/navigation', () => ({ useRouter: () => ({ push: mockPush, replace: mockReplace, - back: jest.fn(), - prefetch: jest.fn(), + back: vi.fn(), + prefetch: vi.fn(), }), usePathname: () => '/current-path', useSearchParams: () => new URLSearchParams('?key=value'), @@ -85,7 +88,7 @@ jest.mock('next/navigation', () => ({ describe('Component', () => { beforeEach(() => { - jest.clearAllMocks() + vi.clearAllMocks() }) it('should navigate on click', () => { @@ -102,7 +105,7 @@ describe('Component', () => { // ⚠️ Important: Use shared state for components that depend on each other let mockPortalOpenState = false -jest.mock('@/app/components/base/portal-to-follow-elem', () => ({ +vi.mock('@/app/components/base/portal-to-follow-elem', () => ({ PortalToFollowElem: ({ children, open, ...props }: any) => { mockPortalOpenState = open || false // Update shared state return
{children}
@@ -119,7 +122,7 @@ jest.mock('@/app/components/base/portal-to-follow-elem', () => ({ describe('Component', () => { beforeEach(() => { - jest.clearAllMocks() + vi.clearAllMocks() mockPortalOpenState = false // ✅ Reset shared state }) }) @@ -130,13 +133,13 @@ describe('Component', () => { ```typescript import * as api from '@/service/api' -jest.mock('@/service/api') +vi.mock('@/service/api') -const mockedApi = api as jest.Mocked +const mockedApi = vi.mocked(api) describe('Component', () => { beforeEach(() => { - jest.clearAllMocks() + vi.clearAllMocks() // Setup default mock implementation mockedApi.fetchData.mockResolvedValue({ data: [] }) @@ -239,32 +242,9 @@ describe('Component with Context', () => { }) ``` -### 7. SWR / React Query +### 7. React Query ```typescript -// SWR -jest.mock('swr', () => ({ - __esModule: true, - default: jest.fn(), -})) - -import useSWR from 'swr' -const mockedUseSWR = useSWR as jest.Mock - -describe('Component with SWR', () => { - it('should show loading state', () => { - mockedUseSWR.mockReturnValue({ - data: undefined, - error: undefined, - isLoading: true, - }) - - render() - expect(screen.getByText(/loading/i)).toBeInTheDocument() - }) -}) - -// React Query import { QueryClient, QueryClientProvider } from '@tanstack/react-query' const createTestQueryClient = () => new QueryClient({ diff --git a/.claude/skills/frontend-testing/guides/workflow.md b/.claude/skills/frontend-testing/references/workflow.md similarity index 96% rename from .claude/skills/frontend-testing/guides/workflow.md rename to .claude/skills/frontend-testing/references/workflow.md index b0f2994bde..009c3e013b 100644 --- a/.claude/skills/frontend-testing/guides/workflow.md +++ b/.claude/skills/frontend-testing/references/workflow.md @@ -35,7 +35,7 @@ When testing a **single component, hook, or utility**: 2. Run `pnpm analyze-component ` (if available) 3. Check complexity score and features detected 4. Write the test file -5. Run test: `pnpm test -- .spec.tsx` +5. Run test: `pnpm test .spec.tsx` 6. Fix any failures 7. Verify coverage meets goals (100% function, >95% branch) ``` @@ -80,7 +80,7 @@ Process files in this recommended order: ``` ┌─────────────────────────────────────────────┐ │ 1. Write test file │ -│ 2. Run: pnpm test -- .spec.tsx │ +│ 2. Run: pnpm test .spec.tsx │ │ 3. If FAIL → Fix immediately, re-run │ │ 4. If PASS → Mark complete in todo list │ │ 5. ONLY THEN proceed to next file │ @@ -95,10 +95,10 @@ After all individual tests pass: ```bash # Run all tests in the directory together -pnpm test -- path/to/directory/ +pnpm test path/to/directory/ # Check coverage -pnpm test -- --coverage path/to/directory/ +pnpm test:coverage path/to/directory/ ``` ## Component Complexity Guidelines @@ -201,9 +201,9 @@ Run pnpm test ← Multiple failures, hard to debug ``` # GOOD: Incremental with verification Write component-a.spec.tsx -Run pnpm test -- component-a.spec.tsx ✅ +Run pnpm test component-a.spec.tsx ✅ Write component-b.spec.tsx -Run pnpm test -- component-b.spec.tsx ✅ +Run pnpm test component-b.spec.tsx ✅ ...continue... ``` diff --git a/.codex/skills b/.codex/skills new file mode 120000 index 0000000000..454b8427cd --- /dev/null +++ b/.codex/skills @@ -0,0 +1 @@ +../.claude/skills \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ddec42e0ee..3998a69c36 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,6 +6,9 @@ "context": "..", "dockerfile": "Dockerfile" }, + "mounts": [ + "source=dify-dev-tmp,target=/tmp,type=volume" + ], "features": { "ghcr.io/devcontainers/features/node:1": { "nodeGypDependencies": true, @@ -34,19 +37,13 @@ }, "postStartCommand": "./.devcontainer/post_start_command.sh", "postCreateCommand": "./.devcontainer/post_create_command.sh" - // Features to add to the dev container. More info: https://containers.dev/features. // "features": {}, - // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], - // Use 'postCreateCommand' to run commands after the container is created. // "postCreateCommand": "python --version", - // Configure tool-specific properties. // "customizations": {}, - // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. - // "remoteUser": "root" -} +} \ No newline at end of file diff --git a/.devcontainer/post_create_command.sh b/.devcontainer/post_create_command.sh index a26fd076ed..220f77e5ce 100755 --- a/.devcontainer/post_create_command.sh +++ b/.devcontainer/post_create_command.sh @@ -1,12 +1,13 @@ #!/bin/bash WORKSPACE_ROOT=$(pwd) +export COREPACK_ENABLE_DOWNLOAD_PROMPT=0 corepack enable cd web && pnpm install pipx install uv echo "alias start-api=\"cd $WORKSPACE_ROOT/api && uv run python -m flask run --host 0.0.0.0 --port=5001 --debug\"" >> ~/.bashrc -echo "alias start-worker=\"cd $WORKSPACE_ROOT/api && uv run python -m celery -A app.celery worker -P threads -c 1 --loglevel INFO -Q dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor\"" >> ~/.bashrc +echo "alias start-worker=\"cd $WORKSPACE_ROOT/api && uv run python -m celery -A app.celery worker -P threads -c 1 --loglevel INFO -Q dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention\"" >> ~/.bashrc echo "alias start-web=\"cd $WORKSPACE_ROOT/web && pnpm dev\"" >> ~/.bashrc echo "alias start-web-prod=\"cd $WORKSPACE_ROOT/web && pnpm build && pnpm start\"" >> ~/.bashrc echo "alias start-containers=\"cd $WORKSPACE_ROOT/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env up -d\"" >> ~/.bashrc diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d6f326d4dc..106c26bbed 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,229 +6,244 @@ * @crazywoola @laipz8200 @Yeuoly +# CODEOWNERS file +/.github/CODEOWNERS @laipz8200 @crazywoola + +# Docs +/docs/ @crazywoola + # Backend (default owner, more specific rules below will override) -api/ @QuantumGhost +/api/ @QuantumGhost # Backend - MCP -api/core/mcp/ @Nov1c444 -api/core/entities/mcp_provider.py @Nov1c444 -api/services/tools/mcp_tools_manage_service.py @Nov1c444 -api/controllers/mcp/ @Nov1c444 -api/controllers/console/app/mcp_server.py @Nov1c444 -api/tests/**/*mcp* @Nov1c444 +/api/core/mcp/ @Nov1c444 +/api/core/entities/mcp_provider.py @Nov1c444 +/api/services/tools/mcp_tools_manage_service.py @Nov1c444 +/api/controllers/mcp/ @Nov1c444 +/api/controllers/console/app/mcp_server.py @Nov1c444 +/api/tests/**/*mcp* @Nov1c444 # Backend - Workflow - Engine (Core graph execution engine) -api/core/workflow/graph_engine/ @laipz8200 @QuantumGhost -api/core/workflow/runtime/ @laipz8200 @QuantumGhost -api/core/workflow/graph/ @laipz8200 @QuantumGhost -api/core/workflow/graph_events/ @laipz8200 @QuantumGhost -api/core/workflow/node_events/ @laipz8200 @QuantumGhost -api/core/model_runtime/ @laipz8200 @QuantumGhost +/api/core/workflow/graph_engine/ @laipz8200 @QuantumGhost +/api/core/workflow/runtime/ @laipz8200 @QuantumGhost +/api/core/workflow/graph/ @laipz8200 @QuantumGhost +/api/core/workflow/graph_events/ @laipz8200 @QuantumGhost +/api/core/workflow/node_events/ @laipz8200 @QuantumGhost +/api/core/model_runtime/ @laipz8200 @QuantumGhost # Backend - Workflow - Nodes (Agent, Iteration, Loop, LLM) -api/core/workflow/nodes/agent/ @Nov1c444 -api/core/workflow/nodes/iteration/ @Nov1c444 -api/core/workflow/nodes/loop/ @Nov1c444 -api/core/workflow/nodes/llm/ @Nov1c444 +/api/core/workflow/nodes/agent/ @Nov1c444 +/api/core/workflow/nodes/iteration/ @Nov1c444 +/api/core/workflow/nodes/loop/ @Nov1c444 +/api/core/workflow/nodes/llm/ @Nov1c444 # Backend - RAG (Retrieval Augmented Generation) -api/core/rag/ @JohnJyong -api/services/rag_pipeline/ @JohnJyong -api/services/dataset_service.py @JohnJyong -api/services/knowledge_service.py @JohnJyong -api/services/external_knowledge_service.py @JohnJyong -api/services/hit_testing_service.py @JohnJyong -api/services/metadata_service.py @JohnJyong -api/services/vector_service.py @JohnJyong -api/services/entities/knowledge_entities/ @JohnJyong -api/services/entities/external_knowledge_entities/ @JohnJyong -api/controllers/console/datasets/ @JohnJyong -api/controllers/service_api/dataset/ @JohnJyong -api/models/dataset.py @JohnJyong -api/tasks/rag_pipeline/ @JohnJyong -api/tasks/add_document_to_index_task.py @JohnJyong -api/tasks/batch_clean_document_task.py @JohnJyong -api/tasks/clean_document_task.py @JohnJyong -api/tasks/clean_notion_document_task.py @JohnJyong -api/tasks/document_indexing_task.py @JohnJyong -api/tasks/document_indexing_sync_task.py @JohnJyong -api/tasks/document_indexing_update_task.py @JohnJyong -api/tasks/duplicate_document_indexing_task.py @JohnJyong -api/tasks/recover_document_indexing_task.py @JohnJyong -api/tasks/remove_document_from_index_task.py @JohnJyong -api/tasks/retry_document_indexing_task.py @JohnJyong -api/tasks/sync_website_document_indexing_task.py @JohnJyong -api/tasks/batch_create_segment_to_index_task.py @JohnJyong -api/tasks/create_segment_to_index_task.py @JohnJyong -api/tasks/delete_segment_from_index_task.py @JohnJyong -api/tasks/disable_segment_from_index_task.py @JohnJyong -api/tasks/disable_segments_from_index_task.py @JohnJyong -api/tasks/enable_segment_to_index_task.py @JohnJyong -api/tasks/enable_segments_to_index_task.py @JohnJyong -api/tasks/clean_dataset_task.py @JohnJyong -api/tasks/deal_dataset_index_update_task.py @JohnJyong -api/tasks/deal_dataset_vector_index_task.py @JohnJyong +/api/core/rag/ @JohnJyong +/api/services/rag_pipeline/ @JohnJyong +/api/services/dataset_service.py @JohnJyong +/api/services/knowledge_service.py @JohnJyong +/api/services/external_knowledge_service.py @JohnJyong +/api/services/hit_testing_service.py @JohnJyong +/api/services/metadata_service.py @JohnJyong +/api/services/vector_service.py @JohnJyong +/api/services/entities/knowledge_entities/ @JohnJyong +/api/services/entities/external_knowledge_entities/ @JohnJyong +/api/controllers/console/datasets/ @JohnJyong +/api/controllers/service_api/dataset/ @JohnJyong +/api/models/dataset.py @JohnJyong +/api/tasks/rag_pipeline/ @JohnJyong +/api/tasks/add_document_to_index_task.py @JohnJyong +/api/tasks/batch_clean_document_task.py @JohnJyong +/api/tasks/clean_document_task.py @JohnJyong +/api/tasks/clean_notion_document_task.py @JohnJyong +/api/tasks/document_indexing_task.py @JohnJyong +/api/tasks/document_indexing_sync_task.py @JohnJyong +/api/tasks/document_indexing_update_task.py @JohnJyong +/api/tasks/duplicate_document_indexing_task.py @JohnJyong +/api/tasks/recover_document_indexing_task.py @JohnJyong +/api/tasks/remove_document_from_index_task.py @JohnJyong +/api/tasks/retry_document_indexing_task.py @JohnJyong +/api/tasks/sync_website_document_indexing_task.py @JohnJyong +/api/tasks/batch_create_segment_to_index_task.py @JohnJyong +/api/tasks/create_segment_to_index_task.py @JohnJyong +/api/tasks/delete_segment_from_index_task.py @JohnJyong +/api/tasks/disable_segment_from_index_task.py @JohnJyong +/api/tasks/disable_segments_from_index_task.py @JohnJyong +/api/tasks/enable_segment_to_index_task.py @JohnJyong +/api/tasks/enable_segments_to_index_task.py @JohnJyong +/api/tasks/clean_dataset_task.py @JohnJyong +/api/tasks/deal_dataset_index_update_task.py @JohnJyong +/api/tasks/deal_dataset_vector_index_task.py @JohnJyong # Backend - Plugins -api/core/plugin/ @Mairuis @Yeuoly @Stream29 -api/services/plugin/ @Mairuis @Yeuoly @Stream29 -api/controllers/console/workspace/plugin.py @Mairuis @Yeuoly @Stream29 -api/controllers/inner_api/plugin/ @Mairuis @Yeuoly @Stream29 -api/tasks/process_tenant_plugin_autoupgrade_check_task.py @Mairuis @Yeuoly @Stream29 +/api/core/plugin/ @Mairuis @Yeuoly @Stream29 +/api/services/plugin/ @Mairuis @Yeuoly @Stream29 +/api/controllers/console/workspace/plugin.py @Mairuis @Yeuoly @Stream29 +/api/controllers/inner_api/plugin/ @Mairuis @Yeuoly @Stream29 +/api/tasks/process_tenant_plugin_autoupgrade_check_task.py @Mairuis @Yeuoly @Stream29 # Backend - Trigger/Schedule/Webhook -api/controllers/trigger/ @Mairuis @Yeuoly -api/controllers/console/app/workflow_trigger.py @Mairuis @Yeuoly -api/controllers/console/workspace/trigger_providers.py @Mairuis @Yeuoly -api/core/trigger/ @Mairuis @Yeuoly -api/core/app/layers/trigger_post_layer.py @Mairuis @Yeuoly -api/services/trigger/ @Mairuis @Yeuoly -api/models/trigger.py @Mairuis @Yeuoly -api/fields/workflow_trigger_fields.py @Mairuis @Yeuoly -api/repositories/workflow_trigger_log_repository.py @Mairuis @Yeuoly -api/repositories/sqlalchemy_workflow_trigger_log_repository.py @Mairuis @Yeuoly -api/libs/schedule_utils.py @Mairuis @Yeuoly -api/services/workflow/scheduler.py @Mairuis @Yeuoly -api/schedule/trigger_provider_refresh_task.py @Mairuis @Yeuoly -api/schedule/workflow_schedule_task.py @Mairuis @Yeuoly -api/tasks/trigger_processing_tasks.py @Mairuis @Yeuoly -api/tasks/trigger_subscription_refresh_tasks.py @Mairuis @Yeuoly -api/tasks/workflow_schedule_tasks.py @Mairuis @Yeuoly -api/tasks/workflow_cfs_scheduler/ @Mairuis @Yeuoly -api/events/event_handlers/sync_plugin_trigger_when_app_created.py @Mairuis @Yeuoly -api/events/event_handlers/update_app_triggers_when_app_published_workflow_updated.py @Mairuis @Yeuoly -api/events/event_handlers/sync_workflow_schedule_when_app_published.py @Mairuis @Yeuoly -api/events/event_handlers/sync_webhook_when_app_created.py @Mairuis @Yeuoly +/api/controllers/trigger/ @Mairuis @Yeuoly +/api/controllers/console/app/workflow_trigger.py @Mairuis @Yeuoly +/api/controllers/console/workspace/trigger_providers.py @Mairuis @Yeuoly +/api/core/trigger/ @Mairuis @Yeuoly +/api/core/app/layers/trigger_post_layer.py @Mairuis @Yeuoly +/api/services/trigger/ @Mairuis @Yeuoly +/api/models/trigger.py @Mairuis @Yeuoly +/api/fields/workflow_trigger_fields.py @Mairuis @Yeuoly +/api/repositories/workflow_trigger_log_repository.py @Mairuis @Yeuoly +/api/repositories/sqlalchemy_workflow_trigger_log_repository.py @Mairuis @Yeuoly +/api/libs/schedule_utils.py @Mairuis @Yeuoly +/api/services/workflow/scheduler.py @Mairuis @Yeuoly +/api/schedule/trigger_provider_refresh_task.py @Mairuis @Yeuoly +/api/schedule/workflow_schedule_task.py @Mairuis @Yeuoly +/api/tasks/trigger_processing_tasks.py @Mairuis @Yeuoly +/api/tasks/trigger_subscription_refresh_tasks.py @Mairuis @Yeuoly +/api/tasks/workflow_schedule_tasks.py @Mairuis @Yeuoly +/api/tasks/workflow_cfs_scheduler/ @Mairuis @Yeuoly +/api/events/event_handlers/sync_plugin_trigger_when_app_created.py @Mairuis @Yeuoly +/api/events/event_handlers/update_app_triggers_when_app_published_workflow_updated.py @Mairuis @Yeuoly +/api/events/event_handlers/sync_workflow_schedule_when_app_published.py @Mairuis @Yeuoly +/api/events/event_handlers/sync_webhook_when_app_created.py @Mairuis @Yeuoly # Backend - Async Workflow -api/services/async_workflow_service.py @Mairuis @Yeuoly -api/tasks/async_workflow_tasks.py @Mairuis @Yeuoly +/api/services/async_workflow_service.py @Mairuis @Yeuoly +/api/tasks/async_workflow_tasks.py @Mairuis @Yeuoly # Backend - Billing -api/services/billing_service.py @hj24 @zyssyz123 -api/controllers/console/billing/ @hj24 @zyssyz123 +/api/services/billing_service.py @hj24 @zyssyz123 +/api/controllers/console/billing/ @hj24 @zyssyz123 # Backend - Enterprise -api/configs/enterprise/ @GarfieldDai @GareArc -api/services/enterprise/ @GarfieldDai @GareArc -api/services/feature_service.py @GarfieldDai @GareArc -api/controllers/console/feature.py @GarfieldDai @GareArc -api/controllers/web/feature.py @GarfieldDai @GareArc +/api/configs/enterprise/ @GarfieldDai @GareArc +/api/services/enterprise/ @GarfieldDai @GareArc +/api/services/feature_service.py @GarfieldDai @GareArc +/api/controllers/console/feature.py @GarfieldDai @GareArc +/api/controllers/web/feature.py @GarfieldDai @GareArc # Backend - Database Migrations -api/migrations/ @snakevash @laipz8200 +/api/migrations/ @snakevash @laipz8200 @MRZHUH + +# Backend - Vector DB Middleware +/api/configs/middleware/vdb/* @JohnJyong # Frontend -web/ @iamjoel +/web/ @iamjoel + +# Frontend - Web Tests +/.github/workflows/web-tests.yml @iamjoel # Frontend - App - Orchestration -web/app/components/workflow/ @iamjoel @zxhlyh -web/app/components/workflow-app/ @iamjoel @zxhlyh -web/app/components/app/configuration/ @iamjoel @zxhlyh -web/app/components/app/app-publisher/ @iamjoel @zxhlyh +/web/app/components/workflow/ @iamjoel @zxhlyh +/web/app/components/workflow-app/ @iamjoel @zxhlyh +/web/app/components/app/configuration/ @iamjoel @zxhlyh +/web/app/components/app/app-publisher/ @iamjoel @zxhlyh # Frontend - WebApp - Chat -web/app/components/base/chat/ @iamjoel @zxhlyh +/web/app/components/base/chat/ @iamjoel @zxhlyh # Frontend - WebApp - Completion -web/app/components/share/text-generation/ @iamjoel @zxhlyh +/web/app/components/share/text-generation/ @iamjoel @zxhlyh # Frontend - App - List and Creation -web/app/components/apps/ @JzoNgKVO @iamjoel -web/app/components/app/create-app-dialog/ @JzoNgKVO @iamjoel -web/app/components/app/create-app-modal/ @JzoNgKVO @iamjoel -web/app/components/app/create-from-dsl-modal/ @JzoNgKVO @iamjoel +/web/app/components/apps/ @JzoNgKVO @iamjoel +/web/app/components/app/create-app-dialog/ @JzoNgKVO @iamjoel +/web/app/components/app/create-app-modal/ @JzoNgKVO @iamjoel +/web/app/components/app/create-from-dsl-modal/ @JzoNgKVO @iamjoel # Frontend - App - API Documentation -web/app/components/develop/ @JzoNgKVO @iamjoel +/web/app/components/develop/ @JzoNgKVO @iamjoel # Frontend - App - Logs and Annotations -web/app/components/app/workflow-log/ @JzoNgKVO @iamjoel -web/app/components/app/log/ @JzoNgKVO @iamjoel -web/app/components/app/log-annotation/ @JzoNgKVO @iamjoel -web/app/components/app/annotation/ @JzoNgKVO @iamjoel +/web/app/components/app/workflow-log/ @JzoNgKVO @iamjoel +/web/app/components/app/log/ @JzoNgKVO @iamjoel +/web/app/components/app/log-annotation/ @JzoNgKVO @iamjoel +/web/app/components/app/annotation/ @JzoNgKVO @iamjoel # Frontend - App - Monitoring -web/app/(commonLayout)/app/(appDetailLayout)/\[appId\]/overview/ @JzoNgKVO @iamjoel -web/app/components/app/overview/ @JzoNgKVO @iamjoel +/web/app/(commonLayout)/app/(appDetailLayout)/\[appId\]/overview/ @JzoNgKVO @iamjoel +/web/app/components/app/overview/ @JzoNgKVO @iamjoel # Frontend - App - Settings -web/app/components/app-sidebar/ @JzoNgKVO @iamjoel +/web/app/components/app-sidebar/ @JzoNgKVO @iamjoel # Frontend - RAG - Hit Testing -web/app/components/datasets/hit-testing/ @JzoNgKVO @iamjoel +/web/app/components/datasets/hit-testing/ @JzoNgKVO @iamjoel # Frontend - RAG - List and Creation -web/app/components/datasets/list/ @iamjoel @WTW0313 -web/app/components/datasets/create/ @iamjoel @WTW0313 -web/app/components/datasets/create-from-pipeline/ @iamjoel @WTW0313 -web/app/components/datasets/external-knowledge-base/ @iamjoel @WTW0313 +/web/app/components/datasets/list/ @iamjoel @WTW0313 +/web/app/components/datasets/create/ @iamjoel @WTW0313 +/web/app/components/datasets/create-from-pipeline/ @iamjoel @WTW0313 +/web/app/components/datasets/external-knowledge-base/ @iamjoel @WTW0313 # Frontend - RAG - Orchestration (general rule first, specific rules below override) -web/app/components/rag-pipeline/ @iamjoel @WTW0313 -web/app/components/rag-pipeline/components/rag-pipeline-main.tsx @iamjoel @zxhlyh -web/app/components/rag-pipeline/store/ @iamjoel @zxhlyh +/web/app/components/rag-pipeline/ @iamjoel @WTW0313 +/web/app/components/rag-pipeline/components/rag-pipeline-main.tsx @iamjoel @zxhlyh +/web/app/components/rag-pipeline/store/ @iamjoel @zxhlyh # Frontend - RAG - Documents List -web/app/components/datasets/documents/list.tsx @iamjoel @WTW0313 -web/app/components/datasets/documents/create-from-pipeline/ @iamjoel @WTW0313 +/web/app/components/datasets/documents/list.tsx @iamjoel @WTW0313 +/web/app/components/datasets/documents/create-from-pipeline/ @iamjoel @WTW0313 # Frontend - RAG - Segments List -web/app/components/datasets/documents/detail/ @iamjoel @WTW0313 +/web/app/components/datasets/documents/detail/ @iamjoel @WTW0313 # Frontend - RAG - Settings -web/app/components/datasets/settings/ @iamjoel @WTW0313 +/web/app/components/datasets/settings/ @iamjoel @WTW0313 # Frontend - Ecosystem - Plugins -web/app/components/plugins/ @iamjoel @zhsama +/web/app/components/plugins/ @iamjoel @zhsama # Frontend - Ecosystem - Tools -web/app/components/tools/ @iamjoel @Yessenia-d +/web/app/components/tools/ @iamjoel @Yessenia-d # Frontend - Ecosystem - MarketPlace -web/app/components/plugins/marketplace/ @iamjoel @Yessenia-d +/web/app/components/plugins/marketplace/ @iamjoel @Yessenia-d # Frontend - Login and Registration -web/app/signin/ @douxc @iamjoel -web/app/signup/ @douxc @iamjoel -web/app/reset-password/ @douxc @iamjoel -web/app/install/ @douxc @iamjoel -web/app/init/ @douxc @iamjoel -web/app/forgot-password/ @douxc @iamjoel -web/app/account/ @douxc @iamjoel +/web/app/signin/ @douxc @iamjoel +/web/app/signup/ @douxc @iamjoel +/web/app/reset-password/ @douxc @iamjoel +/web/app/install/ @douxc @iamjoel +/web/app/init/ @douxc @iamjoel +/web/app/forgot-password/ @douxc @iamjoel +/web/app/account/ @douxc @iamjoel # Frontend - Service Authentication -web/service/base.ts @douxc @iamjoel +/web/service/base.ts @douxc @iamjoel # Frontend - WebApp Authentication and Access Control -web/app/(shareLayout)/components/ @douxc @iamjoel -web/app/(shareLayout)/webapp-signin/ @douxc @iamjoel -web/app/(shareLayout)/webapp-reset-password/ @douxc @iamjoel -web/app/components/app/app-access-control/ @douxc @iamjoel +/web/app/(shareLayout)/components/ @douxc @iamjoel +/web/app/(shareLayout)/webapp-signin/ @douxc @iamjoel +/web/app/(shareLayout)/webapp-reset-password/ @douxc @iamjoel +/web/app/components/app/app-access-control/ @douxc @iamjoel # Frontend - Explore Page -web/app/components/explore/ @CodingOnStar @iamjoel +/web/app/components/explore/ @CodingOnStar @iamjoel # Frontend - Personal Settings -web/app/components/header/account-setting/ @CodingOnStar @iamjoel -web/app/components/header/account-dropdown/ @CodingOnStar @iamjoel +/web/app/components/header/account-setting/ @CodingOnStar @iamjoel +/web/app/components/header/account-dropdown/ @CodingOnStar @iamjoel # Frontend - Analytics -web/app/components/base/ga/ @CodingOnStar @iamjoel +/web/app/components/base/ga/ @CodingOnStar @iamjoel # Frontend - Base Components -web/app/components/base/ @iamjoel @zxhlyh +/web/app/components/base/ @iamjoel @zxhlyh # Frontend - Utils and Hooks -web/utils/classnames.ts @iamjoel @zxhlyh -web/utils/time.ts @iamjoel @zxhlyh -web/utils/format.ts @iamjoel @zxhlyh -web/utils/clipboard.ts @iamjoel @zxhlyh -web/hooks/use-document-title.ts @iamjoel @zxhlyh +/web/utils/classnames.ts @iamjoel @zxhlyh +/web/utils/time.ts @iamjoel @zxhlyh +/web/utils/format.ts @iamjoel @zxhlyh +/web/utils/clipboard.ts @iamjoel @zxhlyh +/web/hooks/use-document-title.ts @iamjoel @zxhlyh # Frontend - Billing and Education -web/app/components/billing/ @iamjoel @zxhlyh -web/app/education-apply/ @iamjoel @zxhlyh +/web/app/components/billing/ @iamjoel @zxhlyh +/web/app/education-apply/ @iamjoel @zxhlyh # Frontend - Workspace -web/app/components/header/account-dropdown/workplace-selector/ @iamjoel @zxhlyh +/web/app/components/header/account-dropdown/workplace-selector/ @iamjoel @zxhlyh + +# Docker +/docker/* @laipz8200 diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 76cbf64fca..152a9caee8 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -22,12 +22,12 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: persist-credentials: false - name: Setup UV and Python - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: enable-cache: true python-version: ${{ matrix.python-version }} @@ -57,7 +57,7 @@ jobs: run: sh .github/workflows/expose_service_ports.sh - name: Set up Sandbox - uses: hoverkraft-tech/compose-action@v2.0.2 + uses: hoverkraft-tech/compose-action@v2 with: compose-file: | docker/docker-compose.middleware.yaml diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index d7a58ce93d..5413f83c27 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -12,12 +12,28 @@ jobs: if: github.repository == 'langgenius/dify' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + + - name: Check Docker Compose inputs + id: docker-compose-changes + uses: tj-actions/changed-files@v46 + with: + files: | + docker/generate_docker_compose + docker/.env.example + docker/docker-compose-template.yaml + docker/docker-compose.yaml - uses: actions/setup-python@v5 with: python-version: "3.11" - - uses: astral-sh/setup-uv@v6 + - uses: astral-sh/setup-uv@v7 + + - name: Generate Docker Compose + if: steps.docker-compose-changes.outputs.any_changed == 'true' + run: | + cd docker + ./generate_docker_compose - run: | cd api @@ -66,27 +82,6 @@ jobs: # mdformat breaks YAML front matter in markdown files. Add --exclude for directories containing YAML front matter. - name: mdformat run: | - uvx --python 3.13 mdformat . --exclude ".claude/skills/**" - - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - package_json_file: web/package.json - run_install: false - - - name: Setup NodeJS - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: pnpm - cache-dependency-path: ./web/package.json - - - name: Web dependencies - working-directory: ./web - run: pnpm install --frozen-lockfile - - - name: oxlint - working-directory: ./web - run: pnpm exec oxlint --config .oxlintrc.json --fix . + uvx --python 3.13 mdformat . --exclude ".claude/skills/**/SKILL.md" - uses: autofix-ci/action@635ffb0c9798bd160680f18fd73371e355b85f27 diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index f7f464a601..bbf89236de 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -90,7 +90,7 @@ jobs: touch "/tmp/digests/${sanitized_digest}" - name: Upload digest - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }} path: /tmp/digests/* diff --git a/.github/workflows/db-migration-test.yml b/.github/workflows/db-migration-test.yml index 101d973466..e20cf9850b 100644 --- a/.github/workflows/db-migration-test.yml +++ b/.github/workflows/db-migration-test.yml @@ -13,13 +13,13 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 persist-credentials: false - name: Setup UV and Python - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: enable-cache: true python-version: "3.12" @@ -63,13 +63,13 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 persist-credentials: false - name: Setup UV and Python - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: enable-cache: true python-version: "3.12" diff --git a/.github/workflows/main-ci.yml b/.github/workflows/main-ci.yml index 876ec23a3d..d6653de950 100644 --- a/.github/workflows/main-ci.yml +++ b/.github/workflows/main-ci.yml @@ -27,7 +27,7 @@ jobs: vdb-changed: ${{ steps.changes.outputs.vdb }} migration-changed: ${{ steps.changes.outputs.migration }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dorny/paths-filter@v3 id: changes with: @@ -38,6 +38,7 @@ jobs: - '.github/workflows/api-tests.yml' web: - 'web/**' + - '.github/workflows/web-tests.yml' vdb: - 'api/core/rag/datasource/**' - 'docker/**' diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 5a8a34be79..d463349686 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -19,13 +19,13 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: persist-credentials: false - name: Check changed files id: changed-files - uses: tj-actions/changed-files@v46 + uses: tj-actions/changed-files@v47 with: files: | api/** @@ -33,7 +33,7 @@ jobs: - name: Setup UV and Python if: steps.changed-files.outputs.any_changed == 'true' - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: enable-cache: false python-version: "3.12" @@ -68,15 +68,17 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: persist-credentials: false - name: Check changed files id: changed-files - uses: tj-actions/changed-files@v46 + uses: tj-actions/changed-files@v47 with: - files: web/** + files: | + web/** + .github/workflows/style.yml - name: Install pnpm uses: pnpm/action-setup@v4 @@ -85,12 +87,12 @@ jobs: run_install: false - name: Setup NodeJS - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 if: steps.changed-files.outputs.any_changed == 'true' with: node-version: 22 cache: pnpm - cache-dependency-path: ./web/package.json + cache-dependency-path: ./web/pnpm-lock.yaml - name: Web dependencies if: steps.changed-files.outputs.any_changed == 'true' @@ -108,50 +110,20 @@ jobs: working-directory: ./web run: pnpm run type-check:tsgo - docker-compose-template: - name: Docker Compose Template - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Check changed files - id: changed-files - uses: tj-actions/changed-files@v46 - with: - files: | - docker/generate_docker_compose - docker/.env.example - docker/docker-compose-template.yaml - docker/docker-compose.yaml - - - name: Generate Docker Compose - if: steps.changed-files.outputs.any_changed == 'true' - run: | - cd docker - ./generate_docker_compose - - - name: Check for changes - if: steps.changed-files.outputs.any_changed == 'true' - run: git diff --exit-code - superlinter: name: SuperLinter runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 persist-credentials: false - name: Check changed files id: changed-files - uses: tj-actions/changed-files@v46 + uses: tj-actions/changed-files@v47 with: files: | **.sh diff --git a/.github/workflows/tool-test-sdks.yaml b/.github/workflows/tool-test-sdks.yaml index b1ccd7417a..0259ef2232 100644 --- a/.github/workflows/tool-test-sdks.yaml +++ b/.github/workflows/tool-test-sdks.yaml @@ -25,12 +25,12 @@ jobs: working-directory: sdks/nodejs-client steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: ${{ matrix.node-version }} cache: '' diff --git a/.github/workflows/translate-i18n-base-on-english.yml b/.github/workflows/translate-i18n-base-on-english.yml index fe8e2ebc2b..a51350f630 100644 --- a/.github/workflows/translate-i18n-base-on-english.yml +++ b/.github/workflows/translate-i18n-base-on-english.yml @@ -1,10 +1,10 @@ -name: Check i18n Files and Create PR +name: Translate i18n Files Based on English on: push: branches: [main] paths: - - 'web/i18n/en-US/*.ts' + - 'web/i18n/en-US/*.json' permissions: contents: write @@ -18,7 +18,7 @@ jobs: run: working-directory: web steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} @@ -28,13 +28,13 @@ jobs: run: | git fetch origin "${{ github.event.before }}" || true git fetch origin "${{ github.sha }}" || true - changed_files=$(git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" -- 'i18n/en-US/*.ts') + changed_files=$(git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" -- 'i18n/en-US/*.json') echo "Changed files: $changed_files" if [ -n "$changed_files" ]; then echo "FILES_CHANGED=true" >> $GITHUB_ENV file_args="" for file in $changed_files; do - filename=$(basename "$file" .ts) + filename=$(basename "$file" .json) file_args="$file_args --file $filename" done echo "FILE_ARGS=$file_args" >> $GITHUB_ENV @@ -51,11 +51,11 @@ jobs: - name: Set up Node.js if: env.FILES_CHANGED == 'true' - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: 'lts/*' cache: pnpm - cache-dependency-path: ./web/package.json + cache-dependency-path: ./web/pnpm-lock.yaml - name: Install dependencies if: env.FILES_CHANGED == 'true' @@ -65,12 +65,7 @@ jobs: - name: Generate i18n translations if: env.FILES_CHANGED == 'true' working-directory: ./web - run: pnpm run auto-gen-i18n ${{ env.FILE_ARGS }} - - - name: Generate i18n type definitions - if: env.FILES_CHANGED == 'true' - working-directory: ./web - run: pnpm run gen:i18n-types + run: pnpm run i18n:gen ${{ env.FILE_ARGS }} - name: Create Pull Request if: env.FILES_CHANGED == 'true' @@ -78,14 +73,13 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: 'chore(i18n): update translations based on en-US changes' - title: 'chore(i18n): translate i18n files and update type definitions' + title: 'chore(i18n): translate i18n files based on en-US changes' body: | - This PR was automatically created to update i18n files and TypeScript type definitions based on changes in en-US locale. + This PR was automatically created to update i18n translation files based on changes in en-US locale. **Triggered by:** ${{ github.sha }} **Changes included:** - Updated translation files for all locales - - Regenerated TypeScript type definitions for type safety branch: chore/automated-i18n-updates-${{ github.sha }} delete-branch: true diff --git a/.github/workflows/vdb-tests.yml b/.github/workflows/vdb-tests.yml index 291171e5c7..7735afdaca 100644 --- a/.github/workflows/vdb-tests.yml +++ b/.github/workflows/vdb-tests.yml @@ -19,19 +19,19 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: persist-credentials: false - name: Free Disk Space - uses: endersonmenezes/free-disk-space@v2 + uses: endersonmenezes/free-disk-space@v3 with: remove_dotnet: true remove_haskell: true remove_tool_cache: true - name: Setup UV and Python - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: enable-cache: true python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/web-tests.yml b/.github/workflows/web-tests.yml index 3313e58614..0fd1d5d22b 100644 --- a/.github/workflows/web-tests.yml +++ b/.github/workflows/web-tests.yml @@ -13,46 +13,356 @@ jobs: runs-on: ubuntu-latest defaults: run: + shell: bash working-directory: ./web steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: persist-credentials: false - - name: Check changed files - id: changed-files - uses: tj-actions/changed-files@v46 - with: - files: web/** - - name: Install pnpm - if: steps.changed-files.outputs.any_changed == 'true' uses: pnpm/action-setup@v4 with: package_json_file: web/package.json run_install: false - name: Setup Node.js - uses: actions/setup-node@v4 - if: steps.changed-files.outputs.any_changed == 'true' + uses: actions/setup-node@v6 with: node-version: 22 cache: pnpm - cache-dependency-path: ./web/package.json + cache-dependency-path: ./web/pnpm-lock.yaml - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' - working-directory: ./web run: pnpm install --frozen-lockfile - - name: Check i18n types synchronization - if: steps.changed-files.outputs.any_changed == 'true' - working-directory: ./web - run: pnpm run check:i18n-types - - name: Run tests - if: steps.changed-files.outputs.any_changed == 'true' - working-directory: ./web - run: pnpm test + run: pnpm test:coverage + + - name: Coverage Summary + if: always() + id: coverage-summary + run: | + set -eo pipefail + + COVERAGE_FILE="coverage/coverage-final.json" + COVERAGE_SUMMARY_FILE="coverage/coverage-summary.json" + + if [ ! -f "$COVERAGE_FILE" ] && [ ! -f "$COVERAGE_SUMMARY_FILE" ]; then + echo "has_coverage=false" >> "$GITHUB_OUTPUT" + echo "### 🚨 Test Coverage Report :test_tube:" >> "$GITHUB_STEP_SUMMARY" + echo "Coverage data not found. Ensure Vitest runs with coverage enabled." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + echo "has_coverage=true" >> "$GITHUB_OUTPUT" + + node <<'NODE' >> "$GITHUB_STEP_SUMMARY" + const fs = require('fs'); + const path = require('path'); + let libCoverage = null; + + try { + libCoverage = require('istanbul-lib-coverage'); + } catch (error) { + libCoverage = null; + } + + const summaryPath = path.join('coverage', 'coverage-summary.json'); + const finalPath = path.join('coverage', 'coverage-final.json'); + + const hasSummary = fs.existsSync(summaryPath); + const hasFinal = fs.existsSync(finalPath); + + if (!hasSummary && !hasFinal) { + console.log('### Test Coverage Summary :test_tube:'); + console.log(''); + console.log('No coverage data found.'); + process.exit(0); + } + + const summary = hasSummary + ? JSON.parse(fs.readFileSync(summaryPath, 'utf8')) + : null; + const coverage = hasFinal + ? JSON.parse(fs.readFileSync(finalPath, 'utf8')) + : null; + + const getLineCoverageFromStatements = (statementMap, statementHits) => { + const lineHits = {}; + + if (!statementMap || !statementHits) { + return lineHits; + } + + Object.entries(statementMap).forEach(([key, statement]) => { + const line = statement?.start?.line; + if (!line) { + return; + } + const hits = statementHits[key] ?? 0; + const previous = lineHits[line]; + lineHits[line] = previous === undefined ? hits : Math.max(previous, hits); + }); + + return lineHits; + }; + + const getFileCoverage = (entry) => ( + libCoverage ? libCoverage.createFileCoverage(entry) : null + ); + + const getLineHits = (entry, fileCoverage) => { + const lineHits = entry.l ?? {}; + if (Object.keys(lineHits).length > 0) { + return lineHits; + } + if (fileCoverage) { + return fileCoverage.getLineCoverage(); + } + return getLineCoverageFromStatements(entry.statementMap ?? {}, entry.s ?? {}); + }; + + const getUncoveredLines = (entry, fileCoverage, lineHits) => { + if (lineHits && Object.keys(lineHits).length > 0) { + return Object.entries(lineHits) + .filter(([, count]) => count === 0) + .map(([line]) => Number(line)) + .sort((a, b) => a - b); + } + if (fileCoverage) { + return fileCoverage.getUncoveredLines(); + } + return []; + }; + + const totals = { + lines: { covered: 0, total: 0 }, + statements: { covered: 0, total: 0 }, + branches: { covered: 0, total: 0 }, + functions: { covered: 0, total: 0 }, + }; + const fileSummaries = []; + + if (summary) { + const totalEntry = summary.total ?? {}; + ['lines', 'statements', 'branches', 'functions'].forEach((key) => { + if (totalEntry[key]) { + totals[key].covered = totalEntry[key].covered ?? 0; + totals[key].total = totalEntry[key].total ?? 0; + } + }); + + Object.entries(summary) + .filter(([file]) => file !== 'total') + .forEach(([file, data]) => { + fileSummaries.push({ + file, + pct: data.lines?.pct ?? data.statements?.pct ?? 0, + lines: { + covered: data.lines?.covered ?? 0, + total: data.lines?.total ?? 0, + }, + }); + }); + } else if (coverage) { + Object.entries(coverage).forEach(([file, entry]) => { + const fileCoverage = getFileCoverage(entry); + const lineHits = getLineHits(entry, fileCoverage); + const statementHits = entry.s ?? {}; + const branchHits = entry.b ?? {}; + const functionHits = entry.f ?? {}; + + const lineTotal = Object.keys(lineHits).length; + const lineCovered = Object.values(lineHits).filter((n) => n > 0).length; + + const statementTotal = Object.keys(statementHits).length; + const statementCovered = Object.values(statementHits).filter((n) => n > 0).length; + + const branchTotal = Object.values(branchHits).reduce((acc, branches) => acc + branches.length, 0); + const branchCovered = Object.values(branchHits).reduce( + (acc, branches) => acc + branches.filter((n) => n > 0).length, + 0, + ); + + const functionTotal = Object.keys(functionHits).length; + const functionCovered = Object.values(functionHits).filter((n) => n > 0).length; + + totals.lines.total += lineTotal; + totals.lines.covered += lineCovered; + totals.statements.total += statementTotal; + totals.statements.covered += statementCovered; + totals.branches.total += branchTotal; + totals.branches.covered += branchCovered; + totals.functions.total += functionTotal; + totals.functions.covered += functionCovered; + + const pct = (covered, tot) => (tot > 0 ? (covered / tot) * 100 : 0); + + fileSummaries.push({ + file, + pct: pct(lineCovered || statementCovered, lineTotal || statementTotal), + lines: { + covered: lineCovered || statementCovered, + total: lineTotal || statementTotal, + }, + }); + }); + } + + const pct = (covered, tot) => (tot > 0 ? ((covered / tot) * 100).toFixed(2) : '0.00'); + + console.log('### Test Coverage Summary :test_tube:'); + console.log(''); + console.log('| Metric | Coverage | Covered / Total |'); + console.log('|--------|----------|-----------------|'); + console.log(`| Lines | ${pct(totals.lines.covered, totals.lines.total)}% | ${totals.lines.covered} / ${totals.lines.total} |`); + console.log(`| Statements | ${pct(totals.statements.covered, totals.statements.total)}% | ${totals.statements.covered} / ${totals.statements.total} |`); + console.log(`| Branches | ${pct(totals.branches.covered, totals.branches.total)}% | ${totals.branches.covered} / ${totals.branches.total} |`); + console.log(`| Functions | ${pct(totals.functions.covered, totals.functions.total)}% | ${totals.functions.covered} / ${totals.functions.total} |`); + + console.log(''); + console.log('
File coverage (lowest lines first)'); + console.log(''); + console.log('```'); + fileSummaries + .sort((a, b) => (a.pct - b.pct) || (b.lines.total - a.lines.total)) + .slice(0, 25) + .forEach(({ file, pct, lines }) => { + console.log(`${pct.toFixed(2)}%\t${lines.covered}/${lines.total}\t${file}`); + }); + console.log('```'); + console.log('
'); + + if (coverage) { + const pctValue = (covered, tot) => { + if (tot === 0) { + return '0'; + } + return ((covered / tot) * 100) + .toFixed(2) + .replace(/\.?0+$/, ''); + }; + + const formatLineRanges = (lines) => { + if (lines.length === 0) { + return ''; + } + const ranges = []; + let start = lines[0]; + let end = lines[0]; + + for (let i = 1; i < lines.length; i += 1) { + const current = lines[i]; + if (current === end + 1) { + end = current; + continue; + } + ranges.push(start === end ? `${start}` : `${start}-${end}`); + start = current; + end = current; + } + ranges.push(start === end ? `${start}` : `${start}-${end}`); + return ranges.join(','); + }; + + const tableTotals = { + statements: { covered: 0, total: 0 }, + branches: { covered: 0, total: 0 }, + functions: { covered: 0, total: 0 }, + lines: { covered: 0, total: 0 }, + }; + const tableRows = Object.entries(coverage) + .map(([file, entry]) => { + const fileCoverage = getFileCoverage(entry); + const lineHits = getLineHits(entry, fileCoverage); + const statementHits = entry.s ?? {}; + const branchHits = entry.b ?? {}; + const functionHits = entry.f ?? {}; + + const lineTotal = Object.keys(lineHits).length; + const lineCovered = Object.values(lineHits).filter((n) => n > 0).length; + const statementTotal = Object.keys(statementHits).length; + const statementCovered = Object.values(statementHits).filter((n) => n > 0).length; + const branchTotal = Object.values(branchHits).reduce((acc, branches) => acc + branches.length, 0); + const branchCovered = Object.values(branchHits).reduce( + (acc, branches) => acc + branches.filter((n) => n > 0).length, + 0, + ); + const functionTotal = Object.keys(functionHits).length; + const functionCovered = Object.values(functionHits).filter((n) => n > 0).length; + + tableTotals.lines.total += lineTotal; + tableTotals.lines.covered += lineCovered; + tableTotals.statements.total += statementTotal; + tableTotals.statements.covered += statementCovered; + tableTotals.branches.total += branchTotal; + tableTotals.branches.covered += branchCovered; + tableTotals.functions.total += functionTotal; + tableTotals.functions.covered += functionCovered; + + const uncoveredLines = getUncoveredLines(entry, fileCoverage, lineHits); + + const filePath = entry.path ?? file; + const relativePath = path.isAbsolute(filePath) + ? path.relative(process.cwd(), filePath) + : filePath; + + return { + file: relativePath || file, + statements: pctValue(statementCovered, statementTotal), + branches: pctValue(branchCovered, branchTotal), + functions: pctValue(functionCovered, functionTotal), + lines: pctValue(lineCovered, lineTotal), + uncovered: formatLineRanges(uncoveredLines), + }; + }) + .sort((a, b) => a.file.localeCompare(b.file)); + + const columns = [ + { key: 'file', header: 'File', align: 'left' }, + { key: 'statements', header: '% Stmts', align: 'right' }, + { key: 'branches', header: '% Branch', align: 'right' }, + { key: 'functions', header: '% Funcs', align: 'right' }, + { key: 'lines', header: '% Lines', align: 'right' }, + { key: 'uncovered', header: 'Uncovered Line #s', align: 'left' }, + ]; + + const allFilesRow = { + file: 'All files', + statements: pctValue(tableTotals.statements.covered, tableTotals.statements.total), + branches: pctValue(tableTotals.branches.covered, tableTotals.branches.total), + functions: pctValue(tableTotals.functions.covered, tableTotals.functions.total), + lines: pctValue(tableTotals.lines.covered, tableTotals.lines.total), + uncovered: '', + }; + + const rowsForOutput = [allFilesRow, ...tableRows]; + const formatRow = (row) => `| ${columns + .map(({ key }) => String(row[key] ?? '')) + .join(' | ')} |`; + const headerRow = `| ${columns.map(({ header }) => header).join(' | ')} |`; + const dividerRow = `| ${columns + .map(({ align }) => (align === 'right' ? '---:' : ':---')) + .join(' | ')} |`; + + console.log(''); + console.log('
Vitest coverage table'); + console.log(''); + console.log(headerRow); + console.log(dividerRow); + rowsForOutput.forEach((row) => console.log(formatRow(row))); + console.log('
'); + } + NODE + + - name: Upload Coverage Artifact + if: steps.coverage-summary.outputs.has_coverage == 'true' + uses: actions/upload-artifact@v6 + with: + name: web-coverage-report + path: web/coverage + retention-days: 30 + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 5ad728c3da..17a2bd5b7b 100644 --- a/.gitignore +++ b/.gitignore @@ -139,7 +139,6 @@ pyrightconfig.json .idea/' .DS_Store -web/.vscode/settings.json # Intellij IDEA Files .idea/* @@ -196,6 +195,7 @@ docker/nginx/ssl/* !docker/nginx/ssl/.gitkeep docker/middleware.env docker/docker-compose.override.yaml +docker/env-backup/* sdks/python-client/build sdks/python-client/dist @@ -205,7 +205,6 @@ sdks/python-client/dify_client.egg-info !.vscode/launch.json.template !.vscode/README.md api/.vscode -web/.vscode # vscode Code History Extension .history @@ -220,15 +219,6 @@ plugins.jsonl # mise mise.toml -# Next.js build output -.next/ - -# PWA generated files -web/public/sw.js -web/public/sw.js.map -web/public/workbox-*.js -web/public/workbox-*.js.map -web/public/fallback-*.js # AI Assistant .roo/ diff --git a/.mcp.json b/.mcp.json deleted file mode 100644 index 8eceaf9ead..0000000000 --- a/.mcp.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "mcpServers": { - "context7": { - "type": "http", - "url": "https://mcp.context7.com/mcp" - }, - "sequential-thinking": { - "type": "stdio", - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-sequential-thinking"], - "env": {} - }, - "github": { - "type": "stdio", - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-github"], - "env": { - "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_PERSONAL_ACCESS_TOKEN}" - } - }, - "fetch": { - "type": "stdio", - "command": "uvx", - "args": ["mcp-server-fetch"], - "env": {} - }, - "playwright": { - "type": "stdio", - "command": "npx", - "args": ["-y", "@playwright/mcp@latest"], - "env": {} - } - } - } \ No newline at end of file diff --git a/.vscode/launch.json.template b/.vscode/launch.json.template index cb934d01b5..bdded1e73e 100644 --- a/.vscode/launch.json.template +++ b/.vscode/launch.json.template @@ -37,7 +37,7 @@ "-c", "1", "-Q", - "dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor", + "dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention", "--loglevel", "INFO" ], diff --git a/api/.env.example b/api/.env.example index 43fe76bb11..99cd2ba558 100644 --- a/api/.env.example +++ b/api/.env.example @@ -116,6 +116,7 @@ ALIYUN_OSS_AUTH_VERSION=v1 ALIYUN_OSS_REGION=your-region # Don't start with '/'. OSS doesn't support leading slash in object names. ALIYUN_OSS_PATH=your-path +ALIYUN_CLOUDBOX_ID=your-cloudbox-id # Google Storage configuration GOOGLE_STORAGE_BUCKET_NAME=your-bucket-name @@ -127,12 +128,14 @@ TENCENT_COS_SECRET_KEY=your-secret-key TENCENT_COS_SECRET_ID=your-secret-id TENCENT_COS_REGION=your-region TENCENT_COS_SCHEME=your-scheme +TENCENT_COS_CUSTOM_DOMAIN=your-custom-domain # Huawei OBS Storage Configuration HUAWEI_OBS_BUCKET_NAME=your-bucket-name HUAWEI_OBS_SECRET_KEY=your-secret-key HUAWEI_OBS_ACCESS_KEY=your-access-key HUAWEI_OBS_SERVER=your-server-url +HUAWEI_OBS_PATH_STYLE=false # Baidu OBS Storage Configuration BAIDU_OBS_BUCKET_NAME=your-bucket-name @@ -690,3 +693,7 @@ ANNOTATION_IMPORT_RATE_LIMIT_PER_MINUTE=5 ANNOTATION_IMPORT_RATE_LIMIT_PER_HOUR=20 # Maximum number of concurrent annotation import tasks per tenant ANNOTATION_IMPORT_MAX_CONCURRENT=5 +# Sandbox expired records clean configuration +SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD=21 +SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE=1000 +SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS=30 diff --git a/api/README.md b/api/README.md index 2dab2ec6e6..794b05d3af 100644 --- a/api/README.md +++ b/api/README.md @@ -84,7 +84,7 @@ 1. If you need to handle and debug the async tasks (e.g. dataset importing and documents indexing), please start the worker service. ```bash -uv run celery -A app.celery worker -P threads -c 2 --loglevel INFO -Q dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor +uv run celery -A app.celery worker -P threads -c 2 --loglevel INFO -Q dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention ``` Additionally, if you want to debug the celery scheduled tasks, you can run the following command in another terminal to start the beat service: diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index e16ca52f46..43dddbd011 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -218,7 +218,7 @@ class PluginConfig(BaseSettings): PLUGIN_DAEMON_TIMEOUT: PositiveFloat | None = Field( description="Timeout in seconds for requests to the plugin daemon (set to None to disable)", - default=300.0, + default=600.0, ) INNER_API_KEY_FOR_PLUGIN: str = Field(description="Inner api key for plugin", default="inner-api-key") @@ -1270,6 +1270,21 @@ class TenantIsolatedTaskQueueConfig(BaseSettings): ) +class SandboxExpiredRecordsCleanConfig(BaseSettings): + SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD: NonNegativeInt = Field( + description="Graceful period in days for sandbox records clean after subscription expiration", + default=21, + ) + SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE: PositiveInt = Field( + description="Maximum number of records to process in each batch", + default=1000, + ) + SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS: PositiveInt = Field( + description="Retention days for sandbox expired workflow_run records and message records", + default=30, + ) + + class FeatureConfig( # place the configs in alphabet order AppExecutionConfig, @@ -1295,6 +1310,7 @@ class FeatureConfig( PositionConfig, RagEtlConfig, RepositoryConfig, + SandboxExpiredRecordsCleanConfig, SecurityConfig, TenantIsolatedTaskQueueConfig, ToolConfig, diff --git a/api/configs/middleware/storage/aliyun_oss_storage_config.py b/api/configs/middleware/storage/aliyun_oss_storage_config.py index 331c486d54..6df14175ae 100644 --- a/api/configs/middleware/storage/aliyun_oss_storage_config.py +++ b/api/configs/middleware/storage/aliyun_oss_storage_config.py @@ -41,3 +41,8 @@ class AliyunOSSStorageConfig(BaseSettings): description="Base path within the bucket to store objects (e.g., 'my-app-data/')", default=None, ) + + ALIYUN_CLOUDBOX_ID: str | None = Field( + description="Cloudbox id for aliyun cloudbox service", + default=None, + ) diff --git a/api/configs/middleware/storage/huawei_obs_storage_config.py b/api/configs/middleware/storage/huawei_obs_storage_config.py index 5b5cd2f750..46b6f2e68d 100644 --- a/api/configs/middleware/storage/huawei_obs_storage_config.py +++ b/api/configs/middleware/storage/huawei_obs_storage_config.py @@ -26,3 +26,8 @@ class HuaweiCloudOBSStorageConfig(BaseSettings): description="Endpoint URL for Huawei Cloud OBS (e.g., 'https://obs.cn-north-4.myhuaweicloud.com')", default=None, ) + + HUAWEI_OBS_PATH_STYLE: bool = Field( + description="Flag to indicate whether to use path-style URLs for OBS requests", + default=False, + ) diff --git a/api/configs/middleware/storage/tencent_cos_storage_config.py b/api/configs/middleware/storage/tencent_cos_storage_config.py index e297e748e9..cdd10740f8 100644 --- a/api/configs/middleware/storage/tencent_cos_storage_config.py +++ b/api/configs/middleware/storage/tencent_cos_storage_config.py @@ -31,3 +31,8 @@ class TencentCloudCOSStorageConfig(BaseSettings): description="Protocol scheme for COS requests: 'https' (recommended) or 'http'", default=None, ) + + TENCENT_COS_CUSTOM_DOMAIN: str | None = Field( + description="Tencent Cloud COS custom domain setting", + default=None, + ) diff --git a/api/controllers/common/file_response.py b/api/controllers/common/file_response.py new file mode 100644 index 0000000000..ca8ea3d52e --- /dev/null +++ b/api/controllers/common/file_response.py @@ -0,0 +1,57 @@ +import os +from email.message import Message +from urllib.parse import quote + +from flask import Response + +HTML_MIME_TYPES = frozenset({"text/html", "application/xhtml+xml"}) +HTML_EXTENSIONS = frozenset({"html", "htm"}) + + +def _normalize_mime_type(mime_type: str | None) -> str: + if not mime_type: + return "" + message = Message() + message["Content-Type"] = mime_type + return message.get_content_type().strip().lower() + + +def _is_html_extension(extension: str | None) -> bool: + if not extension: + return False + return extension.lstrip(".").lower() in HTML_EXTENSIONS + + +def is_html_content(mime_type: str | None, filename: str | None, extension: str | None = None) -> bool: + normalized_mime_type = _normalize_mime_type(mime_type) + if normalized_mime_type in HTML_MIME_TYPES: + return True + + if _is_html_extension(extension): + return True + + if filename: + return _is_html_extension(os.path.splitext(filename)[1]) + + return False + + +def enforce_download_for_html( + response: Response, + *, + mime_type: str | None, + filename: str | None, + extension: str | None = None, +) -> bool: + if not is_html_content(mime_type, filename, extension): + return False + + if filename: + encoded_filename = quote(filename) + response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}" + else: + response.headers["Content-Disposition"] = "attachment" + + response.headers["Content-Type"] = "application/octet-stream" + response.headers["X-Content-Type-Options"] = "nosniff" + return True diff --git a/api/controllers/console/auth/activate.py b/api/controllers/console/auth/activate.py index 6834656a7f..fe70d930fb 100644 --- a/api/controllers/console/auth/activate.py +++ b/api/controllers/console/auth/activate.py @@ -7,9 +7,9 @@ from controllers.console import console_ns from controllers.console.error import AlreadyActivateError from extensions.ext_database import db from libs.datetime_utils import naive_utc_now -from libs.helper import EmailStr, extract_remote_ip, timezone +from libs.helper import EmailStr, timezone from models import AccountStatus -from services.account_service import AccountService, RegisterService +from services.account_service import RegisterService DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" @@ -93,7 +93,6 @@ class ActivateApi(Resource): "ActivationResponse", { "result": fields.String(description="Operation result"), - "data": fields.Raw(description="Login token data"), }, ), ) @@ -117,6 +116,4 @@ class ActivateApi(Resource): account.initialized_at = naive_utc_now() db.session.commit() - token_pair = AccountService.login(account, ip_address=extract_remote_ip(request)) - - return {"result": "success", "data": token_pair.model_dump()} + return {"result": "success"} diff --git a/api/controllers/console/billing/billing.py b/api/controllers/console/billing/billing.py index 7f907dc420..ac039f9c5d 100644 --- a/api/controllers/console/billing/billing.py +++ b/api/controllers/console/billing/billing.py @@ -1,8 +1,9 @@ import base64 +from typing import Literal from flask import request from flask_restx import Resource, fields -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field from werkzeug.exceptions import BadRequest from controllers.console import console_ns @@ -15,22 +16,8 @@ DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" class SubscriptionQuery(BaseModel): - plan: str = Field(..., description="Subscription plan") - interval: str = Field(..., description="Billing interval") - - @field_validator("plan") - @classmethod - def validate_plan(cls, value: str) -> str: - if value not in [CloudPlan.PROFESSIONAL, CloudPlan.TEAM]: - raise ValueError("Invalid plan") - return value - - @field_validator("interval") - @classmethod - def validate_interval(cls, value: str) -> str: - if value not in {"month", "year"}: - raise ValueError("Invalid interval") - return value + plan: Literal[CloudPlan.PROFESSIONAL, CloudPlan.TEAM] = Field(..., description="Subscription plan") + interval: Literal["month", "year"] = Field(..., description="Billing interval") class PartnerTenantsPayload(BaseModel): diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index ea21c4480d..8ceb896d4f 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -146,7 +146,7 @@ class DatasetUpdatePayload(BaseModel): embedding_model: str | None = None embedding_model_provider: str | None = None retrieval_model: dict[str, Any] | None = None - partial_member_list: list[str] | None = None + partial_member_list: list[dict[str, str]] | None = None external_retrieval_model: dict[str, Any] | None = None external_knowledge_id: str | None = None external_knowledge_api_id: str | None = None diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index 6145da31a5..e94768f985 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -572,7 +572,7 @@ class DocumentBatchIndexingEstimateApi(DocumentResource): datasource_type=DatasourceType.NOTION, notion_info=NotionInfo.model_validate( { - "credential_id": data_source_info["credential_id"], + "credential_id": data_source_info.get("credential_id"), "notion_workspace_id": data_source_info["notion_workspace_id"], "notion_obj_id": data_source_info["notion_page_id"], "notion_page_type": data_source_info["type"], diff --git a/api/controllers/console/explore/completion.py b/api/controllers/console/explore/completion.py index 5901eca915..a6e5b2822a 100644 --- a/api/controllers/console/explore/completion.py +++ b/api/controllers/console/explore/completion.py @@ -40,7 +40,7 @@ from .. import console_ns logger = logging.getLogger(__name__) -class CompletionMessagePayload(BaseModel): +class CompletionMessageExplorePayload(BaseModel): inputs: dict[str, Any] query: str = "" files: list[dict[str, Any]] | None = None @@ -71,7 +71,7 @@ class ChatMessagePayload(BaseModel): raise ValueError("must be a valid UUID") from exc -register_schema_models(console_ns, CompletionMessagePayload, ChatMessagePayload) +register_schema_models(console_ns, CompletionMessageExplorePayload, ChatMessagePayload) # define completion api for user @@ -80,13 +80,13 @@ register_schema_models(console_ns, CompletionMessagePayload, ChatMessagePayload) endpoint="installed_app_completion", ) class CompletionApi(InstalledAppResource): - @console_ns.expect(console_ns.models[CompletionMessagePayload.__name__]) + @console_ns.expect(console_ns.models[CompletionMessageExplorePayload.__name__]) def post(self, installed_app): app_model = installed_app.app if app_model.mode != AppMode.COMPLETION: raise NotCompletionAppError() - payload = CompletionMessagePayload.model_validate(console_ns.payload or {}) + payload = CompletionMessageExplorePayload.model_validate(console_ns.payload or {}) args = payload.model_dump(exclude_none=True) streaming = payload.response_mode == "streaming" diff --git a/api/controllers/console/explore/conversation.py b/api/controllers/console/explore/conversation.py index 92da591ab4..51995b8b8a 100644 --- a/api/controllers/console/explore/conversation.py +++ b/api/controllers/console/explore/conversation.py @@ -1,5 +1,4 @@ from typing import Any -from uuid import UUID from flask import request from flask_restx import marshal_with @@ -13,6 +12,7 @@ from controllers.console.explore.wraps import InstalledAppResource from core.app.entities.app_invoke_entities import InvokeFrom from extensions.ext_database import db from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields +from libs.helper import UUIDStrOrEmpty from libs.login import current_user from models import Account from models.model import AppMode @@ -24,7 +24,7 @@ from .. import console_ns class ConversationListQuery(BaseModel): - last_id: UUID | None = None + last_id: UUIDStrOrEmpty | None = None limit: int = Field(default=20, ge=1, le=100) pinned: bool | None = None diff --git a/api/controllers/console/explore/installed_app.py b/api/controllers/console/explore/installed_app.py index 3c95779475..e42db10ba6 100644 --- a/api/controllers/console/explore/installed_app.py +++ b/api/controllers/console/explore/installed_app.py @@ -2,7 +2,8 @@ import logging from typing import Any from flask import request -from flask_restx import Resource, inputs, marshal_with, reqparse +from flask_restx import Resource, marshal_with +from pydantic import BaseModel from sqlalchemy import and_, select from werkzeug.exceptions import BadRequest, Forbidden, NotFound @@ -18,6 +19,15 @@ from services.account_service import TenantService from services.enterprise.enterprise_service import EnterpriseService from services.feature_service import FeatureService + +class InstalledAppCreatePayload(BaseModel): + app_id: str + + +class InstalledAppUpdatePayload(BaseModel): + is_pinned: bool | None = None + + logger = logging.getLogger(__name__) @@ -105,26 +115,25 @@ class InstalledAppsListApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("apps") def post(self): - parser = reqparse.RequestParser().add_argument("app_id", type=str, required=True, help="Invalid app_id") - args = parser.parse_args() + payload = InstalledAppCreatePayload.model_validate(console_ns.payload or {}) - recommended_app = db.session.query(RecommendedApp).where(RecommendedApp.app_id == args["app_id"]).first() + recommended_app = db.session.query(RecommendedApp).where(RecommendedApp.app_id == payload.app_id).first() if recommended_app is None: - raise NotFound("App not found") + raise NotFound("Recommended app not found") _, current_tenant_id = current_account_with_tenant() - app = db.session.query(App).where(App.id == args["app_id"]).first() + app = db.session.query(App).where(App.id == payload.app_id).first() if app is None: - raise NotFound("App not found") + raise NotFound("App entity not found") if not app.is_public: raise Forbidden("You can't install a non-public app") installed_app = ( db.session.query(InstalledApp) - .where(and_(InstalledApp.app_id == args["app_id"], InstalledApp.tenant_id == current_tenant_id)) + .where(and_(InstalledApp.app_id == payload.app_id, InstalledApp.tenant_id == current_tenant_id)) .first() ) @@ -133,7 +142,7 @@ class InstalledAppsListApi(Resource): recommended_app.install_count += 1 new_installed_app = InstalledApp( - app_id=args["app_id"], + app_id=payload.app_id, tenant_id=current_tenant_id, app_owner_tenant_id=app.tenant_id, is_pinned=False, @@ -163,12 +172,11 @@ class InstalledAppApi(InstalledAppResource): return {"result": "success", "message": "App uninstalled successfully"}, 204 def patch(self, installed_app): - parser = reqparse.RequestParser().add_argument("is_pinned", type=inputs.boolean) - args = parser.parse_args() + payload = InstalledAppUpdatePayload.model_validate(console_ns.payload or {}) commit_args = False - if "is_pinned" in args: - installed_app.is_pinned = args["is_pinned"] + if payload.is_pinned is not None: + installed_app.is_pinned = payload.is_pinned commit_args = True if commit_args: diff --git a/api/controllers/console/explore/message.py b/api/controllers/console/explore/message.py index 229b7c8865..d596d60b36 100644 --- a/api/controllers/console/explore/message.py +++ b/api/controllers/console/explore/message.py @@ -1,6 +1,5 @@ import logging from typing import Literal -from uuid import UUID from flask import request from flask_restx import marshal_with @@ -26,6 +25,7 @@ from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotIni from core.model_runtime.errors.invoke import InvokeError from fields.message_fields import message_infinite_scroll_pagination_fields from libs import helper +from libs.helper import UUIDStrOrEmpty from libs.login import current_account_with_tenant from models.model import AppMode from services.app_generate_service import AppGenerateService @@ -44,8 +44,8 @@ logger = logging.getLogger(__name__) class MessageListQuery(BaseModel): - conversation_id: UUID - first_id: UUID | None = None + conversation_id: UUIDStrOrEmpty + first_id: UUIDStrOrEmpty | None = None limit: int = Field(default=20, ge=1, le=100) diff --git a/api/controllers/console/explore/saved_message.py b/api/controllers/console/explore/saved_message.py index 6a9e274a0e..bc7b8e7651 100644 --- a/api/controllers/console/explore/saved_message.py +++ b/api/controllers/console/explore/saved_message.py @@ -1,5 +1,3 @@ -from uuid import UUID - from flask import request from flask_restx import fields, marshal_with from pydantic import BaseModel, Field @@ -10,19 +8,19 @@ from controllers.console import console_ns from controllers.console.explore.error import NotCompletionAppError from controllers.console.explore.wraps import InstalledAppResource from fields.conversation_fields import message_file_fields -from libs.helper import TimestampField +from libs.helper import TimestampField, UUIDStrOrEmpty from libs.login import current_account_with_tenant from services.errors.message import MessageNotExistsError from services.saved_message_service import SavedMessageService class SavedMessageListQuery(BaseModel): - last_id: UUID | None = None + last_id: UUIDStrOrEmpty | None = None limit: int = Field(default=20, ge=1, le=100) class SavedMessageCreatePayload(BaseModel): - message_id: UUID + message_id: UUIDStrOrEmpty register_schema_models(console_ns, SavedMessageListQuery, SavedMessageCreatePayload) diff --git a/api/controllers/console/extension.py b/api/controllers/console/extension.py index 08f29b4655..efa46c9779 100644 --- a/api/controllers/console/extension.py +++ b/api/controllers/console/extension.py @@ -1,14 +1,32 @@ -from flask_restx import Resource, fields, marshal_with, reqparse +from flask import request +from flask_restx import Resource, fields, marshal_with +from pydantic import BaseModel, Field from constants import HIDDEN_VALUE -from controllers.console import console_ns -from controllers.console.wraps import account_initialization_required, setup_required from fields.api_based_extension_fields import api_based_extension_fields from libs.login import current_account_with_tenant, login_required from models.api_based_extension import APIBasedExtension from services.api_based_extension_service import APIBasedExtensionService from services.code_based_extension_service import CodeBasedExtensionService +from ..common.schema import register_schema_models +from . import console_ns +from .wraps import account_initialization_required, setup_required + + +class CodeBasedExtensionQuery(BaseModel): + module: str + + +class APIBasedExtensionPayload(BaseModel): + name: str = Field(description="Extension name") + api_endpoint: str = Field(description="API endpoint URL") + api_key: str = Field(description="API key for authentication") + + +register_schema_models(console_ns, APIBasedExtensionPayload) + + api_based_extension_model = console_ns.model("ApiBasedExtensionModel", api_based_extension_fields) api_based_extension_list_model = fields.List(fields.Nested(api_based_extension_model)) @@ -18,11 +36,7 @@ api_based_extension_list_model = fields.List(fields.Nested(api_based_extension_m class CodeBasedExtensionAPI(Resource): @console_ns.doc("get_code_based_extension") @console_ns.doc(description="Get code-based extension data by module name") - @console_ns.expect( - console_ns.parser().add_argument( - "module", type=str, required=True, location="args", help="Extension module name" - ) - ) + @console_ns.doc(params={"module": "Extension module name"}) @console_ns.response( 200, "Success", @@ -35,10 +49,9 @@ class CodeBasedExtensionAPI(Resource): @login_required @account_initialization_required def get(self): - parser = reqparse.RequestParser().add_argument("module", type=str, required=True, location="args") - args = parser.parse_args() + query = CodeBasedExtensionQuery.model_validate(request.args.to_dict(flat=True)) # type: ignore - return {"module": args["module"], "data": CodeBasedExtensionService.get_code_based_extension(args["module"])} + return {"module": query.module, "data": CodeBasedExtensionService.get_code_based_extension(query.module)} @console_ns.route("/api-based-extension") @@ -56,30 +69,21 @@ class APIBasedExtensionAPI(Resource): @console_ns.doc("create_api_based_extension") @console_ns.doc(description="Create a new API-based extension") - @console_ns.expect( - console_ns.model( - "CreateAPIBasedExtensionRequest", - { - "name": fields.String(required=True, description="Extension name"), - "api_endpoint": fields.String(required=True, description="API endpoint URL"), - "api_key": fields.String(required=True, description="API key for authentication"), - }, - ) - ) + @console_ns.expect(console_ns.models[APIBasedExtensionPayload.__name__]) @console_ns.response(201, "Extension created successfully", api_based_extension_model) @setup_required @login_required @account_initialization_required @marshal_with(api_based_extension_model) def post(self): - args = console_ns.payload + payload = APIBasedExtensionPayload.model_validate(console_ns.payload or {}) _, current_tenant_id = current_account_with_tenant() extension_data = APIBasedExtension( tenant_id=current_tenant_id, - name=args["name"], - api_endpoint=args["api_endpoint"], - api_key=args["api_key"], + name=payload.name, + api_endpoint=payload.api_endpoint, + api_key=payload.api_key, ) return APIBasedExtensionService.save(extension_data) @@ -104,16 +108,7 @@ class APIBasedExtensionDetailAPI(Resource): @console_ns.doc("update_api_based_extension") @console_ns.doc(description="Update API-based extension") @console_ns.doc(params={"id": "Extension ID"}) - @console_ns.expect( - console_ns.model( - "UpdateAPIBasedExtensionRequest", - { - "name": fields.String(required=True, description="Extension name"), - "api_endpoint": fields.String(required=True, description="API endpoint URL"), - "api_key": fields.String(required=True, description="API key for authentication"), - }, - ) - ) + @console_ns.expect(console_ns.models[APIBasedExtensionPayload.__name__]) @console_ns.response(200, "Extension updated successfully", api_based_extension_model) @setup_required @login_required @@ -125,13 +120,13 @@ class APIBasedExtensionDetailAPI(Resource): extension_data_from_db = APIBasedExtensionService.get_with_tenant_id(current_tenant_id, api_based_extension_id) - args = console_ns.payload + payload = APIBasedExtensionPayload.model_validate(console_ns.payload or {}) - extension_data_from_db.name = args["name"] - extension_data_from_db.api_endpoint = args["api_endpoint"] + extension_data_from_db.name = payload.name + extension_data_from_db.api_endpoint = payload.api_endpoint - if args["api_key"] != HIDDEN_VALUE: - extension_data_from_db.api_key = args["api_key"] + if payload.api_key != HIDDEN_VALUE: + extension_data_from_db.api_key = payload.api_key return APIBasedExtensionService.save(extension_data_from_db) diff --git a/api/controllers/console/tag/tags.py b/api/controllers/console/tag/tags.py index 17cfc3ff4b..e9fbb515e4 100644 --- a/api/controllers/console/tag/tags.py +++ b/api/controllers/console/tag/tags.py @@ -1,31 +1,40 @@ +from typing import Literal + from flask import request -from flask_restx import Resource, marshal_with, reqparse +from flask_restx import Resource, marshal_with +from pydantic import BaseModel, Field from werkzeug.exceptions import Forbidden +from controllers.common.schema import register_schema_models from controllers.console import console_ns from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required from fields.tag_fields import dataset_tag_fields from libs.login import current_account_with_tenant, login_required -from models.model import Tag from services.tag_service import TagService -def _validate_name(name): - if not name or len(name) < 1 or len(name) > 50: - raise ValueError("Name must be between 1 to 50 characters.") - return name +class TagBasePayload(BaseModel): + name: str = Field(description="Tag name", min_length=1, max_length=50) + type: Literal["knowledge", "app"] | None = Field(default=None, description="Tag type") -parser_tags = ( - reqparse.RequestParser() - .add_argument( - "name", - nullable=False, - required=True, - help="Name must be between 1 to 50 characters.", - type=_validate_name, - ) - .add_argument("type", type=str, location="json", choices=Tag.TAG_TYPE_LIST, nullable=True, help="Invalid tag type.") +class TagBindingPayload(BaseModel): + tag_ids: list[str] = Field(description="Tag IDs to bind") + target_id: str = Field(description="Target ID to bind tags to") + type: Literal["knowledge", "app"] | None = Field(default=None, description="Tag type") + + +class TagBindingRemovePayload(BaseModel): + tag_id: str = Field(description="Tag ID to remove") + target_id: str = Field(description="Target ID to unbind tag from") + type: Literal["knowledge", "app"] | None = Field(default=None, description="Tag type") + + +register_schema_models( + console_ns, + TagBasePayload, + TagBindingPayload, + TagBindingRemovePayload, ) @@ -43,7 +52,7 @@ class TagListApi(Resource): return tags, 200 - @console_ns.expect(parser_tags) + @console_ns.expect(console_ns.models[TagBasePayload.__name__]) @setup_required @login_required @account_initialization_required @@ -53,22 +62,17 @@ class TagListApi(Resource): if not (current_user.has_edit_permission or current_user.is_dataset_editor): raise Forbidden() - args = parser_tags.parse_args() - tag = TagService.save_tags(args) + payload = TagBasePayload.model_validate(console_ns.payload or {}) + tag = TagService.save_tags(payload.model_dump()) response = {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": 0} return response, 200 -parser_tag_id = reqparse.RequestParser().add_argument( - "name", nullable=False, required=True, help="Name must be between 1 to 50 characters.", type=_validate_name -) - - @console_ns.route("/tags/") class TagUpdateDeleteApi(Resource): - @console_ns.expect(parser_tag_id) + @console_ns.expect(console_ns.models[TagBasePayload.__name__]) @setup_required @login_required @account_initialization_required @@ -79,8 +83,8 @@ class TagUpdateDeleteApi(Resource): if not (current_user.has_edit_permission or current_user.is_dataset_editor): raise Forbidden() - args = parser_tag_id.parse_args() - tag = TagService.update_tags(args, tag_id) + payload = TagBasePayload.model_validate(console_ns.payload or {}) + tag = TagService.update_tags(payload.model_dump(), tag_id) binding_count = TagService.get_tag_binding_count(tag_id) @@ -100,17 +104,9 @@ class TagUpdateDeleteApi(Resource): return 204 -parser_create = ( - reqparse.RequestParser() - .add_argument("tag_ids", type=list, nullable=False, required=True, location="json", help="Tag IDs is required.") - .add_argument("target_id", type=str, nullable=False, required=True, location="json", help="Target ID is required.") - .add_argument("type", type=str, location="json", choices=Tag.TAG_TYPE_LIST, nullable=True, help="Invalid tag type.") -) - - @console_ns.route("/tag-bindings/create") class TagBindingCreateApi(Resource): - @console_ns.expect(parser_create) + @console_ns.expect(console_ns.models[TagBindingPayload.__name__]) @setup_required @login_required @account_initialization_required @@ -120,23 +116,15 @@ class TagBindingCreateApi(Resource): if not (current_user.has_edit_permission or current_user.is_dataset_editor): raise Forbidden() - args = parser_create.parse_args() - TagService.save_tag_binding(args) + payload = TagBindingPayload.model_validate(console_ns.payload or {}) + TagService.save_tag_binding(payload.model_dump()) return {"result": "success"}, 200 -parser_remove = ( - reqparse.RequestParser() - .add_argument("tag_id", type=str, nullable=False, required=True, help="Tag ID is required.") - .add_argument("target_id", type=str, nullable=False, required=True, help="Target ID is required.") - .add_argument("type", type=str, location="json", choices=Tag.TAG_TYPE_LIST, nullable=True, help="Invalid tag type.") -) - - @console_ns.route("/tag-bindings/remove") class TagBindingDeleteApi(Resource): - @console_ns.expect(parser_remove) + @console_ns.expect(console_ns.models[TagBindingRemovePayload.__name__]) @setup_required @login_required @account_initialization_required @@ -146,7 +134,7 @@ class TagBindingDeleteApi(Resource): if not (current_user.has_edit_permission or current_user.is_dataset_editor): raise Forbidden() - args = parser_remove.parse_args() - TagService.delete_tag_binding(args) + payload = TagBindingRemovePayload.model_validate(console_ns.payload or {}) + TagService.delete_tag_binding(payload.model_dump()) return {"result": "success"}, 200 diff --git a/api/controllers/console/workspace/load_balancing_config.py b/api/controllers/console/workspace/load_balancing_config.py index 9bf393ea2e..ccb60b1461 100644 --- a/api/controllers/console/workspace/load_balancing_config.py +++ b/api/controllers/console/workspace/load_balancing_config.py @@ -1,6 +1,8 @@ -from flask_restx import Resource, reqparse +from flask_restx import Resource +from pydantic import BaseModel from werkzeug.exceptions import Forbidden +from controllers.common.schema import register_schema_models from controllers.console import console_ns from controllers.console.wraps import account_initialization_required, setup_required from core.model_runtime.entities.model_entities import ModelType @@ -10,10 +12,20 @@ from models import TenantAccountRole from services.model_load_balancing_service import ModelLoadBalancingService +class LoadBalancingCredentialPayload(BaseModel): + model: str + model_type: ModelType + credentials: dict[str, object] + + +register_schema_models(console_ns, LoadBalancingCredentialPayload) + + @console_ns.route( "/workspaces/current/model-providers//models/load-balancing-configs/credentials-validate" ) class LoadBalancingCredentialsValidateApi(Resource): + @console_ns.expect(console_ns.models[LoadBalancingCredentialPayload.__name__]) @setup_required @login_required @account_initialization_required @@ -24,20 +36,7 @@ class LoadBalancingCredentialsValidateApi(Resource): tenant_id = current_tenant_id - parser = ( - reqparse.RequestParser() - .add_argument("model", type=str, required=True, nullable=False, location="json") - .add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="json", - ) - .add_argument("credentials", type=dict, required=True, nullable=False, location="json") - ) - args = parser.parse_args() + payload = LoadBalancingCredentialPayload.model_validate(console_ns.payload or {}) # validate model load balancing credentials model_load_balancing_service = ModelLoadBalancingService() @@ -49,9 +48,9 @@ class LoadBalancingCredentialsValidateApi(Resource): model_load_balancing_service.validate_load_balancing_credentials( tenant_id=tenant_id, provider=provider, - model=args["model"], - model_type=args["model_type"], - credentials=args["credentials"], + model=payload.model, + model_type=payload.model_type, + credentials=payload.credentials, ) except CredentialsValidateFailedError as ex: result = False @@ -69,6 +68,7 @@ class LoadBalancingCredentialsValidateApi(Resource): "/workspaces/current/model-providers//models/load-balancing-configs//credentials-validate" ) class LoadBalancingConfigCredentialsValidateApi(Resource): + @console_ns.expect(console_ns.models[LoadBalancingCredentialPayload.__name__]) @setup_required @login_required @account_initialization_required @@ -79,20 +79,7 @@ class LoadBalancingConfigCredentialsValidateApi(Resource): tenant_id = current_tenant_id - parser = ( - reqparse.RequestParser() - .add_argument("model", type=str, required=True, nullable=False, location="json") - .add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="json", - ) - .add_argument("credentials", type=dict, required=True, nullable=False, location="json") - ) - args = parser.parse_args() + payload = LoadBalancingCredentialPayload.model_validate(console_ns.payload or {}) # validate model load balancing config credentials model_load_balancing_service = ModelLoadBalancingService() @@ -104,9 +91,9 @@ class LoadBalancingConfigCredentialsValidateApi(Resource): model_load_balancing_service.validate_load_balancing_credentials( tenant_id=tenant_id, provider=provider, - model=args["model"], - model_type=args["model_type"], - credentials=args["credentials"], + model=payload.model, + model_type=payload.model_type, + credentials=payload.credentials, config_id=config_id, ) except CredentialsValidateFailedError as ex: diff --git a/api/controllers/console/workspace/plugin.py b/api/controllers/console/workspace/plugin.py index 805058ba5a..ea74fc0337 100644 --- a/api/controllers/console/workspace/plugin.py +++ b/api/controllers/console/workspace/plugin.py @@ -1,5 +1,6 @@ import io -from typing import Literal +from collections.abc import Mapping +from typing import Any, Literal from flask import request, send_file from flask_restx import Resource @@ -141,6 +142,15 @@ class ParserDynamicOptions(BaseModel): provider_type: Literal["tool", "trigger"] +class ParserDynamicOptionsWithCredentials(BaseModel): + plugin_id: str + provider: str + action: str + parameter: str + credential_id: str + credentials: Mapping[str, Any] + + class PluginPermissionSettingsPayload(BaseModel): install_permission: TenantPluginPermission.InstallPermission = TenantPluginPermission.InstallPermission.EVERYONE debug_permission: TenantPluginPermission.DebugPermission = TenantPluginPermission.DebugPermission.EVERYONE @@ -183,6 +193,7 @@ reg(ParserGithubUpgrade) reg(ParserUninstall) reg(ParserPermissionChange) reg(ParserDynamicOptions) +reg(ParserDynamicOptionsWithCredentials) reg(ParserPreferencesChange) reg(ParserExcludePlugin) reg(ParserReadme) @@ -657,6 +668,37 @@ class PluginFetchDynamicSelectOptionsApi(Resource): return jsonable_encoder({"options": options}) +@console_ns.route("/workspaces/current/plugin/parameters/dynamic-options-with-credentials") +class PluginFetchDynamicSelectOptionsWithCredentialsApi(Resource): + @console_ns.expect(console_ns.models[ParserDynamicOptionsWithCredentials.__name__]) + @setup_required + @login_required + @is_admin_or_owner_required + @account_initialization_required + def post(self): + """Fetch dynamic options using credentials directly (for edit mode).""" + current_user, tenant_id = current_account_with_tenant() + user_id = current_user.id + + args = ParserDynamicOptionsWithCredentials.model_validate(console_ns.payload) + + try: + options = PluginParameterService.get_dynamic_select_options_with_credentials( + tenant_id=tenant_id, + user_id=user_id, + plugin_id=args.plugin_id, + provider=args.provider, + action=args.action, + parameter=args.parameter, + credential_id=args.credential_id, + credentials=args.credentials, + ) + except PluginDaemonClientSideError as e: + raise ValueError(e) + + return jsonable_encoder({"options": options}) + + @console_ns.route("/workspaces/current/plugin/preferences/change") class PluginChangePreferencesApi(Resource): @console_ns.expect(console_ns.models[ParserPreferencesChange.__name__]) diff --git a/api/controllers/console/workspace/tool_providers.py b/api/controllers/console/workspace/tool_providers.py index 2c54aa5a20..e9e7b72718 100644 --- a/api/controllers/console/workspace/tool_providers.py +++ b/api/controllers/console/workspace/tool_providers.py @@ -1,4 +1,5 @@ import io +import logging from urllib.parse import urlparse from flask import make_response, redirect, request, send_file @@ -17,6 +18,7 @@ from controllers.console.wraps import ( is_admin_or_owner_required, setup_required, ) +from core.db.session_factory import session_factory from core.entities.mcp_provider import MCPAuthentication, MCPConfiguration from core.mcp.auth.auth_flow import auth, handle_callback from core.mcp.error import MCPAuthError, MCPError, MCPRefreshTokenError @@ -39,6 +41,8 @@ from services.tools.tools_manage_service import ToolCommonService from services.tools.tools_transform_service import ToolTransformService from services.tools.workflow_tools_manage_service import WorkflowToolManageService +logger = logging.getLogger(__name__) + def is_valid_url(url: str) -> bool: if not url: @@ -944,8 +948,8 @@ class ToolProviderMCPApi(Resource): configuration = MCPConfiguration.model_validate(args["configuration"]) authentication = MCPAuthentication.model_validate(args["authentication"]) if args["authentication"] else None - # Create provider - with Session(db.engine) as session, session.begin(): + # 1) Create provider in a short transaction (no network I/O inside) + with session_factory.create_session() as session, session.begin(): service = MCPToolManageService(session=session) result = service.create_provider( tenant_id=tenant_id, @@ -960,7 +964,29 @@ class ToolProviderMCPApi(Resource): configuration=configuration, authentication=authentication, ) - return jsonable_encoder(result) + + # 2) Try to fetch tools immediately after creation so they appear without a second save. + # Perform network I/O outside any DB session to avoid holding locks. + try: + reconnect = MCPToolManageService.reconnect_with_url( + server_url=args["server_url"], + headers=args.get("headers") or {}, + timeout=configuration.timeout, + sse_read_timeout=configuration.sse_read_timeout, + ) + # Update just-created provider with authed/tools in a new short transaction + with session_factory.create_session() as session, session.begin(): + service = MCPToolManageService(session=session) + db_provider = service.get_provider(provider_id=result.id, tenant_id=tenant_id) + db_provider.authed = reconnect.authed + db_provider.tools = reconnect.tools + + result = ToolTransformService.mcp_provider_to_user_provider(db_provider, for_list=True) + except Exception: + # Best-effort: if initial fetch fails (e.g., auth required), return created provider as-is + logger.warning("Failed to fetch MCP tools after creation", exc_info=True) + + return jsonable_encoder(result) @console_ns.expect(parser_mcp_put) @setup_required @@ -972,17 +998,23 @@ class ToolProviderMCPApi(Resource): authentication = MCPAuthentication.model_validate(args["authentication"]) if args["authentication"] else None _, current_tenant_id = current_account_with_tenant() - # Step 1: Validate server URL change if needed (includes URL format validation and network operation) - validation_result = None + # Step 1: Get provider data for URL validation (short-lived session, no network I/O) + validation_data = None with Session(db.engine) as session: service = MCPToolManageService(session=session) - validation_result = service.validate_server_url_change( - tenant_id=current_tenant_id, provider_id=args["provider_id"], new_server_url=args["server_url"] + validation_data = service.get_provider_for_url_validation( + tenant_id=current_tenant_id, provider_id=args["provider_id"] ) - # No need to check for errors here, exceptions will be raised directly + # Step 2: Perform URL validation with network I/O OUTSIDE of any database session + # This prevents holding database locks during potentially slow network operations + validation_result = MCPToolManageService.validate_server_url_standalone( + tenant_id=current_tenant_id, + new_server_url=args["server_url"], + validation_data=validation_data, + ) - # Step 2: Perform database update in a transaction + # Step 3: Perform database update in a transaction with Session(db.engine) as session, session.begin(): service = MCPToolManageService(session=session) service.update_provider( @@ -999,7 +1031,8 @@ class ToolProviderMCPApi(Resource): authentication=authentication, validation_result=validation_result, ) - return {"result": "success"} + + return {"result": "success"} @console_ns.expect(parser_mcp_delete) @setup_required @@ -1012,7 +1045,8 @@ class ToolProviderMCPApi(Resource): with Session(db.engine) as session, session.begin(): service = MCPToolManageService(session=session) service.delete_provider(tenant_id=current_tenant_id, provider_id=args["provider_id"]) - return {"result": "success"} + + return {"result": "success"} parser_auth = ( diff --git a/api/controllers/console/workspace/trigger_providers.py b/api/controllers/console/workspace/trigger_providers.py index 268473d6d1..497e62b790 100644 --- a/api/controllers/console/workspace/trigger_providers.py +++ b/api/controllers/console/workspace/trigger_providers.py @@ -1,11 +1,15 @@ import logging +from collections.abc import Mapping +from typing import Any from flask import make_response, redirect, request from flask_restx import Resource, reqparse +from pydantic import BaseModel, Field from sqlalchemy.orm import Session from werkzeug.exceptions import BadRequest, Forbidden from configs import dify_config +from constants import HIDDEN_VALUE, UNKNOWN_VALUE from controllers.web.error import NotFoundError from core.model_runtime.utils.encoders import jsonable_encoder from core.plugin.entities.plugin_daemon import CredentialType @@ -32,6 +36,32 @@ from ..wraps import ( logger = logging.getLogger(__name__) +class TriggerSubscriptionUpdateRequest(BaseModel): + """Request payload for updating a trigger subscription""" + + name: str | None = Field(default=None, description="The name for the subscription") + credentials: Mapping[str, Any] | None = Field(default=None, description="The credentials for the subscription") + parameters: Mapping[str, Any] | None = Field(default=None, description="The parameters for the subscription") + properties: Mapping[str, Any] | None = Field(default=None, description="The properties for the subscription") + + +class TriggerSubscriptionVerifyRequest(BaseModel): + """Request payload for verifying subscription credentials.""" + + credentials: Mapping[str, Any] = Field(description="The credentials to verify") + + +console_ns.schema_model( + TriggerSubscriptionUpdateRequest.__name__, + TriggerSubscriptionUpdateRequest.model_json_schema(ref_template="#/definitions/{model}"), +) + +console_ns.schema_model( + TriggerSubscriptionVerifyRequest.__name__, + TriggerSubscriptionVerifyRequest.model_json_schema(ref_template="#/definitions/{model}"), +) + + @console_ns.route("/workspaces/current/trigger-provider//icon") class TriggerProviderIconApi(Resource): @setup_required @@ -155,16 +185,16 @@ parser_api = ( @console_ns.route( - "/workspaces/current/trigger-provider//subscriptions/builder/verify/", + "/workspaces/current/trigger-provider//subscriptions/builder/verify-and-update/", ) -class TriggerSubscriptionBuilderVerifyApi(Resource): +class TriggerSubscriptionBuilderVerifyAndUpdateApi(Resource): @console_ns.expect(parser_api) @setup_required @login_required @edit_permission_required @account_initialization_required def post(self, provider, subscription_builder_id): - """Verify a subscription instance for a trigger provider""" + """Verify and update a subscription instance for a trigger provider""" user = current_user assert user.current_tenant_id is not None @@ -289,6 +319,83 @@ class TriggerSubscriptionBuilderBuildApi(Resource): raise ValueError(str(e)) from e +@console_ns.route( + "/workspaces/current/trigger-provider//subscriptions/update", +) +class TriggerSubscriptionUpdateApi(Resource): + @console_ns.expect(console_ns.models[TriggerSubscriptionUpdateRequest.__name__]) + @setup_required + @login_required + @edit_permission_required + @account_initialization_required + def post(self, subscription_id: str): + """Update a subscription instance""" + user = current_user + assert user.current_tenant_id is not None + + args = TriggerSubscriptionUpdateRequest.model_validate(console_ns.payload) + + subscription = TriggerProviderService.get_subscription_by_id( + tenant_id=user.current_tenant_id, + subscription_id=subscription_id, + ) + if not subscription: + raise NotFoundError(f"Subscription {subscription_id} not found") + + provider_id = TriggerProviderID(subscription.provider_id) + + try: + # rename only + if ( + args.name is not None + and args.credentials is None + and args.parameters is None + and args.properties is None + ): + TriggerProviderService.update_trigger_subscription( + tenant_id=user.current_tenant_id, + subscription_id=subscription_id, + name=args.name, + ) + return 200 + + # rebuild for create automatically by the provider + match subscription.credential_type: + case CredentialType.UNAUTHORIZED: + TriggerProviderService.update_trigger_subscription( + tenant_id=user.current_tenant_id, + subscription_id=subscription_id, + name=args.name, + properties=args.properties, + ) + return 200 + case CredentialType.API_KEY | CredentialType.OAUTH2: + if args.credentials: + new_credentials: dict[str, Any] = { + key: value if value != HIDDEN_VALUE else subscription.credentials.get(key, UNKNOWN_VALUE) + for key, value in args.credentials.items() + } + else: + new_credentials = subscription.credentials + + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=user.current_tenant_id, + name=args.name, + provider_id=provider_id, + subscription_id=subscription_id, + credentials=new_credentials, + parameters=args.parameters or subscription.parameters, + ) + return 200 + case _: + raise BadRequest("Invalid credential type") + except ValueError as e: + raise BadRequest(str(e)) + except Exception as e: + logger.exception("Error updating subscription", exc_info=e) + raise + + @console_ns.route( "/workspaces/current/trigger-provider//subscriptions/delete", ) @@ -576,3 +683,38 @@ class TriggerOAuthClientManageApi(Resource): except Exception as e: logger.exception("Error removing OAuth client", exc_info=e) raise + + +@console_ns.route( + "/workspaces/current/trigger-provider//subscriptions/verify/", +) +class TriggerSubscriptionVerifyApi(Resource): + @console_ns.expect(console_ns.models[TriggerSubscriptionVerifyRequest.__name__]) + @setup_required + @login_required + @edit_permission_required + @account_initialization_required + def post(self, provider, subscription_id): + """Verify credentials for an existing subscription (edit mode only)""" + user = current_user + assert user.current_tenant_id is not None + + verify_request: TriggerSubscriptionVerifyRequest = TriggerSubscriptionVerifyRequest.model_validate( + console_ns.payload + ) + + try: + result = TriggerProviderService.verify_subscription_credentials( + tenant_id=user.current_tenant_id, + user_id=user.id, + provider_id=TriggerProviderID(provider), + subscription_id=subscription_id, + credentials=verify_request.credentials, + ) + return result + except ValueError as e: + logger.warning("Credential verification failed", exc_info=e) + raise BadRequest(str(e)) from e + except Exception as e: + logger.exception("Error verifying subscription credentials", exc_info=e) + raise BadRequest(str(e)) from e diff --git a/api/controllers/files/image_preview.py b/api/controllers/files/image_preview.py index 64f47f426a..04db1c67cb 100644 --- a/api/controllers/files/image_preview.py +++ b/api/controllers/files/image_preview.py @@ -7,6 +7,7 @@ from werkzeug.exceptions import NotFound import services from controllers.common.errors import UnsupportedFileTypeError +from controllers.common.file_response import enforce_download_for_html from controllers.files import files_ns from extensions.ext_database import db from services.account_service import TenantService @@ -138,6 +139,13 @@ class FilePreviewApi(Resource): response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}" response.headers["Content-Type"] = "application/octet-stream" + enforce_download_for_html( + response, + mime_type=upload_file.mime_type, + filename=upload_file.name, + extension=upload_file.extension, + ) + return response diff --git a/api/controllers/files/tool_files.py b/api/controllers/files/tool_files.py index c487a0a915..89aa472015 100644 --- a/api/controllers/files/tool_files.py +++ b/api/controllers/files/tool_files.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Field from werkzeug.exceptions import Forbidden, NotFound from controllers.common.errors import UnsupportedFileTypeError +from controllers.common.file_response import enforce_download_for_html from controllers.files import files_ns from core.tools.signature import verify_tool_file_signature from core.tools.tool_file_manager import ToolFileManager @@ -78,4 +79,11 @@ class ToolFileApi(Resource): encoded_filename = quote(tool_file.name) response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}" + enforce_download_for_html( + response, + mime_type=tool_file.mimetype, + filename=tool_file.name, + extension=extension, + ) + return response diff --git a/api/controllers/service_api/app/conversation.py b/api/controllers/service_api/app/conversation.py index be6d837032..40e4bde389 100644 --- a/api/controllers/service_api/app/conversation.py +++ b/api/controllers/service_api/app/conversation.py @@ -4,7 +4,7 @@ from uuid import UUID from flask import request from flask_restx import Resource from flask_restx._http import HTTPStatus -from pydantic import BaseModel, Field, model_validator +from pydantic import BaseModel, Field, field_validator, model_validator from sqlalchemy.orm import Session from werkzeug.exceptions import BadRequest, NotFound @@ -51,6 +51,32 @@ class ConversationRenamePayload(BaseModel): class ConversationVariablesQuery(BaseModel): last_id: UUID | None = Field(default=None, description="Last variable ID for pagination") limit: int = Field(default=20, ge=1, le=100, description="Number of variables to return") + variable_name: str | None = Field( + default=None, description="Filter variables by name", min_length=1, max_length=255 + ) + + @field_validator("variable_name", mode="before") + @classmethod + def validate_variable_name(cls, v: str | None) -> str | None: + """ + Validate variable_name to prevent injection attacks. + """ + if v is None: + return v + + # Only allow safe characters: alphanumeric, underscore, hyphen, period + if not v.replace("-", "").replace("_", "").replace(".", "").isalnum(): + raise ValueError( + "Variable name can only contain letters, numbers, hyphens (-), underscores (_), and periods (.)" + ) + + # Prevent SQL injection patterns + dangerous_patterns = ["'", '"', ";", "--", "/*", "*/", "xp_", "sp_"] + for pattern in dangerous_patterns: + if pattern in v.lower(): + raise ValueError(f"Variable name contains invalid characters: {pattern}") + + return v class ConversationVariableUpdatePayload(BaseModel): @@ -199,7 +225,7 @@ class ConversationVariablesApi(Resource): try: return ConversationService.get_conversational_variable( - app_model, conversation_id, end_user, query_args.limit, last_id + app_model, conversation_id, end_user, query_args.limit, last_id, query_args.variable_name ) except services.errors.conversation.ConversationNotExistsError: raise NotFound("Conversation Not Exists.") diff --git a/api/controllers/service_api/app/file_preview.py b/api/controllers/service_api/app/file_preview.py index 60f422b88e..f853a124ef 100644 --- a/api/controllers/service_api/app/file_preview.py +++ b/api/controllers/service_api/app/file_preview.py @@ -5,6 +5,7 @@ from flask import Response, request from flask_restx import Resource from pydantic import BaseModel, Field +from controllers.common.file_response import enforce_download_for_html from controllers.common.schema import register_schema_model from controllers.service_api import service_api_ns from controllers.service_api.app.error import ( @@ -183,6 +184,13 @@ class FilePreviewApi(Resource): # Override content-type for downloads to force download response.headers["Content-Type"] = "application/octet-stream" + enforce_download_for_html( + response, + mime_type=upload_file.mime_type, + filename=upload_file.name, + extension=upload_file.extension, + ) + # Add caching headers for performance response.headers["Cache-Control"] = "public, max-age=3600" # Cache for 1 hour diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index 7692aeed23..94faf8dd42 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -13,7 +13,6 @@ from controllers.service_api.dataset.error import DatasetInUseError, DatasetName from controllers.service_api.wraps import ( DatasetApiResource, cloud_edition_billing_rate_limit_check, - validate_dataset_token, ) from core.model_runtime.entities.model_entities import ModelType from core.provider_manager import ProviderManager @@ -49,7 +48,7 @@ class DatasetUpdatePayload(BaseModel): embedding_model: str | None = None embedding_model_provider: str | None = None retrieval_model: RetrievalModel | None = None - partial_member_list: list[str] | None = None + partial_member_list: list[dict[str, str]] | None = None external_retrieval_model: dict[str, Any] | None = None external_knowledge_id: str | None = None external_knowledge_api_id: str | None = None @@ -460,9 +459,8 @@ class DatasetTagsApi(DatasetApiResource): 401: "Unauthorized - invalid API token", } ) - @validate_dataset_token @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns)) - def get(self, _, dataset_id): + def get(self, _): """Get all knowledge type tags.""" assert isinstance(current_user, Account) cid = current_user.current_tenant_id @@ -482,8 +480,7 @@ class DatasetTagsApi(DatasetApiResource): } ) @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns)) - @validate_dataset_token - def post(self, _, dataset_id): + def post(self, _): """Add a knowledge type tag.""" assert isinstance(current_user, Account) if not (current_user.has_edit_permission or current_user.is_dataset_editor): @@ -506,8 +503,7 @@ class DatasetTagsApi(DatasetApiResource): } ) @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns)) - @validate_dataset_token - def patch(self, _, dataset_id): + def patch(self, _): assert isinstance(current_user, Account) if not (current_user.has_edit_permission or current_user.is_dataset_editor): raise Forbidden() @@ -533,9 +529,8 @@ class DatasetTagsApi(DatasetApiResource): 403: "Forbidden - insufficient permissions", } ) - @validate_dataset_token @edit_permission_required - def delete(self, _, dataset_id): + def delete(self, _): """Delete a knowledge type tag.""" payload = TagDeletePayload.model_validate(service_api_ns.payload or {}) TagService.delete_tag(payload.tag_id) @@ -555,8 +550,7 @@ class DatasetTagBindingApi(DatasetApiResource): 403: "Forbidden - insufficient permissions", } ) - @validate_dataset_token - def post(self, _, dataset_id): + def post(self, _): # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator assert isinstance(current_user, Account) if not (current_user.has_edit_permission or current_user.is_dataset_editor): @@ -580,8 +574,7 @@ class DatasetTagUnbindingApi(DatasetApiResource): 403: "Forbidden - insufficient permissions", } ) - @validate_dataset_token - def post(self, _, dataset_id): + def post(self, _): # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator assert isinstance(current_user, Account) if not (current_user.has_edit_permission or current_user.is_dataset_editor): @@ -604,7 +597,6 @@ class DatasetTagsBindingStatusApi(DatasetApiResource): 401: "Unauthorized - invalid API token", } ) - @validate_dataset_token def get(self, _, *args, **kwargs): """Get all knowledge type tags.""" dataset_id = kwargs.get("dataset_id") diff --git a/api/controllers/web/app.py b/api/controllers/web/app.py index 60193f5f15..db3b93a4dc 100644 --- a/api/controllers/web/app.py +++ b/api/controllers/web/app.py @@ -1,14 +1,13 @@ import logging from flask import request -from flask_restx import Resource, marshal_with, reqparse +from flask_restx import Resource, marshal_with +from pydantic import BaseModel, ConfigDict, Field from werkzeug.exceptions import Unauthorized from constants import HEADER_NAME_APP_CODE from controllers.common import fields -from controllers.web import web_ns -from controllers.web.error import AppUnavailableError -from controllers.web.wraps import WebApiResource +from controllers.common.schema import register_schema_models from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict from libs.passport import PassportService from libs.token import extract_webapp_passport @@ -18,9 +17,23 @@ from services.enterprise.enterprise_service import EnterpriseService from services.feature_service import FeatureService from services.webapp_auth_service import WebAppAuthService +from . import web_ns +from .error import AppUnavailableError +from .wraps import WebApiResource + logger = logging.getLogger(__name__) +class AppAccessModeQuery(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + app_id: str | None = Field(default=None, alias="appId", description="Application ID") + app_code: str | None = Field(default=None, alias="appCode", description="Application code") + + +register_schema_models(web_ns, AppAccessModeQuery) + + @web_ns.route("/parameters") class AppParameterApi(WebApiResource): """Resource for app variables.""" @@ -96,21 +109,16 @@ class AppAccessMode(Resource): } ) def get(self): - parser = ( - reqparse.RequestParser() - .add_argument("appId", type=str, required=False, location="args") - .add_argument("appCode", type=str, required=False, location="args") - ) - args = parser.parse_args() + raw_args = request.args.to_dict() + args = AppAccessModeQuery.model_validate(raw_args) features = FeatureService.get_system_features() if not features.webapp_auth.enabled: return {"accessMode": "public"} - app_id = args.get("appId") - if args.get("appCode"): - app_code = args["appCode"] - app_id = AppService.get_app_id_by_code(app_code) + app_id = args.app_id + if args.app_code: + app_id = AppService.get_app_id_by_code(args.app_code) if not app_id: raise ValueError("appId or appCode must be provided") diff --git a/api/controllers/web/audio.py b/api/controllers/web/audio.py index b9fef48c4d..15828cc208 100644 --- a/api/controllers/web/audio.py +++ b/api/controllers/web/audio.py @@ -1,7 +1,8 @@ import logging from flask import request -from flask_restx import fields, marshal_with, reqparse +from flask_restx import fields, marshal_with +from pydantic import BaseModel, field_validator from werkzeug.exceptions import InternalServerError import services @@ -20,6 +21,7 @@ from controllers.web.error import ( from controllers.web.wraps import WebApiResource from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from core.model_runtime.errors.invoke import InvokeError +from libs.helper import uuid_value from models.model import App from services.audio_service import AudioService from services.errors.audio import ( @@ -29,6 +31,25 @@ from services.errors.audio import ( UnsupportedAudioTypeServiceError, ) +from ..common.schema import register_schema_models + + +class TextToAudioPayload(BaseModel): + message_id: str | None = None + voice: str | None = None + text: str | None = None + streaming: bool | None = None + + @field_validator("message_id") + @classmethod + def validate_message_id(cls, value: str | None) -> str | None: + if value is None: + return value + return uuid_value(value) + + +register_schema_models(web_ns, TextToAudioPayload) + logger = logging.getLogger(__name__) @@ -88,6 +109,7 @@ class AudioApi(WebApiResource): @web_ns.route("/text-to-audio") class TextApi(WebApiResource): + @web_ns.expect(web_ns.models[TextToAudioPayload.__name__]) @web_ns.doc("Text to Audio") @web_ns.doc(description="Convert text to audio using text-to-speech service.") @web_ns.doc( @@ -102,18 +124,11 @@ class TextApi(WebApiResource): def post(self, app_model: App, end_user): """Convert text to audio""" try: - parser = ( - reqparse.RequestParser() - .add_argument("message_id", type=str, required=False, location="json") - .add_argument("voice", type=str, location="json") - .add_argument("text", type=str, location="json") - .add_argument("streaming", type=bool, location="json") - ) - args = parser.parse_args() + payload = TextToAudioPayload.model_validate(web_ns.payload or {}) - message_id = args.get("message_id", None) - text = args.get("text", None) - voice = args.get("voice", None) + message_id = payload.message_id + text = payload.text + voice = payload.voice response = AudioService.transcript_tts( app_model=app_model, text=text, voice=voice, end_user=end_user.external_user_id, message_id=message_id ) diff --git a/api/controllers/web/completion.py b/api/controllers/web/completion.py index e8a4698375..a97d745471 100644 --- a/api/controllers/web/completion.py +++ b/api/controllers/web/completion.py @@ -1,9 +1,11 @@ import logging +from typing import Any, Literal -from flask_restx import reqparse +from pydantic import BaseModel, Field, field_validator from werkzeug.exceptions import InternalServerError, NotFound import services +from controllers.common.schema import register_schema_models from controllers.web import web_ns from controllers.web.error import ( AppUnavailableError, @@ -34,25 +36,44 @@ from services.errors.llm import InvokeRateLimitError logger = logging.getLogger(__name__) +class CompletionMessagePayload(BaseModel): + inputs: dict[str, Any] = Field(description="Input variables for the completion") + query: str = Field(default="", description="Query text for completion") + files: list[dict[str, Any]] | None = Field(default=None, description="Files to be processed") + response_mode: Literal["blocking", "streaming"] | None = Field( + default=None, description="Response mode: blocking or streaming" + ) + retriever_from: str = Field(default="web_app", description="Source of retriever") + + +class ChatMessagePayload(BaseModel): + inputs: dict[str, Any] = Field(description="Input variables for the chat") + query: str = Field(description="User query/message") + files: list[dict[str, Any]] | None = Field(default=None, description="Files to be processed") + response_mode: Literal["blocking", "streaming"] | None = Field( + default=None, description="Response mode: blocking or streaming" + ) + conversation_id: str | None = Field(default=None, description="Conversation ID") + parent_message_id: str | None = Field(default=None, description="Parent message ID") + retriever_from: str = Field(default="web_app", description="Source of retriever") + + @field_validator("conversation_id", "parent_message_id") + @classmethod + def validate_uuid(cls, value: str | None) -> str | None: + if value is None: + return value + return uuid_value(value) + + +register_schema_models(web_ns, CompletionMessagePayload, ChatMessagePayload) + + # define completion api for user @web_ns.route("/completion-messages") class CompletionApi(WebApiResource): @web_ns.doc("Create Completion Message") @web_ns.doc(description="Create a completion message for text generation applications.") - @web_ns.doc( - params={ - "inputs": {"description": "Input variables for the completion", "type": "object", "required": True}, - "query": {"description": "Query text for completion", "type": "string", "required": False}, - "files": {"description": "Files to be processed", "type": "array", "required": False}, - "response_mode": { - "description": "Response mode: blocking or streaming", - "type": "string", - "enum": ["blocking", "streaming"], - "required": False, - }, - "retriever_from": {"description": "Source of retriever", "type": "string", "required": False}, - } - ) + @web_ns.expect(web_ns.models[CompletionMessagePayload.__name__]) @web_ns.doc( responses={ 200: "Success", @@ -67,18 +88,10 @@ class CompletionApi(WebApiResource): if app_model.mode != AppMode.COMPLETION: raise NotCompletionAppError() - parser = ( - reqparse.RequestParser() - .add_argument("inputs", type=dict, required=True, location="json") - .add_argument("query", type=str, location="json", default="") - .add_argument("files", type=list, required=False, location="json") - .add_argument("response_mode", type=str, choices=["blocking", "streaming"], location="json") - .add_argument("retriever_from", type=str, required=False, default="web_app", location="json") - ) + payload = CompletionMessagePayload.model_validate(web_ns.payload or {}) + args = payload.model_dump(exclude_none=True) - args = parser.parse_args() - - streaming = args["response_mode"] == "streaming" + streaming = payload.response_mode == "streaming" args["auto_generate_name"] = False try: @@ -142,22 +155,7 @@ class CompletionStopApi(WebApiResource): class ChatApi(WebApiResource): @web_ns.doc("Create Chat Message") @web_ns.doc(description="Create a chat message for conversational applications.") - @web_ns.doc( - params={ - "inputs": {"description": "Input variables for the chat", "type": "object", "required": True}, - "query": {"description": "User query/message", "type": "string", "required": True}, - "files": {"description": "Files to be processed", "type": "array", "required": False}, - "response_mode": { - "description": "Response mode: blocking or streaming", - "type": "string", - "enum": ["blocking", "streaming"], - "required": False, - }, - "conversation_id": {"description": "Conversation UUID", "type": "string", "required": False}, - "parent_message_id": {"description": "Parent message UUID", "type": "string", "required": False}, - "retriever_from": {"description": "Source of retriever", "type": "string", "required": False}, - } - ) + @web_ns.expect(web_ns.models[ChatMessagePayload.__name__]) @web_ns.doc( responses={ 200: "Success", @@ -173,20 +171,10 @@ class ChatApi(WebApiResource): if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}: raise NotChatAppError() - parser = ( - reqparse.RequestParser() - .add_argument("inputs", type=dict, required=True, location="json") - .add_argument("query", type=str, required=True, location="json") - .add_argument("files", type=list, required=False, location="json") - .add_argument("response_mode", type=str, choices=["blocking", "streaming"], location="json") - .add_argument("conversation_id", type=uuid_value, location="json") - .add_argument("parent_message_id", type=uuid_value, required=False, location="json") - .add_argument("retriever_from", type=str, required=False, default="web_app", location="json") - ) + payload = ChatMessagePayload.model_validate(web_ns.payload or {}) + args = payload.model_dump(exclude_none=True) - args = parser.parse_args() - - streaming = args["response_mode"] == "streaming" + streaming = payload.response_mode == "streaming" args["auto_generate_name"] = False try: diff --git a/api/controllers/web/forgot_password.py b/api/controllers/web/forgot_password.py index b9e391e049..690b76655f 100644 --- a/api/controllers/web/forgot_password.py +++ b/api/controllers/web/forgot_password.py @@ -2,10 +2,12 @@ import base64 import secrets from flask import request -from flask_restx import Resource, reqparse +from flask_restx import Resource +from pydantic import BaseModel, Field, field_validator from sqlalchemy import select from sqlalchemy.orm import Session +from controllers.common.schema import register_schema_models from controllers.console.auth.error import ( AuthenticationFailedError, EmailCodeError, @@ -18,14 +20,40 @@ from controllers.console.error import EmailSendIpLimitError from controllers.console.wraps import email_password_login_enabled, only_edition_enterprise, setup_required from controllers.web import web_ns from extensions.ext_database import db -from libs.helper import email, extract_remote_ip +from libs.helper import EmailStr, extract_remote_ip from libs.password import hash_password, valid_password from models import Account from services.account_service import AccountService +class ForgotPasswordSendPayload(BaseModel): + email: EmailStr + language: str | None = None + + +class ForgotPasswordCheckPayload(BaseModel): + email: EmailStr + code: str + token: str = Field(min_length=1) + + +class ForgotPasswordResetPayload(BaseModel): + token: str = Field(min_length=1) + new_password: str + password_confirm: str + + @field_validator("new_password", "password_confirm") + @classmethod + def validate_password(cls, value: str) -> str: + return valid_password(value) + + +register_schema_models(web_ns, ForgotPasswordSendPayload, ForgotPasswordCheckPayload, ForgotPasswordResetPayload) + + @web_ns.route("/forgot-password") class ForgotPasswordSendEmailApi(Resource): + @web_ns.expect(web_ns.models[ForgotPasswordSendPayload.__name__]) @only_edition_enterprise @setup_required @email_password_login_enabled @@ -40,35 +68,31 @@ class ForgotPasswordSendEmailApi(Resource): } ) def post(self): - parser = ( - reqparse.RequestParser() - .add_argument("email", type=email, required=True, location="json") - .add_argument("language", type=str, required=False, location="json") - ) - args = parser.parse_args() + payload = ForgotPasswordSendPayload.model_validate(web_ns.payload or {}) ip_address = extract_remote_ip(request) if AccountService.is_email_send_ip_limit(ip_address): raise EmailSendIpLimitError() - if args["language"] is not None and args["language"] == "zh-Hans": + if payload.language == "zh-Hans": language = "zh-Hans" else: language = "en-US" with Session(db.engine) as session: - account = session.execute(select(Account).filter_by(email=args["email"])).scalar_one_or_none() + account = session.execute(select(Account).filter_by(email=payload.email)).scalar_one_or_none() token = None if account is None: raise AuthenticationFailedError() else: - token = AccountService.send_reset_password_email(account=account, email=args["email"], language=language) + token = AccountService.send_reset_password_email(account=account, email=payload.email, language=language) return {"result": "success", "data": token} @web_ns.route("/forgot-password/validity") class ForgotPasswordCheckApi(Resource): + @web_ns.expect(web_ns.models[ForgotPasswordCheckPayload.__name__]) @only_edition_enterprise @setup_required @email_password_login_enabled @@ -78,45 +102,40 @@ class ForgotPasswordCheckApi(Resource): responses={200: "Token is valid", 400: "Bad request - invalid token format", 401: "Invalid or expired token"} ) def post(self): - parser = ( - reqparse.RequestParser() - .add_argument("email", type=str, required=True, location="json") - .add_argument("code", type=str, required=True, location="json") - .add_argument("token", type=str, required=True, nullable=False, location="json") - ) - args = parser.parse_args() + payload = ForgotPasswordCheckPayload.model_validate(web_ns.payload or {}) - user_email = args["email"] + user_email = payload.email - is_forgot_password_error_rate_limit = AccountService.is_forgot_password_error_rate_limit(args["email"]) + is_forgot_password_error_rate_limit = AccountService.is_forgot_password_error_rate_limit(payload.email) if is_forgot_password_error_rate_limit: raise EmailPasswordResetLimitError() - token_data = AccountService.get_reset_password_data(args["token"]) + token_data = AccountService.get_reset_password_data(payload.token) if token_data is None: raise InvalidTokenError() if user_email != token_data.get("email"): raise InvalidEmailError() - if args["code"] != token_data.get("code"): - AccountService.add_forgot_password_error_rate_limit(args["email"]) + if payload.code != token_data.get("code"): + AccountService.add_forgot_password_error_rate_limit(payload.email) raise EmailCodeError() # Verified, revoke the first token - AccountService.revoke_reset_password_token(args["token"]) + AccountService.revoke_reset_password_token(payload.token) # Refresh token data by generating a new token _, new_token = AccountService.generate_reset_password_token( - user_email, code=args["code"], additional_data={"phase": "reset"} + user_email, code=payload.code, additional_data={"phase": "reset"} ) - AccountService.reset_forgot_password_error_rate_limit(args["email"]) + AccountService.reset_forgot_password_error_rate_limit(payload.email) return {"is_valid": True, "email": token_data.get("email"), "token": new_token} @web_ns.route("/forgot-password/resets") class ForgotPasswordResetApi(Resource): + @web_ns.expect(web_ns.models[ForgotPasswordResetPayload.__name__]) @only_edition_enterprise @setup_required @email_password_login_enabled @@ -131,20 +150,14 @@ class ForgotPasswordResetApi(Resource): } ) def post(self): - parser = ( - reqparse.RequestParser() - .add_argument("token", type=str, required=True, nullable=False, location="json") - .add_argument("new_password", type=valid_password, required=True, nullable=False, location="json") - .add_argument("password_confirm", type=valid_password, required=True, nullable=False, location="json") - ) - args = parser.parse_args() + payload = ForgotPasswordResetPayload.model_validate(web_ns.payload or {}) # Validate passwords match - if args["new_password"] != args["password_confirm"]: + if payload.new_password != payload.password_confirm: raise PasswordMismatchError() # Validate token and get reset data - reset_data = AccountService.get_reset_password_data(args["token"]) + reset_data = AccountService.get_reset_password_data(payload.token) if not reset_data: raise InvalidTokenError() # Must use token in reset phase @@ -152,11 +165,11 @@ class ForgotPasswordResetApi(Resource): raise InvalidTokenError() # Revoke token to prevent reuse - AccountService.revoke_reset_password_token(args["token"]) + AccountService.revoke_reset_password_token(payload.token) # Generate secure salt and hash password salt = secrets.token_bytes(16) - password_hashed = hash_password(args["new_password"], salt) + password_hashed = hash_password(payload.new_password, salt) email = reset_data.get("email", "") @@ -170,7 +183,7 @@ class ForgotPasswordResetApi(Resource): return {"result": "success"} - def _update_existing_account(self, account, password_hashed, salt, session): + def _update_existing_account(self, account: Account, password_hashed, salt, session): # Update existing account credentials account.password = base64.b64encode(password_hashed).decode() account.password_salt = base64.b64encode(salt).decode() diff --git a/api/controllers/web/message.py b/api/controllers/web/message.py index afa935afa6..51ce024a5b 100644 --- a/api/controllers/web/message.py +++ b/api/controllers/web/message.py @@ -1,9 +1,12 @@ import logging +from typing import Literal -from flask_restx import fields, marshal_with, reqparse -from flask_restx.inputs import int_range +from flask import request +from flask_restx import fields, marshal_with +from pydantic import BaseModel, Field, field_validator from werkzeug.exceptions import InternalServerError, NotFound +from controllers.common.schema import register_schema_models from controllers.web import web_ns from controllers.web.error import ( AppMoreLikeThisDisabledError, @@ -38,6 +41,33 @@ from services.message_service import MessageService logger = logging.getLogger(__name__) +class MessageListQuery(BaseModel): + conversation_id: str = Field(description="Conversation UUID") + first_id: str | None = Field(default=None, description="First message ID for pagination") + limit: int = Field(default=20, ge=1, le=100, description="Number of messages to return (1-100)") + + @field_validator("conversation_id", "first_id") + @classmethod + def validate_uuid(cls, value: str | None) -> str | None: + if value is None: + return value + return uuid_value(value) + + +class MessageFeedbackPayload(BaseModel): + rating: Literal["like", "dislike"] | None = Field(default=None, description="Feedback rating") + content: str | None = Field(default=None, description="Feedback content") + + +class MessageMoreLikeThisQuery(BaseModel): + response_mode: Literal["blocking", "streaming"] = Field( + description="Response mode", + ) + + +register_schema_models(web_ns, MessageListQuery, MessageFeedbackPayload, MessageMoreLikeThisQuery) + + @web_ns.route("/messages") class MessageListApi(WebApiResource): message_fields = { @@ -69,7 +99,11 @@ class MessageListApi(WebApiResource): @web_ns.doc( params={ "conversation_id": {"description": "Conversation UUID", "type": "string", "required": True}, - "first_id": {"description": "First message ID for pagination", "type": "string", "required": False}, + "first_id": { + "description": "First message ID for pagination", + "type": "string", + "required": False, + }, "limit": { "description": "Number of messages to return (1-100)", "type": "integer", @@ -94,17 +128,12 @@ class MessageListApi(WebApiResource): if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}: raise NotChatAppError() - parser = ( - reqparse.RequestParser() - .add_argument("conversation_id", required=True, type=uuid_value, location="args") - .add_argument("first_id", type=uuid_value, location="args") - .add_argument("limit", type=int_range(1, 100), required=False, default=20, location="args") - ) - args = parser.parse_args() + raw_args = request.args.to_dict() + query = MessageListQuery.model_validate(raw_args) try: return MessageService.pagination_by_first_id( - app_model, end_user, args["conversation_id"], args["first_id"], args["limit"] + app_model, end_user, query.conversation_id, query.first_id, query.limit ) except ConversationNotExistsError: raise NotFound("Conversation Not Exists.") @@ -129,7 +158,7 @@ class MessageFeedbackApi(WebApiResource): "enum": ["like", "dislike"], "required": False, }, - "content": {"description": "Feedback content/comment", "type": "string", "required": False}, + "content": {"description": "Feedback content", "type": "string", "required": False}, } ) @web_ns.doc( @@ -146,20 +175,15 @@ class MessageFeedbackApi(WebApiResource): def post(self, app_model, end_user, message_id): message_id = str(message_id) - parser = ( - reqparse.RequestParser() - .add_argument("rating", type=str, choices=["like", "dislike", None], location="json") - .add_argument("content", type=str, location="json", default=None) - ) - args = parser.parse_args() + payload = MessageFeedbackPayload.model_validate(web_ns.payload or {}) try: MessageService.create_feedback( app_model=app_model, message_id=message_id, user=end_user, - rating=args.get("rating"), - content=args.get("content"), + rating=payload.rating, + content=payload.content, ) except MessageNotExistsError: raise NotFound("Message Not Exists.") @@ -171,17 +195,7 @@ class MessageFeedbackApi(WebApiResource): class MessageMoreLikeThisApi(WebApiResource): @web_ns.doc("Generate More Like This") @web_ns.doc(description="Generate a new completion similar to an existing message (completion apps only).") - @web_ns.doc( - params={ - "message_id": {"description": "Message UUID", "type": "string", "required": True}, - "response_mode": { - "description": "Response mode", - "type": "string", - "enum": ["blocking", "streaming"], - "required": True, - }, - } - ) + @web_ns.expect(web_ns.models[MessageMoreLikeThisQuery.__name__]) @web_ns.doc( responses={ 200: "Success", @@ -198,12 +212,10 @@ class MessageMoreLikeThisApi(WebApiResource): message_id = str(message_id) - parser = reqparse.RequestParser().add_argument( - "response_mode", type=str, required=True, choices=["blocking", "streaming"], location="args" - ) - args = parser.parse_args() + raw_args = request.args.to_dict() + query = MessageMoreLikeThisQuery.model_validate(raw_args) - streaming = args["response_mode"] == "streaming" + streaming = query.response_mode == "streaming" try: response = AppGenerateService.generate_more_like_this( diff --git a/api/controllers/web/remote_files.py b/api/controllers/web/remote_files.py index dac4b3da94..c1f976829f 100644 --- a/api/controllers/web/remote_files.py +++ b/api/controllers/web/remote_files.py @@ -1,7 +1,8 @@ import urllib.parse import httpx -from flask_restx import marshal_with, reqparse +from flask_restx import marshal_with +from pydantic import BaseModel, Field, HttpUrl import services from controllers.common import helpers @@ -10,14 +11,23 @@ from controllers.common.errors import ( RemoteFileUploadError, UnsupportedFileTypeError, ) -from controllers.web import web_ns -from controllers.web.wraps import WebApiResource from core.file import helpers as file_helpers from core.helper import ssrf_proxy from extensions.ext_database import db from fields.file_fields import build_file_with_signed_url_model, build_remote_file_info_model from services.file_service import FileService +from ..common.schema import register_schema_models +from . import web_ns +from .wraps import WebApiResource + + +class RemoteFileUploadPayload(BaseModel): + url: HttpUrl = Field(description="Remote file URL") + + +register_schema_models(web_ns, RemoteFileUploadPayload) + @web_ns.route("/remote-files/") class RemoteFileInfoApi(WebApiResource): @@ -97,10 +107,8 @@ class RemoteFileUploadApi(WebApiResource): FileTooLargeError: File exceeds size limit UnsupportedFileTypeError: File type not supported """ - parser = reqparse.RequestParser().add_argument("url", type=str, required=True, help="URL is required") - args = parser.parse_args() - - url = args["url"] + payload = RemoteFileUploadPayload.model_validate(web_ns.payload or {}) + url = str(payload.url) try: resp = ssrf_proxy.head(url=url) diff --git a/api/core/app/app_config/entities.py b/api/core/app/app_config/entities.py index 93f2742599..307af3747c 100644 --- a/api/core/app/app_config/entities.py +++ b/api/core/app/app_config/entities.py @@ -1,3 +1,4 @@ +import json from collections.abc import Sequence from enum import StrEnum, auto from typing import Any, Literal @@ -120,7 +121,7 @@ class VariableEntity(BaseModel): allowed_file_types: Sequence[FileType] | None = Field(default_factory=list) allowed_file_extensions: Sequence[str] | None = Field(default_factory=list) allowed_file_upload_methods: Sequence[FileTransferMethod] | None = Field(default_factory=list) - json_schema: dict[str, Any] | None = Field(default=None) + json_schema: str | None = Field(default=None) @field_validator("description", mode="before") @classmethod @@ -134,11 +135,17 @@ class VariableEntity(BaseModel): @field_validator("json_schema") @classmethod - def validate_json_schema(cls, schema: dict[str, Any] | None) -> dict[str, Any] | None: + def validate_json_schema(cls, schema: str | None) -> str | None: if schema is None: return None + try: - Draft7Validator.check_schema(schema) + json_schema = json.loads(schema) + except json.JSONDecodeError: + raise ValueError(f"invalid json_schema value {schema}") + + try: + Draft7Validator.check_schema(json_schema) except SchemaError as e: raise ValueError(f"Invalid JSON schema: {e.message}") return schema diff --git a/api/core/app/apps/base_app_generator.py b/api/core/app/apps/base_app_generator.py index 1b0474142e..a6aace168e 100644 --- a/api/core/app/apps/base_app_generator.py +++ b/api/core/app/apps/base_app_generator.py @@ -1,3 +1,4 @@ +import json from collections.abc import Generator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Union, final @@ -104,8 +105,9 @@ class BaseAppGenerator: variable_entity.type in {VariableEntityType.FILE, VariableEntityType.FILE_LIST} and not variable_entity.required ): - # Treat empty string (frontend default) or empty list as unset - if not value and isinstance(value, (str, list)): + # Treat empty string (frontend default) as unset + # For FILE_LIST, allow empty list [] to pass through + if isinstance(value, str) and not value: return None if variable_entity.type in { @@ -175,6 +177,13 @@ class BaseAppGenerator: value = True elif value == 0: value = False + case VariableEntityType.JSON_OBJECT: + if not isinstance(value, str): + raise ValueError(f"{variable_entity.variable} in input form must be a string") + try: + json.loads(value) + except json.JSONDecodeError: + raise ValueError(f"{variable_entity.variable} in input form must be a valid JSON object") case _: raise AssertionError("this statement should be unreachable.") diff --git a/api/core/app/apps/base_app_queue_manager.py b/api/core/app/apps/base_app_queue_manager.py index 698eee9894..b41bedbea4 100644 --- a/api/core/app/apps/base_app_queue_manager.py +++ b/api/core/app/apps/base_app_queue_manager.py @@ -90,6 +90,7 @@ class AppQueueManager: """ self._clear_task_belong_cache() self._q.put(None) + self._graph_runtime_state = None # Release reference to allow GC to reclaim memory def _clear_task_belong_cache(self) -> None: """ diff --git a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py index c4ea428270..a1852ffe19 100644 --- a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py +++ b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py @@ -345,9 +345,11 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline): self._task_state.llm_result.message.content = current_content if isinstance(event, QueueLLMChunkEvent): + event_type = self._message_cycle_manager.get_message_event_type(message_id=self._message_id) yield self._message_cycle_manager.message_to_stream_response( answer=cast(str, delta_text), message_id=self._message_id, + event_type=event_type, ) else: yield self._agent_message_to_stream_response( diff --git a/api/core/app/task_pipeline/message_cycle_manager.py b/api/core/app/task_pipeline/message_cycle_manager.py index 25c3a99cf8..f2fa3c064b 100644 --- a/api/core/app/task_pipeline/message_cycle_manager.py +++ b/api/core/app/task_pipeline/message_cycle_manager.py @@ -5,7 +5,7 @@ from threading import Thread from typing import Union from flask import Flask, current_app -from sqlalchemy import select +from sqlalchemy import exists, select from sqlalchemy.orm import Session from configs import dify_config @@ -54,6 +54,20 @@ class MessageCycleManager: ): self._application_generate_entity = application_generate_entity self._task_state = task_state + self._message_has_file: set[str] = set() + + def get_message_event_type(self, message_id: str) -> StreamEvent: + if message_id in self._message_has_file: + return StreamEvent.MESSAGE_FILE + + with Session(db.engine, expire_on_commit=False) as session: + has_file = session.query(exists().where(MessageFile.message_id == message_id)).scalar() + + if has_file: + self._message_has_file.add(message_id) + return StreamEvent.MESSAGE_FILE + + return StreamEvent.MESSAGE def generate_conversation_name(self, *, conversation_id: str, query: str) -> Thread | None: """ @@ -224,6 +238,7 @@ class MessageCycleManager: tool_arguments: str | None = None, tool_files: list[str] | None = None, tool_error: str | None = None, + event_type: StreamEvent | None = None, ) -> MessageStreamResponse: """ Message to stream response. @@ -238,22 +253,18 @@ class MessageCycleManager: :param tool_error: error message if tool failed :return: """ - with Session(db.engine, expire_on_commit=False) as session: - message_file = session.scalar(select(MessageFile).where(MessageFile.id == message_id)) - event_type = StreamEvent.MESSAGE_FILE if message_file else StreamEvent.MESSAGE - return MessageStreamResponse( task_id=self._application_generate_entity.task_id, id=message_id, answer=answer, from_variable_selector=from_variable_selector, - event=event_type, chunk_type=chunk_type, tool_call_id=tool_call_id, tool_name=tool_name, tool_arguments=tool_arguments, tool_files=tool_files, tool_error=tool_error, + event=event_type or StreamEvent.MESSAGE, ) def message_replace_to_stream_response(self, answer: str, reason: str = "") -> MessageReplaceStreamResponse: diff --git a/api/core/helper/code_executor/jinja2/jinja2_transformer.py b/api/core/helper/code_executor/jinja2/jinja2_transformer.py index 969125d2f7..5e4807401e 100644 --- a/api/core/helper/code_executor/jinja2/jinja2_transformer.py +++ b/api/core/helper/code_executor/jinja2/jinja2_transformer.py @@ -1,9 +1,14 @@ +from collections.abc import Mapping from textwrap import dedent +from typing import Any from core.helper.code_executor.template_transformer import TemplateTransformer class Jinja2TemplateTransformer(TemplateTransformer): + # Use separate placeholder for base64-encoded template to avoid confusion + _template_b64_placeholder: str = "{{template_b64}}" + @classmethod def transform_response(cls, response: str): """ @@ -13,18 +18,35 @@ class Jinja2TemplateTransformer(TemplateTransformer): """ return {"result": cls.extract_result_str_from_response(response)} + @classmethod + def assemble_runner_script(cls, code: str, inputs: Mapping[str, Any]) -> str: + """ + Override base class to use base64 encoding for template code. + This prevents issues with special characters (quotes, newlines) in templates + breaking the generated Python script. Fixes #26818. + """ + script = cls.get_runner_script() + # Encode template as base64 to safely embed any content including quotes + code_b64 = cls.serialize_code(code) + script = script.replace(cls._template_b64_placeholder, code_b64) + inputs_str = cls.serialize_inputs(inputs) + script = script.replace(cls._inputs_placeholder, inputs_str) + return script + @classmethod def get_runner_script(cls) -> str: runner_script = dedent(f""" - # declare main function - def main(**inputs): - import jinja2 - template = jinja2.Template('''{cls._code_placeholder}''') - return template.render(**inputs) - + import jinja2 import json from base64 import b64decode + # declare main function + def main(**inputs): + # Decode base64-encoded template to handle special characters safely + template_code = b64decode('{cls._template_b64_placeholder}').decode('utf-8') + template = jinja2.Template(template_code) + return template.render(**inputs) + # decode and prepare input dict inputs_obj = json.loads(b64decode('{cls._inputs_placeholder}').decode('utf-8')) diff --git a/api/core/helper/code_executor/template_transformer.py b/api/core/helper/code_executor/template_transformer.py index 3965f8cb31..6fda073913 100644 --- a/api/core/helper/code_executor/template_transformer.py +++ b/api/core/helper/code_executor/template_transformer.py @@ -13,6 +13,15 @@ class TemplateTransformer(ABC): _inputs_placeholder: str = "{{inputs}}" _result_tag: str = "<>" + @classmethod + def serialize_code(cls, code: str) -> str: + """ + Serialize template code to base64 to safely embed in generated script. + This prevents issues with special characters like quotes breaking the script. + """ + code_bytes = code.encode("utf-8") + return b64encode(code_bytes).decode("utf-8") + @classmethod def transform_caller(cls, code: str, inputs: Mapping[str, Any]) -> tuple[str, str]: """ diff --git a/api/core/helper/ssrf_proxy.py b/api/core/helper/ssrf_proxy.py index 6c98aea1be..0b36969cf9 100644 --- a/api/core/helper/ssrf_proxy.py +++ b/api/core/helper/ssrf_proxy.py @@ -72,6 +72,22 @@ def _get_ssrf_client(ssl_verify_enabled: bool) -> httpx.Client: ) +def _get_user_provided_host_header(headers: dict | None) -> str | None: + """ + Extract the user-provided Host header from the headers dict. + + This is needed because when using a forward proxy, httpx may override the Host header. + We preserve the user's explicit Host header to support virtual hosting and other use cases. + """ + if not headers: + return None + # Case-insensitive lookup for Host header + for key, value in headers.items(): + if key.lower() == "host": + return value + return None + + def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs): if "allow_redirects" in kwargs: allow_redirects = kwargs.pop("allow_redirects") @@ -90,10 +106,24 @@ def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs): verify_option = kwargs.pop("ssl_verify", dify_config.HTTP_REQUEST_NODE_SSL_VERIFY) client = _get_ssrf_client(verify_option) + # Preserve user-provided Host header + # When using a forward proxy, httpx may override the Host header based on the URL. + # We extract and preserve any explicitly set Host header to support virtual hosting. + headers = kwargs.get("headers", {}) + user_provided_host = _get_user_provided_host_header(headers) + retries = 0 while retries <= max_retries: try: + # Build the request manually to preserve the Host header + # httpx may override the Host header when using a proxy, so we use + # the request API to explicitly set headers before sending + headers = {k: v for k, v in headers.items() if k.lower() != "host"} + if user_provided_host is not None: + headers["host"] = user_provided_host + kwargs["headers"] = headers response = client.request(method=method, url=url, **kwargs) + # Check for SSRF protection by Squid proxy if response.status_code in (401, 403): # Check if this is a Squid SSRF rejection diff --git a/api/core/helper/tool_provider_cache.py b/api/core/helper/tool_provider_cache.py deleted file mode 100644 index eef5937407..0000000000 --- a/api/core/helper/tool_provider_cache.py +++ /dev/null @@ -1,56 +0,0 @@ -import json -import logging -from typing import Any - -from core.tools.entities.api_entities import ToolProviderTypeApiLiteral -from extensions.ext_redis import redis_client, redis_fallback - -logger = logging.getLogger(__name__) - - -class ToolProviderListCache: - """Cache for tool provider lists""" - - CACHE_TTL = 300 # 5 minutes - - @staticmethod - def _generate_cache_key(tenant_id: str, typ: ToolProviderTypeApiLiteral = None) -> str: - """Generate cache key for tool providers list""" - type_filter = typ or "all" - return f"tool_providers:tenant_id:{tenant_id}:type:{type_filter}" - - @staticmethod - @redis_fallback(default_return=None) - def get_cached_providers(tenant_id: str, typ: ToolProviderTypeApiLiteral = None) -> list[dict[str, Any]] | None: - """Get cached tool providers""" - cache_key = ToolProviderListCache._generate_cache_key(tenant_id, typ) - cached_data = redis_client.get(cache_key) - if cached_data: - try: - return json.loads(cached_data.decode("utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError): - logger.warning("Failed to decode cached tool providers data") - return None - return None - - @staticmethod - @redis_fallback() - def set_cached_providers(tenant_id: str, typ: ToolProviderTypeApiLiteral, providers: list[dict[str, Any]]): - """Cache tool providers""" - cache_key = ToolProviderListCache._generate_cache_key(tenant_id, typ) - redis_client.setex(cache_key, ToolProviderListCache.CACHE_TTL, json.dumps(providers)) - - @staticmethod - @redis_fallback() - def invalidate_cache(tenant_id: str, typ: ToolProviderTypeApiLiteral = None): - """Invalidate cache for tool providers""" - if typ: - # Invalidate specific type cache - cache_key = ToolProviderListCache._generate_cache_key(tenant_id, typ) - redis_client.delete(cache_key) - else: - # Invalidate all caches for this tenant - pattern = f"tool_providers:tenant_id:{tenant_id}:*" - keys = list(redis_client.scan_iter(pattern)) - if keys: - redis_client.delete(*keys) diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 59de4f403d..f1b50f360b 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -396,7 +396,7 @@ class IndexingRunner: datasource_type=DatasourceType.NOTION, notion_info=NotionInfo.model_validate( { - "credential_id": data_source_info["credential_id"], + "credential_id": data_source_info.get("credential_id"), "notion_workspace_id": data_source_info["notion_workspace_id"], "notion_obj_id": data_source_info["notion_page_id"], "notion_page_type": data_source_info["type"], diff --git a/api/core/mcp/auth/auth_flow.py b/api/core/mcp/auth/auth_flow.py index 92787b39dd..aef1afb235 100644 --- a/api/core/mcp/auth/auth_flow.py +++ b/api/core/mcp/auth/auth_flow.py @@ -47,7 +47,11 @@ def build_protected_resource_metadata_discovery_urls( """ Build a list of URLs to try for Protected Resource Metadata discovery. - Per SEP-985, supports fallback when discovery fails at one URL. + Per RFC 9728 Section 5.1, supports fallback when discovery fails at one URL. + Priority order: + 1. URL from WWW-Authenticate header (if provided) + 2. Well-known URI with path: https://example.com/.well-known/oauth-protected-resource/public/mcp + 3. Well-known URI at root: https://example.com/.well-known/oauth-protected-resource """ urls = [] @@ -58,9 +62,18 @@ def build_protected_resource_metadata_discovery_urls( # Fallback: construct from server URL parsed = urlparse(server_url) base_url = f"{parsed.scheme}://{parsed.netloc}" - fallback_url = urljoin(base_url, "/.well-known/oauth-protected-resource") - if fallback_url not in urls: - urls.append(fallback_url) + path = parsed.path.rstrip("/") + + # Priority 2: With path insertion (e.g., /.well-known/oauth-protected-resource/public/mcp) + if path: + path_url = f"{base_url}/.well-known/oauth-protected-resource{path}" + if path_url not in urls: + urls.append(path_url) + + # Priority 3: At root (e.g., /.well-known/oauth-protected-resource) + root_url = f"{base_url}/.well-known/oauth-protected-resource" + if root_url not in urls: + urls.append(root_url) return urls @@ -71,30 +84,34 @@ def build_oauth_authorization_server_metadata_discovery_urls(auth_server_url: st Supports both OAuth 2.0 (RFC 8414) and OpenID Connect discovery. - Per RFC 8414 section 3: - - If issuer has no path: https://example.com/.well-known/oauth-authorization-server - - If issuer has path: https://example.com/.well-known/oauth-authorization-server{path} - - Example: - - issuer: https://example.com/oauth - - metadata: https://example.com/.well-known/oauth-authorization-server/oauth + Per RFC 8414 section 3.1 and section 5, try all possible endpoints: + - OAuth 2.0 with path insertion: https://example.com/.well-known/oauth-authorization-server/tenant1 + - OpenID Connect with path insertion: https://example.com/.well-known/openid-configuration/tenant1 + - OpenID Connect path appending: https://example.com/tenant1/.well-known/openid-configuration + - OAuth 2.0 at root: https://example.com/.well-known/oauth-authorization-server + - OpenID Connect at root: https://example.com/.well-known/openid-configuration """ urls = [] base_url = auth_server_url or server_url parsed = urlparse(base_url) base = f"{parsed.scheme}://{parsed.netloc}" - path = parsed.path.rstrip("/") # Remove trailing slash + path = parsed.path.rstrip("/") + # OAuth 2.0 Authorization Server Metadata at root (MCP-03-26) + urls.append(f"{base}/.well-known/oauth-authorization-server") - # Try OpenID Connect discovery first (more common) - urls.append(urljoin(base + "/", ".well-known/openid-configuration")) + # OpenID Connect Discovery at root + urls.append(f"{base}/.well-known/openid-configuration") - # OAuth 2.0 Authorization Server Metadata (RFC 8414) - # Include the path component if present in the issuer URL if path: - urls.append(urljoin(base, f".well-known/oauth-authorization-server{path}")) - else: - urls.append(urljoin(base, ".well-known/oauth-authorization-server")) + # OpenID Connect Discovery with path insertion + urls.append(f"{base}/.well-known/openid-configuration{path}") + + # OpenID Connect Discovery path appending + urls.append(f"{base}{path}/.well-known/openid-configuration") + + # OAuth 2.0 Authorization Server Metadata with path insertion + urls.append(f"{base}/.well-known/oauth-authorization-server{path}") return urls diff --git a/api/core/mcp/client/sse_client.py b/api/core/mcp/client/sse_client.py index 24ca59ee45..1de1d5a073 100644 --- a/api/core/mcp/client/sse_client.py +++ b/api/core/mcp/client/sse_client.py @@ -61,6 +61,7 @@ class SSETransport: self.timeout = timeout self.sse_read_timeout = sse_read_timeout self.endpoint_url: str | None = None + self.event_source: EventSource | None = None def _validate_endpoint_url(self, endpoint_url: str) -> bool: """Validate that the endpoint URL matches the connection origin. @@ -237,6 +238,9 @@ class SSETransport: write_queue: WriteQueue = queue.Queue() status_queue: StatusQueue = queue.Queue() + # Store event_source for graceful shutdown + self.event_source = event_source + # Start SSE reader thread executor.submit(self.sse_reader, event_source, read_queue, status_queue) @@ -296,6 +300,13 @@ def sse_client( logger.exception("Error connecting to SSE endpoint") raise finally: + # Close the SSE connection to unblock the reader thread + if transport.event_source is not None: + try: + transport.event_source.response.close() + except RuntimeError: + pass + # Clean up queues if read_queue: read_queue.put(None) diff --git a/api/core/mcp/client/streamable_client.py b/api/core/mcp/client/streamable_client.py index 805c16c838..5c3cd0d8f8 100644 --- a/api/core/mcp/client/streamable_client.py +++ b/api/core/mcp/client/streamable_client.py @@ -8,6 +8,7 @@ and session management. import logging import queue +import threading from collections.abc import Callable, Generator from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager @@ -103,6 +104,9 @@ class StreamableHTTPTransport: CONTENT_TYPE: JSON, **self.headers, } + self.stop_event = threading.Event() + self._active_responses: list[httpx.Response] = [] + self._lock = threading.Lock() def _update_headers_with_session(self, base_headers: dict[str, str]) -> dict[str, str]: """Update headers with session ID if available.""" @@ -111,6 +115,30 @@ class StreamableHTTPTransport: headers[MCP_SESSION_ID] = self.session_id return headers + def _register_response(self, response: httpx.Response): + """Register a response for cleanup on shutdown.""" + with self._lock: + self._active_responses.append(response) + + def _unregister_response(self, response: httpx.Response): + """Unregister a response after it's closed.""" + with self._lock: + try: + self._active_responses.remove(response) + except ValueError as e: + logger.debug("Ignoring error during response unregister: %s", e) + + def close_active_responses(self): + """Close all active SSE connections to unblock threads.""" + with self._lock: + responses_to_close = list(self._active_responses) + self._active_responses.clear() + for response in responses_to_close: + try: + response.close() + except RuntimeError as e: + logger.debug("Ignoring error during active response close: %s", e) + def _is_initialization_request(self, message: JSONRPCMessage) -> bool: """Check if the message is an initialization request.""" return isinstance(message.root, JSONRPCRequest) and message.root.method == "initialize" @@ -195,11 +223,21 @@ class StreamableHTTPTransport: event_source.response.raise_for_status() logger.debug("GET SSE connection established") - for sse in event_source.iter_sse(): - self._handle_sse_event(sse, server_to_client_queue) + # Register response for cleanup + self._register_response(event_source.response) + + try: + for sse in event_source.iter_sse(): + if self.stop_event.is_set(): + logger.debug("GET stream received stop signal") + break + self._handle_sse_event(sse, server_to_client_queue) + finally: + self._unregister_response(event_source.response) except Exception as exc: - logger.debug("GET stream error (non-fatal): %s", exc) + if not self.stop_event.is_set(): + logger.debug("GET stream error (non-fatal): %s", exc) def _handle_resumption_request(self, ctx: RequestContext): """Handle a resumption request using GET with SSE.""" @@ -224,15 +262,24 @@ class StreamableHTTPTransport: event_source.response.raise_for_status() logger.debug("Resumption GET SSE connection established") - for sse in event_source.iter_sse(): - is_complete = self._handle_sse_event( - sse, - ctx.server_to_client_queue, - original_request_id, - ctx.metadata.on_resumption_token_update if ctx.metadata else None, - ) - if is_complete: - break + # Register response for cleanup + self._register_response(event_source.response) + + try: + for sse in event_source.iter_sse(): + if self.stop_event.is_set(): + logger.debug("Resumption stream received stop signal") + break + is_complete = self._handle_sse_event( + sse, + ctx.server_to_client_queue, + original_request_id, + ctx.metadata.on_resumption_token_update if ctx.metadata else None, + ) + if is_complete: + break + finally: + self._unregister_response(event_source.response) def _handle_post_request(self, ctx: RequestContext): """Handle a POST request with response processing.""" @@ -266,17 +313,20 @@ class StreamableHTTPTransport: if is_initialization: self._maybe_extract_session_id_from_response(response) - content_type = cast(str, response.headers.get(CONTENT_TYPE, "").lower()) + # Per https://modelcontextprotocol.io/specification/2025-06-18/basic#notifications: + # The server MUST NOT send a response to notifications. + if isinstance(message.root, JSONRPCRequest): + content_type = cast(str, response.headers.get(CONTENT_TYPE, "").lower()) - if content_type.startswith(JSON): - self._handle_json_response(response, ctx.server_to_client_queue) - elif content_type.startswith(SSE): - self._handle_sse_response(response, ctx) - else: - self._handle_unexpected_content_type( - content_type, - ctx.server_to_client_queue, - ) + if content_type.startswith(JSON): + self._handle_json_response(response, ctx.server_to_client_queue) + elif content_type.startswith(SSE): + self._handle_sse_response(response, ctx) + else: + self._handle_unexpected_content_type( + content_type, + ctx.server_to_client_queue, + ) def _handle_json_response( self, @@ -295,17 +345,27 @@ class StreamableHTTPTransport: def _handle_sse_response(self, response: httpx.Response, ctx: RequestContext): """Handle SSE response from the server.""" try: + # Register response for cleanup + self._register_response(response) + event_source = EventSource(response) - for sse in event_source.iter_sse(): - is_complete = self._handle_sse_event( - sse, - ctx.server_to_client_queue, - resumption_callback=(ctx.metadata.on_resumption_token_update if ctx.metadata else None), - ) - if is_complete: - break + try: + for sse in event_source.iter_sse(): + if self.stop_event.is_set(): + logger.debug("SSE response stream received stop signal") + break + is_complete = self._handle_sse_event( + sse, + ctx.server_to_client_queue, + resumption_callback=(ctx.metadata.on_resumption_token_update if ctx.metadata else None), + ) + if is_complete: + break + finally: + self._unregister_response(response) except Exception as e: - ctx.server_to_client_queue.put(e) + if not self.stop_event.is_set(): + ctx.server_to_client_queue.put(e) def _handle_unexpected_content_type( self, @@ -345,6 +405,11 @@ class StreamableHTTPTransport: """ while True: try: + # Check if we should stop + if self.stop_event.is_set(): + logger.debug("Post writer received stop signal") + break + # Read message from client queue with timeout to check stop_event periodically session_message = client_to_server_queue.get(timeout=DEFAULT_QUEUE_READ_TIMEOUT) if session_message is None: @@ -381,7 +446,8 @@ class StreamableHTTPTransport: except queue.Empty: continue except Exception as exc: - server_to_client_queue.put(exc) + if not self.stop_event.is_set(): + server_to_client_queue.put(exc) def terminate_session(self, client: httpx.Client): """Terminate the session by sending a DELETE request.""" @@ -465,6 +531,12 @@ def streamablehttp_client( transport.get_session_id, ) finally: + # Set stop event to signal all threads to stop + transport.stop_event.set() + + # Close all active SSE connections to unblock threads + transport.close_active_responses() + if transport.session_id and terminate_on_close: transport.terminate_session(client) diff --git a/api/core/mcp/mcp_client.py b/api/core/mcp/mcp_client.py index b0e0dab9be..2b0645b558 100644 --- a/api/core/mcp/mcp_client.py +++ b/api/core/mcp/mcp_client.py @@ -59,7 +59,7 @@ class MCPClient: try: logger.debug("Not supported method %s found in URL path, trying default 'mcp' method.", method_name) self.connect_server(sse_client, "sse") - except MCPConnectionError: + except (MCPConnectionError, ValueError): logger.debug("MCP connection failed with 'sse', falling back to 'mcp' method.") self.connect_server(streamablehttp_client, "mcp") diff --git a/api/core/model_runtime/README.md b/api/core/model_runtime/README.md index a6caa7eb1e..b9d2c55210 100644 --- a/api/core/model_runtime/README.md +++ b/api/core/model_runtime/README.md @@ -18,34 +18,20 @@ This module provides the interface for invoking and authenticating various model - Model provider display - ![image-20231210143654461](./docs/en_US/images/index/image-20231210143654461.png) - - Displays a list of all supported providers, including provider names, icons, supported model types list, predefined model list, configuration method, and credentials form rules, etc. For detailed rule design, see: [Schema](./docs/en_US/schema.md). + Displays a list of all supported providers, including provider names, icons, supported model types list, predefined model list, configuration method, and credentials form rules, etc. - Selectable model list display - ![image-20231210144229650](./docs/en_US/images/index/image-20231210144229650.png) - After configuring provider/model credentials, the dropdown (application orchestration interface/default model) allows viewing of the available LLM list. Greyed out items represent predefined model lists from providers without configured credentials, facilitating user review of supported models. - In addition, this list also returns configurable parameter information and rules for LLM, as shown below: - - ![image-20231210144814617](./docs/en_US/images/index/image-20231210144814617.png) - - These parameters are all defined in the backend, allowing different settings for various parameters supported by different models, as detailed in: [Schema](./docs/en_US/schema.md#ParameterRule). + In addition, this list also returns configurable parameter information and rules for LLM. These parameters are all defined in the backend, allowing different settings for various parameters supported by different models. - Provider/model credential authentication - ![image-20231210151548521](./docs/en_US/images/index/image-20231210151548521.png) - - ![image-20231210151628992](./docs/en_US/images/index/image-20231210151628992.png) - - The provider list returns configuration information for the credentials form, which can be authenticated through Runtime's interface. The first image above is a provider credential DEMO, and the second is a model credential DEMO. + The provider list returns configuration information for the credentials form, which can be authenticated through Runtime's interface. ## Structure -![](./docs/en_US/images/index/image-20231210165243632.png) - Model Runtime is divided into three layers: - The outermost layer is the factory method @@ -60,9 +46,6 @@ Model Runtime is divided into three layers: It offers direct invocation of various model types, predefined model configuration information, getting predefined/remote model lists, model credential authentication methods. Different models provide additional special methods, like LLM's pre-computed tokens method, cost information obtaining method, etc., **allowing horizontal expansion** for different models under the same provider (within supported model types). -## Next Steps +## Documentation -- Add new provider configuration: [Link](./docs/en_US/provider_scale_out.md) -- Add new models for existing providers: [Link](./docs/en_US/provider_scale_out.md#AddModel) -- View YAML configuration rules: [Link](./docs/en_US/schema.md) -- Implement interface methods: [Link](./docs/en_US/interfaces.md) +For detailed documentation on how to add new providers or models, please refer to the [Dify documentation](https://docs.dify.ai/). diff --git a/api/core/model_runtime/README_CN.md b/api/core/model_runtime/README_CN.md index dfe614347a..0a8b56b3fe 100644 --- a/api/core/model_runtime/README_CN.md +++ b/api/core/model_runtime/README_CN.md @@ -18,34 +18,20 @@ - 模型供应商展示 - ![image-20231210143654461](./docs/zh_Hans/images/index/image-20231210143654461.png) - -​ 展示所有已支持的供应商列表,除了返回供应商名称、图标之外,还提供了支持的模型类型列表,预定义模型列表、配置方式以及配置凭据的表单规则等等,规则设计详见:[Schema](./docs/zh_Hans/schema.md)。 + 展示所有已支持的供应商列表,除了返回供应商名称、图标之外,还提供了支持的模型类型列表,预定义模型列表、配置方式以及配置凭据的表单规则等等。 - 可选择的模型列表展示 - ![image-20231210144229650](./docs/zh_Hans/images/index/image-20231210144229650.png) + 配置供应商/模型凭据后,可在此下拉(应用编排界面/默认模型)查看可用的 LLM 列表,其中灰色的为未配置凭据供应商的预定义模型列表,方便用户查看已支持的模型。 -​ 配置供应商/模型凭据后,可在此下拉(应用编排界面/默认模型)查看可用的 LLM 列表,其中灰色的为未配置凭据供应商的预定义模型列表,方便用户查看已支持的模型。 - -​ 除此之外,该列表还返回了 LLM 可配置的参数信息和规则,如下图: - -​ ![image-20231210144814617](./docs/zh_Hans/images/index/image-20231210144814617.png) - -​ 这里的参数均为后端定义,相比之前只有 5 种固定参数,这里可为不同模型设置所支持的各种参数,详见:[Schema](./docs/zh_Hans/schema.md#ParameterRule)。 + 除此之外,该列表还返回了 LLM 可配置的参数信息和规则。这里的参数均为后端定义,相比之前只有 5 种固定参数,这里可为不同模型设置所支持的各种参数。 - 供应商/模型凭据鉴权 - ![image-20231210151548521](./docs/zh_Hans/images/index/image-20231210151548521.png) - -![image-20231210151628992](./docs/zh_Hans/images/index/image-20231210151628992.png) - -​ 供应商列表返回了凭据表单的配置信息,可通过 Runtime 提供的接口对凭据进行鉴权,上图 1 为供应商凭据 DEMO,上图 2 为模型凭据 DEMO。 + 供应商列表返回了凭据表单的配置信息,可通过 Runtime 提供的接口对凭据进行鉴权。 ## 结构 -![](./docs/zh_Hans/images/index/image-20231210165243632.png) - Model Runtime 分三层: - 最外层为工厂方法 @@ -59,8 +45,7 @@ Model Runtime 分三层: 对于供应商/模型凭据,有两种情况 - 如 OpenAI 这类中心化供应商,需要定义如**api_key**这类的鉴权凭据 - - 如[**Xinference**](https://github.com/xorbitsai/inference)这类本地部署的供应商,需要定义如**server_url**这类的地址凭据,有时候还需要定义**model_uid**之类的模型类型凭据,就像下面这样,当在供应商层定义了这些凭据后,就可以在前端页面上直接展示,无需修改前端逻辑。 - ![Alt text](docs/zh_Hans/images/index/image.png) + - 如[**Xinference**](https://github.com/xorbitsai/inference)这类本地部署的供应商,需要定义如**server_url**这类的地址凭据,有时候还需要定义**model_uid**之类的模型类型凭据。当在供应商层定义了这些凭据后,就可以在前端页面上直接展示,无需修改前端逻辑。 当配置好凭据后,就可以通过 DifyRuntime 的外部接口直接获取到对应供应商所需要的**Schema**(凭据表单规则),从而在可以在不修改前端逻辑的情况下,提供新的供应商/模型的支持。 @@ -74,20 +59,6 @@ Model Runtime 分三层: - 模型凭据 (**在供应商层定义**):这是一类不经常变动,一般在配置好后就不会再变动的参数,如 **api_key**、**server_url** 等。在 DifyRuntime 中,他们的参数名一般为**credentials: dict[str, any]**,Provider 层的 credentials 会直接被传递到这一层,不需要再单独定义。 -## 下一步 +## 文档 -### [增加新的供应商配置 👈🏻](./docs/zh_Hans/provider_scale_out.md) - -当添加后,这里将会出现一个新的供应商 - -![Alt text](docs/zh_Hans/images/index/image-1.png) - -### [为已存在的供应商新增模型 👈🏻](./docs/zh_Hans/provider_scale_out.md#%E5%A2%9E%E5%8A%A0%E6%A8%A1%E5%9E%8B) - -当添加后,对应供应商的模型列表中将会出现一个新的预定义模型供用户选择,如 GPT-3.5 GPT-4 ChatGLM3-6b 等,而对于支持自定义模型的供应商,则不需要新增模型。 - -![Alt text](docs/zh_Hans/images/index/image-2.png) - -### [接口的具体实现 👈🏻](./docs/zh_Hans/interfaces.md) - -你可以在这里找到你想要查看的接口的具体实现,以及接口的参数和返回值的具体含义。 +有关如何添加新供应商或模型的详细文档,请参阅 [Dify 文档](https://docs.dify.ai/)。 diff --git a/api/core/ops/utils.py b/api/core/ops/utils.py index c00f785034..631e3b77b2 100644 --- a/api/core/ops/utils.py +++ b/api/core/ops/utils.py @@ -54,7 +54,7 @@ def generate_dotted_order(run_id: str, start_time: Union[str, datetime], parent_ generate dotted_order for langsmith """ start_time = datetime.fromisoformat(start_time) if isinstance(start_time, str) else start_time - timestamp = start_time.strftime("%Y%m%dT%H%M%S%f")[:-3] + "Z" + timestamp = start_time.strftime("%Y%m%dT%H%M%S%f") + "Z" current_segment = f"{timestamp}{run_id}" if parent_dotted_order is None: diff --git a/api/core/plugin/entities/parameters.py b/api/core/plugin/entities/parameters.py index 88a3a7bd43..bfa662b9f6 100644 --- a/api/core/plugin/entities/parameters.py +++ b/api/core/plugin/entities/parameters.py @@ -76,7 +76,7 @@ class PluginParameter(BaseModel): auto_generate: PluginParameterAutoGenerate | None = None template: PluginParameterTemplate | None = None required: bool = False - default: Union[float, int, str, bool] | None = None + default: Union[float, int, str, bool, list, dict] | None = None min: Union[float, int] | None = None max: Union[float, int] | None = None precision: int | None = None diff --git a/api/core/plugin/impl/base.py b/api/core/plugin/impl/base.py index a1c84bd5d9..7bb2749afa 100644 --- a/api/core/plugin/impl/base.py +++ b/api/core/plugin/impl/base.py @@ -39,7 +39,7 @@ from core.trigger.errors import ( plugin_daemon_inner_api_baseurl = URL(str(dify_config.PLUGIN_DAEMON_URL)) _plugin_daemon_timeout_config = cast( float | httpx.Timeout | None, - getattr(dify_config, "PLUGIN_DAEMON_TIMEOUT", 300.0), + getattr(dify_config, "PLUGIN_DAEMON_TIMEOUT", 600.0), ) plugin_daemon_request_timeout: httpx.Timeout | None if _plugin_daemon_timeout_config is None: diff --git a/api/core/rag/datasource/keyword/jieba/jieba.py b/api/core/rag/datasource/keyword/jieba/jieba.py index 97052717db..0f19ecadc8 100644 --- a/api/core/rag/datasource/keyword/jieba/jieba.py +++ b/api/core/rag/datasource/keyword/jieba/jieba.py @@ -90,13 +90,17 @@ class Jieba(BaseKeyword): sorted_chunk_indices = self._retrieve_ids_by_query(keyword_table or {}, query, k) documents = [] + + segment_query_stmt = db.session.query(DocumentSegment).where( + DocumentSegment.dataset_id == self.dataset.id, DocumentSegment.index_node_id.in_(sorted_chunk_indices) + ) + if document_ids_filter: + segment_query_stmt = segment_query_stmt.where(DocumentSegment.document_id.in_(document_ids_filter)) + + segments = db.session.execute(segment_query_stmt).scalars().all() + segment_map = {segment.index_node_id: segment for segment in segments} for chunk_index in sorted_chunk_indices: - segment_query = db.session.query(DocumentSegment).where( - DocumentSegment.dataset_id == self.dataset.id, DocumentSegment.index_node_id == chunk_index - ) - if document_ids_filter: - segment_query = segment_query.where(DocumentSegment.document_id.in_(document_ids_filter)) - segment = segment_query.first() + segment = segment_map.get(chunk_index) if segment: documents.append( diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py index a139fba4d0..8ec1ce6242 100644 --- a/api/core/rag/datasource/retrieval_service.py +++ b/api/core/rag/datasource/retrieval_service.py @@ -1,4 +1,5 @@ import concurrent.futures +import logging from concurrent.futures import ThreadPoolExecutor from typing import Any @@ -7,12 +8,13 @@ from sqlalchemy import select from sqlalchemy.orm import Session, load_only from configs import dify_config +from core.db.session_factory import session_factory from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelType from core.rag.data_post_processor.data_post_processor import DataPostProcessor from core.rag.datasource.keyword.keyword_factory import Keyword from core.rag.datasource.vdb.vector_factory import Vector -from core.rag.embedding.retrieval import RetrievalSegments +from core.rag.embedding.retrieval import RetrievalChildChunk, RetrievalSegments from core.rag.entities.metadata_entities import MetadataCondition from core.rag.index_processor.constant.doc_type import DocType from core.rag.index_processor.constant.index_type import IndexStructureType @@ -35,6 +37,8 @@ default_retrieval_model = { "score_threshold_enabled": False, } +logger = logging.getLogger(__name__) + class RetrievalService: # Cache precompiled regular expressions to avoid repeated compilation @@ -105,7 +109,12 @@ class RetrievalService: ) ) - concurrent.futures.wait(futures, timeout=3600, return_when=concurrent.futures.ALL_COMPLETED) + if futures: + for future in concurrent.futures.as_completed(futures, timeout=3600): + if exceptions: + for f in futures: + f.cancel() + break if exceptions: raise ValueError(";\n".join(exceptions)) @@ -138,37 +147,47 @@ class RetrievalService: @classmethod def _deduplicate_documents(cls, documents: list[Document]) -> list[Document]: - """Deduplicate documents based on doc_id to avoid duplicate chunks in hybrid search.""" + """Deduplicate documents in O(n) while preserving first-seen order. + + Rules: + - For provider == "dify" and metadata["doc_id"] exists: keep the doc with the highest + metadata["score"] among duplicates; if a later duplicate has no score, ignore it. + - For non-dify documents (or dify without doc_id): deduplicate by content key + (provider, page_content), keeping the first occurrence. + """ if not documents: return documents - unique_documents = [] - seen_doc_ids = set() + # Map of dedup key -> chosen Document + chosen: dict[tuple, Document] = {} + # Preserve the order of first appearance of each dedup key + order: list[tuple] = [] - for document in documents: - # For dify provider documents, use doc_id for deduplication - if document.provider == "dify" and document.metadata is not None and "doc_id" in document.metadata: - doc_id = document.metadata["doc_id"] - if doc_id not in seen_doc_ids: - seen_doc_ids.add(doc_id) - unique_documents.append(document) - # If duplicate, keep the one with higher score - elif "score" in document.metadata: - # Find existing document with same doc_id and compare scores - for i, existing_doc in enumerate(unique_documents): - if ( - existing_doc.metadata - and existing_doc.metadata.get("doc_id") == doc_id - and existing_doc.metadata.get("score", 0) < document.metadata.get("score", 0) - ): - unique_documents[i] = document - break + for doc in documents: + is_dify = doc.provider == "dify" + doc_id = (doc.metadata or {}).get("doc_id") if is_dify else None + + if is_dify and doc_id: + key = ("dify", doc_id) + if key not in chosen: + chosen[key] = doc + order.append(key) + else: + # Only replace if the new one has a score and it's strictly higher + if "score" in doc.metadata: + new_score = float(doc.metadata.get("score", 0.0)) + old_score = float(chosen[key].metadata.get("score", 0.0)) if chosen[key].metadata else 0.0 + if new_score > old_score: + chosen[key] = doc else: - # For non-dify documents, use content-based deduplication - if document not in unique_documents: - unique_documents.append(document) + # Content-based dedup for non-dify or dify without doc_id + content_key = (doc.provider or "dify", doc.page_content) + if content_key not in chosen: + chosen[content_key] = doc + order.append(content_key) + # If duplicate content appears, we keep the first occurrence (no score comparison) - return unique_documents + return [chosen[k] for k in order] @classmethod def _get_dataset(cls, dataset_id: str) -> Dataset | None: @@ -199,6 +218,7 @@ class RetrievalService: ) all_documents.extend(documents) except Exception as e: + logger.error(e, exc_info=True) exceptions.append(str(e)) @classmethod @@ -292,6 +312,7 @@ class RetrievalService: else: all_documents.extend(documents) except Exception as e: + logger.error(e, exc_info=True) exceptions.append(str(e)) @classmethod @@ -340,6 +361,7 @@ class RetrievalService: else: all_documents.extend(documents) except Exception as e: + logger.error(e, exc_info=True) exceptions.append(str(e)) @staticmethod @@ -370,171 +392,176 @@ class RetrievalService: records = [] include_segment_ids = set() segment_child_map = {} - segment_file_map = {} - with Session(bind=db.engine, expire_on_commit=False) as session: - # Process documents - for document in documents: - segment_id = None - attachment_info = None - child_chunk = None - document_id = document.metadata.get("document_id") - if document_id not in dataset_documents: - continue - dataset_document = dataset_documents[document_id] - if not dataset_document: - continue + valid_dataset_documents = {} + image_doc_ids: list[Any] = [] + child_index_node_ids = [] + index_node_ids = [] + doc_to_document_map = {} + for document in documents: + document_id = document.metadata.get("document_id") + if document_id not in dataset_documents: + continue - if dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX: - # Handle parent-child documents - if document.metadata.get("doc_type") == DocType.IMAGE: - attachment_info_dict = cls.get_segment_attachment_info( - dataset_document.dataset_id, - dataset_document.tenant_id, - document.metadata.get("doc_id") or "", - session, - ) - if attachment_info_dict: - attachment_info = attachment_info_dict["attachment_info"] - segment_id = attachment_info_dict["segment_id"] - else: - child_index_node_id = document.metadata.get("doc_id") - child_chunk_stmt = select(ChildChunk).where(ChildChunk.index_node_id == child_index_node_id) - child_chunk = session.scalar(child_chunk_stmt) + dataset_document = dataset_documents[document_id] + if not dataset_document: + continue + valid_dataset_documents[document_id] = dataset_document - if not child_chunk: - continue - segment_id = child_chunk.segment_id - - if not segment_id: - continue - - segment = ( - session.query(DocumentSegment) - .where( - DocumentSegment.dataset_id == dataset_document.dataset_id, - DocumentSegment.enabled == True, - DocumentSegment.status == "completed", - DocumentSegment.id == segment_id, - ) - .first() - ) - - if not segment: - continue - - if segment.id not in include_segment_ids: - include_segment_ids.add(segment.id) - if child_chunk: - child_chunk_detail = { - "id": child_chunk.id, - "content": child_chunk.content, - "position": child_chunk.position, - "score": document.metadata.get("score", 0.0), - } - map_detail = { - "max_score": document.metadata.get("score", 0.0), - "child_chunks": [child_chunk_detail], - } - segment_child_map[segment.id] = map_detail - record = { - "segment": segment, - } - if attachment_info: - segment_file_map[segment.id] = [attachment_info] - records.append(record) - else: - if child_chunk: - child_chunk_detail = { - "id": child_chunk.id, - "content": child_chunk.content, - "position": child_chunk.position, - "score": document.metadata.get("score", 0.0), - } - if segment.id in segment_child_map: - segment_child_map[segment.id]["child_chunks"].append(child_chunk_detail) - segment_child_map[segment.id]["max_score"] = max( - segment_child_map[segment.id]["max_score"], document.metadata.get("score", 0.0) - ) - else: - segment_child_map[segment.id] = { - "max_score": document.metadata.get("score", 0.0), - "child_chunks": [child_chunk_detail], - } - if attachment_info: - if segment.id in segment_file_map: - segment_file_map[segment.id].append(attachment_info) - else: - segment_file_map[segment.id] = [attachment_info] + if dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX: + doc_id = document.metadata.get("doc_id") or "" + doc_to_document_map[doc_id] = document + if document.metadata.get("doc_type") == DocType.IMAGE: + image_doc_ids.append(doc_id) else: - # Handle normal documents - segment = None - if document.metadata.get("doc_type") == DocType.IMAGE: - attachment_info_dict = cls.get_segment_attachment_info( - dataset_document.dataset_id, - dataset_document.tenant_id, - document.metadata.get("doc_id") or "", - session, - ) - if attachment_info_dict: - attachment_info = attachment_info_dict["attachment_info"] - segment_id = attachment_info_dict["segment_id"] - document_segment_stmt = select(DocumentSegment).where( - DocumentSegment.dataset_id == dataset_document.dataset_id, - DocumentSegment.enabled == True, - DocumentSegment.status == "completed", - DocumentSegment.id == segment_id, - ) - segment = session.scalar(document_segment_stmt) - if segment: - segment_file_map[segment.id] = [attachment_info] - else: - index_node_id = document.metadata.get("doc_id") - if not index_node_id: - continue - document_segment_stmt = select(DocumentSegment).where( - DocumentSegment.dataset_id == dataset_document.dataset_id, - DocumentSegment.enabled == True, - DocumentSegment.status == "completed", - DocumentSegment.index_node_id == index_node_id, - ) - segment = session.scalar(document_segment_stmt) + child_index_node_ids.append(doc_id) + else: + doc_id = document.metadata.get("doc_id") or "" + doc_to_document_map[doc_id] = document + if document.metadata.get("doc_type") == DocType.IMAGE: + image_doc_ids.append(doc_id) + else: + index_node_ids.append(doc_id) - if not segment: - continue - if segment.id not in include_segment_ids: - include_segment_ids.add(segment.id) - record = { - "segment": segment, - "score": document.metadata.get("score"), # type: ignore + image_doc_ids = [i for i in image_doc_ids if i] + child_index_node_ids = [i for i in child_index_node_ids if i] + index_node_ids = [i for i in index_node_ids if i] + + segment_ids: list[str] = [] + index_node_segments: list[DocumentSegment] = [] + segments: list[DocumentSegment] = [] + attachment_map: dict[str, list[dict[str, Any]]] = {} + child_chunk_map: dict[str, list[ChildChunk]] = {} + doc_segment_map: dict[str, list[str]] = {} + + with session_factory.create_session() as session: + attachments = cls.get_segment_attachment_infos(image_doc_ids, session) + + for attachment in attachments: + segment_ids.append(attachment["segment_id"]) + if attachment["segment_id"] in attachment_map: + attachment_map[attachment["segment_id"]].append(attachment["attachment_info"]) + else: + attachment_map[attachment["segment_id"]] = [attachment["attachment_info"]] + if attachment["segment_id"] in doc_segment_map: + doc_segment_map[attachment["segment_id"]].append(attachment["attachment_id"]) + else: + doc_segment_map[attachment["segment_id"]] = [attachment["attachment_id"]] + child_chunk_stmt = select(ChildChunk).where(ChildChunk.index_node_id.in_(child_index_node_ids)) + child_index_nodes = session.execute(child_chunk_stmt).scalars().all() + + for i in child_index_nodes: + segment_ids.append(i.segment_id) + if i.segment_id in child_chunk_map: + child_chunk_map[i.segment_id].append(i) + else: + child_chunk_map[i.segment_id] = [i] + if i.segment_id in doc_segment_map: + doc_segment_map[i.segment_id].append(i.index_node_id) + else: + doc_segment_map[i.segment_id] = [i.index_node_id] + + if index_node_ids: + document_segment_stmt = select(DocumentSegment).where( + DocumentSegment.enabled == True, + DocumentSegment.status == "completed", + DocumentSegment.index_node_id.in_(index_node_ids), + ) + index_node_segments = session.execute(document_segment_stmt).scalars().all() # type: ignore + for index_node_segment in index_node_segments: + doc_segment_map[index_node_segment.id] = [index_node_segment.index_node_id] + if segment_ids: + document_segment_stmt = select(DocumentSegment).where( + DocumentSegment.enabled == True, + DocumentSegment.status == "completed", + DocumentSegment.id.in_(segment_ids), + ) + segments = session.execute(document_segment_stmt).scalars().all() # type: ignore + + if index_node_segments: + segments.extend(index_node_segments) + + for segment in segments: + child_chunks: list[ChildChunk] = child_chunk_map.get(segment.id, []) + attachment_infos: list[dict[str, Any]] = attachment_map.get(segment.id, []) + ds_dataset_document: DatasetDocument | None = valid_dataset_documents.get(segment.document_id) + + if ds_dataset_document and ds_dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX: + if segment.id not in include_segment_ids: + include_segment_ids.add(segment.id) + if child_chunks or attachment_infos: + child_chunk_details = [] + max_score = 0.0 + for child_chunk in child_chunks: + document = doc_to_document_map[child_chunk.index_node_id] + child_chunk_detail = { + "id": child_chunk.id, + "content": child_chunk.content, + "position": child_chunk.position, + "score": document.metadata.get("score", 0.0) if document else 0.0, + } + child_chunk_details.append(child_chunk_detail) + max_score = max(max_score, document.metadata.get("score", 0.0) if document else 0.0) + for attachment_info in attachment_infos: + file_document = doc_to_document_map[attachment_info["id"]] + max_score = max( + max_score, file_document.metadata.get("score", 0.0) if file_document else 0.0 + ) + + map_detail = { + "max_score": max_score, + "child_chunks": child_chunk_details, } - if attachment_info: - segment_file_map[segment.id] = [attachment_info] - records.append(record) - else: - if attachment_info: - attachment_infos = segment_file_map.get(segment.id, []) - if attachment_info not in attachment_infos: - attachment_infos.append(attachment_info) - segment_file_map[segment.id] = attachment_infos + segment_child_map[segment.id] = map_detail + record: dict[str, Any] = { + "segment": segment, + } + records.append(record) + else: + if segment.id not in include_segment_ids: + include_segment_ids.add(segment.id) + max_score = 0.0 + segment_document = doc_to_document_map.get(segment.index_node_id) + if segment_document: + max_score = max(max_score, segment_document.metadata.get("score", 0.0)) + for attachment_info in attachment_infos: + file_doc = doc_to_document_map.get(attachment_info["id"]) + if file_doc: + max_score = max(max_score, file_doc.metadata.get("score", 0.0)) + record = { + "segment": segment, + "score": max_score, + } + records.append(record) # Add child chunks information to records for record in records: if record["segment"].id in segment_child_map: record["child_chunks"] = segment_child_map[record["segment"].id].get("child_chunks") # type: ignore - record["score"] = segment_child_map[record["segment"].id]["max_score"] - if record["segment"].id in segment_file_map: - record["files"] = segment_file_map[record["segment"].id] # type: ignore[assignment] + record["score"] = segment_child_map[record["segment"].id]["max_score"] # type: ignore + if record["segment"].id in attachment_map: + record["files"] = attachment_map[record["segment"].id] # type: ignore[assignment] - result = [] + result: list[RetrievalSegments] = [] for record in records: # Extract segment segment = record["segment"] # Extract child_chunks, ensuring it's a list or None - child_chunks = record.get("child_chunks") - if not isinstance(child_chunks, list): - child_chunks = None + raw_child_chunks = record.get("child_chunks") + child_chunks_list: list[RetrievalChildChunk] | None = None + if isinstance(raw_child_chunks, list): + # Sort by score descending + sorted_chunks = sorted(raw_child_chunks, key=lambda x: x.get("score", 0.0), reverse=True) + child_chunks_list = [ + RetrievalChildChunk( + id=chunk["id"], + content=chunk["content"], + score=chunk.get("score", 0.0), + position=chunk["position"], + ) + for chunk in sorted_chunks + ] # Extract files, ensuring it's a list or None files = record.get("files") @@ -551,11 +578,11 @@ class RetrievalService: # Create RetrievalSegments object retrieval_segment = RetrievalSegments( - segment=segment, child_chunks=child_chunks, score=score, files=files + segment=segment, child_chunks=child_chunks_list, score=score, files=files ) result.append(retrieval_segment) - return result + return sorted(result, key=lambda x: x.score if x.score is not None else 0.0, reverse=True) except Exception as e: db.session.rollback() raise e @@ -565,6 +592,8 @@ class RetrievalService: flask_app: Flask, retrieval_method: RetrievalMethod, dataset: Dataset, + all_documents: list[Document], + exceptions: list[str], query: str | None = None, top_k: int = 4, score_threshold: float | None = 0.0, @@ -573,8 +602,6 @@ class RetrievalService: weights: dict | None = None, document_ids_filter: list[str] | None = None, attachment_id: str | None = None, - all_documents: list[Document] = [], - exceptions: list[str] = [], ): if not query and not attachment_id: return @@ -647,7 +674,14 @@ class RetrievalService: document_ids_filter=document_ids_filter, ) ) - concurrent.futures.wait(futures, timeout=300, return_when=concurrent.futures.ALL_COMPLETED) + # Use as_completed for early error propagation - cancel remaining futures on first error + if futures: + for future in concurrent.futures.as_completed(futures, timeout=300): + if future.exception(): + # Cancel remaining futures to avoid unnecessary waiting + for f in futures: + f.cancel() + break if exceptions: raise ValueError(";\n".join(exceptions)) @@ -696,3 +730,37 @@ class RetrievalService: } return {"attachment_info": attachment_info, "segment_id": attachment_binding.segment_id} return None + + @classmethod + def get_segment_attachment_infos(cls, attachment_ids: list[str], session: Session) -> list[dict[str, Any]]: + attachment_infos = [] + upload_files = session.query(UploadFile).where(UploadFile.id.in_(attachment_ids)).all() + if upload_files: + upload_file_ids = [upload_file.id for upload_file in upload_files] + attachment_bindings = ( + session.query(SegmentAttachmentBinding) + .where(SegmentAttachmentBinding.attachment_id.in_(upload_file_ids)) + .all() + ) + attachment_binding_map = {binding.attachment_id: binding for binding in attachment_bindings} + + if attachment_bindings: + for upload_file in upload_files: + attachment_binding = attachment_binding_map.get(upload_file.id) + attachment_info = { + "id": upload_file.id, + "name": upload_file.name, + "extension": "." + upload_file.extension, + "mime_type": upload_file.mime_type, + "source_url": sign_upload_file(upload_file.id, upload_file.extension), + "size": upload_file.size, + } + if attachment_binding: + attachment_infos.append( + { + "attachment_id": attachment_binding.attachment_id, + "attachment_info": attachment_info, + "segment_id": attachment_binding.segment_id, + } + ) + return attachment_infos diff --git a/api/core/rag/datasource/vdb/oracle/oraclevector.py b/api/core/rag/datasource/vdb/oracle/oraclevector.py index d82ab89a34..cb05c22b55 100644 --- a/api/core/rag/datasource/vdb/oracle/oraclevector.py +++ b/api/core/rag/datasource/vdb/oracle/oraclevector.py @@ -289,7 +289,8 @@ class OracleVector(BaseVector): words = pseg.cut(query) current_entity = "" for word, pos in words: - if pos in {"nr", "Ng", "eng", "nz", "n", "ORG", "v"}: # nr: 人名,ns: 地名,nt: 机构名 + # `nr`: Person, `ns`: Location, `nt`: Organization + if pos in {"nr", "Ng", "eng", "nz", "n", "ORG", "v"}: current_entity += word else: if current_entity: diff --git a/api/core/rag/datasource/vdb/pgvector/pgvector.py b/api/core/rag/datasource/vdb/pgvector/pgvector.py index 445a0a7f8b..0615b8312c 100644 --- a/api/core/rag/datasource/vdb/pgvector/pgvector.py +++ b/api/core/rag/datasource/vdb/pgvector/pgvector.py @@ -255,7 +255,10 @@ class PGVector(BaseVector): return with self._get_cursor() as cur: - cur.execute("CREATE EXTENSION IF NOT EXISTS vector") + cur.execute("SELECT 1 FROM pg_extension WHERE extname = 'vector'") + if not cur.fetchone(): + cur.execute("CREATE EXTENSION vector") + cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name, dimension=dimension)) # PG hnsw index only support 2000 dimension or less # ref: https://github.com/pgvector/pgvector?tab=readme-ov-file#indexing diff --git a/api/core/rag/datasource/vdb/pyvastbase/vastbase_vector.py b/api/core/rag/datasource/vdb/pyvastbase/vastbase_vector.py index 86b6ace3f6..d080e8da58 100644 --- a/api/core/rag/datasource/vdb/pyvastbase/vastbase_vector.py +++ b/api/core/rag/datasource/vdb/pyvastbase/vastbase_vector.py @@ -213,7 +213,7 @@ class VastbaseVector(BaseVector): with self._get_cursor() as cur: cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name, dimension=dimension)) - # Vastbase 支持的向量维度取值范围为 [1,16000] + # Vastbase supports vector dimensions in the range [1, 16,000] if dimension <= 16000: cur.execute(SQL_CREATE_INDEX.format(table_name=self.table_name)) redis_client.set(collection_exist_cache_key, 1, ex=3600) diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index 789ac8557d..5d6223db06 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -25,7 +25,7 @@ class FirecrawlApp: } if params: json_data.update(params) - response = self._post_request(f"{self.base_url}/v2/scrape", json_data, headers) + response = self._post_request(self._build_url("v2/scrape"), json_data, headers) if response.status_code == 200: response_data = response.json() data = response_data["data"] @@ -42,7 +42,7 @@ class FirecrawlApp: json_data = {"url": url} if params: json_data.update(params) - response = self._post_request(f"{self.base_url}/v2/crawl", json_data, headers) + response = self._post_request(self._build_url("v2/crawl"), json_data, headers) if response.status_code == 200: # There's also another two fields in the response: "success" (bool) and "url" (str) job_id = response.json().get("id") @@ -58,7 +58,7 @@ class FirecrawlApp: if params: # Pass through provided params, including optional "sitemap": "only" | "include" | "skip" json_data.update(params) - response = self._post_request(f"{self.base_url}/v2/map", json_data, headers) + response = self._post_request(self._build_url("v2/map"), json_data, headers) if response.status_code == 200: return cast(dict[str, Any], response.json()) elif response.status_code in {402, 409, 500, 429, 408}: @@ -69,7 +69,7 @@ class FirecrawlApp: def check_crawl_status(self, job_id) -> dict[str, Any]: headers = self._prepare_headers() - response = self._get_request(f"{self.base_url}/v2/crawl/{job_id}", headers) + response = self._get_request(self._build_url(f"v2/crawl/{job_id}"), headers) if response.status_code == 200: crawl_status_response = response.json() if crawl_status_response.get("status") == "completed": @@ -120,6 +120,10 @@ class FirecrawlApp: def _prepare_headers(self) -> dict[str, Any]: return {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"} + def _build_url(self, path: str) -> str: + # ensure exactly one slash between base and path, regardless of user-provided base_url + return f"{self.base_url.rstrip('/')}/{path.lstrip('/')}" + def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5) -> httpx.Response: for attempt in range(retries): response = httpx.post(url, headers=headers, json=data) @@ -139,7 +143,11 @@ class FirecrawlApp: return response def _handle_error(self, response, action): - error_message = response.json().get("error", "Unknown error occurred") + try: + payload = response.json() + error_message = payload.get("error") or payload.get("message") or response.text or "Unknown error occurred" + except json.JSONDecodeError: + error_message = response.text or "Unknown error occurred" raise Exception(f"Failed to {action}. Status code: {response.status_code}. Error: {error_message}") # type: ignore[return] def search(self, query: str, params: dict[str, Any] | None = None) -> dict[str, Any]: @@ -160,7 +168,7 @@ class FirecrawlApp: } if params: json_data.update(params) - response = self._post_request(f"{self.base_url}/v2/search", json_data, headers) + response = self._post_request(self._build_url("v2/search"), json_data, headers) if response.status_code == 200: response_data = response.json() if not response_data.get("success"): diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.py index e87ab38349..372af8fd94 100644 --- a/api/core/rag/extractor/notion_extractor.py +++ b/api/core/rag/extractor/notion_extractor.py @@ -48,13 +48,21 @@ class NotionExtractor(BaseExtractor): if notion_access_token: self._notion_access_token = notion_access_token else: - self._notion_access_token = self._get_access_token(tenant_id, self._credential_id) - if not self._notion_access_token: + try: + self._notion_access_token = self._get_access_token(tenant_id, self._credential_id) + except Exception as e: + logger.warning( + ( + "Failed to get Notion access token from datasource credentials: %s, " + "falling back to environment variable NOTION_INTEGRATION_TOKEN" + ), + e, + ) integration_token = dify_config.NOTION_INTEGRATION_TOKEN if integration_token is None: raise ValueError( "Must specify `integration_token` or set environment variable `NOTION_INTEGRATION_TOKEN`." - ) + ) from e self._notion_access_token = integration_token diff --git a/api/core/rag/extractor/word_extractor.py b/api/core/rag/extractor/word_extractor.py index 044b118635..f67f613e9d 100644 --- a/api/core/rag/extractor/word_extractor.py +++ b/api/core/rag/extractor/word_extractor.py @@ -83,6 +83,7 @@ class WordExtractor(BaseExtractor): def _extract_images_from_docx(self, doc): image_count = 0 image_map = {} + base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL for r_id, rel in doc.part.rels.items(): if "image" in rel.target_ref: @@ -121,8 +122,7 @@ class WordExtractor(BaseExtractor): used_at=naive_utc_now(), ) db.session.add(upload_file) - # Use r_id as key for external images since target_part is undefined - image_map[r_id] = f"![image]({dify_config.FILES_URL}/files/{upload_file.id}/file-preview)" + image_map[r_id] = f"![image]({base_url}/files/{upload_file.id}/file-preview)" else: image_ext = rel.target_ref.split(".")[-1] if image_ext is None: @@ -150,10 +150,7 @@ class WordExtractor(BaseExtractor): used_at=naive_utc_now(), ) db.session.add(upload_file) - # Use target_part as key for internal images - image_map[rel.target_part] = ( - f"![image]({dify_config.FILES_URL}/files/{upload_file.id}/file-preview)" - ) + image_map[rel.target_part] = f"![image]({base_url}/files/{upload_file.id}/file-preview)" db.session.commit() return image_map diff --git a/api/core/rag/index_processor/index_processor_base.py b/api/core/rag/index_processor/index_processor_base.py index 8a28eb477a..e36b54eedd 100644 --- a/api/core/rag/index_processor/index_processor_base.py +++ b/api/core/rag/index_processor/index_processor_base.py @@ -231,7 +231,7 @@ class BaseIndexProcessor(ABC): if not filename: parsed_url = urlparse(image_url) - # unquote 处理 URL 中的中文 + # Decode percent-encoded characters in the URL path. path = unquote(parsed_url.path) filename = os.path.basename(path) diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index 635eab73f0..4ec59940e3 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -7,7 +7,7 @@ from collections.abc import Generator, Mapping from typing import Any, Union, cast from flask import Flask, current_app -from sqlalchemy import and_, or_, select +from sqlalchemy import and_, literal, or_, select from sqlalchemy.orm import Session from core.app.app_config.entities import ( @@ -151,20 +151,14 @@ class DatasetRetrieval: if ModelFeature.TOOL_CALL in features or ModelFeature.MULTI_TOOL_CALL in features: planning_strategy = PlanningStrategy.ROUTER available_datasets = [] - for dataset_id in dataset_ids: - # get dataset from dataset id - dataset_stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id) - dataset = db.session.scalar(dataset_stmt) - # pass if dataset is not available - if not dataset: + dataset_stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id.in_(dataset_ids)) + datasets: list[Dataset] = db.session.execute(dataset_stmt).scalars().all() # type: ignore + for dataset in datasets: + if dataset.available_document_count == 0 and dataset.provider != "external": continue - - # pass if dataset is not available - if dataset and dataset.available_document_count == 0 and dataset.provider != "external": - continue - available_datasets.append(dataset) + if inputs: inputs = {key: str(value) for key, value in inputs.items()} else: @@ -282,26 +276,35 @@ class DatasetRetrieval: ) context_files.append(attachment_info) if show_retrieve_source: + dataset_ids = [record.segment.dataset_id for record in records] + document_ids = [record.segment.document_id for record in records] + dataset_document_stmt = select(DatasetDocument).where( + DatasetDocument.id.in_(document_ids), + DatasetDocument.enabled == True, + DatasetDocument.archived == False, + ) + documents = db.session.execute(dataset_document_stmt).scalars().all() # type: ignore + dataset_stmt = select(Dataset).where( + Dataset.id.in_(dataset_ids), + ) + datasets = db.session.execute(dataset_stmt).scalars().all() # type: ignore + dataset_map = {i.id: i for i in datasets} + document_map = {i.id: i for i in documents} for record in records: segment = record.segment - dataset = db.session.query(Dataset).filter_by(id=segment.dataset_id).first() - dataset_document_stmt = select(DatasetDocument).where( - DatasetDocument.id == segment.document_id, - DatasetDocument.enabled == True, - DatasetDocument.archived == False, - ) - document = db.session.scalar(dataset_document_stmt) - if dataset and document: + dataset_item = dataset_map.get(segment.dataset_id) + document_item = document_map.get(segment.document_id) + if dataset_item and document_item: source = RetrievalSourceMetadata( - dataset_id=dataset.id, - dataset_name=dataset.name, - document_id=document.id, - document_name=document.name, - data_source_type=document.data_source_type, + dataset_id=dataset_item.id, + dataset_name=dataset_item.name, + document_id=document_item.id, + document_name=document_item.name, + data_source_type=document_item.data_source_type, segment_id=segment.id, retriever_from=invoke_from.to_source(), score=record.score or 0.0, - doc_metadata=document.doc_metadata, + doc_metadata=document_item.doc_metadata, ) if invoke_from.to_source() == "dev": @@ -513,6 +516,9 @@ class DatasetRetrieval: ].embedding_model_provider weights["vector_setting"]["embedding_model_name"] = available_datasets[0].embedding_model with measure_time() as timer: + cancel_event = threading.Event() + thread_exceptions: list[Exception] = [] + if query: query_thread = threading.Thread( target=self._multiple_retrieve_thread, @@ -531,6 +537,8 @@ class DatasetRetrieval: "score_threshold": score_threshold, "query": query, "attachment_id": None, + "cancel_event": cancel_event, + "thread_exceptions": thread_exceptions, }, ) all_threads.append(query_thread) @@ -554,12 +562,25 @@ class DatasetRetrieval: "score_threshold": score_threshold, "query": None, "attachment_id": attachment_id, + "cancel_event": cancel_event, + "thread_exceptions": thread_exceptions, }, ) all_threads.append(attachment_thread) attachment_thread.start() - for thread in all_threads: - thread.join() + + # Poll threads with short timeout to detect errors quickly (fail-fast) + while any(t.is_alive() for t in all_threads): + for thread in all_threads: + thread.join(timeout=0.1) + if thread_exceptions: + cancel_event.set() + break + if thread_exceptions: + break + + if thread_exceptions: + raise thread_exceptions[0] self._on_query(query, attachment_ids, dataset_ids, app_id, user_from, user_id) if all_documents: @@ -1033,7 +1054,7 @@ class DatasetRetrieval: if automatic_metadata_filters: conditions = [] for sequence, filter in enumerate(automatic_metadata_filters): - self._process_metadata_filter_func( + self.process_metadata_filter_func( sequence, filter.get("condition"), # type: ignore filter.get("metadata_name"), # type: ignore @@ -1069,7 +1090,7 @@ class DatasetRetrieval: value=expected_value, ) ) - filters = self._process_metadata_filter_func( + filters = self.process_metadata_filter_func( sequence, condition.comparison_operator, metadata_name, @@ -1165,8 +1186,9 @@ class DatasetRetrieval: return None return automatic_metadata_filters - def _process_metadata_filter_func( - self, sequence: int, condition: str, metadata_name: str, value: Any | None, filters: list + @classmethod + def process_metadata_filter_func( + cls, sequence: int, condition: str, metadata_name: str, value: Any | None, filters: list ): if value is None and condition not in ("empty", "not empty"): return filters @@ -1215,6 +1237,20 @@ class DatasetRetrieval: case "≥" | ">=": filters.append(DatasetDocument.doc_metadata[metadata_name].as_float() >= value) + case "in" | "not in": + if isinstance(value, str): + value_list = [v.strip() for v in value.split(",") if v.strip()] + elif isinstance(value, (list, tuple)): + value_list = [str(v) for v in value if v is not None] + else: + value_list = [str(value)] if value is not None else [] + + if not value_list: + # `field in []` is False, `field not in []` is True + filters.append(literal(condition == "not in")) + else: + op = json_field.in_ if condition == "in" else json_field.notin_ + filters.append(op(value_list)) case _: pass @@ -1386,40 +1422,53 @@ class DatasetRetrieval: score_threshold: float, query: str | None, attachment_id: str | None, + cancel_event: threading.Event | None = None, + thread_exceptions: list[Exception] | None = None, ): - with flask_app.app_context(): - threads = [] - all_documents_item: list[Document] = [] - index_type = None - for dataset in available_datasets: - index_type = dataset.indexing_technique - document_ids_filter = None - if dataset.provider != "external": - if metadata_condition and not metadata_filter_document_ids: - continue - if metadata_filter_document_ids: - document_ids = metadata_filter_document_ids.get(dataset.id, []) - if document_ids: - document_ids_filter = document_ids - else: + try: + with flask_app.app_context(): + threads = [] + all_documents_item: list[Document] = [] + index_type = None + for dataset in available_datasets: + # Check for cancellation signal + if cancel_event and cancel_event.is_set(): + break + index_type = dataset.indexing_technique + document_ids_filter = None + if dataset.provider != "external": + if metadata_condition and not metadata_filter_document_ids: continue - retrieval_thread = threading.Thread( - target=self._retriever, - kwargs={ - "flask_app": flask_app, - "dataset_id": dataset.id, - "query": query, - "top_k": top_k, - "all_documents": all_documents_item, - "document_ids_filter": document_ids_filter, - "metadata_condition": metadata_condition, - "attachment_ids": [attachment_id] if attachment_id else None, - }, - ) - threads.append(retrieval_thread) - retrieval_thread.start() - for thread in threads: - thread.join() + if metadata_filter_document_ids: + document_ids = metadata_filter_document_ids.get(dataset.id, []) + if document_ids: + document_ids_filter = document_ids + else: + continue + retrieval_thread = threading.Thread( + target=self._retriever, + kwargs={ + "flask_app": flask_app, + "dataset_id": dataset.id, + "query": query, + "top_k": top_k, + "all_documents": all_documents_item, + "document_ids_filter": document_ids_filter, + "metadata_condition": metadata_condition, + "attachment_ids": [attachment_id] if attachment_id else None, + }, + ) + threads.append(retrieval_thread) + retrieval_thread.start() + + # Poll threads with short timeout to respond quickly to cancellation + while any(t.is_alive() for t in threads): + for thread in threads: + thread.join(timeout=0.1) + if cancel_event and cancel_event.is_set(): + break + if cancel_event and cancel_event.is_set(): + break if reranking_enable: # do rerank for searched documents @@ -1452,3 +1501,8 @@ class DatasetRetrieval: all_documents_item = all_documents_item[:top_k] if top_k else all_documents_item if all_documents_item: all_documents.extend(all_documents_item) + except Exception as e: + if cancel_event: + cancel_event.set() + if thread_exceptions is not None: + thread_exceptions.append(e) diff --git a/api/core/rag/splitter/fixed_text_splitter.py b/api/core/rag/splitter/fixed_text_splitter.py index 801d2a2a52..b65cb14d8e 100644 --- a/api/core/rag/splitter/fixed_text_splitter.py +++ b/api/core/rag/splitter/fixed_text_splitter.py @@ -2,6 +2,7 @@ from __future__ import annotations +import codecs import re from typing import Any @@ -52,7 +53,7 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter) def __init__(self, fixed_separator: str = "\n\n", separators: list[str] | None = None, **kwargs: Any): """Create a new TextSplitter.""" super().__init__(**kwargs) - self._fixed_separator = fixed_separator + self._fixed_separator = codecs.decode(fixed_separator, "unicode_escape") self._separators = separators or ["\n\n", "\n", "。", ". ", " ", ""] def split_text(self, text: str) -> list[str]: @@ -94,7 +95,8 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter) splits = re.split(r" +", text) else: splits = text.split(separator) - splits = [item + separator if i < len(splits) else item for i, item in enumerate(splits)] + if self._keep_separator: + splits = [s + separator for s in splits[:-1]] + splits[-1:] else: splits = list(text) if separator == "\n": @@ -103,7 +105,7 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter) splits = [s for s in splits if (s not in {"", "\n"})] _good_splits = [] _good_splits_lengths = [] # cache the lengths of the splits - _separator = separator if self._keep_separator else "" + _separator = "" if self._keep_separator else separator s_lens = self._length_function(splits) if separator != "": for s, s_len in zip(splits, s_lens): diff --git a/api/core/tools/entities/tool_entities.py b/api/core/tools/entities/tool_entities.py index 353f3a646a..583a3584f7 100644 --- a/api/core/tools/entities/tool_entities.py +++ b/api/core/tools/entities/tool_entities.py @@ -153,11 +153,11 @@ class ToolInvokeMessage(BaseModel): @classmethod def transform_variable_value(cls, values): """ - Only basic types and lists are allowed. + Only basic types, lists, and None are allowed. """ value = values.get("variable_value") - if not isinstance(value, dict | list | str | int | float | bool): - raise ValueError("Only basic types and lists are allowed.") + if value is not None and not isinstance(value, dict | list | str | int | float | bool): + raise ValueError("Only basic types, lists, and None are allowed.") # if stream is true, the value must be a string if values.get("stream"): diff --git a/api/core/tools/mcp_tool/tool.py b/api/core/tools/mcp_tool/tool.py index fbaf31ad09..96917045e3 100644 --- a/api/core/tools/mcp_tool/tool.py +++ b/api/core/tools/mcp_tool/tool.py @@ -6,7 +6,15 @@ from typing import Any from core.mcp.auth_client import MCPClientWithAuthRetry from core.mcp.error import MCPConnectionError -from core.mcp.types import AudioContent, CallToolResult, ImageContent, TextContent +from core.mcp.types import ( + AudioContent, + BlobResourceContents, + CallToolResult, + EmbeddedResource, + ImageContent, + TextContent, + TextResourceContents, +) from core.tools.__base.tool import Tool from core.tools.__base.tool_runtime import ToolRuntime from core.tools.entities.tool_entities import ToolEntity, ToolInvokeMessage, ToolProviderType @@ -53,10 +61,19 @@ class MCPTool(Tool): for content in result.content: if isinstance(content, TextContent): yield from self._process_text_content(content) - elif isinstance(content, ImageContent): - yield self._process_image_content(content) - elif isinstance(content, AudioContent): - yield self._process_audio_content(content) + elif isinstance(content, ImageContent | AudioContent): + yield self.create_blob_message( + blob=base64.b64decode(content.data), meta={"mime_type": content.mimeType} + ) + elif isinstance(content, EmbeddedResource): + resource = content.resource + if isinstance(resource, TextResourceContents): + yield self.create_text_message(resource.text) + elif isinstance(resource, BlobResourceContents): + mime_type = resource.mimeType or "application/octet-stream" + yield self.create_blob_message(blob=base64.b64decode(resource.blob), meta={"mime_type": mime_type}) + else: + raise ToolInvokeError(f"Unsupported embedded resource type: {type(resource)}") else: logger.warning("Unsupported content type=%s", type(content)) @@ -101,14 +118,6 @@ class MCPTool(Tool): for item in json_list: yield self.create_json_message(item) - def _process_image_content(self, content: ImageContent) -> ToolInvokeMessage: - """Process image content and return a blob message.""" - return self.create_blob_message(blob=base64.b64decode(content.data), meta={"mime_type": content.mimeType}) - - def _process_audio_content(self, content: AudioContent) -> ToolInvokeMessage: - """Process audio content and return a blob message.""" - return self.create_blob_message(blob=base64.b64decode(content.data), meta={"mime_type": content.mimeType}) - def fork_tool_runtime(self, runtime: ToolRuntime) -> "MCPTool": return MCPTool( entity=self.entity, diff --git a/api/core/tools/workflow_as_tool/provider.py b/api/core/tools/workflow_as_tool/provider.py index 0439fb1d60..2bd973f831 100644 --- a/api/core/tools/workflow_as_tool/provider.py +++ b/api/core/tools/workflow_as_tool/provider.py @@ -5,6 +5,7 @@ from sqlalchemy.orm import Session from core.app.app_config.entities import VariableEntity, VariableEntityType from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager +from core.db.session_factory import session_factory from core.plugin.entities.parameters import PluginParameterOption from core.tools.__base.tool_provider import ToolProviderController from core.tools.__base.tool_runtime import ToolRuntime @@ -47,33 +48,30 @@ class WorkflowToolProviderController(ToolProviderController): @classmethod def from_db(cls, db_provider: WorkflowToolProvider) -> "WorkflowToolProviderController": - with Session(db.engine, expire_on_commit=False) as session, session.begin(): - provider = session.get(WorkflowToolProvider, db_provider.id) if db_provider.id else None - if not provider: - raise ValueError("workflow provider not found") - app = session.get(App, provider.app_id) + with session_factory.create_session() as session, session.begin(): + app = session.get(App, db_provider.app_id) if not app: raise ValueError("app not found") - user = session.get(Account, provider.user_id) if provider.user_id else None + user = session.get(Account, db_provider.user_id) if db_provider.user_id else None controller = WorkflowToolProviderController( entity=ToolProviderEntity( identity=ToolProviderIdentity( author=user.name if user else "", - name=provider.label, - label=I18nObject(en_US=provider.label, zh_Hans=provider.label), - description=I18nObject(en_US=provider.description, zh_Hans=provider.description), - icon=provider.icon, + name=db_provider.label, + label=I18nObject(en_US=db_provider.label, zh_Hans=db_provider.label), + description=I18nObject(en_US=db_provider.description, zh_Hans=db_provider.description), + icon=db_provider.icon, ), credentials_schema=[], plugin_id=None, ), - provider_id=provider.id or "", + provider_id="", ) controller.tools = [ - controller._get_db_provider_tool(provider, app, session=session, user=user), + controller._get_db_provider_tool(db_provider, app, session=session, user=user), ] return controller diff --git a/api/core/trigger/utils/encryption.py b/api/core/trigger/utils/encryption.py index 026a65aa23..b12291e299 100644 --- a/api/core/trigger/utils/encryption.py +++ b/api/core/trigger/utils/encryption.py @@ -67,12 +67,16 @@ def create_trigger_provider_encrypter_for_subscription( def delete_cache_for_subscription(tenant_id: str, provider_id: str, subscription_id: str): - cache = TriggerProviderCredentialsCache( + TriggerProviderCredentialsCache( tenant_id=tenant_id, provider_id=provider_id, credential_id=subscription_id, - ) - cache.delete() + ).delete() + TriggerProviderPropertiesCache( + tenant_id=tenant_id, + provider_id=provider_id, + subscription_id=subscription_id, + ).delete() def create_trigger_provider_encrypter_for_properties( diff --git a/api/core/workflow/enums.py b/api/core/workflow/enums.py index 32c6b2d6e5..5ea7cf1a07 100644 --- a/api/core/workflow/enums.py +++ b/api/core/workflow/enums.py @@ -249,6 +249,7 @@ class WorkflowNodeExecutionMetadataKey(StrEnum): DATASOURCE_INFO = "datasource_info" LLM_CONTENT_SEQUENCE = "llm_content_sequence" LLM_TRACE = "llm_trace" + COMPLETED_REASON = "completed_reason" # completed reason for loop node class WorkflowNodeExecutionStatus(StrEnum): diff --git a/api/core/workflow/nodes/http_request/executor.py b/api/core/workflow/nodes/http_request/executor.py index f0c84872fb..931c6113a7 100644 --- a/api/core/workflow/nodes/http_request/executor.py +++ b/api/core/workflow/nodes/http_request/executor.py @@ -86,6 +86,11 @@ class Executor: node_data.authorization.config.api_key = variable_pool.convert_template( node_data.authorization.config.api_key ).text + # Validate that API key is not empty after template conversion + if not node_data.authorization.config.api_key or not node_data.authorization.config.api_key.strip(): + raise AuthorizationConfigError( + "API key is required for authorization but was empty. Please provide a valid API key." + ) self.url = node_data.url self.method = node_data.method diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index adc474bd60..8670a71aa3 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -6,7 +6,7 @@ from collections import defaultdict from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, cast -from sqlalchemy import and_, func, literal, or_, select +from sqlalchemy import and_, func, or_, select from sqlalchemy.orm import sessionmaker from core.app.app_config.entities import DatasetRetrieveConfigEntity @@ -460,7 +460,7 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD if automatic_metadata_filters: conditions = [] for sequence, filter in enumerate(automatic_metadata_filters): - self._process_metadata_filter_func( + DatasetRetrieval.process_metadata_filter_func( sequence, filter.get("condition", ""), filter.get("metadata_name", ""), @@ -504,7 +504,7 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD value=expected_value, ) ) - filters = self._process_metadata_filter_func( + filters = DatasetRetrieval.process_metadata_filter_func( sequence, condition.comparison_operator, metadata_name, @@ -603,87 +603,6 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD return [], usage return automatic_metadata_filters, usage - def _process_metadata_filter_func( - self, sequence: int, condition: str, metadata_name: str, value: Any, filters: list[Any] - ) -> list[Any]: - if value is None and condition not in ("empty", "not empty"): - return filters - - json_field = Document.doc_metadata[metadata_name].as_string() - - match condition: - case "contains": - filters.append(json_field.like(f"%{value}%")) - - case "not contains": - filters.append(json_field.notlike(f"%{value}%")) - - case "start with": - filters.append(json_field.like(f"{value}%")) - - case "end with": - filters.append(json_field.like(f"%{value}")) - case "in": - if isinstance(value, str): - value_list = [v.strip() for v in value.split(",") if v.strip()] - elif isinstance(value, (list, tuple)): - value_list = [str(v) for v in value if v is not None] - else: - value_list = [str(value)] if value is not None else [] - - if not value_list: - filters.append(literal(False)) - else: - filters.append(json_field.in_(value_list)) - - case "not in": - if isinstance(value, str): - value_list = [v.strip() for v in value.split(",") if v.strip()] - elif isinstance(value, (list, tuple)): - value_list = [str(v) for v in value if v is not None] - else: - value_list = [str(value)] if value is not None else [] - - if not value_list: - filters.append(literal(True)) - else: - filters.append(json_field.notin_(value_list)) - - case "is" | "=": - if isinstance(value, str): - filters.append(json_field == value) - elif isinstance(value, (int, float)): - filters.append(Document.doc_metadata[metadata_name].as_float() == value) - - case "is not" | "≠": - if isinstance(value, str): - filters.append(json_field != value) - elif isinstance(value, (int, float)): - filters.append(Document.doc_metadata[metadata_name].as_float() != value) - - case "empty": - filters.append(Document.doc_metadata[metadata_name].is_(None)) - - case "not empty": - filters.append(Document.doc_metadata[metadata_name].isnot(None)) - - case "before" | "<": - filters.append(Document.doc_metadata[metadata_name].as_float() < value) - - case "after" | ">": - filters.append(Document.doc_metadata[metadata_name].as_float() > value) - - case "≤" | "<=": - filters.append(Document.doc_metadata[metadata_name].as_float() <= value) - - case "≥" | ">=": - filters.append(Document.doc_metadata[metadata_name].as_float() >= value) - - case _: - pass - - return filters - @classmethod def _extract_variable_selector_to_variable_mapping( cls, diff --git a/api/core/workflow/nodes/loop/entities.py b/api/core/workflow/nodes/loop/entities.py index 4fcad888e4..92a8702fc3 100644 --- a/api/core/workflow/nodes/loop/entities.py +++ b/api/core/workflow/nodes/loop/entities.py @@ -1,3 +1,4 @@ +from enum import StrEnum from typing import Annotated, Any, Literal from pydantic import AfterValidator, BaseModel, Field, field_validator @@ -96,3 +97,8 @@ class LoopState(BaseLoopState): Get current output. """ return self.current_output + + +class LoopCompletedReason(StrEnum): + LOOP_BREAK = "loop_break" + LOOP_COMPLETED = "loop_completed" diff --git a/api/core/workflow/nodes/loop/loop_node.py b/api/core/workflow/nodes/loop/loop_node.py index 1c26bbc2d0..1f9fc8a115 100644 --- a/api/core/workflow/nodes/loop/loop_node.py +++ b/api/core/workflow/nodes/loop/loop_node.py @@ -29,7 +29,7 @@ from core.workflow.node_events import ( ) from core.workflow.nodes.base import LLMUsageTrackingMixin from core.workflow.nodes.base.node import Node -from core.workflow.nodes.loop.entities import LoopNodeData, LoopVariableData +from core.workflow.nodes.loop.entities import LoopCompletedReason, LoopNodeData, LoopVariableData from core.workflow.utils.condition.processor import ConditionProcessor from factories.variable_factory import TypeMismatchError, build_segment_with_type, segment_to_variable from libs.datetime_utils import naive_utc_now @@ -96,6 +96,7 @@ class LoopNode(LLMUsageTrackingMixin, Node[LoopNodeData]): loop_duration_map: dict[str, float] = {} single_loop_variable_map: dict[str, dict[str, Any]] = {} # single loop variable output loop_usage = LLMUsage.empty_usage() + loop_node_ids = self._extract_loop_node_ids_from_config(self.graph_config, self._node_id) # Start Loop event yield LoopStartedEvent( @@ -118,6 +119,8 @@ class LoopNode(LLMUsageTrackingMixin, Node[LoopNodeData]): loop_count = 0 for i in range(loop_count): + # Clear stale variables from previous loop iterations to avoid streaming old values + self._clear_loop_subgraph_variables(loop_node_ids) graph_engine = self._create_graph_engine(start_at=start_at, root_node_id=root_node_id) loop_start_time = naive_utc_now() @@ -177,7 +180,11 @@ class LoopNode(LLMUsageTrackingMixin, Node[LoopNodeData]): WorkflowNodeExecutionMetadataKey.TOTAL_TOKENS: loop_usage.total_tokens, WorkflowNodeExecutionMetadataKey.TOTAL_PRICE: loop_usage.total_price, WorkflowNodeExecutionMetadataKey.CURRENCY: loop_usage.currency, - "completed_reason": "loop_break" if reach_break_condition else "loop_completed", + WorkflowNodeExecutionMetadataKey.COMPLETED_REASON: ( + LoopCompletedReason.LOOP_BREAK + if reach_break_condition + else LoopCompletedReason.LOOP_COMPLETED.value + ), WorkflowNodeExecutionMetadataKey.LOOP_DURATION_MAP: loop_duration_map, WorkflowNodeExecutionMetadataKey.LOOP_VARIABLE_MAP: single_loop_variable_map, }, @@ -274,6 +281,17 @@ class LoopNode(LLMUsageTrackingMixin, Node[LoopNodeData]): if WorkflowNodeExecutionMetadataKey.LOOP_ID not in current_metadata: event.node_run_result.metadata = {**current_metadata, **loop_metadata} + def _clear_loop_subgraph_variables(self, loop_node_ids: set[str]) -> None: + """ + Remove variables produced by loop sub-graph nodes from previous iterations. + + Keeping stale variables causes a freshly created response coordinator in the + next iteration to fall back to outdated values when no stream chunks exist. + """ + variable_pool = self.graph_runtime_state.variable_pool + for node_id in loop_node_ids: + variable_pool.remove([node_id]) + @classmethod def _extract_variable_selector_to_variable_mapping( cls, diff --git a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py index 93db417b15..08e0542d61 100644 --- a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py +++ b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py @@ -281,7 +281,7 @@ class ParameterExtractorNode(Node[ParameterExtractorNodeData]): # handle invoke result - text = invoke_result.message.content or "" + text = invoke_result.message.get_text_content() if not isinstance(text, str): raise InvalidTextContentTypeError(f"Invalid text content type: {type(text)}. Expected str.") diff --git a/api/core/workflow/nodes/start/start_node.py b/api/core/workflow/nodes/start/start_node.py index 38effa79f7..36fc5078c5 100644 --- a/api/core/workflow/nodes/start/start_node.py +++ b/api/core/workflow/nodes/start/start_node.py @@ -1,3 +1,4 @@ +import json from typing import Any from jsonschema import Draft7Validator, ValidationError @@ -42,15 +43,25 @@ class StartNode(Node[StartNodeData]): if value is None and variable.required: raise ValueError(f"{key} is required in input form") - if not isinstance(value, dict): - raise ValueError(f"{key} must be a JSON object") - schema = variable.json_schema if not schema: continue + if not value: + continue + try: - Draft7Validator(schema).validate(value) + json_schema = json.loads(schema) + except json.JSONDecodeError as e: + raise ValueError(f"{schema} must be a valid JSON object") + + try: + json_value = json.loads(value) + except json.JSONDecodeError as e: + raise ValueError(f"{value} must be a valid JSON object") + + try: + Draft7Validator(json_schema).validate(json_value) except ValidationError as e: raise ValueError(f"JSON object for '{key}' does not match schema: {e.message}") - node_inputs[key] = value + node_inputs[key] = json_value diff --git a/api/docker/entrypoint.sh b/api/docker/entrypoint.sh index 6313085e64..5a69eb15ac 100755 --- a/api/docker/entrypoint.sh +++ b/api/docker/entrypoint.sh @@ -34,10 +34,10 @@ if [[ "${MODE}" == "worker" ]]; then if [[ -z "${CELERY_QUEUES}" ]]; then if [[ "${EDITION}" == "CLOUD" ]]; then # Cloud edition: separate queues for dataset and trigger tasks - DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor" + DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention" else # Community edition (SELF_HOSTED): dataset, pipeline and workflow have separate queues - DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor" + DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention" fi else DEFAULT_QUEUES="${CELERY_QUEUES}" @@ -69,6 +69,53 @@ if [[ "${MODE}" == "worker" ]]; then elif [[ "${MODE}" == "beat" ]]; then exec celery -A app.celery beat --loglevel ${LOG_LEVEL:-INFO} + +elif [[ "${MODE}" == "job" ]]; then + # Job mode: Run a one-time Flask command and exit + # Pass Flask command and arguments via container args + # Example K8s usage: + # args: + # - create-tenant + # - --email + # - admin@example.com + # + # Example Docker usage: + # docker run -e MODE=job dify-api:latest create-tenant --email admin@example.com + + if [[ $# -eq 0 ]]; then + echo "Error: No command specified for job mode." + echo "" + echo "Usage examples:" + echo " Kubernetes:" + echo " args: [create-tenant, --email, admin@example.com]" + echo "" + echo " Docker:" + echo " docker run -e MODE=job dify-api create-tenant --email admin@example.com" + echo "" + echo "Available commands:" + echo " create-tenant, reset-password, reset-email, upgrade-db," + echo " vdb-migrate, install-plugins, and more..." + echo "" + echo "Run 'flask --help' to see all available commands." + exit 1 + fi + + echo "Running Flask job command: flask $*" + + # Temporarily disable exit on error to capture exit code + set +e + flask "$@" + JOB_EXIT_CODE=$? + set -e + + if [[ ${JOB_EXIT_CODE} -eq 0 ]]; then + echo "Job completed successfully." + else + echo "Job failed with exit code ${JOB_EXIT_CODE}." + fi + + exit ${JOB_EXIT_CODE} + else if [[ "${DEBUG}" == "true" ]]; then exec flask run --host=${DIFY_BIND_ADDRESS:-0.0.0.0} --port=${DIFY_PORT:-5001} --debug diff --git a/api/extensions/storage/aliyun_oss_storage.py b/api/extensions/storage/aliyun_oss_storage.py index 2283581f62..3d7ef99c9e 100644 --- a/api/extensions/storage/aliyun_oss_storage.py +++ b/api/extensions/storage/aliyun_oss_storage.py @@ -26,6 +26,7 @@ class AliyunOssStorage(BaseStorage): self.bucket_name, connect_timeout=30, region=region, + cloudbox_id=dify_config.ALIYUN_CLOUDBOX_ID, ) def save(self, filename, data): diff --git a/api/extensions/storage/huawei_obs_storage.py b/api/extensions/storage/huawei_obs_storage.py index 74fed26f65..72cb59abbe 100644 --- a/api/extensions/storage/huawei_obs_storage.py +++ b/api/extensions/storage/huawei_obs_storage.py @@ -17,6 +17,7 @@ class HuaweiObsStorage(BaseStorage): access_key_id=dify_config.HUAWEI_OBS_ACCESS_KEY, secret_access_key=dify_config.HUAWEI_OBS_SECRET_KEY, server=dify_config.HUAWEI_OBS_SERVER, + path_style=dify_config.HUAWEI_OBS_PATH_STYLE, ) def save(self, filename, data): diff --git a/api/extensions/storage/opendal_storage.py b/api/extensions/storage/opendal_storage.py index a084844d72..83c5c2d12f 100644 --- a/api/extensions/storage/opendal_storage.py +++ b/api/extensions/storage/opendal_storage.py @@ -87,15 +87,16 @@ class OpenDALStorage(BaseStorage): if not self.exists(path): raise FileNotFoundError("Path not found") - all_files = self.op.scan(path=path) + # Use the new OpenDAL 0.46.0+ API with recursive listing + lister = self.op.list(path, recursive=True) if files and directories: logger.debug("files and directories on %s scanned", path) - return [f.path for f in all_files] + return [entry.path for entry in lister] if files: logger.debug("files on %s scanned", path) - return [f.path for f in all_files if not f.path.endswith("/")] + return [entry.path for entry in lister if not entry.metadata.is_dir] elif directories: logger.debug("directories on %s scanned", path) - return [f.path for f in all_files if f.path.endswith("/")] + return [entry.path for entry in lister if entry.metadata.is_dir] else: raise ValueError("At least one of files or directories must be True") diff --git a/api/extensions/storage/tencent_cos_storage.py b/api/extensions/storage/tencent_cos_storage.py index ea5d982efc..cf092c6973 100644 --- a/api/extensions/storage/tencent_cos_storage.py +++ b/api/extensions/storage/tencent_cos_storage.py @@ -13,12 +13,20 @@ class TencentCosStorage(BaseStorage): super().__init__() self.bucket_name = dify_config.TENCENT_COS_BUCKET_NAME - config = CosConfig( - Region=dify_config.TENCENT_COS_REGION, - SecretId=dify_config.TENCENT_COS_SECRET_ID, - SecretKey=dify_config.TENCENT_COS_SECRET_KEY, - Scheme=dify_config.TENCENT_COS_SCHEME, - ) + if dify_config.TENCENT_COS_CUSTOM_DOMAIN: + config = CosConfig( + Domain=dify_config.TENCENT_COS_CUSTOM_DOMAIN, + SecretId=dify_config.TENCENT_COS_SECRET_ID, + SecretKey=dify_config.TENCENT_COS_SECRET_KEY, + Scheme=dify_config.TENCENT_COS_SCHEME, + ) + else: + config = CosConfig( + Region=dify_config.TENCENT_COS_REGION, + SecretId=dify_config.TENCENT_COS_SECRET_ID, + SecretKey=dify_config.TENCENT_COS_SECRET_KEY, + Scheme=dify_config.TENCENT_COS_SCHEME, + ) self.client = CosS3Client(config) def save(self, filename, data): diff --git a/api/libs/helper.py b/api/libs/helper.py index 4a7afe0bda..74e1808e49 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -11,6 +11,7 @@ from collections.abc import Generator, Mapping from datetime import datetime from hashlib import sha256 from typing import TYPE_CHECKING, Annotated, Any, Optional, Union, cast +from uuid import UUID from zoneinfo import available_timezones from flask import Response, stream_with_context @@ -119,6 +120,19 @@ def uuid_value(value: Any) -> str: raise ValueError(error) +def normalize_uuid(value: str | UUID) -> str: + if not value: + return "" + + try: + return uuid_value(value) + except ValueError as exc: + raise ValueError("must be a valid UUID") from exc + + +UUIDStrOrEmpty = Annotated[str, AfterValidator(normalize_uuid)] + + def alphanumeric(value: str): # check if the value is alphanumeric and underlined if re.match(r"^[a-zA-Z0-9_]+$", value): diff --git a/api/pyproject.toml b/api/pyproject.toml index 6fcbc0f25c..dbc6a2eb83 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dify-api" -version = "1.11.1" +version = "1.11.2" requires-python = ">=3.11,<3.13" dependencies = [ @@ -12,7 +12,7 @@ dependencies = [ "bs4~=0.0.1", "cachetools~=5.3.0", "celery~=5.5.2", - "chardet~=5.1.0", + "charset-normalizer>=3.4.4", "flask~=3.1.2", "flask-compress>=1.17,<1.18", "flask-cors~=6.0.0", @@ -32,6 +32,7 @@ dependencies = [ "httpx[socks]~=0.27.0", "jieba==0.42.1", "json-repair>=0.41.1", + "jsonschema>=4.25.1", "langfuse~=2.51.3", "langsmith~=0.1.77", "markdown~=3.5.1", @@ -68,7 +69,7 @@ dependencies = [ "pydantic-extra-types~=2.10.3", "pydantic-settings~=2.11.0", "pyjwt~=2.10.1", - "pypdfium2==4.30.0", + "pypdfium2==5.2.0", "python-docx~=1.1.0", "python-dotenv==1.0.1", "pyyaml~=6.0.1", diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index 1dd6faea5d..deba0b79e8 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -155,6 +155,7 @@ class AppDslService: parsed_url.scheme == "https" and parsed_url.netloc == "github.com" and parsed_url.path.endswith((".yml", ".yaml")) + and "/blob/" in parsed_url.path ): yaml_url = yaml_url.replace("https://github.com", "https://raw.githubusercontent.com") yaml_url = yaml_url.replace("/blob/", "/") diff --git a/api/services/app_generate_service.py b/api/services/app_generate_service.py index 4514c86f7c..cc58899dc4 100644 --- a/api/services/app_generate_service.py +++ b/api/services/app_generate_service.py @@ -14,7 +14,8 @@ from enums.quota_type import QuotaType, unlimited from extensions.otel import AppGenerateHandler, trace_span from models.model import Account, App, AppMode, EndUser from models.workflow import Workflow -from services.errors.app import InvokeRateLimitError, QuotaExceededError, WorkflowIdFormatError, WorkflowNotFoundError +from services.errors.app import QuotaExceededError, WorkflowIdFormatError, WorkflowNotFoundError +from services.errors.llm import InvokeRateLimitError from services.workflow_service import WorkflowService diff --git a/api/services/async_workflow_service.py b/api/services/async_workflow_service.py index e100582511..bc73b7c8c2 100644 --- a/api/services/async_workflow_service.py +++ b/api/services/async_workflow_service.py @@ -21,7 +21,7 @@ from models.model import App, EndUser from models.trigger import WorkflowTriggerLog from models.workflow import Workflow from repositories.sqlalchemy_workflow_trigger_log_repository import SQLAlchemyWorkflowTriggerLogRepository -from services.errors.app import InvokeRateLimitError, QuotaExceededError, WorkflowNotFoundError +from services.errors.app import QuotaExceededError, WorkflowNotFoundError, WorkflowQuotaLimitError from services.workflow.entities import AsyncTriggerResponse, TriggerData, WorkflowTaskData from services.workflow.queue_dispatcher import QueueDispatcherManager, QueuePriority from services.workflow_service import WorkflowService @@ -141,7 +141,7 @@ class AsyncWorkflowService: trigger_log_repo.update(trigger_log) session.commit() - raise InvokeRateLimitError( + raise WorkflowQuotaLimitError( f"Workflow execution quota limit reached for tenant {trigger_data.tenant_id}" ) from e diff --git a/api/services/auth/firecrawl/firecrawl.py b/api/services/auth/firecrawl/firecrawl.py index d455475bfc..b002706931 100644 --- a/api/services/auth/firecrawl/firecrawl.py +++ b/api/services/auth/firecrawl/firecrawl.py @@ -26,7 +26,7 @@ class FirecrawlAuth(ApiKeyAuthBase): "limit": 1, "scrapeOptions": {"onlyMainContent": True}, } - response = self._post_request(f"{self.base_url}/v1/crawl", options, headers) + response = self._post_request(self._build_url("v1/crawl"), options, headers) if response.status_code == 200: return True else: @@ -35,15 +35,17 @@ class FirecrawlAuth(ApiKeyAuthBase): def _prepare_headers(self): return {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"} + def _build_url(self, path: str) -> str: + # ensure exactly one slash between base and path, regardless of user-provided base_url + return f"{self.base_url.rstrip('/')}/{path.lstrip('/')}" + def _post_request(self, url, data, headers): return httpx.post(url, headers=headers, json=data) def _handle_error(self, response): - if response.status_code in {402, 409, 500}: - error_message = response.json().get("error", "Unknown error occurred") - raise Exception(f"Failed to authorize. Status code: {response.status_code}. Error: {error_message}") - else: - if response.text: - error_message = json.loads(response.text).get("error", "Unknown error occurred") - raise Exception(f"Failed to authorize. Status code: {response.status_code}. Error: {error_message}") - raise Exception(f"Unexpected error occurred while trying to authorize. Status code: {response.status_code}") + try: + payload = response.json() + except json.JSONDecodeError: + payload = {} + error_message = payload.get("error") or payload.get("message") or (response.text or "Unknown error occurred") + raise Exception(f"Failed to authorize. Status code: {response.status_code}. Error: {error_message}") diff --git a/api/services/billing_service.py b/api/services/billing_service.py index 54e1c9d285..26ce8cad33 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -1,8 +1,13 @@ +import json +import logging import os +from collections.abc import Sequence from typing import Literal import httpx +from pydantic import TypeAdapter from tenacity import retry, retry_if_exception_type, stop_before_delay, wait_fixed +from typing_extensions import TypedDict from werkzeug.exceptions import InternalServerError from enums.cloud_plan import CloudPlan @@ -11,6 +16,15 @@ from extensions.ext_redis import redis_client from libs.helper import RateLimiter from models import Account, TenantAccountJoin, TenantAccountRole +logger = logging.getLogger(__name__) + + +class SubscriptionPlan(TypedDict): + """Tenant subscriptionplan information.""" + + plan: str + expiration_date: int + class BillingService: base_url = os.environ.get("BILLING_API_URL", "BILLING_API_URL") @@ -18,6 +32,11 @@ class BillingService: compliance_download_rate_limiter = RateLimiter("compliance_download_rate_limiter", 4, 60) + # Redis key prefix for tenant plan cache + _PLAN_CACHE_KEY_PREFIX = "tenant_plan:" + # Cache TTL: 10 minutes + _PLAN_CACHE_TTL = 600 + @classmethod def get_info(cls, tenant_id: str): params = {"tenant_id": tenant_id} @@ -239,3 +258,135 @@ class BillingService: def sync_partner_tenants_bindings(cls, account_id: str, partner_key: str, click_id: str): payload = {"account_id": account_id, "click_id": click_id} return cls._send_request("PUT", f"/partners/{partner_key}/tenants", json=payload) + + @classmethod + def get_plan_bulk(cls, tenant_ids: Sequence[str]) -> dict[str, SubscriptionPlan]: + """ + Bulk fetch billing subscription plan via billing API. + Payload: {"tenant_ids": ["t1", "t2", ...]} (max 200 per request) + Returns: + Mapping of tenant_id -> {plan: str, expiration_date: int} + """ + results: dict[str, SubscriptionPlan] = {} + subscription_adapter = TypeAdapter(SubscriptionPlan) + + chunk_size = 200 + for i in range(0, len(tenant_ids), chunk_size): + chunk = tenant_ids[i : i + chunk_size] + try: + resp = cls._send_request("POST", "/subscription/plan/batch", json={"tenant_ids": chunk}) + data = resp.get("data", {}) + + for tenant_id, plan in data.items(): + try: + subscription_plan = subscription_adapter.validate_python(plan) + results[tenant_id] = subscription_plan + except Exception: + logger.exception( + "get_plan_bulk: failed to validate subscription plan for tenant(%s)", tenant_id + ) + continue + except Exception: + logger.exception("get_plan_bulk: failed to fetch billing info batch for tenants: %s", chunk) + continue + + return results + + @classmethod + def _make_plan_cache_key(cls, tenant_id: str) -> str: + return f"{cls._PLAN_CACHE_KEY_PREFIX}{tenant_id}" + + @classmethod + def get_plan_bulk_with_cache(cls, tenant_ids: Sequence[str]) -> dict[str, SubscriptionPlan]: + """ + Bulk fetch billing subscription plan with cache to reduce billing API loads in batch job scenarios. + + NOTE: if you want to high data consistency, use get_plan_bulk instead. + + Returns: + Mapping of tenant_id -> {plan: str, expiration_date: int} + """ + tenant_plans: dict[str, SubscriptionPlan] = {} + + if not tenant_ids: + return tenant_plans + + subscription_adapter = TypeAdapter(SubscriptionPlan) + + # Step 1: Batch fetch from Redis cache using mget + redis_keys = [cls._make_plan_cache_key(tenant_id) for tenant_id in tenant_ids] + try: + cached_values = redis_client.mget(redis_keys) + + if len(cached_values) != len(tenant_ids): + raise Exception( + "get_plan_bulk_with_cache: unexpected error: redis mget failed: cached values length mismatch" + ) + + # Map cached values back to tenant_ids + cache_misses: list[str] = [] + + for tenant_id, cached_value in zip(tenant_ids, cached_values): + if cached_value: + try: + # Redis returns bytes, decode to string and parse JSON + json_str = cached_value.decode("utf-8") if isinstance(cached_value, bytes) else cached_value + plan_dict = json.loads(json_str) + subscription_plan = subscription_adapter.validate_python(plan_dict) + tenant_plans[tenant_id] = subscription_plan + except Exception: + logger.exception( + "get_plan_bulk_with_cache: process tenant(%s) failed, add to cache misses", tenant_id + ) + cache_misses.append(tenant_id) + else: + cache_misses.append(tenant_id) + + logger.info( + "get_plan_bulk_with_cache: cache hits=%s, cache misses=%s", + len(tenant_plans), + len(cache_misses), + ) + except Exception: + logger.exception("get_plan_bulk_with_cache: redis mget failed, falling back to API") + cache_misses = list(tenant_ids) + + # Step 2: Fetch missing plans from billing API + if cache_misses: + bulk_plans = BillingService.get_plan_bulk(cache_misses) + + if bulk_plans: + plans_to_cache: dict[str, SubscriptionPlan] = {} + + for tenant_id, subscription_plan in bulk_plans.items(): + tenant_plans[tenant_id] = subscription_plan + plans_to_cache[tenant_id] = subscription_plan + + # Step 3: Batch update Redis cache using pipeline + if plans_to_cache: + try: + pipe = redis_client.pipeline() + for tenant_id, subscription_plan in plans_to_cache.items(): + redis_key = cls._make_plan_cache_key(tenant_id) + # Serialize dict to JSON string + json_str = json.dumps(subscription_plan) + pipe.setex(redis_key, cls._PLAN_CACHE_TTL, json_str) + pipe.execute() + + logger.info( + "get_plan_bulk_with_cache: cached %s new tenant plans to Redis", + len(plans_to_cache), + ) + except Exception: + logger.exception("get_plan_bulk_with_cache: redis pipeline failed") + + return tenant_plans + + @classmethod + def get_expired_subscription_cleanup_whitelist(cls) -> Sequence[str]: + resp = cls._send_request("GET", "/subscription/cleanup/whitelist") + data = resp.get("data", []) + tenant_whitelist = [] + for item in data: + tenant_whitelist.append(item["tenant_id"]) + return tenant_whitelist diff --git a/api/services/conversation_service.py b/api/services/conversation_service.py index 5253199552..659e7406fb 100644 --- a/api/services/conversation_service.py +++ b/api/services/conversation_service.py @@ -6,7 +6,9 @@ from typing import Any, Union from sqlalchemy import asc, desc, func, or_, select from sqlalchemy.orm import Session +from configs import dify_config from core.app.entities.app_invoke_entities import InvokeFrom +from core.db.session_factory import session_factory from core.llm_generator.llm_generator import LLMGenerator from core.variables.types import SegmentType from core.workflow.nodes.variable_assigner.common.impl import conversation_variable_updater_factory @@ -202,6 +204,7 @@ class ConversationService: user: Union[Account, EndUser] | None, limit: int, last_id: str | None, + variable_name: str | None = None, ) -> InfiniteScrollPagination: conversation = cls.get_conversation(app_model, conversation_id, user) @@ -212,7 +215,25 @@ class ConversationService: .order_by(ConversationVariable.created_at) ) - with Session(db.engine) as session: + # Apply variable_name filter if provided + if variable_name: + # Filter using JSON extraction to match variable names case-insensitively + escaped_variable_name = variable_name.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + # Filter using JSON extraction to match variable names case-insensitively + if dify_config.DB_TYPE in ["mysql", "oceanbase", "seekdb"]: + stmt = stmt.where( + func.json_extract(ConversationVariable.data, "$.name").ilike( + f"%{escaped_variable_name}%", escape="\\" + ) + ) + elif dify_config.DB_TYPE == "postgresql": + stmt = stmt.where( + func.json_extract_path_text(ConversationVariable.data, "name").ilike( + f"%{escaped_variable_name}%", escape="\\" + ) + ) + + with session_factory.create_session() as session: if last_id: last_variable = session.scalar(stmt.where(ConversationVariable.id == last_id)) if not last_variable: @@ -279,7 +300,7 @@ class ConversationService: .where(ConversationVariable.id == variable_id) ) - with Session(db.engine) as session: + with session_factory.create_session() as session: existing_variable = session.scalar(stmt) if not existing_variable: raise ConversationVariableNotExistsError() diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 970192fde5..ac4b25c5dc 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -3458,7 +3458,7 @@ class SegmentService: if keyword: query = query.where(DocumentSegment.content.ilike(f"%{keyword}%")) - query = query.order_by(DocumentSegment.position.asc()) + query = query.order_by(DocumentSegment.position.asc(), DocumentSegment.id.asc()) paginated_segments = db.paginate(select=query, page=page, per_page=limit, max_per_page=100, error_out=False) return paginated_segments.items, paginated_segments.total diff --git a/api/services/enterprise/enterprise_service.py b/api/services/enterprise/enterprise_service.py index 83d0fcf296..c0cc0e5233 100644 --- a/api/services/enterprise/enterprise_service.py +++ b/api/services/enterprise/enterprise_service.py @@ -110,5 +110,5 @@ class EnterpriseService: if not app_id: raise ValueError("app_id must be provided.") - body = {"appId": app_id} - EnterpriseRequest.send_request("DELETE", "/webapp/clean", json=body) + params = {"appId": app_id} + EnterpriseRequest.send_request("DELETE", "/webapp/clean", params=params) diff --git a/api/services/entities/knowledge_entities/rag_pipeline_entities.py b/api/services/entities/knowledge_entities/rag_pipeline_entities.py index a97ccab914..cbb0efcc2a 100644 --- a/api/services/entities/knowledge_entities/rag_pipeline_entities.py +++ b/api/services/entities/knowledge_entities/rag_pipeline_entities.py @@ -23,7 +23,7 @@ class RagPipelineDatasetCreateEntity(BaseModel): description: str icon_info: IconInfo permission: str - partial_member_list: list[str] | None = None + partial_member_list: list[dict[str, str]] | None = None yaml_content: str | None = None diff --git a/api/services/errors/app.py b/api/services/errors/app.py index 24e4760acc..60e59e97dc 100644 --- a/api/services/errors/app.py +++ b/api/services/errors/app.py @@ -18,8 +18,8 @@ class WorkflowIdFormatError(Exception): pass -class InvokeRateLimitError(Exception): - """Raised when rate limit is exceeded for workflow invocations.""" +class WorkflowQuotaLimitError(Exception): + """Raised when workflow execution quota is exceeded (for async/background workflows).""" pass diff --git a/api/services/plugin/plugin_parameter_service.py b/api/services/plugin/plugin_parameter_service.py index c517d9f966..40565c56ed 100644 --- a/api/services/plugin/plugin_parameter_service.py +++ b/api/services/plugin/plugin_parameter_service.py @@ -105,3 +105,49 @@ class PluginParameterService: ) .options ) + + @staticmethod + def get_dynamic_select_options_with_credentials( + tenant_id: str, + user_id: str, + plugin_id: str, + provider: str, + action: str, + parameter: str, + credential_id: str, + credentials: Mapping[str, Any], + ) -> Sequence[PluginParameterOption]: + """ + Get dynamic select options using provided credentials directly. + Used for edit mode when credentials have been modified but not yet saved. + + Security: credential_id is validated against tenant_id to ensure + users can only access their own credentials. + """ + from constants import HIDDEN_VALUE + + # Get original subscription to replace hidden values (with tenant_id check for security) + original_subscription = TriggerProviderService.get_subscription_by_id(tenant_id, credential_id) + if not original_subscription: + raise ValueError(f"Subscription {credential_id} not found") + + # Replace [__HIDDEN__] with original values + resolved_credentials: dict[str, Any] = { + key: (original_subscription.credentials.get(key) if value == HIDDEN_VALUE else value) + for key, value in credentials.items() + } + + return ( + DynamicSelectClient() + .fetch_dynamic_select_options( + tenant_id, + user_id, + plugin_id, + provider, + action, + resolved_credentials, + original_subscription.credential_type or CredentialType.UNAUTHORIZED.value, + parameter, + ) + .options + ) diff --git a/api/services/tools/api_tools_manage_service.py b/api/services/tools/api_tools_manage_service.py index b3b6e36346..250d29f335 100644 --- a/api/services/tools/api_tools_manage_service.py +++ b/api/services/tools/api_tools_manage_service.py @@ -7,7 +7,6 @@ from httpx import get from sqlalchemy import select from core.entities.provider_entities import ProviderConfig -from core.helper.tool_provider_cache import ToolProviderListCache from core.model_runtime.utils.encoders import jsonable_encoder from core.tools.__base.tool_runtime import ToolRuntime from core.tools.custom_tool.provider import ApiToolProviderController @@ -178,9 +177,6 @@ class ApiToolManageService: # update labels ToolLabelManager.update_tool_labels(provider_controller, labels) - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) - return {"result": "success"} @staticmethod @@ -322,9 +318,6 @@ class ApiToolManageService: # update labels ToolLabelManager.update_tool_labels(provider_controller, labels) - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) - return {"result": "success"} @staticmethod @@ -347,9 +340,6 @@ class ApiToolManageService: db.session.delete(provider) db.session.commit() - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) - return {"result": "success"} @staticmethod diff --git a/api/services/tools/builtin_tools_manage_service.py b/api/services/tools/builtin_tools_manage_service.py index cf1d39fa25..6797a67dde 100644 --- a/api/services/tools/builtin_tools_manage_service.py +++ b/api/services/tools/builtin_tools_manage_service.py @@ -12,7 +12,6 @@ from constants import HIDDEN_VALUE, UNKNOWN_VALUE from core.helper.name_generator import generate_incremental_name from core.helper.position_helper import is_filtered from core.helper.provider_cache import NoOpProviderCredentialCache, ToolProviderCredentialsCache -from core.helper.tool_provider_cache import ToolProviderListCache from core.plugin.entities.plugin_daemon import CredentialType from core.tools.builtin_tool.provider import BuiltinToolProviderController from core.tools.builtin_tool.providers._positions import BuiltinToolProviderSort @@ -205,9 +204,6 @@ class BuiltinToolManageService: db_provider.name = name session.commit() - - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) except Exception as e: session.rollback() raise ValueError(str(e)) @@ -286,12 +282,10 @@ class BuiltinToolManageService: session.add(db_provider) session.commit() - - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) except Exception as e: session.rollback() raise ValueError(str(e)) + return {"result": "success"} @staticmethod @@ -409,9 +403,6 @@ class BuiltinToolManageService: ) cache.delete() - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) - return {"result": "success"} @staticmethod @@ -434,8 +425,6 @@ class BuiltinToolManageService: target_provider.is_default = True session.commit() - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) return {"result": "success"} @staticmethod diff --git a/api/services/tools/mcp_tools_manage_service.py b/api/services/tools/mcp_tools_manage_service.py index d641fe0315..0be106f597 100644 --- a/api/services/tools/mcp_tools_manage_service.py +++ b/api/services/tools/mcp_tools_manage_service.py @@ -15,7 +15,6 @@ from sqlalchemy.orm import Session from core.entities.mcp_provider import MCPAuthentication, MCPConfiguration, MCPProviderEntity from core.helper import encrypter from core.helper.provider_cache import NoOpProviderCredentialCache -from core.helper.tool_provider_cache import ToolProviderListCache from core.mcp.auth.auth_flow import auth from core.mcp.auth_client import MCPClientWithAuthRetry from core.mcp.error import MCPAuthError, MCPError @@ -65,6 +64,15 @@ class ServerUrlValidationResult(BaseModel): return self.needs_validation and self.validation_passed and self.reconnect_result is not None +class ProviderUrlValidationData(BaseModel): + """Data required for URL validation, extracted from database to perform network operations outside of session""" + + current_server_url_hash: str + headers: dict[str, str] + timeout: float | None + sse_read_timeout: float | None + + class MCPToolManageService: """Service class for managing MCP tools and providers.""" @@ -166,9 +174,6 @@ class MCPToolManageService: self._session.add(mcp_tool) self._session.flush() - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) - mcp_providers = ToolTransformService.mcp_provider_to_user_provider(mcp_tool, for_list=True) return mcp_providers @@ -192,7 +197,7 @@ class MCPToolManageService: Update an MCP provider. Args: - validation_result: Pre-validation result from validate_server_url_change. + validation_result: Pre-validation result from validate_server_url_standalone. If provided and contains reconnect_result, it will be used instead of performing network operations. """ @@ -251,8 +256,6 @@ class MCPToolManageService: # Flush changes to database self._session.flush() - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) except IntegrityError as e: self._handle_integrity_error(e, name, server_url, server_identifier) @@ -261,9 +264,6 @@ class MCPToolManageService: mcp_tool = self.get_provider(provider_id=provider_id, tenant_id=tenant_id) self._session.delete(mcp_tool) - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) - def list_providers( self, *, tenant_id: str, for_list: bool = False, include_sensitive: bool = True ) -> list[ToolProviderApiEntity]: @@ -319,8 +319,14 @@ class MCPToolManageService: except MCPError as e: raise ValueError(f"Failed to connect to MCP server: {e}") - # Update database with retrieved tools - db_provider.tools = json.dumps([tool.model_dump() for tool in tools]) + # Update database with retrieved tools (ensure description is a non-null string) + tools_payload = [] + for tool in tools: + data = tool.model_dump() + if data.get("description") is None: + data["description"] = "" + tools_payload.append(data) + db_provider.tools = json.dumps(tools_payload) db_provider.authed = True db_provider.updated_at = datetime.now() self._session.flush() @@ -546,30 +552,39 @@ class MCPToolManageService: ) return self.execute_auth_actions(auth_result) - def _reconnect_provider(self, *, server_url: str, provider: MCPToolProvider) -> ReconnectResult: - """Attempt to reconnect to MCP provider with new server URL.""" + def get_provider_for_url_validation(self, *, tenant_id: str, provider_id: str) -> ProviderUrlValidationData: + """ + Get provider data required for URL validation. + This method performs database read and should be called within a session. + + Returns: + ProviderUrlValidationData: Data needed for standalone URL validation + """ + provider = self.get_provider(provider_id=provider_id, tenant_id=tenant_id) provider_entity = provider.to_entity() - headers = provider_entity.headers + return ProviderUrlValidationData( + current_server_url_hash=provider.server_url_hash, + headers=provider_entity.headers, + timeout=provider_entity.timeout, + sse_read_timeout=provider_entity.sse_read_timeout, + ) - try: - tools = self._retrieve_remote_mcp_tools(server_url, headers, provider_entity) - return ReconnectResult( - authed=True, - tools=json.dumps([tool.model_dump() for tool in tools]), - encrypted_credentials=EMPTY_CREDENTIALS_JSON, - ) - except MCPAuthError: - return ReconnectResult(authed=False, tools=EMPTY_TOOLS_JSON, encrypted_credentials=EMPTY_CREDENTIALS_JSON) - except MCPError as e: - raise ValueError(f"Failed to re-connect MCP server: {e}") from e - - def validate_server_url_change( - self, *, tenant_id: str, provider_id: str, new_server_url: str + @staticmethod + def validate_server_url_standalone( + *, + tenant_id: str, + new_server_url: str, + validation_data: ProviderUrlValidationData, ) -> ServerUrlValidationResult: """ Validate server URL change by attempting to connect to the new server. - This method should be called BEFORE update_provider to perform network operations - outside of the database transaction. + This method performs network operations and MUST be called OUTSIDE of any database session + to avoid holding locks during network I/O. + + Args: + tenant_id: Tenant ID for encryption + new_server_url: The new server URL to validate + validation_data: Provider data obtained from get_provider_for_url_validation Returns: ServerUrlValidationResult: Validation result with connection status and tools if successful @@ -579,25 +594,30 @@ class MCPToolManageService: return ServerUrlValidationResult(needs_validation=False) # Validate URL format - if not self._is_valid_url(new_server_url): + parsed = urlparse(new_server_url) + if not all([parsed.scheme, parsed.netloc]) or parsed.scheme not in ["http", "https"]: raise ValueError("Server URL is not valid.") # Always encrypt and hash the URL encrypted_server_url = encrypter.encrypt_token(tenant_id, new_server_url) new_server_url_hash = hashlib.sha256(new_server_url.encode()).hexdigest() - # Get current provider - provider = self.get_provider(provider_id=provider_id, tenant_id=tenant_id) - # Check if URL is actually different - if new_server_url_hash == provider.server_url_hash: + if new_server_url_hash == validation_data.current_server_url_hash: # URL hasn't changed, but still return the encrypted data return ServerUrlValidationResult( - needs_validation=False, encrypted_server_url=encrypted_server_url, server_url_hash=new_server_url_hash + needs_validation=False, + encrypted_server_url=encrypted_server_url, + server_url_hash=new_server_url_hash, ) - # Perform validation by attempting to connect - reconnect_result = self._reconnect_provider(server_url=new_server_url, provider=provider) + # Perform network validation - this is the expensive operation that should be outside session + reconnect_result = MCPToolManageService._reconnect_with_url( + server_url=new_server_url, + headers=validation_data.headers, + timeout=validation_data.timeout, + sse_read_timeout=validation_data.sse_read_timeout, + ) return ServerUrlValidationResult( needs_validation=True, validation_passed=True, @@ -606,6 +626,60 @@ class MCPToolManageService: server_url_hash=new_server_url_hash, ) + @staticmethod + def reconnect_with_url( + *, + server_url: str, + headers: dict[str, str], + timeout: float | None, + sse_read_timeout: float | None, + ) -> ReconnectResult: + return MCPToolManageService._reconnect_with_url( + server_url=server_url, + headers=headers, + timeout=timeout, + sse_read_timeout=sse_read_timeout, + ) + + @staticmethod + def _reconnect_with_url( + *, + server_url: str, + headers: dict[str, str], + timeout: float | None, + sse_read_timeout: float | None, + ) -> ReconnectResult: + """ + Attempt to connect to MCP server with given URL. + This is a static method that performs network I/O without database access. + """ + from core.mcp.mcp_client import MCPClient + + try: + with MCPClient( + server_url=server_url, + headers=headers, + timeout=timeout, + sse_read_timeout=sse_read_timeout, + ) as mcp_client: + tools = mcp_client.list_tools() + # Ensure tool descriptions are non-null in payload + tools_payload = [] + for t in tools: + d = t.model_dump() + if d.get("description") is None: + d["description"] = "" + tools_payload.append(d) + return ReconnectResult( + authed=True, + tools=json.dumps(tools_payload), + encrypted_credentials=EMPTY_CREDENTIALS_JSON, + ) + except MCPAuthError: + return ReconnectResult(authed=False, tools=EMPTY_TOOLS_JSON, encrypted_credentials=EMPTY_CREDENTIALS_JSON) + except MCPError as e: + raise ValueError(f"Failed to re-connect MCP server: {e}") from e + def _build_tool_provider_response( self, db_provider: MCPToolProvider, provider_entity: MCPProviderEntity, tools: list ) -> ToolProviderApiEntity: diff --git a/api/services/tools/tools_manage_service.py b/api/services/tools/tools_manage_service.py index 038c462f15..51e9120b8d 100644 --- a/api/services/tools/tools_manage_service.py +++ b/api/services/tools/tools_manage_service.py @@ -1,6 +1,5 @@ import logging -from core.helper.tool_provider_cache import ToolProviderListCache from core.tools.entities.api_entities import ToolProviderTypeApiLiteral from core.tools.tool_manager import ToolManager from services.tools.tools_transform_service import ToolTransformService @@ -16,14 +15,6 @@ class ToolCommonService: :return: the list of tool providers """ - # Try to get from cache first - cached_result = ToolProviderListCache.get_cached_providers(tenant_id, typ) - if cached_result is not None: - logger.debug("Returning cached tool providers for tenant %s, type %s", tenant_id, typ) - return cached_result - - # Cache miss - fetch from database - logger.debug("Cache miss for tool providers, fetching from database for tenant %s, type %s", tenant_id, typ) providers = ToolManager.list_providers_from_api(user_id, tenant_id, typ) # add icon @@ -32,7 +23,4 @@ class ToolCommonService: result = [provider.to_dict() for provider in providers] - # Cache the result - ToolProviderListCache.set_cached_providers(tenant_id, typ, result) - return result diff --git a/api/services/tools/workflow_tools_manage_service.py b/api/services/tools/workflow_tools_manage_service.py index fe77ff2dc5..ab5d5480df 100644 --- a/api/services/tools/workflow_tools_manage_service.py +++ b/api/services/tools/workflow_tools_manage_service.py @@ -7,7 +7,6 @@ from typing import Any from sqlalchemy import or_, select from sqlalchemy.orm import Session -from core.helper.tool_provider_cache import ToolProviderListCache from core.model_runtime.utils.encoders import jsonable_encoder from core.tools.__base.tool_provider import ToolProviderController from core.tools.entities.api_entities import ToolApiEntity, ToolProviderApiEntity @@ -68,34 +67,31 @@ class WorkflowToolManageService: if workflow is None: raise ValueError(f"Workflow not found for app {workflow_app_id}") - with Session(db.engine, expire_on_commit=False) as session, session.begin(): - workflow_tool_provider = WorkflowToolProvider( - tenant_id=tenant_id, - user_id=user_id, - app_id=workflow_app_id, - name=name, - label=label, - icon=json.dumps(icon), - description=description, - parameter_configuration=json.dumps(parameters), - privacy_policy=privacy_policy, - version=workflow.version, - ) - session.add(workflow_tool_provider) + workflow_tool_provider = WorkflowToolProvider( + tenant_id=tenant_id, + user_id=user_id, + app_id=workflow_app_id, + name=name, + label=label, + icon=json.dumps(icon), + description=description, + parameter_configuration=json.dumps(parameters), + privacy_policy=privacy_policy, + version=workflow.version, + ) try: WorkflowToolProviderController.from_db(workflow_tool_provider) except Exception as e: raise ValueError(str(e)) + with Session(db.engine, expire_on_commit=False) as session, session.begin(): + session.add(workflow_tool_provider) + if labels is not None: ToolLabelManager.update_tool_labels( ToolTransformService.workflow_provider_to_controller(workflow_tool_provider), labels ) - - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) - return {"result": "success"} @classmethod @@ -183,9 +179,6 @@ class WorkflowToolManageService: ToolTransformService.workflow_provider_to_controller(workflow_tool_provider), labels ) - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) - return {"result": "success"} @classmethod @@ -248,9 +241,6 @@ class WorkflowToolManageService: db.session.commit() - # Invalidate tool providers cache - ToolProviderListCache.invalidate_cache(tenant_id) - return {"result": "success"} @classmethod diff --git a/api/services/trigger/trigger_provider_service.py b/api/services/trigger/trigger_provider_service.py index 668e4c5be2..ef77c33c1b 100644 --- a/api/services/trigger/trigger_provider_service.py +++ b/api/services/trigger/trigger_provider_service.py @@ -94,16 +94,23 @@ class TriggerProviderService: provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id) for subscription in subscriptions: - encrypter, _ = create_trigger_provider_encrypter_for_subscription( + credential_encrypter, _ = create_trigger_provider_encrypter_for_subscription( tenant_id=tenant_id, controller=provider_controller, subscription=subscription, ) subscription.credentials = dict( - encrypter.mask_credentials(dict(encrypter.decrypt(subscription.credentials))) + credential_encrypter.mask_credentials(dict(credential_encrypter.decrypt(subscription.credentials))) ) - subscription.properties = dict(encrypter.mask_credentials(dict(encrypter.decrypt(subscription.properties)))) - subscription.parameters = dict(encrypter.mask_credentials(dict(encrypter.decrypt(subscription.parameters)))) + properties_encrypter, _ = create_trigger_provider_encrypter_for_properties( + tenant_id=tenant_id, + controller=provider_controller, + subscription=subscription, + ) + subscription.properties = dict( + properties_encrypter.mask_credentials(dict(properties_encrypter.decrypt(subscription.properties))) + ) + subscription.parameters = dict(subscription.parameters) count = workflows_in_use_map.get(subscription.id) subscription.workflows_in_use = count if count is not None else 0 @@ -209,6 +216,101 @@ class TriggerProviderService: logger.exception("Failed to add trigger provider") raise ValueError(str(e)) + @classmethod + def update_trigger_subscription( + cls, + tenant_id: str, + subscription_id: str, + name: str | None = None, + properties: Mapping[str, Any] | None = None, + parameters: Mapping[str, Any] | None = None, + credentials: Mapping[str, Any] | None = None, + credential_expires_at: int | None = None, + expires_at: int | None = None, + ) -> None: + """ + Update an existing trigger subscription. + + :param tenant_id: Tenant ID + :param subscription_id: Subscription instance ID + :param name: Optional new name for this subscription + :param properties: Optional new properties + :param parameters: Optional new parameters + :param credentials: Optional new credentials + :param credential_expires_at: Optional new credential expiration timestamp + :param expires_at: Optional new expiration timestamp + :return: Success response with updated subscription info + """ + with Session(db.engine, expire_on_commit=False) as session: + # Use distributed lock to prevent race conditions on the same subscription + lock_key = f"trigger_subscription_update_lock:{tenant_id}_{subscription_id}" + with redis_client.lock(lock_key, timeout=20): + subscription: TriggerSubscription | None = ( + session.query(TriggerSubscription).filter_by(tenant_id=tenant_id, id=subscription_id).first() + ) + if not subscription: + raise ValueError(f"Trigger subscription {subscription_id} not found") + + provider_id = TriggerProviderID(subscription.provider_id) + provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id) + + # Check for name uniqueness if name is being updated + if name is not None and name != subscription.name: + existing = ( + session.query(TriggerSubscription) + .filter_by(tenant_id=tenant_id, provider_id=str(provider_id), name=name) + .first() + ) + if existing: + raise ValueError(f"Subscription name '{name}' already exists for this provider") + subscription.name = name + + # Update properties if provided + if properties is not None: + properties_encrypter, _ = create_provider_encrypter( + tenant_id=tenant_id, + config=provider_controller.get_properties_schema(), + cache=NoOpProviderCredentialCache(), + ) + # Handle hidden values - preserve original encrypted values + original_properties = properties_encrypter.decrypt(subscription.properties) + new_properties: dict[str, Any] = { + key: value if value != HIDDEN_VALUE else original_properties.get(key, UNKNOWN_VALUE) + for key, value in properties.items() + } + subscription.properties = dict(properties_encrypter.encrypt(new_properties)) + + # Update parameters if provided + if parameters is not None: + subscription.parameters = dict(parameters) + + # Update credentials if provided + if credentials is not None: + credential_type = CredentialType.of(subscription.credential_type) + credential_encrypter, _ = create_provider_encrypter( + tenant_id=tenant_id, + config=provider_controller.get_credential_schema_config(credential_type), + cache=NoOpProviderCredentialCache(), + ) + subscription.credentials = dict(credential_encrypter.encrypt(dict(credentials))) + + # Update credential expiration timestamp if provided + if credential_expires_at is not None: + subscription.credential_expires_at = credential_expires_at + + # Update expiration timestamp if provided + if expires_at is not None: + subscription.expires_at = expires_at + + session.commit() + + # Clear subscription cache + delete_cache_for_subscription( + tenant_id=tenant_id, + provider_id=subscription.provider_id, + subscription_id=subscription.id, + ) + @classmethod def get_subscription_by_id(cls, tenant_id: str, subscription_id: str | None = None) -> TriggerSubscription | None: """ @@ -257,17 +359,18 @@ class TriggerProviderService: raise ValueError(f"Trigger provider subscription {subscription_id} not found") credential_type: CredentialType = CredentialType.of(subscription.credential_type) + provider_id = TriggerProviderID(subscription.provider_id) + provider_controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider( + tenant_id=tenant_id, provider_id=provider_id + ) + encrypter, _ = create_trigger_provider_encrypter_for_subscription( + tenant_id=tenant_id, + controller=provider_controller, + subscription=subscription, + ) + is_auto_created: bool = credential_type in [CredentialType.OAUTH2, CredentialType.API_KEY] if is_auto_created: - provider_id = TriggerProviderID(subscription.provider_id) - provider_controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider( - tenant_id=tenant_id, provider_id=provider_id - ) - encrypter, _ = create_trigger_provider_encrypter_for_subscription( - tenant_id=tenant_id, - controller=provider_controller, - subscription=subscription, - ) try: TriggerManager.unsubscribe_trigger( tenant_id=tenant_id, @@ -280,8 +383,8 @@ class TriggerProviderService: except Exception as e: logger.exception("Error unsubscribing trigger", exc_info=e) - # Clear cache session.delete(subscription) + # Clear cache delete_cache_for_subscription( tenant_id=tenant_id, provider_id=subscription.provider_id, @@ -688,3 +791,188 @@ class TriggerProviderService: ) subscription.properties = dict(properties_encrypter.decrypt(subscription.properties)) return subscription + + @classmethod + def verify_subscription_credentials( + cls, + tenant_id: str, + user_id: str, + provider_id: TriggerProviderID, + subscription_id: str, + credentials: Mapping[str, Any], + ) -> dict[str, Any]: + """ + Verify credentials for an existing subscription without updating it. + + This is used in edit mode to validate new credentials before rebuild. + + :param tenant_id: Tenant ID + :param user_id: User ID + :param provider_id: Provider identifier + :param subscription_id: Subscription ID + :param credentials: New credentials to verify + :return: dict with 'verified' boolean + """ + provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id) + if not provider_controller: + raise ValueError(f"Provider {provider_id} not found") + + subscription = cls.get_subscription_by_id( + tenant_id=tenant_id, + subscription_id=subscription_id, + ) + if not subscription: + raise ValueError(f"Subscription {subscription_id} not found") + + credential_type = CredentialType.of(subscription.credential_type) + + # For API Key, validate the new credentials + if credential_type == CredentialType.API_KEY: + new_credentials: dict[str, Any] = { + key: value if value != HIDDEN_VALUE else subscription.credentials.get(key, UNKNOWN_VALUE) + for key, value in credentials.items() + } + try: + provider_controller.validate_credentials(user_id, credentials=new_credentials) + return {"verified": True} + except Exception as e: + raise ValueError(f"Invalid credentials: {e}") from e + + return {"verified": True} + + @classmethod + def rebuild_trigger_subscription( + cls, + tenant_id: str, + provider_id: TriggerProviderID, + subscription_id: str, + credentials: Mapping[str, Any], + parameters: Mapping[str, Any], + name: str | None = None, + ) -> None: + """ + Create a subscription builder for rebuilding an existing subscription. + + This method creates a builder pre-filled with data from the rebuild request, + keeping the same subscription_id and endpoint_id so the webhook URL remains unchanged. + + :param tenant_id: Tenant ID + :param name: Name for the subscription + :param subscription_id: Subscription ID + :param provider_id: Provider identifier + :param credentials: Credentials for the subscription + :param parameters: Parameters for the subscription + :return: SubscriptionBuilderApiEntity + """ + provider_controller = TriggerManager.get_trigger_provider(tenant_id, provider_id) + if not provider_controller: + raise ValueError(f"Provider {provider_id} not found") + + # Use distributed lock to prevent race conditions on the same subscription + lock_key = f"trigger_subscription_rebuild_lock:{tenant_id}_{subscription_id}" + with redis_client.lock(lock_key, timeout=20): + with Session(db.engine, expire_on_commit=False) as session: + try: + # Get subscription within the transaction + subscription: TriggerSubscription | None = ( + session.query(TriggerSubscription).filter_by(tenant_id=tenant_id, id=subscription_id).first() + ) + if not subscription: + raise ValueError(f"Subscription {subscription_id} not found") + + credential_type = CredentialType.of(subscription.credential_type) + if credential_type not in [CredentialType.OAUTH2, CredentialType.API_KEY]: + raise ValueError("Credential type not supported for rebuild") + + # Decrypt existing credentials for merging + credential_encrypter, _ = create_trigger_provider_encrypter_for_subscription( + tenant_id=tenant_id, + controller=provider_controller, + subscription=subscription, + ) + decrypted_credentials = dict(credential_encrypter.decrypt(subscription.credentials)) + + # Merge credentials: if caller passed HIDDEN_VALUE, retain existing decrypted value + merged_credentials: dict[str, Any] = { + key: value if value != HIDDEN_VALUE else decrypted_credentials.get(key, UNKNOWN_VALUE) + for key, value in credentials.items() + } + + user_id = subscription.user_id + + # TODO: Trying to invoke update api of the plugin trigger provider + + # FALLBACK: If the update api is not implemented, + # delete the previous subscription and create a new one + + # Unsubscribe the previous subscription (external call, but we'll handle errors) + try: + TriggerManager.unsubscribe_trigger( + tenant_id=tenant_id, + user_id=user_id, + provider_id=provider_id, + subscription=subscription.to_entity(), + credentials=decrypted_credentials, + credential_type=credential_type, + ) + except Exception as e: + logger.exception("Error unsubscribing trigger during rebuild", exc_info=e) + # Continue anyway - the subscription might already be deleted externally + + # Create a new subscription with the same subscription_id and endpoint_id (external call) + new_subscription: TriggerSubscriptionEntity = TriggerManager.subscribe_trigger( + tenant_id=tenant_id, + user_id=user_id, + provider_id=provider_id, + endpoint=generate_plugin_trigger_endpoint_url(subscription.endpoint_id), + parameters=parameters, + credentials=merged_credentials, + credential_type=credential_type, + ) + + # Update the subscription in the same transaction + # Inline update logic to reuse the same session + if name is not None and name != subscription.name: + existing = ( + session.query(TriggerSubscription) + .filter_by(tenant_id=tenant_id, provider_id=str(provider_id), name=name) + .first() + ) + if existing and existing.id != subscription.id: + raise ValueError(f"Subscription name '{name}' already exists for this provider") + subscription.name = name + + # Update parameters + subscription.parameters = dict(parameters) + + # Update credentials with merged (and encrypted) values + subscription.credentials = dict(credential_encrypter.encrypt(merged_credentials)) + + # Update properties + if new_subscription.properties: + properties_encrypter, _ = create_provider_encrypter( + tenant_id=tenant_id, + config=provider_controller.get_properties_schema(), + cache=NoOpProviderCredentialCache(), + ) + subscription.properties = dict(properties_encrypter.encrypt(dict(new_subscription.properties))) + + # Update expiration timestamp + if new_subscription.expires_at is not None: + subscription.expires_at = new_subscription.expires_at + + # Commit the transaction + session.commit() + + # Clear subscription cache + delete_cache_for_subscription( + tenant_id=tenant_id, + provider_id=subscription.provider_id, + subscription_id=subscription.id, + ) + + except Exception as e: + # Rollback on any error + session.rollback() + logger.exception("Failed to rebuild trigger subscription", exc_info=e) + raise diff --git a/api/services/trigger/trigger_subscription_builder_service.py b/api/services/trigger/trigger_subscription_builder_service.py index 571393c782..37f852da3e 100644 --- a/api/services/trigger/trigger_subscription_builder_service.py +++ b/api/services/trigger/trigger_subscription_builder_service.py @@ -453,11 +453,12 @@ class TriggerSubscriptionBuilderService: if not subscription_builder: return None - # response to validation endpoint - controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider( - tenant_id=subscription_builder.tenant_id, provider_id=TriggerProviderID(subscription_builder.provider_id) - ) try: + # response to validation endpoint + controller: PluginTriggerProviderController = TriggerManager.get_trigger_provider( + tenant_id=subscription_builder.tenant_id, + provider_id=TriggerProviderID(subscription_builder.provider_id), + ) dispatch_response: TriggerDispatchResponse = controller.dispatch( request=request, subscription=subscription_builder.to_subscription(), diff --git a/api/services/trigger/webhook_service.py b/api/services/trigger/webhook_service.py index 5c4607d400..4159f5f8f4 100644 --- a/api/services/trigger/webhook_service.py +++ b/api/services/trigger/webhook_service.py @@ -863,10 +863,18 @@ class WebhookService: not_found_in_cache.append(node_id) continue - with Session(db.engine) as session: - try: - # lock the concurrent webhook trigger creation - redis_client.lock(f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:apps:{app.id}:lock", timeout=10) + lock_key = f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:apps:{app.id}:lock" + lock = redis_client.lock(lock_key, timeout=10) + lock_acquired = False + + try: + # acquire the lock with blocking and timeout + lock_acquired = lock.acquire(blocking=True, blocking_timeout=10) + if not lock_acquired: + logger.warning("Failed to acquire lock for webhook sync, app %s", app.id) + raise RuntimeError("Failed to acquire lock for webhook trigger synchronization") + + with Session(db.engine) as session: # fetch the non-cached nodes from DB all_records = session.scalars( select(WorkflowWebhookTrigger).where( @@ -903,11 +911,16 @@ class WebhookService: session.delete(nodes_id_in_db[node_id]) redis_client.delete(f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:{node_id}") session.commit() - except Exception: - logger.exception("Failed to sync webhook relationships for app %s", app.id) - raise - finally: - redis_client.delete(f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:apps:{app.id}:lock") + except Exception: + logger.exception("Failed to sync webhook relationships for app %s", app.id) + raise + finally: + # release the lock only if it was acquired + if lock_acquired: + try: + lock.release() + except Exception: + logger.exception("Failed to release lock for webhook sync, app %s", app.id) @classmethod def generate_webhook_id(cls) -> str: diff --git a/api/tasks/document_indexing_sync_task.py b/api/tasks/document_indexing_sync_task.py index 4c1f38c3bb..5fc2597c92 100644 --- a/api/tasks/document_indexing_sync_task.py +++ b/api/tasks/document_indexing_sync_task.py @@ -2,7 +2,6 @@ import logging import time import click -import sqlalchemy as sa from celery import shared_task from sqlalchemy import select @@ -12,7 +11,7 @@ from core.rag.index_processor.index_processor_factory import IndexProcessorFacto from extensions.ext_database import db from libs.datetime_utils import naive_utc_now from models.dataset import Dataset, Document, DocumentSegment -from models.source import DataSourceOauthBinding +from services.datasource_provider_service import DatasourceProviderService logger = logging.getLogger(__name__) @@ -48,27 +47,36 @@ def document_indexing_sync_task(dataset_id: str, document_id: str): page_id = data_source_info["notion_page_id"] page_type = data_source_info["type"] page_edited_time = data_source_info["last_edited_time"] + credential_id = data_source_info.get("credential_id") - data_source_binding = ( - db.session.query(DataSourceOauthBinding) - .where( - sa.and_( - DataSourceOauthBinding.tenant_id == document.tenant_id, - DataSourceOauthBinding.provider == "notion", - DataSourceOauthBinding.disabled == False, - DataSourceOauthBinding.source_info["workspace_id"] == f'"{workspace_id}"', - ) - ) - .first() + # Get credentials from datasource provider + datasource_provider_service = DatasourceProviderService() + credential = datasource_provider_service.get_datasource_credentials( + tenant_id=document.tenant_id, + credential_id=credential_id, + provider="notion_datasource", + plugin_id="langgenius/notion_datasource", ) - if not data_source_binding: - raise ValueError("Data source binding not found.") + + if not credential: + logger.error( + "Datasource credential not found for document %s, tenant_id: %s, credential_id: %s", + document_id, + document.tenant_id, + credential_id, + ) + document.indexing_status = "error" + document.error = "Datasource credential not found. Please reconnect your Notion workspace." + document.stopped_at = naive_utc_now() + db.session.commit() + db.session.close() + return loader = NotionExtractor( notion_workspace_id=workspace_id, notion_obj_id=page_id, notion_page_type=page_type, - notion_access_token=data_source_binding.access_token, + notion_access_token=credential.get("integration_secret"), tenant_id=document.tenant_id, ) diff --git a/api/tests/integration_tests/workflow/nodes/code_executor/test_code_jinja2.py b/api/tests/integration_tests/workflow/nodes/code_executor/test_code_jinja2.py index 94903cf796..c8eb9ec3e4 100644 --- a/api/tests/integration_tests/workflow/nodes/code_executor/test_code_jinja2.py +++ b/api/tests/integration_tests/workflow/nodes/code_executor/test_code_jinja2.py @@ -7,11 +7,14 @@ CODE_LANGUAGE = CodeLanguage.JINJA2 def test_jinja2(): + """Test basic Jinja2 template rendering.""" template = "Hello {{template}}" + # Template must be base64 encoded to match the new safe embedding approach + template_b64 = base64.b64encode(template.encode("utf-8")).decode("utf-8") inputs = base64.b64encode(b'{"template": "World"}').decode("utf-8") code = ( Jinja2TemplateTransformer.get_runner_script() - .replace(Jinja2TemplateTransformer._code_placeholder, template) + .replace(Jinja2TemplateTransformer._template_b64_placeholder, template_b64) .replace(Jinja2TemplateTransformer._inputs_placeholder, inputs) ) result = CodeExecutor.execute_code( @@ -21,6 +24,7 @@ def test_jinja2(): def test_jinja2_with_code_template(): + """Test template rendering via the high-level workflow API.""" result = CodeExecutor.execute_workflow_code_template( language=CODE_LANGUAGE, code="Hello {{template}}", inputs={"template": "World"} ) @@ -28,7 +32,64 @@ def test_jinja2_with_code_template(): def test_jinja2_get_runner_script(): + """Test that runner script contains required placeholders.""" runner_script = Jinja2TemplateTransformer.get_runner_script() - assert runner_script.count(Jinja2TemplateTransformer._code_placeholder) == 1 + assert runner_script.count(Jinja2TemplateTransformer._template_b64_placeholder) == 1 assert runner_script.count(Jinja2TemplateTransformer._inputs_placeholder) == 1 assert runner_script.count(Jinja2TemplateTransformer._result_tag) == 2 + + +def test_jinja2_template_with_special_characters(): + """ + Test that templates with special characters (quotes, newlines) render correctly. + This is a regression test for issue #26818 where textarea pre-fill values + containing special characters would break template rendering. + """ + # Template with triple quotes, single quotes, double quotes, and newlines + template = """ + + + +

Status: "{{ status }}"

+
'''code block'''
+ +""" + inputs = {"task": {"Task ID": "TASK-123", "Issues": "Line 1\nLine 2\nLine 3"}, "status": "completed"} + + result = CodeExecutor.execute_workflow_code_template(language=CODE_LANGUAGE, code=template, inputs=inputs) + + # Verify the template rendered correctly with all special characters + output = result["result"] + assert 'value="TASK-123"' in output + assert "" in output + assert 'Status: "completed"' in output + assert "'''code block'''" in output + + +def test_jinja2_template_with_html_textarea_prefill(): + """ + Specific test for HTML textarea with Jinja2 variable pre-fill. + Verifies fix for issue #26818. + """ + template = "" + notes_content = "This is a multi-line note.\nWith special chars: 'single' and \"double\" quotes." + inputs = {"notes": notes_content} + + result = CodeExecutor.execute_workflow_code_template(language=CODE_LANGUAGE, code=template, inputs=inputs) + + expected_output = f"" + assert result["result"] == expected_output + + +def test_jinja2_assemble_runner_script_encodes_template(): + """Test that assemble_runner_script properly base64 encodes the template.""" + template = "Hello {{ name }}!" + inputs = {"name": "World"} + + script = Jinja2TemplateTransformer.assemble_runner_script(template, inputs) + + # The template should be base64 encoded in the script + template_b64 = base64.b64encode(template.encode("utf-8")).decode("utf-8") + assert template_b64 in script + # The raw template should NOT appear in the script (it's encoded) + assert "Hello {{ name }}!" not in script diff --git a/api/tests/integration_tests/workflow/nodes/test_http.py b/api/tests/integration_tests/workflow/nodes/test_http.py index e75258a2a2..d814da8ec7 100644 --- a/api/tests/integration_tests/workflow/nodes/test_http.py +++ b/api/tests/integration_tests/workflow/nodes/test_http.py @@ -6,6 +6,7 @@ import pytest from core.app.entities.app_invoke_entities import InvokeFrom from core.workflow.entities import GraphInitParams +from core.workflow.enums import WorkflowNodeExecutionStatus from core.workflow.graph import Graph from core.workflow.nodes.http_request.node import HttpRequestNode from core.workflow.nodes.node_factory import DifyNodeFactory @@ -169,13 +170,14 @@ def test_custom_authorization_header(setup_http_mock): @pytest.mark.parametrize("setup_http_mock", [["none"]], indirect=True) -def test_custom_auth_with_empty_api_key_does_not_set_header(setup_http_mock): - """Test: In custom authentication mode, when the api_key is empty, no header should be set.""" +def test_custom_auth_with_empty_api_key_raises_error(setup_http_mock): + """Test: In custom authentication mode, when the api_key is empty, AuthorizationConfigError should be raised.""" from core.workflow.nodes.http_request.entities import ( HttpRequestNodeAuthorization, HttpRequestNodeData, HttpRequestNodeTimeout, ) + from core.workflow.nodes.http_request.exc import AuthorizationConfigError from core.workflow.nodes.http_request.executor import Executor from core.workflow.runtime import VariablePool from core.workflow.system_variable import SystemVariable @@ -208,16 +210,13 @@ def test_custom_auth_with_empty_api_key_does_not_set_header(setup_http_mock): ssl_verify=True, ) - # Create executor - executor = Executor( - node_data=node_data, timeout=HttpRequestNodeTimeout(connect=10, read=30, write=10), variable_pool=variable_pool - ) - - # Get assembled headers - headers = executor._assembling_headers() - - # When api_key is empty, the custom header should NOT be set - assert "X-Custom-Auth" not in headers + # Create executor should raise AuthorizationConfigError + with pytest.raises(AuthorizationConfigError, match="API key is required"): + Executor( + node_data=node_data, + timeout=HttpRequestNodeTimeout(connect=10, read=30, write=10), + variable_pool=variable_pool, + ) @pytest.mark.parametrize("setup_http_mock", [["none"]], indirect=True) @@ -305,9 +304,10 @@ def test_basic_authorization_with_custom_header_ignored(setup_http_mock): @pytest.mark.parametrize("setup_http_mock", [["none"]], indirect=True) def test_custom_authorization_with_empty_api_key(setup_http_mock): """ - Test that custom authorization doesn't set header when api_key is empty. - This test verifies the fix for issue #23554. + Test that custom authorization raises error when api_key is empty. + This test verifies the fix for issue #21830. """ + node = init_http_node( config={ "id": "1", @@ -333,11 +333,10 @@ def test_custom_authorization_with_empty_api_key(setup_http_mock): ) result = node._run() - assert result.process_data is not None - data = result.process_data.get("request", "") - - # Custom header should NOT be set when api_key is empty - assert "X-Custom-Auth:" not in data + # Should fail with AuthorizationConfigError + assert result.status == WorkflowNodeExecutionStatus.FAILED + assert "API key is required" in result.error + assert result.error_type == "AuthorizationConfigError" @pytest.mark.parametrize("setup_http_mock", [["none"]], indirect=True) diff --git a/api/tests/test_containers_integration_tests/services/test_billing_service.py b/api/tests/test_containers_integration_tests/services/test_billing_service.py new file mode 100644 index 0000000000..76708b36b1 --- /dev/null +++ b/api/tests/test_containers_integration_tests/services/test_billing_service.py @@ -0,0 +1,365 @@ +import json +from unittest.mock import patch + +import pytest + +from extensions.ext_redis import redis_client +from services.billing_service import BillingService + + +class TestBillingServiceGetPlanBulkWithCache: + """ + Comprehensive integration tests for get_plan_bulk_with_cache using testcontainers. + + This test class covers all major scenarios: + - Cache hit/miss scenarios + - Redis operation failures and fallback behavior + - Invalid cache data handling + - TTL expiration handling + - Error recovery and logging + """ + + @pytest.fixture(autouse=True) + def setup_redis_cleanup(self, flask_app_with_containers): + """Clean up Redis cache before and after each test.""" + with flask_app_with_containers.app_context(): + # Clean up before test + yield + # Clean up after test + # Delete all test cache keys + pattern = f"{BillingService._PLAN_CACHE_KEY_PREFIX}*" + keys = redis_client.keys(pattern) + if keys: + redis_client.delete(*keys) + + def _create_test_plan_data(self, plan: str = "sandbox", expiration_date: int = 1735689600): + """Helper to create test SubscriptionPlan data.""" + return {"plan": plan, "expiration_date": expiration_date} + + def _set_cache(self, tenant_id: str, plan_data: dict, ttl: int = 600): + """Helper to set cache data in Redis.""" + cache_key = BillingService._make_plan_cache_key(tenant_id) + json_str = json.dumps(plan_data) + redis_client.setex(cache_key, ttl, json_str) + + def _get_cache(self, tenant_id: str): + """Helper to get cache data from Redis.""" + cache_key = BillingService._make_plan_cache_key(tenant_id) + value = redis_client.get(cache_key) + if value: + if isinstance(value, bytes): + return value.decode("utf-8") + return value + return None + + def test_get_plan_bulk_with_cache_all_cache_hit(self, flask_app_with_containers): + """Test bulk plan retrieval when all tenants are in cache.""" + with flask_app_with_containers.app_context(): + # Arrange + tenant_ids = ["tenant-1", "tenant-2", "tenant-3"] + expected_plans = { + "tenant-1": self._create_test_plan_data("sandbox", 1735689600), + "tenant-2": self._create_test_plan_data("professional", 1767225600), + "tenant-3": self._create_test_plan_data("team", 1798761600), + } + + # Pre-populate cache + for tenant_id, plan_data in expected_plans.items(): + self._set_cache(tenant_id, plan_data) + + # Act + with patch.object(BillingService, "get_plan_bulk") as mock_get_plan_bulk: + result = BillingService.get_plan_bulk_with_cache(tenant_ids) + + # Assert + assert len(result) == 3 + assert result["tenant-1"]["plan"] == "sandbox" + assert result["tenant-1"]["expiration_date"] == 1735689600 + assert result["tenant-2"]["plan"] == "professional" + assert result["tenant-2"]["expiration_date"] == 1767225600 + assert result["tenant-3"]["plan"] == "team" + assert result["tenant-3"]["expiration_date"] == 1798761600 + + # Verify API was not called + mock_get_plan_bulk.assert_not_called() + + def test_get_plan_bulk_with_cache_all_cache_miss(self, flask_app_with_containers): + """Test bulk plan retrieval when all tenants are not in cache.""" + with flask_app_with_containers.app_context(): + # Arrange + tenant_ids = ["tenant-1", "tenant-2"] + expected_plans = { + "tenant-1": self._create_test_plan_data("sandbox", 1735689600), + "tenant-2": self._create_test_plan_data("professional", 1767225600), + } + + # Act + with patch.object(BillingService, "get_plan_bulk", return_value=expected_plans) as mock_get_plan_bulk: + result = BillingService.get_plan_bulk_with_cache(tenant_ids) + + # Assert + assert len(result) == 2 + assert result["tenant-1"]["plan"] == "sandbox" + assert result["tenant-2"]["plan"] == "professional" + + # Verify API was called with correct tenant_ids + mock_get_plan_bulk.assert_called_once_with(tenant_ids) + + # Verify data was written to cache + cached_1 = self._get_cache("tenant-1") + cached_2 = self._get_cache("tenant-2") + assert cached_1 is not None + assert cached_2 is not None + + # Verify cache content + cached_data_1 = json.loads(cached_1) + cached_data_2 = json.loads(cached_2) + assert cached_data_1 == expected_plans["tenant-1"] + assert cached_data_2 == expected_plans["tenant-2"] + + # Verify TTL is set + cache_key_1 = BillingService._make_plan_cache_key("tenant-1") + ttl_1 = redis_client.ttl(cache_key_1) + assert ttl_1 > 0 + assert ttl_1 <= 600 # Should be <= 600 seconds + + def test_get_plan_bulk_with_cache_partial_cache_hit(self, flask_app_with_containers): + """Test bulk plan retrieval when some tenants are in cache, some are not.""" + with flask_app_with_containers.app_context(): + # Arrange + tenant_ids = ["tenant-1", "tenant-2", "tenant-3"] + # Pre-populate cache for tenant-1 and tenant-2 + self._set_cache("tenant-1", self._create_test_plan_data("sandbox", 1735689600)) + self._set_cache("tenant-2", self._create_test_plan_data("professional", 1767225600)) + + # tenant-3 is not in cache + missing_plan = {"tenant-3": self._create_test_plan_data("team", 1798761600)} + + # Act + with patch.object(BillingService, "get_plan_bulk", return_value=missing_plan) as mock_get_plan_bulk: + result = BillingService.get_plan_bulk_with_cache(tenant_ids) + + # Assert + assert len(result) == 3 + assert result["tenant-1"]["plan"] == "sandbox" + assert result["tenant-2"]["plan"] == "professional" + assert result["tenant-3"]["plan"] == "team" + + # Verify API was called only for missing tenant + mock_get_plan_bulk.assert_called_once_with(["tenant-3"]) + + # Verify tenant-3 data was written to cache + cached_3 = self._get_cache("tenant-3") + assert cached_3 is not None + cached_data_3 = json.loads(cached_3) + assert cached_data_3 == missing_plan["tenant-3"] + + def test_get_plan_bulk_with_cache_redis_mget_failure(self, flask_app_with_containers): + """Test fallback to API when Redis mget fails.""" + with flask_app_with_containers.app_context(): + # Arrange + tenant_ids = ["tenant-1", "tenant-2"] + expected_plans = { + "tenant-1": self._create_test_plan_data("sandbox", 1735689600), + "tenant-2": self._create_test_plan_data("professional", 1767225600), + } + + # Act + with ( + patch.object(redis_client, "mget", side_effect=Exception("Redis connection error")), + patch.object(BillingService, "get_plan_bulk", return_value=expected_plans) as mock_get_plan_bulk, + ): + result = BillingService.get_plan_bulk_with_cache(tenant_ids) + + # Assert + assert len(result) == 2 + assert result["tenant-1"]["plan"] == "sandbox" + assert result["tenant-2"]["plan"] == "professional" + + # Verify API was called for all tenants (fallback) + mock_get_plan_bulk.assert_called_once_with(tenant_ids) + + # Verify data was written to cache after fallback + cached_1 = self._get_cache("tenant-1") + cached_2 = self._get_cache("tenant-2") + assert cached_1 is not None + assert cached_2 is not None + + def test_get_plan_bulk_with_cache_invalid_json_in_cache(self, flask_app_with_containers): + """Test fallback to API when cache contains invalid JSON.""" + with flask_app_with_containers.app_context(): + # Arrange + tenant_ids = ["tenant-1", "tenant-2", "tenant-3"] + + # Set valid cache for tenant-1 + self._set_cache("tenant-1", self._create_test_plan_data("sandbox", 1735689600)) + + # Set invalid JSON for tenant-2 + cache_key_2 = BillingService._make_plan_cache_key("tenant-2") + redis_client.setex(cache_key_2, 600, "invalid json {") + + # tenant-3 is not in cache + expected_plans = { + "tenant-2": self._create_test_plan_data("professional", 1767225600), + "tenant-3": self._create_test_plan_data("team", 1798761600), + } + + # Act + with patch.object(BillingService, "get_plan_bulk", return_value=expected_plans) as mock_get_plan_bulk: + result = BillingService.get_plan_bulk_with_cache(tenant_ids) + + # Assert + assert len(result) == 3 + assert result["tenant-1"]["plan"] == "sandbox" # From cache + assert result["tenant-2"]["plan"] == "professional" # From API (fallback) + assert result["tenant-3"]["plan"] == "team" # From API + + # Verify API was called for tenant-2 and tenant-3 + mock_get_plan_bulk.assert_called_once_with(["tenant-2", "tenant-3"]) + + # Verify tenant-2's invalid JSON was replaced with correct data in cache + cached_2 = self._get_cache("tenant-2") + assert cached_2 is not None + cached_data_2 = json.loads(cached_2) + assert cached_data_2 == expected_plans["tenant-2"] + assert cached_data_2["plan"] == "professional" + assert cached_data_2["expiration_date"] == 1767225600 + + # Verify tenant-2 cache has correct TTL + cache_key_2_new = BillingService._make_plan_cache_key("tenant-2") + ttl_2 = redis_client.ttl(cache_key_2_new) + assert ttl_2 > 0 + assert ttl_2 <= 600 + + # Verify tenant-3 data was also written to cache + cached_3 = self._get_cache("tenant-3") + assert cached_3 is not None + cached_data_3 = json.loads(cached_3) + assert cached_data_3 == expected_plans["tenant-3"] + + def test_get_plan_bulk_with_cache_invalid_plan_data_in_cache(self, flask_app_with_containers): + """Test fallback to API when cache data doesn't match SubscriptionPlan schema.""" + with flask_app_with_containers.app_context(): + # Arrange + tenant_ids = ["tenant-1", "tenant-2", "tenant-3"] + + # Set valid cache for tenant-1 + self._set_cache("tenant-1", self._create_test_plan_data("sandbox", 1735689600)) + + # Set invalid plan data for tenant-2 (missing expiration_date) + cache_key_2 = BillingService._make_plan_cache_key("tenant-2") + invalid_data = json.dumps({"plan": "professional"}) # Missing expiration_date + redis_client.setex(cache_key_2, 600, invalid_data) + + # tenant-3 is not in cache + expected_plans = { + "tenant-2": self._create_test_plan_data("professional", 1767225600), + "tenant-3": self._create_test_plan_data("team", 1798761600), + } + + # Act + with patch.object(BillingService, "get_plan_bulk", return_value=expected_plans) as mock_get_plan_bulk: + result = BillingService.get_plan_bulk_with_cache(tenant_ids) + + # Assert + assert len(result) == 3 + assert result["tenant-1"]["plan"] == "sandbox" # From cache + assert result["tenant-2"]["plan"] == "professional" # From API (fallback) + assert result["tenant-3"]["plan"] == "team" # From API + + # Verify API was called for tenant-2 and tenant-3 + mock_get_plan_bulk.assert_called_once_with(["tenant-2", "tenant-3"]) + + def test_get_plan_bulk_with_cache_redis_pipeline_failure(self, flask_app_with_containers): + """Test that pipeline failure doesn't affect return value.""" + with flask_app_with_containers.app_context(): + # Arrange + tenant_ids = ["tenant-1", "tenant-2"] + expected_plans = { + "tenant-1": self._create_test_plan_data("sandbox", 1735689600), + "tenant-2": self._create_test_plan_data("professional", 1767225600), + } + + # Act + with ( + patch.object(BillingService, "get_plan_bulk", return_value=expected_plans), + patch.object(redis_client, "pipeline") as mock_pipeline, + ): + # Create a mock pipeline that fails on execute + mock_pipe = mock_pipeline.return_value + mock_pipe.execute.side_effect = Exception("Pipeline execution failed") + + result = BillingService.get_plan_bulk_with_cache(tenant_ids) + + # Assert - Function should still return correct result despite pipeline failure + assert len(result) == 2 + assert result["tenant-1"]["plan"] == "sandbox" + assert result["tenant-2"]["plan"] == "professional" + + # Verify pipeline was attempted + mock_pipeline.assert_called_once() + + def test_get_plan_bulk_with_cache_empty_tenant_ids(self, flask_app_with_containers): + """Test with empty tenant_ids list.""" + with flask_app_with_containers.app_context(): + # Act + with patch.object(BillingService, "get_plan_bulk") as mock_get_plan_bulk: + result = BillingService.get_plan_bulk_with_cache([]) + + # Assert + assert result == {} + assert len(result) == 0 + + # Verify no API calls + mock_get_plan_bulk.assert_not_called() + + # Verify no Redis operations (mget with empty list would return empty list) + # But we should check that mget was not called at all + # Since we can't easily verify this without more mocking, we just verify the result + + def test_get_plan_bulk_with_cache_ttl_expired(self, flask_app_with_containers): + """Test that expired cache keys are treated as cache misses.""" + with flask_app_with_containers.app_context(): + # Arrange + tenant_ids = ["tenant-1", "tenant-2"] + + # Set cache for tenant-1 with very short TTL (1 second) to simulate expiration + self._set_cache("tenant-1", self._create_test_plan_data("sandbox", 1735689600), ttl=1) + + # Wait for TTL to expire (key will be deleted by Redis) + import time + + time.sleep(2) + + # Verify cache is expired (key doesn't exist) + cache_key_1 = BillingService._make_plan_cache_key("tenant-1") + exists = redis_client.exists(cache_key_1) + assert exists == 0 # Key doesn't exist (expired) + + # tenant-2 is not in cache + expected_plans = { + "tenant-1": self._create_test_plan_data("sandbox", 1735689600), + "tenant-2": self._create_test_plan_data("professional", 1767225600), + } + + # Act + with patch.object(BillingService, "get_plan_bulk", return_value=expected_plans) as mock_get_plan_bulk: + result = BillingService.get_plan_bulk_with_cache(tenant_ids) + + # Assert + assert len(result) == 2 + assert result["tenant-1"]["plan"] == "sandbox" + assert result["tenant-2"]["plan"] == "professional" + + # Verify API was called for both tenants (tenant-1 expired, tenant-2 missing) + mock_get_plan_bulk.assert_called_once_with(tenant_ids) + + # Verify both were written to cache with correct TTL + cache_key_1_new = BillingService._make_plan_cache_key("tenant-1") + cache_key_2 = BillingService._make_plan_cache_key("tenant-2") + ttl_1_new = redis_client.ttl(cache_key_1_new) + ttl_2 = redis_client.ttl(cache_key_2) + assert ttl_1_new > 0 + assert ttl_1_new <= 600 + assert ttl_2 > 0 + assert ttl_2 <= 600 diff --git a/api/tests/test_containers_integration_tests/services/test_trigger_provider_service.py b/api/tests/test_containers_integration_tests/services/test_trigger_provider_service.py new file mode 100644 index 0000000000..8322b9414e --- /dev/null +++ b/api/tests/test_containers_integration_tests/services/test_trigger_provider_service.py @@ -0,0 +1,682 @@ +from unittest.mock import MagicMock, patch + +import pytest +from faker import Faker + +from constants import HIDDEN_VALUE, UNKNOWN_VALUE +from core.plugin.entities.plugin_daemon import CredentialType +from core.trigger.entities.entities import Subscription as TriggerSubscriptionEntity +from extensions.ext_database import db +from models.provider_ids import TriggerProviderID +from models.trigger import TriggerSubscription +from services.trigger.trigger_provider_service import TriggerProviderService + + +class TestTriggerProviderService: + """Integration tests for TriggerProviderService using testcontainers.""" + + @pytest.fixture + def mock_external_service_dependencies(self): + """Mock setup for external service dependencies.""" + with ( + patch("services.trigger.trigger_provider_service.TriggerManager") as mock_trigger_manager, + patch("services.trigger.trigger_provider_service.redis_client") as mock_redis_client, + patch("services.trigger.trigger_provider_service.delete_cache_for_subscription") as mock_delete_cache, + patch("services.account_service.FeatureService") as mock_account_feature_service, + ): + # Setup default mock returns + mock_provider_controller = MagicMock() + mock_provider_controller.get_credential_schema_config.return_value = MagicMock() + mock_provider_controller.get_properties_schema.return_value = MagicMock() + mock_trigger_manager.get_trigger_provider.return_value = mock_provider_controller + + # Mock redis lock + mock_lock = MagicMock() + mock_lock.__enter__ = MagicMock(return_value=None) + mock_lock.__exit__ = MagicMock(return_value=None) + mock_redis_client.lock.return_value = mock_lock + + # Setup account feature service mock + mock_account_feature_service.get_system_features.return_value.is_allow_register = True + + yield { + "trigger_manager": mock_trigger_manager, + "redis_client": mock_redis_client, + "delete_cache": mock_delete_cache, + "provider_controller": mock_provider_controller, + "account_feature_service": mock_account_feature_service, + } + + def _create_test_account_and_tenant(self, db_session_with_containers, mock_external_service_dependencies): + """ + Helper method to create a test account and tenant for testing. + + Args: + db_session_with_containers: Database session from testcontainers infrastructure + mock_external_service_dependencies: Mock dependencies + + Returns: + tuple: (account, tenant) - Created account and tenant instances + """ + fake = Faker() + + from services.account_service import AccountService, TenantService + + # Setup mocks for account creation + mock_external_service_dependencies[ + "account_feature_service" + ].get_system_features.return_value.is_allow_register = True + mock_external_service_dependencies[ + "trigger_manager" + ].get_trigger_provider.return_value = mock_external_service_dependencies["provider_controller"] + + # Create account and tenant + account = AccountService.create_account( + email=fake.email(), + name=fake.name(), + interface_language="en-US", + password=fake.password(length=12), + ) + TenantService.create_owner_tenant_if_not_exist(account, name=fake.company()) + tenant = account.current_tenant + + return account, tenant + + def _create_test_subscription( + self, + db_session_with_containers, + tenant_id, + user_id, + provider_id, + credential_type, + credentials, + mock_external_service_dependencies, + ): + """ + Helper method to create a test trigger subscription. + + Args: + db_session_with_containers: Database session + tenant_id: Tenant ID + user_id: User ID + provider_id: Provider ID + credential_type: Credential type + credentials: Credentials dict + mock_external_service_dependencies: Mock dependencies + + Returns: + TriggerSubscription: Created subscription instance + """ + fake = Faker() + from core.helper.provider_cache import NoOpProviderCredentialCache + from core.helper.provider_encryption import create_provider_encrypter + + # Use mock provider controller to encrypt credentials + provider_controller = mock_external_service_dependencies["provider_controller"] + + # Create encrypter for credentials + credential_encrypter, _ = create_provider_encrypter( + tenant_id=tenant_id, + config=provider_controller.get_credential_schema_config(credential_type), + cache=NoOpProviderCredentialCache(), + ) + + subscription = TriggerSubscription( + name=fake.word(), + tenant_id=tenant_id, + user_id=user_id, + provider_id=str(provider_id), + endpoint_id=fake.uuid4(), + parameters={"param1": "value1"}, + properties={"prop1": "value1"}, + credentials=dict(credential_encrypter.encrypt(credentials)), + credential_type=credential_type.value, + credential_expires_at=-1, + expires_at=-1, + ) + + db.session.add(subscription) + db.session.commit() + db.session.refresh(subscription) + + return subscription + + def test_rebuild_trigger_subscription_success_with_merged_credentials( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test successful rebuild with credential merging (HIDDEN_VALUE handling). + + This test verifies: + - Credentials are properly merged (HIDDEN_VALUE replaced with existing values) + - Single transaction wraps all operations + - Merged credentials are used for subscribe and update + - Database state is correctly updated + """ + fake = Faker() + account, tenant = self._create_test_account_and_tenant( + db_session_with_containers, mock_external_service_dependencies + ) + + provider_id = TriggerProviderID("test_org/test_plugin/test_provider") + credential_type = CredentialType.API_KEY + + # Create initial subscription with credentials + original_credentials = {"api_key": "original-secret-key", "api_secret": "original-secret"} + subscription = self._create_test_subscription( + db_session_with_containers, + tenant.id, + account.id, + provider_id, + credential_type, + original_credentials, + mock_external_service_dependencies, + ) + + # Prepare new credentials with HIDDEN_VALUE for api_key (should keep original) + # and new value for api_secret (should update) + new_credentials = { + "api_key": HIDDEN_VALUE, # Should be replaced with original + "api_secret": "new-secret-value", # Should be updated + } + + # Mock subscribe_trigger to return a new subscription entity + new_subscription_entity = TriggerSubscriptionEntity( + endpoint=subscription.endpoint_id, + parameters={"param1": "value1"}, + properties={"prop1": "new_prop_value"}, + expires_at=1234567890, + ) + mock_external_service_dependencies["trigger_manager"].subscribe_trigger.return_value = new_subscription_entity + + # Mock unsubscribe_trigger + mock_external_service_dependencies["trigger_manager"].unsubscribe_trigger.return_value = MagicMock() + + # Execute rebuild + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=tenant.id, + provider_id=provider_id, + subscription_id=subscription.id, + credentials=new_credentials, + parameters={"param1": "updated_value"}, + name="updated_name", + ) + + # Verify unsubscribe was called with decrypted original credentials + mock_external_service_dependencies["trigger_manager"].unsubscribe_trigger.assert_called_once() + unsubscribe_call_args = mock_external_service_dependencies["trigger_manager"].unsubscribe_trigger.call_args + assert unsubscribe_call_args.kwargs["tenant_id"] == tenant.id + assert unsubscribe_call_args.kwargs["provider_id"] == provider_id + assert unsubscribe_call_args.kwargs["credential_type"] == credential_type + + # Verify subscribe was called with merged credentials (api_key from original, api_secret new) + mock_external_service_dependencies["trigger_manager"].subscribe_trigger.assert_called_once() + subscribe_call_args = mock_external_service_dependencies["trigger_manager"].subscribe_trigger.call_args + subscribe_credentials = subscribe_call_args.kwargs["credentials"] + assert subscribe_credentials["api_key"] == original_credentials["api_key"] # Merged from original + assert subscribe_credentials["api_secret"] == "new-secret-value" # New value + + # Verify database state was updated + db.session.refresh(subscription) + assert subscription.name == "updated_name" + assert subscription.parameters == {"param1": "updated_value"} + + # Verify credentials in DB were updated with merged values (decrypt to check) + from core.helper.provider_cache import NoOpProviderCredentialCache + from core.helper.provider_encryption import create_provider_encrypter + + # Use mock provider controller to decrypt credentials + provider_controller = mock_external_service_dependencies["provider_controller"] + credential_encrypter, _ = create_provider_encrypter( + tenant_id=tenant.id, + config=provider_controller.get_credential_schema_config(credential_type), + cache=NoOpProviderCredentialCache(), + ) + decrypted_db_credentials = dict(credential_encrypter.decrypt(subscription.credentials)) + assert decrypted_db_credentials["api_key"] == original_credentials["api_key"] + assert decrypted_db_credentials["api_secret"] == "new-secret-value" + + # Verify cache was cleared + mock_external_service_dependencies["delete_cache"].assert_called_once_with( + tenant_id=tenant.id, + provider_id=subscription.provider_id, + subscription_id=subscription.id, + ) + + def test_rebuild_trigger_subscription_with_all_new_credentials( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test rebuild when all credentials are new (no HIDDEN_VALUE). + + This test verifies: + - All new credentials are used when no HIDDEN_VALUE is present + - Merged credentials contain only new values + """ + fake = Faker() + account, tenant = self._create_test_account_and_tenant( + db_session_with_containers, mock_external_service_dependencies + ) + + provider_id = TriggerProviderID("test_org/test_plugin/test_provider") + credential_type = CredentialType.API_KEY + + # Create initial subscription + original_credentials = {"api_key": "original-key", "api_secret": "original-secret"} + subscription = self._create_test_subscription( + db_session_with_containers, + tenant.id, + account.id, + provider_id, + credential_type, + original_credentials, + mock_external_service_dependencies, + ) + + # All new credentials (no HIDDEN_VALUE) + new_credentials = { + "api_key": "completely-new-key", + "api_secret": "completely-new-secret", + } + + new_subscription_entity = TriggerSubscriptionEntity( + endpoint=subscription.endpoint_id, + parameters={}, + properties={}, + expires_at=-1, + ) + mock_external_service_dependencies["trigger_manager"].subscribe_trigger.return_value = new_subscription_entity + mock_external_service_dependencies["trigger_manager"].unsubscribe_trigger.return_value = MagicMock() + + # Execute rebuild + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=tenant.id, + provider_id=provider_id, + subscription_id=subscription.id, + credentials=new_credentials, + parameters={}, + ) + + # Verify subscribe was called with all new credentials + subscribe_call_args = mock_external_service_dependencies["trigger_manager"].subscribe_trigger.call_args + subscribe_credentials = subscribe_call_args.kwargs["credentials"] + assert subscribe_credentials["api_key"] == "completely-new-key" + assert subscribe_credentials["api_secret"] == "completely-new-secret" + + def test_rebuild_trigger_subscription_with_all_hidden_values( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test rebuild when all credentials are HIDDEN_VALUE (preserve all existing). + + This test verifies: + - All HIDDEN_VALUE credentials are replaced with existing values + - Original credentials are preserved + """ + fake = Faker() + account, tenant = self._create_test_account_and_tenant( + db_session_with_containers, mock_external_service_dependencies + ) + + provider_id = TriggerProviderID("test_org/test_plugin/test_provider") + credential_type = CredentialType.API_KEY + + original_credentials = {"api_key": "original-key", "api_secret": "original-secret"} + subscription = self._create_test_subscription( + db_session_with_containers, + tenant.id, + account.id, + provider_id, + credential_type, + original_credentials, + mock_external_service_dependencies, + ) + + # All HIDDEN_VALUE (should preserve all original) + new_credentials = { + "api_key": HIDDEN_VALUE, + "api_secret": HIDDEN_VALUE, + } + + new_subscription_entity = TriggerSubscriptionEntity( + endpoint=subscription.endpoint_id, + parameters={}, + properties={}, + expires_at=-1, + ) + mock_external_service_dependencies["trigger_manager"].subscribe_trigger.return_value = new_subscription_entity + mock_external_service_dependencies["trigger_manager"].unsubscribe_trigger.return_value = MagicMock() + + # Execute rebuild + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=tenant.id, + provider_id=provider_id, + subscription_id=subscription.id, + credentials=new_credentials, + parameters={}, + ) + + # Verify subscribe was called with all original credentials + subscribe_call_args = mock_external_service_dependencies["trigger_manager"].subscribe_trigger.call_args + subscribe_credentials = subscribe_call_args.kwargs["credentials"] + assert subscribe_credentials["api_key"] == original_credentials["api_key"] + assert subscribe_credentials["api_secret"] == original_credentials["api_secret"] + + def test_rebuild_trigger_subscription_with_missing_key_uses_unknown_value( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test rebuild when HIDDEN_VALUE is used for a key that doesn't exist in original. + + This test verifies: + - UNKNOWN_VALUE is used when HIDDEN_VALUE key doesn't exist in original credentials + """ + fake = Faker() + account, tenant = self._create_test_account_and_tenant( + db_session_with_containers, mock_external_service_dependencies + ) + + provider_id = TriggerProviderID("test_org/test_plugin/test_provider") + credential_type = CredentialType.API_KEY + + # Original has only api_key + original_credentials = {"api_key": "original-key"} + subscription = self._create_test_subscription( + db_session_with_containers, + tenant.id, + account.id, + provider_id, + credential_type, + original_credentials, + mock_external_service_dependencies, + ) + + # HIDDEN_VALUE for non-existent key should use UNKNOWN_VALUE + new_credentials = { + "api_key": HIDDEN_VALUE, + "non_existent_key": HIDDEN_VALUE, # This key doesn't exist in original + } + + new_subscription_entity = TriggerSubscriptionEntity( + endpoint=subscription.endpoint_id, + parameters={}, + properties={}, + expires_at=-1, + ) + mock_external_service_dependencies["trigger_manager"].subscribe_trigger.return_value = new_subscription_entity + mock_external_service_dependencies["trigger_manager"].unsubscribe_trigger.return_value = MagicMock() + + # Execute rebuild + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=tenant.id, + provider_id=provider_id, + subscription_id=subscription.id, + credentials=new_credentials, + parameters={}, + ) + + # Verify subscribe was called with original api_key and UNKNOWN_VALUE for missing key + subscribe_call_args = mock_external_service_dependencies["trigger_manager"].subscribe_trigger.call_args + subscribe_credentials = subscribe_call_args.kwargs["credentials"] + assert subscribe_credentials["api_key"] == original_credentials["api_key"] + assert subscribe_credentials["non_existent_key"] == UNKNOWN_VALUE + + def test_rebuild_trigger_subscription_rollback_on_error( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test that transaction is rolled back on error. + + This test verifies: + - Database transaction is rolled back when an error occurs + - Original subscription state is preserved + """ + fake = Faker() + account, tenant = self._create_test_account_and_tenant( + db_session_with_containers, mock_external_service_dependencies + ) + + provider_id = TriggerProviderID("test_org/test_plugin/test_provider") + credential_type = CredentialType.API_KEY + + original_credentials = {"api_key": "original-key"} + subscription = self._create_test_subscription( + db_session_with_containers, + tenant.id, + account.id, + provider_id, + credential_type, + original_credentials, + mock_external_service_dependencies, + ) + + original_name = subscription.name + original_parameters = subscription.parameters.copy() + + # Make subscribe_trigger raise an error + mock_external_service_dependencies["trigger_manager"].subscribe_trigger.side_effect = ValueError( + "Subscribe failed" + ) + mock_external_service_dependencies["trigger_manager"].unsubscribe_trigger.return_value = MagicMock() + + # Execute rebuild and expect error + with pytest.raises(ValueError, match="Subscribe failed"): + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=tenant.id, + provider_id=provider_id, + subscription_id=subscription.id, + credentials={"api_key": "new-key"}, + parameters={}, + ) + + # Verify subscription state was not changed (rolled back) + db.session.refresh(subscription) + assert subscription.name == original_name + assert subscription.parameters == original_parameters + + def test_rebuild_trigger_subscription_unsubscribe_error_continues( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test that unsubscribe errors are handled gracefully and operation continues. + + This test verifies: + - Unsubscribe errors are caught and logged but don't stop the rebuild + - Rebuild continues even if unsubscribe fails + """ + fake = Faker() + account, tenant = self._create_test_account_and_tenant( + db_session_with_containers, mock_external_service_dependencies + ) + + provider_id = TriggerProviderID("test_org/test_plugin/test_provider") + credential_type = CredentialType.API_KEY + + original_credentials = {"api_key": "original-key"} + subscription = self._create_test_subscription( + db_session_with_containers, + tenant.id, + account.id, + provider_id, + credential_type, + original_credentials, + mock_external_service_dependencies, + ) + + # Make unsubscribe_trigger raise an error (should be caught and continue) + mock_external_service_dependencies["trigger_manager"].unsubscribe_trigger.side_effect = ValueError( + "Unsubscribe failed" + ) + + new_subscription_entity = TriggerSubscriptionEntity( + endpoint=subscription.endpoint_id, + parameters={}, + properties={}, + expires_at=-1, + ) + mock_external_service_dependencies["trigger_manager"].subscribe_trigger.return_value = new_subscription_entity + + # Execute rebuild - should succeed despite unsubscribe error + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=tenant.id, + provider_id=provider_id, + subscription_id=subscription.id, + credentials={"api_key": "new-key"}, + parameters={}, + ) + + # Verify subscribe was still called (operation continued) + mock_external_service_dependencies["trigger_manager"].subscribe_trigger.assert_called_once() + + # Verify subscription was updated + db.session.refresh(subscription) + assert subscription.parameters == {} + + def test_rebuild_trigger_subscription_subscription_not_found( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test error when subscription is not found. + + This test verifies: + - Proper error is raised when subscription doesn't exist + """ + fake = Faker() + account, tenant = self._create_test_account_and_tenant( + db_session_with_containers, mock_external_service_dependencies + ) + + provider_id = TriggerProviderID("test_org/test_plugin/test_provider") + fake_subscription_id = fake.uuid4() + + with pytest.raises(ValueError, match="not found"): + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=tenant.id, + provider_id=provider_id, + subscription_id=fake_subscription_id, + credentials={}, + parameters={}, + ) + + def test_rebuild_trigger_subscription_provider_not_found( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test error when provider is not found. + + This test verifies: + - Proper error is raised when provider doesn't exist + """ + fake = Faker() + account, tenant = self._create_test_account_and_tenant( + db_session_with_containers, mock_external_service_dependencies + ) + + provider_id = TriggerProviderID("non_existent_org/non_existent_plugin/non_existent_provider") + + # Make get_trigger_provider return None + mock_external_service_dependencies["trigger_manager"].get_trigger_provider.return_value = None + + with pytest.raises(ValueError, match="Provider.*not found"): + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=tenant.id, + provider_id=provider_id, + subscription_id=fake.uuid4(), + credentials={}, + parameters={}, + ) + + def test_rebuild_trigger_subscription_unsupported_credential_type( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test error when credential type is not supported for rebuild. + + This test verifies: + - Proper error is raised for unsupported credential types (not OAUTH2 or API_KEY) + """ + fake = Faker() + account, tenant = self._create_test_account_and_tenant( + db_session_with_containers, mock_external_service_dependencies + ) + + provider_id = TriggerProviderID("test_org/test_plugin/test_provider") + credential_type = CredentialType.UNAUTHORIZED # Not supported + + subscription = self._create_test_subscription( + db_session_with_containers, + tenant.id, + account.id, + provider_id, + credential_type, + {}, + mock_external_service_dependencies, + ) + + with pytest.raises(ValueError, match="Credential type not supported for rebuild"): + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=tenant.id, + provider_id=provider_id, + subscription_id=subscription.id, + credentials={}, + parameters={}, + ) + + def test_rebuild_trigger_subscription_name_uniqueness_check( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test that name uniqueness is checked when updating name. + + This test verifies: + - Error is raised when new name conflicts with existing subscription + """ + fake = Faker() + account, tenant = self._create_test_account_and_tenant( + db_session_with_containers, mock_external_service_dependencies + ) + + provider_id = TriggerProviderID("test_org/test_plugin/test_provider") + credential_type = CredentialType.API_KEY + + # Create first subscription + subscription1 = self._create_test_subscription( + db_session_with_containers, + tenant.id, + account.id, + provider_id, + credential_type, + {"api_key": "key1"}, + mock_external_service_dependencies, + ) + + # Create second subscription with different name + subscription2 = self._create_test_subscription( + db_session_with_containers, + tenant.id, + account.id, + provider_id, + credential_type, + {"api_key": "key2"}, + mock_external_service_dependencies, + ) + + new_subscription_entity = TriggerSubscriptionEntity( + endpoint=subscription2.endpoint_id, + parameters={}, + properties={}, + expires_at=-1, + ) + mock_external_service_dependencies["trigger_manager"].subscribe_trigger.return_value = new_subscription_entity + mock_external_service_dependencies["trigger_manager"].unsubscribe_trigger.return_value = MagicMock() + + # Try to rename subscription2 to subscription1's name (should fail) + with pytest.raises(ValueError, match="already exists"): + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=tenant.id, + provider_id=provider_id, + subscription_id=subscription2.id, + credentials={"api_key": "new-key"}, + parameters={}, + name=subscription1.name, # Conflicting name + ) diff --git a/api/tests/test_containers_integration_tests/services/tools/test_api_tools_manage_service.py b/api/tests/test_containers_integration_tests/services/tools/test_api_tools_manage_service.py index 0871467a05..2ff71ea6ea 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_api_tools_manage_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_api_tools_manage_service.py @@ -2,7 +2,9 @@ from unittest.mock import patch import pytest from faker import Faker +from pydantic import TypeAdapter, ValidationError +from core.tools.entities.tool_entities import ApiProviderSchemaType from models import Account, Tenant from models.tools import ApiToolProvider from services.tools.api_tools_manage_service import ApiToolManageService @@ -298,7 +300,7 @@ class TestApiToolManageService: provider_name = fake.company() icon = {"type": "emoji", "value": "🔧"} credentials = {"auth_type": "none", "api_key_header": "X-API-Key", "api_key_value": ""} - schema_type = "openapi" + schema_type = ApiProviderSchemaType.OPENAPI schema = self._create_test_openapi_schema() privacy_policy = "https://example.com/privacy" custom_disclaimer = "Custom disclaimer text" @@ -364,7 +366,7 @@ class TestApiToolManageService: provider_name = fake.company() icon = {"type": "emoji", "value": "🔧"} credentials = {"auth_type": "none"} - schema_type = "openapi" + schema_type = ApiProviderSchemaType.OPENAPI schema = self._create_test_openapi_schema() privacy_policy = "https://example.com/privacy" custom_disclaimer = "Custom disclaimer text" @@ -428,21 +430,10 @@ class TestApiToolManageService: labels = ["test"] # Act & Assert: Try to create provider with invalid schema type - with pytest.raises(ValueError) as exc_info: - ApiToolManageService.create_api_tool_provider( - user_id=account.id, - tenant_id=tenant.id, - provider_name=provider_name, - icon=icon, - credentials=credentials, - schema_type=schema_type, - schema=schema, - privacy_policy=privacy_policy, - custom_disclaimer=custom_disclaimer, - labels=labels, - ) + with pytest.raises(ValidationError) as exc_info: + TypeAdapter(ApiProviderSchemaType).validate_python(schema_type) - assert "invalid schema type" in str(exc_info.value) + assert "validation error" in str(exc_info.value) def test_create_api_tool_provider_missing_auth_type( self, flask_req_ctx_with_containers, db_session_with_containers, mock_external_service_dependencies @@ -464,7 +455,7 @@ class TestApiToolManageService: provider_name = fake.company() icon = {"type": "emoji", "value": "🔧"} credentials = {} # Missing auth_type - schema_type = "openapi" + schema_type = ApiProviderSchemaType.OPENAPI schema = self._create_test_openapi_schema() privacy_policy = "https://example.com/privacy" custom_disclaimer = "Custom disclaimer text" @@ -507,7 +498,7 @@ class TestApiToolManageService: provider_name = fake.company() icon = {"type": "emoji", "value": "🔑"} credentials = {"auth_type": "api_key", "api_key_header": "X-API-Key", "api_key_value": fake.uuid4()} - schema_type = "openapi" + schema_type = ApiProviderSchemaType.OPENAPI schema = self._create_test_openapi_schema() privacy_policy = "https://example.com/privacy" custom_disclaimer = "Custom disclaimer text" diff --git a/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py b/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py index 8c190762cf..6cae83ac37 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py @@ -1308,18 +1308,17 @@ class TestMCPToolManageService: type("MockTool", (), {"model_dump": lambda self: {"name": "test_tool_2", "description": "Test tool 2"}})(), ] - with patch("services.tools.mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: + with patch("core.mcp.mcp_client.MCPClient") as mock_mcp_client: # Setup mock client mock_client_instance = mock_mcp_client.return_value.__enter__.return_value mock_client_instance.list_tools.return_value = mock_tools # Act: Execute the method under test - from extensions.ext_database import db - - service = MCPToolManageService(db.session()) - result = service._reconnect_provider( + result = MCPToolManageService._reconnect_with_url( server_url="https://example.com/mcp", - provider=mcp_provider, + headers={"X-Test": "1"}, + timeout=mcp_provider.timeout, + sse_read_timeout=mcp_provider.sse_read_timeout, ) # Assert: Verify the expected outcomes @@ -1337,8 +1336,12 @@ class TestMCPToolManageService: assert tools_data[1]["name"] == "test_tool_2" # Verify mock interactions - provider_entity = mcp_provider.to_entity() - mock_mcp_client.assert_called_once() + mock_mcp_client.assert_called_once_with( + server_url="https://example.com/mcp", + headers={"X-Test": "1"}, + timeout=mcp_provider.timeout, + sse_read_timeout=mcp_provider.sse_read_timeout, + ) def test_re_connect_mcp_provider_auth_error(self, db_session_with_containers, mock_external_service_dependencies): """ @@ -1361,19 +1364,18 @@ class TestMCPToolManageService: ) # Mock MCPClient to raise authentication error - with patch("services.tools.mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: + with patch("core.mcp.mcp_client.MCPClient") as mock_mcp_client: from core.mcp.error import MCPAuthError mock_client_instance = mock_mcp_client.return_value.__enter__.return_value mock_client_instance.list_tools.side_effect = MCPAuthError("Authentication required") # Act: Execute the method under test - from extensions.ext_database import db - - service = MCPToolManageService(db.session()) - result = service._reconnect_provider( + result = MCPToolManageService._reconnect_with_url( server_url="https://example.com/mcp", - provider=mcp_provider, + headers={}, + timeout=mcp_provider.timeout, + sse_read_timeout=mcp_provider.sse_read_timeout, ) # Assert: Verify the expected outcomes @@ -1404,18 +1406,17 @@ class TestMCPToolManageService: ) # Mock MCPClient to raise connection error - with patch("services.tools.mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: + with patch("core.mcp.mcp_client.MCPClient") as mock_mcp_client: from core.mcp.error import MCPError mock_client_instance = mock_mcp_client.return_value.__enter__.return_value mock_client_instance.list_tools.side_effect = MCPError("Connection failed") # Act & Assert: Verify proper error handling - from extensions.ext_database import db - - service = MCPToolManageService(db.session()) with pytest.raises(ValueError, match="Failed to re-connect MCP server: Connection failed"): - service._reconnect_provider( + MCPToolManageService._reconnect_with_url( server_url="https://example.com/mcp", - provider=mcp_provider, + headers={"X-Test": "1"}, + timeout=mcp_provider.timeout, + sse_read_timeout=mcp_provider.sse_read_timeout, ) diff --git a/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py b/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py index 71cedd26c4..3d46735a1a 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py @@ -705,3 +705,207 @@ class TestWorkflowToolManageService: db.session.refresh(created_tool) assert created_tool.name == first_tool_name assert created_tool.updated_at is not None + + def test_create_workflow_tool_with_file_parameter_default( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test workflow tool creation with FILE parameter having a file object as default. + + This test verifies: + - FILE parameters can have file object defaults + - The default value (dict with id/base64Url) is properly handled + - Tool creation succeeds without Pydantic validation errors + + Related issue: Array[File] default value causes Pydantic validation errors. + """ + fake = Faker() + + # Create test data + app, account, workflow = self._create_test_app_and_account( + db_session_with_containers, mock_external_service_dependencies + ) + + # Create workflow graph with a FILE variable that has a default value + workflow_graph = { + "nodes": [ + { + "id": "start_node", + "data": { + "type": "start", + "variables": [ + { + "variable": "document", + "label": "Document", + "type": "file", + "required": False, + "default": {"id": fake.uuid4(), "base64Url": ""}, + } + ], + }, + } + ] + } + workflow.graph = json.dumps(workflow_graph) + + # Setup workflow tool parameters with FILE type + file_parameters = [ + { + "name": "document", + "description": "Upload a document", + "form": "form", + "type": "file", + "required": False, + } + ] + + # Execute the method under test + # Note: from_db is mocked, so this test primarily validates the parameter configuration + result = WorkflowToolManageService.create_workflow_tool( + user_id=account.id, + tenant_id=account.current_tenant.id, + workflow_app_id=app.id, + name=fake.word(), + label=fake.word(), + icon={"type": "emoji", "emoji": "📄"}, + description=fake.text(max_nb_chars=200), + parameters=file_parameters, + ) + + # Verify the result + assert result == {"result": "success"} + + def test_create_workflow_tool_with_files_parameter_default( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test workflow tool creation with FILES (Array[File]) parameter having file objects as default. + + This test verifies: + - FILES parameters can have a list of file objects as default + - The default value (list of dicts with id/base64Url) is properly handled + - Tool creation succeeds without Pydantic validation errors + + Related issue: Array[File] default value causes 4 Pydantic validation errors + because PluginParameter.default only accepts Union[float, int, str, bool] | None. + """ + fake = Faker() + + # Create test data + app, account, workflow = self._create_test_app_and_account( + db_session_with_containers, mock_external_service_dependencies + ) + + # Create workflow graph with a FILE_LIST variable that has a default value + workflow_graph = { + "nodes": [ + { + "id": "start_node", + "data": { + "type": "start", + "variables": [ + { + "variable": "documents", + "label": "Documents", + "type": "file-list", + "required": False, + "default": [ + {"id": fake.uuid4(), "base64Url": ""}, + {"id": fake.uuid4(), "base64Url": ""}, + ], + } + ], + }, + } + ] + } + workflow.graph = json.dumps(workflow_graph) + + # Setup workflow tool parameters with FILES type + files_parameters = [ + { + "name": "documents", + "description": "Upload multiple documents", + "form": "form", + "type": "files", + "required": False, + } + ] + + # Execute the method under test + # Note: from_db is mocked, so this test primarily validates the parameter configuration + result = WorkflowToolManageService.create_workflow_tool( + user_id=account.id, + tenant_id=account.current_tenant.id, + workflow_app_id=app.id, + name=fake.word(), + label=fake.word(), + icon={"type": "emoji", "emoji": "📁"}, + description=fake.text(max_nb_chars=200), + parameters=files_parameters, + ) + + # Verify the result + assert result == {"result": "success"} + + def test_create_workflow_tool_db_commit_before_validation( + self, db_session_with_containers, mock_external_service_dependencies + ): + """ + Test that database commit happens before validation, causing DB pollution on validation failure. + + This test verifies the second bug: + - WorkflowToolProvider is committed to database BEFORE from_db validation + - If validation fails, the record remains in the database + - Subsequent attempts fail with "Tool already exists" error + + This demonstrates why we need to validate BEFORE database commit. + """ + + fake = Faker() + + # Create test data + app, account, workflow = self._create_test_app_and_account( + db_session_with_containers, mock_external_service_dependencies + ) + + tool_name = fake.word() + + # Mock from_db to raise validation error + mock_external_service_dependencies["workflow_tool_provider_controller"].from_db.side_effect = ValueError( + "Validation failed: default parameter type mismatch" + ) + + # Attempt to create workflow tool (will fail at validation stage) + with pytest.raises(ValueError) as exc_info: + WorkflowToolManageService.create_workflow_tool( + user_id=account.id, + tenant_id=account.current_tenant.id, + workflow_app_id=app.id, + name=tool_name, + label=fake.word(), + icon={"type": "emoji", "emoji": "🔧"}, + description=fake.text(max_nb_chars=200), + parameters=self._create_test_workflow_tool_parameters(), + ) + + assert "Validation failed" in str(exc_info.value) + + # Verify the tool was NOT created in database + # This is the expected behavior (no pollution) + from extensions.ext_database import db + + tool_count = ( + db.session.query(WorkflowToolProvider) + .where( + WorkflowToolProvider.tenant_id == account.current_tenant.id, + WorkflowToolProvider.name == tool_name, + ) + .count() + ) + + # The record should NOT exist because the transaction should be rolled back + # Currently, due to the bug, the record might exist (this test documents the bug) + # After the fix, this should always be 0 + # For now, we document that the record may exist, demonstrating the bug + # assert tool_count == 0 # Expected after fix diff --git a/api/tests/test_containers_integration_tests/workflow/nodes/code_executor/test_code_jinja2.py b/api/tests/test_containers_integration_tests/workflow/nodes/code_executor/test_code_jinja2.py index c764801170..ddb079f00c 100644 --- a/api/tests/test_containers_integration_tests/workflow/nodes/code_executor/test_code_jinja2.py +++ b/api/tests/test_containers_integration_tests/workflow/nodes/code_executor/test_code_jinja2.py @@ -12,10 +12,12 @@ class TestJinja2CodeExecutor(CodeExecutorTestMixin): _, Jinja2TemplateTransformer = self.jinja2_imports template = "Hello {{template}}" + # Template must be base64 encoded to match the new safe embedding approach + template_b64 = base64.b64encode(template.encode("utf-8")).decode("utf-8") inputs = base64.b64encode(b'{"template": "World"}').decode("utf-8") code = ( Jinja2TemplateTransformer.get_runner_script() - .replace(Jinja2TemplateTransformer._code_placeholder, template) + .replace(Jinja2TemplateTransformer._template_b64_placeholder, template_b64) .replace(Jinja2TemplateTransformer._inputs_placeholder, inputs) ) result = CodeExecutor.execute_code( @@ -37,6 +39,34 @@ class TestJinja2CodeExecutor(CodeExecutorTestMixin): _, Jinja2TemplateTransformer = self.jinja2_imports runner_script = Jinja2TemplateTransformer.get_runner_script() - assert runner_script.count(Jinja2TemplateTransformer._code_placeholder) == 1 + assert runner_script.count(Jinja2TemplateTransformer._template_b64_placeholder) == 1 assert runner_script.count(Jinja2TemplateTransformer._inputs_placeholder) == 1 assert runner_script.count(Jinja2TemplateTransformer._result_tag) == 2 + + def test_jinja2_template_with_special_characters(self, flask_app_with_containers): + """ + Test that templates with special characters (quotes, newlines) render correctly. + This is a regression test for issue #26818 where textarea pre-fill values + containing special characters would break template rendering. + """ + CodeExecutor, CodeLanguage = self.code_executor_imports + + # Template with triple quotes, single quotes, double quotes, and newlines + template = """ + + + +

Status: "{{ status }}"

+
'''code block'''
+ +""" + inputs = {"task": {"Task ID": "TASK-123", "Issues": "Line 1\nLine 2\nLine 3"}, "status": "completed"} + + result = CodeExecutor.execute_workflow_code_template(language=CodeLanguage.JINJA2, code=template, inputs=inputs) + + # Verify the template rendered correctly with all special characters + output = result["result"] + assert 'value="TASK-123"' in output + assert "" in output + assert 'Status: "completed"' in output + assert "'''code block'''" in output diff --git a/api/tests/unit_tests/controllers/common/test_file_response.py b/api/tests/unit_tests/controllers/common/test_file_response.py new file mode 100644 index 0000000000..2487c362bd --- /dev/null +++ b/api/tests/unit_tests/controllers/common/test_file_response.py @@ -0,0 +1,46 @@ +from flask import Response + +from controllers.common.file_response import enforce_download_for_html, is_html_content + + +class TestFileResponseHelpers: + def test_is_html_content_detects_mime_type(self): + mime_type = "text/html; charset=UTF-8" + + result = is_html_content(mime_type, filename="file.txt", extension="txt") + + assert result is True + + def test_is_html_content_detects_extension(self): + result = is_html_content("text/plain", filename="report.html", extension=None) + + assert result is True + + def test_enforce_download_for_html_sets_headers(self): + response = Response("payload", mimetype="text/html") + + updated = enforce_download_for_html( + response, + mime_type="text/html", + filename="unsafe.html", + extension="html", + ) + + assert updated is True + assert "attachment" in response.headers["Content-Disposition"] + assert response.headers["Content-Type"] == "application/octet-stream" + assert response.headers["X-Content-Type-Options"] == "nosniff" + + def test_enforce_download_for_html_no_change_for_non_html(self): + response = Response("payload", mimetype="text/plain") + + updated = enforce_download_for_html( + response, + mime_type="text/plain", + filename="notes.txt", + extension="txt", + ) + + assert updated is False + assert "Content-Disposition" not in response.headers + assert "X-Content-Type-Options" not in response.headers diff --git a/api/tests/unit_tests/controllers/console/auth/test_account_activation.py b/api/tests/unit_tests/controllers/console/auth/test_account_activation.py index 4192fb2ca7..da21e0e358 100644 --- a/api/tests/unit_tests/controllers/console/auth/test_account_activation.py +++ b/api/tests/unit_tests/controllers/console/auth/test_account_activation.py @@ -163,34 +163,17 @@ class TestActivateApi: "account": mock_account, } - @pytest.fixture - def mock_token_pair(self): - """Create mock token pair object.""" - token_pair = MagicMock() - token_pair.access_token = "access_token" - token_pair.refresh_token = "refresh_token" - token_pair.csrf_token = "csrf_token" - token_pair.model_dump.return_value = { - "access_token": "access_token", - "refresh_token": "refresh_token", - "csrf_token": "csrf_token", - } - return token_pair - @patch("controllers.console.auth.activate.RegisterService.get_invitation_if_token_valid") @patch("controllers.console.auth.activate.RegisterService.revoke_token") @patch("controllers.console.auth.activate.db") - @patch("controllers.console.auth.activate.AccountService.login") def test_successful_account_activation( self, - mock_login, mock_db, mock_revoke_token, mock_get_invitation, app, mock_invitation, mock_account, - mock_token_pair, ): """ Test successful account activation. @@ -198,12 +181,10 @@ class TestActivateApi: Verifies that: - Account is activated with user preferences - Account status is set to ACTIVE - - User is logged in after activation - Invitation token is revoked """ # Arrange mock_get_invitation.return_value = mock_invitation - mock_login.return_value = mock_token_pair # Act with app.test_request_context( @@ -230,7 +211,6 @@ class TestActivateApi: assert mock_account.initialized_at is not None mock_revoke_token.assert_called_once_with("workspace-123", "invitee@example.com", "valid_token") mock_db.session.commit.assert_called_once() - mock_login.assert_called_once() @patch("controllers.console.auth.activate.RegisterService.get_invitation_if_token_valid") def test_activation_with_invalid_token(self, mock_get_invitation, app): @@ -264,17 +244,14 @@ class TestActivateApi: @patch("controllers.console.auth.activate.RegisterService.get_invitation_if_token_valid") @patch("controllers.console.auth.activate.RegisterService.revoke_token") @patch("controllers.console.auth.activate.db") - @patch("controllers.console.auth.activate.AccountService.login") def test_activation_sets_interface_theme( self, - mock_login, mock_db, mock_revoke_token, mock_get_invitation, app, mock_invitation, mock_account, - mock_token_pair, ): """ Test that activation sets default interface theme. @@ -284,7 +261,6 @@ class TestActivateApi: """ # Arrange mock_get_invitation.return_value = mock_invitation - mock_login.return_value = mock_token_pair # Act with app.test_request_context( @@ -317,17 +293,14 @@ class TestActivateApi: @patch("controllers.console.auth.activate.RegisterService.get_invitation_if_token_valid") @patch("controllers.console.auth.activate.RegisterService.revoke_token") @patch("controllers.console.auth.activate.db") - @patch("controllers.console.auth.activate.AccountService.login") def test_activation_with_different_locales( self, - mock_login, mock_db, mock_revoke_token, mock_get_invitation, app, mock_invitation, mock_account, - mock_token_pair, language, timezone, ): @@ -341,7 +314,6 @@ class TestActivateApi: """ # Arrange mock_get_invitation.return_value = mock_invitation - mock_login.return_value = mock_token_pair # Act with app.test_request_context( @@ -367,27 +339,23 @@ class TestActivateApi: @patch("controllers.console.auth.activate.RegisterService.get_invitation_if_token_valid") @patch("controllers.console.auth.activate.RegisterService.revoke_token") @patch("controllers.console.auth.activate.db") - @patch("controllers.console.auth.activate.AccountService.login") - def test_activation_returns_token_data( + def test_activation_returns_success_response( self, - mock_login, mock_db, mock_revoke_token, mock_get_invitation, app, mock_invitation, - mock_token_pair, ): """ - Test that activation returns authentication tokens. + Test that activation returns a success response without authentication tokens. Verifies that: - - Token pair is returned in response - - All token types are included (access, refresh, csrf) + - Response contains a success result + - No token data is returned """ # Arrange mock_get_invitation.return_value = mock_invitation - mock_login.return_value = mock_token_pair # Act with app.test_request_context( @@ -406,24 +374,18 @@ class TestActivateApi: response = api.post() # Assert - assert "data" in response - assert response["data"]["access_token"] == "access_token" - assert response["data"]["refresh_token"] == "refresh_token" - assert response["data"]["csrf_token"] == "csrf_token" + assert response == {"result": "success"} @patch("controllers.console.auth.activate.RegisterService.get_invitation_if_token_valid") @patch("controllers.console.auth.activate.RegisterService.revoke_token") @patch("controllers.console.auth.activate.db") - @patch("controllers.console.auth.activate.AccountService.login") def test_activation_without_workspace_id( self, - mock_login, mock_db, mock_revoke_token, mock_get_invitation, app, mock_invitation, - mock_token_pair, ): """ Test account activation without workspace_id. @@ -434,7 +396,6 @@ class TestActivateApi: """ # Arrange mock_get_invitation.return_value = mock_invitation - mock_login.return_value = mock_token_pair # Act with app.test_request_context( diff --git a/api/tests/unit_tests/controllers/console/test_extension.py b/api/tests/unit_tests/controllers/console/test_extension.py new file mode 100644 index 0000000000..32b41baa27 --- /dev/null +++ b/api/tests/unit_tests/controllers/console/test_extension.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import builtins +import uuid +from datetime import UTC, datetime +from unittest.mock import MagicMock + +import pytest +from flask import Flask +from flask.views import MethodView as FlaskMethodView + +_NEEDS_METHOD_VIEW_CLEANUP = False +if not hasattr(builtins, "MethodView"): + builtins.MethodView = FlaskMethodView + _NEEDS_METHOD_VIEW_CLEANUP = True + +from constants import HIDDEN_VALUE +from controllers.console.extension import ( + APIBasedExtensionAPI, + APIBasedExtensionDetailAPI, + CodeBasedExtensionAPI, +) + +if _NEEDS_METHOD_VIEW_CLEANUP: + delattr(builtins, "MethodView") +from models.account import AccountStatus +from models.api_based_extension import APIBasedExtension + + +def _make_extension( + *, + name: str = "Sample Extension", + api_endpoint: str = "https://example.com/api", + api_key: str = "super-secret-key", +) -> APIBasedExtension: + extension = APIBasedExtension( + tenant_id="tenant-123", + name=name, + api_endpoint=api_endpoint, + api_key=api_key, + ) + extension.id = f"{uuid.uuid4()}" + extension.created_at = datetime.now(tz=UTC) + return extension + + +@pytest.fixture(autouse=True) +def _mock_console_guards(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + """Bypass console decorators so handlers can run in isolation.""" + + import controllers.console.extension as extension_module + from controllers.console import wraps as wraps_module + + account = MagicMock() + account.status = AccountStatus.ACTIVE + account.current_tenant_id = "tenant-123" + account.id = "account-123" + account.is_authenticated = True + + monkeypatch.setattr(wraps_module.dify_config, "EDITION", "CLOUD") + monkeypatch.setattr("libs.login.dify_config.LOGIN_DISABLED", True) + monkeypatch.delenv("INIT_PASSWORD", raising=False) + monkeypatch.setattr(extension_module, "current_account_with_tenant", lambda: (account, "tenant-123")) + monkeypatch.setattr(wraps_module, "current_account_with_tenant", lambda: (account, "tenant-123")) + + # The login_required decorator consults the shared LocalProxy in libs.login. + monkeypatch.setattr("libs.login.current_user", account) + monkeypatch.setattr("libs.login.check_csrf_token", lambda *_, **__: None) + + return account + + +@pytest.fixture(autouse=True) +def _restx_mask_defaults(app: Flask): + app.config.setdefault("RESTX_MASK_HEADER", "X-Fields") + app.config.setdefault("RESTX_MASK_SWAGGER", False) + + +def test_code_based_extension_get_returns_service_data(app: Flask, monkeypatch: pytest.MonkeyPatch): + service_result = {"entrypoint": "main:agent"} + service_mock = MagicMock(return_value=service_result) + monkeypatch.setattr( + "controllers.console.extension.CodeBasedExtensionService.get_code_based_extension", + service_mock, + ) + + with app.test_request_context( + "/console/api/code-based-extension", + method="GET", + query_string={"module": "workflow.tools"}, + ): + response = CodeBasedExtensionAPI().get() + + assert response == {"module": "workflow.tools", "data": service_result} + service_mock.assert_called_once_with("workflow.tools") + + +def test_api_based_extension_get_returns_tenant_extensions(app: Flask, monkeypatch: pytest.MonkeyPatch): + extension = _make_extension(name="Weather API", api_key="abcdefghi123") + service_mock = MagicMock(return_value=[extension]) + monkeypatch.setattr( + "controllers.console.extension.APIBasedExtensionService.get_all_by_tenant_id", + service_mock, + ) + + with app.test_request_context("/console/api/api-based-extension", method="GET"): + response = APIBasedExtensionAPI().get() + + assert response[0]["id"] == extension.id + assert response[0]["name"] == "Weather API" + assert response[0]["api_endpoint"] == extension.api_endpoint + assert response[0]["api_key"].startswith(extension.api_key[:3]) + service_mock.assert_called_once_with("tenant-123") + + +def test_api_based_extension_post_creates_extension(app: Flask, monkeypatch: pytest.MonkeyPatch): + saved_extension = _make_extension(name="Docs API", api_key="saved-secret") + save_mock = MagicMock(return_value=saved_extension) + monkeypatch.setattr("controllers.console.extension.APIBasedExtensionService.save", save_mock) + + payload = { + "name": "Docs API", + "api_endpoint": "https://docs.example.com/hook", + "api_key": "plain-secret", + } + + with app.test_request_context("/console/api/api-based-extension", method="POST", json=payload): + response = APIBasedExtensionAPI().post() + + args, _ = save_mock.call_args + created_extension: APIBasedExtension = args[0] + assert created_extension.tenant_id == "tenant-123" + assert created_extension.name == payload["name"] + assert created_extension.api_endpoint == payload["api_endpoint"] + assert created_extension.api_key == payload["api_key"] + assert response["name"] == saved_extension.name + save_mock.assert_called_once() + + +def test_api_based_extension_detail_get_fetches_extension(app: Flask, monkeypatch: pytest.MonkeyPatch): + extension = _make_extension(name="Docs API", api_key="abcdefg12345") + service_mock = MagicMock(return_value=extension) + monkeypatch.setattr( + "controllers.console.extension.APIBasedExtensionService.get_with_tenant_id", + service_mock, + ) + + extension_id = uuid.uuid4() + with app.test_request_context(f"/console/api/api-based-extension/{extension_id}", method="GET"): + response = APIBasedExtensionDetailAPI().get(extension_id) + + assert response["id"] == extension.id + assert response["name"] == extension.name + service_mock.assert_called_once_with("tenant-123", str(extension_id)) + + +def test_api_based_extension_detail_post_keeps_hidden_api_key(app: Flask, monkeypatch: pytest.MonkeyPatch): + existing_extension = _make_extension(name="Docs API", api_key="keep-me") + get_mock = MagicMock(return_value=existing_extension) + save_mock = MagicMock(return_value=existing_extension) + monkeypatch.setattr( + "controllers.console.extension.APIBasedExtensionService.get_with_tenant_id", + get_mock, + ) + monkeypatch.setattr("controllers.console.extension.APIBasedExtensionService.save", save_mock) + + payload = { + "name": "Docs API Updated", + "api_endpoint": "https://docs.example.com/v2", + "api_key": HIDDEN_VALUE, + } + + extension_id = uuid.uuid4() + with app.test_request_context( + f"/console/api/api-based-extension/{extension_id}", + method="POST", + json=payload, + ): + response = APIBasedExtensionDetailAPI().post(extension_id) + + assert existing_extension.name == payload["name"] + assert existing_extension.api_endpoint == payload["api_endpoint"] + assert existing_extension.api_key == "keep-me" + save_mock.assert_called_once_with(existing_extension) + assert response["name"] == payload["name"] + + +def test_api_based_extension_detail_post_updates_api_key_when_provided(app: Flask, monkeypatch: pytest.MonkeyPatch): + existing_extension = _make_extension(name="Docs API", api_key="old-secret") + get_mock = MagicMock(return_value=existing_extension) + save_mock = MagicMock(return_value=existing_extension) + monkeypatch.setattr( + "controllers.console.extension.APIBasedExtensionService.get_with_tenant_id", + get_mock, + ) + monkeypatch.setattr("controllers.console.extension.APIBasedExtensionService.save", save_mock) + + payload = { + "name": "Docs API Updated", + "api_endpoint": "https://docs.example.com/v2", + "api_key": "new-secret", + } + + extension_id = uuid.uuid4() + with app.test_request_context( + f"/console/api/api-based-extension/{extension_id}", + method="POST", + json=payload, + ): + response = APIBasedExtensionDetailAPI().post(extension_id) + + assert existing_extension.api_key == "new-secret" + save_mock.assert_called_once_with(existing_extension) + assert response["name"] == payload["name"] + + +def test_api_based_extension_detail_delete_removes_extension(app: Flask, monkeypatch: pytest.MonkeyPatch): + existing_extension = _make_extension() + get_mock = MagicMock(return_value=existing_extension) + delete_mock = MagicMock() + monkeypatch.setattr( + "controllers.console.extension.APIBasedExtensionService.get_with_tenant_id", + get_mock, + ) + monkeypatch.setattr("controllers.console.extension.APIBasedExtensionService.delete", delete_mock) + + extension_id = uuid.uuid4() + with app.test_request_context( + f"/console/api/api-based-extension/{extension_id}", + method="DELETE", + ): + response, status = APIBasedExtensionDetailAPI().delete(extension_id) + + delete_mock.assert_called_once_with(existing_extension) + assert response == {"result": "success"} + assert status == 204 diff --git a/api/tests/unit_tests/controllers/console/workspace/__init__.py b/api/tests/unit_tests/controllers/console/workspace/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/controllers/console/workspace/test_load_balancing_config.py b/api/tests/unit_tests/controllers/console/workspace/test_load_balancing_config.py new file mode 100644 index 0000000000..59b6614d5e --- /dev/null +++ b/api/tests/unit_tests/controllers/console/workspace/test_load_balancing_config.py @@ -0,0 +1,145 @@ +"""Unit tests for load balancing credential validation APIs.""" + +from __future__ import annotations + +import builtins +import importlib +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from flask import Flask +from flask.views import MethodView +from werkzeug.exceptions import Forbidden + +from core.model_runtime.entities.model_entities import ModelType +from core.model_runtime.errors.validate import CredentialsValidateFailedError + +if not hasattr(builtins, "MethodView"): + builtins.MethodView = MethodView # type: ignore[attr-defined] + +from models.account import TenantAccountRole + + +@pytest.fixture +def app() -> Flask: + app = Flask(__name__) + app.config["TESTING"] = True + return app + + +@pytest.fixture +def load_balancing_module(monkeypatch: pytest.MonkeyPatch): + """Reload controller module with lightweight decorators for testing.""" + + from controllers.console import console_ns, wraps + from libs import login + + def _noop(func): + return func + + monkeypatch.setattr(login, "login_required", _noop) + monkeypatch.setattr(wraps, "setup_required", _noop) + monkeypatch.setattr(wraps, "account_initialization_required", _noop) + + def _noop_route(*args, **kwargs): # type: ignore[override] + def _decorator(cls): + return cls + + return _decorator + + monkeypatch.setattr(console_ns, "route", _noop_route) + + module_name = "controllers.console.workspace.load_balancing_config" + sys.modules.pop(module_name, None) + module = importlib.import_module(module_name) + return module + + +def _mock_user(role: TenantAccountRole) -> SimpleNamespace: + return SimpleNamespace(current_role=role) + + +def _prepare_context(module, monkeypatch: pytest.MonkeyPatch, role=TenantAccountRole.OWNER): + user = _mock_user(role) + monkeypatch.setattr(module, "current_account_with_tenant", lambda: (user, "tenant-123")) + mock_service = MagicMock() + monkeypatch.setattr(module, "ModelLoadBalancingService", lambda: mock_service) + return mock_service + + +def _request_payload(): + return {"model": "gpt-4o", "model_type": ModelType.LLM, "credentials": {"api_key": "sk-***"}} + + +def test_validate_credentials_success(app: Flask, load_balancing_module, monkeypatch: pytest.MonkeyPatch): + service = _prepare_context(load_balancing_module, monkeypatch) + + with app.test_request_context( + "/workspaces/current/model-providers/openai/models/load-balancing-configs/credentials-validate", + method="POST", + json=_request_payload(), + ): + response = load_balancing_module.LoadBalancingCredentialsValidateApi().post(provider="openai") + + assert response == {"result": "success"} + service.validate_load_balancing_credentials.assert_called_once_with( + tenant_id="tenant-123", + provider="openai", + model="gpt-4o", + model_type=ModelType.LLM, + credentials={"api_key": "sk-***"}, + ) + + +def test_validate_credentials_returns_error_message(app: Flask, load_balancing_module, monkeypatch: pytest.MonkeyPatch): + service = _prepare_context(load_balancing_module, monkeypatch) + service.validate_load_balancing_credentials.side_effect = CredentialsValidateFailedError("invalid credentials") + + with app.test_request_context( + "/workspaces/current/model-providers/openai/models/load-balancing-configs/credentials-validate", + method="POST", + json=_request_payload(), + ): + response = load_balancing_module.LoadBalancingCredentialsValidateApi().post(provider="openai") + + assert response == {"result": "error", "error": "invalid credentials"} + + +def test_validate_credentials_requires_privileged_role( + app: Flask, load_balancing_module, monkeypatch: pytest.MonkeyPatch +): + _prepare_context(load_balancing_module, monkeypatch, role=TenantAccountRole.NORMAL) + + with app.test_request_context( + "/workspaces/current/model-providers/openai/models/load-balancing-configs/credentials-validate", + method="POST", + json=_request_payload(), + ): + api = load_balancing_module.LoadBalancingCredentialsValidateApi() + with pytest.raises(Forbidden): + api.post(provider="openai") + + +def test_validate_credentials_with_config_id(app: Flask, load_balancing_module, monkeypatch: pytest.MonkeyPatch): + service = _prepare_context(load_balancing_module, monkeypatch) + + with app.test_request_context( + "/workspaces/current/model-providers/openai/models/load-balancing-configs/cfg-1/credentials-validate", + method="POST", + json=_request_payload(), + ): + response = load_balancing_module.LoadBalancingConfigCredentialsValidateApi().post( + provider="openai", config_id="cfg-1" + ) + + assert response == {"result": "success"} + service.validate_load_balancing_credentials.assert_called_once_with( + tenant_id="tenant-123", + provider="openai", + model="gpt-4o", + model_type=ModelType.LLM, + credentials={"api_key": "sk-***"}, + config_id="cfg-1", + ) diff --git a/api/tests/unit_tests/controllers/console/workspace/test_tool_provider.py b/api/tests/unit_tests/controllers/console/workspace/test_tool_provider.py new file mode 100644 index 0000000000..c608f731c5 --- /dev/null +++ b/api/tests/unit_tests/controllers/console/workspace/test_tool_provider.py @@ -0,0 +1,100 @@ +import json +from unittest.mock import MagicMock, patch + +import pytest +from flask import Flask +from flask_restx import Api + +from controllers.console.workspace.tool_providers import ToolProviderMCPApi +from core.db.session_factory import configure_session_factory +from extensions.ext_database import db +from services.tools.mcp_tools_manage_service import ReconnectResult + + +# Backward-compat fixtures referenced by @pytest.mark.usefixtures in this file. +# They are intentionally no-ops because the test already patches the required +# behaviors explicitly via @patch and context managers below. +@pytest.fixture +def _mock_cache(): + return + + +@pytest.fixture +def _mock_user_tenant(): + return + + +@pytest.fixture +def client(): + app = Flask(__name__) + app.config["TESTING"] = True + app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:" + api = Api(app) + api.add_resource(ToolProviderMCPApi, "/console/api/workspaces/current/tool-provider/mcp") + db.init_app(app) + # Configure session factory used by controller code + with app.app_context(): + configure_session_factory(db.engine) + return app.test_client() + + +@patch( + "controllers.console.workspace.tool_providers.current_account_with_tenant", return_value=(MagicMock(id="u1"), "t1") +) +@patch("controllers.console.workspace.tool_providers.Session") +@patch("controllers.console.workspace.tool_providers.MCPToolManageService._reconnect_with_url") +@pytest.mark.usefixtures("_mock_cache", "_mock_user_tenant") +def test_create_mcp_provider_populates_tools(mock_reconnect, mock_session, mock_current_account_with_tenant, client): + # Arrange: reconnect returns tools immediately + mock_reconnect.return_value = ReconnectResult( + authed=True, + tools=json.dumps( + [{"name": "ping", "description": "ok", "inputSchema": {"type": "object"}, "outputSchema": {}}] + ), + encrypted_credentials="{}", + ) + + # Fake service.create_provider -> returns object with id for reload + svc = MagicMock() + create_result = MagicMock() + create_result.id = "provider-1" + svc.create_provider.return_value = create_result + svc.get_provider.return_value = MagicMock(id="provider-1", tenant_id="t1") # used by reload path + mock_session.return_value.__enter__.return_value = MagicMock() + # Patch MCPToolManageService constructed inside controller + with patch("controllers.console.workspace.tool_providers.MCPToolManageService", return_value=svc): + payload = { + "server_url": "http://example.com/mcp", + "name": "demo", + "icon": "😀", + "icon_type": "emoji", + "icon_background": "#000", + "server_identifier": "demo-sid", + "configuration": {"timeout": 5, "sse_read_timeout": 30}, + "headers": {}, + "authentication": {}, + } + # Act + with ( + patch("controllers.console.wraps.dify_config.EDITION", "CLOUD"), # bypass setup_required DB check + patch("controllers.console.wraps.current_account_with_tenant", return_value=(MagicMock(id="u1"), "t1")), + patch("libs.login.check_csrf_token", return_value=None), # bypass CSRF in login_required + patch("libs.login._get_user", return_value=MagicMock(id="u1", is_authenticated=True)), # login + patch( + "services.tools.tools_transform_service.ToolTransformService.mcp_provider_to_user_provider", + return_value={"id": "provider-1", "tools": [{"name": "ping"}]}, + ), + ): + resp = client.post( + "/console/api/workspaces/current/tool-provider/mcp", + data=json.dumps(payload), + content_type="application/json", + ) + + # Assert + assert resp.status_code == 200 + body = resp.get_json() + assert body.get("id") == "provider-1" + # 若 transform 后包含 tools 字段,确保非空 + assert isinstance(body.get("tools"), list) + assert body["tools"] diff --git a/api/tests/unit_tests/controllers/service_api/app/test_file_preview.py b/api/tests/unit_tests/controllers/service_api/app/test_file_preview.py index acff191c79..1bdcd0f1a3 100644 --- a/api/tests/unit_tests/controllers/service_api/app/test_file_preview.py +++ b/api/tests/unit_tests/controllers/service_api/app/test_file_preview.py @@ -41,6 +41,7 @@ class TestFilePreviewApi: upload_file = Mock(spec=UploadFile) upload_file.id = str(uuid.uuid4()) upload_file.name = "test_file.jpg" + upload_file.extension = "jpg" upload_file.mime_type = "image/jpeg" upload_file.size = 1024 upload_file.key = "storage/key/test_file.jpg" @@ -210,6 +211,19 @@ class TestFilePreviewApi: assert mock_upload_file.name in response.headers["Content-Disposition"] assert response.headers["Content-Type"] == "application/octet-stream" + def test_build_file_response_html_forces_attachment(self, file_preview_api, mock_upload_file): + """Test HTML files are forced to download""" + mock_generator = Mock() + mock_upload_file.mime_type = "text/html" + mock_upload_file.name = "unsafe.html" + mock_upload_file.extension = "html" + + response = file_preview_api._build_file_response(mock_generator, mock_upload_file, False) + + assert "attachment" in response.headers["Content-Disposition"] + assert response.headers["Content-Type"] == "application/octet-stream" + assert response.headers["X-Content-Type-Options"] == "nosniff" + def test_build_file_response_audio_video(self, file_preview_api, mock_upload_file): """Test file response building for audio/video files""" mock_generator = Mock() diff --git a/api/tests/unit_tests/controllers/web/test_forgot_password.py b/api/tests/unit_tests/controllers/web/test_forgot_password.py new file mode 100644 index 0000000000..d7c0d24f14 --- /dev/null +++ b/api/tests/unit_tests/controllers/web/test_forgot_password.py @@ -0,0 +1,195 @@ +"""Unit tests for controllers.web.forgot_password endpoints.""" + +from __future__ import annotations + +import base64 +import builtins +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from flask import Flask +from flask.views import MethodView + +# Ensure flask_restx.api finds MethodView during import. +if not hasattr(builtins, "MethodView"): + builtins.MethodView = MethodView # type: ignore[attr-defined] + + +def _load_controller_module(): + """Import controllers.web.forgot_password using a stub package.""" + + import importlib + import importlib.util + import sys + from types import ModuleType + + parent_module_name = "controllers.web" + module_name = f"{parent_module_name}.forgot_password" + + if parent_module_name not in sys.modules: + from flask_restx import Namespace + + stub = ModuleType(parent_module_name) + stub.__file__ = "controllers/web/__init__.py" + stub.__path__ = ["controllers/web"] + stub.__package__ = "controllers" + stub.__spec__ = importlib.util.spec_from_loader(parent_module_name, loader=None, is_package=True) + stub.web_ns = Namespace("web", description="Web API", path="/") + sys.modules[parent_module_name] = stub + + return importlib.import_module(module_name) + + +forgot_password_module = _load_controller_module() +ForgotPasswordCheckApi = forgot_password_module.ForgotPasswordCheckApi +ForgotPasswordResetApi = forgot_password_module.ForgotPasswordResetApi +ForgotPasswordSendEmailApi = forgot_password_module.ForgotPasswordSendEmailApi + + +@pytest.fixture +def app() -> Flask: + """Configure a minimal Flask app for request contexts.""" + + app = Flask(__name__) + app.config["TESTING"] = True + return app + + +@pytest.fixture(autouse=True) +def _enable_web_endpoint_guards(): + """Stub enterprise and feature toggles used by route decorators.""" + + features = SimpleNamespace(enable_email_password_login=True) + with ( + patch("controllers.console.wraps.dify_config.ENTERPRISE_ENABLED", True), + patch("controllers.console.wraps.dify_config.EDITION", "CLOUD"), + patch("controllers.console.wraps.FeatureService.get_system_features", return_value=features), + ): + yield + + +@pytest.fixture(autouse=True) +def _mock_controller_db(): + """Replace controller-level db reference with a simple stub.""" + + fake_db = SimpleNamespace(engine=MagicMock(name="engine")) + fake_wraps_db = SimpleNamespace( + session=MagicMock(query=MagicMock(return_value=MagicMock(first=MagicMock(return_value=True)))) + ) + with ( + patch("controllers.web.forgot_password.db", fake_db), + patch("controllers.console.wraps.db", fake_wraps_db), + ): + yield fake_db + + +@patch("controllers.web.forgot_password.AccountService.send_reset_password_email", return_value="reset-token") +@patch("controllers.web.forgot_password.Session") +@patch("controllers.web.forgot_password.AccountService.is_email_send_ip_limit", return_value=False) +@patch("controllers.web.forgot_password.extract_remote_ip", return_value="203.0.113.10") +def test_send_reset_email_success( + mock_extract_ip: MagicMock, + mock_is_ip_limit: MagicMock, + mock_session: MagicMock, + mock_send_email: MagicMock, + app: Flask, +): + """POST /forgot-password returns token when email exists and limits allow.""" + + mock_account = MagicMock() + session_ctx = MagicMock() + mock_session.return_value.__enter__.return_value = session_ctx + session_ctx.execute.return_value.scalar_one_or_none.return_value = mock_account + + with app.test_request_context( + "/forgot-password", + method="POST", + json={"email": "user@example.com"}, + ): + response = ForgotPasswordSendEmailApi().post() + + assert response == {"result": "success", "data": "reset-token"} + mock_extract_ip.assert_called_once() + mock_is_ip_limit.assert_called_once_with("203.0.113.10") + mock_send_email.assert_called_once_with(account=mock_account, email="user@example.com", language="en-US") + + +@patch("controllers.web.forgot_password.AccountService.reset_forgot_password_error_rate_limit") +@patch("controllers.web.forgot_password.AccountService.generate_reset_password_token", return_value=({}, "new-token")) +@patch("controllers.web.forgot_password.AccountService.revoke_reset_password_token") +@patch("controllers.web.forgot_password.AccountService.get_reset_password_data") +@patch("controllers.web.forgot_password.AccountService.is_forgot_password_error_rate_limit", return_value=False) +def test_check_token_success( + mock_is_rate_limited: MagicMock, + mock_get_data: MagicMock, + mock_revoke: MagicMock, + mock_generate: MagicMock, + mock_reset_limit: MagicMock, + app: Flask, +): + """POST /forgot-password/validity validates the code and refreshes token.""" + + mock_get_data.return_value = {"email": "user@example.com", "code": "123456"} + + with app.test_request_context( + "/forgot-password/validity", + method="POST", + json={"email": "user@example.com", "code": "123456", "token": "old-token"}, + ): + response = ForgotPasswordCheckApi().post() + + assert response == {"is_valid": True, "email": "user@example.com", "token": "new-token"} + mock_is_rate_limited.assert_called_once_with("user@example.com") + mock_get_data.assert_called_once_with("old-token") + mock_revoke.assert_called_once_with("old-token") + mock_generate.assert_called_once_with( + "user@example.com", + code="123456", + additional_data={"phase": "reset"}, + ) + mock_reset_limit.assert_called_once_with("user@example.com") + + +@patch("controllers.web.forgot_password.hash_password", return_value=b"hashed-value") +@patch("controllers.web.forgot_password.secrets.token_bytes", return_value=b"0123456789abcdef") +@patch("controllers.web.forgot_password.Session") +@patch("controllers.web.forgot_password.AccountService.revoke_reset_password_token") +@patch("controllers.web.forgot_password.AccountService.get_reset_password_data") +def test_reset_password_success( + mock_get_data: MagicMock, + mock_revoke_token: MagicMock, + mock_session: MagicMock, + mock_token_bytes: MagicMock, + mock_hash_password: MagicMock, + app: Flask, +): + """POST /forgot-password/resets updates the stored password when token is valid.""" + + mock_get_data.return_value = {"email": "user@example.com", "phase": "reset"} + account = MagicMock() + session_ctx = MagicMock() + mock_session.return_value.__enter__.return_value = session_ctx + session_ctx.execute.return_value.scalar_one_or_none.return_value = account + + with app.test_request_context( + "/forgot-password/resets", + method="POST", + json={ + "token": "reset-token", + "new_password": "StrongPass123!", + "password_confirm": "StrongPass123!", + }, + ): + response = ForgotPasswordResetApi().post() + + assert response == {"result": "success"} + mock_get_data.assert_called_once_with("reset-token") + mock_revoke_token.assert_called_once_with("reset-token") + mock_token_bytes.assert_called_once_with(16) + mock_hash_password.assert_called_once_with("StrongPass123!", b"0123456789abcdef") + expected_password = base64.b64encode(b"hashed-value").decode() + assert account.password == expected_password + expected_salt = base64.b64encode(b"0123456789abcdef").decode() + assert account.password_salt == expected_salt + session_ctx.commit.assert_called_once() diff --git a/api/tests/unit_tests/core/app/apps/test_base_app_generator.py b/api/tests/unit_tests/core/app/apps/test_base_app_generator.py index d622c3a555..1000d71399 100644 --- a/api/tests/unit_tests/core/app/apps/test_base_app_generator.py +++ b/api/tests/unit_tests/core/app/apps/test_base_app_generator.py @@ -287,7 +287,7 @@ def test_validate_inputs_optional_file_with_empty_string(): def test_validate_inputs_optional_file_list_with_empty_list(): - """Test that optional FILE_LIST variable with empty list returns None""" + """Test that optional FILE_LIST variable with empty list returns empty list (not None)""" base_app_generator = BaseAppGenerator() var_file_list = VariableEntity( @@ -302,6 +302,28 @@ def test_validate_inputs_optional_file_list_with_empty_list(): value=[], ) + # Empty list should be preserved, not converted to None + # This allows downstream components like document_extractor to handle empty lists properly + assert result == [] + + +def test_validate_inputs_optional_file_list_with_empty_string(): + """Test that optional FILE_LIST variable with empty string returns None""" + base_app_generator = BaseAppGenerator() + + var_file_list = VariableEntity( + variable="test_file_list", + label="test_file_list", + type=VariableEntityType.FILE_LIST, + required=False, + ) + + result = base_app_generator._validate_inputs( + variable_entity=var_file_list, + value="", + ) + + # Empty string should be treated as unset assert result is None diff --git a/api/tests/unit_tests/core/app/task_pipeline/test_easy_ui_based_generate_task_pipeline.py b/api/tests/unit_tests/core/app/task_pipeline/test_easy_ui_based_generate_task_pipeline.py new file mode 100644 index 0000000000..40f58c9ddf --- /dev/null +++ b/api/tests/unit_tests/core/app/task_pipeline/test_easy_ui_based_generate_task_pipeline.py @@ -0,0 +1,420 @@ +from types import SimpleNamespace +from unittest.mock import ANY, Mock, patch + +import pytest + +from core.app.apps.base_app_queue_manager import AppQueueManager +from core.app.entities.app_invoke_entities import ChatAppGenerateEntity +from core.app.entities.queue_entities import ( + QueueAgentMessageEvent, + QueueErrorEvent, + QueueLLMChunkEvent, + QueueMessageEndEvent, + QueueMessageFileEvent, + QueuePingEvent, +) +from core.app.entities.task_entities import ( + EasyUITaskState, + ErrorStreamResponse, + MessageEndStreamResponse, + MessageFileStreamResponse, + MessageReplaceStreamResponse, + MessageStreamResponse, + PingStreamResponse, + StreamEvent, +) +from core.app.task_pipeline.easy_ui_based_generate_task_pipeline import EasyUIBasedGenerateTaskPipeline +from core.base.tts import AppGeneratorTTSPublisher +from core.model_runtime.entities.llm_entities import LLMResult as RuntimeLLMResult +from core.model_runtime.entities.message_entities import TextPromptMessageContent +from core.ops.ops_trace_manager import TraceQueueManager +from models.model import AppMode + + +class TestEasyUIBasedGenerateTaskPipelineProcessStreamResponse: + """Test cases for EasyUIBasedGenerateTaskPipeline._process_stream_response method.""" + + @pytest.fixture + def mock_application_generate_entity(self): + """Create a mock application generate entity.""" + entity = Mock(spec=ChatAppGenerateEntity) + entity.task_id = "test-task-id" + entity.app_id = "test-app-id" + # minimal app_config used by pipeline internals + entity.app_config = SimpleNamespace( + tenant_id="test-tenant-id", + app_id="test-app-id", + app_mode=AppMode.CHAT, + app_model_config_dict={}, + additional_features=None, + sensitive_word_avoidance=None, + ) + # minimal model_conf for LLMResult init + entity.model_conf = SimpleNamespace( + model="test-model", + provider_model_bundle=SimpleNamespace(model_type_instance=Mock()), + credentials={}, + ) + return entity + + @pytest.fixture + def mock_queue_manager(self): + """Create a mock queue manager.""" + manager = Mock(spec=AppQueueManager) + return manager + + @pytest.fixture + def mock_message_cycle_manager(self): + """Create a mock message cycle manager.""" + manager = Mock() + manager.get_message_event_type.return_value = StreamEvent.MESSAGE + manager.message_to_stream_response.return_value = Mock(spec=MessageStreamResponse) + manager.message_file_to_stream_response.return_value = Mock(spec=MessageFileStreamResponse) + manager.message_replace_to_stream_response.return_value = Mock(spec=MessageReplaceStreamResponse) + manager.handle_retriever_resources = Mock() + manager.handle_annotation_reply.return_value = None + return manager + + @pytest.fixture + def mock_conversation(self): + """Create a mock conversation.""" + conversation = Mock() + conversation.id = "test-conversation-id" + conversation.mode = "chat" + return conversation + + @pytest.fixture + def mock_message(self): + """Create a mock message.""" + message = Mock() + message.id = "test-message-id" + message.created_at = Mock() + message.created_at.timestamp.return_value = 1234567890 + return message + + @pytest.fixture + def mock_task_state(self): + """Create a mock task state.""" + task_state = Mock(spec=EasyUITaskState) + + # Create LLM result mock + llm_result = Mock(spec=RuntimeLLMResult) + llm_result.prompt_messages = [] + llm_result.message = Mock() + llm_result.message.content = "" + + task_state.llm_result = llm_result + task_state.answer = "" + + return task_state + + @pytest.fixture + def pipeline( + self, + mock_application_generate_entity, + mock_queue_manager, + mock_conversation, + mock_message, + mock_message_cycle_manager, + mock_task_state, + ): + """Create an EasyUIBasedGenerateTaskPipeline instance with mocked dependencies.""" + with patch( + "core.app.task_pipeline.easy_ui_based_generate_task_pipeline.EasyUITaskState", return_value=mock_task_state + ): + pipeline = EasyUIBasedGenerateTaskPipeline( + application_generate_entity=mock_application_generate_entity, + queue_manager=mock_queue_manager, + conversation=mock_conversation, + message=mock_message, + stream=True, + ) + pipeline._message_cycle_manager = mock_message_cycle_manager + pipeline._task_state = mock_task_state + return pipeline + + def test_get_message_event_type_called_once_when_first_llm_chunk_arrives( + self, pipeline, mock_message_cycle_manager + ): + """Expect get_message_event_type to be called when processing the first LLM chunk event.""" + # Setup a minimal LLM chunk event + chunk = Mock() + chunk.delta.message.content = "hi" + chunk.prompt_messages = [] + llm_chunk_event = Mock(spec=QueueLLMChunkEvent) + llm_chunk_event.chunk = chunk + mock_queue_message = Mock() + mock_queue_message.event = llm_chunk_event + pipeline.queue_manager.listen.return_value = [mock_queue_message] + + # Execute + list(pipeline._process_stream_response(publisher=None, trace_manager=None)) + + # Assert + mock_message_cycle_manager.get_message_event_type.assert_called_once_with(message_id="test-message-id") + + def test_llm_chunk_event_with_text_content(self, pipeline, mock_message_cycle_manager, mock_task_state): + """Test handling of LLM chunk events with text content.""" + # Setup + chunk = Mock() + chunk.delta.message.content = "Hello, world!" + chunk.prompt_messages = [] + + llm_chunk_event = Mock(spec=QueueLLMChunkEvent) + llm_chunk_event.chunk = chunk + + mock_queue_message = Mock() + mock_queue_message.event = llm_chunk_event + pipeline.queue_manager.listen.return_value = [mock_queue_message] + + mock_message_cycle_manager.get_message_event_type.return_value = StreamEvent.MESSAGE + + # Execute + responses = list(pipeline._process_stream_response(publisher=None, trace_manager=None)) + + # Assert + assert len(responses) == 1 + mock_message_cycle_manager.message_to_stream_response.assert_called_once_with( + answer="Hello, world!", message_id="test-message-id", event_type=StreamEvent.MESSAGE + ) + assert mock_task_state.llm_result.message.content == "Hello, world!" + + def test_llm_chunk_event_with_list_content(self, pipeline, mock_message_cycle_manager, mock_task_state): + """Test handling of LLM chunk events with list content.""" + # Setup + text_content = Mock(spec=TextPromptMessageContent) + text_content.data = "Hello" + + chunk = Mock() + chunk.delta.message.content = [text_content, " world!"] + chunk.prompt_messages = [] + + llm_chunk_event = Mock(spec=QueueLLMChunkEvent) + llm_chunk_event.chunk = chunk + + mock_queue_message = Mock() + mock_queue_message.event = llm_chunk_event + pipeline.queue_manager.listen.return_value = [mock_queue_message] + + mock_message_cycle_manager.get_message_event_type.return_value = StreamEvent.MESSAGE + + # Execute + responses = list(pipeline._process_stream_response(publisher=None, trace_manager=None)) + + # Assert + assert len(responses) == 1 + mock_message_cycle_manager.message_to_stream_response.assert_called_once_with( + answer="Hello world!", message_id="test-message-id", event_type=StreamEvent.MESSAGE + ) + assert mock_task_state.llm_result.message.content == "Hello world!" + + def test_agent_message_event(self, pipeline, mock_message_cycle_manager, mock_task_state): + """Test handling of agent message events.""" + # Setup + chunk = Mock() + chunk.delta.message.content = "Agent response" + + agent_message_event = Mock(spec=QueueAgentMessageEvent) + agent_message_event.chunk = chunk + + mock_queue_message = Mock() + mock_queue_message.event = agent_message_event + pipeline.queue_manager.listen.return_value = [mock_queue_message] + + # Ensure method under assertion is a mock to track calls + pipeline._agent_message_to_stream_response = Mock(return_value=Mock()) + + # Execute + responses = list(pipeline._process_stream_response(publisher=None, trace_manager=None)) + + # Assert + assert len(responses) == 1 + # Agent messages should use _agent_message_to_stream_response + pipeline._agent_message_to_stream_response.assert_called_once_with( + answer="Agent response", message_id="test-message-id" + ) + + def test_message_end_event(self, pipeline, mock_message_cycle_manager, mock_task_state): + """Test handling of message end events.""" + # Setup + llm_result = Mock(spec=RuntimeLLMResult) + llm_result.message = Mock() + llm_result.message.content = "Final response" + + message_end_event = Mock(spec=QueueMessageEndEvent) + message_end_event.llm_result = llm_result + + mock_queue_message = Mock() + mock_queue_message.event = message_end_event + pipeline.queue_manager.listen.return_value = [mock_queue_message] + + pipeline._save_message = Mock() + pipeline._message_end_to_stream_response = Mock(return_value=Mock(spec=MessageEndStreamResponse)) + + # Patch db.engine used inside pipeline for session creation + with patch( + "core.app.task_pipeline.easy_ui_based_generate_task_pipeline.db", new=SimpleNamespace(engine=Mock()) + ): + # Execute + responses = list(pipeline._process_stream_response(publisher=None, trace_manager=None)) + + # Assert + assert len(responses) == 1 + assert mock_task_state.llm_result == llm_result + pipeline._save_message.assert_called_once() + pipeline._message_end_to_stream_response.assert_called_once() + + def test_error_event(self, pipeline): + """Test handling of error events.""" + # Setup + error_event = Mock(spec=QueueErrorEvent) + error_event.error = Exception("Test error") + + mock_queue_message = Mock() + mock_queue_message.event = error_event + pipeline.queue_manager.listen.return_value = [mock_queue_message] + + pipeline.handle_error = Mock(return_value=Exception("Test error")) + pipeline.error_to_stream_response = Mock(return_value=Mock(spec=ErrorStreamResponse)) + + # Patch db.engine used inside pipeline for session creation + with patch( + "core.app.task_pipeline.easy_ui_based_generate_task_pipeline.db", new=SimpleNamespace(engine=Mock()) + ): + # Execute + responses = list(pipeline._process_stream_response(publisher=None, trace_manager=None)) + + # Assert + assert len(responses) == 1 + pipeline.handle_error.assert_called_once() + pipeline.error_to_stream_response.assert_called_once() + + def test_ping_event(self, pipeline): + """Test handling of ping events.""" + # Setup + ping_event = Mock(spec=QueuePingEvent) + + mock_queue_message = Mock() + mock_queue_message.event = ping_event + pipeline.queue_manager.listen.return_value = [mock_queue_message] + + pipeline.ping_stream_response = Mock(return_value=Mock(spec=PingStreamResponse)) + + # Execute + responses = list(pipeline._process_stream_response(publisher=None, trace_manager=None)) + + # Assert + assert len(responses) == 1 + pipeline.ping_stream_response.assert_called_once() + + def test_file_event(self, pipeline, mock_message_cycle_manager): + """Test handling of file events.""" + # Setup + file_event = Mock(spec=QueueMessageFileEvent) + file_event.message_file_id = "file-id" + + mock_queue_message = Mock() + mock_queue_message.event = file_event + pipeline.queue_manager.listen.return_value = [mock_queue_message] + + file_response = Mock(spec=MessageFileStreamResponse) + mock_message_cycle_manager.message_file_to_stream_response.return_value = file_response + + # Execute + responses = list(pipeline._process_stream_response(publisher=None, trace_manager=None)) + + # Assert + assert len(responses) == 1 + assert responses[0] == file_response + mock_message_cycle_manager.message_file_to_stream_response.assert_called_once_with(file_event) + + def test_publisher_is_called_with_messages(self, pipeline): + """Test that publisher publishes messages when provided.""" + # Setup + publisher = Mock(spec=AppGeneratorTTSPublisher) + + ping_event = Mock(spec=QueuePingEvent) + mock_queue_message = Mock() + mock_queue_message.event = ping_event + pipeline.queue_manager.listen.return_value = [mock_queue_message] + + pipeline.ping_stream_response = Mock(return_value=Mock(spec=PingStreamResponse)) + + # Execute + list(pipeline._process_stream_response(publisher=publisher, trace_manager=None)) + + # Assert + # Called once with message and once with None at the end + assert publisher.publish.call_count == 2 + publisher.publish.assert_any_call(mock_queue_message) + publisher.publish.assert_any_call(None) + + def test_trace_manager_passed_to_save_message(self, pipeline): + """Test that trace manager is passed to _save_message.""" + # Setup + trace_manager = Mock(spec=TraceQueueManager) + + message_end_event = Mock(spec=QueueMessageEndEvent) + message_end_event.llm_result = None + + mock_queue_message = Mock() + mock_queue_message.event = message_end_event + pipeline.queue_manager.listen.return_value = [mock_queue_message] + + pipeline._save_message = Mock() + pipeline._message_end_to_stream_response = Mock(return_value=Mock(spec=MessageEndStreamResponse)) + + # Patch db.engine used inside pipeline for session creation + with patch( + "core.app.task_pipeline.easy_ui_based_generate_task_pipeline.db", new=SimpleNamespace(engine=Mock()) + ): + # Execute + list(pipeline._process_stream_response(publisher=None, trace_manager=trace_manager)) + + # Assert + pipeline._save_message.assert_called_once_with(session=ANY, trace_manager=trace_manager) + + def test_multiple_events_sequence(self, pipeline, mock_message_cycle_manager, mock_task_state): + """Test handling multiple events in sequence.""" + # Setup + chunk1 = Mock() + chunk1.delta.message.content = "Hello" + chunk1.prompt_messages = [] + + chunk2 = Mock() + chunk2.delta.message.content = " world!" + chunk2.prompt_messages = [] + + llm_chunk_event1 = Mock(spec=QueueLLMChunkEvent) + llm_chunk_event1.chunk = chunk1 + + ping_event = Mock(spec=QueuePingEvent) + + llm_chunk_event2 = Mock(spec=QueueLLMChunkEvent) + llm_chunk_event2.chunk = chunk2 + + mock_queue_messages = [ + Mock(event=llm_chunk_event1), + Mock(event=ping_event), + Mock(event=llm_chunk_event2), + ] + pipeline.queue_manager.listen.return_value = mock_queue_messages + + mock_message_cycle_manager.get_message_event_type.return_value = StreamEvent.MESSAGE + pipeline.ping_stream_response = Mock(return_value=Mock(spec=PingStreamResponse)) + + # Execute + responses = list(pipeline._process_stream_response(publisher=None, trace_manager=None)) + + # Assert + assert len(responses) == 3 + assert mock_task_state.llm_result.message.content == "Hello world!" + + # Verify calls to message_to_stream_response + assert mock_message_cycle_manager.message_to_stream_response.call_count == 2 + mock_message_cycle_manager.message_to_stream_response.assert_any_call( + answer="Hello", message_id="test-message-id", event_type=StreamEvent.MESSAGE + ) + mock_message_cycle_manager.message_to_stream_response.assert_any_call( + answer=" world!", message_id="test-message-id", event_type=StreamEvent.MESSAGE + ) diff --git a/api/tests/unit_tests/core/app/task_pipeline/test_message_cycle_manager_optimization.py b/api/tests/unit_tests/core/app/task_pipeline/test_message_cycle_manager_optimization.py new file mode 100644 index 0000000000..5ef7f0d7f4 --- /dev/null +++ b/api/tests/unit_tests/core/app/task_pipeline/test_message_cycle_manager_optimization.py @@ -0,0 +1,166 @@ +"""Unit tests for the message cycle manager optimization.""" + +from types import SimpleNamespace +from unittest.mock import ANY, Mock, patch + +import pytest +from flask import current_app + +from core.app.entities.task_entities import MessageStreamResponse, StreamEvent +from core.app.task_pipeline.message_cycle_manager import MessageCycleManager + + +class TestMessageCycleManagerOptimization: + """Test cases for the message cycle manager optimization that prevents N+1 queries.""" + + @pytest.fixture + def mock_application_generate_entity(self): + """Create a mock application generate entity.""" + entity = Mock() + entity.task_id = "test-task-id" + return entity + + @pytest.fixture + def message_cycle_manager(self, mock_application_generate_entity): + """Create a message cycle manager instance.""" + task_state = Mock() + return MessageCycleManager(application_generate_entity=mock_application_generate_entity, task_state=task_state) + + def test_get_message_event_type_with_message_file(self, message_cycle_manager): + """Test get_message_event_type returns MESSAGE_FILE when message has files.""" + with ( + patch("core.app.task_pipeline.message_cycle_manager.Session") as mock_session_class, + patch("core.app.task_pipeline.message_cycle_manager.db", new=SimpleNamespace(engine=Mock())), + ): + # Setup mock session and message file + mock_session = Mock() + mock_session_class.return_value.__enter__.return_value = mock_session + + mock_message_file = Mock() + # Current implementation uses session.query(...).scalar() + mock_session.query.return_value.scalar.return_value = mock_message_file + + # Execute + with current_app.app_context(): + result = message_cycle_manager.get_message_event_type("test-message-id") + + # Assert + assert result == StreamEvent.MESSAGE_FILE + mock_session.query.return_value.scalar.assert_called_once() + + def test_get_message_event_type_without_message_file(self, message_cycle_manager): + """Test get_message_event_type returns MESSAGE when message has no files.""" + with ( + patch("core.app.task_pipeline.message_cycle_manager.Session") as mock_session_class, + patch("core.app.task_pipeline.message_cycle_manager.db", new=SimpleNamespace(engine=Mock())), + ): + # Setup mock session and no message file + mock_session = Mock() + mock_session_class.return_value.__enter__.return_value = mock_session + # Current implementation uses session.query(...).scalar() + mock_session.query.return_value.scalar.return_value = None + + # Execute + with current_app.app_context(): + result = message_cycle_manager.get_message_event_type("test-message-id") + + # Assert + assert result == StreamEvent.MESSAGE + mock_session.query.return_value.scalar.assert_called_once() + + def test_message_to_stream_response_with_precomputed_event_type(self, message_cycle_manager): + """MessageCycleManager.message_to_stream_response expects a valid event_type; callers should precompute it.""" + with ( + patch("core.app.task_pipeline.message_cycle_manager.Session") as mock_session_class, + patch("core.app.task_pipeline.message_cycle_manager.db", new=SimpleNamespace(engine=Mock())), + ): + # Setup mock session and message file + mock_session = Mock() + mock_session_class.return_value.__enter__.return_value = mock_session + + mock_message_file = Mock() + # Current implementation uses session.query(...).scalar() + mock_session.query.return_value.scalar.return_value = mock_message_file + + # Execute: compute event type once, then pass to message_to_stream_response + with current_app.app_context(): + event_type = message_cycle_manager.get_message_event_type("test-message-id") + result = message_cycle_manager.message_to_stream_response( + answer="Hello world", message_id="test-message-id", event_type=event_type + ) + + # Assert + assert isinstance(result, MessageStreamResponse) + assert result.answer == "Hello world" + assert result.id == "test-message-id" + assert result.event == StreamEvent.MESSAGE_FILE + mock_session.query.return_value.scalar.assert_called_once() + + def test_message_to_stream_response_with_event_type_skips_query(self, message_cycle_manager): + """Test that message_to_stream_response skips database query when event_type is provided.""" + with patch("core.app.task_pipeline.message_cycle_manager.Session") as mock_session_class: + # Execute with event_type provided + result = message_cycle_manager.message_to_stream_response( + answer="Hello world", message_id="test-message-id", event_type=StreamEvent.MESSAGE + ) + + # Assert + assert isinstance(result, MessageStreamResponse) + assert result.answer == "Hello world" + assert result.id == "test-message-id" + assert result.event == StreamEvent.MESSAGE + # Should not query database when event_type is provided + mock_session_class.assert_not_called() + + def test_message_to_stream_response_with_from_variable_selector(self, message_cycle_manager): + """Test message_to_stream_response with from_variable_selector parameter.""" + result = message_cycle_manager.message_to_stream_response( + answer="Hello world", + message_id="test-message-id", + from_variable_selector=["var1", "var2"], + event_type=StreamEvent.MESSAGE, + ) + + assert isinstance(result, MessageStreamResponse) + assert result.answer == "Hello world" + assert result.id == "test-message-id" + assert result.from_variable_selector == ["var1", "var2"] + assert result.event == StreamEvent.MESSAGE + + def test_optimization_usage_example(self, message_cycle_manager): + """Test the optimization pattern that should be used by callers.""" + # Step 1: Get event type once (this queries database) + with ( + patch("core.app.task_pipeline.message_cycle_manager.Session") as mock_session_class, + patch("core.app.task_pipeline.message_cycle_manager.db", new=SimpleNamespace(engine=Mock())), + ): + mock_session = Mock() + mock_session_class.return_value.__enter__.return_value = mock_session + # Current implementation uses session.query(...).scalar() + mock_session.query.return_value.scalar.return_value = None # No files + with current_app.app_context(): + event_type = message_cycle_manager.get_message_event_type("test-message-id") + + # Should query database once + mock_session_class.assert_called_once_with(ANY, expire_on_commit=False) + assert event_type == StreamEvent.MESSAGE + + # Step 2: Use event_type for multiple calls (no additional queries) + with patch("core.app.task_pipeline.message_cycle_manager.Session") as mock_session_class: + mock_session_class.return_value.__enter__.return_value = Mock() + + chunk1_response = message_cycle_manager.message_to_stream_response( + answer="Chunk 1", message_id="test-message-id", event_type=event_type + ) + + chunk2_response = message_cycle_manager.message_to_stream_response( + answer="Chunk 2", message_id="test-message-id", event_type=event_type + ) + + # Should not query database again + mock_session_class.assert_not_called() + + assert chunk1_response.event == StreamEvent.MESSAGE + assert chunk2_response.event == StreamEvent.MESSAGE + assert chunk1_response.answer == "Chunk 1" + assert chunk2_response.answer == "Chunk 2" diff --git a/api/tests/unit_tests/core/datasource/test_notion_provider.py b/api/tests/unit_tests/core/datasource/test_notion_provider.py index 9e7255bc3f..e4bd7d3bdf 100644 --- a/api/tests/unit_tests/core/datasource/test_notion_provider.py +++ b/api/tests/unit_tests/core/datasource/test_notion_provider.py @@ -96,7 +96,7 @@ class TestNotionExtractorAuthentication: def test_init_with_integration_token_fallback(self, mock_get_token, mock_config, mock_document_model): """Test NotionExtractor falls back to integration token when credential not found.""" # Arrange - mock_get_token.return_value = None + mock_get_token.side_effect = Exception("No credential id found") mock_config.NOTION_INTEGRATION_TOKEN = "integration-token-fallback" # Act @@ -105,7 +105,7 @@ class TestNotionExtractorAuthentication: notion_obj_id="page-456", notion_page_type="page", tenant_id="tenant-789", - credential_id="cred-123", + credential_id=None, document_model=mock_document_model, ) @@ -117,7 +117,7 @@ class TestNotionExtractorAuthentication: def test_init_missing_credentials_raises_error(self, mock_get_token, mock_config, mock_document_model): """Test NotionExtractor raises error when no credentials available.""" # Arrange - mock_get_token.return_value = None + mock_get_token.side_effect = Exception("No credential id found") mock_config.NOTION_INTEGRATION_TOKEN = None # Act & Assert @@ -127,7 +127,7 @@ class TestNotionExtractorAuthentication: notion_obj_id="page-456", notion_page_type="page", tenant_id="tenant-789", - credential_id="cred-123", + credential_id=None, document_model=mock_document_model, ) assert "Must specify `integration_token`" in str(exc_info.value) diff --git a/api/tests/unit_tests/core/helper/test_ssrf_proxy.py b/api/tests/unit_tests/core/helper/test_ssrf_proxy.py index 37749f0c66..beae1d0358 100644 --- a/api/tests/unit_tests/core/helper/test_ssrf_proxy.py +++ b/api/tests/unit_tests/core/helper/test_ssrf_proxy.py @@ -1,52 +1,109 @@ -import secrets from unittest.mock import MagicMock, patch import pytest -from core.helper.ssrf_proxy import SSRF_DEFAULT_MAX_RETRIES, STATUS_FORCELIST, make_request +from core.helper.ssrf_proxy import ( + SSRF_DEFAULT_MAX_RETRIES, + _get_user_provided_host_header, + make_request, +) -@patch("httpx.Client.request") -def test_successful_request(mock_request): +@patch("core.helper.ssrf_proxy._get_ssrf_client") +def test_successful_request(mock_get_client): + mock_client = MagicMock() mock_response = MagicMock() mock_response.status_code = 200 - mock_request.return_value = mock_response + mock_client.send.return_value = mock_response + mock_client.request.return_value = mock_response + mock_get_client.return_value = mock_client response = make_request("GET", "http://example.com") assert response.status_code == 200 -@patch("httpx.Client.request") -def test_retry_exceed_max_retries(mock_request): +@patch("core.helper.ssrf_proxy._get_ssrf_client") +def test_retry_exceed_max_retries(mock_get_client): + mock_client = MagicMock() mock_response = MagicMock() mock_response.status_code = 500 - - side_effects = [mock_response] * SSRF_DEFAULT_MAX_RETRIES - mock_request.side_effect = side_effects + mock_client.send.return_value = mock_response + mock_client.request.return_value = mock_response + mock_get_client.return_value = mock_client with pytest.raises(Exception) as e: make_request("GET", "http://example.com", max_retries=SSRF_DEFAULT_MAX_RETRIES - 1) assert str(e.value) == f"Reached maximum retries ({SSRF_DEFAULT_MAX_RETRIES - 1}) for URL http://example.com" -@patch("httpx.Client.request") -def test_retry_logic_success(mock_request): - side_effects = [] +class TestGetUserProvidedHostHeader: + """Tests for _get_user_provided_host_header function.""" - for _ in range(SSRF_DEFAULT_MAX_RETRIES): - status_code = secrets.choice(STATUS_FORCELIST) - mock_response = MagicMock() - mock_response.status_code = status_code - side_effects.append(mock_response) + def test_returns_none_when_headers_is_none(self): + assert _get_user_provided_host_header(None) is None - mock_response_200 = MagicMock() - mock_response_200.status_code = 200 - side_effects.append(mock_response_200) + def test_returns_none_when_headers_is_empty(self): + assert _get_user_provided_host_header({}) is None - mock_request.side_effect = side_effects + def test_returns_none_when_host_header_not_present(self): + headers = {"Content-Type": "application/json", "Authorization": "Bearer token"} + assert _get_user_provided_host_header(headers) is None - response = make_request("GET", "http://example.com", max_retries=SSRF_DEFAULT_MAX_RETRIES) + def test_returns_host_header_lowercase(self): + headers = {"host": "example.com"} + assert _get_user_provided_host_header(headers) == "example.com" + + def test_returns_host_header_uppercase(self): + headers = {"HOST": "example.com"} + assert _get_user_provided_host_header(headers) == "example.com" + + def test_returns_host_header_mixed_case(self): + headers = {"HoSt": "example.com"} + assert _get_user_provided_host_header(headers) == "example.com" + + def test_returns_host_header_from_multiple_headers(self): + headers = {"Content-Type": "application/json", "Host": "api.example.com", "Authorization": "Bearer token"} + assert _get_user_provided_host_header(headers) == "api.example.com" + + def test_returns_first_host_header_when_duplicates(self): + headers = {"host": "first.com", "Host": "second.com"} + # Should return the first one encountered (iteration order is preserved in dict) + result = _get_user_provided_host_header(headers) + assert result in ("first.com", "second.com") + + +@patch("core.helper.ssrf_proxy._get_ssrf_client") +def test_host_header_preservation_without_user_header(mock_get_client): + """Test that when no Host header is provided, the default behavior is maintained.""" + mock_client = MagicMock() + mock_request = MagicMock() + mock_request.headers = {} + mock_response = MagicMock() + mock_response.status_code = 200 + mock_client.send.return_value = mock_response + mock_client.request.return_value = mock_response + mock_get_client.return_value = mock_client + + response = make_request("GET", "http://example.com") + + assert response.status_code == 200 + # Host should not be set if not provided by user + assert "Host" not in mock_request.headers or mock_request.headers.get("Host") is None + + +@patch("core.helper.ssrf_proxy._get_ssrf_client") +def test_host_header_preservation_with_user_header(mock_get_client): + """Test that user-provided Host header is preserved in the request.""" + mock_client = MagicMock() + mock_request = MagicMock() + mock_request.headers = {} + mock_response = MagicMock() + mock_response.status_code = 200 + mock_client.send.return_value = mock_response + mock_client.request.return_value = mock_response + mock_get_client.return_value = mock_client + + custom_host = "custom.example.com:8080" + response = make_request("GET", "http://example.com", headers={"Host": custom_host}) assert response.status_code == 200 - assert mock_request.call_count == SSRF_DEFAULT_MAX_RETRIES + 1 - assert mock_request.call_args_list[0][1].get("method") == "GET" diff --git a/api/tests/unit_tests/core/helper/test_tool_provider_cache.py b/api/tests/unit_tests/core/helper/test_tool_provider_cache.py deleted file mode 100644 index 00f7c9d7e9..0000000000 --- a/api/tests/unit_tests/core/helper/test_tool_provider_cache.py +++ /dev/null @@ -1,129 +0,0 @@ -import json -from unittest.mock import patch - -import pytest -from redis.exceptions import RedisError - -from core.helper.tool_provider_cache import ToolProviderListCache -from core.tools.entities.api_entities import ToolProviderTypeApiLiteral - - -@pytest.fixture -def mock_redis_client(): - """Fixture: Mock Redis client""" - with patch("core.helper.tool_provider_cache.redis_client") as mock: - yield mock - - -class TestToolProviderListCache: - """Test class for ToolProviderListCache""" - - def test_generate_cache_key(self): - """Test cache key generation logic""" - # Scenario 1: Specify typ (valid literal value) - tenant_id = "tenant_123" - typ: ToolProviderTypeApiLiteral = "builtin" - expected_key = f"tool_providers:tenant_id:{tenant_id}:type:{typ}" - assert ToolProviderListCache._generate_cache_key(tenant_id, typ) == expected_key - - # Scenario 2: typ is None (defaults to "all") - expected_key_all = f"tool_providers:tenant_id:{tenant_id}:type:all" - assert ToolProviderListCache._generate_cache_key(tenant_id) == expected_key_all - - def test_get_cached_providers_hit(self, mock_redis_client): - """Test get cached providers - cache hit and successful decoding""" - tenant_id = "tenant_123" - typ: ToolProviderTypeApiLiteral = "api" - mock_providers = [{"id": "tool", "name": "test_provider"}] - mock_redis_client.get.return_value = json.dumps(mock_providers).encode("utf-8") - - result = ToolProviderListCache.get_cached_providers(tenant_id, typ) - - mock_redis_client.get.assert_called_once_with(ToolProviderListCache._generate_cache_key(tenant_id, typ)) - assert result == mock_providers - - def test_get_cached_providers_decode_error(self, mock_redis_client): - """Test get cached providers - cache hit but decoding failed""" - tenant_id = "tenant_123" - mock_redis_client.get.return_value = b"invalid_json_data" - - result = ToolProviderListCache.get_cached_providers(tenant_id) - - assert result is None - mock_redis_client.get.assert_called_once() - - def test_get_cached_providers_miss(self, mock_redis_client): - """Test get cached providers - cache miss""" - tenant_id = "tenant_123" - mock_redis_client.get.return_value = None - - result = ToolProviderListCache.get_cached_providers(tenant_id) - - assert result is None - mock_redis_client.get.assert_called_once() - - def test_set_cached_providers(self, mock_redis_client): - """Test set cached providers""" - tenant_id = "tenant_123" - typ: ToolProviderTypeApiLiteral = "builtin" - mock_providers = [{"id": "tool", "name": "test_provider"}] - cache_key = ToolProviderListCache._generate_cache_key(tenant_id, typ) - - ToolProviderListCache.set_cached_providers(tenant_id, typ, mock_providers) - - mock_redis_client.setex.assert_called_once_with( - cache_key, ToolProviderListCache.CACHE_TTL, json.dumps(mock_providers) - ) - - def test_invalidate_cache_specific_type(self, mock_redis_client): - """Test invalidate cache - specific type""" - tenant_id = "tenant_123" - typ: ToolProviderTypeApiLiteral = "workflow" - cache_key = ToolProviderListCache._generate_cache_key(tenant_id, typ) - - ToolProviderListCache.invalidate_cache(tenant_id, typ) - - mock_redis_client.delete.assert_called_once_with(cache_key) - - def test_invalidate_cache_all_types(self, mock_redis_client): - """Test invalidate cache - clear all tenant cache""" - tenant_id = "tenant_123" - mock_keys = [ - b"tool_providers:tenant_id:tenant_123:type:all", - b"tool_providers:tenant_id:tenant_123:type:builtin", - ] - mock_redis_client.scan_iter.return_value = mock_keys - - ToolProviderListCache.invalidate_cache(tenant_id) - - mock_redis_client.scan_iter.assert_called_once_with(f"tool_providers:tenant_id:{tenant_id}:*") - mock_redis_client.delete.assert_called_once_with(*mock_keys) - - def test_invalidate_cache_no_keys(self, mock_redis_client): - """Test invalidate cache - no cache keys for tenant""" - tenant_id = "tenant_123" - mock_redis_client.scan_iter.return_value = [] - - ToolProviderListCache.invalidate_cache(tenant_id) - - mock_redis_client.delete.assert_not_called() - - def test_redis_fallback_default_return(self, mock_redis_client): - """Test redis_fallback decorator - default return value (Redis error)""" - mock_redis_client.get.side_effect = RedisError("Redis connection error") - - result = ToolProviderListCache.get_cached_providers("tenant_123") - - assert result is None - mock_redis_client.get.assert_called_once() - - def test_redis_fallback_no_default(self, mock_redis_client): - """Test redis_fallback decorator - no default return value (Redis error)""" - mock_redis_client.setex.side_effect = RedisError("Redis connection error") - - try: - ToolProviderListCache.set_cached_providers("tenant_123", "mcp", []) - except RedisError: - pytest.fail("set_cached_providers should not raise RedisError (handled by fallback)") - - mock_redis_client.setex.assert_called_once() diff --git a/api/tests/unit_tests/core/ops/test_utils.py b/api/tests/unit_tests/core/ops/test_utils.py index 7cc2772acf..e1084001b7 100644 --- a/api/tests/unit_tests/core/ops/test_utils.py +++ b/api/tests/unit_tests/core/ops/test_utils.py @@ -1,6 +1,9 @@ +import re +from datetime import datetime + import pytest -from core.ops.utils import validate_project_name, validate_url, validate_url_with_path +from core.ops.utils import generate_dotted_order, validate_project_name, validate_url, validate_url_with_path class TestValidateUrl: @@ -136,3 +139,51 @@ class TestValidateProjectName: """Test custom default name""" result = validate_project_name("", "Custom Default") assert result == "Custom Default" + + +class TestGenerateDottedOrder: + """Test cases for generate_dotted_order function""" + + def test_dotted_order_has_6_digit_microseconds(self): + """Test that timestamp includes full 6-digit microseconds for LangSmith API compatibility. + + LangSmith API expects timestamps in format: YYYYMMDDTHHMMSSffffffZ (6-digit microseconds). + Previously, the code truncated to 3 digits which caused API errors: + 'cannot parse .111 as .000000' + """ + start_time = datetime(2025, 12, 23, 4, 19, 55, 111000) + run_id = "test-run-id" + result = generate_dotted_order(run_id, start_time) + + # Extract timestamp portion (before the run_id) + timestamp_match = re.match(r"^(\d{8}T\d{6})(\d+)Z", result) + assert timestamp_match is not None, "Timestamp format should match YYYYMMDDTHHMMSSffffffZ" + + microseconds = timestamp_match.group(2) + assert len(microseconds) == 6, f"Microseconds should be 6 digits, got {len(microseconds)}: {microseconds}" + + def test_dotted_order_format_matches_langsmith_expected(self): + """Test that dotted_order format matches LangSmith API expected format.""" + start_time = datetime(2025, 1, 15, 10, 30, 45, 123456) + run_id = "abc123" + result = generate_dotted_order(run_id, start_time) + + # LangSmith expects: YYYYMMDDTHHMMSSffffffZ followed by run_id + assert result == "20250115T103045123456Zabc123" + + def test_dotted_order_with_parent(self): + """Test dotted_order generation with parent order uses dot separator.""" + start_time = datetime(2025, 12, 23, 4, 19, 55, 111000) + run_id = "child-run-id" + parent_order = "20251223T041955000000Zparent-run-id" + result = generate_dotted_order(run_id, start_time, parent_order) + + assert result == "20251223T041955000000Zparent-run-id.20251223T041955111000Zchild-run-id" + + def test_dotted_order_without_parent_has_no_dot(self): + """Test dotted_order generation without parent has no dot separator.""" + start_time = datetime(2025, 12, 23, 4, 19, 55, 111000) + run_id = "test-run-id" + result = generate_dotted_order(run_id, start_time, None) + + assert "." not in result diff --git a/api/tests/unit_tests/core/rag/datasource/vdb/pgvector/__init__.py b/api/tests/unit_tests/core/rag/datasource/vdb/pgvector/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/core/rag/datasource/vdb/pgvector/test_pgvector.py b/api/tests/unit_tests/core/rag/datasource/vdb/pgvector/test_pgvector.py new file mode 100644 index 0000000000..4998a9858f --- /dev/null +++ b/api/tests/unit_tests/core/rag/datasource/vdb/pgvector/test_pgvector.py @@ -0,0 +1,327 @@ +import unittest +from unittest.mock import MagicMock, patch + +import pytest + +from core.rag.datasource.vdb.pgvector.pgvector import ( + PGVector, + PGVectorConfig, +) + + +class TestPGVector(unittest.TestCase): + def setUp(self): + self.config = PGVectorConfig( + host="localhost", + port=5432, + user="test_user", + password="test_password", + database="test_db", + min_connection=1, + max_connection=5, + pg_bigm=False, + ) + self.collection_name = "test_collection" + + @patch("core.rag.datasource.vdb.pgvector.pgvector.psycopg2.pool.SimpleConnectionPool") + def test_init(self, mock_pool_class): + """Test PGVector initialization.""" + mock_pool = MagicMock() + mock_pool_class.return_value = mock_pool + + pgvector = PGVector(self.collection_name, self.config) + + assert pgvector._collection_name == self.collection_name + assert pgvector.table_name == f"embedding_{self.collection_name}" + assert pgvector.get_type() == "pgvector" + assert pgvector.pool is not None + assert pgvector.pg_bigm is False + assert pgvector.index_hash is not None + + @patch("core.rag.datasource.vdb.pgvector.pgvector.psycopg2.pool.SimpleConnectionPool") + def test_init_with_pg_bigm(self, mock_pool_class): + """Test PGVector initialization with pg_bigm enabled.""" + config = PGVectorConfig( + host="localhost", + port=5432, + user="test_user", + password="test_password", + database="test_db", + min_connection=1, + max_connection=5, + pg_bigm=True, + ) + mock_pool = MagicMock() + mock_pool_class.return_value = mock_pool + + pgvector = PGVector(self.collection_name, config) + + assert pgvector.pg_bigm is True + + @patch("core.rag.datasource.vdb.pgvector.pgvector.psycopg2.pool.SimpleConnectionPool") + @patch("core.rag.datasource.vdb.pgvector.pgvector.redis_client") + def test_create_collection_basic(self, mock_redis, mock_pool_class): + """Test basic collection creation.""" + # Mock Redis operations + mock_lock = MagicMock() + mock_lock.__enter__ = MagicMock() + mock_lock.__exit__ = MagicMock() + mock_redis.lock.return_value = mock_lock + mock_redis.get.return_value = None + mock_redis.set.return_value = None + + # Mock the connection pool + mock_pool = MagicMock() + mock_pool_class.return_value = mock_pool + + # Mock connection and cursor + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_pool.getconn.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + mock_cursor.fetchone.return_value = [1] # vector extension exists + + pgvector = PGVector(self.collection_name, self.config) + pgvector._create_collection(1536) + + # Verify SQL execution calls + assert mock_cursor.execute.called + + # Check that CREATE TABLE was called with correct dimension + create_table_calls = [call for call in mock_cursor.execute.call_args_list if "CREATE TABLE" in str(call)] + assert len(create_table_calls) == 1 + assert "vector(1536)" in create_table_calls[0][0][0] + + # Check that CREATE INDEX was called (dimension <= 2000) + create_index_calls = [ + call for call in mock_cursor.execute.call_args_list if "CREATE INDEX" in str(call) and "hnsw" in str(call) + ] + assert len(create_index_calls) == 1 + + # Verify Redis cache was set + mock_redis.set.assert_called_once() + + @patch("core.rag.datasource.vdb.pgvector.pgvector.psycopg2.pool.SimpleConnectionPool") + @patch("core.rag.datasource.vdb.pgvector.pgvector.redis_client") + def test_create_collection_with_large_dimension(self, mock_redis, mock_pool_class): + """Test collection creation with dimension > 2000 (no HNSW index).""" + # Mock Redis operations + mock_lock = MagicMock() + mock_lock.__enter__ = MagicMock() + mock_lock.__exit__ = MagicMock() + mock_redis.lock.return_value = mock_lock + mock_redis.get.return_value = None + mock_redis.set.return_value = None + + # Mock the connection pool + mock_pool = MagicMock() + mock_pool_class.return_value = mock_pool + + # Mock connection and cursor + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_pool.getconn.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + mock_cursor.fetchone.return_value = [1] # vector extension exists + + pgvector = PGVector(self.collection_name, self.config) + pgvector._create_collection(3072) # Dimension > 2000 + + # Check that CREATE TABLE was called + create_table_calls = [call for call in mock_cursor.execute.call_args_list if "CREATE TABLE" in str(call)] + assert len(create_table_calls) == 1 + assert "vector(3072)" in create_table_calls[0][0][0] + + # Check that HNSW index was NOT created (dimension > 2000) + hnsw_index_calls = [call for call in mock_cursor.execute.call_args_list if "hnsw" in str(call)] + assert len(hnsw_index_calls) == 0 + + @patch("core.rag.datasource.vdb.pgvector.pgvector.psycopg2.pool.SimpleConnectionPool") + @patch("core.rag.datasource.vdb.pgvector.pgvector.redis_client") + def test_create_collection_with_pg_bigm(self, mock_redis, mock_pool_class): + """Test collection creation with pg_bigm enabled.""" + config = PGVectorConfig( + host="localhost", + port=5432, + user="test_user", + password="test_password", + database="test_db", + min_connection=1, + max_connection=5, + pg_bigm=True, + ) + + # Mock Redis operations + mock_lock = MagicMock() + mock_lock.__enter__ = MagicMock() + mock_lock.__exit__ = MagicMock() + mock_redis.lock.return_value = mock_lock + mock_redis.get.return_value = None + mock_redis.set.return_value = None + + # Mock the connection pool + mock_pool = MagicMock() + mock_pool_class.return_value = mock_pool + + # Mock connection and cursor + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_pool.getconn.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + mock_cursor.fetchone.return_value = [1] # vector extension exists + + pgvector = PGVector(self.collection_name, config) + pgvector._create_collection(1536) + + # Check that pg_bigm index was created + bigm_index_calls = [call for call in mock_cursor.execute.call_args_list if "gin_bigm_ops" in str(call)] + assert len(bigm_index_calls) == 1 + + @patch("core.rag.datasource.vdb.pgvector.pgvector.psycopg2.pool.SimpleConnectionPool") + @patch("core.rag.datasource.vdb.pgvector.pgvector.redis_client") + def test_create_collection_creates_vector_extension(self, mock_redis, mock_pool_class): + """Test that vector extension is created if it doesn't exist.""" + # Mock Redis operations + mock_lock = MagicMock() + mock_lock.__enter__ = MagicMock() + mock_lock.__exit__ = MagicMock() + mock_redis.lock.return_value = mock_lock + mock_redis.get.return_value = None + mock_redis.set.return_value = None + + # Mock the connection pool + mock_pool = MagicMock() + mock_pool_class.return_value = mock_pool + + # Mock connection and cursor + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_pool.getconn.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + # First call: vector extension doesn't exist + mock_cursor.fetchone.return_value = None + + pgvector = PGVector(self.collection_name, self.config) + pgvector._create_collection(1536) + + # Check that CREATE EXTENSION was called + create_extension_calls = [ + call for call in mock_cursor.execute.call_args_list if "CREATE EXTENSION vector" in str(call) + ] + assert len(create_extension_calls) == 1 + + @patch("core.rag.datasource.vdb.pgvector.pgvector.psycopg2.pool.SimpleConnectionPool") + @patch("core.rag.datasource.vdb.pgvector.pgvector.redis_client") + def test_create_collection_with_cache_hit(self, mock_redis, mock_pool_class): + """Test that collection creation is skipped when cache exists.""" + # Mock Redis operations - cache exists + mock_lock = MagicMock() + mock_lock.__enter__ = MagicMock() + mock_lock.__exit__ = MagicMock() + mock_redis.lock.return_value = mock_lock + mock_redis.get.return_value = 1 # Cache exists + + # Mock the connection pool + mock_pool = MagicMock() + mock_pool_class.return_value = mock_pool + + # Mock connection and cursor + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_pool.getconn.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + + pgvector = PGVector(self.collection_name, self.config) + pgvector._create_collection(1536) + + # Check that no SQL was executed (early return due to cache) + assert mock_cursor.execute.call_count == 0 + + @patch("core.rag.datasource.vdb.pgvector.pgvector.psycopg2.pool.SimpleConnectionPool") + @patch("core.rag.datasource.vdb.pgvector.pgvector.redis_client") + def test_create_collection_with_redis_lock(self, mock_redis, mock_pool_class): + """Test that Redis lock is used during collection creation.""" + # Mock Redis operations + mock_lock = MagicMock() + mock_lock.__enter__ = MagicMock() + mock_lock.__exit__ = MagicMock() + mock_redis.lock.return_value = mock_lock + mock_redis.get.return_value = None + mock_redis.set.return_value = None + + # Mock the connection pool + mock_pool = MagicMock() + mock_pool_class.return_value = mock_pool + + # Mock connection and cursor + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_pool.getconn.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + mock_cursor.fetchone.return_value = [1] # vector extension exists + + pgvector = PGVector(self.collection_name, self.config) + pgvector._create_collection(1536) + + # Verify Redis lock was acquired with correct lock name + mock_redis.lock.assert_called_once_with("vector_indexing_test_collection_lock", timeout=20) + + # Verify lock context manager was entered and exited + mock_lock.__enter__.assert_called_once() + mock_lock.__exit__.assert_called_once() + + @patch("core.rag.datasource.vdb.pgvector.pgvector.psycopg2.pool.SimpleConnectionPool") + def test_get_cursor_context_manager(self, mock_pool_class): + """Test that _get_cursor properly manages connection lifecycle.""" + mock_pool = MagicMock() + mock_pool_class.return_value = mock_pool + + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_pool.getconn.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + + pgvector = PGVector(self.collection_name, self.config) + + with pgvector._get_cursor() as cur: + assert cur == mock_cursor + + # Verify connection lifecycle methods were called + mock_pool.getconn.assert_called_once() + mock_cursor.close.assert_called_once() + mock_conn.commit.assert_called_once() + mock_pool.putconn.assert_called_once_with(mock_conn) + + +@pytest.mark.parametrize( + "invalid_config_override", + [ + {"host": ""}, # Test empty host + {"port": 0}, # Test invalid port + {"user": ""}, # Test empty user + {"password": ""}, # Test empty password + {"database": ""}, # Test empty database + {"min_connection": 0}, # Test invalid min_connection + {"max_connection": 0}, # Test invalid max_connection + {"min_connection": 10, "max_connection": 5}, # Test min > max + ], +) +def test_config_validation_parametrized(invalid_config_override): + """Test configuration validation for various invalid inputs using parametrize.""" + config = { + "host": "localhost", + "port": 5432, + "user": "test_user", + "password": "test_password", + "database": "test_db", + "min_connection": 1, + "max_connection": 5, + } + config.update(invalid_config_override) + + with pytest.raises(ValueError): + PGVectorConfig(**config) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py index b4ee1b91b4..4ee04ddebc 100644 --- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py +++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py @@ -1,5 +1,7 @@ import os +from unittest.mock import MagicMock +import pytest from pytest_mock import MockerFixture from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp @@ -25,3 +27,35 @@ def test_firecrawl_web_extractor_crawl_mode(mocker: MockerFixture): assert job_id is not None assert isinstance(job_id, str) + + +def test_build_url_normalizes_slashes_for_crawl(mocker: MockerFixture): + api_key = "fc-" + base_urls = ["https://custom.firecrawl.dev", "https://custom.firecrawl.dev/"] + for base in base_urls: + app = FirecrawlApp(api_key=api_key, base_url=base) + mock_post = mocker.patch("httpx.post") + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"id": "job123"} + mock_post.return_value = mock_resp + app.crawl_url("https://example.com", params=None) + called_url = mock_post.call_args[0][0] + assert called_url == "https://custom.firecrawl.dev/v2/crawl" + + +def test_error_handler_handles_non_json_error_bodies(mocker: MockerFixture): + api_key = "fc-" + app = FirecrawlApp(api_key=api_key, base_url="https://custom.firecrawl.dev/") + mock_post = mocker.patch("httpx.post") + mock_resp = MagicMock() + mock_resp.status_code = 404 + mock_resp.text = "Not Found" + mock_resp.json.side_effect = Exception("Not JSON") + mock_post.return_value = mock_resp + + with pytest.raises(Exception) as excinfo: + app.scrape_url("https://example.com") + + # Should not raise a JSONDecodeError; current behavior reports status code only + assert str(excinfo.value) == "Failed to scrape URL. Status code: 404" diff --git a/api/tests/unit_tests/core/rag/extractor/test_word_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_word_extractor.py index fd0b0e2e44..3203aab8c3 100644 --- a/api/tests/unit_tests/core/rag/extractor/test_word_extractor.py +++ b/api/tests/unit_tests/core/rag/extractor/test_word_extractor.py @@ -132,3 +132,36 @@ def test_extract_images_from_docx(monkeypatch): # DB interactions should be recorded assert len(db_stub.session.added) == 2 assert db_stub.session.committed is True + + +def test_extract_images_from_docx_uses_internal_files_url(): + """Test that INTERNAL_FILES_URL takes precedence over FILES_URL for plugin access.""" + # Test the URL generation logic directly + from configs import dify_config + + # Mock the configuration values + original_files_url = getattr(dify_config, "FILES_URL", None) + original_internal_files_url = getattr(dify_config, "INTERNAL_FILES_URL", None) + + try: + # Set both URLs - INTERNAL should take precedence + dify_config.FILES_URL = "http://external.example.com" + dify_config.INTERNAL_FILES_URL = "http://internal.docker:5001" + + # Test the URL generation logic (same as in word_extractor.py) + upload_file_id = "test_file_id" + + # This is the pattern we fixed in the word extractor + base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL + generated_url = f"{base_url}/files/{upload_file_id}/file-preview" + + # Verify that INTERNAL_FILES_URL is used instead of FILES_URL + assert "http://internal.docker:5001" in generated_url, f"Expected internal URL, got: {generated_url}" + assert "http://external.example.com" not in generated_url, f"Should not use external URL, got: {generated_url}" + + finally: + # Restore original values + if original_files_url is not None: + dify_config.FILES_URL = original_files_url + if original_internal_files_url is not None: + dify_config.INTERNAL_FILES_URL = original_internal_files_url diff --git a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py index affd6c648f..6306d665e7 100644 --- a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py +++ b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py @@ -421,7 +421,18 @@ class TestRetrievalService: # In real code, this waits for all futures to complete # In tests, futures complete immediately, so wait is a no-op with patch("core.rag.datasource.retrieval_service.concurrent.futures.wait"): - yield mock_executor + # Mock concurrent.futures.as_completed for early error propagation + # In real code, this yields futures as they complete + # In tests, we yield all futures immediately since they're already done + def mock_as_completed(futures_list, timeout=None): + """Mock as_completed that yields futures immediately.""" + yield from futures_list + + with patch( + "core.rag.datasource.retrieval_service.concurrent.futures.as_completed", + side_effect=mock_as_completed, + ): + yield mock_executor # ==================== Vector Search Tests ==================== diff --git a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_metadata_filter.py b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_metadata_filter.py new file mode 100644 index 0000000000..07d6e51e4b --- /dev/null +++ b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_metadata_filter.py @@ -0,0 +1,873 @@ +""" +Unit tests for DatasetRetrieval.process_metadata_filter_func. + +This module provides comprehensive test coverage for the process_metadata_filter_func +method in the DatasetRetrieval class, which is responsible for building SQLAlchemy +filter expressions based on metadata filtering conditions. + +Conditions Tested: +================== +1. **String Conditions**: contains, not contains, start with, end with +2. **Equality Conditions**: is / =, is not / ≠ +3. **Null Conditions**: empty, not empty +4. **Numeric Comparisons**: before / <, after / >, ≤ / <=, ≥ / >= +5. **List Conditions**: in +6. **Edge Cases**: None values, different data types (str, int, float) + +Test Architecture: +================== +- Direct instantiation of DatasetRetrieval +- Mocking of DatasetDocument model attributes +- Verification of SQLAlchemy filter expressions +- Follows Arrange-Act-Assert (AAA) pattern + +Running Tests: +============== + # Run all tests in this module + uv run --project api pytest \ + api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_metadata_filter.py -v + + # Run a specific test + uv run --project api pytest \ + api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_metadata_filter.py::\ +TestProcessMetadataFilterFunc::test_contains_condition -v +""" + +from unittest.mock import MagicMock + +import pytest + +from core.rag.retrieval.dataset_retrieval import DatasetRetrieval + + +class TestProcessMetadataFilterFunc: + """ + Comprehensive test suite for process_metadata_filter_func method. + + This test class validates all metadata filtering conditions supported by + the DatasetRetrieval class, including string operations, numeric comparisons, + null checks, and list operations. + + Method Signature: + ================== + def process_metadata_filter_func( + self, sequence: int, condition: str, metadata_name: str, value: Any | None, filters: list + ) -> list: + + The method builds SQLAlchemy filter expressions by: + 1. Validating value is not None (except for empty/not empty conditions) + 2. Using DatasetDocument.doc_metadata JSON field operations + 3. Adding appropriate SQLAlchemy expressions to the filters list + 4. Returning the updated filters list + + Mocking Strategy: + ================== + - Mock DatasetDocument.doc_metadata to avoid database dependencies + - Verify filter expressions are created correctly + - Test with various data types (str, int, float, list) + """ + + @pytest.fixture + def retrieval(self): + """ + Create a DatasetRetrieval instance for testing. + + Returns: + DatasetRetrieval: Instance to test process_metadata_filter_func + """ + return DatasetRetrieval() + + @pytest.fixture + def mock_doc_metadata(self): + """ + Mock the DatasetDocument.doc_metadata JSON field. + + The method uses DatasetDocument.doc_metadata[metadata_name] to access + JSON fields. We mock this to avoid database dependencies. + + Returns: + Mock: Mocked doc_metadata attribute + """ + mock_metadata_field = MagicMock() + + # Create mock for string access + mock_string_access = MagicMock() + mock_string_access.like = MagicMock() + mock_string_access.notlike = MagicMock() + mock_string_access.__eq__ = MagicMock(return_value=MagicMock()) + mock_string_access.__ne__ = MagicMock(return_value=MagicMock()) + mock_string_access.in_ = MagicMock(return_value=MagicMock()) + + # Create mock for float access (for numeric comparisons) + mock_float_access = MagicMock() + mock_float_access.__eq__ = MagicMock(return_value=MagicMock()) + mock_float_access.__ne__ = MagicMock(return_value=MagicMock()) + mock_float_access.__lt__ = MagicMock(return_value=MagicMock()) + mock_float_access.__gt__ = MagicMock(return_value=MagicMock()) + mock_float_access.__le__ = MagicMock(return_value=MagicMock()) + mock_float_access.__ge__ = MagicMock(return_value=MagicMock()) + + # Create mock for null checks + mock_null_access = MagicMock() + mock_null_access.is_ = MagicMock(return_value=MagicMock()) + mock_null_access.isnot = MagicMock(return_value=MagicMock()) + + # Setup __getitem__ to return appropriate mock based on usage + def getitem_side_effect(name): + if name in ["author", "title", "category"]: + return mock_string_access + elif name in ["year", "price", "rating"]: + return mock_float_access + else: + return mock_string_access + + mock_metadata_field.__getitem__ = MagicMock(side_effect=getitem_side_effect) + mock_metadata_field.as_string.return_value = mock_string_access + mock_metadata_field.as_float.return_value = mock_float_access + mock_metadata_field[metadata_name:str].is_ = mock_null_access.is_ + mock_metadata_field[metadata_name:str].isnot = mock_null_access.isnot + + return mock_metadata_field + + # ==================== String Condition Tests ==================== + + def test_contains_condition_string_value(self, retrieval): + """ + Test 'contains' condition with string value. + + Verifies: + - Filters list is populated with LIKE expression + - Pattern matching uses %value% syntax + """ + filters = [] + sequence = 0 + condition = "contains" + metadata_name = "author" + value = "John" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_not_contains_condition(self, retrieval): + """ + Test 'not contains' condition. + + Verifies: + - Filters list is populated with NOT LIKE expression + - Pattern matching uses %value% syntax with negation + """ + filters = [] + sequence = 0 + condition = "not contains" + metadata_name = "title" + value = "banned" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_start_with_condition(self, retrieval): + """ + Test 'start with' condition. + + Verifies: + - Filters list is populated with LIKE expression + - Pattern matching uses value% syntax + """ + filters = [] + sequence = 0 + condition = "start with" + metadata_name = "category" + value = "tech" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_end_with_condition(self, retrieval): + """ + Test 'end with' condition. + + Verifies: + - Filters list is populated with LIKE expression + - Pattern matching uses %value syntax + """ + filters = [] + sequence = 0 + condition = "end with" + metadata_name = "filename" + value = ".pdf" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + # ==================== Equality Condition Tests ==================== + + def test_is_condition_with_string_value(self, retrieval): + """ + Test 'is' (=) condition with string value. + + Verifies: + - Filters list is populated with equality expression + - String comparison is used + """ + filters = [] + sequence = 0 + condition = "is" + metadata_name = "author" + value = "Jane Doe" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_equals_condition_with_string_value(self, retrieval): + """ + Test '=' condition with string value. + + Verifies: + - Same behavior as 'is' condition + - String comparison is used + """ + filters = [] + sequence = 0 + condition = "=" + metadata_name = "category" + value = "technology" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_is_condition_with_int_value(self, retrieval): + """ + Test 'is' condition with integer value. + + Verifies: + - Numeric comparison is used + - as_float() is called on the metadata field + """ + filters = [] + sequence = 0 + condition = "is" + metadata_name = "year" + value = 2023 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_is_condition_with_float_value(self, retrieval): + """ + Test 'is' condition with float value. + + Verifies: + - Numeric comparison is used + - as_float() is called on the metadata field + """ + filters = [] + sequence = 0 + condition = "is" + metadata_name = "price" + value = 19.99 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_is_not_condition_with_string_value(self, retrieval): + """ + Test 'is not' (≠) condition with string value. + + Verifies: + - Filters list is populated with inequality expression + - String comparison is used + """ + filters = [] + sequence = 0 + condition = "is not" + metadata_name = "author" + value = "Unknown" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_not_equals_condition(self, retrieval): + """ + Test '≠' condition with string value. + + Verifies: + - Same behavior as 'is not' condition + - Inequality expression is used + """ + filters = [] + sequence = 0 + condition = "≠" + metadata_name = "category" + value = "archived" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_is_not_condition_with_numeric_value(self, retrieval): + """ + Test 'is not' condition with numeric value. + + Verifies: + - Numeric inequality comparison is used + - as_float() is called on the metadata field + """ + filters = [] + sequence = 0 + condition = "is not" + metadata_name = "year" + value = 2000 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + # ==================== Null Condition Tests ==================== + + def test_empty_condition(self, retrieval): + """ + Test 'empty' condition (null check). + + Verifies: + - Filters list is populated with IS NULL expression + - Value can be None for this condition + """ + filters = [] + sequence = 0 + condition = "empty" + metadata_name = "author" + value = None + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_not_empty_condition(self, retrieval): + """ + Test 'not empty' condition (not null check). + + Verifies: + - Filters list is populated with IS NOT NULL expression + - Value can be None for this condition + """ + filters = [] + sequence = 0 + condition = "not empty" + metadata_name = "description" + value = None + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + # ==================== Numeric Comparison Tests ==================== + + def test_before_condition(self, retrieval): + """ + Test 'before' (<) condition. + + Verifies: + - Filters list is populated with less than expression + - Numeric comparison is used + """ + filters = [] + sequence = 0 + condition = "before" + metadata_name = "year" + value = 2020 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_less_than_condition(self, retrieval): + """ + Test '<' condition. + + Verifies: + - Same behavior as 'before' condition + - Less than expression is used + """ + filters = [] + sequence = 0 + condition = "<" + metadata_name = "price" + value = 100.0 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_after_condition(self, retrieval): + """ + Test 'after' (>) condition. + + Verifies: + - Filters list is populated with greater than expression + - Numeric comparison is used + """ + filters = [] + sequence = 0 + condition = "after" + metadata_name = "year" + value = 2020 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_greater_than_condition(self, retrieval): + """ + Test '>' condition. + + Verifies: + - Same behavior as 'after' condition + - Greater than expression is used + """ + filters = [] + sequence = 0 + condition = ">" + metadata_name = "rating" + value = 4.5 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_less_than_or_equal_condition_unicode(self, retrieval): + """ + Test '≤' condition. + + Verifies: + - Filters list is populated with less than or equal expression + - Numeric comparison is used + """ + filters = [] + sequence = 0 + condition = "≤" + metadata_name = "price" + value = 50.0 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_less_than_or_equal_condition_ascii(self, retrieval): + """ + Test '<=' condition. + + Verifies: + - Same behavior as '≤' condition + - Less than or equal expression is used + """ + filters = [] + sequence = 0 + condition = "<=" + metadata_name = "year" + value = 2023 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_greater_than_or_equal_condition_unicode(self, retrieval): + """ + Test '≥' condition. + + Verifies: + - Filters list is populated with greater than or equal expression + - Numeric comparison is used + """ + filters = [] + sequence = 0 + condition = "≥" + metadata_name = "rating" + value = 3.5 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_greater_than_or_equal_condition_ascii(self, retrieval): + """ + Test '>=' condition. + + Verifies: + - Same behavior as '≥' condition + - Greater than or equal expression is used + """ + filters = [] + sequence = 0 + condition = ">=" + metadata_name = "year" + value = 2000 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + # ==================== List/In Condition Tests ==================== + + def test_in_condition_with_comma_separated_string(self, retrieval): + """ + Test 'in' condition with comma-separated string value. + + Verifies: + - String is split into list + - Whitespace is trimmed from each value + - IN expression is created + """ + filters = [] + sequence = 0 + condition = "in" + metadata_name = "category" + value = "tech, science, AI " + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_in_condition_with_list_value(self, retrieval): + """ + Test 'in' condition with list value. + + Verifies: + - List is processed correctly + - None values are filtered out + - IN expression is created with valid values + """ + filters = [] + sequence = 0 + condition = "in" + metadata_name = "tags" + value = ["python", "javascript", None, "golang"] + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_in_condition_with_tuple_value(self, retrieval): + """ + Test 'in' condition with tuple value. + + Verifies: + - Tuple is processed like a list + - IN expression is created + """ + filters = [] + sequence = 0 + condition = "in" + metadata_name = "category" + value = ("tech", "science", "ai") + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_in_condition_with_empty_string(self, retrieval): + """ + Test 'in' condition with empty string value. + + Verifies: + - Empty string results in literal(False) filter + - No valid values to match + """ + filters = [] + sequence = 0 + condition = "in" + metadata_name = "category" + value = "" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + # Verify it's a literal(False) expression + # This is a bit tricky to test without access to the actual expression + + def test_in_condition_with_only_whitespace(self, retrieval): + """ + Test 'in' condition with whitespace-only string value. + + Verifies: + - Whitespace-only string results in literal(False) filter + - All values are stripped and filtered out + """ + filters = [] + sequence = 0 + condition = "in" + metadata_name = "category" + value = " , , " + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_in_condition_with_single_string(self, retrieval): + """ + Test 'in' condition with single non-comma string. + + Verifies: + - Single string is treated as single-item list + - IN expression is created with one value + """ + filters = [] + sequence = 0 + condition = "in" + metadata_name = "category" + value = "technology" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + # ==================== Edge Case Tests ==================== + + def test_none_value_with_non_empty_condition(self, retrieval): + """ + Test None value with conditions that require value. + + Verifies: + - Original filters list is returned unchanged + - No filter is added for None values (except empty/not empty) + """ + filters = [] + sequence = 0 + condition = "contains" + metadata_name = "author" + value = None + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 0 # No filter added + + def test_none_value_with_equals_condition(self, retrieval): + """ + Test None value with 'is' (=) condition. + + Verifies: + - Original filters list is returned unchanged + - No filter is added for None values + """ + filters = [] + sequence = 0 + condition = "is" + metadata_name = "author" + value = None + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 0 + + def test_none_value_with_numeric_condition(self, retrieval): + """ + Test None value with numeric comparison condition. + + Verifies: + - Original filters list is returned unchanged + - No filter is added for None values + """ + filters = [] + sequence = 0 + condition = ">" + metadata_name = "year" + value = None + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 0 + + def test_existing_filters_preserved(self, retrieval): + """ + Test that existing filters are preserved. + + Verifies: + - Existing filters in the list are not removed + - New filters are appended to the list + """ + existing_filter = MagicMock() + filters = [existing_filter] + sequence = 0 + condition = "contains" + metadata_name = "author" + value = "test" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 2 + assert filters[0] == existing_filter + + def test_multiple_filters_accumulated(self, retrieval): + """ + Test multiple calls to accumulate filters. + + Verifies: + - Each call adds a new filter to the list + - All filters are preserved across calls + """ + filters = [] + + # First filter + retrieval.process_metadata_filter_func(0, "contains", "author", "John", filters) + assert len(filters) == 1 + + # Second filter + retrieval.process_metadata_filter_func(1, ">", "year", 2020, filters) + assert len(filters) == 2 + + # Third filter + retrieval.process_metadata_filter_func(2, "is", "category", "tech", filters) + assert len(filters) == 3 + + def test_unknown_condition(self, retrieval): + """ + Test unknown/unsupported condition. + + Verifies: + - Original filters list is returned unchanged + - No filter is added for unknown conditions + """ + filters = [] + sequence = 0 + condition = "unknown_condition" + metadata_name = "author" + value = "test" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 0 + + def test_empty_string_value_with_contains(self, retrieval): + """ + Test empty string value with 'contains' condition. + + Verifies: + - Filter is added even with empty string + - LIKE expression is created + """ + filters = [] + sequence = 0 + condition = "contains" + metadata_name = "author" + value = "" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_special_characters_in_value(self, retrieval): + """ + Test special characters in value string. + + Verifies: + - Special characters are handled in value + - LIKE expression is created correctly + """ + filters = [] + sequence = 0 + condition = "contains" + metadata_name = "title" + value = "C++ & Python's features" + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_zero_value_with_numeric_condition(self, retrieval): + """ + Test zero value with numeric comparison condition. + + Verifies: + - Zero is treated as valid value + - Numeric comparison is performed + """ + filters = [] + sequence = 0 + condition = ">" + metadata_name = "price" + value = 0 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_negative_value_with_numeric_condition(self, retrieval): + """ + Test negative value with numeric comparison condition. + + Verifies: + - Negative numbers are handled correctly + - Numeric comparison is performed + """ + filters = [] + sequence = 0 + condition = "<" + metadata_name = "temperature" + value = -10.5 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 + + def test_float_value_with_integer_comparison(self, retrieval): + """ + Test float value with numeric comparison condition. + + Verifies: + - Float values work correctly + - Numeric comparison is performed + """ + filters = [] + sequence = 0 + condition = ">=" + metadata_name = "rating" + value = 4.5 + + result = retrieval.process_metadata_filter_func(sequence, condition, metadata_name, value, filters) + + assert result == filters + assert len(filters) == 1 diff --git a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py index 7d246ac3cc..943a9e5712 100644 --- a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py +++ b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py @@ -901,6 +901,13 @@ class TestFixedRecursiveCharacterTextSplitter: # Verify no empty chunks assert all(len(chunk) > 0 for chunk in result) + def test_double_slash_n(self): + data = "chunk 1\n\nsubchunk 1.\nsubchunk 2.\n\n---\n\nchunk 2\n\nsubchunk 1\nsubchunk 2." + separator = "\\n\\n---\\n\\n" + splitter = FixedRecursiveCharacterTextSplitter(fixed_separator=separator) + chunks = splitter.split_text(data) + assert chunks == ["chunk 1\n\nsubchunk 1.\nsubchunk 2.", "chunk 2\n\nsubchunk 1\nsubchunk 2."] + # ============================================================================ # Test Metadata Preservation diff --git a/api/tests/unit_tests/core/workflow/nodes/http_request/test_http_request_executor.py b/api/tests/unit_tests/core/workflow/nodes/http_request/test_http_request_executor.py index f040a92b6f..27df938102 100644 --- a/api/tests/unit_tests/core/workflow/nodes/http_request/test_http_request_executor.py +++ b/api/tests/unit_tests/core/workflow/nodes/http_request/test_http_request_executor.py @@ -1,3 +1,5 @@ +import pytest + from core.workflow.nodes.http_request import ( BodyData, HttpRequestNodeAuthorization, @@ -5,6 +7,7 @@ from core.workflow.nodes.http_request import ( HttpRequestNodeData, ) from core.workflow.nodes.http_request.entities import HttpRequestNodeTimeout +from core.workflow.nodes.http_request.exc import AuthorizationConfigError from core.workflow.nodes.http_request.executor import Executor from core.workflow.runtime import VariablePool from core.workflow.system_variable import SystemVariable @@ -348,3 +351,127 @@ def test_init_params(): executor = create_executor("key1:value1\n\nkey2:value2\n\n") executor._init_params() assert executor.params == [("key1", "value1"), ("key2", "value2")] + + +def test_empty_api_key_raises_error_bearer(): + """Test that empty API key raises AuthorizationConfigError for bearer auth.""" + variable_pool = VariablePool(system_variables=SystemVariable.empty()) + node_data = HttpRequestNodeData( + title="test", + method="get", + url="http://example.com", + headers="", + params="", + authorization=HttpRequestNodeAuthorization( + type="api-key", + config={"type": "bearer", "api_key": ""}, + ), + ) + timeout = HttpRequestNodeTimeout(connect=10, read=30, write=30) + + with pytest.raises(AuthorizationConfigError, match="API key is required"): + Executor( + node_data=node_data, + timeout=timeout, + variable_pool=variable_pool, + ) + + +def test_empty_api_key_raises_error_basic(): + """Test that empty API key raises AuthorizationConfigError for basic auth.""" + variable_pool = VariablePool(system_variables=SystemVariable.empty()) + node_data = HttpRequestNodeData( + title="test", + method="get", + url="http://example.com", + headers="", + params="", + authorization=HttpRequestNodeAuthorization( + type="api-key", + config={"type": "basic", "api_key": ""}, + ), + ) + timeout = HttpRequestNodeTimeout(connect=10, read=30, write=30) + + with pytest.raises(AuthorizationConfigError, match="API key is required"): + Executor( + node_data=node_data, + timeout=timeout, + variable_pool=variable_pool, + ) + + +def test_empty_api_key_raises_error_custom(): + """Test that empty API key raises AuthorizationConfigError for custom auth.""" + variable_pool = VariablePool(system_variables=SystemVariable.empty()) + node_data = HttpRequestNodeData( + title="test", + method="get", + url="http://example.com", + headers="", + params="", + authorization=HttpRequestNodeAuthorization( + type="api-key", + config={"type": "custom", "api_key": "", "header": "X-Custom-Auth"}, + ), + ) + timeout = HttpRequestNodeTimeout(connect=10, read=30, write=30) + + with pytest.raises(AuthorizationConfigError, match="API key is required"): + Executor( + node_data=node_data, + timeout=timeout, + variable_pool=variable_pool, + ) + + +def test_whitespace_only_api_key_raises_error(): + """Test that whitespace-only API key raises AuthorizationConfigError.""" + variable_pool = VariablePool(system_variables=SystemVariable.empty()) + node_data = HttpRequestNodeData( + title="test", + method="get", + url="http://example.com", + headers="", + params="", + authorization=HttpRequestNodeAuthorization( + type="api-key", + config={"type": "bearer", "api_key": " "}, + ), + ) + timeout = HttpRequestNodeTimeout(connect=10, read=30, write=30) + + with pytest.raises(AuthorizationConfigError, match="API key is required"): + Executor( + node_data=node_data, + timeout=timeout, + variable_pool=variable_pool, + ) + + +def test_valid_api_key_works(): + """Test that valid API key works correctly for bearer auth.""" + variable_pool = VariablePool(system_variables=SystemVariable.empty()) + node_data = HttpRequestNodeData( + title="test", + method="get", + url="http://example.com", + headers="", + params="", + authorization=HttpRequestNodeAuthorization( + type="api-key", + config={"type": "bearer", "api_key": "valid-api-key-123"}, + ), + ) + timeout = HttpRequestNodeTimeout(connect=10, read=30, write=30) + + executor = Executor( + node_data=node_data, + timeout=timeout, + variable_pool=variable_pool, + ) + + # Should not raise an error + headers = executor._assembling_headers() + assert "Authorization" in headers + assert headers["Authorization"] == "Bearer valid-api-key-123" diff --git a/api/tests/unit_tests/core/workflow/nodes/test_start_node_json_object.py b/api/tests/unit_tests/core/workflow/nodes/test_start_node_json_object.py index 83799c9508..539e72edb5 100644 --- a/api/tests/unit_tests/core/workflow/nodes/test_start_node_json_object.py +++ b/api/tests/unit_tests/core/workflow/nodes/test_start_node_json_object.py @@ -1,3 +1,4 @@ +import json import time import pytest @@ -46,14 +47,16 @@ def make_start_node(user_inputs, variables): def test_json_object_valid_schema(): - schema = { - "type": "object", - "properties": { - "age": {"type": "number"}, - "name": {"type": "string"}, - }, - "required": ["age"], - } + schema = json.dumps( + { + "type": "object", + "properties": { + "age": {"type": "number"}, + "name": {"type": "string"}, + }, + "required": ["age"], + } + ) variables = [ VariableEntity( @@ -65,7 +68,7 @@ def test_json_object_valid_schema(): ) ] - user_inputs = {"profile": {"age": 20, "name": "Tom"}} + user_inputs = {"profile": json.dumps({"age": 20, "name": "Tom"})} node = make_start_node(user_inputs, variables) result = node._run() @@ -74,12 +77,23 @@ def test_json_object_valid_schema(): def test_json_object_invalid_json_string(): + schema = json.dumps( + { + "type": "object", + "properties": { + "age": {"type": "number"}, + "name": {"type": "string"}, + }, + "required": ["age", "name"], + } + ) variables = [ VariableEntity( variable="profile", label="profile", type=VariableEntityType.JSON_OBJECT, required=True, + json_schema=schema, ) ] @@ -88,38 +102,21 @@ def test_json_object_invalid_json_string(): node = make_start_node(user_inputs, variables) - with pytest.raises(ValueError, match="profile must be a JSON object"): - node._run() - - -@pytest.mark.parametrize("value", ["[1, 2, 3]", "123"]) -def test_json_object_valid_json_but_not_object(value): - variables = [ - VariableEntity( - variable="profile", - label="profile", - type=VariableEntityType.JSON_OBJECT, - required=True, - ) - ] - - user_inputs = {"profile": value} - - node = make_start_node(user_inputs, variables) - - with pytest.raises(ValueError, match="profile must be a JSON object"): + with pytest.raises(ValueError, match='{"age": 20, "name": "Tom" must be a valid JSON object'): node._run() def test_json_object_does_not_match_schema(): - schema = { - "type": "object", - "properties": { - "age": {"type": "number"}, - "name": {"type": "string"}, - }, - "required": ["age", "name"], - } + schema = json.dumps( + { + "type": "object", + "properties": { + "age": {"type": "number"}, + "name": {"type": "string"}, + }, + "required": ["age", "name"], + } + ) variables = [ VariableEntity( @@ -132,7 +129,7 @@ def test_json_object_does_not_match_schema(): ] # age is a string, which violates the schema (expects number) - user_inputs = {"profile": {"age": "twenty", "name": "Tom"}} + user_inputs = {"profile": json.dumps({"age": "twenty", "name": "Tom"})} node = make_start_node(user_inputs, variables) @@ -141,14 +138,16 @@ def test_json_object_does_not_match_schema(): def test_json_object_missing_required_schema_field(): - schema = { - "type": "object", - "properties": { - "age": {"type": "number"}, - "name": {"type": "string"}, - }, - "required": ["age", "name"], - } + schema = json.dumps( + { + "type": "object", + "properties": { + "age": {"type": "number"}, + "name": {"type": "string"}, + }, + "required": ["age", "name"], + } + ) variables = [ VariableEntity( @@ -161,7 +160,7 @@ def test_json_object_missing_required_schema_field(): ] # Missing required field "name" - user_inputs = {"profile": {"age": 20}} + user_inputs = {"profile": json.dumps({"age": 20})} node = make_start_node(user_inputs, variables) @@ -214,7 +213,7 @@ def test_json_object_optional_variable_not_provided(): variable="profile", label="profile", type=VariableEntityType.JSON_OBJECT, - required=False, + required=True, ) ] @@ -223,5 +222,5 @@ def test_json_object_optional_variable_not_provided(): node = make_start_node(user_inputs, variables) # Current implementation raises a validation error even when the variable is optional - with pytest.raises(ValueError, match="profile must be a JSON object"): + with pytest.raises(ValueError, match="profile is required in input form"): node._run() diff --git a/api/tests/unit_tests/oss/tencent_cos/test_tencent_cos.py b/api/tests/unit_tests/oss/tencent_cos/test_tencent_cos.py index 303f0493bd..a0fed1aa14 100644 --- a/api/tests/unit_tests/oss/tencent_cos/test_tencent_cos.py +++ b/api/tests/unit_tests/oss/tencent_cos/test_tencent_cos.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from qcloud_cos import CosConfig @@ -18,3 +18,72 @@ class TestTencentCos(BaseStorageTest): with patch.object(CosConfig, "__init__", return_value=None): self.storage = TencentCosStorage() self.storage.bucket_name = get_example_bucket() + + +class TestTencentCosConfiguration: + """Tests for TencentCosStorage initialization with different configurations.""" + + def test_init_with_custom_domain(self): + """Test initialization with custom domain configured.""" + # Mock dify_config to return custom domain configuration + mock_dify_config = MagicMock() + mock_dify_config.TENCENT_COS_CUSTOM_DOMAIN = "cos.example.com" + mock_dify_config.TENCENT_COS_SECRET_ID = "test-secret-id" + mock_dify_config.TENCENT_COS_SECRET_KEY = "test-secret-key" + mock_dify_config.TENCENT_COS_SCHEME = "https" + + # Mock CosConfig and CosS3Client + mock_config_instance = MagicMock() + mock_client = MagicMock() + + with ( + patch("extensions.storage.tencent_cos_storage.dify_config", mock_dify_config), + patch( + "extensions.storage.tencent_cos_storage.CosConfig", return_value=mock_config_instance + ) as mock_cos_config, + patch("extensions.storage.tencent_cos_storage.CosS3Client", return_value=mock_client), + ): + TencentCosStorage() + + # Verify CosConfig was called with Domain parameter (not Region) + mock_cos_config.assert_called_once() + call_kwargs = mock_cos_config.call_args[1] + assert "Domain" in call_kwargs + assert call_kwargs["Domain"] == "cos.example.com" + assert "Region" not in call_kwargs + assert call_kwargs["SecretId"] == "test-secret-id" + assert call_kwargs["SecretKey"] == "test-secret-key" + assert call_kwargs["Scheme"] == "https" + + def test_init_with_region(self): + """Test initialization with region configured (no custom domain).""" + # Mock dify_config to return region configuration + mock_dify_config = MagicMock() + mock_dify_config.TENCENT_COS_CUSTOM_DOMAIN = None + mock_dify_config.TENCENT_COS_REGION = "ap-guangzhou" + mock_dify_config.TENCENT_COS_SECRET_ID = "test-secret-id" + mock_dify_config.TENCENT_COS_SECRET_KEY = "test-secret-key" + mock_dify_config.TENCENT_COS_SCHEME = "https" + + # Mock CosConfig and CosS3Client + mock_config_instance = MagicMock() + mock_client = MagicMock() + + with ( + patch("extensions.storage.tencent_cos_storage.dify_config", mock_dify_config), + patch( + "extensions.storage.tencent_cos_storage.CosConfig", return_value=mock_config_instance + ) as mock_cos_config, + patch("extensions.storage.tencent_cos_storage.CosS3Client", return_value=mock_client), + ): + TencentCosStorage() + + # Verify CosConfig was called with Region parameter (not Domain) + mock_cos_config.assert_called_once() + call_kwargs = mock_cos_config.call_args[1] + assert "Region" in call_kwargs + assert call_kwargs["Region"] == "ap-guangzhou" + assert "Domain" not in call_kwargs + assert call_kwargs["SecretId"] == "test-secret-id" + assert call_kwargs["SecretKey"] == "test-secret-key" + assert call_kwargs["Scheme"] == "https" diff --git a/api/tests/unit_tests/services/auth/test_firecrawl_auth.py b/api/tests/unit_tests/services/auth/test_firecrawl_auth.py index b5ee55706d..ab50d6a92c 100644 --- a/api/tests/unit_tests/services/auth/test_firecrawl_auth.py +++ b/api/tests/unit_tests/services/auth/test_firecrawl_auth.py @@ -1,3 +1,4 @@ +import json from unittest.mock import MagicMock, patch import httpx @@ -110,9 +111,11 @@ class TestFirecrawlAuth: @pytest.mark.parametrize( ("status_code", "response_text", "has_json_error", "expected_error_contains"), [ - (403, '{"error": "Forbidden"}', True, "Failed to authorize. Status code: 403. Error: Forbidden"), - (404, "", True, "Unexpected error occurred while trying to authorize. Status code: 404"), - (401, "Not JSON", True, "Expecting value"), # JSON decode error + (403, '{"error": "Forbidden"}', False, "Failed to authorize. Status code: 403. Error: Forbidden"), + # empty body falls back to generic message + (404, "", True, "Failed to authorize. Status code: 404. Error: Unknown error occurred"), + # non-JSON body is surfaced directly + (401, "Not JSON", True, "Failed to authorize. Status code: 401. Error: Not JSON"), ], ) @patch("services.auth.firecrawl.firecrawl.httpx.post") @@ -124,12 +127,14 @@ class TestFirecrawlAuth: mock_response.status_code = status_code mock_response.text = response_text if has_json_error: - mock_response.json.side_effect = Exception("Not JSON") + mock_response.json.side_effect = json.JSONDecodeError("Not JSON", "", 0) + else: + mock_response.json.return_value = {"error": "Forbidden"} mock_post.return_value = mock_response with pytest.raises(Exception) as exc_info: auth_instance.validate_credentials() - assert expected_error_contains in str(exc_info.value) + assert str(exc_info.value) == expected_error_contains @pytest.mark.parametrize( ("exception_type", "exception_message"), @@ -164,20 +169,21 @@ class TestFirecrawlAuth: @patch("services.auth.firecrawl.firecrawl.httpx.post") def test_should_use_custom_base_url_in_validation(self, mock_post): - """Test that custom base URL is used in validation""" + """Test that custom base URL is used in validation and normalized""" mock_response = MagicMock() mock_response.status_code = 200 mock_post.return_value = mock_response - credentials = { - "auth_type": "bearer", - "config": {"api_key": "test_api_key_123", "base_url": "https://custom.firecrawl.dev"}, - } - auth = FirecrawlAuth(credentials) - result = auth.validate_credentials() + for base in ("https://custom.firecrawl.dev", "https://custom.firecrawl.dev/"): + credentials = { + "auth_type": "bearer", + "config": {"api_key": "test_api_key_123", "base_url": base}, + } + auth = FirecrawlAuth(credentials) + result = auth.validate_credentials() - assert result is True - assert mock_post.call_args[0][0] == "https://custom.firecrawl.dev/v1/crawl" + assert result is True + assert mock_post.call_args[0][0] == "https://custom.firecrawl.dev/v1/crawl" @patch("services.auth.firecrawl.firecrawl.httpx.post") def test_should_handle_timeout_with_retry_suggestion(self, mock_post, auth_instance): diff --git a/api/tests/unit_tests/services/test_app_dsl_service_import_yaml_url.py b/api/tests/unit_tests/services/test_app_dsl_service_import_yaml_url.py new file mode 100644 index 0000000000..41c1d0ea2a --- /dev/null +++ b/api/tests/unit_tests/services/test_app_dsl_service_import_yaml_url.py @@ -0,0 +1,71 @@ +from unittest.mock import MagicMock + +import httpx + +from models import Account +from services import app_dsl_service +from services.app_dsl_service import AppDslService, ImportMode, ImportStatus + + +def _build_response(url: str, status_code: int, content: bytes = b"") -> httpx.Response: + request = httpx.Request("GET", url) + return httpx.Response(status_code=status_code, request=request, content=content) + + +def _pending_yaml_content(version: str = "99.0.0") -> bytes: + return (f'version: "{version}"\nkind: app\napp:\n name: Loop Test\n mode: workflow\n').encode() + + +def _account_mock() -> MagicMock: + account = MagicMock(spec=Account) + account.current_tenant_id = "tenant-1" + return account + + +def test_import_app_yaml_url_user_attachments_keeps_original_url(monkeypatch): + yaml_url = "https://github.com/user-attachments/files/24290802/loop-test.yml" + raw_url = "https://raw.githubusercontent.com/user-attachments/files/24290802/loop-test.yml" + yaml_bytes = _pending_yaml_content() + + def fake_get(url: str, **kwargs): + if url == raw_url: + return _build_response(url, status_code=404) + assert url == yaml_url + return _build_response(url, status_code=200, content=yaml_bytes) + + monkeypatch.setattr(app_dsl_service.ssrf_proxy, "get", fake_get) + + service = AppDslService(MagicMock()) + result = service.import_app( + account=_account_mock(), + import_mode=ImportMode.YAML_URL, + yaml_url=yaml_url, + ) + + assert result.status == ImportStatus.PENDING + assert result.imported_dsl_version == "99.0.0" + + +def test_import_app_yaml_url_github_blob_rewrites_to_raw(monkeypatch): + yaml_url = "https://github.com/acme/repo/blob/main/app.yml" + raw_url = "https://raw.githubusercontent.com/acme/repo/main/app.yml" + yaml_bytes = _pending_yaml_content() + + requested_urls: list[str] = [] + + def fake_get(url: str, **kwargs): + requested_urls.append(url) + assert url == raw_url + return _build_response(url, status_code=200, content=yaml_bytes) + + monkeypatch.setattr(app_dsl_service.ssrf_proxy, "get", fake_get) + + service = AppDslService(MagicMock()) + result = service.import_app( + account=_account_mock(), + import_mode=ImportMode.YAML_URL, + yaml_url=yaml_url, + ) + + assert result.status == ImportStatus.PENDING + assert requested_urls == [raw_url] diff --git a/api/tests/unit_tests/services/test_billing_service.py b/api/tests/unit_tests/services/test_billing_service.py index 915aee3fa7..d00743278e 100644 --- a/api/tests/unit_tests/services/test_billing_service.py +++ b/api/tests/unit_tests/services/test_billing_service.py @@ -1156,6 +1156,235 @@ class TestBillingServiceEdgeCases: assert "Only team owner or team admin can perform this action" in str(exc_info.value) +class TestBillingServiceSubscriptionOperations: + """Unit tests for subscription operations in BillingService. + + Tests cover: + - Bulk plan retrieval with chunking + - Expired subscription cleanup whitelist retrieval + """ + + @pytest.fixture + def mock_send_request(self): + """Mock _send_request method.""" + with patch.object(BillingService, "_send_request") as mock: + yield mock + + def test_get_plan_bulk_with_empty_list(self, mock_send_request): + """Test bulk plan retrieval with empty tenant list.""" + # Arrange + tenant_ids = [] + + # Act + result = BillingService.get_plan_bulk(tenant_ids) + + # Assert + assert result == {} + mock_send_request.assert_not_called() + + def test_get_plan_bulk_with_chunking(self, mock_send_request): + """Test bulk plan retrieval with more than 200 tenants (chunking logic).""" + # Arrange - 250 tenants to test chunking (chunk_size = 200) + tenant_ids = [f"tenant-{i}" for i in range(250)] + + # First chunk: tenants 0-199 + first_chunk_response = { + "data": {f"tenant-{i}": {"plan": "sandbox", "expiration_date": 1735689600} for i in range(200)} + } + + # Second chunk: tenants 200-249 + second_chunk_response = { + "data": {f"tenant-{i}": {"plan": "professional", "expiration_date": 1767225600} for i in range(200, 250)} + } + + mock_send_request.side_effect = [first_chunk_response, second_chunk_response] + + # Act + result = BillingService.get_plan_bulk(tenant_ids) + + # Assert + assert len(result) == 250 + assert result["tenant-0"]["plan"] == "sandbox" + assert result["tenant-199"]["plan"] == "sandbox" + assert result["tenant-200"]["plan"] == "professional" + assert result["tenant-249"]["plan"] == "professional" + assert mock_send_request.call_count == 2 + + # Verify first chunk call + first_call = mock_send_request.call_args_list[0] + assert first_call[0][0] == "POST" + assert first_call[0][1] == "/subscription/plan/batch" + assert len(first_call[1]["json"]["tenant_ids"]) == 200 + + # Verify second chunk call + second_call = mock_send_request.call_args_list[1] + assert len(second_call[1]["json"]["tenant_ids"]) == 50 + + def test_get_plan_bulk_with_partial_batch_failure(self, mock_send_request): + """Test bulk plan retrieval when one batch fails but others succeed.""" + # Arrange - 250 tenants, second batch will fail + tenant_ids = [f"tenant-{i}" for i in range(250)] + + # First chunk succeeds + first_chunk_response = { + "data": {f"tenant-{i}": {"plan": "sandbox", "expiration_date": 1735689600} for i in range(200)} + } + + # Second chunk fails - need to create a mock that raises when called + def side_effect_func(*args, **kwargs): + if mock_send_request.call_count == 1: + return first_chunk_response + else: + raise ValueError("API error") + + mock_send_request.side_effect = side_effect_func + + # Act + result = BillingService.get_plan_bulk(tenant_ids) + + # Assert - should only have data from first batch + assert len(result) == 200 + assert result["tenant-0"]["plan"] == "sandbox" + assert result["tenant-199"]["plan"] == "sandbox" + assert "tenant-200" not in result + assert mock_send_request.call_count == 2 + + def test_get_plan_bulk_with_all_batches_failing(self, mock_send_request): + """Test bulk plan retrieval when all batches fail.""" + # Arrange + tenant_ids = [f"tenant-{i}" for i in range(250)] + + # All chunks fail + def side_effect_func(*args, **kwargs): + raise ValueError("API error") + + mock_send_request.side_effect = side_effect_func + + # Act + result = BillingService.get_plan_bulk(tenant_ids) + + # Assert - should return empty dict + assert result == {} + assert mock_send_request.call_count == 2 + + def test_get_plan_bulk_with_exactly_200_tenants(self, mock_send_request): + """Test bulk plan retrieval with exactly 200 tenants (boundary condition).""" + # Arrange + tenant_ids = [f"tenant-{i}" for i in range(200)] + mock_send_request.return_value = { + "data": {f"tenant-{i}": {"plan": "sandbox", "expiration_date": 1735689600} for i in range(200)} + } + + # Act + result = BillingService.get_plan_bulk(tenant_ids) + + # Assert + assert len(result) == 200 + assert mock_send_request.call_count == 1 + + def test_get_plan_bulk_with_empty_data_response(self, mock_send_request): + """Test bulk plan retrieval with empty data in response.""" + # Arrange + tenant_ids = ["tenant-1", "tenant-2"] + mock_send_request.return_value = {"data": {}} + + # Act + result = BillingService.get_plan_bulk(tenant_ids) + + # Assert + assert result == {} + + def test_get_plan_bulk_with_invalid_tenant_plan_skipped(self, mock_send_request): + """Test bulk plan retrieval when one tenant has invalid plan data (should skip that tenant).""" + # Arrange + tenant_ids = ["tenant-valid-1", "tenant-invalid", "tenant-valid-2"] + + # Response with one invalid tenant plan (missing expiration_date) and two valid ones + mock_send_request.return_value = { + "data": { + "tenant-valid-1": {"plan": "sandbox", "expiration_date": 1735689600}, + "tenant-invalid": {"plan": "professional"}, # Missing expiration_date field + "tenant-valid-2": {"plan": "team", "expiration_date": 1767225600}, + } + } + + # Act + with patch("services.billing_service.logger") as mock_logger: + result = BillingService.get_plan_bulk(tenant_ids) + + # Assert - should only contain valid tenants + assert len(result) == 2 + assert "tenant-valid-1" in result + assert "tenant-valid-2" in result + assert "tenant-invalid" not in result + + # Verify valid tenants have correct data + assert result["tenant-valid-1"]["plan"] == "sandbox" + assert result["tenant-valid-1"]["expiration_date"] == 1735689600 + assert result["tenant-valid-2"]["plan"] == "team" + assert result["tenant-valid-2"]["expiration_date"] == 1767225600 + + # Verify exception was logged for the invalid tenant + mock_logger.exception.assert_called_once() + log_call_args = mock_logger.exception.call_args[0] + assert "get_plan_bulk: failed to validate subscription plan for tenant" in log_call_args[0] + assert "tenant-invalid" in log_call_args[1] + + def test_get_expired_subscription_cleanup_whitelist_success(self, mock_send_request): + """Test successful retrieval of expired subscription cleanup whitelist.""" + # Arrange + api_response = [ + { + "created_at": "2025-10-16T01:56:17", + "tenant_id": "36bd55ec-2ea9-4d75-a9ea-1f26aeb4ffe6", + "contact": "example@dify.ai", + "id": "36bd55ec-2ea9-4d75-a9ea-1f26aeb4ffe5", + "expired_at": "2026-01-01T01:56:17", + "updated_at": "2025-10-16T01:56:17", + }, + { + "created_at": "2025-10-16T02:00:00", + "tenant_id": "tenant-2", + "contact": "test@example.com", + "id": "whitelist-id-2", + "expired_at": "2026-02-01T00:00:00", + "updated_at": "2025-10-16T02:00:00", + }, + { + "created_at": "2025-10-16T03:00:00", + "tenant_id": "tenant-3", + "contact": "another@example.com", + "id": "whitelist-id-3", + "expired_at": "2026-03-01T00:00:00", + "updated_at": "2025-10-16T03:00:00", + }, + ] + mock_send_request.return_value = {"data": api_response} + + # Act + result = BillingService.get_expired_subscription_cleanup_whitelist() + + # Assert - should return only tenant_ids + assert result == ["36bd55ec-2ea9-4d75-a9ea-1f26aeb4ffe6", "tenant-2", "tenant-3"] + assert len(result) == 3 + assert result[0] == "36bd55ec-2ea9-4d75-a9ea-1f26aeb4ffe6" + assert result[1] == "tenant-2" + assert result[2] == "tenant-3" + mock_send_request.assert_called_once_with("GET", "/subscription/cleanup/whitelist") + + def test_get_expired_subscription_cleanup_whitelist_empty_list(self, mock_send_request): + """Test retrieval of empty cleanup whitelist.""" + # Arrange + mock_send_request.return_value = {"data": []} + + # Act + result = BillingService.get_expired_subscription_cleanup_whitelist() + + # Assert + assert result == [] + assert len(result) == 0 + + class TestBillingServiceIntegrationScenarios: """Integration-style tests simulating real-world usage scenarios. diff --git a/api/tests/unit_tests/services/test_dataset_service_get_segments.py b/api/tests/unit_tests/services/test_dataset_service_get_segments.py new file mode 100644 index 0000000000..360c8a3c7d --- /dev/null +++ b/api/tests/unit_tests/services/test_dataset_service_get_segments.py @@ -0,0 +1,472 @@ +""" +Unit tests for SegmentService.get_segments method. + +Tests the retrieval of document segments with pagination and filtering: +- Basic pagination (page, limit) +- Status filtering +- Keyword search +- Ordering by position and id (to avoid duplicate data) +""" + +from unittest.mock import Mock, create_autospec, patch + +import pytest + +from models.dataset import DocumentSegment + + +class SegmentServiceTestDataFactory: + """ + Factory class for creating test data and mock objects for segment tests. + """ + + @staticmethod + def create_segment_mock( + segment_id: str = "segment-123", + document_id: str = "doc-123", + tenant_id: str = "tenant-123", + dataset_id: str = "dataset-123", + position: int = 1, + content: str = "Test content", + status: str = "completed", + **kwargs, + ) -> Mock: + """ + Create a mock document segment. + + Args: + segment_id: Unique identifier for the segment + document_id: Parent document ID + tenant_id: Tenant ID the segment belongs to + dataset_id: Parent dataset ID + position: Position within the document + content: Segment text content + status: Indexing status + **kwargs: Additional attributes + + Returns: + Mock: DocumentSegment mock object + """ + segment = create_autospec(DocumentSegment, instance=True) + segment.id = segment_id + segment.document_id = document_id + segment.tenant_id = tenant_id + segment.dataset_id = dataset_id + segment.position = position + segment.content = content + segment.status = status + for key, value in kwargs.items(): + setattr(segment, key, value) + return segment + + +class TestSegmentServiceGetSegments: + """ + Comprehensive unit tests for SegmentService.get_segments method. + + Tests cover: + - Basic pagination functionality + - Status list filtering + - Keyword search filtering + - Ordering (position + id for uniqueness) + - Empty results + - Combined filters + """ + + @pytest.fixture + def mock_segment_service_dependencies(self): + """ + Common mock setup for segment service dependencies. + + Patches: + - db: Database operations and pagination + - select: SQLAlchemy query builder + """ + with ( + patch("services.dataset_service.db") as mock_db, + patch("services.dataset_service.select") as mock_select, + ): + yield { + "db": mock_db, + "select": mock_select, + } + + def test_get_segments_basic_pagination(self, mock_segment_service_dependencies): + """ + Test basic pagination functionality. + + Verifies: + - Query is built with document_id and tenant_id filters + - Pagination uses correct page and limit parameters + - Returns segments and total count + """ + # Arrange + document_id = "doc-123" + tenant_id = "tenant-123" + page = 1 + limit = 20 + + # Create mock segments + segment1 = SegmentServiceTestDataFactory.create_segment_mock( + segment_id="seg-1", position=1, content="First segment" + ) + segment2 = SegmentServiceTestDataFactory.create_segment_mock( + segment_id="seg-2", position=2, content="Second segment" + ) + + # Mock pagination result + mock_paginated = Mock() + mock_paginated.items = [segment1, segment2] + mock_paginated.total = 2 + + mock_segment_service_dependencies["db"].paginate.return_value = mock_paginated + + # Mock select builder + mock_query = Mock() + mock_segment_service_dependencies["select"].return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.order_by.return_value = mock_query + + # Act + from services.dataset_service import SegmentService + + items, total = SegmentService.get_segments(document_id=document_id, tenant_id=tenant_id, page=page, limit=limit) + + # Assert + assert len(items) == 2 + assert total == 2 + assert items[0].id == "seg-1" + assert items[1].id == "seg-2" + mock_segment_service_dependencies["db"].paginate.assert_called_once() + call_kwargs = mock_segment_service_dependencies["db"].paginate.call_args[1] + assert call_kwargs["page"] == page + assert call_kwargs["per_page"] == limit + assert call_kwargs["max_per_page"] == 100 + assert call_kwargs["error_out"] is False + + def test_get_segments_with_status_filter(self, mock_segment_service_dependencies): + """ + Test filtering by status list. + + Verifies: + - Status list filter is applied to query + - Only segments with matching status are returned + """ + # Arrange + document_id = "doc-123" + tenant_id = "tenant-123" + status_list = ["completed", "indexing"] + + segment1 = SegmentServiceTestDataFactory.create_segment_mock(segment_id="seg-1", status="completed") + segment2 = SegmentServiceTestDataFactory.create_segment_mock(segment_id="seg-2", status="indexing") + + mock_paginated = Mock() + mock_paginated.items = [segment1, segment2] + mock_paginated.total = 2 + + mock_segment_service_dependencies["db"].paginate.return_value = mock_paginated + + mock_query = Mock() + mock_segment_service_dependencies["select"].return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.order_by.return_value = mock_query + + # Act + from services.dataset_service import SegmentService + + items, total = SegmentService.get_segments( + document_id=document_id, tenant_id=tenant_id, status_list=status_list + ) + + # Assert + assert len(items) == 2 + assert total == 2 + # Verify where was called multiple times (base filters + status filter) + assert mock_query.where.call_count >= 2 + + def test_get_segments_with_empty_status_list(self, mock_segment_service_dependencies): + """ + Test with empty status list. + + Verifies: + - Empty status list is handled correctly + - No status filter is applied to avoid WHERE false condition + """ + # Arrange + document_id = "doc-123" + tenant_id = "tenant-123" + status_list = [] + + segment = SegmentServiceTestDataFactory.create_segment_mock(segment_id="seg-1") + + mock_paginated = Mock() + mock_paginated.items = [segment] + mock_paginated.total = 1 + + mock_segment_service_dependencies["db"].paginate.return_value = mock_paginated + + mock_query = Mock() + mock_segment_service_dependencies["select"].return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.order_by.return_value = mock_query + + # Act + from services.dataset_service import SegmentService + + items, total = SegmentService.get_segments( + document_id=document_id, tenant_id=tenant_id, status_list=status_list + ) + + # Assert + assert len(items) == 1 + assert total == 1 + # Should only be called once (base filters, no status filter) + assert mock_query.where.call_count == 1 + + def test_get_segments_with_keyword_search(self, mock_segment_service_dependencies): + """ + Test keyword search functionality. + + Verifies: + - Keyword filter uses ilike for case-insensitive search + - Search pattern includes wildcards (%keyword%) + """ + # Arrange + document_id = "doc-123" + tenant_id = "tenant-123" + keyword = "search term" + + segment = SegmentServiceTestDataFactory.create_segment_mock( + segment_id="seg-1", content="This contains search term" + ) + + mock_paginated = Mock() + mock_paginated.items = [segment] + mock_paginated.total = 1 + + mock_segment_service_dependencies["db"].paginate.return_value = mock_paginated + + mock_query = Mock() + mock_segment_service_dependencies["select"].return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.order_by.return_value = mock_query + + # Act + from services.dataset_service import SegmentService + + items, total = SegmentService.get_segments(document_id=document_id, tenant_id=tenant_id, keyword=keyword) + + # Assert + assert len(items) == 1 + assert total == 1 + # Verify where was called for base filters + keyword filter + assert mock_query.where.call_count == 2 + + def test_get_segments_ordering_by_position_and_id(self, mock_segment_service_dependencies): + """ + Test ordering by position and id. + + Verifies: + - Results are ordered by position ASC + - Results are secondarily ordered by id ASC to ensure uniqueness + - This prevents duplicate data across pages when positions are not unique + """ + # Arrange + document_id = "doc-123" + tenant_id = "tenant-123" + + # Create segments with same position but different ids + segment1 = SegmentServiceTestDataFactory.create_segment_mock( + segment_id="seg-1", position=1, content="Content 1" + ) + segment2 = SegmentServiceTestDataFactory.create_segment_mock( + segment_id="seg-2", position=1, content="Content 2" + ) + segment3 = SegmentServiceTestDataFactory.create_segment_mock( + segment_id="seg-3", position=2, content="Content 3" + ) + + mock_paginated = Mock() + mock_paginated.items = [segment1, segment2, segment3] + mock_paginated.total = 3 + + mock_segment_service_dependencies["db"].paginate.return_value = mock_paginated + + mock_query = Mock() + mock_segment_service_dependencies["select"].return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.order_by.return_value = mock_query + + # Act + from services.dataset_service import SegmentService + + items, total = SegmentService.get_segments(document_id=document_id, tenant_id=tenant_id) + + # Assert + assert len(items) == 3 + assert total == 3 + mock_query.order_by.assert_called_once() + + def test_get_segments_empty_results(self, mock_segment_service_dependencies): + """ + Test when no segments match the criteria. + + Verifies: + - Empty list is returned for items + - Total count is 0 + """ + # Arrange + document_id = "non-existent-doc" + tenant_id = "tenant-123" + + mock_paginated = Mock() + mock_paginated.items = [] + mock_paginated.total = 0 + + mock_segment_service_dependencies["db"].paginate.return_value = mock_paginated + + mock_query = Mock() + mock_segment_service_dependencies["select"].return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.order_by.return_value = mock_query + + # Act + from services.dataset_service import SegmentService + + items, total = SegmentService.get_segments(document_id=document_id, tenant_id=tenant_id) + + # Assert + assert items == [] + assert total == 0 + + def test_get_segments_combined_filters(self, mock_segment_service_dependencies): + """ + Test with multiple filters combined. + + Verifies: + - All filters work together correctly + - Status list and keyword search both applied + """ + # Arrange + document_id = "doc-123" + tenant_id = "tenant-123" + status_list = ["completed"] + keyword = "important" + page = 2 + limit = 10 + + segment = SegmentServiceTestDataFactory.create_segment_mock( + segment_id="seg-1", + status="completed", + content="This is important information", + ) + + mock_paginated = Mock() + mock_paginated.items = [segment] + mock_paginated.total = 1 + + mock_segment_service_dependencies["db"].paginate.return_value = mock_paginated + + mock_query = Mock() + mock_segment_service_dependencies["select"].return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.order_by.return_value = mock_query + + # Act + from services.dataset_service import SegmentService + + items, total = SegmentService.get_segments( + document_id=document_id, + tenant_id=tenant_id, + status_list=status_list, + keyword=keyword, + page=page, + limit=limit, + ) + + # Assert + assert len(items) == 1 + assert total == 1 + # Verify filters: base + status + keyword + assert mock_query.where.call_count == 3 + # Verify pagination parameters + call_kwargs = mock_segment_service_dependencies["db"].paginate.call_args[1] + assert call_kwargs["page"] == page + assert call_kwargs["per_page"] == limit + + def test_get_segments_with_none_status_list(self, mock_segment_service_dependencies): + """ + Test with None status list. + + Verifies: + - None status list is handled correctly + - No status filter is applied + """ + # Arrange + document_id = "doc-123" + tenant_id = "tenant-123" + + segment = SegmentServiceTestDataFactory.create_segment_mock(segment_id="seg-1") + + mock_paginated = Mock() + mock_paginated.items = [segment] + mock_paginated.total = 1 + + mock_segment_service_dependencies["db"].paginate.return_value = mock_paginated + + mock_query = Mock() + mock_segment_service_dependencies["select"].return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.order_by.return_value = mock_query + + # Act + from services.dataset_service import SegmentService + + items, total = SegmentService.get_segments( + document_id=document_id, + tenant_id=tenant_id, + status_list=None, + ) + + # Assert + assert len(items) == 1 + assert total == 1 + # Should only be called once (base filters only, no status filter) + assert mock_query.where.call_count == 1 + + def test_get_segments_pagination_max_per_page_limit(self, mock_segment_service_dependencies): + """ + Test that max_per_page is correctly set to 100. + + Verifies: + - max_per_page parameter is set to 100 + - This prevents excessive page sizes + """ + # Arrange + document_id = "doc-123" + tenant_id = "tenant-123" + limit = 200 # Request more than max_per_page + + mock_paginated = Mock() + mock_paginated.items = [] + mock_paginated.total = 0 + + mock_segment_service_dependencies["db"].paginate.return_value = mock_paginated + + mock_query = Mock() + mock_segment_service_dependencies["select"].return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.order_by.return_value = mock_query + + # Act + from services.dataset_service import SegmentService + + SegmentService.get_segments( + document_id=document_id, + tenant_id=tenant_id, + limit=limit, + ) + + # Assert + call_kwargs = mock_segment_service_dependencies["db"].paginate.call_args[1] + assert call_kwargs["max_per_page"] == 100 diff --git a/api/tests/unit_tests/tasks/test_document_indexing_sync_task.py b/api/tests/unit_tests/tasks/test_document_indexing_sync_task.py new file mode 100644 index 0000000000..374abe0368 --- /dev/null +++ b/api/tests/unit_tests/tasks/test_document_indexing_sync_task.py @@ -0,0 +1,520 @@ +""" +Unit tests for document indexing sync task. + +This module tests the document indexing sync task functionality including: +- Syncing Notion documents when updated +- Validating document and data source existence +- Credential validation and retrieval +- Cleaning old segments before re-indexing +- Error handling and edge cases +""" + +import uuid +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from core.indexing_runner import DocumentIsPausedError, IndexingRunner +from models.dataset import Dataset, Document, DocumentSegment +from tasks.document_indexing_sync_task import document_indexing_sync_task + +# ============================================================================ +# Fixtures +# ============================================================================ + + +@pytest.fixture +def tenant_id(): + """Generate a unique tenant ID for testing.""" + return str(uuid.uuid4()) + + +@pytest.fixture +def dataset_id(): + """Generate a unique dataset ID for testing.""" + return str(uuid.uuid4()) + + +@pytest.fixture +def document_id(): + """Generate a unique document ID for testing.""" + return str(uuid.uuid4()) + + +@pytest.fixture +def notion_workspace_id(): + """Generate a Notion workspace ID for testing.""" + return str(uuid.uuid4()) + + +@pytest.fixture +def notion_page_id(): + """Generate a Notion page ID for testing.""" + return str(uuid.uuid4()) + + +@pytest.fixture +def credential_id(): + """Generate a credential ID for testing.""" + return str(uuid.uuid4()) + + +@pytest.fixture +def mock_dataset(dataset_id, tenant_id): + """Create a mock Dataset object.""" + dataset = Mock(spec=Dataset) + dataset.id = dataset_id + dataset.tenant_id = tenant_id + dataset.indexing_technique = "high_quality" + dataset.embedding_model_provider = "openai" + dataset.embedding_model = "text-embedding-ada-002" + return dataset + + +@pytest.fixture +def mock_document(document_id, dataset_id, tenant_id, notion_workspace_id, notion_page_id, credential_id): + """Create a mock Document object with Notion data source.""" + doc = Mock(spec=Document) + doc.id = document_id + doc.dataset_id = dataset_id + doc.tenant_id = tenant_id + doc.data_source_type = "notion_import" + doc.indexing_status = "completed" + doc.error = None + doc.stopped_at = None + doc.processing_started_at = None + doc.doc_form = "text_model" + doc.data_source_info_dict = { + "notion_workspace_id": notion_workspace_id, + "notion_page_id": notion_page_id, + "type": "page", + "last_edited_time": "2024-01-01T00:00:00Z", + "credential_id": credential_id, + } + return doc + + +@pytest.fixture +def mock_document_segments(document_id): + """Create mock DocumentSegment objects.""" + segments = [] + for i in range(3): + segment = Mock(spec=DocumentSegment) + segment.id = str(uuid.uuid4()) + segment.document_id = document_id + segment.index_node_id = f"node-{document_id}-{i}" + segments.append(segment) + return segments + + +@pytest.fixture +def mock_db_session(): + """Mock database session.""" + with patch("tasks.document_indexing_sync_task.db.session") as mock_session: + mock_query = MagicMock() + mock_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + mock_session.scalars.return_value = MagicMock() + yield mock_session + + +@pytest.fixture +def mock_datasource_provider_service(): + """Mock DatasourceProviderService.""" + with patch("tasks.document_indexing_sync_task.DatasourceProviderService") as mock_service_class: + mock_service = MagicMock() + mock_service.get_datasource_credentials.return_value = {"integration_secret": "test_token"} + mock_service_class.return_value = mock_service + yield mock_service + + +@pytest.fixture +def mock_notion_extractor(): + """Mock NotionExtractor.""" + with patch("tasks.document_indexing_sync_task.NotionExtractor") as mock_extractor_class: + mock_extractor = MagicMock() + mock_extractor.get_notion_last_edited_time.return_value = "2024-01-02T00:00:00Z" # Updated time + mock_extractor_class.return_value = mock_extractor + yield mock_extractor + + +@pytest.fixture +def mock_index_processor_factory(): + """Mock IndexProcessorFactory.""" + with patch("tasks.document_indexing_sync_task.IndexProcessorFactory") as mock_factory: + mock_processor = MagicMock() + mock_processor.clean = Mock() + mock_factory.return_value.init_index_processor.return_value = mock_processor + yield mock_factory + + +@pytest.fixture +def mock_indexing_runner(): + """Mock IndexingRunner.""" + with patch("tasks.document_indexing_sync_task.IndexingRunner") as mock_runner_class: + mock_runner = MagicMock(spec=IndexingRunner) + mock_runner.run = Mock() + mock_runner_class.return_value = mock_runner + yield mock_runner + + +# ============================================================================ +# Tests for document_indexing_sync_task +# ============================================================================ + + +class TestDocumentIndexingSyncTask: + """Tests for the document_indexing_sync_task function.""" + + def test_document_not_found(self, mock_db_session, dataset_id, document_id): + """Test that task handles document not found gracefully.""" + # Arrange + mock_db_session.query.return_value.where.return_value.first.return_value = None + + # Act + document_indexing_sync_task(dataset_id, document_id) + + # Assert + mock_db_session.close.assert_called_once() + + def test_missing_notion_workspace_id(self, mock_db_session, mock_document, dataset_id, document_id): + """Test that task raises error when notion_workspace_id is missing.""" + # Arrange + mock_document.data_source_info_dict = {"notion_page_id": "page123", "type": "page"} + mock_db_session.query.return_value.where.return_value.first.return_value = mock_document + + # Act & Assert + with pytest.raises(ValueError, match="no notion page found"): + document_indexing_sync_task(dataset_id, document_id) + + def test_missing_notion_page_id(self, mock_db_session, mock_document, dataset_id, document_id): + """Test that task raises error when notion_page_id is missing.""" + # Arrange + mock_document.data_source_info_dict = {"notion_workspace_id": "ws123", "type": "page"} + mock_db_session.query.return_value.where.return_value.first.return_value = mock_document + + # Act & Assert + with pytest.raises(ValueError, match="no notion page found"): + document_indexing_sync_task(dataset_id, document_id) + + def test_empty_data_source_info(self, mock_db_session, mock_document, dataset_id, document_id): + """Test that task raises error when data_source_info is empty.""" + # Arrange + mock_document.data_source_info_dict = None + mock_db_session.query.return_value.where.return_value.first.return_value = mock_document + + # Act & Assert + with pytest.raises(ValueError, match="no notion page found"): + document_indexing_sync_task(dataset_id, document_id) + + def test_credential_not_found( + self, + mock_db_session, + mock_datasource_provider_service, + mock_document, + dataset_id, + document_id, + ): + """Test that task handles missing credentials by updating document status.""" + # Arrange + mock_db_session.query.return_value.where.return_value.first.return_value = mock_document + mock_datasource_provider_service.get_datasource_credentials.return_value = None + + # Act + document_indexing_sync_task(dataset_id, document_id) + + # Assert + assert mock_document.indexing_status == "error" + assert "Datasource credential not found" in mock_document.error + assert mock_document.stopped_at is not None + mock_db_session.commit.assert_called() + mock_db_session.close.assert_called() + + def test_page_not_updated( + self, + mock_db_session, + mock_datasource_provider_service, + mock_notion_extractor, + mock_document, + dataset_id, + document_id, + ): + """Test that task does nothing when page has not been updated.""" + # Arrange + mock_db_session.query.return_value.where.return_value.first.return_value = mock_document + # Return same time as stored in document + mock_notion_extractor.get_notion_last_edited_time.return_value = "2024-01-01T00:00:00Z" + + # Act + document_indexing_sync_task(dataset_id, document_id) + + # Assert + # Document status should remain unchanged + assert mock_document.indexing_status == "completed" + # No session operations should be performed beyond the initial query + mock_db_session.close.assert_not_called() + + def test_successful_sync_when_page_updated( + self, + mock_db_session, + mock_datasource_provider_service, + mock_notion_extractor, + mock_index_processor_factory, + mock_indexing_runner, + mock_dataset, + mock_document, + mock_document_segments, + dataset_id, + document_id, + ): + """Test successful sync flow when Notion page has been updated.""" + # Arrange + mock_db_session.query.return_value.where.return_value.first.side_effect = [mock_document, mock_dataset] + mock_db_session.scalars.return_value.all.return_value = mock_document_segments + # NotionExtractor returns updated time + mock_notion_extractor.get_notion_last_edited_time.return_value = "2024-01-02T00:00:00Z" + + # Act + document_indexing_sync_task(dataset_id, document_id) + + # Assert + # Verify document status was updated to parsing + assert mock_document.indexing_status == "parsing" + assert mock_document.processing_started_at is not None + + # Verify segments were cleaned + mock_processor = mock_index_processor_factory.return_value.init_index_processor.return_value + mock_processor.clean.assert_called_once() + + # Verify segments were deleted from database + for segment in mock_document_segments: + mock_db_session.delete.assert_any_call(segment) + + # Verify indexing runner was called + mock_indexing_runner.run.assert_called_once_with([mock_document]) + + # Verify session operations + assert mock_db_session.commit.called + mock_db_session.close.assert_called_once() + + def test_dataset_not_found_during_cleaning( + self, + mock_db_session, + mock_datasource_provider_service, + mock_notion_extractor, + mock_document, + dataset_id, + document_id, + ): + """Test that task handles dataset not found during cleaning phase.""" + # Arrange + mock_db_session.query.return_value.where.return_value.first.side_effect = [mock_document, None] + mock_notion_extractor.get_notion_last_edited_time.return_value = "2024-01-02T00:00:00Z" + + # Act + document_indexing_sync_task(dataset_id, document_id) + + # Assert + # Document should still be set to parsing + assert mock_document.indexing_status == "parsing" + # Session should be closed after error + mock_db_session.close.assert_called_once() + + def test_cleaning_error_continues_to_indexing( + self, + mock_db_session, + mock_datasource_provider_service, + mock_notion_extractor, + mock_index_processor_factory, + mock_indexing_runner, + mock_dataset, + mock_document, + dataset_id, + document_id, + ): + """Test that indexing continues even if cleaning fails.""" + # Arrange + mock_db_session.query.return_value.where.return_value.first.side_effect = [mock_document, mock_dataset] + mock_db_session.scalars.return_value.all.side_effect = Exception("Cleaning error") + mock_notion_extractor.get_notion_last_edited_time.return_value = "2024-01-02T00:00:00Z" + + # Act + document_indexing_sync_task(dataset_id, document_id) + + # Assert + # Indexing should still be attempted despite cleaning error + mock_indexing_runner.run.assert_called_once_with([mock_document]) + mock_db_session.close.assert_called_once() + + def test_indexing_runner_document_paused_error( + self, + mock_db_session, + mock_datasource_provider_service, + mock_notion_extractor, + mock_index_processor_factory, + mock_indexing_runner, + mock_dataset, + mock_document, + mock_document_segments, + dataset_id, + document_id, + ): + """Test that DocumentIsPausedError is handled gracefully.""" + # Arrange + mock_db_session.query.return_value.where.return_value.first.side_effect = [mock_document, mock_dataset] + mock_db_session.scalars.return_value.all.return_value = mock_document_segments + mock_notion_extractor.get_notion_last_edited_time.return_value = "2024-01-02T00:00:00Z" + mock_indexing_runner.run.side_effect = DocumentIsPausedError("Document paused") + + # Act + document_indexing_sync_task(dataset_id, document_id) + + # Assert + # Session should be closed after handling error + mock_db_session.close.assert_called_once() + + def test_indexing_runner_general_error( + self, + mock_db_session, + mock_datasource_provider_service, + mock_notion_extractor, + mock_index_processor_factory, + mock_indexing_runner, + mock_dataset, + mock_document, + mock_document_segments, + dataset_id, + document_id, + ): + """Test that general exceptions during indexing are handled.""" + # Arrange + mock_db_session.query.return_value.where.return_value.first.side_effect = [mock_document, mock_dataset] + mock_db_session.scalars.return_value.all.return_value = mock_document_segments + mock_notion_extractor.get_notion_last_edited_time.return_value = "2024-01-02T00:00:00Z" + mock_indexing_runner.run.side_effect = Exception("Indexing error") + + # Act + document_indexing_sync_task(dataset_id, document_id) + + # Assert + # Session should be closed after error + mock_db_session.close.assert_called_once() + + def test_notion_extractor_initialized_with_correct_params( + self, + mock_db_session, + mock_datasource_provider_service, + mock_notion_extractor, + mock_document, + dataset_id, + document_id, + notion_workspace_id, + notion_page_id, + ): + """Test that NotionExtractor is initialized with correct parameters.""" + # Arrange + mock_db_session.query.return_value.where.return_value.first.return_value = mock_document + mock_notion_extractor.get_notion_last_edited_time.return_value = "2024-01-01T00:00:00Z" # No update + + # Act + with patch("tasks.document_indexing_sync_task.NotionExtractor") as mock_extractor_class: + mock_extractor = MagicMock() + mock_extractor.get_notion_last_edited_time.return_value = "2024-01-01T00:00:00Z" + mock_extractor_class.return_value = mock_extractor + + document_indexing_sync_task(dataset_id, document_id) + + # Assert + mock_extractor_class.assert_called_once_with( + notion_workspace_id=notion_workspace_id, + notion_obj_id=notion_page_id, + notion_page_type="page", + notion_access_token="test_token", + tenant_id=mock_document.tenant_id, + ) + + def test_datasource_credentials_requested_correctly( + self, + mock_db_session, + mock_datasource_provider_service, + mock_notion_extractor, + mock_document, + dataset_id, + document_id, + credential_id, + ): + """Test that datasource credentials are requested with correct parameters.""" + # Arrange + mock_db_session.query.return_value.where.return_value.first.return_value = mock_document + mock_notion_extractor.get_notion_last_edited_time.return_value = "2024-01-01T00:00:00Z" + + # Act + document_indexing_sync_task(dataset_id, document_id) + + # Assert + mock_datasource_provider_service.get_datasource_credentials.assert_called_once_with( + tenant_id=mock_document.tenant_id, + credential_id=credential_id, + provider="notion_datasource", + plugin_id="langgenius/notion_datasource", + ) + + def test_credential_id_missing_uses_none( + self, + mock_db_session, + mock_datasource_provider_service, + mock_notion_extractor, + mock_document, + dataset_id, + document_id, + ): + """Test that task handles missing credential_id by passing None.""" + # Arrange + mock_document.data_source_info_dict = { + "notion_workspace_id": "ws123", + "notion_page_id": "page123", + "type": "page", + "last_edited_time": "2024-01-01T00:00:00Z", + } + mock_db_session.query.return_value.where.return_value.first.return_value = mock_document + mock_notion_extractor.get_notion_last_edited_time.return_value = "2024-01-01T00:00:00Z" + + # Act + document_indexing_sync_task(dataset_id, document_id) + + # Assert + mock_datasource_provider_service.get_datasource_credentials.assert_called_once_with( + tenant_id=mock_document.tenant_id, + credential_id=None, + provider="notion_datasource", + plugin_id="langgenius/notion_datasource", + ) + + def test_index_processor_clean_called_with_correct_params( + self, + mock_db_session, + mock_datasource_provider_service, + mock_notion_extractor, + mock_index_processor_factory, + mock_indexing_runner, + mock_dataset, + mock_document, + mock_document_segments, + dataset_id, + document_id, + ): + """Test that index processor clean is called with correct parameters.""" + # Arrange + mock_db_session.query.return_value.where.return_value.first.side_effect = [mock_document, mock_dataset] + mock_db_session.scalars.return_value.all.return_value = mock_document_segments + mock_notion_extractor.get_notion_last_edited_time.return_value = "2024-01-02T00:00:00Z" + + # Act + document_indexing_sync_task(dataset_id, document_id) + + # Assert + mock_processor = mock_index_processor_factory.return_value.init_index_processor.return_value + expected_node_ids = [seg.index_node_id for seg in mock_document_segments] + mock_processor.clean.assert_called_once_with( + mock_dataset, expected_node_ids, with_keywords=True, delete_child_chunks=True + ) diff --git a/api/tests/unit_tests/tools/test_mcp_tool.py b/api/tests/unit_tests/tools/test_mcp_tool.py new file mode 100644 index 0000000000..a527773e4e --- /dev/null +++ b/api/tests/unit_tests/tools/test_mcp_tool.py @@ -0,0 +1,122 @@ +import base64 +from unittest.mock import Mock, patch + +import pytest + +from core.mcp.types import ( + AudioContent, + BlobResourceContents, + CallToolResult, + EmbeddedResource, + ImageContent, + TextResourceContents, +) +from core.tools.__base.tool_runtime import ToolRuntime +from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_entities import ToolEntity, ToolIdentity, ToolInvokeMessage +from core.tools.mcp_tool.tool import MCPTool + + +def _make_mcp_tool(output_schema: dict | None = None) -> MCPTool: + identity = ToolIdentity( + author="test", + name="test_mcp_tool", + label=I18nObject(en_US="Test MCP Tool", zh_Hans="测试MCP工具"), + provider="test_provider", + ) + entity = ToolEntity(identity=identity, output_schema=output_schema or {}) + runtime = Mock(spec=ToolRuntime) + runtime.credentials = {} + return MCPTool( + entity=entity, + runtime=runtime, + tenant_id="test_tenant", + icon="", + server_url="https://server.invalid", + provider_id="provider_1", + headers={}, + ) + + +class TestMCPToolInvoke: + @pytest.mark.parametrize( + ("content_factory", "mime_type"), + [ + ( + lambda b64, mt: ImageContent(type="image", data=b64, mimeType=mt), + "image/png", + ), + ( + lambda b64, mt: AudioContent(type="audio", data=b64, mimeType=mt), + "audio/mpeg", + ), + ], + ) + def test_invoke_image_or_audio_yields_blob(self, content_factory, mime_type) -> None: + tool = _make_mcp_tool() + raw = b"\x00\x01test-bytes\x02" + b64 = base64.b64encode(raw).decode() + content = content_factory(b64, mime_type) + result = CallToolResult(content=[content]) + + with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): + messages = list(tool._invoke(user_id="test_user", tool_parameters={})) + + assert len(messages) == 1 + msg = messages[0] + assert msg.type == ToolInvokeMessage.MessageType.BLOB + assert isinstance(msg.message, ToolInvokeMessage.BlobMessage) + assert msg.message.blob == raw + assert msg.meta == {"mime_type": mime_type} + + def test_invoke_embedded_text_resource_yields_text(self) -> None: + tool = _make_mcp_tool() + text_resource = TextResourceContents(uri="file://test.txt", mimeType="text/plain", text="hello world") + content = EmbeddedResource(type="resource", resource=text_resource) + result = CallToolResult(content=[content]) + + with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): + messages = list(tool._invoke(user_id="test_user", tool_parameters={})) + + assert len(messages) == 1 + msg = messages[0] + assert msg.type == ToolInvokeMessage.MessageType.TEXT + assert isinstance(msg.message, ToolInvokeMessage.TextMessage) + assert msg.message.text == "hello world" + + @pytest.mark.parametrize( + ("mime_type", "expected_mime"), + [("application/pdf", "application/pdf"), (None, "application/octet-stream")], + ) + def test_invoke_embedded_blob_resource_yields_blob(self, mime_type, expected_mime) -> None: + tool = _make_mcp_tool() + raw = b"binary-data" + b64 = base64.b64encode(raw).decode() + blob_resource = BlobResourceContents(uri="file://doc.bin", mimeType=mime_type, blob=b64) + content = EmbeddedResource(type="resource", resource=blob_resource) + result = CallToolResult(content=[content]) + + with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): + messages = list(tool._invoke(user_id="test_user", tool_parameters={})) + + assert len(messages) == 1 + msg = messages[0] + assert msg.type == ToolInvokeMessage.MessageType.BLOB + assert isinstance(msg.message, ToolInvokeMessage.BlobMessage) + assert msg.message.blob == raw + assert msg.meta == {"mime_type": expected_mime} + + def test_invoke_yields_variables_when_structured_content_and_schema(self) -> None: + tool = _make_mcp_tool(output_schema={"type": "object"}) + result = CallToolResult(content=[], structuredContent={"a": 1, "b": "x"}) + + with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): + messages = list(tool._invoke(user_id="test_user", tool_parameters={})) + + # Expect two variable messages corresponding to keys a and b + assert len(messages) == 2 + var_msgs = [m for m in messages if isinstance(m.message, ToolInvokeMessage.VariableMessage)] + assert {m.message.variable_name for m in var_msgs} == {"a", "b"} + # Validate values + values = {m.message.variable_name: m.message.variable_value for m in var_msgs} + assert values == {"a": 1, "b": "x"} diff --git a/api/uv.lock b/api/uv.lock index 726abf6920..4ccd229eec 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -1368,7 +1368,7 @@ wheels = [ [[package]] name = "dify-api" -version = "1.11.1" +version = "1.11.2" source = { virtual = "." } dependencies = [ { name = "aliyun-log-python-sdk" }, @@ -1380,7 +1380,7 @@ dependencies = [ { name = "bs4" }, { name = "cachetools" }, { name = "celery" }, - { name = "chardet" }, + { name = "charset-normalizer" }, { name = "croniter" }, { name = "flask" }, { name = "flask-compress" }, @@ -1403,6 +1403,7 @@ dependencies = [ { name = "httpx-sse" }, { name = "jieba" }, { name = "json-repair" }, + { name = "jsonschema" }, { name = "langfuse" }, { name = "langsmith" }, { name = "litellm" }, @@ -1577,7 +1578,7 @@ requires-dist = [ { name = "bs4", specifier = "~=0.0.1" }, { name = "cachetools", specifier = "~=5.3.0" }, { name = "celery", specifier = "~=5.5.2" }, - { name = "chardet", specifier = "~=5.1.0" }, + { name = "charset-normalizer", specifier = ">=3.4.4" }, { name = "croniter", specifier = ">=6.0.0" }, { name = "flask", specifier = "~=3.1.2" }, { name = "flask-compress", specifier = ">=1.17,<1.18" }, @@ -1600,6 +1601,7 @@ requires-dist = [ { name = "httpx-sse", specifier = "~=0.4.0" }, { name = "jieba", specifier = "==0.42.1" }, { name = "json-repair", specifier = ">=0.41.1" }, + { name = "jsonschema", specifier = ">=4.25.1" }, { name = "langfuse", specifier = "~=2.51.3" }, { name = "langsmith", specifier = "~=0.1.77" }, { name = "litellm", specifier = "==1.77.1" }, @@ -1634,7 +1636,7 @@ requires-dist = [ { name = "pydantic-extra-types", specifier = "~=2.10.3" }, { name = "pydantic-settings", specifier = "~=2.11.0" }, { name = "pyjwt", specifier = "~=2.10.1" }, - { name = "pypdfium2", specifier = "==4.30.0" }, + { name = "pypdfium2", specifier = "==5.2.0" }, { name = "python-docx", specifier = "~=1.1.0" }, { name = "python-dotenv", specifier = "==1.0.1" }, { name = "pyyaml", specifier = "~=6.0.1" }, @@ -3070,11 +3072,11 @@ wheels = [ [[package]] name = "json-repair" -version = "0.54.1" +version = "0.54.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/46/d3a4d9a3dad39bb4a2ad16b8adb9fe2e8611b20b71197fe33daa6768e85d/json_repair-0.54.1.tar.gz", hash = "sha256:d010bc31f1fc66e7c36dc33bff5f8902674498ae5cb8e801ad455a53b455ad1d", size = 38555, upload-time = "2025-11-19T14:55:24.265Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/86/48b12ac02032f121ac7e5f11a32143edca6c1e3d19ffc54d6fb9ca0aafd0/json_repair-0.54.3.tar.gz", hash = "sha256:e50feec9725e52ac91f12184609754684ac1656119dfbd31de09bdaf9a1d8bf6", size = 38626, upload-time = "2025-12-15T09:41:58.594Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/96/c9aad7ee949cc1bf15df91f347fbc2d3bd10b30b80c7df689ce6fe9332b5/json_repair-0.54.1-py3-none-any.whl", hash = "sha256:016160c5db5d5fe443164927bb58d2dfbba5f43ad85719fa9bc51c713a443ab1", size = 29311, upload-time = "2025-11-19T14:55:22.886Z" }, + { url = "https://files.pythonhosted.org/packages/e9/08/abe317237add63c3e62f18a981bccf92112b431835b43d844aedaf61f4a0/json_repair-0.54.3-py3-none-any.whl", hash = "sha256:4cdc132ee27d4780576f71bf27a113877046224a808bfc17392e079cb344fb81", size = 29357, upload-time = "2025-12-15T09:41:57.436Z" }, ] [[package]] @@ -4991,22 +4993,31 @@ wheels = [ [[package]] name = "pypdfium2" -version = "4.30.0" +version = "5.2.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/14/838b3ba247a0ba92e4df5d23f2bea9478edcfd72b78a39d6ca36ccd84ad2/pypdfium2-4.30.0.tar.gz", hash = "sha256:48b5b7e5566665bc1015b9d69c1ebabe21f6aee468b509531c3c8318eeee2e16", size = 140239, upload-time = "2024-05-09T18:33:17.552Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/ab/73c7d24e4eac9ba952569403b32b7cca9412fc5b9bef54fdbd669551389f/pypdfium2-5.2.0.tar.gz", hash = "sha256:43863625231ce999c1ebbed6721a88de818b2ab4d909c1de558d413b9a400256", size = 269999, upload-time = "2025-12-12T13:20:15.353Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/9a/c8ff5cc352c1b60b0b97642ae734f51edbab6e28b45b4fcdfe5306ee3c83/pypdfium2-4.30.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:b33ceded0b6ff5b2b93bc1fe0ad4b71aa6b7e7bd5875f1ca0cdfb6ba6ac01aab", size = 2837254, upload-time = "2024-05-09T18:32:48.653Z" }, - { url = "https://files.pythonhosted.org/packages/21/8b/27d4d5409f3c76b985f4ee4afe147b606594411e15ac4dc1c3363c9a9810/pypdfium2-4.30.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:4e55689f4b06e2d2406203e771f78789bd4f190731b5d57383d05cf611d829de", size = 2707624, upload-time = "2024-05-09T18:32:51.458Z" }, - { url = "https://files.pythonhosted.org/packages/11/63/28a73ca17c24b41a205d658e177d68e198d7dde65a8c99c821d231b6ee3d/pypdfium2-4.30.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e6e50f5ce7f65a40a33d7c9edc39f23140c57e37144c2d6d9e9262a2a854854", size = 2793126, upload-time = "2024-05-09T18:32:53.581Z" }, - { url = "https://files.pythonhosted.org/packages/d1/96/53b3ebf0955edbd02ac6da16a818ecc65c939e98fdeb4e0958362bd385c8/pypdfium2-4.30.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3d0dd3ecaffd0b6dbda3da663220e705cb563918249bda26058c6036752ba3a2", size = 2591077, upload-time = "2024-05-09T18:32:55.99Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ee/0394e56e7cab8b5b21f744d988400948ef71a9a892cbeb0b200d324ab2c7/pypdfium2-4.30.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc3bf29b0db8c76cdfaac1ec1cde8edf211a7de7390fbf8934ad2aa9b4d6dfad", size = 2864431, upload-time = "2024-05-09T18:32:57.911Z" }, - { url = "https://files.pythonhosted.org/packages/65/cd/3f1edf20a0ef4a212a5e20a5900e64942c5a374473671ac0780eaa08ea80/pypdfium2-4.30.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1f78d2189e0ddf9ac2b7a9b9bd4f0c66f54d1389ff6c17e9fd9dc034d06eb3f", size = 2812008, upload-time = "2024-05-09T18:32:59.886Z" }, - { url = "https://files.pythonhosted.org/packages/c8/91/2d517db61845698f41a2a974de90762e50faeb529201c6b3574935969045/pypdfium2-4.30.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:5eda3641a2da7a7a0b2f4dbd71d706401a656fea521b6b6faa0675b15d31a163", size = 6181543, upload-time = "2024-05-09T18:33:02.597Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c4/ed1315143a7a84b2c7616569dfb472473968d628f17c231c39e29ae9d780/pypdfium2-4.30.0-py3-none-musllinux_1_1_i686.whl", hash = "sha256:0dfa61421b5eb68e1188b0b2231e7ba35735aef2d867d86e48ee6cab6975195e", size = 6175911, upload-time = "2024-05-09T18:33:05.376Z" }, - { url = "https://files.pythonhosted.org/packages/7a/c4/9e62d03f414e0e3051c56d5943c3bf42aa9608ede4e19dc96438364e9e03/pypdfium2-4.30.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:f33bd79e7a09d5f7acca3b0b69ff6c8a488869a7fab48fdf400fec6e20b9c8be", size = 6267430, upload-time = "2024-05-09T18:33:08.067Z" }, - { url = "https://files.pythonhosted.org/packages/90/47/eda4904f715fb98561e34012826e883816945934a851745570521ec89520/pypdfium2-4.30.0-py3-none-win32.whl", hash = "sha256:ee2410f15d576d976c2ab2558c93d392a25fb9f6635e8dd0a8a3a5241b275e0e", size = 2775951, upload-time = "2024-05-09T18:33:10.567Z" }, - { url = "https://files.pythonhosted.org/packages/25/bd/56d9ec6b9f0fc4e0d95288759f3179f0fcd34b1a1526b75673d2f6d5196f/pypdfium2-4.30.0-py3-none-win_amd64.whl", hash = "sha256:90dbb2ac07be53219f56be09961eb95cf2473f834d01a42d901d13ccfad64b4c", size = 2892098, upload-time = "2024-05-09T18:33:13.107Z" }, - { url = "https://files.pythonhosted.org/packages/be/7a/097801205b991bc3115e8af1edb850d30aeaf0118520b016354cf5ccd3f6/pypdfium2-4.30.0-py3-none-win_arm64.whl", hash = "sha256:119b2969a6d6b1e8d55e99caaf05290294f2d0fe49c12a3f17102d01c441bd29", size = 2752118, upload-time = "2024-05-09T18:33:15.489Z" }, + { url = "https://files.pythonhosted.org/packages/fb/0c/9108ae5266ee4cdf495f99205c44d4b5c83b4eb227c2b610d35c9e9fe961/pypdfium2-5.2.0-py3-none-android_23_arm64_v8a.whl", hash = "sha256:1ba4187a45ce4cf08f2a8c7e0f8970c36b9aa1770c8a3412a70781c1d80fb145", size = 2763268, upload-time = "2025-12-12T13:19:37.354Z" }, + { url = "https://files.pythonhosted.org/packages/35/8c/55f5c8a2c6b293f5c020be4aa123eaa891e797c514e5eccd8cb042740d37/pypdfium2-5.2.0-py3-none-android_23_armeabi_v7a.whl", hash = "sha256:80c55e10a8c9242f0901d35a9a306dd09accce8e497507bb23fcec017d45fe2e", size = 2301821, upload-time = "2025-12-12T13:19:39.484Z" }, + { url = "https://files.pythonhosted.org/packages/5e/7d/efa013e3795b41c59dd1e472f7201c241232c3a6553be4917e3a26b9f225/pypdfium2-5.2.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:73523ae69cd95c084c1342096893b2143ea73c36fdde35494780ba431e6a7d6e", size = 2816428, upload-time = "2025-12-12T13:19:41.735Z" }, + { url = "https://files.pythonhosted.org/packages/ec/ae/8c30af6ff2ab41a7cb84753ee79dd1e0a8932c9bda9fe19759d69cbbf115/pypdfium2-5.2.0-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:19c501d22ef5eb98e42416d22cc3ac66d4808b436e3d06686392f24d8d9f708d", size = 2939486, upload-time = "2025-12-12T13:19:43.176Z" }, + { url = "https://files.pythonhosted.org/packages/64/64/454a73c49a04c2c290917ad86184e4da959e9e5aba94b3b046328c89be93/pypdfium2-5.2.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ed15a3f58d6ee4905f0d0a731e30b381b457c30689512589c7f57950b0cdcec", size = 2979235, upload-time = "2025-12-12T13:19:44.635Z" }, + { url = "https://files.pythonhosted.org/packages/4e/29/f1cab8e31192dd367dc7b1afa71f45cfcb8ff0b176f1d2a0f528faf04052/pypdfium2-5.2.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:329cd1e9f068e8729e0d0b79a070d6126f52bc48ff1e40505cb207a5e20ce0ba", size = 2763001, upload-time = "2025-12-12T13:19:47.598Z" }, + { url = "https://files.pythonhosted.org/packages/bc/5d/e95fad8fdac960854173469c4b6931d5de5e09d05e6ee7d9756f8b95eef0/pypdfium2-5.2.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:325259759886e66619504df4721fef3b8deabf8a233e4f4a66e0c32ebae60c2f", size = 3057024, upload-time = "2025-12-12T13:19:49.179Z" }, + { url = "https://files.pythonhosted.org/packages/f4/32/468591d017ab67f8142d40f4db8163b6d8bb404fe0d22da75a5c661dc144/pypdfium2-5.2.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5683e8f08ab38ed05e0e59e611451ec74332803d4e78f8c45658ea1d372a17af", size = 3448598, upload-time = "2025-12-12T13:19:50.979Z" }, + { url = "https://files.pythonhosted.org/packages/f9/a5/57b4e389b77ab5f7e9361dc7fc03b5378e678ba81b21e791e85350fbb235/pypdfium2-5.2.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da4815426a5adcf03bf4d2c5f26c0ff8109dbfaf2c3415984689931bc6006ef9", size = 2993946, upload-time = "2025-12-12T13:19:53.154Z" }, + { url = "https://files.pythonhosted.org/packages/84/3a/e03e9978f817632aa56183bb7a4989284086fdd45de3245ead35f147179b/pypdfium2-5.2.0-py3-none-manylinux_2_27_s390x.manylinux_2_28_s390x.whl", hash = "sha256:64bf5c039b2c314dab1fd158bfff99db96299a5b5c6d96fc056071166056f1de", size = 3673148, upload-time = "2025-12-12T13:19:54.528Z" }, + { url = "https://files.pythonhosted.org/packages/13/ee/e581506806553afa4b7939d47bf50dca35c1151b8cc960f4542a6eb135ce/pypdfium2-5.2.0-py3-none-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:76b42a17748ac7dc04d5ef04d0561c6a0a4b546d113ec1d101d59650c6a340f7", size = 2964757, upload-time = "2025-12-12T13:19:56.406Z" }, + { url = "https://files.pythonhosted.org/packages/00/be/3715c652aff30f12284523dd337843d0efe3e721020f0ec303a99ffffd8d/pypdfium2-5.2.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9d4367d471439fae846f0aba91ff9e8d66e524edcf3c8d6e02fe96fa306e13b9", size = 4130319, upload-time = "2025-12-12T13:19:57.889Z" }, + { url = "https://files.pythonhosted.org/packages/b0/0b/28aa2ede9004dd4192266bbad394df0896787f7c7bcfa4d1a6e091ad9a2c/pypdfium2-5.2.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:613f6bb2b47d76b66c0bf2ca581c7c33e3dd9dcb29d65d8c34fef4135f933149", size = 3746488, upload-time = "2025-12-12T13:19:59.469Z" }, + { url = "https://files.pythonhosted.org/packages/bc/04/1b791e1219652bbfc51df6498267d8dcec73ad508b99388b2890902ccd9d/pypdfium2-5.2.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c03fad3f2fa68d358f5dd4deb07e438482fa26fae439c49d127576d969769ca1", size = 4336534, upload-time = "2025-12-12T13:20:01.28Z" }, + { url = "https://files.pythonhosted.org/packages/4f/e3/6f00f963bb702ffd2e3e2d9c7286bc3bb0bebcdfa96ca897d466f66976c6/pypdfium2-5.2.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:f10be1900ae21879d02d9f4d58c2d2db3a2e6da611736a8e9decc22d1fb02909", size = 4375079, upload-time = "2025-12-12T13:20:03.117Z" }, + { url = "https://files.pythonhosted.org/packages/3a/2a/7ec2b191b5e1b7716a0dfc14e6860e89bb355fb3b94ed0c1d46db526858c/pypdfium2-5.2.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:97c1a126d30378726872f94866e38c055740cae80313638dafd1cd448d05e7c0", size = 3928648, upload-time = "2025-12-12T13:20:05.041Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c3/c6d972fa095ff3ace76f9d3a91ceaf8a9dbbe0d9a5a84ac1d6178a46630e/pypdfium2-5.2.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:c369f183a90781b788af9a357a877bc8caddc24801e8346d0bf23f3295f89f3a", size = 4997772, upload-time = "2025-12-12T13:20:06.453Z" }, + { url = "https://files.pythonhosted.org/packages/22/45/2c64584b7a3ca5c4652280a884f4b85b8ed24e27662adeebdc06d991c917/pypdfium2-5.2.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b391f1cceb454934b612a05b54e90f98aafeffe5e73830d71700b17f0812226b", size = 4180046, upload-time = "2025-12-12T13:20:08.715Z" }, + { url = "https://files.pythonhosted.org/packages/d6/99/8d1ff87b626649400e62a2840e6e10fe258443ba518798e071fee4cd86f9/pypdfium2-5.2.0-py3-none-win32.whl", hash = "sha256:c68067938f617c37e4d17b18de7cac231fc7ce0eb7b6653b7283ebe8764d4999", size = 2990175, upload-time = "2025-12-12T13:20:10.241Z" }, + { url = "https://files.pythonhosted.org/packages/93/fc/114fff8895b620aac4984808e93d01b6d7b93e342a1635fcfe2a5f39cf39/pypdfium2-5.2.0-py3-none-win_amd64.whl", hash = "sha256:eb0591b720e8aaeab9475c66d653655ec1be0464b946f3f48a53922e843f0f3b", size = 3098615, upload-time = "2025-12-12T13:20:11.795Z" }, + { url = "https://files.pythonhosted.org/packages/08/97/eb738bff5998760d6e0cbcb7dd04cbf1a95a97b997fac6d4e57562a58992/pypdfium2-5.2.0-py3-none-win_arm64.whl", hash = "sha256:5dd1ef579f19fa3719aee4959b28bda44b1072405756708b5e83df8806a19521", size = 2939479, upload-time = "2025-12-12T13:20:13.815Z" }, ] [[package]] diff --git a/dev/start-worker b/dev/start-worker index a01da11d86..7876620188 100755 --- a/dev/start-worker +++ b/dev/start-worker @@ -37,6 +37,7 @@ show_help() { echo " pipeline - Standard pipeline tasks" echo " triggered_workflow_dispatcher - Trigger dispatcher tasks" echo " trigger_refresh_executor - Trigger refresh tasks" + echo " retention - Retention tasks" } # Parse command line arguments @@ -105,10 +106,10 @@ if [[ -z "${QUEUES}" ]]; then # Configure queues based on edition if [[ "${EDITION}" == "CLOUD" ]]; then # Cloud edition: separate queues for dataset and trigger tasks - QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor" + QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention" else # Community edition (SELF_HOSTED): dataset and workflow have separate queues - QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor" + QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention" fi echo "No queues specified, using edition-based defaults: ${QUEUES}" diff --git a/docker/.env.example b/docker/.env.example index dd0d083da3..0e09d6869d 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -399,6 +399,7 @@ CONSOLE_CORS_ALLOW_ORIGINS=* COOKIE_DOMAIN= # When the frontend and backend run on different subdomains, set NEXT_PUBLIC_COOKIE_DOMAIN=1. NEXT_PUBLIC_COOKIE_DOMAIN= +NEXT_PUBLIC_BATCH_CONCURRENCY=5 # ------------------------------ # File Storage Configuration @@ -468,6 +469,7 @@ ALIYUN_OSS_REGION=ap-southeast-1 ALIYUN_OSS_AUTH_VERSION=v4 # Don't start with '/'. OSS doesn't support leading slash in object names. ALIYUN_OSS_PATH=your-path +ALIYUN_CLOUDBOX_ID=your-cloudbox-id # Tencent COS Configuration # @@ -476,6 +478,7 @@ TENCENT_COS_SECRET_KEY=your-secret-key TENCENT_COS_SECRET_ID=your-secret-id TENCENT_COS_REGION=your-region TENCENT_COS_SCHEME=your-scheme +TENCENT_COS_CUSTOM_DOMAIN=your-custom-domain # Oracle Storage Configuration # @@ -491,6 +494,7 @@ HUAWEI_OBS_BUCKET_NAME=your-bucket-name HUAWEI_OBS_SECRET_KEY=your-secret-key HUAWEI_OBS_ACCESS_KEY=your-access-key HUAWEI_OBS_SERVER=your-server-url +HUAWEI_OBS_PATH_STYLE=false # Volcengine TOS Configuration # @@ -1369,7 +1373,10 @@ PLUGIN_STDIO_BUFFER_SIZE=1024 PLUGIN_STDIO_MAX_BUFFER_SIZE=5242880 PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120 +# Plugin Daemon side timeout (configure to match the API side below) PLUGIN_MAX_EXECUTION_TIMEOUT=600 +# API side timeout (configure to match the Plugin Daemon side above) +PLUGIN_DAEMON_TIMEOUT=600.0 # PIP_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple PIP_MIRROR_URL= @@ -1479,4 +1486,9 @@ ANNOTATION_IMPORT_RATE_LIMIT_PER_HOUR=20 ANNOTATION_IMPORT_MAX_CONCURRENT=5 # The API key of amplitude -AMPLITUDE_API_KEY= \ No newline at end of file +AMPLITUDE_API_KEY= + +# Sandbox expired records clean configuration +SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD=21 +SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE=1000 +SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS=30 diff --git a/docker/README.md b/docker/README.md index 375570f106..4c40317f37 100644 --- a/docker/README.md +++ b/docker/README.md @@ -23,6 +23,10 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T - Navigate to the `docker` directory. - Copy the `.env.example` file to a new file named `.env` by running `cp .env.example .env`. - Customize the `.env` file as needed. Refer to the `.env.example` file for detailed configuration options. + - **Optional (Recommended for upgrades)**: + You may use the environment synchronization tool to help keep your `.env` file aligned with the latest `.env.example` updates, while preserving your custom settings. + This is especially useful when upgrading Dify or managing a large, customized `.env` file. + See the [Environment Variables Synchronization](#environment-variables-synchronization) section below. 1. **Running the Services**: - Execute `docker compose up` from the `docker` directory to start the services. - To specify a vector database, set the `VECTOR_STORE` variable in your `.env` file to your desired vector database service, such as `milvus`, `weaviate`, or `opensearch`. @@ -111,6 +115,47 @@ The `.env.example` file provided in the Docker setup is extensive and covers a w - Each service like `nginx`, `redis`, `db`, and vector databases have specific environment variables that are directly referenced in the `docker-compose.yaml`. +### Environment Variables Synchronization + +When upgrading Dify or pulling the latest changes, new environment variables may be introduced in `.env.example`. + +To help keep your existing `.env` file up to date **without losing your custom values**, an optional environment variables synchronization tool is provided. + +> This tool performs a **one-way synchronization** from `.env.example` to `.env`. +> Existing values in `.env` are never overwritten automatically. + +#### `dify-env-sync.sh` (Optional) + +This script compares your current `.env` file with the latest `.env.example` template and helps safely apply new or updated environment variables. + +**What it does** + +- Creates a backup of the current `.env` file before making any changes +- Synchronizes newly added environment variables from `.env.example` +- Preserves all existing custom values in `.env` +- Displays differences and variables removed from `.env.example` for review + +**Backup behavior** + +Before synchronization, the current `.env` file is saved to the `env-backup/` directory with a timestamped filename +(e.g. `env-backup/.env.backup_20231218_143022`). + +**When to use** + +- After upgrading Dify to a newer version +- When `.env.example` has been updated with new environment variables +- When managing a large or heavily customized `.env` file + +**Usage** + +```bash +# Grant execution permission (first time only) +chmod +x dify-env-sync.sh + +# Run the synchronization +./dify-env-sync.sh +``` + ### Additional Information - **Continuous Improvement Phase**: We are actively seeking feedback from the community to refine and enhance the deployment process. As more users adopt this new method, we will continue to make improvements based on your experiences and suggestions. diff --git a/docker/dify-env-sync.sh b/docker/dify-env-sync.sh new file mode 100755 index 0000000000..edeeae3abc --- /dev/null +++ b/docker/dify-env-sync.sh @@ -0,0 +1,465 @@ +#!/bin/bash + +# ================================================================ +# Dify Environment Variables Synchronization Script +# +# Features: +# - Synchronize latest settings from .env.example to .env +# - Preserve custom settings in existing .env +# - Add new environment variables +# - Detect removed environment variables +# - Create backup files +# ================================================================ + +set -eo pipefail # Exit on error and pipe failures (safer for complex variable handling) + +# Error handling function +# Arguments: +# $1 - Line number where error occurred +# $2 - Error code +handle_error() { + local line_no=$1 + local error_code=$2 + echo -e "\033[0;31m[ERROR]\033[0m Script error: line $line_no with error code $error_code" >&2 + echo -e "\033[0;31m[ERROR]\033[0m Debug info: current working directory $(pwd)" >&2 + exit $error_code +} + +# Set error trap +trap 'handle_error ${LINENO} $?' ERR + +# Color settings for output +readonly RED='\033[0;31m' +readonly GREEN='\033[0;32m' +readonly YELLOW='\033[1;33m' +readonly BLUE='\033[0;34m' +readonly NC='\033[0m' # No Color + +# Logging functions +# Print informational message in blue +# Arguments: $1 - Message to print +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +# Print success message in green +# Arguments: $1 - Message to print +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +# Print warning message in yellow +# Arguments: $1 - Message to print +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" >&2 +} + +# Print error message in red to stderr +# Arguments: $1 - Message to print +log_error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +# Check for required files and create .env if missing +# Verifies that .env.example exists and creates .env from template if needed +check_files() { + log_info "Checking required files..." + + if [[ ! -f ".env.example" ]]; then + log_error ".env.example file not found" + exit 1 + fi + + if [[ ! -f ".env" ]]; then + log_warning ".env file does not exist. Creating from .env.example." + cp ".env.example" ".env" + log_success ".env file created" + fi + + log_success "Required files verified" +} + +# Create timestamped backup of .env file +# Creates env-backup directory if needed and backs up current .env file +create_backup() { + local timestamp=$(date +"%Y%m%d_%H%M%S") + local backup_dir="env-backup" + + # Create backup directory if it doesn't exist + if [[ ! -d "$backup_dir" ]]; then + mkdir -p "$backup_dir" + log_info "Created backup directory: $backup_dir" + fi + + if [[ -f ".env" ]]; then + local backup_file="${backup_dir}/.env.backup_${timestamp}" + cp ".env" "$backup_file" + log_success "Backed up existing .env to $backup_file" + fi +} + +# Detect differences between .env and .env.example (optimized for large files) +detect_differences() { + log_info "Detecting differences between .env and .env.example..." + + # Create secure temporary directory + local temp_dir=$(mktemp -d) + local temp_diff="$temp_dir/env_diff" + + # Store diff file path as global variable + declare -g DIFF_FILE="$temp_diff" + declare -g TEMP_DIR="$temp_dir" + + # Initialize difference file + > "$temp_diff" + + # Use awk for efficient comparison (much faster for large files) + local diff_count=$(awk -F= ' + BEGIN { OFS="\x01" } + FNR==NR { + if (!/^[[:space:]]*#/ && !/^[[:space:]]*$/ && /=/) { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", $1) + key = $1 + value = substr($0, index($0,"=")+1) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + env_values[key] = value + } + next + } + { + if (!/^[[:space:]]*#/ && !/^[[:space:]]*$/ && /=/) { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", $1) + key = $1 + example_value = substr($0, index($0,"=")+1) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", example_value) + + if (key in env_values && env_values[key] != example_value) { + print key, env_values[key], example_value > "'$temp_diff'" + diff_count++ + } + } + } + END { print diff_count } + ' .env .env.example) + + if [[ $diff_count -gt 0 ]]; then + log_success "Detected differences in $diff_count environment variables" + # Show detailed differences + show_differences_detail + else + log_info "No differences detected" + fi +} + +# Parse environment variable line +# Extracts key-value pairs from .env file format lines +# Arguments: +# $1 - Line to parse +# Returns: +# 0 - Success, outputs "key|value" format +# 1 - Skip (empty line, comment, or invalid format) +parse_env_line() { + local line="$1" + local key="" + local value="" + + # Skip empty lines or comment lines + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && return 1 + + # Split by = + if [[ "$line" =~ ^([^=]+)=(.*)$ ]]; then + key="${BASH_REMATCH[1]}" + value="${BASH_REMATCH[2]}" + + # Remove leading and trailing whitespace + key=$(echo "$key" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + value=$(echo "$value" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + + if [[ -n "$key" ]]; then + echo "$key|$value" + return 0 + fi + fi + + return 1 +} + +# Show detailed differences +show_differences_detail() { + log_info "" + log_info "=== Environment Variable Differences ===" + + # Read differences from the already created diff file + if [[ ! -s "$DIFF_FILE" ]]; then + log_info "No differences to display" + return + fi + + # Display differences + local count=1 + while IFS=$'\x01' read -r key env_value example_value; do + echo "" + echo -e "${YELLOW}[$count] $key${NC}" + echo -e " ${GREEN}.env (current)${NC} : ${env_value}" + echo -e " ${BLUE}.env.example (recommended)${NC}: ${example_value}" + + # Analyze value changes + analyze_value_change "$env_value" "$example_value" + ((count++)) + done < "$DIFF_FILE" + + echo "" + log_info "=== Difference Analysis Complete ===" + log_info "Note: Consider changing to the recommended values above." + log_info "Current implementation preserves .env values." + echo "" +} + +# Analyze value changes +analyze_value_change() { + local current_value="$1" + local recommended_value="$2" + + # Analyze value characteristics + local analysis="" + + # Empty value check + if [[ -z "$current_value" && -n "$recommended_value" ]]; then + analysis=" ${RED}→ Setting from empty to recommended value${NC}" + elif [[ -n "$current_value" && -z "$recommended_value" ]]; then + analysis=" ${RED}→ Recommended value changed to empty${NC}" + # Numeric check - using arithmetic evaluation for robust comparison + elif [[ "$current_value" =~ ^[0-9]+$ && "$recommended_value" =~ ^[0-9]+$ ]]; then + # Use arithmetic evaluation to handle leading zeros correctly + if (( 10#$current_value < 10#$recommended_value )); then + analysis=" ${BLUE}→ Numeric increase (${current_value} < ${recommended_value})${NC}" + elif (( 10#$current_value > 10#$recommended_value )); then + analysis=" ${YELLOW}→ Numeric decrease (${current_value} > ${recommended_value})${NC}" + fi + # Boolean check + elif [[ "$current_value" =~ ^(true|false)$ && "$recommended_value" =~ ^(true|false)$ ]]; then + if [[ "$current_value" != "$recommended_value" ]]; then + analysis=" ${BLUE}→ Boolean value change (${current_value} → ${recommended_value})${NC}" + fi + # URL/endpoint check + elif [[ "$current_value" =~ ^https?:// || "$recommended_value" =~ ^https?:// ]]; then + analysis=" ${BLUE}→ URL/endpoint change${NC}" + # File path check + elif [[ "$current_value" =~ ^/ || "$recommended_value" =~ ^/ ]]; then + analysis=" ${BLUE}→ File path change${NC}" + else + # Length comparison + local current_len=${#current_value} + local recommended_len=${#recommended_value} + if [[ $current_len -ne $recommended_len ]]; then + analysis=" ${YELLOW}→ String length change (${current_len} → ${recommended_len} characters)${NC}" + fi + fi + + if [[ -n "$analysis" ]]; then + echo -e "$analysis" + fi +} + +# Synchronize .env file with .env.example while preserving custom values +# Creates a new .env file based on .env.example structure, preserving existing custom values +# Global variables used: DIFF_FILE, TEMP_DIR +sync_env_file() { + log_info "Starting partial synchronization of .env file..." + + local new_env_file=".env.new" + local preserved_count=0 + local updated_count=0 + + # Pre-process diff file for efficient lookup + local lookup_file="" + if [[ -f "$DIFF_FILE" && -s "$DIFF_FILE" ]]; then + lookup_file="${DIFF_FILE}.lookup" + # Create sorted lookup file for fast search + sort "$DIFF_FILE" > "$lookup_file" + log_info "Created lookup file for $(wc -l < "$DIFF_FILE") preserved values" + fi + + # Use AWK for efficient processing (much faster than bash loop for large files) + log_info "Processing $(wc -l < .env.example) lines with AWK..." + + local preserved_keys_file="${TEMP_DIR}/preserved_keys" + local awk_preserved_count_file="${TEMP_DIR}/awk_preserved_count" + local awk_updated_count_file="${TEMP_DIR}/awk_updated_count" + + awk -F'=' -v lookup_file="$lookup_file" -v preserved_file="$preserved_keys_file" \ + -v preserved_count_file="$awk_preserved_count_file" -v updated_count_file="$awk_updated_count_file" ' + BEGIN { + preserved_count = 0 + updated_count = 0 + + # Load preserved values if lookup file exists + if (lookup_file != "") { + while ((getline line < lookup_file) > 0) { + split(line, parts, "\x01") + key = parts[1] + value = parts[2] + preserved_values[key] = value + } + close(lookup_file) + } + } + + # Process each line + { + # Check if this is an environment variable line + if (/^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*=/) { + # Extract key + key = $1 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", key) + + # Check if key should be preserved + if (key in preserved_values) { + print key "=" preserved_values[key] + print key > preserved_file + preserved_count++ + } else { + print $0 + updated_count++ + } + } else { + # Not an env var line, preserve as-is + print $0 + } + } + + END { + print preserved_count > preserved_count_file + print updated_count > updated_count_file + } + ' .env.example > "$new_env_file" + + # Read counters and preserved keys + if [[ -f "$awk_preserved_count_file" ]]; then + preserved_count=$(cat "$awk_preserved_count_file") + fi + if [[ -f "$awk_updated_count_file" ]]; then + updated_count=$(cat "$awk_updated_count_file") + fi + + # Show what was preserved + if [[ -f "$preserved_keys_file" ]]; then + while read -r key; do + [[ -n "$key" ]] && log_info " Preserved: $key (.env value)" + done < "$preserved_keys_file" + fi + + # Clean up lookup file + [[ -n "$lookup_file" ]] && rm -f "$lookup_file" + + # Replace the original .env file + if mv "$new_env_file" ".env"; then + log_success "Successfully created new .env file" + else + log_error "Failed to replace .env file" + rm -f "$new_env_file" + return 1 + fi + + # Clean up difference file and temporary directory + if [[ -n "${TEMP_DIR:-}" ]]; then + rm -rf "${TEMP_DIR}" + unset TEMP_DIR + fi + if [[ -n "${DIFF_FILE:-}" ]]; then + unset DIFF_FILE + fi + + log_success "Partial synchronization of .env file completed" + log_info " Preserved .env values: $preserved_count" + log_info " Updated to .env.example values: $updated_count" +} + +# Detect removed environment variables +detect_removed_variables() { + log_info "Detecting removed environment variables..." + + if [[ ! -f ".env" ]]; then + return + fi + + # Use temporary files for efficient lookup + local temp_dir="${TEMP_DIR:-$(mktemp -d)}" + local temp_example_keys="$temp_dir/example_keys" + local temp_current_keys="$temp_dir/current_keys" + local cleanup_temp_dir="" + + # Set flag if we created a new temp directory + if [[ -z "${TEMP_DIR:-}" ]]; then + cleanup_temp_dir="$temp_dir" + fi + + # Get keys from .env.example and .env, sorted for comm + awk -F= '!/^[[:space:]]*#/ && /=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $1); print $1}' .env.example | sort > "$temp_example_keys" + awk -F= '!/^[[:space:]]*#/ && /=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $1); print $1}' .env | sort > "$temp_current_keys" + + # Get keys from existing .env and check for removals + local removed_vars=() + while IFS= read -r var; do + removed_vars+=("$var") + done < <(comm -13 "$temp_example_keys" "$temp_current_keys") + + # Clean up temporary files if we created a new temp directory + if [[ -n "$cleanup_temp_dir" ]]; then + rm -rf "$cleanup_temp_dir" + fi + + if [[ ${#removed_vars[@]} -gt 0 ]]; then + log_warning "The following environment variables have been removed from .env.example:" + for var in "${removed_vars[@]}"; do + log_warning " - $var" + done + log_warning "Consider manually removing these variables from .env" + else + log_success "No removed environment variables found" + fi +} + +# Show statistics +show_statistics() { + log_info "Synchronization statistics:" + + local total_example=$(grep -c "^[^#]*=" .env.example 2>/dev/null || echo "0") + local total_env=$(grep -c "^[^#]*=" .env 2>/dev/null || echo "0") + + log_info " .env.example environment variables: $total_example" + log_info " .env environment variables: $total_env" +} + +# Main execution function +# Orchestrates the complete synchronization process in the correct order +main() { + log_info "=== Dify Environment Variables Synchronization Script ===" + log_info "Execution started: $(date)" + + # Check prerequisites + check_files + + # Create backup + create_backup + + # Detect differences + detect_differences + + # Detect removed variables (before sync) + detect_removed_variables + + # Synchronize environment file + sync_env_file + + # Show statistics + show_statistics + + log_success "=== Synchronization process completed successfully ===" + log_info "Execution finished: $(date)" +} + +# Execute main function only when script is run directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi \ No newline at end of file diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 4f6194b9e4..3c88cddf8c 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -21,7 +21,7 @@ services: # API service api: - image: langgenius/dify-api:1.11.1 + image: langgenius/dify-api:1.11.2 restart: always environment: # Use the shared environment variables. @@ -34,6 +34,7 @@ services: PLUGIN_REMOTE_INSTALL_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost} PLUGIN_REMOTE_INSTALL_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003} PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + PLUGIN_DAEMON_TIMEOUT: ${PLUGIN_DAEMON_TIMEOUT:-600.0} INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} depends_on: init_permissions: @@ -62,7 +63,7 @@ services: # worker service # The Celery worker for processing all queues (dataset, workflow, mail, etc.) worker: - image: langgenius/dify-api:1.11.1 + image: langgenius/dify-api:1.11.2 restart: always environment: # Use the shared environment variables. @@ -101,7 +102,7 @@ services: # worker_beat service # Celery beat for scheduling periodic tasks. worker_beat: - image: langgenius/dify-api:1.11.1 + image: langgenius/dify-api:1.11.2 restart: always environment: # Use the shared environment variables. @@ -131,7 +132,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.11.1 + image: langgenius/dify-web:1.11.2 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -269,7 +270,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.5.1-local + image: langgenius/dify-plugin-daemon:0.5.2-local restart: always environment: # Use the shared environment variables. diff --git a/docker/docker-compose.middleware.yaml b/docker/docker-compose.middleware.yaml index 68ef217bbd..dba61d1816 100644 --- a/docker/docker-compose.middleware.yaml +++ b/docker/docker-compose.middleware.yaml @@ -123,7 +123,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.5.1-local + image: langgenius/dify-plugin-daemon:0.5.2-local restart: always env_file: - ./middleware.env diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index aca4325880..1c8d8d03e3 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -108,6 +108,7 @@ x-shared-env: &shared-api-worker-env CONSOLE_CORS_ALLOW_ORIGINS: ${CONSOLE_CORS_ALLOW_ORIGINS:-*} COOKIE_DOMAIN: ${COOKIE_DOMAIN:-} NEXT_PUBLIC_COOKIE_DOMAIN: ${NEXT_PUBLIC_COOKIE_DOMAIN:-} + NEXT_PUBLIC_BATCH_CONCURRENCY: ${NEXT_PUBLIC_BATCH_CONCURRENCY:-5} STORAGE_TYPE: ${STORAGE_TYPE:-opendal} OPENDAL_SCHEME: ${OPENDAL_SCHEME:-fs} OPENDAL_FS_ROOT: ${OPENDAL_FS_ROOT:-storage} @@ -134,11 +135,13 @@ x-shared-env: &shared-api-worker-env ALIYUN_OSS_REGION: ${ALIYUN_OSS_REGION:-ap-southeast-1} ALIYUN_OSS_AUTH_VERSION: ${ALIYUN_OSS_AUTH_VERSION:-v4} ALIYUN_OSS_PATH: ${ALIYUN_OSS_PATH:-your-path} + ALIYUN_CLOUDBOX_ID: ${ALIYUN_CLOUDBOX_ID:-your-cloudbox-id} TENCENT_COS_BUCKET_NAME: ${TENCENT_COS_BUCKET_NAME:-your-bucket-name} TENCENT_COS_SECRET_KEY: ${TENCENT_COS_SECRET_KEY:-your-secret-key} TENCENT_COS_SECRET_ID: ${TENCENT_COS_SECRET_ID:-your-secret-id} TENCENT_COS_REGION: ${TENCENT_COS_REGION:-your-region} TENCENT_COS_SCHEME: ${TENCENT_COS_SCHEME:-your-scheme} + TENCENT_COS_CUSTOM_DOMAIN: ${TENCENT_COS_CUSTOM_DOMAIN:-your-custom-domain} OCI_ENDPOINT: ${OCI_ENDPOINT:-https://your-object-storage-namespace.compat.objectstorage.us-ashburn-1.oraclecloud.com} OCI_BUCKET_NAME: ${OCI_BUCKET_NAME:-your-bucket-name} OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-your-access-key} @@ -148,6 +151,7 @@ x-shared-env: &shared-api-worker-env HUAWEI_OBS_SECRET_KEY: ${HUAWEI_OBS_SECRET_KEY:-your-secret-key} HUAWEI_OBS_ACCESS_KEY: ${HUAWEI_OBS_ACCESS_KEY:-your-access-key} HUAWEI_OBS_SERVER: ${HUAWEI_OBS_SERVER:-your-server-url} + HUAWEI_OBS_PATH_STYLE: ${HUAWEI_OBS_PATH_STYLE:-false} VOLCENGINE_TOS_BUCKET_NAME: ${VOLCENGINE_TOS_BUCKET_NAME:-your-bucket-name} VOLCENGINE_TOS_SECRET_KEY: ${VOLCENGINE_TOS_SECRET_KEY:-your-secret-key} VOLCENGINE_TOS_ACCESS_KEY: ${VOLCENGINE_TOS_ACCESS_KEY:-your-access-key} @@ -591,6 +595,7 @@ x-shared-env: &shared-api-worker-env PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880} PLUGIN_PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} + PLUGIN_DAEMON_TIMEOUT: ${PLUGIN_DAEMON_TIMEOUT:-600.0} PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} @@ -663,6 +668,9 @@ x-shared-env: &shared-api-worker-env ANNOTATION_IMPORT_RATE_LIMIT_PER_HOUR: ${ANNOTATION_IMPORT_RATE_LIMIT_PER_HOUR:-20} ANNOTATION_IMPORT_MAX_CONCURRENT: ${ANNOTATION_IMPORT_MAX_CONCURRENT:-5} AMPLITUDE_API_KEY: ${AMPLITUDE_API_KEY:-} + SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD: ${SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD:-21} + SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE: ${SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE:-1000} + SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS: ${SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS:-30} services: # Init container to fix permissions @@ -686,7 +694,7 @@ services: # API service api: - image: langgenius/dify-api:1.11.1 + image: langgenius/dify-api:1.11.2 restart: always environment: # Use the shared environment variables. @@ -699,6 +707,7 @@ services: PLUGIN_REMOTE_INSTALL_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost} PLUGIN_REMOTE_INSTALL_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003} PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + PLUGIN_DAEMON_TIMEOUT: ${PLUGIN_DAEMON_TIMEOUT:-600.0} INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} depends_on: init_permissions: @@ -727,7 +736,7 @@ services: # worker service # The Celery worker for processing all queues (dataset, workflow, mail, etc.) worker: - image: langgenius/dify-api:1.11.1 + image: langgenius/dify-api:1.11.2 restart: always environment: # Use the shared environment variables. @@ -766,7 +775,7 @@ services: # worker_beat service # Celery beat for scheduling periodic tasks. worker_beat: - image: langgenius/dify-api:1.11.1 + image: langgenius/dify-api:1.11.2 restart: always environment: # Use the shared environment variables. @@ -796,7 +805,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.11.1 + image: langgenius/dify-web:1.11.2 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -934,7 +943,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.5.1-local + image: langgenius/dify-plugin-daemon:0.5.2-local restart: always environment: # Use the shared environment variables. diff --git a/docs/fr-FR/README.md b/docs/fr-FR/README.md index 03f3221798..291c8dab40 100644 --- a/docs/fr-FR/README.md +++ b/docs/fr-FR/README.md @@ -61,14 +61,14 @@

langgenius%2Fdify | Trendshift

-Dify est une plateforme de développement d'applications LLM open source. Son interface intuitive combine un flux de travail d'IA, un pipeline RAG, des capacités d'agent, une gestion de modèles, des fonctionnalités d'observabilité, et plus encore, vous permettant de passer rapidement du prototype à la production. Voici une liste des fonctionnalités principales: +Dify est une plateforme de développement d'applications LLM open source. Sa interface intuitive combine un flux de travail d'IA, un pipeline RAG, des capacités d'agent, une gestion de modèles, des fonctionnalités d'observabilité, et plus encore, vous permettant de passer rapidement du prototype à la production. Voici une liste des fonctionnalités principales:

**1. Flux de travail** : Construisez et testez des flux de travail d'IA puissants sur un canevas visuel, en utilisant toutes les fonctionnalités suivantes et plus encore. **2. Prise en charge complète des modèles** : -Intégration transparente avec des centaines de LLM propriétaires / open source provenant de dizaines de fournisseurs d'inférence et de solutions auto-hébergées, couvrant GPT, Mistral, Llama3, et tous les modèles compatibles avec l'API OpenAI. Une liste complète des fournisseurs de modèles pris en charge se trouve [ici](https://docs.dify.ai/getting-started/readme/model-providers). +Intégration transparente avec des centaines de LLM propriétaires / open source offerts par dizaines de fournisseurs d'inférence et de solutions auto-hébergées, couvrant GPT, Mistral, Llama3, et tous les modèles compatibles avec l'API OpenAI. Une liste complète des fournisseurs de modèles pris en charge se trouve [ici](https://docs.dify.ai/getting-started/readme/model-providers). ![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3) @@ -79,7 +79,7 @@ Interface intuitive pour créer des prompts, comparer les performances des modè Des capacités RAG étendues qui couvrent tout, de l'ingestion de documents à la récupération, avec un support prêt à l'emploi pour l'extraction de texte à partir de PDF, PPT et autres formats de document courants. **5. Capacités d'agent** : -Vous pouvez définir des agents basés sur l'appel de fonction LLM ou ReAct, et ajouter des outils pré-construits ou personnalisés pour l'agent. Dify fournit plus de 50 outils intégrés pour les agents d'IA, tels que la recherche Google, DALL·E, Stable Diffusion et WolframAlpha. +Vous pouvez définir des agents basés sur l'appel de fonctions LLM ou ReAct, et ajouter des outils pré-construits ou personnalisés pour l'agent. Dify fournit plus de 50 outils intégrés pour les agents d'IA, tels que la recherche Google, DALL·E, Stable Diffusion et WolframAlpha. **6. LLMOps** : Surveillez et analysez les journaux d'application et les performances au fil du temps. Vous pouvez continuellement améliorer les prompts, les ensembles de données et les modèles en fonction des données de production et des annotations. diff --git a/sdks/nodejs-client/.gitignore b/sdks/nodejs-client/.gitignore index 1d40ff2ece..33cfbd3b18 100644 --- a/sdks/nodejs-client/.gitignore +++ b/sdks/nodejs-client/.gitignore @@ -1,48 +1,40 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. +# Dependencies +node_modules/ -# dependencies -/node_modules -/.pnp -.pnp.js +# Build output +dist/ -# testing -/coverage +# Testing +coverage/ -# next.js -/.next/ -/out/ +# IDE +.idea/ +.vscode/ +*.swp +*.swo -# production -/build - -# misc +# OS .DS_Store -*.pem +Thumbs.db -# debug +# Debug logs npm-debug.log* yarn-debug.log* yarn-error.log* -.pnpm-debug.log* +pnpm-debug.log* -# local env files -.env*.local +# Environment +.env +.env.local +.env.*.local -# vercel -.vercel - -# typescript +# TypeScript *.tsbuildinfo -next-env.d.ts -# npm +# Lock files (use pnpm-lock.yaml in CI if needed) package-lock.json +yarn.lock -# yarn -.pnp.cjs -.pnp.loader.mjs -.yarn/ -.yarnrc.yml - -# pmpm -pnpm-lock.yaml +# Misc +*.pem +*.tgz diff --git a/sdks/nodejs-client/LICENSE b/sdks/nodejs-client/LICENSE new file mode 100644 index 0000000000..ff17417e1f --- /dev/null +++ b/sdks/nodejs-client/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2023 LangGenius + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/sdks/nodejs-client/README.md b/sdks/nodejs-client/README.md index 3a5688bcbe..f8c2803c08 100644 --- a/sdks/nodejs-client/README.md +++ b/sdks/nodejs-client/README.md @@ -13,54 +13,92 @@ npm install dify-client After installing the SDK, you can use it in your project like this: ```js -import { DifyClient, ChatClient, CompletionClient } from 'dify-client' +import { + DifyClient, + ChatClient, + CompletionClient, + WorkflowClient, + KnowledgeBaseClient, + WorkspaceClient +} from 'dify-client' -const API_KEY = 'your-api-key-here' -const user = `random-user-id` +const API_KEY = 'your-app-api-key' +const DATASET_API_KEY = 'your-dataset-api-key' +const user = 'random-user-id' const query = 'Please tell me a short story in 10 words or less.' -const remote_url_files = [{ - type: 'image', - transfer_method: 'remote_url', - url: 'your_url_address' -}] -// Create a completion client -const completionClient = new CompletionClient(API_KEY) -// Create a completion message -completionClient.createCompletionMessage({'query': query}, user) -// Create a completion message with vision model -completionClient.createCompletionMessage({'query': 'Describe the picture.'}, user, false, remote_url_files) - -// Create a chat client const chatClient = new ChatClient(API_KEY) -// Create a chat message in stream mode -const response = await chatClient.createChatMessage({}, query, user, true, null) -const stream = response.data; -stream.on('data', data => { - console.log(data); -}); -stream.on('end', () => { - console.log('stream done'); -}); -// Create a chat message with vision model -chatClient.createChatMessage({}, 'Describe the picture.', user, false, null, remote_url_files) -// Fetch conversations -chatClient.getConversations(user) -// Fetch conversation messages -chatClient.getConversationMessages(conversationId, user) -// Rename conversation -chatClient.renameConversation(conversationId, name, user) - - +const completionClient = new CompletionClient(API_KEY) +const workflowClient = new WorkflowClient(API_KEY) +const kbClient = new KnowledgeBaseClient(DATASET_API_KEY) +const workspaceClient = new WorkspaceClient(DATASET_API_KEY) const client = new DifyClient(API_KEY) -// Fetch application parameters -client.getApplicationParameters(user) -// Provide feedback for a message -client.messageFeedback(messageId, rating, user) + +// App core +await client.getApplicationParameters(user) +await client.messageFeedback('message-id', 'like', user) + +// Completion (blocking) +await completionClient.createCompletionMessage({ + inputs: { query }, + user, + response_mode: 'blocking' +}) + +// Chat (streaming) +const stream = await chatClient.createChatMessage({ + inputs: {}, + query, + user, + response_mode: 'streaming' +}) +for await (const event of stream) { + console.log(event.event, event.data) +} + +// Chatflow (advanced chat via workflow_id) +await chatClient.createChatMessage({ + inputs: {}, + query, + user, + workflow_id: 'workflow-id', + response_mode: 'blocking' +}) + +// Workflow run (blocking or streaming) +await workflowClient.run({ + inputs: { query }, + user, + response_mode: 'blocking' +}) + +// Knowledge base (dataset token required) +await kbClient.listDatasets({ page: 1, limit: 20 }) +await kbClient.createDataset({ name: 'KB', indexing_technique: 'economy' }) + +// RAG pipeline (may require service API route registration) +const pipelineStream = await kbClient.runPipeline('dataset-id', { + inputs: {}, + datasource_type: 'online_document', + datasource_info_list: [], + start_node_id: 'start-node-id', + is_published: true, + response_mode: 'streaming' +}) +for await (const event of pipelineStream) { + console.log(event.data) +} + +// Workspace models (dataset token required) +await workspaceClient.getModelsByType('text-embedding') ``` -Replace 'your-api-key-here' with your actual Dify API key.Replace 'your-app-id-here' with your actual Dify APP ID. +Notes: + +- App endpoints use an app API token; knowledge base and workspace endpoints use a dataset API token. +- Chat/completion require a stable `user` identifier in the request payload. +- For streaming responses, iterate the returned AsyncIterable. Use `stream.toText()` to collect text. ## License diff --git a/sdks/nodejs-client/babel.config.cjs b/sdks/nodejs-client/babel.config.cjs deleted file mode 100644 index 392abb66d8..0000000000 --- a/sdks/nodejs-client/babel.config.cjs +++ /dev/null @@ -1,12 +0,0 @@ -module.exports = { - presets: [ - [ - "@babel/preset-env", - { - targets: { - node: "current", - }, - }, - ], - ], -}; diff --git a/sdks/nodejs-client/eslint.config.js b/sdks/nodejs-client/eslint.config.js new file mode 100644 index 0000000000..9e659f5d28 --- /dev/null +++ b/sdks/nodejs-client/eslint.config.js @@ -0,0 +1,45 @@ +import js from "@eslint/js"; +import tsParser from "@typescript-eslint/parser"; +import tsPlugin from "@typescript-eslint/eslint-plugin"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; + +const tsconfigRootDir = path.dirname(fileURLToPath(import.meta.url)); +const typeCheckedRules = + tsPlugin.configs["recommended-type-checked"]?.rules ?? + tsPlugin.configs.recommendedTypeChecked?.rules ?? + {}; + +export default [ + { + ignores: ["dist", "node_modules", "scripts", "tests", "**/*.test.*", "**/*.spec.*"], + }, + js.configs.recommended, + { + files: ["src/**/*.ts"], + languageOptions: { + parser: tsParser, + ecmaVersion: "latest", + parserOptions: { + project: "./tsconfig.json", + tsconfigRootDir, + sourceType: "module", + }, + }, + plugins: { + "@typescript-eslint": tsPlugin, + }, + rules: { + ...tsPlugin.configs.recommended.rules, + ...typeCheckedRules, + "no-undef": "off", + "no-unused-vars": "off", + "@typescript-eslint/no-unsafe-call": "error", + "@typescript-eslint/no-unsafe-return": "error", + "@typescript-eslint/consistent-type-imports": [ + "error", + { prefer: "type-imports", fixStyle: "separate-type-imports" }, + ], + }, + }, +]; diff --git a/sdks/nodejs-client/index.d.ts b/sdks/nodejs-client/index.d.ts deleted file mode 100644 index 3ea4b9d153..0000000000 --- a/sdks/nodejs-client/index.d.ts +++ /dev/null @@ -1,107 +0,0 @@ -// Types.d.ts -export const BASE_URL: string; - -export type RequestMethods = 'GET' | 'POST' | 'PATCH' | 'DELETE'; - -interface Params { - [key: string]: any; -} - -interface HeaderParams { - [key: string]: string; -} - -interface User { -} - -interface DifyFileBase { - type: "image" -} - -export interface DifyRemoteFile extends DifyFileBase { - transfer_method: "remote_url" - url: string -} - -export interface DifyLocalFile extends DifyFileBase { - transfer_method: "local_file" - upload_file_id: string -} - -export type DifyFile = DifyRemoteFile | DifyLocalFile; - -export declare class DifyClient { - constructor(apiKey: string, baseUrl?: string); - - updateApiKey(apiKey: string): void; - - sendRequest( - method: RequestMethods, - endpoint: string, - data?: any, - params?: Params, - stream?: boolean, - headerParams?: HeaderParams - ): Promise; - - messageFeedback(message_id: string, rating: number, user: User): Promise; - - getApplicationParameters(user: User): Promise; - - fileUpload(data: FormData): Promise; - - textToAudio(text: string ,user: string, streaming?: boolean): Promise; - - getMeta(user: User): Promise; -} - -export declare class CompletionClient extends DifyClient { - createCompletionMessage( - inputs: any, - user: User, - stream?: boolean, - files?: DifyFile[] | null - ): Promise; -} - -export declare class ChatClient extends DifyClient { - createChatMessage( - inputs: any, - query: string, - user: User, - stream?: boolean, - conversation_id?: string | null, - files?: DifyFile[] | null - ): Promise; - - getSuggested(message_id: string, user: User): Promise; - - stopMessage(task_id: string, user: User) : Promise; - - - getConversations( - user: User, - first_id?: string | null, - limit?: number | null, - pinned?: boolean | null - ): Promise; - - getConversationMessages( - user: User, - conversation_id?: string, - first_id?: string | null, - limit?: number | null - ): Promise; - - renameConversation(conversation_id: string, name: string, user: User,auto_generate:boolean): Promise; - - deleteConversation(conversation_id: string, user: User): Promise; - - audioToText(data: FormData): Promise; -} - -export declare class WorkflowClient extends DifyClient { - run(inputs: any, user: User, stream?: boolean,): Promise; - - stop(task_id: string, user: User): Promise; -} diff --git a/sdks/nodejs-client/index.js b/sdks/nodejs-client/index.js deleted file mode 100644 index 9743ae358c..0000000000 --- a/sdks/nodejs-client/index.js +++ /dev/null @@ -1,351 +0,0 @@ -import axios from "axios"; -export const BASE_URL = "https://api.dify.ai/v1"; - -export const routes = { - // app's - feedback: { - method: "POST", - url: (message_id) => `/messages/${message_id}/feedbacks`, - }, - application: { - method: "GET", - url: () => `/parameters`, - }, - fileUpload: { - method: "POST", - url: () => `/files/upload`, - }, - textToAudio: { - method: "POST", - url: () => `/text-to-audio`, - }, - getMeta: { - method: "GET", - url: () => `/meta`, - }, - - // completion's - createCompletionMessage: { - method: "POST", - url: () => `/completion-messages`, - }, - - // chat's - createChatMessage: { - method: "POST", - url: () => `/chat-messages`, - }, - getSuggested:{ - method: "GET", - url: (message_id) => `/messages/${message_id}/suggested`, - }, - stopChatMessage: { - method: "POST", - url: (task_id) => `/chat-messages/${task_id}/stop`, - }, - getConversations: { - method: "GET", - url: () => `/conversations`, - }, - getConversationMessages: { - method: "GET", - url: () => `/messages`, - }, - renameConversation: { - method: "POST", - url: (conversation_id) => `/conversations/${conversation_id}/name`, - }, - deleteConversation: { - method: "DELETE", - url: (conversation_id) => `/conversations/${conversation_id}`, - }, - audioToText: { - method: "POST", - url: () => `/audio-to-text`, - }, - - // workflow‘s - runWorkflow: { - method: "POST", - url: () => `/workflows/run`, - }, - stopWorkflow: { - method: "POST", - url: (task_id) => `/workflows/tasks/${task_id}/stop`, - } - -}; - -export class DifyClient { - constructor(apiKey, baseUrl = BASE_URL) { - this.apiKey = apiKey; - this.baseUrl = baseUrl; - } - - updateApiKey(apiKey) { - this.apiKey = apiKey; - } - - async sendRequest( - method, - endpoint, - data = null, - params = null, - stream = false, - headerParams = {} - ) { - const isFormData = - (typeof FormData !== "undefined" && data instanceof FormData) || - (data && data.constructor && data.constructor.name === "FormData"); - const headers = { - Authorization: `Bearer ${this.apiKey}`, - ...(isFormData ? {} : { "Content-Type": "application/json" }), - ...headerParams, - }; - - const url = `${this.baseUrl}${endpoint}`; - let response; - if (stream) { - response = await axios({ - method, - url, - data, - params, - headers, - responseType: "stream", - }); - } else { - response = await axios({ - method, - url, - ...(method !== "GET" && { data }), - params, - headers, - responseType: "json", - }); - } - - return response; - } - - messageFeedback(message_id, rating, user) { - const data = { - rating, - user, - }; - return this.sendRequest( - routes.feedback.method, - routes.feedback.url(message_id), - data - ); - } - - getApplicationParameters(user) { - const params = { user }; - return this.sendRequest( - routes.application.method, - routes.application.url(), - null, - params - ); - } - - fileUpload(data) { - return this.sendRequest( - routes.fileUpload.method, - routes.fileUpload.url(), - data - ); - } - - textToAudio(text, user, streaming = false) { - const data = { - text, - user, - streaming - }; - return this.sendRequest( - routes.textToAudio.method, - routes.textToAudio.url(), - data, - null, - streaming - ); - } - - getMeta(user) { - const params = { user }; - return this.sendRequest( - routes.getMeta.method, - routes.getMeta.url(), - null, - params - ); - } -} - -export class CompletionClient extends DifyClient { - createCompletionMessage(inputs, user, stream = false, files = null) { - const data = { - inputs, - user, - response_mode: stream ? "streaming" : "blocking", - files, - }; - return this.sendRequest( - routes.createCompletionMessage.method, - routes.createCompletionMessage.url(), - data, - null, - stream - ); - } - - runWorkflow(inputs, user, stream = false, files = null) { - const data = { - inputs, - user, - response_mode: stream ? "streaming" : "blocking", - }; - return this.sendRequest( - routes.runWorkflow.method, - routes.runWorkflow.url(), - data, - null, - stream - ); - } -} - -export class ChatClient extends DifyClient { - createChatMessage( - inputs, - query, - user, - stream = false, - conversation_id = null, - files = null - ) { - const data = { - inputs, - query, - user, - response_mode: stream ? "streaming" : "blocking", - files, - }; - if (conversation_id) data.conversation_id = conversation_id; - - return this.sendRequest( - routes.createChatMessage.method, - routes.createChatMessage.url(), - data, - null, - stream - ); - } - - getSuggested(message_id, user) { - const data = { user }; - return this.sendRequest( - routes.getSuggested.method, - routes.getSuggested.url(message_id), - data - ); - } - - stopMessage(task_id, user) { - const data = { user }; - return this.sendRequest( - routes.stopChatMessage.method, - routes.stopChatMessage.url(task_id), - data - ); - } - - getConversations(user, first_id = null, limit = null, pinned = null) { - const params = { user, first_id: first_id, limit, pinned }; - return this.sendRequest( - routes.getConversations.method, - routes.getConversations.url(), - null, - params - ); - } - - getConversationMessages( - user, - conversation_id = "", - first_id = null, - limit = null - ) { - const params = { user }; - - if (conversation_id) params.conversation_id = conversation_id; - - if (first_id) params.first_id = first_id; - - if (limit) params.limit = limit; - - return this.sendRequest( - routes.getConversationMessages.method, - routes.getConversationMessages.url(), - null, - params - ); - } - - renameConversation(conversation_id, name, user, auto_generate) { - const data = { name, user, auto_generate }; - return this.sendRequest( - routes.renameConversation.method, - routes.renameConversation.url(conversation_id), - data - ); - } - - deleteConversation(conversation_id, user) { - const data = { user }; - return this.sendRequest( - routes.deleteConversation.method, - routes.deleteConversation.url(conversation_id), - data - ); - } - - - audioToText(data) { - return this.sendRequest( - routes.audioToText.method, - routes.audioToText.url(), - data - ); - } - -} - -export class WorkflowClient extends DifyClient { - run(inputs,user,stream) { - const data = { - inputs, - response_mode: stream ? "streaming" : "blocking", - user - }; - - return this.sendRequest( - routes.runWorkflow.method, - routes.runWorkflow.url(), - data, - null, - stream - ); - } - - stop(task_id, user) { - const data = { user }; - return this.sendRequest( - routes.stopWorkflow.method, - routes.stopWorkflow.url(task_id), - data - ); - } -} diff --git a/sdks/nodejs-client/index.test.js b/sdks/nodejs-client/index.test.js deleted file mode 100644 index e3a1715238..0000000000 --- a/sdks/nodejs-client/index.test.js +++ /dev/null @@ -1,141 +0,0 @@ -import { DifyClient, WorkflowClient, BASE_URL, routes } from "."; - -import axios from 'axios' - -jest.mock('axios') - -afterEach(() => { - jest.resetAllMocks() -}) - -describe('Client', () => { - let difyClient - beforeEach(() => { - difyClient = new DifyClient('test') - }) - - test('should create a client', () => { - expect(difyClient).toBeDefined(); - }) - // test updateApiKey - test('should update the api key', () => { - difyClient.updateApiKey('test2'); - expect(difyClient.apiKey).toBe('test2'); - }) -}); - -describe('Send Requests', () => { - let difyClient - - beforeEach(() => { - difyClient = new DifyClient('test') - }) - - it('should make a successful request to the application parameter', async () => { - const method = 'GET' - const endpoint = routes.application.url() - const expectedResponse = { data: 'response' } - axios.mockResolvedValue(expectedResponse) - - await difyClient.sendRequest(method, endpoint) - - expect(axios).toHaveBeenCalledWith({ - method, - url: `${BASE_URL}${endpoint}`, - params: null, - headers: { - Authorization: `Bearer ${difyClient.apiKey}`, - 'Content-Type': 'application/json', - }, - responseType: 'json', - }) - - }) - - it('should handle errors from the API', async () => { - const method = 'GET' - const endpoint = '/test-endpoint' - const errorMessage = 'Request failed with status code 404' - axios.mockRejectedValue(new Error(errorMessage)) - - await expect(difyClient.sendRequest(method, endpoint)).rejects.toThrow( - errorMessage - ) - }) - - it('uses the getMeta route configuration', async () => { - axios.mockResolvedValue({ data: 'ok' }) - await difyClient.getMeta('end-user') - - expect(axios).toHaveBeenCalledWith({ - method: routes.getMeta.method, - url: `${BASE_URL}${routes.getMeta.url()}`, - params: { user: 'end-user' }, - headers: { - Authorization: `Bearer ${difyClient.apiKey}`, - 'Content-Type': 'application/json', - }, - responseType: 'json', - }) - }) -}) - -describe('File uploads', () => { - let difyClient - const OriginalFormData = global.FormData - - beforeAll(() => { - global.FormData = class FormDataMock {} - }) - - afterAll(() => { - global.FormData = OriginalFormData - }) - - beforeEach(() => { - difyClient = new DifyClient('test') - }) - - it('does not override multipart boundary headers for FormData', async () => { - const form = new FormData() - axios.mockResolvedValue({ data: 'ok' }) - - await difyClient.fileUpload(form) - - expect(axios).toHaveBeenCalledWith({ - method: routes.fileUpload.method, - url: `${BASE_URL}${routes.fileUpload.url()}`, - data: form, - params: null, - headers: { - Authorization: `Bearer ${difyClient.apiKey}`, - }, - responseType: 'json', - }) - }) -}) - -describe('Workflow client', () => { - let workflowClient - - beforeEach(() => { - workflowClient = new WorkflowClient('test') - }) - - it('uses tasks stop path for workflow stop', async () => { - axios.mockResolvedValue({ data: 'stopped' }) - await workflowClient.stop('task-1', 'end-user') - - expect(axios).toHaveBeenCalledWith({ - method: routes.stopWorkflow.method, - url: `${BASE_URL}${routes.stopWorkflow.url('task-1')}`, - data: { user: 'end-user' }, - params: null, - headers: { - Authorization: `Bearer ${workflowClient.apiKey}`, - 'Content-Type': 'application/json', - }, - responseType: 'json', - }) - }) -}) diff --git a/sdks/nodejs-client/jest.config.cjs b/sdks/nodejs-client/jest.config.cjs deleted file mode 100644 index ea0fb34ad1..0000000000 --- a/sdks/nodejs-client/jest.config.cjs +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - testEnvironment: "node", - transform: { - "^.+\\.[tj]sx?$": "babel-jest", - }, -}; diff --git a/sdks/nodejs-client/package.json b/sdks/nodejs-client/package.json index c6bb0a9c1f..afbb58fee1 100644 --- a/sdks/nodejs-client/package.json +++ b/sdks/nodejs-client/package.json @@ -1,30 +1,70 @@ { "name": "dify-client", - "version": "2.3.2", + "version": "3.0.0", "description": "This is the Node.js SDK for the Dify.AI API, which allows you to easily integrate Dify.AI into your Node.js applications.", - "main": "index.js", "type": "module", - "types":"index.d.ts", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "engines": { + "node": ">=18.0.0" + }, + "files": [ + "dist", + "README.md", + "LICENSE" + ], "keywords": [ "Dify", "Dify.AI", - "LLM" + "LLM", + "AI", + "SDK", + "API" ], - "author": "Joel", + "author": "LangGenius", "contributors": [ - " <<427733928@qq.com>> (https://github.com/crazywoola)" + "Joel (https://github.com/iamjoel)", + "lyzno1 (https://github.com/lyzno1)", + "crazywoola <427733928@qq.com> (https://github.com/crazywoola)" ], + "repository": { + "type": "git", + "url": "https://github.com/langgenius/dify.git", + "directory": "sdks/nodejs-client" + }, + "bugs": { + "url": "https://github.com/langgenius/dify/issues" + }, + "homepage": "https://dify.ai", "license": "MIT", "scripts": { - "test": "jest" + "build": "tsup", + "lint": "eslint", + "lint:fix": "eslint --fix", + "type-check": "tsc -p tsconfig.json --noEmit", + "test": "vitest run", + "test:coverage": "vitest run --coverage", + "publish:check": "./scripts/publish.sh --dry-run", + "publish:npm": "./scripts/publish.sh" }, "dependencies": { - "axios": "^1.3.5" + "axios": "^1.13.2" }, "devDependencies": { - "@babel/core": "^7.21.8", - "@babel/preset-env": "^7.21.5", - "babel-jest": "^29.5.0", - "jest": "^29.5.0" + "@eslint/js": "^9.39.2", + "@types/node": "^25.0.3", + "@typescript-eslint/eslint-plugin": "^8.50.1", + "@typescript-eslint/parser": "^8.50.1", + "@vitest/coverage-v8": "4.0.16", + "eslint": "^9.39.2", + "tsup": "^8.5.1", + "typescript": "^5.9.3", + "vitest": "^4.0.16" } } diff --git a/sdks/nodejs-client/pnpm-lock.yaml b/sdks/nodejs-client/pnpm-lock.yaml new file mode 100644 index 0000000000..6febed2ea6 --- /dev/null +++ b/sdks/nodejs-client/pnpm-lock.yaml @@ -0,0 +1,2353 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + axios: + specifier: ^1.13.2 + version: 1.13.2 + devDependencies: + '@eslint/js': + specifier: ^9.39.2 + version: 9.39.2 + '@types/node': + specifier: ^25.0.3 + version: 25.0.3 + '@typescript-eslint/eslint-plugin': + specifier: ^8.50.1 + version: 8.50.1(@typescript-eslint/parser@8.50.1(eslint@9.39.2)(typescript@5.9.3))(eslint@9.39.2)(typescript@5.9.3) + '@typescript-eslint/parser': + specifier: ^8.50.1 + version: 8.50.1(eslint@9.39.2)(typescript@5.9.3) + '@vitest/coverage-v8': + specifier: 4.0.16 + version: 4.0.16(vitest@4.0.16(@types/node@25.0.3)) + eslint: + specifier: ^9.39.2 + version: 9.39.2 + tsup: + specifier: ^8.5.1 + version: 8.5.1(postcss@8.5.6)(typescript@5.9.3) + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^4.0.16 + version: 4.0.16(@types/node@25.0.3) + +packages: + + '@babel/helper-string-parser@7.27.1': + resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.28.5': + resolution: {integrity: sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.28.5': + resolution: {integrity: sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==} + engines: {node: '>=6.0.0'} + hasBin: true + + '@babel/types@7.28.5': + resolution: {integrity: sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==} + engines: {node: '>=6.9.0'} + + '@bcoe/v8-coverage@1.0.2': + resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} + engines: {node: '>=18'} + + '@esbuild/aix-ppc64@0.27.2': + resolution: {integrity: sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.27.2': + resolution: {integrity: sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.27.2': + resolution: {integrity: sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.27.2': + resolution: {integrity: sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.27.2': + resolution: {integrity: sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.2': + resolution: {integrity: sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.27.2': + resolution: {integrity: sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.2': + resolution: {integrity: sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.27.2': + resolution: {integrity: sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.27.2': + resolution: {integrity: sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.27.2': + resolution: {integrity: sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.27.2': + resolution: {integrity: sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.27.2': + resolution: {integrity: sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.27.2': + resolution: {integrity: sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.2': + resolution: {integrity: sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.27.2': + resolution: {integrity: sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.27.2': + resolution: {integrity: sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.27.2': + resolution: {integrity: sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.2': + resolution: {integrity: sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.27.2': + resolution: {integrity: sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.2': + resolution: {integrity: sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.27.2': + resolution: {integrity: sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.27.2': + resolution: {integrity: sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.27.2': + resolution: {integrity: sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.27.2': + resolution: {integrity: sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.27.2': + resolution: {integrity: sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@eslint-community/eslint-utils@4.9.0': + resolution: {integrity: sha512-ayVFHdtZ+hsq1t2Dy24wCmGXGe4q9Gu3smhLYALJrr473ZH27MsnSL+LKUlimp4BWJqMDMLmPpx/Q9R3OAlL4g==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + + '@eslint-community/regexpp@4.12.2': + resolution: {integrity: sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + + '@eslint/config-array@0.21.1': + resolution: {integrity: sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/config-helpers@0.4.2': + resolution: {integrity: sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/core@0.17.0': + resolution: {integrity: sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/eslintrc@3.3.3': + resolution: {integrity: sha512-Kr+LPIUVKz2qkx1HAMH8q1q6azbqBAsXJUxBl/ODDuVPX45Z9DfwB8tPjTi6nNZ8BuM3nbJxC5zCAg5elnBUTQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/js@9.39.2': + resolution: {integrity: sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/object-schema@2.1.7': + resolution: {integrity: sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/plugin-kit@0.4.1': + resolution: {integrity: sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@humanfs/core@0.19.1': + resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} + engines: {node: '>=18.18.0'} + + '@humanfs/node@0.16.7': + resolution: {integrity: sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==} + engines: {node: '>=18.18.0'} + + '@humanwhocodes/module-importer@1.0.1': + resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} + engines: {node: '>=12.22'} + + '@humanwhocodes/retry@0.4.3': + resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} + engines: {node: '>=18.18'} + + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@rollup/rollup-android-arm-eabi@4.54.0': + resolution: {integrity: sha512-OywsdRHrFvCdvsewAInDKCNyR3laPA2mc9bRYJ6LBp5IyvF3fvXbbNR0bSzHlZVFtn6E0xw2oZlyjg4rKCVcng==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.54.0': + resolution: {integrity: sha512-Skx39Uv+u7H224Af+bDgNinitlmHyQX1K/atIA32JP3JQw6hVODX5tkbi2zof/E69M1qH2UoN3Xdxgs90mmNYw==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.54.0': + resolution: {integrity: sha512-k43D4qta/+6Fq+nCDhhv9yP2HdeKeP56QrUUTW7E6PhZP1US6NDqpJj4MY0jBHlJivVJD5P8NxrjuobZBJTCRw==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.54.0': + resolution: {integrity: sha512-cOo7biqwkpawslEfox5Vs8/qj83M/aZCSSNIWpVzfU2CYHa2G3P1UN5WF01RdTHSgCkri7XOlTdtk17BezlV3A==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.54.0': + resolution: {integrity: sha512-miSvuFkmvFbgJ1BevMa4CPCFt5MPGw094knM64W9I0giUIMMmRYcGW/JWZDriaw/k1kOBtsWh1z6nIFV1vPNtA==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.54.0': + resolution: {integrity: sha512-KGXIs55+b/ZfZsq9aR026tmr/+7tq6VG6MsnrvF4H8VhwflTIuYh+LFUlIsRdQSgrgmtM3fVATzEAj4hBQlaqQ==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.54.0': + resolution: {integrity: sha512-EHMUcDwhtdRGlXZsGSIuXSYwD5kOT9NVnx9sqzYiwAc91wfYOE1g1djOEDseZJKKqtHAHGwnGPQu3kytmfaXLQ==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm-musleabihf@4.54.0': + resolution: {integrity: sha512-+pBrqEjaakN2ySv5RVrj/qLytYhPKEUwk+e3SFU5jTLHIcAtqh2rLrd/OkbNuHJpsBgxsD8ccJt5ga/SeG0JmA==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm64-gnu@4.54.0': + resolution: {integrity: sha512-NSqc7rE9wuUaRBsBp5ckQ5CVz5aIRKCwsoa6WMF7G01sX3/qHUw/z4pv+D+ahL1EIKy6Enpcnz1RY8pf7bjwng==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-arm64-musl@4.54.0': + resolution: {integrity: sha512-gr5vDbg3Bakga5kbdpqx81m2n9IX8M6gIMlQQIXiLTNeQW6CucvuInJ91EuCJ/JYvc+rcLLsDFcfAD1K7fMofg==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-loong64-gnu@4.54.0': + resolution: {integrity: sha512-gsrtB1NA3ZYj2vq0Rzkylo9ylCtW/PhpLEivlgWe0bpgtX5+9j9EZa0wtZiCjgu6zmSeZWyI/e2YRX1URozpIw==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-ppc64-gnu@4.54.0': + resolution: {integrity: sha512-y3qNOfTBStmFNq+t4s7Tmc9hW2ENtPg8FeUD/VShI7rKxNW7O4fFeaYbMsd3tpFlIg1Q8IapFgy7Q9i2BqeBvA==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-riscv64-gnu@4.54.0': + resolution: {integrity: sha512-89sepv7h2lIVPsFma8iwmccN7Yjjtgz0Rj/Ou6fEqg3HDhpCa+Et+YSufy27i6b0Wav69Qv4WBNl3Rs6pwhebQ==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-riscv64-musl@4.54.0': + resolution: {integrity: sha512-ZcU77ieh0M2Q8Ur7D5X7KvK+UxbXeDHwiOt/CPSBTI1fBmeDMivW0dPkdqkT4rOgDjrDDBUed9x4EgraIKoR2A==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-s390x-gnu@4.54.0': + resolution: {integrity: sha512-2AdWy5RdDF5+4YfG/YesGDDtbyJlC9LHmL6rZw6FurBJ5n4vFGupsOBGfwMRjBYH7qRQowT8D/U4LoSvVwOhSQ==} + cpu: [s390x] + os: [linux] + + '@rollup/rollup-linux-x64-gnu@4.54.0': + resolution: {integrity: sha512-WGt5J8Ij/rvyqpFexxk3ffKqqbLf9AqrTBbWDk7ApGUzaIs6V+s2s84kAxklFwmMF/vBNGrVdYgbblCOFFezMQ==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-linux-x64-musl@4.54.0': + resolution: {integrity: sha512-JzQmb38ATzHjxlPHuTH6tE7ojnMKM2kYNzt44LO/jJi8BpceEC8QuXYA908n8r3CNuG/B3BV8VR3Hi1rYtmPiw==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-openharmony-arm64@4.54.0': + resolution: {integrity: sha512-huT3fd0iC7jigGh7n3q/+lfPcXxBi+om/Rs3yiFxjvSxbSB6aohDFXbWvlspaqjeOh+hx7DDHS+5Es5qRkWkZg==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.54.0': + resolution: {integrity: sha512-c2V0W1bsKIKfbLMBu/WGBz6Yci8nJ/ZJdheE0EwB73N3MvHYKiKGs3mVilX4Gs70eGeDaMqEob25Tw2Gb9Nqyw==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.54.0': + resolution: {integrity: sha512-woEHgqQqDCkAzrDhvDipnSirm5vxUXtSKDYTVpZG3nUdW/VVB5VdCYA2iReSj/u3yCZzXID4kuKG7OynPnB3WQ==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.54.0': + resolution: {integrity: sha512-dzAc53LOuFvHwbCEOS0rPbXp6SIhAf2txMP5p6mGyOXXw5mWY8NGGbPMPrs4P1WItkfApDathBj/NzMLUZ9rtQ==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.54.0': + resolution: {integrity: sha512-hYT5d3YNdSh3mbCU1gwQyPgQd3T2ne0A3KG8KSBdav5TiBg6eInVmV+TeR5uHufiIgSFg0XsOWGW5/RhNcSvPg==} + cpu: [x64] + os: [win32] + + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + + '@types/estree@1.0.8': + resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + + '@types/node@25.0.3': + resolution: {integrity: sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==} + + '@typescript-eslint/eslint-plugin@8.50.1': + resolution: {integrity: sha512-PKhLGDq3JAg0Jk/aK890knnqduuI/Qj+udH7wCf0217IGi4gt+acgCyPVe79qoT+qKUvHMDQkwJeKW9fwl8Cyw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + '@typescript-eslint/parser': ^8.50.1 + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/parser@8.50.1': + resolution: {integrity: sha512-hM5faZwg7aVNa819m/5r7D0h0c9yC4DUlWAOvHAtISdFTc8xB86VmX5Xqabrama3wIPJ/q9RbGS1worb6JfnMg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/project-service@8.50.1': + resolution: {integrity: sha512-E1ur1MCVf+YiP89+o4Les/oBAVzmSbeRB0MQLfSlYtbWU17HPxZ6Bhs5iYmKZRALvEuBoXIZMOIRRc/P++Ortg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/scope-manager@8.50.1': + resolution: {integrity: sha512-mfRx06Myt3T4vuoHaKi8ZWNTPdzKPNBhiblze5N50//TSHOAQQevl/aolqA/BcqqbJ88GUnLqjjcBc8EWdBcVw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/tsconfig-utils@8.50.1': + resolution: {integrity: sha512-ooHmotT/lCWLXi55G4mvaUF60aJa012QzvLK0Y+Mp4WdSt17QhMhWOaBWeGTFVkb2gDgBe19Cxy1elPXylslDw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/type-utils@8.50.1': + resolution: {integrity: sha512-7J3bf022QZE42tYMO6SL+6lTPKFk/WphhRPe9Tw/el+cEwzLz1Jjz2PX3GtGQVxooLDKeMVmMt7fWpYRdG5Etg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/types@8.50.1': + resolution: {integrity: sha512-v5lFIS2feTkNyMhd7AucE/9j/4V9v5iIbpVRncjk/K0sQ6Sb+Np9fgYS/63n6nwqahHQvbmujeBL7mp07Q9mlA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/typescript-estree@8.50.1': + resolution: {integrity: sha512-woHPdW+0gj53aM+cxchymJCrh0cyS7BTIdcDxWUNsclr9VDkOSbqC13juHzxOmQ22dDkMZEpZB+3X1WpUvzgVQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/utils@8.50.1': + resolution: {integrity: sha512-lCLp8H1T9T7gPbEuJSnHwnSuO9mDf8mfK/Nion5mZmiEaQD9sWf9W4dfeFqRyqRjF06/kBuTmAqcs9sewM2NbQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/visitor-keys@8.50.1': + resolution: {integrity: sha512-IrDKrw7pCRUR94zeuCSUWQ+w8JEf5ZX5jl/e6AHGSLi1/zIr0lgutfn/7JpfCey+urpgQEdrZVYzCaVVKiTwhQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@vitest/coverage-v8@4.0.16': + resolution: {integrity: sha512-2rNdjEIsPRzsdu6/9Eq0AYAzYdpP6Bx9cje9tL3FE5XzXRQF1fNU9pe/1yE8fCrS0HD+fBtt6gLPh6LI57tX7A==} + peerDependencies: + '@vitest/browser': 4.0.16 + vitest: 4.0.16 + peerDependenciesMeta: + '@vitest/browser': + optional: true + + '@vitest/expect@4.0.16': + resolution: {integrity: sha512-eshqULT2It7McaJkQGLkPjPjNph+uevROGuIMJdG3V+0BSR2w9u6J9Lwu+E8cK5TETlfou8GRijhafIMhXsimA==} + + '@vitest/mocker@4.0.16': + resolution: {integrity: sha512-yb6k4AZxJTB+q9ycAvsoxGn+j/po0UaPgajllBgt1PzoMAAmJGYFdDk0uCcRcxb3BrME34I6u8gHZTQlkqSZpg==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0-0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@4.0.16': + resolution: {integrity: sha512-eNCYNsSty9xJKi/UdVD8Ou16alu7AYiS2fCPRs0b1OdhJiV89buAXQLpTbe+X8V9L6qrs9CqyvU7OaAopJYPsA==} + + '@vitest/runner@4.0.16': + resolution: {integrity: sha512-VWEDm5Wv9xEo80ctjORcTQRJ539EGPB3Pb9ApvVRAY1U/WkHXmmYISqU5E79uCwcW7xYUV38gwZD+RV755fu3Q==} + + '@vitest/snapshot@4.0.16': + resolution: {integrity: sha512-sf6NcrYhYBsSYefxnry+DR8n3UV4xWZwWxYbCJUt2YdvtqzSPR7VfGrY0zsv090DAbjFZsi7ZaMi1KnSRyK1XA==} + + '@vitest/spy@4.0.16': + resolution: {integrity: sha512-4jIOWjKP0ZUaEmJm00E0cOBLU+5WE0BpeNr3XN6TEF05ltro6NJqHWxXD0kA8/Zc8Nh23AT8WQxwNG+WeROupw==} + + '@vitest/utils@4.0.16': + resolution: {integrity: sha512-h8z9yYhV3e1LEfaQ3zdypIrnAg/9hguReGZoS7Gl0aBG5xgA410zBqECqmaF/+RkTggRsfnzc1XaAHA6bmUufA==} + + acorn-jsx@5.3.2: + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + + acorn@8.15.0: + resolution: {integrity: sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==} + engines: {node: '>=0.4.0'} + hasBin: true + + ajv@6.12.6: + resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + any-promise@1.3.0: + resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} + + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + ast-v8-to-istanbul@0.3.10: + resolution: {integrity: sha512-p4K7vMz2ZSk3wN8l5o3y2bJAoZXT3VuJI5OLTATY/01CYWumWvwkUw0SqDBnNq6IiTO3qDa1eSQDibAV8g7XOQ==} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + axios@1.13.2: + resolution: {integrity: sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==} + + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + + brace-expansion@1.1.12: + resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} + + brace-expansion@2.0.2: + resolution: {integrity: sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==} + + bundle-require@5.1.0: + resolution: {integrity: sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + peerDependencies: + esbuild: '>=0.18' + + cac@6.7.14: + resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} + engines: {node: '>=8'} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + callsites@3.1.0: + resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} + engines: {node: '>=6'} + + chai@6.2.2: + resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} + engines: {node: '>=18'} + + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + chokidar@4.0.3: + resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} + engines: {node: '>= 14.16.0'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + commander@4.1.1: + resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} + engines: {node: '>= 6'} + + concat-map@0.0.1: + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + + confbox@0.1.8: + resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} + + consola@3.4.2: + resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} + engines: {node: ^14.18.0 || >=16.10.0} + + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deep-is@0.1.4: + resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + + esbuild@0.27.2: + resolution: {integrity: sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==} + engines: {node: '>=18'} + hasBin: true + + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + eslint-scope@8.4.0: + resolution: {integrity: sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint-visitor-keys@3.4.3: + resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + eslint-visitor-keys@4.2.1: + resolution: {integrity: sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint@9.39.2: + resolution: {integrity: sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + hasBin: true + peerDependencies: + jiti: '*' + peerDependenciesMeta: + jiti: + optional: true + + espree@10.4.0: + resolution: {integrity: sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + esquery@1.6.0: + resolution: {integrity: sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==} + engines: {node: '>=0.10'} + + esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + + expect-type@1.3.0: + resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} + engines: {node: '>=12.0.0'} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + + fast-levenshtein@2.0.6: + resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + file-entry-cache@8.0.0: + resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} + engines: {node: '>=16.0.0'} + + find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + + fix-dts-default-cjs-exports@1.0.1: + resolution: {integrity: sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==} + + flat-cache@4.0.1: + resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} + engines: {node: '>=16'} + + flatted@3.3.3: + resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} + + follow-redirects@1.15.11: + resolution: {integrity: sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + form-data@4.0.5: + resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + engines: {node: '>= 6'} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + + globals@14.0.0: + resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} + engines: {node: '>=18'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + + html-escaper@2.0.2: + resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + + ignore@5.3.2: + resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} + engines: {node: '>= 4'} + + ignore@7.0.5: + resolution: {integrity: sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==} + engines: {node: '>= 4'} + + import-fresh@3.3.1: + resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} + engines: {node: '>=6'} + + imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + + is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + + is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + istanbul-lib-coverage@3.2.2: + resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==} + engines: {node: '>=8'} + + istanbul-lib-report@3.0.1: + resolution: {integrity: sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==} + engines: {node: '>=10'} + + istanbul-lib-source-maps@5.0.6: + resolution: {integrity: sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==} + engines: {node: '>=10'} + + istanbul-reports@3.2.0: + resolution: {integrity: sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==} + engines: {node: '>=8'} + + joycon@3.1.1: + resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} + engines: {node: '>=10'} + + js-tokens@9.0.1: + resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} + + js-yaml@4.1.1: + resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} + hasBin: true + + json-buffer@3.0.1: + resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + + json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + + json-stable-stringify-without-jsonify@1.0.1: + resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + + keyv@4.5.4: + resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + + levn@0.4.1: + resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} + engines: {node: '>= 0.8.0'} + + lilconfig@3.1.3: + resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} + engines: {node: '>=14'} + + lines-and-columns@1.2.4: + resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + + load-tsconfig@0.2.5: + resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + + lodash.merge@4.6.2: + resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + magicast@0.5.1: + resolution: {integrity: sha512-xrHS24IxaLrvuo613F719wvOIv9xPHFWQHuvGUBmPnCA/3MQxKI3b+r7n1jAoDHmsbC5bRhTZYR77invLAxVnw==} + + make-dir@4.0.0: + resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} + engines: {node: '>=10'} + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + minimatch@3.1.2: + resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + + minimatch@9.0.5: + resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} + engines: {node: '>=16 || 14 >=14.17'} + + mlly@1.8.0: + resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + mz@2.7.0: + resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} + + nanoid@3.3.11: + resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + natural-compare@1.4.0: + resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + + object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + + obug@2.1.1: + resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + + optionator@0.9.4: + resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} + engines: {node: '>= 0.8.0'} + + p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + + p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + + parent-module@1.0.1: + resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} + engines: {node: '>=6'} + + path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.3: + resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==} + engines: {node: '>=12'} + + pirates@4.0.7: + resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} + engines: {node: '>= 6'} + + pkg-types@1.3.1: + resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + + postcss-load-config@6.0.1: + resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} + engines: {node: '>= 18'} + peerDependencies: + jiti: '>=1.21.0' + postcss: '>=8.0.9' + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + jiti: + optional: true + postcss: + optional: true + tsx: + optional: true + yaml: + optional: true + + postcss@8.5.6: + resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} + engines: {node: ^10 || ^12 || >=14} + + prelude-ls@1.2.1: + resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} + engines: {node: '>= 0.8.0'} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + readdirp@4.1.2: + resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} + engines: {node: '>= 14.18.0'} + + resolve-from@4.0.0: + resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} + engines: {node: '>=4'} + + resolve-from@5.0.0: + resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} + engines: {node: '>=8'} + + rollup@4.54.0: + resolution: {integrity: sha512-3nk8Y3a9Ea8szgKhinMlGMhGMw89mqule3KWczxhIzqudyHdCIOHw8WJlj/r329fACjKLEh13ZSk7oE22kyeIw==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + semver@7.7.3: + resolution: {integrity: sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==} + engines: {node: '>=10'} + hasBin: true + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + source-map@0.7.6: + resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} + engines: {node: '>= 12'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@3.10.0: + resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + + strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + + sucrase@3.35.1: + resolution: {integrity: sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==} + engines: {node: '>=16 || 14 >=14.17'} + hasBin: true + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + thenify-all@1.6.0: + resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} + engines: {node: '>=0.8'} + + thenify@3.3.1: + resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + + tinyexec@1.0.2: + resolution: {integrity: sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==} + engines: {node: '>=18'} + + tinyglobby@0.2.15: + resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} + engines: {node: '>=12.0.0'} + + tinyrainbow@3.0.3: + resolution: {integrity: sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==} + engines: {node: '>=14.0.0'} + + tree-kill@1.2.2: + resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} + hasBin: true + + ts-api-utils@2.1.0: + resolution: {integrity: sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==} + engines: {node: '>=18.12'} + peerDependencies: + typescript: '>=4.8.4' + + ts-interface-checker@0.1.13: + resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} + + tsup@8.5.1: + resolution: {integrity: sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing==} + engines: {node: '>=18'} + hasBin: true + peerDependencies: + '@microsoft/api-extractor': ^7.36.0 + '@swc/core': ^1 + postcss: ^8.4.12 + typescript: '>=4.5.0' + peerDependenciesMeta: + '@microsoft/api-extractor': + optional: true + '@swc/core': + optional: true + postcss: + optional: true + typescript: + optional: true + + type-check@0.4.0: + resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} + engines: {node: '>= 0.8.0'} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + ufo@1.6.1: + resolution: {integrity: sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA==} + + undici-types@7.16.0: + resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + + vite@7.3.0: + resolution: {integrity: sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + jiti: '>=1.21.0' + less: ^4.0.0 + lightningcss: ^1.21.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + jiti: + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@4.0.16: + resolution: {integrity: sha512-E4t7DJ9pESL6E3I8nFjPa4xGUd3PmiWDLsDztS2qXSJWfHtbQnwAWylaBvSNY48I3vr8PTqIZlyK8TE3V3CA4Q==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.0.16 + '@vitest/browser-preview': 4.0.16 + '@vitest/browser-webdriverio': 4.0.16 + '@vitest/ui': 4.0.16 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + + word-wrap@1.2.5: + resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} + engines: {node: '>=0.10.0'} + + yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + +snapshots: + + '@babel/helper-string-parser@7.27.1': {} + + '@babel/helper-validator-identifier@7.28.5': {} + + '@babel/parser@7.28.5': + dependencies: + '@babel/types': 7.28.5 + + '@babel/types@7.28.5': + dependencies: + '@babel/helper-string-parser': 7.27.1 + '@babel/helper-validator-identifier': 7.28.5 + + '@bcoe/v8-coverage@1.0.2': {} + + '@esbuild/aix-ppc64@0.27.2': + optional: true + + '@esbuild/android-arm64@0.27.2': + optional: true + + '@esbuild/android-arm@0.27.2': + optional: true + + '@esbuild/android-x64@0.27.2': + optional: true + + '@esbuild/darwin-arm64@0.27.2': + optional: true + + '@esbuild/darwin-x64@0.27.2': + optional: true + + '@esbuild/freebsd-arm64@0.27.2': + optional: true + + '@esbuild/freebsd-x64@0.27.2': + optional: true + + '@esbuild/linux-arm64@0.27.2': + optional: true + + '@esbuild/linux-arm@0.27.2': + optional: true + + '@esbuild/linux-ia32@0.27.2': + optional: true + + '@esbuild/linux-loong64@0.27.2': + optional: true + + '@esbuild/linux-mips64el@0.27.2': + optional: true + + '@esbuild/linux-ppc64@0.27.2': + optional: true + + '@esbuild/linux-riscv64@0.27.2': + optional: true + + '@esbuild/linux-s390x@0.27.2': + optional: true + + '@esbuild/linux-x64@0.27.2': + optional: true + + '@esbuild/netbsd-arm64@0.27.2': + optional: true + + '@esbuild/netbsd-x64@0.27.2': + optional: true + + '@esbuild/openbsd-arm64@0.27.2': + optional: true + + '@esbuild/openbsd-x64@0.27.2': + optional: true + + '@esbuild/openharmony-arm64@0.27.2': + optional: true + + '@esbuild/sunos-x64@0.27.2': + optional: true + + '@esbuild/win32-arm64@0.27.2': + optional: true + + '@esbuild/win32-ia32@0.27.2': + optional: true + + '@esbuild/win32-x64@0.27.2': + optional: true + + '@eslint-community/eslint-utils@4.9.0(eslint@9.39.2)': + dependencies: + eslint: 9.39.2 + eslint-visitor-keys: 3.4.3 + + '@eslint-community/regexpp@4.12.2': {} + + '@eslint/config-array@0.21.1': + dependencies: + '@eslint/object-schema': 2.1.7 + debug: 4.4.3 + minimatch: 3.1.2 + transitivePeerDependencies: + - supports-color + + '@eslint/config-helpers@0.4.2': + dependencies: + '@eslint/core': 0.17.0 + + '@eslint/core@0.17.0': + dependencies: + '@types/json-schema': 7.0.15 + + '@eslint/eslintrc@3.3.3': + dependencies: + ajv: 6.12.6 + debug: 4.4.3 + espree: 10.4.0 + globals: 14.0.0 + ignore: 5.3.2 + import-fresh: 3.3.1 + js-yaml: 4.1.1 + minimatch: 3.1.2 + strip-json-comments: 3.1.1 + transitivePeerDependencies: + - supports-color + + '@eslint/js@9.39.2': {} + + '@eslint/object-schema@2.1.7': {} + + '@eslint/plugin-kit@0.4.1': + dependencies: + '@eslint/core': 0.17.0 + levn: 0.4.1 + + '@humanfs/core@0.19.1': {} + + '@humanfs/node@0.16.7': + dependencies: + '@humanfs/core': 0.19.1 + '@humanwhocodes/retry': 0.4.3 + + '@humanwhocodes/module-importer@1.0.1': {} + + '@humanwhocodes/retry@0.4.3': {} + + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@rollup/rollup-android-arm-eabi@4.54.0': + optional: true + + '@rollup/rollup-android-arm64@4.54.0': + optional: true + + '@rollup/rollup-darwin-arm64@4.54.0': + optional: true + + '@rollup/rollup-darwin-x64@4.54.0': + optional: true + + '@rollup/rollup-freebsd-arm64@4.54.0': + optional: true + + '@rollup/rollup-freebsd-x64@4.54.0': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.54.0': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.54.0': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.54.0': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.54.0': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.54.0': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.54.0': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.54.0': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.54.0': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.54.0': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.54.0': + optional: true + + '@rollup/rollup-linux-x64-musl@4.54.0': + optional: true + + '@rollup/rollup-openharmony-arm64@4.54.0': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.54.0': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.54.0': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.54.0': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.54.0': + optional: true + + '@standard-schema/spec@1.1.0': {} + + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + + '@types/deep-eql@4.0.2': {} + + '@types/estree@1.0.8': {} + + '@types/json-schema@7.0.15': {} + + '@types/node@25.0.3': + dependencies: + undici-types: 7.16.0 + + '@typescript-eslint/eslint-plugin@8.50.1(@typescript-eslint/parser@8.50.1(eslint@9.39.2)(typescript@5.9.3))(eslint@9.39.2)(typescript@5.9.3)': + dependencies: + '@eslint-community/regexpp': 4.12.2 + '@typescript-eslint/parser': 8.50.1(eslint@9.39.2)(typescript@5.9.3) + '@typescript-eslint/scope-manager': 8.50.1 + '@typescript-eslint/type-utils': 8.50.1(eslint@9.39.2)(typescript@5.9.3) + '@typescript-eslint/utils': 8.50.1(eslint@9.39.2)(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.50.1 + eslint: 9.39.2 + ignore: 7.0.5 + natural-compare: 1.4.0 + ts-api-utils: 2.1.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/parser@8.50.1(eslint@9.39.2)(typescript@5.9.3)': + dependencies: + '@typescript-eslint/scope-manager': 8.50.1 + '@typescript-eslint/types': 8.50.1 + '@typescript-eslint/typescript-estree': 8.50.1(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.50.1 + debug: 4.4.3 + eslint: 9.39.2 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/project-service@8.50.1(typescript@5.9.3)': + dependencies: + '@typescript-eslint/tsconfig-utils': 8.50.1(typescript@5.9.3) + '@typescript-eslint/types': 8.50.1 + debug: 4.4.3 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/scope-manager@8.50.1': + dependencies: + '@typescript-eslint/types': 8.50.1 + '@typescript-eslint/visitor-keys': 8.50.1 + + '@typescript-eslint/tsconfig-utils@8.50.1(typescript@5.9.3)': + dependencies: + typescript: 5.9.3 + + '@typescript-eslint/type-utils@8.50.1(eslint@9.39.2)(typescript@5.9.3)': + dependencies: + '@typescript-eslint/types': 8.50.1 + '@typescript-eslint/typescript-estree': 8.50.1(typescript@5.9.3) + '@typescript-eslint/utils': 8.50.1(eslint@9.39.2)(typescript@5.9.3) + debug: 4.4.3 + eslint: 9.39.2 + ts-api-utils: 2.1.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/types@8.50.1': {} + + '@typescript-eslint/typescript-estree@8.50.1(typescript@5.9.3)': + dependencies: + '@typescript-eslint/project-service': 8.50.1(typescript@5.9.3) + '@typescript-eslint/tsconfig-utils': 8.50.1(typescript@5.9.3) + '@typescript-eslint/types': 8.50.1 + '@typescript-eslint/visitor-keys': 8.50.1 + debug: 4.4.3 + minimatch: 9.0.5 + semver: 7.7.3 + tinyglobby: 0.2.15 + ts-api-utils: 2.1.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/utils@8.50.1(eslint@9.39.2)(typescript@5.9.3)': + dependencies: + '@eslint-community/eslint-utils': 4.9.0(eslint@9.39.2) + '@typescript-eslint/scope-manager': 8.50.1 + '@typescript-eslint/types': 8.50.1 + '@typescript-eslint/typescript-estree': 8.50.1(typescript@5.9.3) + eslint: 9.39.2 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/visitor-keys@8.50.1': + dependencies: + '@typescript-eslint/types': 8.50.1 + eslint-visitor-keys: 4.2.1 + + '@vitest/coverage-v8@4.0.16(vitest@4.0.16(@types/node@25.0.3))': + dependencies: + '@bcoe/v8-coverage': 1.0.2 + '@vitest/utils': 4.0.16 + ast-v8-to-istanbul: 0.3.10 + istanbul-lib-coverage: 3.2.2 + istanbul-lib-report: 3.0.1 + istanbul-lib-source-maps: 5.0.6 + istanbul-reports: 3.2.0 + magicast: 0.5.1 + obug: 2.1.1 + std-env: 3.10.0 + tinyrainbow: 3.0.3 + vitest: 4.0.16(@types/node@25.0.3) + transitivePeerDependencies: + - supports-color + + '@vitest/expect@4.0.16': + dependencies: + '@standard-schema/spec': 1.1.0 + '@types/chai': 5.2.3 + '@vitest/spy': 4.0.16 + '@vitest/utils': 4.0.16 + chai: 6.2.2 + tinyrainbow: 3.0.3 + + '@vitest/mocker@4.0.16(vite@7.3.0(@types/node@25.0.3))': + dependencies: + '@vitest/spy': 4.0.16 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.3.0(@types/node@25.0.3) + + '@vitest/pretty-format@4.0.16': + dependencies: + tinyrainbow: 3.0.3 + + '@vitest/runner@4.0.16': + dependencies: + '@vitest/utils': 4.0.16 + pathe: 2.0.3 + + '@vitest/snapshot@4.0.16': + dependencies: + '@vitest/pretty-format': 4.0.16 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@4.0.16': {} + + '@vitest/utils@4.0.16': + dependencies: + '@vitest/pretty-format': 4.0.16 + tinyrainbow: 3.0.3 + + acorn-jsx@5.3.2(acorn@8.15.0): + dependencies: + acorn: 8.15.0 + + acorn@8.15.0: {} + + ajv@6.12.6: + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + any-promise@1.3.0: {} + + argparse@2.0.1: {} + + assertion-error@2.0.1: {} + + ast-v8-to-istanbul@0.3.10: + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + estree-walker: 3.0.3 + js-tokens: 9.0.1 + + asynckit@0.4.0: {} + + axios@1.13.2: + dependencies: + follow-redirects: 1.15.11 + form-data: 4.0.5 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + + balanced-match@1.0.2: {} + + brace-expansion@1.1.12: + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + + brace-expansion@2.0.2: + dependencies: + balanced-match: 1.0.2 + + bundle-require@5.1.0(esbuild@0.27.2): + dependencies: + esbuild: 0.27.2 + load-tsconfig: 0.2.5 + + cac@6.7.14: {} + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + callsites@3.1.0: {} + + chai@6.2.2: {} + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + chokidar@4.0.3: + dependencies: + readdirp: 4.1.2 + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + commander@4.1.1: {} + + concat-map@0.0.1: {} + + confbox@0.1.8: {} + + consola@3.4.2: {} + + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + deep-is@0.1.4: {} + + delayed-stream@1.0.0: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-module-lexer@1.7.0: {} + + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + + esbuild@0.27.2: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.2 + '@esbuild/android-arm': 0.27.2 + '@esbuild/android-arm64': 0.27.2 + '@esbuild/android-x64': 0.27.2 + '@esbuild/darwin-arm64': 0.27.2 + '@esbuild/darwin-x64': 0.27.2 + '@esbuild/freebsd-arm64': 0.27.2 + '@esbuild/freebsd-x64': 0.27.2 + '@esbuild/linux-arm': 0.27.2 + '@esbuild/linux-arm64': 0.27.2 + '@esbuild/linux-ia32': 0.27.2 + '@esbuild/linux-loong64': 0.27.2 + '@esbuild/linux-mips64el': 0.27.2 + '@esbuild/linux-ppc64': 0.27.2 + '@esbuild/linux-riscv64': 0.27.2 + '@esbuild/linux-s390x': 0.27.2 + '@esbuild/linux-x64': 0.27.2 + '@esbuild/netbsd-arm64': 0.27.2 + '@esbuild/netbsd-x64': 0.27.2 + '@esbuild/openbsd-arm64': 0.27.2 + '@esbuild/openbsd-x64': 0.27.2 + '@esbuild/openharmony-arm64': 0.27.2 + '@esbuild/sunos-x64': 0.27.2 + '@esbuild/win32-arm64': 0.27.2 + '@esbuild/win32-ia32': 0.27.2 + '@esbuild/win32-x64': 0.27.2 + + escape-string-regexp@4.0.0: {} + + eslint-scope@8.4.0: + dependencies: + esrecurse: 4.3.0 + estraverse: 5.3.0 + + eslint-visitor-keys@3.4.3: {} + + eslint-visitor-keys@4.2.1: {} + + eslint@9.39.2: + dependencies: + '@eslint-community/eslint-utils': 4.9.0(eslint@9.39.2) + '@eslint-community/regexpp': 4.12.2 + '@eslint/config-array': 0.21.1 + '@eslint/config-helpers': 0.4.2 + '@eslint/core': 0.17.0 + '@eslint/eslintrc': 3.3.3 + '@eslint/js': 9.39.2 + '@eslint/plugin-kit': 0.4.1 + '@humanfs/node': 0.16.7 + '@humanwhocodes/module-importer': 1.0.1 + '@humanwhocodes/retry': 0.4.3 + '@types/estree': 1.0.8 + ajv: 6.12.6 + chalk: 4.1.2 + cross-spawn: 7.0.6 + debug: 4.4.3 + escape-string-regexp: 4.0.0 + eslint-scope: 8.4.0 + eslint-visitor-keys: 4.2.1 + espree: 10.4.0 + esquery: 1.6.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 8.0.0 + find-up: 5.0.0 + glob-parent: 6.0.2 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + json-stable-stringify-without-jsonify: 1.0.1 + lodash.merge: 4.6.2 + minimatch: 3.1.2 + natural-compare: 1.4.0 + optionator: 0.9.4 + transitivePeerDependencies: + - supports-color + + espree@10.4.0: + dependencies: + acorn: 8.15.0 + acorn-jsx: 5.3.2(acorn@8.15.0) + eslint-visitor-keys: 4.2.1 + + esquery@1.6.0: + dependencies: + estraverse: 5.3.0 + + esrecurse@4.3.0: + dependencies: + estraverse: 5.3.0 + + estraverse@5.3.0: {} + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.8 + + esutils@2.0.3: {} + + expect-type@1.3.0: {} + + fast-deep-equal@3.1.3: {} + + fast-json-stable-stringify@2.1.0: {} + + fast-levenshtein@2.0.6: {} + + fdir@6.5.0(picomatch@4.0.3): + optionalDependencies: + picomatch: 4.0.3 + + file-entry-cache@8.0.0: + dependencies: + flat-cache: 4.0.1 + + find-up@5.0.0: + dependencies: + locate-path: 6.0.0 + path-exists: 4.0.0 + + fix-dts-default-cjs-exports@1.0.1: + dependencies: + magic-string: 0.30.21 + mlly: 1.8.0 + rollup: 4.54.0 + + flat-cache@4.0.1: + dependencies: + flatted: 3.3.3 + keyv: 4.5.4 + + flatted@3.3.3: {} + + follow-redirects@1.15.11: {} + + form-data@4.0.5: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + + fsevents@2.3.3: + optional: true + + function-bind@1.1.2: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + + glob-parent@6.0.2: + dependencies: + is-glob: 4.0.3 + + globals@14.0.0: {} + + gopd@1.2.0: {} + + has-flag@4.0.0: {} + + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + + html-escaper@2.0.2: {} + + ignore@5.3.2: {} + + ignore@7.0.5: {} + + import-fresh@3.3.1: + dependencies: + parent-module: 1.0.1 + resolve-from: 4.0.0 + + imurmurhash@0.1.4: {} + + is-extglob@2.1.1: {} + + is-glob@4.0.3: + dependencies: + is-extglob: 2.1.1 + + isexe@2.0.0: {} + + istanbul-lib-coverage@3.2.2: {} + + istanbul-lib-report@3.0.1: + dependencies: + istanbul-lib-coverage: 3.2.2 + make-dir: 4.0.0 + supports-color: 7.2.0 + + istanbul-lib-source-maps@5.0.6: + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + debug: 4.4.3 + istanbul-lib-coverage: 3.2.2 + transitivePeerDependencies: + - supports-color + + istanbul-reports@3.2.0: + dependencies: + html-escaper: 2.0.2 + istanbul-lib-report: 3.0.1 + + joycon@3.1.1: {} + + js-tokens@9.0.1: {} + + js-yaml@4.1.1: + dependencies: + argparse: 2.0.1 + + json-buffer@3.0.1: {} + + json-schema-traverse@0.4.1: {} + + json-stable-stringify-without-jsonify@1.0.1: {} + + keyv@4.5.4: + dependencies: + json-buffer: 3.0.1 + + levn@0.4.1: + dependencies: + prelude-ls: 1.2.1 + type-check: 0.4.0 + + lilconfig@3.1.3: {} + + lines-and-columns@1.2.4: {} + + load-tsconfig@0.2.5: {} + + locate-path@6.0.0: + dependencies: + p-locate: 5.0.0 + + lodash.merge@4.6.2: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + magicast@0.5.1: + dependencies: + '@babel/parser': 7.28.5 + '@babel/types': 7.28.5 + source-map-js: 1.2.1 + + make-dir@4.0.0: + dependencies: + semver: 7.7.3 + + math-intrinsics@1.1.0: {} + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + minimatch@3.1.2: + dependencies: + brace-expansion: 1.1.12 + + minimatch@9.0.5: + dependencies: + brace-expansion: 2.0.2 + + mlly@1.8.0: + dependencies: + acorn: 8.15.0 + pathe: 2.0.3 + pkg-types: 1.3.1 + ufo: 1.6.1 + + ms@2.1.3: {} + + mz@2.7.0: + dependencies: + any-promise: 1.3.0 + object-assign: 4.1.1 + thenify-all: 1.6.0 + + nanoid@3.3.11: {} + + natural-compare@1.4.0: {} + + object-assign@4.1.1: {} + + obug@2.1.1: {} + + optionator@0.9.4: + dependencies: + deep-is: 0.1.4 + fast-levenshtein: 2.0.6 + levn: 0.4.1 + prelude-ls: 1.2.1 + type-check: 0.4.0 + word-wrap: 1.2.5 + + p-limit@3.1.0: + dependencies: + yocto-queue: 0.1.0 + + p-locate@5.0.0: + dependencies: + p-limit: 3.1.0 + + parent-module@1.0.1: + dependencies: + callsites: 3.1.0 + + path-exists@4.0.0: {} + + path-key@3.1.1: {} + + pathe@2.0.3: {} + + picocolors@1.1.1: {} + + picomatch@4.0.3: {} + + pirates@4.0.7: {} + + pkg-types@1.3.1: + dependencies: + confbox: 0.1.8 + mlly: 1.8.0 + pathe: 2.0.3 + + postcss-load-config@6.0.1(postcss@8.5.6): + dependencies: + lilconfig: 3.1.3 + optionalDependencies: + postcss: 8.5.6 + + postcss@8.5.6: + dependencies: + nanoid: 3.3.11 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + prelude-ls@1.2.1: {} + + proxy-from-env@1.1.0: {} + + punycode@2.3.1: {} + + readdirp@4.1.2: {} + + resolve-from@4.0.0: {} + + resolve-from@5.0.0: {} + + rollup@4.54.0: + dependencies: + '@types/estree': 1.0.8 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.54.0 + '@rollup/rollup-android-arm64': 4.54.0 + '@rollup/rollup-darwin-arm64': 4.54.0 + '@rollup/rollup-darwin-x64': 4.54.0 + '@rollup/rollup-freebsd-arm64': 4.54.0 + '@rollup/rollup-freebsd-x64': 4.54.0 + '@rollup/rollup-linux-arm-gnueabihf': 4.54.0 + '@rollup/rollup-linux-arm-musleabihf': 4.54.0 + '@rollup/rollup-linux-arm64-gnu': 4.54.0 + '@rollup/rollup-linux-arm64-musl': 4.54.0 + '@rollup/rollup-linux-loong64-gnu': 4.54.0 + '@rollup/rollup-linux-ppc64-gnu': 4.54.0 + '@rollup/rollup-linux-riscv64-gnu': 4.54.0 + '@rollup/rollup-linux-riscv64-musl': 4.54.0 + '@rollup/rollup-linux-s390x-gnu': 4.54.0 + '@rollup/rollup-linux-x64-gnu': 4.54.0 + '@rollup/rollup-linux-x64-musl': 4.54.0 + '@rollup/rollup-openharmony-arm64': 4.54.0 + '@rollup/rollup-win32-arm64-msvc': 4.54.0 + '@rollup/rollup-win32-ia32-msvc': 4.54.0 + '@rollup/rollup-win32-x64-gnu': 4.54.0 + '@rollup/rollup-win32-x64-msvc': 4.54.0 + fsevents: 2.3.3 + + semver@7.7.3: {} + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + siginfo@2.0.0: {} + + source-map-js@1.2.1: {} + + source-map@0.7.6: {} + + stackback@0.0.2: {} + + std-env@3.10.0: {} + + strip-json-comments@3.1.1: {} + + sucrase@3.35.1: + dependencies: + '@jridgewell/gen-mapping': 0.3.13 + commander: 4.1.1 + lines-and-columns: 1.2.4 + mz: 2.7.0 + pirates: 4.0.7 + tinyglobby: 0.2.15 + ts-interface-checker: 0.1.13 + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + thenify-all@1.6.0: + dependencies: + thenify: 3.3.1 + + thenify@3.3.1: + dependencies: + any-promise: 1.3.0 + + tinybench@2.9.0: {} + + tinyexec@0.3.2: {} + + tinyexec@1.0.2: {} + + tinyglobby@0.2.15: + dependencies: + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + + tinyrainbow@3.0.3: {} + + tree-kill@1.2.2: {} + + ts-api-utils@2.1.0(typescript@5.9.3): + dependencies: + typescript: 5.9.3 + + ts-interface-checker@0.1.13: {} + + tsup@8.5.1(postcss@8.5.6)(typescript@5.9.3): + dependencies: + bundle-require: 5.1.0(esbuild@0.27.2) + cac: 6.7.14 + chokidar: 4.0.3 + consola: 3.4.2 + debug: 4.4.3 + esbuild: 0.27.2 + fix-dts-default-cjs-exports: 1.0.1 + joycon: 3.1.1 + picocolors: 1.1.1 + postcss-load-config: 6.0.1(postcss@8.5.6) + resolve-from: 5.0.0 + rollup: 4.54.0 + source-map: 0.7.6 + sucrase: 3.35.1 + tinyexec: 0.3.2 + tinyglobby: 0.2.15 + tree-kill: 1.2.2 + optionalDependencies: + postcss: 8.5.6 + typescript: 5.9.3 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + + type-check@0.4.0: + dependencies: + prelude-ls: 1.2.1 + + typescript@5.9.3: {} + + ufo@1.6.1: {} + + undici-types@7.16.0: {} + + uri-js@4.4.1: + dependencies: + punycode: 2.3.1 + + vite@7.3.0(@types/node@25.0.3): + dependencies: + esbuild: 0.27.2 + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + postcss: 8.5.6 + rollup: 4.54.0 + tinyglobby: 0.2.15 + optionalDependencies: + '@types/node': 25.0.3 + fsevents: 2.3.3 + + vitest@4.0.16(@types/node@25.0.3): + dependencies: + '@vitest/expect': 4.0.16 + '@vitest/mocker': 4.0.16(vite@7.3.0(@types/node@25.0.3)) + '@vitest/pretty-format': 4.0.16 + '@vitest/runner': 4.0.16 + '@vitest/snapshot': 4.0.16 + '@vitest/spy': 4.0.16 + '@vitest/utils': 4.0.16 + es-module-lexer: 1.7.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.1 + pathe: 2.0.3 + picomatch: 4.0.3 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 1.0.2 + tinyglobby: 0.2.15 + tinyrainbow: 3.0.3 + vite: 7.3.0(@types/node@25.0.3) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 25.0.3 + transitivePeerDependencies: + - jiti + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - terser + - tsx + - yaml + + which@2.0.2: + dependencies: + isexe: 2.0.0 + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + + word-wrap@1.2.5: {} + + yocto-queue@0.1.0: {} diff --git a/sdks/nodejs-client/pnpm-workspace.yaml b/sdks/nodejs-client/pnpm-workspace.yaml new file mode 100644 index 0000000000..efc037aa84 --- /dev/null +++ b/sdks/nodejs-client/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +onlyBuiltDependencies: + - esbuild diff --git a/sdks/nodejs-client/scripts/publish.sh b/sdks/nodejs-client/scripts/publish.sh new file mode 100755 index 0000000000..043cac046d --- /dev/null +++ b/sdks/nodejs-client/scripts/publish.sh @@ -0,0 +1,261 @@ +#!/usr/bin/env bash +# +# Dify Node.js SDK Publish Script +# ================================ +# A beautiful and reliable script to publish the SDK to npm +# +# Usage: +# ./scripts/publish.sh # Normal publish +# ./scripts/publish.sh --dry-run # Test without publishing +# ./scripts/publish.sh --skip-tests # Skip tests (not recommended) +# + +set -euo pipefail + +# ============================================================================ +# Colors and Formatting +# ============================================================================ +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +MAGENTA='\033[0;35m' +CYAN='\033[0;36m' +BOLD='\033[1m' +DIM='\033[2m' +NC='\033[0m' # No Color + +# ============================================================================ +# Helper Functions +# ============================================================================ +print_banner() { + echo -e "${CYAN}" + echo "╔═══════════════════════════════════════════════════════════════╗" + echo "║ ║" + echo "║ 🚀 Dify Node.js SDK Publish Script 🚀 ║" + echo "║ ║" + echo "╚═══════════════════════════════════════════════════════════════╝" + echo -e "${NC}" +} + +info() { + echo -e "${BLUE}ℹ ${NC}$1" +} + +success() { + echo -e "${GREEN}✔ ${NC}$1" +} + +warning() { + echo -e "${YELLOW}⚠ ${NC}$1" +} + +error() { + echo -e "${RED}✖ ${NC}$1" +} + +step() { + echo -e "\n${MAGENTA}▶ ${BOLD}$1${NC}" +} + +divider() { + echo -e "${DIM}─────────────────────────────────────────────────────────────────${NC}" +} + +# ============================================================================ +# Configuration +# ============================================================================ +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +DRY_RUN=false +SKIP_TESTS=false + +# Parse arguments +for arg in "$@"; do + case $arg in + --dry-run) + DRY_RUN=true + ;; + --skip-tests) + SKIP_TESTS=true + ;; + --help|-h) + echo "Usage: $0 [options]" + echo "" + echo "Options:" + echo " --dry-run Run without actually publishing" + echo " --skip-tests Skip running tests (not recommended)" + echo " --help, -h Show this help message" + exit 0 + ;; + esac +done + +# ============================================================================ +# Main Script +# ============================================================================ +main() { + print_banner + cd "$PROJECT_DIR" + + # Show mode + if [[ "$DRY_RUN" == true ]]; then + warning "Running in DRY-RUN mode - no actual publish will occur" + divider + fi + + # ======================================================================== + # Step 1: Environment Check + # ======================================================================== + step "Step 1/6: Checking environment..." + + # Check Node.js + if ! command -v node &> /dev/null; then + error "Node.js is not installed" + exit 1 + fi + NODE_VERSION=$(node -v) + success "Node.js: $NODE_VERSION" + + # Check npm + if ! command -v npm &> /dev/null; then + error "npm is not installed" + exit 1 + fi + NPM_VERSION=$(npm -v) + success "npm: v$NPM_VERSION" + + # Check pnpm (optional, for local dev) + if command -v pnpm &> /dev/null; then + PNPM_VERSION=$(pnpm -v) + success "pnpm: v$PNPM_VERSION" + else + info "pnpm not found (optional)" + fi + + # Check npm login status + if ! npm whoami &> /dev/null; then + error "Not logged in to npm. Run 'npm login' first." + exit 1 + fi + NPM_USER=$(npm whoami) + success "Logged in as: ${BOLD}$NPM_USER${NC}" + + # ======================================================================== + # Step 2: Read Package Info + # ======================================================================== + step "Step 2/6: Reading package info..." + + PACKAGE_NAME=$(node -p "require('./package.json').name") + PACKAGE_VERSION=$(node -p "require('./package.json').version") + + success "Package: ${BOLD}$PACKAGE_NAME${NC}" + success "Version: ${BOLD}$PACKAGE_VERSION${NC}" + + # Check if version already exists on npm + if npm view "$PACKAGE_NAME@$PACKAGE_VERSION" version &> /dev/null; then + error "Version $PACKAGE_VERSION already exists on npm!" + echo "" + info "Current published versions:" + npm view "$PACKAGE_NAME" versions --json 2>/dev/null | tail -5 + echo "" + warning "Please update the version in package.json before publishing." + exit 1 + fi + success "Version $PACKAGE_VERSION is available" + + # ======================================================================== + # Step 3: Install Dependencies + # ======================================================================== + step "Step 3/6: Installing dependencies..." + + if command -v pnpm &> /dev/null; then + pnpm install --frozen-lockfile 2>/dev/null || pnpm install + else + npm ci 2>/dev/null || npm install + fi + success "Dependencies installed" + + # ======================================================================== + # Step 4: Run Tests + # ======================================================================== + step "Step 4/6: Running tests..." + + if [[ "$SKIP_TESTS" == true ]]; then + warning "Skipping tests (--skip-tests flag)" + else + if command -v pnpm &> /dev/null; then + pnpm test + else + npm test + fi + success "All tests passed" + fi + + # ======================================================================== + # Step 5: Build + # ======================================================================== + step "Step 5/6: Building package..." + + # Clean previous build + rm -rf dist + + if command -v pnpm &> /dev/null; then + pnpm run build + else + npm run build + fi + success "Build completed" + + # Verify build output + if [[ ! -f "dist/index.js" ]]; then + error "Build failed - dist/index.js not found" + exit 1 + fi + if [[ ! -f "dist/index.d.ts" ]]; then + error "Build failed - dist/index.d.ts not found" + exit 1 + fi + success "Build output verified" + + # ======================================================================== + # Step 6: Publish + # ======================================================================== + step "Step 6/6: Publishing to npm..." + + divider + echo -e "${CYAN}Package contents:${NC}" + npm pack --dry-run 2>&1 | head -30 + divider + + if [[ "$DRY_RUN" == true ]]; then + warning "DRY-RUN: Skipping actual publish" + echo "" + info "To publish for real, run without --dry-run flag" + else + echo "" + echo -e "${YELLOW}About to publish ${BOLD}$PACKAGE_NAME@$PACKAGE_VERSION${NC}${YELLOW} to npm${NC}" + echo -e "${DIM}Press Enter to continue, or Ctrl+C to cancel...${NC}" + read -r + + npm publish --access public + + echo "" + success "🎉 Successfully published ${BOLD}$PACKAGE_NAME@$PACKAGE_VERSION${NC} to npm!" + echo "" + echo -e "${GREEN}Install with:${NC}" + echo -e " ${CYAN}npm install $PACKAGE_NAME${NC}" + echo -e " ${CYAN}pnpm add $PACKAGE_NAME${NC}" + echo -e " ${CYAN}yarn add $PACKAGE_NAME${NC}" + echo "" + echo -e "${GREEN}View on npm:${NC}" + echo -e " ${CYAN}https://www.npmjs.com/package/$PACKAGE_NAME${NC}" + fi + + divider + echo -e "${GREEN}${BOLD}✨ All done!${NC}" +} + +# Run main function +main "$@" diff --git a/sdks/nodejs-client/src/client/base.test.js b/sdks/nodejs-client/src/client/base.test.js new file mode 100644 index 0000000000..5e1b21d0f1 --- /dev/null +++ b/sdks/nodejs-client/src/client/base.test.js @@ -0,0 +1,175 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DifyClient } from "./base"; +import { ValidationError } from "../errors/dify-error"; +import { createHttpClientWithSpies } from "../../tests/test-utils"; + +describe("DifyClient base", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("getRoot calls root endpoint", async () => { + const { client, request } = createHttpClientWithSpies(); + const dify = new DifyClient(client); + + await dify.getRoot(); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/", + }); + }); + + it("getApplicationParameters includes optional user", async () => { + const { client, request } = createHttpClientWithSpies(); + const dify = new DifyClient(client); + + await dify.getApplicationParameters(); + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/parameters", + query: undefined, + }); + + await dify.getApplicationParameters("user-1"); + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/parameters", + query: { user: "user-1" }, + }); + }); + + it("getMeta includes optional user", async () => { + const { client, request } = createHttpClientWithSpies(); + const dify = new DifyClient(client); + + await dify.getMeta("user-1"); + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/meta", + query: { user: "user-1" }, + }); + }); + + it("getInfo and getSite support optional user", async () => { + const { client, request } = createHttpClientWithSpies(); + const dify = new DifyClient(client); + + await dify.getInfo(); + await dify.getSite("user"); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/info", + query: undefined, + }); + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/site", + query: { user: "user" }, + }); + }); + + it("messageFeedback builds payload from request object", async () => { + const { client, request } = createHttpClientWithSpies(); + const dify = new DifyClient(client); + + await dify.messageFeedback({ + messageId: "msg", + user: "user", + rating: "like", + content: "good", + }); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/messages/msg/feedbacks", + data: { user: "user", rating: "like", content: "good" }, + }); + }); + + it("fileUpload appends user to form data", async () => { + const { client, request } = createHttpClientWithSpies(); + const dify = new DifyClient(client); + const form = { append: vi.fn(), getHeaders: () => ({}) }; + + await dify.fileUpload(form, "user"); + + expect(form.append).toHaveBeenCalledWith("user", "user"); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/files/upload", + data: form, + }); + }); + + it("filePreview uses arraybuffer response", async () => { + const { client, request } = createHttpClientWithSpies(); + const dify = new DifyClient(client); + + await dify.filePreview("file", "user", true); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/files/file/preview", + query: { user: "user", as_attachment: "true" }, + responseType: "arraybuffer", + }); + }); + + it("audioToText appends user and sends form", async () => { + const { client, request } = createHttpClientWithSpies(); + const dify = new DifyClient(client); + const form = { append: vi.fn(), getHeaders: () => ({}) }; + + await dify.audioToText(form, "user"); + + expect(form.append).toHaveBeenCalledWith("user", "user"); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/audio-to-text", + data: form, + }); + }); + + it("textToAudio supports streaming and message id", async () => { + const { client, request, requestBinaryStream } = createHttpClientWithSpies(); + const dify = new DifyClient(client); + + await dify.textToAudio({ + user: "user", + message_id: "msg", + streaming: true, + }); + + expect(requestBinaryStream).toHaveBeenCalledWith({ + method: "POST", + path: "/text-to-audio", + data: { + user: "user", + message_id: "msg", + streaming: true, + }, + }); + + await dify.textToAudio("hello", "user", false, "voice"); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/text-to-audio", + data: { + text: "hello", + user: "user", + streaming: false, + voice: "voice", + }, + responseType: "arraybuffer", + }); + }); + + it("textToAudio requires text or message id", async () => { + const { client } = createHttpClientWithSpies(); + const dify = new DifyClient(client); + + expect(() => dify.textToAudio({ user: "user" })).toThrow(ValidationError); + }); +}); diff --git a/sdks/nodejs-client/src/client/base.ts b/sdks/nodejs-client/src/client/base.ts new file mode 100644 index 0000000000..0fa535a488 --- /dev/null +++ b/sdks/nodejs-client/src/client/base.ts @@ -0,0 +1,284 @@ +import type { + BinaryStream, + DifyClientConfig, + DifyResponse, + MessageFeedbackRequest, + QueryParams, + RequestMethod, + TextToAudioRequest, +} from "../types/common"; +import { HttpClient } from "../http/client"; +import { ensureNonEmptyString, ensureRating } from "./validation"; +import { FileUploadError, ValidationError } from "../errors/dify-error"; +import { isFormData } from "../http/form-data"; + +const toConfig = ( + init: string | DifyClientConfig, + baseUrl?: string +): DifyClientConfig => { + if (typeof init === "string") { + return { + apiKey: init, + baseUrl, + }; + } + return init; +}; + +const appendUserToFormData = (form: unknown, user: string): void => { + if (!isFormData(form)) { + throw new FileUploadError("FormData is required for file uploads"); + } + if (typeof form.append === "function") { + form.append("user", user); + } +}; + +export class DifyClient { + protected http: HttpClient; + + constructor(config: string | DifyClientConfig | HttpClient, baseUrl?: string) { + if (config instanceof HttpClient) { + this.http = config; + } else { + this.http = new HttpClient(toConfig(config, baseUrl)); + } + } + + updateApiKey(apiKey: string): void { + ensureNonEmptyString(apiKey, "apiKey"); + this.http.updateApiKey(apiKey); + } + + getHttpClient(): HttpClient { + return this.http; + } + + sendRequest( + method: RequestMethod, + endpoint: string, + data: unknown = null, + params: QueryParams | null = null, + stream = false, + headerParams: Record = {} + ): ReturnType { + return this.http.requestRaw({ + method, + path: endpoint, + data, + query: params ?? undefined, + headers: headerParams, + responseType: stream ? "stream" : "json", + }); + } + + getRoot(): Promise> { + return this.http.request({ + method: "GET", + path: "/", + }); + } + + getApplicationParameters(user?: string): Promise> { + if (user) { + ensureNonEmptyString(user, "user"); + } + return this.http.request({ + method: "GET", + path: "/parameters", + query: user ? { user } : undefined, + }); + } + + async getParameters(user?: string): Promise> { + return this.getApplicationParameters(user); + } + + getMeta(user?: string): Promise> { + if (user) { + ensureNonEmptyString(user, "user"); + } + return this.http.request({ + method: "GET", + path: "/meta", + query: user ? { user } : undefined, + }); + } + + messageFeedback( + request: MessageFeedbackRequest + ): Promise>>; + messageFeedback( + messageId: string, + rating: "like" | "dislike" | null, + user: string, + content?: string + ): Promise>>; + messageFeedback( + messageIdOrRequest: string | MessageFeedbackRequest, + rating?: "like" | "dislike" | null, + user?: string, + content?: string + ): Promise>> { + let messageId: string; + const payload: Record = {}; + + if (typeof messageIdOrRequest === "string") { + messageId = messageIdOrRequest; + ensureNonEmptyString(messageId, "messageId"); + ensureNonEmptyString(user, "user"); + payload.user = user; + if (rating !== undefined && rating !== null) { + ensureRating(rating); + payload.rating = rating; + } + if (content !== undefined) { + payload.content = content; + } + } else { + const request = messageIdOrRequest; + messageId = request.messageId; + ensureNonEmptyString(messageId, "messageId"); + ensureNonEmptyString(request.user, "user"); + payload.user = request.user; + if (request.rating !== undefined && request.rating !== null) { + ensureRating(request.rating); + payload.rating = request.rating; + } + if (request.content !== undefined) { + payload.content = request.content; + } + } + + return this.http.request({ + method: "POST", + path: `/messages/${messageId}/feedbacks`, + data: payload, + }); + } + + getInfo(user?: string): Promise> { + if (user) { + ensureNonEmptyString(user, "user"); + } + return this.http.request({ + method: "GET", + path: "/info", + query: user ? { user } : undefined, + }); + } + + getSite(user?: string): Promise> { + if (user) { + ensureNonEmptyString(user, "user"); + } + return this.http.request({ + method: "GET", + path: "/site", + query: user ? { user } : undefined, + }); + } + + fileUpload(form: unknown, user: string): Promise> { + if (!isFormData(form)) { + throw new FileUploadError("FormData is required for file uploads"); + } + ensureNonEmptyString(user, "user"); + appendUserToFormData(form, user); + return this.http.request({ + method: "POST", + path: "/files/upload", + data: form, + }); + } + + filePreview( + fileId: string, + user: string, + asAttachment?: boolean + ): Promise> { + ensureNonEmptyString(fileId, "fileId"); + ensureNonEmptyString(user, "user"); + return this.http.request({ + method: "GET", + path: `/files/${fileId}/preview`, + query: { + user, + as_attachment: asAttachment ? "true" : undefined, + }, + responseType: "arraybuffer", + }); + } + + audioToText(form: unknown, user: string): Promise> { + if (!isFormData(form)) { + throw new FileUploadError("FormData is required for audio uploads"); + } + ensureNonEmptyString(user, "user"); + appendUserToFormData(form, user); + return this.http.request({ + method: "POST", + path: "/audio-to-text", + data: form, + }); + } + + textToAudio( + request: TextToAudioRequest + ): Promise | BinaryStream>; + textToAudio( + text: string, + user: string, + streaming?: boolean, + voice?: string + ): Promise | BinaryStream>; + textToAudio( + textOrRequest: string | TextToAudioRequest, + user?: string, + streaming = false, + voice?: string + ): Promise | BinaryStream> { + let payload: TextToAudioRequest; + + if (typeof textOrRequest === "string") { + ensureNonEmptyString(textOrRequest, "text"); + ensureNonEmptyString(user, "user"); + payload = { + text: textOrRequest, + user, + streaming, + }; + if (voice) { + payload.voice = voice; + } + } else { + payload = { ...textOrRequest }; + ensureNonEmptyString(payload.user, "user"); + if (payload.text !== undefined && payload.text !== null) { + ensureNonEmptyString(payload.text, "text"); + } + if (payload.message_id !== undefined && payload.message_id !== null) { + ensureNonEmptyString(payload.message_id, "messageId"); + } + if (!payload.text && !payload.message_id) { + throw new ValidationError("text or message_id is required"); + } + payload.streaming = payload.streaming ?? false; + } + + if (payload.streaming) { + return this.http.requestBinaryStream({ + method: "POST", + path: "/text-to-audio", + data: payload, + }); + } + + return this.http.request({ + method: "POST", + path: "/text-to-audio", + data: payload, + responseType: "arraybuffer", + }); + } +} diff --git a/sdks/nodejs-client/src/client/chat.test.js b/sdks/nodejs-client/src/client/chat.test.js new file mode 100644 index 0000000000..a97c9d4a5c --- /dev/null +++ b/sdks/nodejs-client/src/client/chat.test.js @@ -0,0 +1,239 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { ChatClient } from "./chat"; +import { ValidationError } from "../errors/dify-error"; +import { createHttpClientWithSpies } from "../../tests/test-utils"; + +describe("ChatClient", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("creates chat messages in blocking mode", async () => { + const { client, request } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + await chat.createChatMessage({ input: "x" }, "hello", "user", false, null); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/chat-messages", + data: { + inputs: { input: "x" }, + query: "hello", + user: "user", + response_mode: "blocking", + files: undefined, + }, + }); + }); + + it("creates chat messages in streaming mode", async () => { + const { client, requestStream } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + await chat.createChatMessage({ + inputs: { input: "x" }, + query: "hello", + user: "user", + response_mode: "streaming", + }); + + expect(requestStream).toHaveBeenCalledWith({ + method: "POST", + path: "/chat-messages", + data: { + inputs: { input: "x" }, + query: "hello", + user: "user", + response_mode: "streaming", + }, + }); + }); + + it("stops chat messages", async () => { + const { client, request } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + await chat.stopChatMessage("task", "user"); + await chat.stopMessage("task", "user"); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/chat-messages/task/stop", + data: { user: "user" }, + }); + }); + + it("gets suggested questions", async () => { + const { client, request } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + await chat.getSuggested("msg", "user"); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/messages/msg/suggested", + query: { user: "user" }, + }); + }); + + it("submits message feedback", async () => { + const { client, request } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + await chat.messageFeedback("msg", "like", "user", "good"); + await chat.messageFeedback({ + messageId: "msg", + user: "user", + rating: "dislike", + }); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/messages/msg/feedbacks", + data: { user: "user", rating: "like", content: "good" }, + }); + }); + + it("lists app feedbacks", async () => { + const { client, request } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + await chat.getAppFeedbacks(2, 5); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/app/feedbacks", + query: { page: 2, limit: 5 }, + }); + }); + + it("lists conversations and messages", async () => { + const { client, request } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + await chat.getConversations("user", "last", 10, "-updated_at"); + await chat.getConversationMessages("user", "conv", "first", 5); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/conversations", + query: { + user: "user", + last_id: "last", + limit: 10, + sort_by: "-updated_at", + }, + }); + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/messages", + query: { + user: "user", + conversation_id: "conv", + first_id: "first", + limit: 5, + }, + }); + }); + + it("renames conversations with optional auto-generate", async () => { + const { client, request } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + await chat.renameConversation("conv", "name", "user", false); + await chat.renameConversation("conv", "user", { autoGenerate: true }); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/conversations/conv/name", + data: { user: "user", auto_generate: false, name: "name" }, + }); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/conversations/conv/name", + data: { user: "user", auto_generate: true }, + }); + }); + + it("requires name when autoGenerate is false", async () => { + const { client } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + expect(() => + chat.renameConversation("conv", "", "user", false) + ).toThrow(ValidationError); + }); + + it("deletes conversations", async () => { + const { client, request } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + await chat.deleteConversation("conv", "user"); + + expect(request).toHaveBeenCalledWith({ + method: "DELETE", + path: "/conversations/conv", + data: { user: "user" }, + }); + }); + + it("manages conversation variables", async () => { + const { client, request } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + await chat.getConversationVariables("conv", "user", "last", 10, "name"); + await chat.updateConversationVariable("conv", "var", "user", "value"); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/conversations/conv/variables", + query: { + user: "user", + last_id: "last", + limit: 10, + variable_name: "name", + }, + }); + expect(request).toHaveBeenCalledWith({ + method: "PUT", + path: "/conversations/conv/variables/var", + data: { user: "user", value: "value" }, + }); + }); + + it("handles annotation APIs", async () => { + const { client, request } = createHttpClientWithSpies(); + const chat = new ChatClient(client); + + await chat.annotationReplyAction("enable", { + score_threshold: 0.5, + embedding_provider_name: "prov", + embedding_model_name: "model", + }); + await chat.getAnnotationReplyStatus("enable", "job"); + await chat.listAnnotations({ page: 1, limit: 10, keyword: "k" }); + await chat.createAnnotation({ question: "q", answer: "a" }); + await chat.updateAnnotation("id", { question: "q", answer: "a" }); + await chat.deleteAnnotation("id"); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/apps/annotation-reply/enable", + data: { + score_threshold: 0.5, + embedding_provider_name: "prov", + embedding_model_name: "model", + }, + }); + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/apps/annotation-reply/enable/status/job", + }); + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/apps/annotations", + query: { page: 1, limit: 10, keyword: "k" }, + }); + }); +}); diff --git a/sdks/nodejs-client/src/client/chat.ts b/sdks/nodejs-client/src/client/chat.ts new file mode 100644 index 0000000000..745c999552 --- /dev/null +++ b/sdks/nodejs-client/src/client/chat.ts @@ -0,0 +1,377 @@ +import { DifyClient } from "./base"; +import type { ChatMessageRequest, ChatMessageResponse } from "../types/chat"; +import type { + AnnotationCreateRequest, + AnnotationListOptions, + AnnotationReplyActionRequest, + AnnotationResponse, +} from "../types/annotation"; +import type { + DifyResponse, + DifyStream, + QueryParams, +} from "../types/common"; +import { + ensureNonEmptyString, + ensureOptionalInt, + ensureOptionalString, +} from "./validation"; + +export class ChatClient extends DifyClient { + createChatMessage( + request: ChatMessageRequest + ): Promise | DifyStream>; + createChatMessage( + inputs: Record, + query: string, + user: string, + stream?: boolean, + conversationId?: string | null, + files?: Array> | null + ): Promise | DifyStream>; + createChatMessage( + inputOrRequest: ChatMessageRequest | Record, + query?: string, + user?: string, + stream = false, + conversationId?: string | null, + files?: Array> | null + ): Promise | DifyStream> { + let payload: ChatMessageRequest; + let shouldStream = stream; + + if (query === undefined && "user" in (inputOrRequest as ChatMessageRequest)) { + payload = inputOrRequest as ChatMessageRequest; + shouldStream = payload.response_mode === "streaming"; + } else { + ensureNonEmptyString(query, "query"); + ensureNonEmptyString(user, "user"); + payload = { + inputs: inputOrRequest as Record, + query, + user, + response_mode: stream ? "streaming" : "blocking", + files, + }; + if (conversationId) { + payload.conversation_id = conversationId; + } + } + + ensureNonEmptyString(payload.user, "user"); + ensureNonEmptyString(payload.query, "query"); + + if (shouldStream) { + return this.http.requestStream({ + method: "POST", + path: "/chat-messages", + data: payload, + }); + } + + return this.http.request({ + method: "POST", + path: "/chat-messages", + data: payload, + }); + } + + stopChatMessage( + taskId: string, + user: string + ): Promise> { + ensureNonEmptyString(taskId, "taskId"); + ensureNonEmptyString(user, "user"); + return this.http.request({ + method: "POST", + path: `/chat-messages/${taskId}/stop`, + data: { user }, + }); + } + + stopMessage( + taskId: string, + user: string + ): Promise> { + return this.stopChatMessage(taskId, user); + } + + getSuggested( + messageId: string, + user: string + ): Promise> { + ensureNonEmptyString(messageId, "messageId"); + ensureNonEmptyString(user, "user"); + return this.http.request({ + method: "GET", + path: `/messages/${messageId}/suggested`, + query: { user }, + }); + } + + // Note: messageFeedback is inherited from DifyClient + + getAppFeedbacks( + page?: number, + limit?: number + ): Promise>> { + ensureOptionalInt(page, "page"); + ensureOptionalInt(limit, "limit"); + return this.http.request({ + method: "GET", + path: "/app/feedbacks", + query: { + page, + limit, + }, + }); + } + + getConversations( + user: string, + lastId?: string | null, + limit?: number | null, + sortByOrPinned?: string | boolean | null + ): Promise>> { + ensureNonEmptyString(user, "user"); + ensureOptionalString(lastId, "lastId"); + ensureOptionalInt(limit, "limit"); + + const params: QueryParams = { user }; + if (lastId) { + params.last_id = lastId; + } + if (limit) { + params.limit = limit; + } + if (typeof sortByOrPinned === "string") { + params.sort_by = sortByOrPinned; + } else if (typeof sortByOrPinned === "boolean") { + params.pinned = sortByOrPinned; + } + + return this.http.request({ + method: "GET", + path: "/conversations", + query: params, + }); + } + + getConversationMessages( + user: string, + conversationId: string, + firstId?: string | null, + limit?: number | null + ): Promise>> { + ensureNonEmptyString(user, "user"); + ensureNonEmptyString(conversationId, "conversationId"); + ensureOptionalString(firstId, "firstId"); + ensureOptionalInt(limit, "limit"); + + const params: QueryParams = { user }; + params.conversation_id = conversationId; + if (firstId) { + params.first_id = firstId; + } + if (limit) { + params.limit = limit; + } + + return this.http.request({ + method: "GET", + path: "/messages", + query: params, + }); + } + + renameConversation( + conversationId: string, + name: string, + user: string, + autoGenerate?: boolean + ): Promise>>; + renameConversation( + conversationId: string, + user: string, + options?: { name?: string | null; autoGenerate?: boolean } + ): Promise>>; + renameConversation( + conversationId: string, + nameOrUser: string, + userOrOptions?: string | { name?: string | null; autoGenerate?: boolean }, + autoGenerate?: boolean + ): Promise>> { + ensureNonEmptyString(conversationId, "conversationId"); + + let name: string | null | undefined; + let user: string; + let resolvedAutoGenerate: boolean; + + if (typeof userOrOptions === "string" || userOrOptions === undefined) { + name = nameOrUser; + user = userOrOptions ?? ""; + resolvedAutoGenerate = autoGenerate ?? false; + } else { + user = nameOrUser; + name = userOrOptions.name; + resolvedAutoGenerate = userOrOptions.autoGenerate ?? false; + } + + ensureNonEmptyString(user, "user"); + if (!resolvedAutoGenerate) { + ensureNonEmptyString(name, "name"); + } + + const payload: Record = { + user, + auto_generate: resolvedAutoGenerate, + }; + if (typeof name === "string" && name.trim().length > 0) { + payload.name = name; + } + + return this.http.request({ + method: "POST", + path: `/conversations/${conversationId}/name`, + data: payload, + }); + } + + deleteConversation( + conversationId: string, + user: string + ): Promise>> { + ensureNonEmptyString(conversationId, "conversationId"); + ensureNonEmptyString(user, "user"); + return this.http.request({ + method: "DELETE", + path: `/conversations/${conversationId}`, + data: { user }, + }); + } + + getConversationVariables( + conversationId: string, + user: string, + lastId?: string | null, + limit?: number | null, + variableName?: string | null + ): Promise>> { + ensureNonEmptyString(conversationId, "conversationId"); + ensureNonEmptyString(user, "user"); + ensureOptionalString(lastId, "lastId"); + ensureOptionalInt(limit, "limit"); + ensureOptionalString(variableName, "variableName"); + + return this.http.request({ + method: "GET", + path: `/conversations/${conversationId}/variables`, + query: { + user, + last_id: lastId ?? undefined, + limit: limit ?? undefined, + variable_name: variableName ?? undefined, + }, + }); + } + + updateConversationVariable( + conversationId: string, + variableId: string, + user: string, + value: unknown + ): Promise>> { + ensureNonEmptyString(conversationId, "conversationId"); + ensureNonEmptyString(variableId, "variableId"); + ensureNonEmptyString(user, "user"); + return this.http.request({ + method: "PUT", + path: `/conversations/${conversationId}/variables/${variableId}`, + data: { + user, + value, + }, + }); + } + + annotationReplyAction( + action: "enable" | "disable", + request: AnnotationReplyActionRequest + ): Promise> { + ensureNonEmptyString(action, "action"); + ensureNonEmptyString(request.embedding_provider_name, "embedding_provider_name"); + ensureNonEmptyString(request.embedding_model_name, "embedding_model_name"); + return this.http.request({ + method: "POST", + path: `/apps/annotation-reply/${action}`, + data: request, + }); + } + + getAnnotationReplyStatus( + action: "enable" | "disable", + jobId: string + ): Promise> { + ensureNonEmptyString(action, "action"); + ensureNonEmptyString(jobId, "jobId"); + return this.http.request({ + method: "GET", + path: `/apps/annotation-reply/${action}/status/${jobId}`, + }); + } + + listAnnotations( + options?: AnnotationListOptions + ): Promise> { + ensureOptionalInt(options?.page, "page"); + ensureOptionalInt(options?.limit, "limit"); + ensureOptionalString(options?.keyword, "keyword"); + return this.http.request({ + method: "GET", + path: "/apps/annotations", + query: { + page: options?.page, + limit: options?.limit, + keyword: options?.keyword ?? undefined, + }, + }); + } + + createAnnotation( + request: AnnotationCreateRequest + ): Promise> { + ensureNonEmptyString(request.question, "question"); + ensureNonEmptyString(request.answer, "answer"); + return this.http.request({ + method: "POST", + path: "/apps/annotations", + data: request, + }); + } + + updateAnnotation( + annotationId: string, + request: AnnotationCreateRequest + ): Promise> { + ensureNonEmptyString(annotationId, "annotationId"); + ensureNonEmptyString(request.question, "question"); + ensureNonEmptyString(request.answer, "answer"); + return this.http.request({ + method: "PUT", + path: `/apps/annotations/${annotationId}`, + data: request, + }); + } + + deleteAnnotation( + annotationId: string + ): Promise> { + ensureNonEmptyString(annotationId, "annotationId"); + return this.http.request({ + method: "DELETE", + path: `/apps/annotations/${annotationId}`, + }); + } + + // Note: audioToText is inherited from DifyClient +} diff --git a/sdks/nodejs-client/src/client/completion.test.js b/sdks/nodejs-client/src/client/completion.test.js new file mode 100644 index 0000000000..b79cf3fb8f --- /dev/null +++ b/sdks/nodejs-client/src/client/completion.test.js @@ -0,0 +1,83 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { CompletionClient } from "./completion"; +import { createHttpClientWithSpies } from "../../tests/test-utils"; + +describe("CompletionClient", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("creates completion messages in blocking mode", async () => { + const { client, request } = createHttpClientWithSpies(); + const completion = new CompletionClient(client); + + await completion.createCompletionMessage({ input: "x" }, "user", false); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/completion-messages", + data: { + inputs: { input: "x" }, + user: "user", + files: undefined, + response_mode: "blocking", + }, + }); + }); + + it("creates completion messages in streaming mode", async () => { + const { client, requestStream } = createHttpClientWithSpies(); + const completion = new CompletionClient(client); + + await completion.createCompletionMessage({ + inputs: { input: "x" }, + user: "user", + response_mode: "streaming", + }); + + expect(requestStream).toHaveBeenCalledWith({ + method: "POST", + path: "/completion-messages", + data: { + inputs: { input: "x" }, + user: "user", + response_mode: "streaming", + }, + }); + }); + + it("stops completion messages", async () => { + const { client, request } = createHttpClientWithSpies(); + const completion = new CompletionClient(client); + + await completion.stopCompletionMessage("task", "user"); + await completion.stop("task", "user"); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/completion-messages/task/stop", + data: { user: "user" }, + }); + }); + + it("supports deprecated runWorkflow", async () => { + const { client, request, requestStream } = createHttpClientWithSpies(); + const completion = new CompletionClient(client); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await completion.runWorkflow({ input: "x" }, "user", false); + await completion.runWorkflow({ input: "x" }, "user", true); + + expect(warn).toHaveBeenCalled(); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/workflows/run", + data: { inputs: { input: "x" }, user: "user", response_mode: "blocking" }, + }); + expect(requestStream).toHaveBeenCalledWith({ + method: "POST", + path: "/workflows/run", + data: { inputs: { input: "x" }, user: "user", response_mode: "streaming" }, + }); + }); +}); diff --git a/sdks/nodejs-client/src/client/completion.ts b/sdks/nodejs-client/src/client/completion.ts new file mode 100644 index 0000000000..9e39898e8b --- /dev/null +++ b/sdks/nodejs-client/src/client/completion.ts @@ -0,0 +1,111 @@ +import { DifyClient } from "./base"; +import type { CompletionRequest, CompletionResponse } from "../types/completion"; +import type { DifyResponse, DifyStream } from "../types/common"; +import { ensureNonEmptyString } from "./validation"; + +const warned = new Set(); +const warnOnce = (message: string): void => { + if (warned.has(message)) { + return; + } + warned.add(message); + console.warn(message); +}; + +export class CompletionClient extends DifyClient { + createCompletionMessage( + request: CompletionRequest + ): Promise | DifyStream>; + createCompletionMessage( + inputs: Record, + user: string, + stream?: boolean, + files?: Array> | null + ): Promise | DifyStream>; + createCompletionMessage( + inputOrRequest: CompletionRequest | Record, + user?: string, + stream = false, + files?: Array> | null + ): Promise | DifyStream> { + let payload: CompletionRequest; + let shouldStream = stream; + + if (user === undefined && "user" in (inputOrRequest as CompletionRequest)) { + payload = inputOrRequest as CompletionRequest; + shouldStream = payload.response_mode === "streaming"; + } else { + ensureNonEmptyString(user, "user"); + payload = { + inputs: inputOrRequest as Record, + user, + files, + response_mode: stream ? "streaming" : "blocking", + }; + } + + ensureNonEmptyString(payload.user, "user"); + + if (shouldStream) { + return this.http.requestStream({ + method: "POST", + path: "/completion-messages", + data: payload, + }); + } + + return this.http.request({ + method: "POST", + path: "/completion-messages", + data: payload, + }); + } + + stopCompletionMessage( + taskId: string, + user: string + ): Promise> { + ensureNonEmptyString(taskId, "taskId"); + ensureNonEmptyString(user, "user"); + return this.http.request({ + method: "POST", + path: `/completion-messages/${taskId}/stop`, + data: { user }, + }); + } + + stop( + taskId: string, + user: string + ): Promise> { + return this.stopCompletionMessage(taskId, user); + } + + runWorkflow( + inputs: Record, + user: string, + stream = false + ): Promise> | DifyStream>> { + warnOnce( + "CompletionClient.runWorkflow is deprecated. Use WorkflowClient.run instead." + ); + ensureNonEmptyString(user, "user"); + const payload = { + inputs, + user, + response_mode: stream ? "streaming" : "blocking", + }; + if (stream) { + return this.http.requestStream>({ + method: "POST", + path: "/workflows/run", + data: payload, + }); + } + return this.http.request>({ + method: "POST", + path: "/workflows/run", + data: payload, + }); + } +} diff --git a/sdks/nodejs-client/src/client/knowledge-base.test.js b/sdks/nodejs-client/src/client/knowledge-base.test.js new file mode 100644 index 0000000000..4381b39e56 --- /dev/null +++ b/sdks/nodejs-client/src/client/knowledge-base.test.js @@ -0,0 +1,249 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { KnowledgeBaseClient } from "./knowledge-base"; +import { createHttpClientWithSpies } from "../../tests/test-utils"; + +describe("KnowledgeBaseClient", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("handles dataset and tag operations", async () => { + const { client, request } = createHttpClientWithSpies(); + const kb = new KnowledgeBaseClient(client); + + await kb.listDatasets({ + page: 1, + limit: 2, + keyword: "k", + includeAll: true, + tagIds: ["t1"], + }); + await kb.createDataset({ name: "dataset" }); + await kb.getDataset("ds"); + await kb.updateDataset("ds", { name: "new" }); + await kb.deleteDataset("ds"); + await kb.updateDocumentStatus("ds", "enable", ["doc1"]); + + await kb.listTags(); + await kb.createTag({ name: "tag" }); + await kb.updateTag({ tag_id: "tag", name: "name" }); + await kb.deleteTag({ tag_id: "tag" }); + await kb.bindTags({ tag_ids: ["tag"], target_id: "doc" }); + await kb.unbindTags({ tag_id: "tag", target_id: "doc" }); + await kb.getDatasetTags("ds"); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/datasets", + query: { + page: 1, + limit: 2, + keyword: "k", + include_all: true, + tag_ids: ["t1"], + }, + }); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets", + data: { name: "dataset" }, + }); + expect(request).toHaveBeenCalledWith({ + method: "PATCH", + path: "/datasets/ds", + data: { name: "new" }, + }); + expect(request).toHaveBeenCalledWith({ + method: "PATCH", + path: "/datasets/ds/documents/status/enable", + data: { document_ids: ["doc1"] }, + }); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/tags/binding", + data: { tag_ids: ["tag"], target_id: "doc" }, + }); + }); + + it("handles document operations", async () => { + const { client, request } = createHttpClientWithSpies(); + const kb = new KnowledgeBaseClient(client); + const form = { append: vi.fn(), getHeaders: () => ({}) }; + + await kb.createDocumentByText("ds", { name: "doc", text: "text" }); + await kb.updateDocumentByText("ds", "doc", { name: "doc2" }); + await kb.createDocumentByFile("ds", form); + await kb.updateDocumentByFile("ds", "doc", form); + await kb.listDocuments("ds", { page: 1, limit: 20, keyword: "k" }); + await kb.getDocument("ds", "doc", { metadata: "all" }); + await kb.deleteDocument("ds", "doc"); + await kb.getDocumentIndexingStatus("ds", "batch"); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/ds/document/create_by_text", + data: { name: "doc", text: "text" }, + }); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/ds/documents/doc/update_by_text", + data: { name: "doc2" }, + }); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/ds/document/create_by_file", + data: form, + }); + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/datasets/ds/documents", + query: { page: 1, limit: 20, keyword: "k", status: undefined }, + }); + }); + + it("handles segments and child chunks", async () => { + const { client, request } = createHttpClientWithSpies(); + const kb = new KnowledgeBaseClient(client); + + await kb.createSegments("ds", "doc", { segments: [{ content: "x" }] }); + await kb.listSegments("ds", "doc", { page: 1, limit: 10, keyword: "k" }); + await kb.getSegment("ds", "doc", "seg"); + await kb.updateSegment("ds", "doc", "seg", { + segment: { content: "y" }, + }); + await kb.deleteSegment("ds", "doc", "seg"); + + await kb.createChildChunk("ds", "doc", "seg", { content: "c" }); + await kb.listChildChunks("ds", "doc", "seg", { page: 1, limit: 10 }); + await kb.updateChildChunk("ds", "doc", "seg", "child", { + content: "c2", + }); + await kb.deleteChildChunk("ds", "doc", "seg", "child"); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/ds/documents/doc/segments", + data: { segments: [{ content: "x" }] }, + }); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/ds/documents/doc/segments/seg", + data: { segment: { content: "y" } }, + }); + expect(request).toHaveBeenCalledWith({ + method: "PATCH", + path: "/datasets/ds/documents/doc/segments/seg/child_chunks/child", + data: { content: "c2" }, + }); + }); + + it("handles metadata and retrieval", async () => { + const { client, request } = createHttpClientWithSpies(); + const kb = new KnowledgeBaseClient(client); + + await kb.listMetadata("ds"); + await kb.createMetadata("ds", { name: "m", type: "string" }); + await kb.updateMetadata("ds", "mid", { name: "m2" }); + await kb.deleteMetadata("ds", "mid"); + await kb.listBuiltInMetadata("ds"); + await kb.updateBuiltInMetadata("ds", "enable"); + await kb.updateDocumentsMetadata("ds", { + operation_data: [ + { document_id: "doc", metadata_list: [{ id: "m", name: "n" }] }, + ], + }); + await kb.hitTesting("ds", { query: "q" }); + await kb.retrieve("ds", { query: "q" }); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/datasets/ds/metadata", + }); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/ds/metadata", + data: { name: "m", type: "string" }, + }); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/ds/hit-testing", + data: { query: "q" }, + }); + }); + + it("handles pipeline operations", async () => { + const { client, request, requestStream } = createHttpClientWithSpies(); + const kb = new KnowledgeBaseClient(client); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const form = { append: vi.fn(), getHeaders: () => ({}) }; + + await kb.listDatasourcePlugins("ds", { isPublished: true }); + await kb.runDatasourceNode("ds", "node", { + inputs: { input: "x" }, + datasource_type: "custom", + is_published: true, + }); + await kb.runPipeline("ds", { + inputs: { input: "x" }, + datasource_type: "custom", + datasource_info_list: [], + start_node_id: "start", + is_published: true, + response_mode: "streaming", + }); + await kb.runPipeline("ds", { + inputs: { input: "x" }, + datasource_type: "custom", + datasource_info_list: [], + start_node_id: "start", + is_published: true, + response_mode: "blocking", + }); + await kb.uploadPipelineFile(form); + + expect(warn).toHaveBeenCalled(); + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/datasets/ds/pipeline/datasource-plugins", + query: { is_published: true }, + }); + expect(requestStream).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/ds/pipeline/datasource/nodes/node/run", + data: { + inputs: { input: "x" }, + datasource_type: "custom", + is_published: true, + }, + }); + expect(requestStream).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/ds/pipeline/run", + data: { + inputs: { input: "x" }, + datasource_type: "custom", + datasource_info_list: [], + start_node_id: "start", + is_published: true, + response_mode: "streaming", + }, + }); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/ds/pipeline/run", + data: { + inputs: { input: "x" }, + datasource_type: "custom", + datasource_info_list: [], + start_node_id: "start", + is_published: true, + response_mode: "blocking", + }, + }); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/datasets/pipeline/file-upload", + data: form, + }); + }); +}); diff --git a/sdks/nodejs-client/src/client/knowledge-base.ts b/sdks/nodejs-client/src/client/knowledge-base.ts new file mode 100644 index 0000000000..7a0e39898b --- /dev/null +++ b/sdks/nodejs-client/src/client/knowledge-base.ts @@ -0,0 +1,706 @@ +import { DifyClient } from "./base"; +import type { + DatasetCreateRequest, + DatasetListOptions, + DatasetTagBindingRequest, + DatasetTagCreateRequest, + DatasetTagDeleteRequest, + DatasetTagUnbindingRequest, + DatasetTagUpdateRequest, + DatasetUpdateRequest, + DocumentGetOptions, + DocumentListOptions, + DocumentStatusAction, + DocumentTextCreateRequest, + DocumentTextUpdateRequest, + SegmentCreateRequest, + SegmentListOptions, + SegmentUpdateRequest, + ChildChunkCreateRequest, + ChildChunkListOptions, + ChildChunkUpdateRequest, + MetadataCreateRequest, + MetadataOperationRequest, + MetadataUpdateRequest, + HitTestingRequest, + DatasourcePluginListOptions, + DatasourceNodeRunRequest, + PipelineRunRequest, + KnowledgeBaseResponse, + PipelineStreamEvent, +} from "../types/knowledge-base"; +import type { DifyResponse, DifyStream, QueryParams } from "../types/common"; +import { + ensureNonEmptyString, + ensureOptionalBoolean, + ensureOptionalInt, + ensureOptionalString, + ensureStringArray, +} from "./validation"; +import { FileUploadError, ValidationError } from "../errors/dify-error"; +import { isFormData } from "../http/form-data"; + +const warned = new Set(); +const warnOnce = (message: string): void => { + if (warned.has(message)) { + return; + } + warned.add(message); + console.warn(message); +}; + +const ensureFormData = (form: unknown, context: string): void => { + if (!isFormData(form)) { + throw new FileUploadError(`${context} requires FormData`); + } +}; + +const ensureNonEmptyArray = (value: unknown, name: string): void => { + if (!Array.isArray(value) || value.length === 0) { + throw new ValidationError(`${name} must be a non-empty array`); + } +}; + +const warnPipelineRoutes = (): void => { + warnOnce( + "RAG pipeline endpoints may be unavailable unless the service API registers dataset/rag_pipeline routes." + ); +}; + +export class KnowledgeBaseClient extends DifyClient { + async listDatasets( + options?: DatasetListOptions + ): Promise> { + ensureOptionalInt(options?.page, "page"); + ensureOptionalInt(options?.limit, "limit"); + ensureOptionalString(options?.keyword, "keyword"); + ensureOptionalBoolean(options?.includeAll, "includeAll"); + + const query: QueryParams = { + page: options?.page, + limit: options?.limit, + keyword: options?.keyword ?? undefined, + include_all: options?.includeAll ?? undefined, + }; + + if (options?.tagIds && options.tagIds.length > 0) { + ensureStringArray(options.tagIds, "tagIds"); + query.tag_ids = options.tagIds; + } + + return this.http.request({ + method: "GET", + path: "/datasets", + query, + }); + } + + async createDataset( + request: DatasetCreateRequest + ): Promise> { + ensureNonEmptyString(request.name, "name"); + return this.http.request({ + method: "POST", + path: "/datasets", + data: request, + }); + } + + async getDataset(datasetId: string): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + return this.http.request({ + method: "GET", + path: `/datasets/${datasetId}`, + }); + } + + async updateDataset( + datasetId: string, + request: DatasetUpdateRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + if (request.name !== undefined && request.name !== null) { + ensureNonEmptyString(request.name, "name"); + } + return this.http.request({ + method: "PATCH", + path: `/datasets/${datasetId}`, + data: request, + }); + } + + async deleteDataset(datasetId: string): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + return this.http.request({ + method: "DELETE", + path: `/datasets/${datasetId}`, + }); + } + + async updateDocumentStatus( + datasetId: string, + action: DocumentStatusAction, + documentIds: string[] + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(action, "action"); + ensureStringArray(documentIds, "documentIds"); + return this.http.request({ + method: "PATCH", + path: `/datasets/${datasetId}/documents/status/${action}`, + data: { + document_ids: documentIds, + }, + }); + } + + async listTags(): Promise> { + return this.http.request({ + method: "GET", + path: "/datasets/tags", + }); + } + + async createTag( + request: DatasetTagCreateRequest + ): Promise> { + ensureNonEmptyString(request.name, "name"); + return this.http.request({ + method: "POST", + path: "/datasets/tags", + data: request, + }); + } + + async updateTag( + request: DatasetTagUpdateRequest + ): Promise> { + ensureNonEmptyString(request.tag_id, "tag_id"); + ensureNonEmptyString(request.name, "name"); + return this.http.request({ + method: "PATCH", + path: "/datasets/tags", + data: request, + }); + } + + async deleteTag( + request: DatasetTagDeleteRequest + ): Promise> { + ensureNonEmptyString(request.tag_id, "tag_id"); + return this.http.request({ + method: "DELETE", + path: "/datasets/tags", + data: request, + }); + } + + async bindTags( + request: DatasetTagBindingRequest + ): Promise> { + ensureStringArray(request.tag_ids, "tag_ids"); + ensureNonEmptyString(request.target_id, "target_id"); + return this.http.request({ + method: "POST", + path: "/datasets/tags/binding", + data: request, + }); + } + + async unbindTags( + request: DatasetTagUnbindingRequest + ): Promise> { + ensureNonEmptyString(request.tag_id, "tag_id"); + ensureNonEmptyString(request.target_id, "target_id"); + return this.http.request({ + method: "POST", + path: "/datasets/tags/unbinding", + data: request, + }); + } + + async getDatasetTags( + datasetId: string + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + return this.http.request({ + method: "GET", + path: `/datasets/${datasetId}/tags`, + }); + } + + async createDocumentByText( + datasetId: string, + request: DocumentTextCreateRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(request.name, "name"); + ensureNonEmptyString(request.text, "text"); + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/document/create_by_text`, + data: request, + }); + } + + async updateDocumentByText( + datasetId: string, + documentId: string, + request: DocumentTextUpdateRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + if (request.name !== undefined && request.name !== null) { + ensureNonEmptyString(request.name, "name"); + } + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/documents/${documentId}/update_by_text`, + data: request, + }); + } + + async createDocumentByFile( + datasetId: string, + form: unknown + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureFormData(form, "createDocumentByFile"); + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/document/create_by_file`, + data: form, + }); + } + + async updateDocumentByFile( + datasetId: string, + documentId: string, + form: unknown + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + ensureFormData(form, "updateDocumentByFile"); + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/documents/${documentId}/update_by_file`, + data: form, + }); + } + + async listDocuments( + datasetId: string, + options?: DocumentListOptions + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureOptionalInt(options?.page, "page"); + ensureOptionalInt(options?.limit, "limit"); + ensureOptionalString(options?.keyword, "keyword"); + ensureOptionalString(options?.status, "status"); + + return this.http.request({ + method: "GET", + path: `/datasets/${datasetId}/documents`, + query: { + page: options?.page, + limit: options?.limit, + keyword: options?.keyword ?? undefined, + status: options?.status ?? undefined, + }, + }); + } + + async getDocument( + datasetId: string, + documentId: string, + options?: DocumentGetOptions + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + if (options?.metadata) { + const allowed = new Set(["all", "only", "without"]); + if (!allowed.has(options.metadata)) { + throw new ValidationError("metadata must be one of all, only, without"); + } + } + return this.http.request({ + method: "GET", + path: `/datasets/${datasetId}/documents/${documentId}`, + query: { + metadata: options?.metadata ?? undefined, + }, + }); + } + + async deleteDocument( + datasetId: string, + documentId: string + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + return this.http.request({ + method: "DELETE", + path: `/datasets/${datasetId}/documents/${documentId}`, + }); + } + + async getDocumentIndexingStatus( + datasetId: string, + batch: string + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(batch, "batch"); + return this.http.request({ + method: "GET", + path: `/datasets/${datasetId}/documents/${batch}/indexing-status`, + }); + } + + async createSegments( + datasetId: string, + documentId: string, + request: SegmentCreateRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + ensureNonEmptyArray(request.segments, "segments"); + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/documents/${documentId}/segments`, + data: request, + }); + } + + async listSegments( + datasetId: string, + documentId: string, + options?: SegmentListOptions + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + ensureOptionalInt(options?.page, "page"); + ensureOptionalInt(options?.limit, "limit"); + ensureOptionalString(options?.keyword, "keyword"); + if (options?.status && options.status.length > 0) { + ensureStringArray(options.status, "status"); + } + + const query: QueryParams = { + page: options?.page, + limit: options?.limit, + keyword: options?.keyword ?? undefined, + }; + if (options?.status && options.status.length > 0) { + query.status = options.status; + } + + return this.http.request({ + method: "GET", + path: `/datasets/${datasetId}/documents/${documentId}/segments`, + query, + }); + } + + async getSegment( + datasetId: string, + documentId: string, + segmentId: string + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + ensureNonEmptyString(segmentId, "segmentId"); + return this.http.request({ + method: "GET", + path: `/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}`, + }); + } + + async updateSegment( + datasetId: string, + documentId: string, + segmentId: string, + request: SegmentUpdateRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + ensureNonEmptyString(segmentId, "segmentId"); + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}`, + data: request, + }); + } + + async deleteSegment( + datasetId: string, + documentId: string, + segmentId: string + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + ensureNonEmptyString(segmentId, "segmentId"); + return this.http.request({ + method: "DELETE", + path: `/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}`, + }); + } + + async createChildChunk( + datasetId: string, + documentId: string, + segmentId: string, + request: ChildChunkCreateRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + ensureNonEmptyString(segmentId, "segmentId"); + ensureNonEmptyString(request.content, "content"); + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/child_chunks`, + data: request, + }); + } + + async listChildChunks( + datasetId: string, + documentId: string, + segmentId: string, + options?: ChildChunkListOptions + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + ensureNonEmptyString(segmentId, "segmentId"); + ensureOptionalInt(options?.page, "page"); + ensureOptionalInt(options?.limit, "limit"); + ensureOptionalString(options?.keyword, "keyword"); + + return this.http.request({ + method: "GET", + path: `/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/child_chunks`, + query: { + page: options?.page, + limit: options?.limit, + keyword: options?.keyword ?? undefined, + }, + }); + } + + async updateChildChunk( + datasetId: string, + documentId: string, + segmentId: string, + childChunkId: string, + request: ChildChunkUpdateRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + ensureNonEmptyString(segmentId, "segmentId"); + ensureNonEmptyString(childChunkId, "childChunkId"); + ensureNonEmptyString(request.content, "content"); + return this.http.request({ + method: "PATCH", + path: `/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/child_chunks/${childChunkId}`, + data: request, + }); + } + + async deleteChildChunk( + datasetId: string, + documentId: string, + segmentId: string, + childChunkId: string + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(documentId, "documentId"); + ensureNonEmptyString(segmentId, "segmentId"); + ensureNonEmptyString(childChunkId, "childChunkId"); + return this.http.request({ + method: "DELETE", + path: `/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/child_chunks/${childChunkId}`, + }); + } + + async listMetadata( + datasetId: string + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + return this.http.request({ + method: "GET", + path: `/datasets/${datasetId}/metadata`, + }); + } + + async createMetadata( + datasetId: string, + request: MetadataCreateRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(request.name, "name"); + ensureNonEmptyString(request.type, "type"); + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/metadata`, + data: request, + }); + } + + async updateMetadata( + datasetId: string, + metadataId: string, + request: MetadataUpdateRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(metadataId, "metadataId"); + ensureNonEmptyString(request.name, "name"); + return this.http.request({ + method: "PATCH", + path: `/datasets/${datasetId}/metadata/${metadataId}`, + data: request, + }); + } + + async deleteMetadata( + datasetId: string, + metadataId: string + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(metadataId, "metadataId"); + return this.http.request({ + method: "DELETE", + path: `/datasets/${datasetId}/metadata/${metadataId}`, + }); + } + + async listBuiltInMetadata( + datasetId: string + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + return this.http.request({ + method: "GET", + path: `/datasets/${datasetId}/metadata/built-in`, + }); + } + + async updateBuiltInMetadata( + datasetId: string, + action: "enable" | "disable" + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(action, "action"); + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/metadata/built-in/${action}`, + }); + } + + async updateDocumentsMetadata( + datasetId: string, + request: MetadataOperationRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyArray(request.operation_data, "operation_data"); + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/documents/metadata`, + data: request, + }); + } + + async hitTesting( + datasetId: string, + request: HitTestingRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + if (request.query !== undefined && request.query !== null) { + ensureOptionalString(request.query, "query"); + } + if (request.attachment_ids && request.attachment_ids.length > 0) { + ensureStringArray(request.attachment_ids, "attachment_ids"); + } + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/hit-testing`, + data: request, + }); + } + + async retrieve( + datasetId: string, + request: HitTestingRequest + ): Promise> { + ensureNonEmptyString(datasetId, "datasetId"); + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/retrieve`, + data: request, + }); + } + + async listDatasourcePlugins( + datasetId: string, + options?: DatasourcePluginListOptions + ): Promise> { + warnPipelineRoutes(); + ensureNonEmptyString(datasetId, "datasetId"); + ensureOptionalBoolean(options?.isPublished, "isPublished"); + return this.http.request({ + method: "GET", + path: `/datasets/${datasetId}/pipeline/datasource-plugins`, + query: { + is_published: options?.isPublished ?? undefined, + }, + }); + } + + async runDatasourceNode( + datasetId: string, + nodeId: string, + request: DatasourceNodeRunRequest + ): Promise> { + warnPipelineRoutes(); + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(nodeId, "nodeId"); + ensureNonEmptyString(request.datasource_type, "datasource_type"); + return this.http.requestStream({ + method: "POST", + path: `/datasets/${datasetId}/pipeline/datasource/nodes/${nodeId}/run`, + data: request, + }); + } + + async runPipeline( + datasetId: string, + request: PipelineRunRequest + ): Promise | DifyStream> { + warnPipelineRoutes(); + ensureNonEmptyString(datasetId, "datasetId"); + ensureNonEmptyString(request.datasource_type, "datasource_type"); + ensureNonEmptyString(request.start_node_id, "start_node_id"); + const shouldStream = request.response_mode === "streaming"; + if (shouldStream) { + return this.http.requestStream({ + method: "POST", + path: `/datasets/${datasetId}/pipeline/run`, + data: request, + }); + } + return this.http.request({ + method: "POST", + path: `/datasets/${datasetId}/pipeline/run`, + data: request, + }); + } + + async uploadPipelineFile( + form: unknown + ): Promise> { + warnPipelineRoutes(); + ensureFormData(form, "uploadPipelineFile"); + return this.http.request({ + method: "POST", + path: "/datasets/pipeline/file-upload", + data: form, + }); + } +} diff --git a/sdks/nodejs-client/src/client/validation.test.js b/sdks/nodejs-client/src/client/validation.test.js new file mode 100644 index 0000000000..65bfa471a6 --- /dev/null +++ b/sdks/nodejs-client/src/client/validation.test.js @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; +import { + ensureNonEmptyString, + ensureOptionalBoolean, + ensureOptionalInt, + ensureOptionalString, + ensureOptionalStringArray, + ensureRating, + ensureStringArray, + validateParams, +} from "./validation"; + +const makeLongString = (length) => "a".repeat(length); + +describe("validation utilities", () => { + it("ensureNonEmptyString throws on empty or whitespace", () => { + expect(() => ensureNonEmptyString("", "name")).toThrow(); + expect(() => ensureNonEmptyString(" ", "name")).toThrow(); + }); + + it("ensureNonEmptyString throws on overly long strings", () => { + expect(() => + ensureNonEmptyString(makeLongString(10001), "name") + ).toThrow(); + }); + + it("ensureOptionalString ignores undefined and validates when set", () => { + expect(() => ensureOptionalString(undefined, "opt")).not.toThrow(); + expect(() => ensureOptionalString("", "opt")).toThrow(); + }); + + it("ensureOptionalString throws on overly long strings", () => { + expect(() => ensureOptionalString(makeLongString(10001), "opt")).toThrow(); + }); + + it("ensureOptionalInt validates integer", () => { + expect(() => ensureOptionalInt(undefined, "limit")).not.toThrow(); + expect(() => ensureOptionalInt(1.2, "limit")).toThrow(); + }); + + it("ensureOptionalBoolean validates boolean", () => { + expect(() => ensureOptionalBoolean(undefined, "flag")).not.toThrow(); + expect(() => ensureOptionalBoolean("yes", "flag")).toThrow(); + }); + + it("ensureStringArray enforces size and content", () => { + expect(() => ensureStringArray([], "items")).toThrow(); + expect(() => ensureStringArray([""], "items")).toThrow(); + expect(() => + ensureStringArray(Array.from({ length: 1001 }, () => "a"), "items") + ).toThrow(); + expect(() => ensureStringArray(["ok"], "items")).not.toThrow(); + }); + + it("ensureOptionalStringArray ignores undefined", () => { + expect(() => ensureOptionalStringArray(undefined, "tags")).not.toThrow(); + }); + + it("ensureOptionalStringArray validates when set", () => { + expect(() => ensureOptionalStringArray(["valid"], "tags")).not.toThrow(); + expect(() => ensureOptionalStringArray([], "tags")).toThrow(); + expect(() => ensureOptionalStringArray([""], "tags")).toThrow(); + }); + + it("ensureRating validates allowed values", () => { + expect(() => ensureRating(undefined)).not.toThrow(); + expect(() => ensureRating("like")).not.toThrow(); + expect(() => ensureRating("bad")).toThrow(); + }); + + it("validateParams enforces generic rules", () => { + expect(() => validateParams({ user: 123 })).toThrow(); + expect(() => validateParams({ rating: "bad" })).toThrow(); + expect(() => validateParams({ page: 1.1 })).toThrow(); + expect(() => validateParams({ files: "bad" })).toThrow(); + // Empty strings are allowed for optional params (e.g., keyword: "" means no filter) + expect(() => validateParams({ keyword: "" })).not.toThrow(); + expect(() => validateParams({ name: makeLongString(10001) })).toThrow(); + expect(() => + validateParams({ items: Array.from({ length: 1001 }, () => "a") }) + ).toThrow(); + expect(() => + validateParams({ + data: Object.fromEntries( + Array.from({ length: 101 }, (_, i) => [String(i), i]) + ), + }) + ).toThrow(); + expect(() => validateParams({ user: "u", page: 1 })).not.toThrow(); + }); +}); diff --git a/sdks/nodejs-client/src/client/validation.ts b/sdks/nodejs-client/src/client/validation.ts new file mode 100644 index 0000000000..6aeec36bdc --- /dev/null +++ b/sdks/nodejs-client/src/client/validation.ts @@ -0,0 +1,136 @@ +import { ValidationError } from "../errors/dify-error"; + +const MAX_STRING_LENGTH = 10000; +const MAX_LIST_LENGTH = 1000; +const MAX_DICT_LENGTH = 100; + +export function ensureNonEmptyString( + value: unknown, + name: string +): asserts value is string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new ValidationError(`${name} must be a non-empty string`); + } + if (value.length > MAX_STRING_LENGTH) { + throw new ValidationError( + `${name} exceeds maximum length of ${MAX_STRING_LENGTH} characters` + ); + } +} + +/** + * Validates optional string fields that must be non-empty when provided. + * Use this for fields like `name` that are optional but should not be empty strings. + * + * For filter parameters that accept empty strings (e.g., `keyword: ""`), + * use `validateParams` which allows empty strings for optional params. + */ +export function ensureOptionalString(value: unknown, name: string): void { + if (value === undefined || value === null) { + return; + } + if (typeof value !== "string" || value.trim().length === 0) { + throw new ValidationError(`${name} must be a non-empty string when set`); + } + if (value.length > MAX_STRING_LENGTH) { + throw new ValidationError( + `${name} exceeds maximum length of ${MAX_STRING_LENGTH} characters` + ); + } +} + +export function ensureOptionalInt(value: unknown, name: string): void { + if (value === undefined || value === null) { + return; + } + if (!Number.isInteger(value)) { + throw new ValidationError(`${name} must be an integer when set`); + } +} + +export function ensureOptionalBoolean(value: unknown, name: string): void { + if (value === undefined || value === null) { + return; + } + if (typeof value !== "boolean") { + throw new ValidationError(`${name} must be a boolean when set`); + } +} + +export function ensureStringArray(value: unknown, name: string): void { + if (!Array.isArray(value) || value.length === 0) { + throw new ValidationError(`${name} must be a non-empty string array`); + } + if (value.length > MAX_LIST_LENGTH) { + throw new ValidationError( + `${name} exceeds maximum size of ${MAX_LIST_LENGTH} items` + ); + } + value.forEach((item) => { + if (typeof item !== "string" || item.trim().length === 0) { + throw new ValidationError(`${name} must contain non-empty strings`); + } + }); +} + +export function ensureOptionalStringArray(value: unknown, name: string): void { + if (value === undefined || value === null) { + return; + } + ensureStringArray(value, name); +} + +export function ensureRating(value: unknown): void { + if (value === undefined || value === null) { + return; + } + if (value !== "like" && value !== "dislike") { + throw new ValidationError("rating must be either 'like' or 'dislike'"); + } +} + +export function validateParams(params: Record): void { + Object.entries(params).forEach(([key, value]) => { + if (value === undefined || value === null) { + return; + } + + // Only check max length for strings; empty strings are allowed for optional params + // Required fields are validated at method level via ensureNonEmptyString + if (typeof value === "string") { + if (value.length > MAX_STRING_LENGTH) { + throw new ValidationError( + `Parameter '${key}' exceeds maximum length of ${MAX_STRING_LENGTH} characters` + ); + } + } else if (Array.isArray(value)) { + if (value.length > MAX_LIST_LENGTH) { + throw new ValidationError( + `Parameter '${key}' exceeds maximum size of ${MAX_LIST_LENGTH} items` + ); + } + } else if (typeof value === "object") { + if (Object.keys(value as Record).length > MAX_DICT_LENGTH) { + throw new ValidationError( + `Parameter '${key}' exceeds maximum size of ${MAX_DICT_LENGTH} items` + ); + } + } + + if (key === "user" && typeof value !== "string") { + throw new ValidationError(`Parameter '${key}' must be a string`); + } + if ( + (key === "page" || key === "limit" || key === "page_size") && + !Number.isInteger(value) + ) { + throw new ValidationError(`Parameter '${key}' must be an integer`); + } + if (key === "files" && !Array.isArray(value) && typeof value !== "object") { + throw new ValidationError(`Parameter '${key}' must be a list or dict`); + } + if (key === "rating" && value !== "like" && value !== "dislike") { + throw new ValidationError(`Parameter '${key}' must be 'like' or 'dislike'`); + } + }); +} diff --git a/sdks/nodejs-client/src/client/workflow.test.js b/sdks/nodejs-client/src/client/workflow.test.js new file mode 100644 index 0000000000..79c419b55a --- /dev/null +++ b/sdks/nodejs-client/src/client/workflow.test.js @@ -0,0 +1,119 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { WorkflowClient } from "./workflow"; +import { createHttpClientWithSpies } from "../../tests/test-utils"; + +describe("WorkflowClient", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("runs workflows with blocking and streaming modes", async () => { + const { client, request, requestStream } = createHttpClientWithSpies(); + const workflow = new WorkflowClient(client); + + await workflow.run({ inputs: { input: "x" }, user: "user" }); + await workflow.run({ input: "x" }, "user", true); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/workflows/run", + data: { + inputs: { input: "x" }, + user: "user", + }, + }); + expect(requestStream).toHaveBeenCalledWith({ + method: "POST", + path: "/workflows/run", + data: { + inputs: { input: "x" }, + user: "user", + response_mode: "streaming", + }, + }); + }); + + it("runs workflow by id", async () => { + const { client, request, requestStream } = createHttpClientWithSpies(); + const workflow = new WorkflowClient(client); + + await workflow.runById("wf", { + inputs: { input: "x" }, + user: "user", + response_mode: "blocking", + }); + await workflow.runById("wf", { + inputs: { input: "x" }, + user: "user", + response_mode: "streaming", + }); + + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/workflows/wf/run", + data: { + inputs: { input: "x" }, + user: "user", + response_mode: "blocking", + }, + }); + expect(requestStream).toHaveBeenCalledWith({ + method: "POST", + path: "/workflows/wf/run", + data: { + inputs: { input: "x" }, + user: "user", + response_mode: "streaming", + }, + }); + }); + + it("gets run details and stops workflow", async () => { + const { client, request } = createHttpClientWithSpies(); + const workflow = new WorkflowClient(client); + + await workflow.getRun("run"); + await workflow.stop("task", "user"); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/workflows/run/run", + }); + expect(request).toHaveBeenCalledWith({ + method: "POST", + path: "/workflows/tasks/task/stop", + data: { user: "user" }, + }); + }); + + it("fetches workflow logs", async () => { + const { client, request } = createHttpClientWithSpies(); + const workflow = new WorkflowClient(client); + + // Use createdByEndUserSessionId to filter by user session (backend API parameter) + await workflow.getLogs({ + keyword: "k", + status: "succeeded", + startTime: "2024-01-01", + endTime: "2024-01-02", + createdByEndUserSessionId: "session-123", + page: 1, + limit: 20, + }); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/workflows/logs", + query: { + keyword: "k", + status: "succeeded", + created_at__before: "2024-01-02", + created_at__after: "2024-01-01", + created_by_end_user_session_id: "session-123", + created_by_account: undefined, + page: 1, + limit: 20, + }, + }); + }); +}); diff --git a/sdks/nodejs-client/src/client/workflow.ts b/sdks/nodejs-client/src/client/workflow.ts new file mode 100644 index 0000000000..ae4d5861fa --- /dev/null +++ b/sdks/nodejs-client/src/client/workflow.ts @@ -0,0 +1,165 @@ +import { DifyClient } from "./base"; +import type { WorkflowRunRequest, WorkflowRunResponse } from "../types/workflow"; +import type { DifyResponse, DifyStream, QueryParams } from "../types/common"; +import { + ensureNonEmptyString, + ensureOptionalInt, + ensureOptionalString, +} from "./validation"; + +export class WorkflowClient extends DifyClient { + run( + request: WorkflowRunRequest + ): Promise | DifyStream>; + run( + inputs: Record, + user: string, + stream?: boolean + ): Promise | DifyStream>; + run( + inputOrRequest: WorkflowRunRequest | Record, + user?: string, + stream = false + ): Promise | DifyStream> { + let payload: WorkflowRunRequest; + let shouldStream = stream; + + if (user === undefined && "user" in (inputOrRequest as WorkflowRunRequest)) { + payload = inputOrRequest as WorkflowRunRequest; + shouldStream = payload.response_mode === "streaming"; + } else { + ensureNonEmptyString(user, "user"); + payload = { + inputs: inputOrRequest as Record, + user, + response_mode: stream ? "streaming" : "blocking", + }; + } + + ensureNonEmptyString(payload.user, "user"); + + if (shouldStream) { + return this.http.requestStream({ + method: "POST", + path: "/workflows/run", + data: payload, + }); + } + + return this.http.request({ + method: "POST", + path: "/workflows/run", + data: payload, + }); + } + + runById( + workflowId: string, + request: WorkflowRunRequest + ): Promise | DifyStream> { + ensureNonEmptyString(workflowId, "workflowId"); + ensureNonEmptyString(request.user, "user"); + if (request.response_mode === "streaming") { + return this.http.requestStream({ + method: "POST", + path: `/workflows/${workflowId}/run`, + data: request, + }); + } + return this.http.request({ + method: "POST", + path: `/workflows/${workflowId}/run`, + data: request, + }); + } + + getRun(workflowRunId: string): Promise> { + ensureNonEmptyString(workflowRunId, "workflowRunId"); + return this.http.request({ + method: "GET", + path: `/workflows/run/${workflowRunId}`, + }); + } + + stop( + taskId: string, + user: string + ): Promise> { + ensureNonEmptyString(taskId, "taskId"); + ensureNonEmptyString(user, "user"); + return this.http.request({ + method: "POST", + path: `/workflows/tasks/${taskId}/stop`, + data: { user }, + }); + } + + /** + * Get workflow execution logs with filtering options. + * + * Note: The backend API filters by `createdByEndUserSessionId` (end user session ID) + * or `createdByAccount` (account ID), not by a generic `user` parameter. + */ + getLogs(options?: { + keyword?: string; + status?: string; + createdAtBefore?: string; + createdAtAfter?: string; + createdByEndUserSessionId?: string; + createdByAccount?: string; + page?: number; + limit?: number; + startTime?: string; + endTime?: string; + }): Promise>> { + if (options?.keyword) { + ensureOptionalString(options.keyword, "keyword"); + } + if (options?.status) { + ensureOptionalString(options.status, "status"); + } + if (options?.createdAtBefore) { + ensureOptionalString(options.createdAtBefore, "createdAtBefore"); + } + if (options?.createdAtAfter) { + ensureOptionalString(options.createdAtAfter, "createdAtAfter"); + } + if (options?.createdByEndUserSessionId) { + ensureOptionalString( + options.createdByEndUserSessionId, + "createdByEndUserSessionId" + ); + } + if (options?.createdByAccount) { + ensureOptionalString(options.createdByAccount, "createdByAccount"); + } + if (options?.startTime) { + ensureOptionalString(options.startTime, "startTime"); + } + if (options?.endTime) { + ensureOptionalString(options.endTime, "endTime"); + } + ensureOptionalInt(options?.page, "page"); + ensureOptionalInt(options?.limit, "limit"); + + const createdAtAfter = options?.createdAtAfter ?? options?.startTime; + const createdAtBefore = options?.createdAtBefore ?? options?.endTime; + + const query: QueryParams = { + keyword: options?.keyword, + status: options?.status, + created_at__before: createdAtBefore, + created_at__after: createdAtAfter, + created_by_end_user_session_id: options?.createdByEndUserSessionId, + created_by_account: options?.createdByAccount, + page: options?.page, + limit: options?.limit, + }; + + return this.http.request({ + method: "GET", + path: "/workflows/logs", + query, + }); + } +} diff --git a/sdks/nodejs-client/src/client/workspace.test.js b/sdks/nodejs-client/src/client/workspace.test.js new file mode 100644 index 0000000000..f8fb6e375a --- /dev/null +++ b/sdks/nodejs-client/src/client/workspace.test.js @@ -0,0 +1,21 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { WorkspaceClient } from "./workspace"; +import { createHttpClientWithSpies } from "../../tests/test-utils"; + +describe("WorkspaceClient", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("gets models by type", async () => { + const { client, request } = createHttpClientWithSpies(); + const workspace = new WorkspaceClient(client); + + await workspace.getModelsByType("llm"); + + expect(request).toHaveBeenCalledWith({ + method: "GET", + path: "/workspaces/current/models/model-types/llm", + }); + }); +}); diff --git a/sdks/nodejs-client/src/client/workspace.ts b/sdks/nodejs-client/src/client/workspace.ts new file mode 100644 index 0000000000..daee540c99 --- /dev/null +++ b/sdks/nodejs-client/src/client/workspace.ts @@ -0,0 +1,16 @@ +import { DifyClient } from "./base"; +import type { WorkspaceModelType, WorkspaceModelsResponse } from "../types/workspace"; +import type { DifyResponse } from "../types/common"; +import { ensureNonEmptyString } from "./validation"; + +export class WorkspaceClient extends DifyClient { + async getModelsByType( + modelType: WorkspaceModelType + ): Promise> { + ensureNonEmptyString(modelType, "modelType"); + return this.http.request({ + method: "GET", + path: `/workspaces/current/models/model-types/${modelType}`, + }); + } +} diff --git a/sdks/nodejs-client/src/errors/dify-error.test.js b/sdks/nodejs-client/src/errors/dify-error.test.js new file mode 100644 index 0000000000..d151eca391 --- /dev/null +++ b/sdks/nodejs-client/src/errors/dify-error.test.js @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { + APIError, + AuthenticationError, + DifyError, + FileUploadError, + NetworkError, + RateLimitError, + TimeoutError, + ValidationError, +} from "./dify-error"; + +describe("Dify errors", () => { + it("sets base error fields", () => { + const err = new DifyError("base", { + statusCode: 400, + responseBody: { message: "bad" }, + requestId: "req", + retryAfter: 1, + }); + expect(err.name).toBe("DifyError"); + expect(err.statusCode).toBe(400); + expect(err.responseBody).toEqual({ message: "bad" }); + expect(err.requestId).toBe("req"); + expect(err.retryAfter).toBe(1); + }); + + it("creates specific error types", () => { + expect(new APIError("api").name).toBe("APIError"); + expect(new AuthenticationError("auth").name).toBe("AuthenticationError"); + expect(new RateLimitError("rate").name).toBe("RateLimitError"); + expect(new ValidationError("val").name).toBe("ValidationError"); + expect(new NetworkError("net").name).toBe("NetworkError"); + expect(new TimeoutError("timeout").name).toBe("TimeoutError"); + expect(new FileUploadError("upload").name).toBe("FileUploadError"); + }); +}); diff --git a/sdks/nodejs-client/src/errors/dify-error.ts b/sdks/nodejs-client/src/errors/dify-error.ts new file mode 100644 index 0000000000..e393e1b132 --- /dev/null +++ b/sdks/nodejs-client/src/errors/dify-error.ts @@ -0,0 +1,75 @@ +export type DifyErrorOptions = { + statusCode?: number; + responseBody?: unknown; + requestId?: string; + retryAfter?: number; + cause?: unknown; +}; + +export class DifyError extends Error { + statusCode?: number; + responseBody?: unknown; + requestId?: string; + retryAfter?: number; + + constructor(message: string, options: DifyErrorOptions = {}) { + super(message); + this.name = "DifyError"; + this.statusCode = options.statusCode; + this.responseBody = options.responseBody; + this.requestId = options.requestId; + this.retryAfter = options.retryAfter; + if (options.cause) { + (this as { cause?: unknown }).cause = options.cause; + } + } +} + +export class APIError extends DifyError { + constructor(message: string, options: DifyErrorOptions = {}) { + super(message, options); + this.name = "APIError"; + } +} + +export class AuthenticationError extends APIError { + constructor(message: string, options: DifyErrorOptions = {}) { + super(message, options); + this.name = "AuthenticationError"; + } +} + +export class RateLimitError extends APIError { + constructor(message: string, options: DifyErrorOptions = {}) { + super(message, options); + this.name = "RateLimitError"; + } +} + +export class ValidationError extends APIError { + constructor(message: string, options: DifyErrorOptions = {}) { + super(message, options); + this.name = "ValidationError"; + } +} + +export class NetworkError extends DifyError { + constructor(message: string, options: DifyErrorOptions = {}) { + super(message, options); + this.name = "NetworkError"; + } +} + +export class TimeoutError extends DifyError { + constructor(message: string, options: DifyErrorOptions = {}) { + super(message, options); + this.name = "TimeoutError"; + } +} + +export class FileUploadError extends DifyError { + constructor(message: string, options: DifyErrorOptions = {}) { + super(message, options); + this.name = "FileUploadError"; + } +} diff --git a/sdks/nodejs-client/src/http/client.test.js b/sdks/nodejs-client/src/http/client.test.js new file mode 100644 index 0000000000..05892547ed --- /dev/null +++ b/sdks/nodejs-client/src/http/client.test.js @@ -0,0 +1,304 @@ +import axios from "axios"; +import { Readable } from "node:stream"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + APIError, + AuthenticationError, + FileUploadError, + NetworkError, + RateLimitError, + TimeoutError, + ValidationError, +} from "../errors/dify-error"; +import { HttpClient } from "./client"; + +describe("HttpClient", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + it("builds requests with auth headers and JSON content type", async () => { + const mockRequest = vi.fn().mockResolvedValue({ + status: 200, + data: { ok: true }, + headers: { "x-request-id": "req" }, + }); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + + const client = new HttpClient({ apiKey: "test" }); + const response = await client.request({ + method: "POST", + path: "/chat-messages", + data: { user: "u" }, + }); + + expect(response.requestId).toBe("req"); + const config = mockRequest.mock.calls[0][0]; + expect(config.headers.Authorization).toBe("Bearer test"); + expect(config.headers["Content-Type"]).toBe("application/json"); + expect(config.responseType).toBe("json"); + }); + + it("serializes array query params", async () => { + const mockRequest = vi.fn().mockResolvedValue({ + status: 200, + data: "ok", + headers: {}, + }); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + + const client = new HttpClient({ apiKey: "test" }); + await client.requestRaw({ + method: "GET", + path: "/datasets", + query: { tag_ids: ["a", "b"], limit: 2 }, + }); + + const config = mockRequest.mock.calls[0][0]; + const queryString = config.paramsSerializer.serialize({ + tag_ids: ["a", "b"], + limit: 2, + }); + expect(queryString).toBe("tag_ids=a&tag_ids=b&limit=2"); + }); + + it("returns SSE stream helpers", async () => { + const mockRequest = vi.fn().mockResolvedValue({ + status: 200, + data: Readable.from(["data: {\"text\":\"hi\"}\n\n"]), + headers: { "x-request-id": "req" }, + }); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + + const client = new HttpClient({ apiKey: "test" }); + const stream = await client.requestStream({ + method: "POST", + path: "/chat-messages", + data: { user: "u" }, + }); + + expect(stream.status).toBe(200); + expect(stream.requestId).toBe("req"); + await expect(stream.toText()).resolves.toBe("hi"); + }); + + it("returns binary stream helpers", async () => { + const mockRequest = vi.fn().mockResolvedValue({ + status: 200, + data: Readable.from(["chunk"]), + headers: { "x-request-id": "req" }, + }); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + + const client = new HttpClient({ apiKey: "test" }); + const stream = await client.requestBinaryStream({ + method: "POST", + path: "/text-to-audio", + data: { user: "u", text: "hi" }, + }); + + expect(stream.status).toBe(200); + expect(stream.requestId).toBe("req"); + }); + + it("respects form-data headers", async () => { + const mockRequest = vi.fn().mockResolvedValue({ + status: 200, + data: "ok", + headers: {}, + }); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + + const client = new HttpClient({ apiKey: "test" }); + const form = { + append: () => {}, + getHeaders: () => ({ "content-type": "multipart/form-data; boundary=abc" }), + }; + + await client.requestRaw({ + method: "POST", + path: "/files/upload", + data: form, + }); + + const config = mockRequest.mock.calls[0][0]; + expect(config.headers["content-type"]).toBe( + "multipart/form-data; boundary=abc" + ); + expect(config.headers["Content-Type"]).toBeUndefined(); + }); + + it("maps 401 and 429 errors", async () => { + const mockRequest = vi.fn(); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + const client = new HttpClient({ apiKey: "test", maxRetries: 0 }); + + mockRequest.mockRejectedValueOnce({ + isAxiosError: true, + response: { + status: 401, + data: { message: "unauthorized" }, + headers: {}, + }, + }); + await expect( + client.requestRaw({ method: "GET", path: "/meta" }) + ).rejects.toBeInstanceOf(AuthenticationError); + + mockRequest.mockRejectedValueOnce({ + isAxiosError: true, + response: { + status: 429, + data: { message: "rate" }, + headers: { "retry-after": "2" }, + }, + }); + const error = await client + .requestRaw({ method: "GET", path: "/meta" }) + .catch((err) => err); + expect(error).toBeInstanceOf(RateLimitError); + expect(error.retryAfter).toBe(2); + }); + + it("maps validation and upload errors", async () => { + const mockRequest = vi.fn(); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + const client = new HttpClient({ apiKey: "test", maxRetries: 0 }); + + mockRequest.mockRejectedValueOnce({ + isAxiosError: true, + response: { + status: 422, + data: { message: "invalid" }, + headers: {}, + }, + }); + await expect( + client.requestRaw({ method: "POST", path: "/chat-messages", data: { user: "u" } }) + ).rejects.toBeInstanceOf(ValidationError); + + mockRequest.mockRejectedValueOnce({ + isAxiosError: true, + config: { url: "/files/upload" }, + response: { + status: 400, + data: { message: "bad upload" }, + headers: {}, + }, + }); + await expect( + client.requestRaw({ method: "POST", path: "/files/upload", data: { user: "u" } }) + ).rejects.toBeInstanceOf(FileUploadError); + }); + + it("maps timeout and network errors", async () => { + const mockRequest = vi.fn(); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + const client = new HttpClient({ apiKey: "test", maxRetries: 0 }); + + mockRequest.mockRejectedValueOnce({ + isAxiosError: true, + code: "ECONNABORTED", + message: "timeout", + }); + await expect( + client.requestRaw({ method: "GET", path: "/meta" }) + ).rejects.toBeInstanceOf(TimeoutError); + + mockRequest.mockRejectedValueOnce({ + isAxiosError: true, + message: "network", + }); + await expect( + client.requestRaw({ method: "GET", path: "/meta" }) + ).rejects.toBeInstanceOf(NetworkError); + }); + + it("retries on timeout errors", async () => { + const mockRequest = vi.fn(); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + const client = new HttpClient({ apiKey: "test", maxRetries: 1, retryDelay: 0 }); + + mockRequest + .mockRejectedValueOnce({ + isAxiosError: true, + code: "ECONNABORTED", + message: "timeout", + }) + .mockResolvedValueOnce({ status: 200, data: "ok", headers: {} }); + + await client.requestRaw({ method: "GET", path: "/meta" }); + expect(mockRequest).toHaveBeenCalledTimes(2); + }); + + it("validates query parameters before request", async () => { + const mockRequest = vi.fn(); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + const client = new HttpClient({ apiKey: "test" }); + + await expect( + client.requestRaw({ method: "GET", path: "/meta", query: { user: 1 } }) + ).rejects.toBeInstanceOf(ValidationError); + expect(mockRequest).not.toHaveBeenCalled(); + }); + + it("returns APIError for other http failures", async () => { + const mockRequest = vi.fn(); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + const client = new HttpClient({ apiKey: "test", maxRetries: 0 }); + + mockRequest.mockRejectedValueOnce({ + isAxiosError: true, + response: { status: 500, data: { message: "server" }, headers: {} }, + }); + + await expect( + client.requestRaw({ method: "GET", path: "/meta" }) + ).rejects.toBeInstanceOf(APIError); + }); + + it("logs requests and responses when enableLogging is true", async () => { + const mockRequest = vi.fn().mockResolvedValue({ + status: 200, + data: { ok: true }, + headers: {}, + }); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + const consoleInfo = vi.spyOn(console, "info").mockImplementation(() => {}); + + const client = new HttpClient({ apiKey: "test", enableLogging: true }); + await client.requestRaw({ method: "GET", path: "/meta" }); + + expect(consoleInfo).toHaveBeenCalledWith( + expect.stringContaining("dify-client-node response 200 GET") + ); + consoleInfo.mockRestore(); + }); + + it("logs retry attempts when enableLogging is true", async () => { + const mockRequest = vi.fn(); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + const consoleInfo = vi.spyOn(console, "info").mockImplementation(() => {}); + + const client = new HttpClient({ + apiKey: "test", + maxRetries: 1, + retryDelay: 0, + enableLogging: true, + }); + + mockRequest + .mockRejectedValueOnce({ + isAxiosError: true, + code: "ECONNABORTED", + message: "timeout", + }) + .mockResolvedValueOnce({ status: 200, data: "ok", headers: {} }); + + await client.requestRaw({ method: "GET", path: "/meta" }); + + expect(consoleInfo).toHaveBeenCalledWith( + expect.stringContaining("dify-client-node retry") + ); + consoleInfo.mockRestore(); + }); +}); diff --git a/sdks/nodejs-client/src/http/client.ts b/sdks/nodejs-client/src/http/client.ts new file mode 100644 index 0000000000..44b63c9903 --- /dev/null +++ b/sdks/nodejs-client/src/http/client.ts @@ -0,0 +1,368 @@ +import axios from "axios"; +import type { + AxiosError, + AxiosInstance, + AxiosRequestConfig, + AxiosResponse, +} from "axios"; +import type { Readable } from "node:stream"; +import { + DEFAULT_BASE_URL, + DEFAULT_MAX_RETRIES, + DEFAULT_RETRY_DELAY_SECONDS, + DEFAULT_TIMEOUT_SECONDS, +} from "../types/common"; +import type { + DifyClientConfig, + DifyResponse, + Headers, + QueryParams, + RequestMethod, +} from "../types/common"; +import type { DifyError } from "../errors/dify-error"; +import { + APIError, + AuthenticationError, + FileUploadError, + NetworkError, + RateLimitError, + TimeoutError, + ValidationError, +} from "../errors/dify-error"; +import { getFormDataHeaders, isFormData } from "./form-data"; +import { createBinaryStream, createSseStream } from "./sse"; +import { getRetryDelayMs, shouldRetry, sleep } from "./retry"; +import { validateParams } from "../client/validation"; + +const DEFAULT_USER_AGENT = "dify-client-node"; + +export type RequestOptions = { + method: RequestMethod; + path: string; + query?: QueryParams; + data?: unknown; + headers?: Headers; + responseType?: AxiosRequestConfig["responseType"]; +}; + +export type HttpClientSettings = Required< + Omit +> & { + apiKey: string; +}; + +const normalizeSettings = (config: DifyClientConfig): HttpClientSettings => ({ + apiKey: config.apiKey, + baseUrl: config.baseUrl ?? DEFAULT_BASE_URL, + timeout: config.timeout ?? DEFAULT_TIMEOUT_SECONDS, + maxRetries: config.maxRetries ?? DEFAULT_MAX_RETRIES, + retryDelay: config.retryDelay ?? DEFAULT_RETRY_DELAY_SECONDS, + enableLogging: config.enableLogging ?? false, +}); + +const normalizeHeaders = (headers: AxiosResponse["headers"]): Headers => { + const result: Headers = {}; + if (!headers) { + return result; + } + Object.entries(headers).forEach(([key, value]) => { + if (Array.isArray(value)) { + result[key.toLowerCase()] = value.join(", "); + } else if (typeof value === "string") { + result[key.toLowerCase()] = value; + } else if (typeof value === "number") { + result[key.toLowerCase()] = value.toString(); + } + }); + return result; +}; + +const resolveRequestId = (headers: Headers): string | undefined => + headers["x-request-id"] ?? headers["x-requestid"]; + +const buildRequestUrl = (baseUrl: string, path: string): string => { + const trimmed = baseUrl.replace(/\/+$/, ""); + return `${trimmed}${path}`; +}; + +const buildQueryString = (params?: QueryParams): string => { + if (!params) { + return ""; + } + const searchParams = new URLSearchParams(); + Object.entries(params).forEach(([key, value]) => { + if (value === undefined || value === null) { + return; + } + if (Array.isArray(value)) { + value.forEach((item) => { + searchParams.append(key, String(item)); + }); + return; + } + searchParams.append(key, String(value)); + }); + return searchParams.toString(); +}; + +const parseRetryAfterSeconds = (headerValue?: string): number | undefined => { + if (!headerValue) { + return undefined; + } + const asNumber = Number.parseInt(headerValue, 10); + if (!Number.isNaN(asNumber)) { + return asNumber; + } + const asDate = Date.parse(headerValue); + if (!Number.isNaN(asDate)) { + const diff = asDate - Date.now(); + return diff > 0 ? Math.ceil(diff / 1000) : 0; + } + return undefined; +}; + +const isReadableStream = (value: unknown): value is Readable => { + if (!value || typeof value !== "object") { + return false; + } + return typeof (value as { pipe?: unknown }).pipe === "function"; +}; + +const isUploadLikeRequest = (config?: AxiosRequestConfig): boolean => { + const url = (config?.url ?? "").toLowerCase(); + if (!url) { + return false; + } + return ( + url.includes("upload") || + url.includes("/files/") || + url.includes("audio-to-text") || + url.includes("create_by_file") || + url.includes("update_by_file") + ); +}; + +const resolveErrorMessage = (status: number, responseBody: unknown): string => { + if (typeof responseBody === "string" && responseBody.trim().length > 0) { + return responseBody; + } + if ( + responseBody && + typeof responseBody === "object" && + "message" in responseBody + ) { + const message = (responseBody as Record).message; + if (typeof message === "string" && message.trim().length > 0) { + return message; + } + } + return `Request failed with status code ${status}`; +}; + +const mapAxiosError = (error: unknown): DifyError => { + if (axios.isAxiosError(error)) { + const axiosError = error as AxiosError; + if (axiosError.response) { + const status = axiosError.response.status; + const headers = normalizeHeaders(axiosError.response.headers); + const requestId = resolveRequestId(headers); + const responseBody = axiosError.response.data; + const message = resolveErrorMessage(status, responseBody); + + if (status === 401) { + return new AuthenticationError(message, { + statusCode: status, + responseBody, + requestId, + }); + } + if (status === 429) { + const retryAfter = parseRetryAfterSeconds(headers["retry-after"]); + return new RateLimitError(message, { + statusCode: status, + responseBody, + requestId, + retryAfter, + }); + } + if (status === 422) { + return new ValidationError(message, { + statusCode: status, + responseBody, + requestId, + }); + } + if (status === 400) { + if (isUploadLikeRequest(axiosError.config)) { + return new FileUploadError(message, { + statusCode: status, + responseBody, + requestId, + }); + } + } + return new APIError(message, { + statusCode: status, + responseBody, + requestId, + }); + } + if (axiosError.code === "ECONNABORTED") { + return new TimeoutError("Request timed out", { cause: axiosError }); + } + return new NetworkError(axiosError.message, { cause: axiosError }); + } + if (error instanceof Error) { + return new NetworkError(error.message, { cause: error }); + } + return new NetworkError("Unexpected network error", { cause: error }); +}; + +export class HttpClient { + private axios: AxiosInstance; + private settings: HttpClientSettings; + + constructor(config: DifyClientConfig) { + this.settings = normalizeSettings(config); + this.axios = axios.create({ + baseURL: this.settings.baseUrl, + timeout: this.settings.timeout * 1000, + }); + } + + updateApiKey(apiKey: string): void { + this.settings.apiKey = apiKey; + } + + getSettings(): HttpClientSettings { + return { ...this.settings }; + } + + async request(options: RequestOptions): Promise> { + const response = await this.requestRaw(options); + const headers = normalizeHeaders(response.headers); + return { + data: response.data as T, + status: response.status, + headers, + requestId: resolveRequestId(headers), + }; + } + + async requestStream(options: RequestOptions) { + const response = await this.requestRaw({ + ...options, + responseType: "stream", + }); + const headers = normalizeHeaders(response.headers); + return createSseStream(response.data as Readable, { + status: response.status, + headers, + requestId: resolveRequestId(headers), + }); + } + + async requestBinaryStream(options: RequestOptions) { + const response = await this.requestRaw({ + ...options, + responseType: "stream", + }); + const headers = normalizeHeaders(response.headers); + return createBinaryStream(response.data as Readable, { + status: response.status, + headers, + requestId: resolveRequestId(headers), + }); + } + + async requestRaw(options: RequestOptions): Promise { + const { method, path, query, data, headers, responseType } = options; + const { apiKey, enableLogging, maxRetries, retryDelay, timeout } = + this.settings; + + if (query) { + validateParams(query as Record); + } + if ( + data && + typeof data === "object" && + !Array.isArray(data) && + !isFormData(data) && + !isReadableStream(data) + ) { + validateParams(data as Record); + } + + const requestHeaders: Headers = { + Authorization: `Bearer ${apiKey}`, + ...headers, + }; + if ( + typeof process !== "undefined" && + !!process.versions?.node && + !requestHeaders["User-Agent"] && + !requestHeaders["user-agent"] + ) { + requestHeaders["User-Agent"] = DEFAULT_USER_AGENT; + } + + if (isFormData(data)) { + Object.assign(requestHeaders, getFormDataHeaders(data)); + } else if (data && method !== "GET") { + requestHeaders["Content-Type"] = "application/json"; + } + + const url = buildRequestUrl(this.settings.baseUrl, path); + + if (enableLogging) { + console.info(`dify-client-node request ${method} ${url}`); + } + + const axiosConfig: AxiosRequestConfig = { + method, + url: path, + params: query, + paramsSerializer: { + serialize: (params) => buildQueryString(params as QueryParams), + }, + headers: requestHeaders, + responseType: responseType ?? "json", + timeout: timeout * 1000, + }; + + if (method !== "GET" && data !== undefined) { + axiosConfig.data = data; + } + + let attempt = 0; + // `attempt` is a zero-based retry counter + // Total attempts = 1 (initial) + maxRetries + // e.g., maxRetries=3 means: attempt 0 (initial), then retries at 1, 2, 3 + while (true) { + try { + const response = await this.axios.request(axiosConfig); + if (enableLogging) { + console.info( + `dify-client-node response ${response.status} ${method} ${url}` + ); + } + return response; + } catch (error) { + const mapped = mapAxiosError(error); + if (!shouldRetry(mapped, attempt, maxRetries)) { + throw mapped; + } + const retryAfterSeconds = + mapped instanceof RateLimitError ? mapped.retryAfter : undefined; + const delay = getRetryDelayMs(attempt + 1, retryDelay, retryAfterSeconds); + if (enableLogging) { + console.info( + `dify-client-node retry ${attempt + 1} in ${delay}ms for ${method} ${url}` + ); + } + attempt += 1; + await sleep(delay); + } + } + } +} diff --git a/sdks/nodejs-client/src/http/form-data.test.js b/sdks/nodejs-client/src/http/form-data.test.js new file mode 100644 index 0000000000..2938e41435 --- /dev/null +++ b/sdks/nodejs-client/src/http/form-data.test.js @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { getFormDataHeaders, isFormData } from "./form-data"; + +describe("form-data helpers", () => { + it("detects form-data like objects", () => { + const formLike = { + append: () => {}, + getHeaders: () => ({ "content-type": "multipart/form-data" }), + }; + expect(isFormData(formLike)).toBe(true); + expect(isFormData({})).toBe(false); + }); + + it("returns headers from form-data", () => { + const formLike = { + append: () => {}, + getHeaders: () => ({ "content-type": "multipart/form-data" }), + }; + expect(getFormDataHeaders(formLike)).toEqual({ + "content-type": "multipart/form-data", + }); + }); +}); diff --git a/sdks/nodejs-client/src/http/form-data.ts b/sdks/nodejs-client/src/http/form-data.ts new file mode 100644 index 0000000000..2efa23e54e --- /dev/null +++ b/sdks/nodejs-client/src/http/form-data.ts @@ -0,0 +1,31 @@ +import type { Headers } from "../types/common"; + +export type FormDataLike = { + append: (...args: unknown[]) => void; + getHeaders?: () => Headers; + constructor?: { name?: string }; +}; + +export const isFormData = (value: unknown): value is FormDataLike => { + if (!value || typeof value !== "object") { + return false; + } + if (typeof FormData !== "undefined" && value instanceof FormData) { + return true; + } + const candidate = value as FormDataLike; + if (typeof candidate.append !== "function") { + return false; + } + if (typeof candidate.getHeaders === "function") { + return true; + } + return candidate.constructor?.name === "FormData"; +}; + +export const getFormDataHeaders = (form: FormDataLike): Headers => { + if (typeof form.getHeaders === "function") { + return form.getHeaders(); + } + return {}; +}; diff --git a/sdks/nodejs-client/src/http/retry.test.js b/sdks/nodejs-client/src/http/retry.test.js new file mode 100644 index 0000000000..fc017f631b --- /dev/null +++ b/sdks/nodejs-client/src/http/retry.test.js @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { getRetryDelayMs, shouldRetry } from "./retry"; +import { NetworkError, RateLimitError, TimeoutError } from "../errors/dify-error"; + +const withMockedRandom = (value, fn) => { + const original = Math.random; + Math.random = () => value; + try { + fn(); + } finally { + Math.random = original; + } +}; + +describe("retry helpers", () => { + it("getRetryDelayMs honors retry-after header", () => { + expect(getRetryDelayMs(1, 1, 3)).toBe(3000); + }); + + it("getRetryDelayMs uses exponential backoff with jitter", () => { + withMockedRandom(0, () => { + expect(getRetryDelayMs(1, 1)).toBe(1000); + expect(getRetryDelayMs(2, 1)).toBe(2000); + expect(getRetryDelayMs(3, 1)).toBe(4000); + }); + }); + + it("shouldRetry respects max retries", () => { + expect(shouldRetry(new TimeoutError("timeout"), 3, 3)).toBe(false); + }); + + it("shouldRetry retries on network, timeout, and rate limit", () => { + expect(shouldRetry(new TimeoutError("timeout"), 0, 3)).toBe(true); + expect(shouldRetry(new NetworkError("network"), 0, 3)).toBe(true); + expect(shouldRetry(new RateLimitError("limit"), 0, 3)).toBe(true); + expect(shouldRetry(new Error("other"), 0, 3)).toBe(false); + }); +}); diff --git a/sdks/nodejs-client/src/http/retry.ts b/sdks/nodejs-client/src/http/retry.ts new file mode 100644 index 0000000000..3776b78d5f --- /dev/null +++ b/sdks/nodejs-client/src/http/retry.ts @@ -0,0 +1,40 @@ +import { RateLimitError, NetworkError, TimeoutError } from "../errors/dify-error"; + +export const sleep = (ms: number): Promise => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +export const getRetryDelayMs = ( + attempt: number, + retryDelaySeconds: number, + retryAfterSeconds?: number +): number => { + if (retryAfterSeconds && retryAfterSeconds > 0) { + return retryAfterSeconds * 1000; + } + const base = retryDelaySeconds * 1000; + const exponential = base * Math.pow(2, Math.max(0, attempt - 1)); + const jitter = Math.random() * base; + return exponential + jitter; +}; + +export const shouldRetry = ( + error: unknown, + attempt: number, + maxRetries: number +): boolean => { + if (attempt >= maxRetries) { + return false; + } + if (error instanceof TimeoutError) { + return true; + } + if (error instanceof NetworkError) { + return true; + } + if (error instanceof RateLimitError) { + return true; + } + return false; +}; diff --git a/sdks/nodejs-client/src/http/sse.test.js b/sdks/nodejs-client/src/http/sse.test.js new file mode 100644 index 0000000000..fff85fd29b --- /dev/null +++ b/sdks/nodejs-client/src/http/sse.test.js @@ -0,0 +1,76 @@ +import { Readable } from "node:stream"; +import { describe, expect, it } from "vitest"; +import { createBinaryStream, createSseStream, parseSseStream } from "./sse"; + +describe("sse parsing", () => { + it("parses event and data lines", async () => { + const stream = Readable.from([ + "event: message\n", + "data: {\"answer\":\"hi\"}\n", + "\n", + ]); + const events = []; + for await (const event of parseSseStream(stream)) { + events.push(event); + } + expect(events).toHaveLength(1); + expect(events[0].event).toBe("message"); + expect(events[0].data).toEqual({ answer: "hi" }); + }); + + it("handles multi-line data payloads", async () => { + const stream = Readable.from(["data: line1\n", "data: line2\n", "\n"]); + const events = []; + for await (const event of parseSseStream(stream)) { + events.push(event); + } + expect(events[0].raw).toBe("line1\nline2"); + expect(events[0].data).toBe("line1\nline2"); + }); + + it("createSseStream exposes toText", async () => { + const stream = Readable.from([ + "data: {\"answer\":\"hello\"}\n\n", + "data: {\"delta\":\" world\"}\n\n", + ]); + const sseStream = createSseStream(stream, { + status: 200, + headers: {}, + requestId: "req", + }); + const text = await sseStream.toText(); + expect(text).toBe("hello world"); + }); + + it("toText extracts text from string data", async () => { + const stream = Readable.from(["data: plain text\n\n"]); + const sseStream = createSseStream(stream, { status: 200, headers: {} }); + const text = await sseStream.toText(); + expect(text).toBe("plain text"); + }); + + it("toText extracts text field from object", async () => { + const stream = Readable.from(['data: {"text":"hello"}\n\n']); + const sseStream = createSseStream(stream, { status: 200, headers: {} }); + const text = await sseStream.toText(); + expect(text).toBe("hello"); + }); + + it("toText returns empty for invalid data", async () => { + const stream = Readable.from(["data: null\n\n", "data: 123\n\n"]); + const sseStream = createSseStream(stream, { status: 200, headers: {} }); + const text = await sseStream.toText(); + expect(text).toBe(""); + }); + + it("createBinaryStream exposes metadata", () => { + const stream = Readable.from(["chunk"]); + const binary = createBinaryStream(stream, { + status: 200, + headers: { "content-type": "audio/mpeg" }, + requestId: "req", + }); + expect(binary.status).toBe(200); + expect(binary.headers["content-type"]).toBe("audio/mpeg"); + }); +}); diff --git a/sdks/nodejs-client/src/http/sse.ts b/sdks/nodejs-client/src/http/sse.ts new file mode 100644 index 0000000000..ed5a17fe39 --- /dev/null +++ b/sdks/nodejs-client/src/http/sse.ts @@ -0,0 +1,133 @@ +import type { Readable } from "node:stream"; +import { StringDecoder } from "node:string_decoder"; +import type { BinaryStream, DifyStream, Headers, StreamEvent } from "../types/common"; + +const readLines = async function* (stream: Readable): AsyncIterable { + const decoder = new StringDecoder("utf8"); + let buffered = ""; + for await (const chunk of stream) { + buffered += decoder.write(chunk as Buffer); + let index = buffered.indexOf("\n"); + while (index >= 0) { + let line = buffered.slice(0, index); + buffered = buffered.slice(index + 1); + if (line.endsWith("\r")) { + line = line.slice(0, -1); + } + yield line; + index = buffered.indexOf("\n"); + } + } + buffered += decoder.end(); + if (buffered) { + yield buffered; + } +}; + +const parseMaybeJson = (value: string): unknown => { + if (!value) { + return null; + } + try { + return JSON.parse(value); + } catch { + return value; + } +}; + +export const parseSseStream = async function* ( + stream: Readable +): AsyncIterable> { + let eventName: string | undefined; + const dataLines: string[] = []; + + const emitEvent = function* (): Iterable> { + if (!eventName && dataLines.length === 0) { + return; + } + const raw = dataLines.join("\n"); + const parsed = parseMaybeJson(raw) as T | string | null; + yield { + event: eventName, + data: parsed, + raw, + }; + eventName = undefined; + dataLines.length = 0; + }; + + for await (const line of readLines(stream)) { + if (!line) { + yield* emitEvent(); + continue; + } + if (line.startsWith(":")) { + continue; + } + if (line.startsWith("event:")) { + eventName = line.slice("event:".length).trim(); + continue; + } + if (line.startsWith("data:")) { + dataLines.push(line.slice("data:".length).trimStart()); + continue; + } + } + + yield* emitEvent(); +}; + +const extractTextFromEvent = (data: unknown): string => { + if (typeof data === "string") { + return data; + } + if (!data || typeof data !== "object") { + return ""; + } + const record = data as Record; + if (typeof record.answer === "string") { + return record.answer; + } + if (typeof record.text === "string") { + return record.text; + } + if (typeof record.delta === "string") { + return record.delta; + } + return ""; +}; + +export const createSseStream = ( + stream: Readable, + meta: { status: number; headers: Headers; requestId?: string } +): DifyStream => { + const iterator = parseSseStream(stream)[Symbol.asyncIterator](); + const iterable = { + [Symbol.asyncIterator]: () => iterator, + data: stream, + status: meta.status, + headers: meta.headers, + requestId: meta.requestId, + toReadable: () => stream, + toText: async () => { + let text = ""; + for await (const event of iterable) { + text += extractTextFromEvent(event.data); + } + return text; + }, + } satisfies DifyStream; + + return iterable; +}; + +export const createBinaryStream = ( + stream: Readable, + meta: { status: number; headers: Headers; requestId?: string } +): BinaryStream => ({ + data: stream, + status: meta.status, + headers: meta.headers, + requestId: meta.requestId, + toReadable: () => stream, +}); diff --git a/sdks/nodejs-client/src/index.test.js b/sdks/nodejs-client/src/index.test.js new file mode 100644 index 0000000000..289f4d9b1b --- /dev/null +++ b/sdks/nodejs-client/src/index.test.js @@ -0,0 +1,227 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { ChatClient, DifyClient, WorkflowClient, BASE_URL, routes } from "./index"; +import axios from "axios"; + +const mockRequest = vi.fn(); + +const setupAxiosMock = () => { + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); +}; + +beforeEach(() => { + vi.restoreAllMocks(); + mockRequest.mockReset(); + setupAxiosMock(); +}); + +describe("Client", () => { + it("should create a client", () => { + new DifyClient("test"); + + expect(axios.create).toHaveBeenCalledWith({ + baseURL: BASE_URL, + timeout: 60000, + }); + }); + + it("should update the api key", () => { + const difyClient = new DifyClient("test"); + difyClient.updateApiKey("test2"); + + expect(difyClient.getHttpClient().getSettings().apiKey).toBe("test2"); + }); +}); + +describe("Send Requests", () => { + it("should make a successful request to the application parameter", async () => { + const difyClient = new DifyClient("test"); + const method = "GET"; + const endpoint = routes.application.url(); + mockRequest.mockResolvedValue({ + status: 200, + data: "response", + headers: {}, + }); + + await difyClient.sendRequest(method, endpoint); + + const requestConfig = mockRequest.mock.calls[0][0]; + expect(requestConfig).toMatchObject({ + method, + url: endpoint, + params: undefined, + responseType: "json", + timeout: 60000, + }); + expect(requestConfig.headers.Authorization).toBe("Bearer test"); + }); + + it("uses the getMeta route configuration", async () => { + const difyClient = new DifyClient("test"); + mockRequest.mockResolvedValue({ status: 200, data: "ok", headers: {} }); + + await difyClient.getMeta("end-user"); + + expect(mockRequest).toHaveBeenCalledWith(expect.objectContaining({ + method: routes.getMeta.method, + url: routes.getMeta.url(), + params: { user: "end-user" }, + headers: expect.objectContaining({ + Authorization: "Bearer test", + }), + responseType: "json", + timeout: 60000, + })); + }); +}); + +describe("File uploads", () => { + const OriginalFormData = globalThis.FormData; + + beforeAll(() => { + globalThis.FormData = class FormDataMock { + append() {} + + getHeaders() { + return { + "content-type": "multipart/form-data; boundary=test", + }; + } + }; + }); + + afterAll(() => { + globalThis.FormData = OriginalFormData; + }); + + it("does not override multipart boundary headers for FormData", async () => { + const difyClient = new DifyClient("test"); + const form = new globalThis.FormData(); + mockRequest.mockResolvedValue({ status: 200, data: "ok", headers: {} }); + + await difyClient.fileUpload(form, "end-user"); + + expect(mockRequest).toHaveBeenCalledWith(expect.objectContaining({ + method: routes.fileUpload.method, + url: routes.fileUpload.url(), + params: undefined, + headers: expect.objectContaining({ + Authorization: "Bearer test", + "content-type": "multipart/form-data; boundary=test", + }), + responseType: "json", + timeout: 60000, + data: form, + })); + }); +}); + +describe("Workflow client", () => { + it("uses tasks stop path for workflow stop", async () => { + const workflowClient = new WorkflowClient("test"); + mockRequest.mockResolvedValue({ status: 200, data: "stopped", headers: {} }); + + await workflowClient.stop("task-1", "end-user"); + + expect(mockRequest).toHaveBeenCalledWith(expect.objectContaining({ + method: routes.stopWorkflow.method, + url: routes.stopWorkflow.url("task-1"), + params: undefined, + headers: expect.objectContaining({ + Authorization: "Bearer test", + "Content-Type": "application/json", + }), + responseType: "json", + timeout: 60000, + data: { user: "end-user" }, + })); + }); + + it("maps workflow log filters to service api params", async () => { + const workflowClient = new WorkflowClient("test"); + mockRequest.mockResolvedValue({ status: 200, data: "ok", headers: {} }); + + await workflowClient.getLogs({ + createdAtAfter: "2024-01-01T00:00:00Z", + createdAtBefore: "2024-01-02T00:00:00Z", + createdByEndUserSessionId: "sess-1", + createdByAccount: "acc-1", + page: 2, + limit: 10, + }); + + expect(mockRequest).toHaveBeenCalledWith(expect.objectContaining({ + method: "GET", + url: "/workflows/logs", + params: { + created_at__after: "2024-01-01T00:00:00Z", + created_at__before: "2024-01-02T00:00:00Z", + created_by_end_user_session_id: "sess-1", + created_by_account: "acc-1", + page: 2, + limit: 10, + }, + headers: expect.objectContaining({ + Authorization: "Bearer test", + }), + responseType: "json", + timeout: 60000, + })); + }); +}); + +describe("Chat client", () => { + it("places user in query for suggested messages", async () => { + const chatClient = new ChatClient("test"); + mockRequest.mockResolvedValue({ status: 200, data: "ok", headers: {} }); + + await chatClient.getSuggested("msg-1", "end-user"); + + expect(mockRequest).toHaveBeenCalledWith(expect.objectContaining({ + method: routes.getSuggested.method, + url: routes.getSuggested.url("msg-1"), + params: { user: "end-user" }, + headers: expect.objectContaining({ + Authorization: "Bearer test", + }), + responseType: "json", + timeout: 60000, + })); + }); + + it("uses last_id when listing conversations", async () => { + const chatClient = new ChatClient("test"); + mockRequest.mockResolvedValue({ status: 200, data: "ok", headers: {} }); + + await chatClient.getConversations("end-user", "last-1", 10); + + expect(mockRequest).toHaveBeenCalledWith(expect.objectContaining({ + method: routes.getConversations.method, + url: routes.getConversations.url(), + params: { user: "end-user", last_id: "last-1", limit: 10 }, + headers: expect.objectContaining({ + Authorization: "Bearer test", + }), + responseType: "json", + timeout: 60000, + })); + }); + + it("lists app feedbacks without user params", async () => { + const chatClient = new ChatClient("test"); + mockRequest.mockResolvedValue({ status: 200, data: "ok", headers: {} }); + + await chatClient.getAppFeedbacks(1, 20); + + expect(mockRequest).toHaveBeenCalledWith(expect.objectContaining({ + method: "GET", + url: "/app/feedbacks", + params: { page: 1, limit: 20 }, + headers: expect.objectContaining({ + Authorization: "Bearer test", + }), + responseType: "json", + timeout: 60000, + })); + }); +}); diff --git a/sdks/nodejs-client/src/index.ts b/sdks/nodejs-client/src/index.ts new file mode 100644 index 0000000000..3ba3757baa --- /dev/null +++ b/sdks/nodejs-client/src/index.ts @@ -0,0 +1,103 @@ +import { DEFAULT_BASE_URL } from "./types/common"; + +export const BASE_URL = DEFAULT_BASE_URL; + +export const routes = { + feedback: { + method: "POST", + url: (messageId: string) => `/messages/${messageId}/feedbacks`, + }, + application: { + method: "GET", + url: () => "/parameters", + }, + fileUpload: { + method: "POST", + url: () => "/files/upload", + }, + filePreview: { + method: "GET", + url: (fileId: string) => `/files/${fileId}/preview`, + }, + textToAudio: { + method: "POST", + url: () => "/text-to-audio", + }, + audioToText: { + method: "POST", + url: () => "/audio-to-text", + }, + getMeta: { + method: "GET", + url: () => "/meta", + }, + getInfo: { + method: "GET", + url: () => "/info", + }, + getSite: { + method: "GET", + url: () => "/site", + }, + createCompletionMessage: { + method: "POST", + url: () => "/completion-messages", + }, + stopCompletionMessage: { + method: "POST", + url: (taskId: string) => `/completion-messages/${taskId}/stop`, + }, + createChatMessage: { + method: "POST", + url: () => "/chat-messages", + }, + getSuggested: { + method: "GET", + url: (messageId: string) => `/messages/${messageId}/suggested`, + }, + stopChatMessage: { + method: "POST", + url: (taskId: string) => `/chat-messages/${taskId}/stop`, + }, + getConversations: { + method: "GET", + url: () => "/conversations", + }, + getConversationMessages: { + method: "GET", + url: () => "/messages", + }, + renameConversation: { + method: "POST", + url: (conversationId: string) => `/conversations/${conversationId}/name`, + }, + deleteConversation: { + method: "DELETE", + url: (conversationId: string) => `/conversations/${conversationId}`, + }, + runWorkflow: { + method: "POST", + url: () => "/workflows/run", + }, + stopWorkflow: { + method: "POST", + url: (taskId: string) => `/workflows/tasks/${taskId}/stop`, + }, +}; + +export { DifyClient } from "./client/base"; +export { ChatClient } from "./client/chat"; +export { CompletionClient } from "./client/completion"; +export { WorkflowClient } from "./client/workflow"; +export { KnowledgeBaseClient } from "./client/knowledge-base"; +export { WorkspaceClient } from "./client/workspace"; + +export * from "./errors/dify-error"; +export * from "./types/common"; +export * from "./types/annotation"; +export * from "./types/chat"; +export * from "./types/completion"; +export * from "./types/knowledge-base"; +export * from "./types/workflow"; +export * from "./types/workspace"; +export { HttpClient } from "./http/client"; diff --git a/sdks/nodejs-client/src/types/annotation.ts b/sdks/nodejs-client/src/types/annotation.ts new file mode 100644 index 0000000000..dcbd644dab --- /dev/null +++ b/sdks/nodejs-client/src/types/annotation.ts @@ -0,0 +1,18 @@ +export type AnnotationCreateRequest = { + question: string; + answer: string; +}; + +export type AnnotationReplyActionRequest = { + score_threshold: number; + embedding_provider_name: string; + embedding_model_name: string; +}; + +export type AnnotationListOptions = { + page?: number; + limit?: number; + keyword?: string; +}; + +export type AnnotationResponse = Record; diff --git a/sdks/nodejs-client/src/types/chat.ts b/sdks/nodejs-client/src/types/chat.ts new file mode 100644 index 0000000000..5b627f6cf6 --- /dev/null +++ b/sdks/nodejs-client/src/types/chat.ts @@ -0,0 +1,17 @@ +import type { StreamEvent } from "./common"; + +export type ChatMessageRequest = { + inputs?: Record; + query: string; + user: string; + response_mode?: "blocking" | "streaming"; + files?: Array> | null; + conversation_id?: string; + auto_generate_name?: boolean; + workflow_id?: string; + retriever_from?: "app" | "dataset"; +}; + +export type ChatMessageResponse = Record; + +export type ChatStreamEvent = StreamEvent>; diff --git a/sdks/nodejs-client/src/types/common.ts b/sdks/nodejs-client/src/types/common.ts new file mode 100644 index 0000000000..00b0fcc756 --- /dev/null +++ b/sdks/nodejs-client/src/types/common.ts @@ -0,0 +1,71 @@ +export const DEFAULT_BASE_URL = "https://api.dify.ai/v1"; +export const DEFAULT_TIMEOUT_SECONDS = 60; +export const DEFAULT_MAX_RETRIES = 3; +export const DEFAULT_RETRY_DELAY_SECONDS = 1; + +export type RequestMethod = "GET" | "POST" | "PATCH" | "PUT" | "DELETE"; + +export type QueryParamValue = + | string + | number + | boolean + | Array + | undefined; + +export type QueryParams = Record; + +export type Headers = Record; + +export type DifyClientConfig = { + apiKey: string; + baseUrl?: string; + timeout?: number; + maxRetries?: number; + retryDelay?: number; + enableLogging?: boolean; +}; + +export type DifyResponse = { + data: T; + status: number; + headers: Headers; + requestId?: string; +}; + +export type MessageFeedbackRequest = { + messageId: string; + user: string; + rating?: "like" | "dislike" | null; + content?: string | null; +}; + +export type TextToAudioRequest = { + user: string; + text?: string; + message_id?: string; + streaming?: boolean; + voice?: string; +}; + +export type StreamEvent = { + event?: string; + data: T | string | null; + raw: string; +}; + +export type DifyStream = AsyncIterable> & { + data: NodeJS.ReadableStream; + status: number; + headers: Headers; + requestId?: string; + toText(): Promise; + toReadable(): NodeJS.ReadableStream; +}; + +export type BinaryStream = { + data: NodeJS.ReadableStream; + status: number; + headers: Headers; + requestId?: string; + toReadable(): NodeJS.ReadableStream; +}; diff --git a/sdks/nodejs-client/src/types/completion.ts b/sdks/nodejs-client/src/types/completion.ts new file mode 100644 index 0000000000..4074137c5d --- /dev/null +++ b/sdks/nodejs-client/src/types/completion.ts @@ -0,0 +1,13 @@ +import type { StreamEvent } from "./common"; + +export type CompletionRequest = { + inputs?: Record; + response_mode?: "blocking" | "streaming"; + user: string; + files?: Array> | null; + retriever_from?: "app" | "dataset"; +}; + +export type CompletionResponse = Record; + +export type CompletionStreamEvent = StreamEvent>; diff --git a/sdks/nodejs-client/src/types/knowledge-base.ts b/sdks/nodejs-client/src/types/knowledge-base.ts new file mode 100644 index 0000000000..a4ddef50ea --- /dev/null +++ b/sdks/nodejs-client/src/types/knowledge-base.ts @@ -0,0 +1,184 @@ +export type DatasetListOptions = { + page?: number; + limit?: number; + keyword?: string | null; + tagIds?: string[]; + includeAll?: boolean; +}; + +export type DatasetCreateRequest = { + name: string; + description?: string; + indexing_technique?: "high_quality" | "economy"; + permission?: string | null; + external_knowledge_api_id?: string | null; + provider?: string; + external_knowledge_id?: string | null; + retrieval_model?: Record | null; + embedding_model?: string | null; + embedding_model_provider?: string | null; +}; + +export type DatasetUpdateRequest = { + name?: string; + description?: string | null; + indexing_technique?: "high_quality" | "economy" | null; + permission?: string | null; + embedding_model?: string | null; + embedding_model_provider?: string | null; + retrieval_model?: Record | null; + partial_member_list?: Array> | null; + external_retrieval_model?: Record | null; + external_knowledge_id?: string | null; + external_knowledge_api_id?: string | null; +}; + +export type DocumentStatusAction = "enable" | "disable" | "archive" | "un_archive"; + +export type DatasetTagCreateRequest = { + name: string; +}; + +export type DatasetTagUpdateRequest = { + tag_id: string; + name: string; +}; + +export type DatasetTagDeleteRequest = { + tag_id: string; +}; + +export type DatasetTagBindingRequest = { + tag_ids: string[]; + target_id: string; +}; + +export type DatasetTagUnbindingRequest = { + tag_id: string; + target_id: string; +}; + +export type DocumentTextCreateRequest = { + name: string; + text: string; + process_rule?: Record | null; + original_document_id?: string | null; + doc_form?: string; + doc_language?: string; + indexing_technique?: string | null; + retrieval_model?: Record | null; + embedding_model?: string | null; + embedding_model_provider?: string | null; +}; + +export type DocumentTextUpdateRequest = { + name?: string | null; + text?: string | null; + process_rule?: Record | null; + doc_form?: string; + doc_language?: string; + retrieval_model?: Record | null; +}; + +export type DocumentListOptions = { + page?: number; + limit?: number; + keyword?: string | null; + status?: string | null; +}; + +export type DocumentGetOptions = { + metadata?: "all" | "only" | "without"; +}; + +export type SegmentCreateRequest = { + segments: Array>; +}; + +export type SegmentUpdateRequest = { + segment: { + content?: string | null; + answer?: string | null; + keywords?: string[] | null; + regenerate_child_chunks?: boolean; + enabled?: boolean | null; + attachment_ids?: string[] | null; + }; +}; + +export type SegmentListOptions = { + page?: number; + limit?: number; + status?: string[]; + keyword?: string | null; +}; + +export type ChildChunkCreateRequest = { + content: string; +}; + +export type ChildChunkUpdateRequest = { + content: string; +}; + +export type ChildChunkListOptions = { + page?: number; + limit?: number; + keyword?: string | null; +}; + +export type MetadataCreateRequest = { + type: "string" | "number" | "time"; + name: string; +}; + +export type MetadataUpdateRequest = { + name: string; + value?: string | number | null; +}; + +export type DocumentMetadataDetail = { + id: string; + name: string; + value?: string | number | null; +}; + +export type DocumentMetadataOperation = { + document_id: string; + metadata_list: DocumentMetadataDetail[]; + partial_update?: boolean; +}; + +export type MetadataOperationRequest = { + operation_data: DocumentMetadataOperation[]; +}; + +export type HitTestingRequest = { + query?: string | null; + retrieval_model?: Record | null; + external_retrieval_model?: Record | null; + attachment_ids?: string[] | null; +}; + +export type DatasourcePluginListOptions = { + isPublished?: boolean; +}; + +export type DatasourceNodeRunRequest = { + inputs: Record; + datasource_type: string; + credential_id?: string | null; + is_published: boolean; +}; + +export type PipelineRunRequest = { + inputs: Record; + datasource_type: string; + datasource_info_list: Array>; + start_node_id: string; + is_published: boolean; + response_mode: "streaming" | "blocking"; +}; + +export type KnowledgeBaseResponse = Record; +export type PipelineStreamEvent = Record; diff --git a/sdks/nodejs-client/src/types/workflow.ts b/sdks/nodejs-client/src/types/workflow.ts new file mode 100644 index 0000000000..2b507c7352 --- /dev/null +++ b/sdks/nodejs-client/src/types/workflow.ts @@ -0,0 +1,12 @@ +import type { StreamEvent } from "./common"; + +export type WorkflowRunRequest = { + inputs?: Record; + user: string; + response_mode?: "blocking" | "streaming"; + files?: Array> | null; +}; + +export type WorkflowRunResponse = Record; + +export type WorkflowStreamEvent = StreamEvent>; diff --git a/sdks/nodejs-client/src/types/workspace.ts b/sdks/nodejs-client/src/types/workspace.ts new file mode 100644 index 0000000000..0ab6743063 --- /dev/null +++ b/sdks/nodejs-client/src/types/workspace.ts @@ -0,0 +1,2 @@ +export type WorkspaceModelType = string; +export type WorkspaceModelsResponse = Record; diff --git a/sdks/nodejs-client/tests/test-utils.js b/sdks/nodejs-client/tests/test-utils.js new file mode 100644 index 0000000000..0d42514e9a --- /dev/null +++ b/sdks/nodejs-client/tests/test-utils.js @@ -0,0 +1,30 @@ +import axios from "axios"; +import { vi } from "vitest"; +import { HttpClient } from "../src/http/client"; + +export const createHttpClient = (configOverrides = {}) => { + const mockRequest = vi.fn(); + vi.spyOn(axios, "create").mockReturnValue({ request: mockRequest }); + const client = new HttpClient({ apiKey: "test", ...configOverrides }); + return { client, mockRequest }; +}; + +export const createHttpClientWithSpies = (configOverrides = {}) => { + const { client, mockRequest } = createHttpClient(configOverrides); + const request = vi + .spyOn(client, "request") + .mockResolvedValue({ data: "ok", status: 200, headers: {} }); + const requestStream = vi + .spyOn(client, "requestStream") + .mockResolvedValue({ data: null }); + const requestBinaryStream = vi + .spyOn(client, "requestBinaryStream") + .mockResolvedValue({ data: null }); + return { + client, + mockRequest, + request, + requestStream, + requestBinaryStream, + }; +}; diff --git a/sdks/nodejs-client/tsconfig.json b/sdks/nodejs-client/tsconfig.json new file mode 100644 index 0000000000..d2da9a2a59 --- /dev/null +++ b/sdks/nodejs-client/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "rootDir": "src", + "outDir": "dist", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts"] +} diff --git a/sdks/nodejs-client/tsup.config.ts b/sdks/nodejs-client/tsup.config.ts new file mode 100644 index 0000000000..522382c2a5 --- /dev/null +++ b/sdks/nodejs-client/tsup.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + clean: true, + sourcemap: true, + splitting: false, + treeshake: true, + outDir: "dist", +}); diff --git a/sdks/nodejs-client/vitest.config.ts b/sdks/nodejs-client/vitest.config.ts new file mode 100644 index 0000000000..5a0a8637a2 --- /dev/null +++ b/sdks/nodejs-client/vitest.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + include: ["**/*.test.js"], + coverage: { + provider: "v8", + reporter: ["text", "text-summary"], + include: ["src/**/*.ts"], + exclude: ["src/**/*.test.*", "src/**/*.spec.*"], + }, + }, +});