From 6024dbe98d50d37ee462190a86b18e577ef11241 Mon Sep 17 00:00:00 2001
From: twwu <twwu@dify.ai>
Date: Thu, 8 May 2025 18:29:49 +0800
Subject: [PATCH] refactor: simplify type definitions in form components and
 update related configurations

---
 .../base/form/form-scenarios/base/field.tsx   | 10 +++++-----
 .../base/form/form-scenarios/base/index.tsx   |  8 ++++----
 .../base/form/form-scenarios/base/types.ts    | 19 +++++++++----------
 .../base/form/form-scenarios/base/utils.ts    |  2 +-
 .../form/form-scenarios/input-field/field.tsx | 10 +++++-----
 .../form/form-scenarios/input-field/types.ts  |  8 ++++----
 .../form/form-scenarios/input-field/utils.ts  |  2 +-
 .../form/form-scenarios/node-panel/field.tsx  | 10 +++++-----
 .../form/form-scenarios/node-panel/types.ts   |  8 ++++----
 web/app/components/base/form/index.tsx        |  2 ++
 .../data-source/website/base/options.tsx      | 13 ++++---------
 .../data-source/website/firecrawl/hooks.ts    |  3 +--
 .../data-source/website/jina-reader/hooks.ts  |  3 +--
 .../data-source/website/water-crawl/hooks.ts  |  3 +--
 web/models/pipeline.ts                        | 15 +++++++++++++++
 web/service/use-pipeline.ts                   | 10 ++++++++++
 16 files changed, 72 insertions(+), 54 deletions(-)

diff --git a/web/app/components/base/form/form-scenarios/base/field.tsx b/web/app/components/base/form/form-scenarios/base/field.tsx
index f763ffd00c..637496548f 100644
--- a/web/app/components/base/form/form-scenarios/base/field.tsx
+++ b/web/app/components/base/form/form-scenarios/base/field.tsx
@@ -3,15 +3,15 @@ import { type BaseConfiguration, BaseFieldType } from './types'
 import { withForm } from '../..'
 import { useStore } from '@tanstack/react-form'
 
-type BaseFieldProps<T> = {
-  initialData?: T
-  config: BaseConfiguration<T>
+type BaseFieldProps = {
+  initialData?: Record<string, any>
+  config: BaseConfiguration
 }
 
-const BaseField = <T,>({
+const BaseField = ({
   initialData,
   config,
-}: BaseFieldProps<T>) => withForm({
+}: BaseFieldProps) => withForm({
   defaultValues: initialData,
   render: function Render({
     form,
diff --git a/web/app/components/base/form/form-scenarios/base/index.tsx b/web/app/components/base/form/form-scenarios/base/index.tsx
index d133e304be..bee4952303 100644
--- a/web/app/components/base/form/form-scenarios/base/index.tsx
+++ b/web/app/components/base/form/form-scenarios/base/index.tsx
@@ -4,14 +4,14 @@ import BaseField from './field'
 import type { BaseFormProps } from './types'
 import { generateZodSchema } from './utils'
 
-const BaseForm = <T,>({
+const BaseForm = ({
   initialData,
   configurations,
   onSubmit,
   CustomActions,
-}: BaseFormProps<T>) => {
+}: BaseFormProps) => {
   const schema = useMemo(() => {
-    const schema = generateZodSchema<T>(configurations)
+    const schema = generateZodSchema(configurations)
     return schema
   }, [configurations])
 
@@ -44,7 +44,7 @@ const BaseForm = <T,>({
     >
       <div className='flex flex-col gap-4 px-4 py-2'>
         {configurations.map((config, index) => {
-          const FieldComponent = BaseField<T>({
+          const FieldComponent = BaseField({
             initialData,
             config,
           })
diff --git a/web/app/components/base/form/form-scenarios/base/types.ts b/web/app/components/base/form/form-scenarios/base/types.ts
index 3c4e50b723..a756a912fa 100644
--- a/web/app/components/base/form/form-scenarios/base/types.ts
+++ b/web/app/components/base/form/form-scenarios/base/types.ts
@@ -1,4 +1,3 @@
-import type { DeepKeys } from '@tanstack/react-form'
 import type { FormType } from '../..'
 import type { Option } from '../../../select/pure'
 
@@ -9,8 +8,8 @@ export enum BaseFieldType {
   select = 'select',
 }
 
-export type ShowCondition<T> = {
-  variable: DeepKeys<T>
+export type ShowCondition = {
+  variable: string
   value: any
 }
 
@@ -29,21 +28,21 @@ export type SelectConfiguration = {
   }
 }
 
-export type BaseConfiguration<T> = {
+export type BaseConfiguration = {
   label: string
-  variable: DeepKeys<T> // Variable name
+  variable: string // Variable name
   maxLength?: number // Max length for text input
   placeholder?: string
   required: boolean
   showOptional?: boolean // show optional label
-  showConditions: ShowCondition<T>[] // Show this field only when all conditions are met
+  showConditions: ShowCondition[] // Show this field only when all conditions are met
   type: BaseFieldType
   tooltip?: string // Tooltip for this field
 } & NumberConfiguration & Partial<SelectConfiguration>
 
-export type BaseFormProps<T> = {
-  initialData?: T
-  configurations: BaseConfiguration<T>[]
+export type BaseFormProps = {
+  initialData?: Record<string, any>
+  configurations: BaseConfiguration[]
   CustomActions?: (form: FormType) => React.ReactNode
-  onSubmit: (value: T) => void
+  onSubmit: (value: Record<string, any>) => void
 }
diff --git a/web/app/components/base/form/form-scenarios/base/utils.ts b/web/app/components/base/form/form-scenarios/base/utils.ts
index d92abe109e..a89c4fb2c7 100644
--- a/web/app/components/base/form/form-scenarios/base/utils.ts
+++ b/web/app/components/base/form/form-scenarios/base/utils.ts
@@ -2,7 +2,7 @@ import type { ZodNumber, ZodSchema, ZodString } from 'zod'
 import { z } from 'zod'
 import { type BaseConfiguration, BaseFieldType } from './types'
 
-export const generateZodSchema = <T>(fields: BaseConfiguration<T>[]) => {
+export const generateZodSchema = (fields: BaseConfiguration[]) => {
   const shape: Record<string, ZodSchema> = {}
 
   fields.forEach((field) => {
diff --git a/web/app/components/base/form/form-scenarios/input-field/field.tsx b/web/app/components/base/form/form-scenarios/input-field/field.tsx
index 125622fa4d..ab16b9a482 100644
--- a/web/app/components/base/form/form-scenarios/input-field/field.tsx
+++ b/web/app/components/base/form/form-scenarios/input-field/field.tsx
@@ -3,15 +3,15 @@ import { type InputFieldConfiguration, InputFieldType } from './types'
 import { withForm } from '../..'
 import { useStore } from '@tanstack/react-form'
 
-type InputFieldProps<T> = {
-  initialData?: T
-  config: InputFieldConfiguration<T>
+type InputFieldProps = {
+  initialData?: Record<string, any>
+  config: InputFieldConfiguration
 }
 
-const InputField = <T,>({
+const InputField = ({
   initialData,
   config,
-}: InputFieldProps<T>) => withForm({
+}: InputFieldProps) => withForm({
   defaultValues: initialData,
   render: function Render({
     form,
diff --git a/web/app/components/base/form/form-scenarios/input-field/types.ts b/web/app/components/base/form/form-scenarios/input-field/types.ts
index b6c97d506d..5ffbacb721 100644
--- a/web/app/components/base/form/form-scenarios/input-field/types.ts
+++ b/web/app/components/base/form/form-scenarios/input-field/types.ts
@@ -23,17 +23,17 @@ export type NumberSliderConfiguration = {
   min?: number
 }
 
-export type InputFieldConfiguration<T> = {
+export type InputFieldConfiguration = {
   label: string
-  variable: DeepKeys<T> // Variable name
+  variable: string // Variable name
   maxLength?: number // Max length for text input
   placeholder?: string
   required: boolean
   showOptional?: boolean // show optional label
-  showConditions: ShowCondition<T>[] // Show this field only when all conditions are met
+  showConditions: ShowCondition[] // Show this field only when all conditions are met
   type: InputFieldType
   tooltip?: string // Tooltip for this field
-  listeners?: FieldListeners<T, DeepKeys<T>> // Listener for this field
+  listeners?: FieldListeners<Record<string, any>, DeepKeys<Record<string, any>>> // Listener for this field
 } & NumberConfiguration & Partial<InputTypeSelectConfiguration>
 & Partial<NumberSliderConfiguration>
 & Partial<SelectConfiguration>
diff --git a/web/app/components/base/form/form-scenarios/input-field/utils.ts b/web/app/components/base/form/form-scenarios/input-field/utils.ts
index 797cc32e95..b0852f6816 100644
--- a/web/app/components/base/form/form-scenarios/input-field/utils.ts
+++ b/web/app/components/base/form/form-scenarios/input-field/utils.ts
@@ -3,7 +3,7 @@ import { z } from 'zod'
 import { type InputFieldConfiguration, InputFieldType } from './types'
 import { SupportedFileTypes, TransferMethod } from '@/app/components/rag-pipeline/components/input-field/editor/form/schema'
 
-export const generateZodSchema = <T>(fields: InputFieldConfiguration<T>[]) => {
+export const generateZodSchema = (fields: InputFieldConfiguration[]) => {
   const shape: Record<string, ZodSchema> = {}
 
   fields.forEach((field) => {
diff --git a/web/app/components/base/form/form-scenarios/node-panel/field.tsx b/web/app/components/base/form/form-scenarios/node-panel/field.tsx
index 00cc401960..069700787b 100644
--- a/web/app/components/base/form/form-scenarios/node-panel/field.tsx
+++ b/web/app/components/base/form/form-scenarios/node-panel/field.tsx
@@ -3,15 +3,15 @@ import { type InputFieldConfiguration, InputFieldType } from './types'
 import { withForm } from '../..'
 import { useStore } from '@tanstack/react-form'
 
-type InputFieldProps<T> = {
-  initialData?: T
-  config: InputFieldConfiguration<T>
+type InputFieldProps = {
+  initialData?: Record<string, any>
+  config: InputFieldConfiguration
 }
 
-const NodePanelField = <T,>({
+const NodePanelField = ({
   initialData,
   config,
-}: InputFieldProps<T>) => withForm({
+}: InputFieldProps) => withForm({
   defaultValues: initialData,
   render: function Render({
     form,
diff --git a/web/app/components/base/form/form-scenarios/node-panel/types.ts b/web/app/components/base/form/form-scenarios/node-panel/types.ts
index 9ff663a57a..327ee9b159 100644
--- a/web/app/components/base/form/form-scenarios/node-panel/types.ts
+++ b/web/app/components/base/form/form-scenarios/node-panel/types.ts
@@ -24,17 +24,17 @@ export type NumberSliderConfiguration = {
   min?: number
 }
 
-export type InputFieldConfiguration<T> = {
+export type InputFieldConfiguration = {
   label: string
-  variable: DeepKeys<T> // Variable name
+  variable: string // Variable name
   maxLength?: number // Max length for text input
   placeholder?: string
   required: boolean
   showOptional?: boolean // show optional label
-  showConditions: ShowCondition<T>[] // Show this field only when all conditions are met
+  showConditions: ShowCondition[] // Show this field only when all conditions are met
   type: InputFieldType
   tooltip?: string // Tooltip for this field
-  listeners?: FieldListeners<T, DeepKeys<T>> // Listener for this field
+  listeners?: FieldListeners<Record<string, any>, DeepKeys<Record<string, any>>> // Listener for this field
 } & NumberConfiguration & Partial<InputTypeSelectConfiguration>
 & Partial<NumberSliderConfiguration>
 & Partial<SelectConfiguration>
diff --git a/web/app/components/base/form/index.tsx b/web/app/components/base/form/index.tsx
index a2c3e6b1ee..a7185940ba 100644
--- a/web/app/components/base/form/index.tsx
+++ b/web/app/components/base/form/index.tsx
@@ -10,6 +10,7 @@ import InputTypeSelectField from './components/field/input-type-select'
 import FileTypesField from './components/field/file-types'
 import UploadMethodField from './components/field/upload-method'
 import NumberSliderField from './components/field/number-slider'
+import VariableOrConstantInputField from './components/field/variable-selector'
 
 export const { fieldContext, useFieldContext, formContext, useFormContext }
   = createFormHookContexts()
@@ -26,6 +27,7 @@ export const { useAppForm, withForm } = createFormHook({
     FileTypesField,
     UploadMethodField,
     NumberSliderField,
+    VariableOrConstantInputField,
   },
   formComponents: {
     Actions,
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/options.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/options.tsx
index f1d24d4838..d1d2742fbb 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/options.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/options.tsx
@@ -3,7 +3,6 @@ import { useAppForm } from '@/app/components/base/form'
 import BaseField from '@/app/components/base/form/form-scenarios/base/field'
 import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
 import { ArrowDownRoundFill } from '@/app/components/base/icons/src/vender/solid/general'
-import type { CrawlOptions } from '@/models/datasets'
 import cn from '@/utils/classnames'
 import { RiPlayLargeLine } from '@remixicon/react'
 import { useBoolean } from 'ahooks'
@@ -14,17 +13,13 @@ import type { ZodSchema } from 'zod'
 
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
-export type FormData = {
-  url: string
-} & CrawlOptions
-
 type OptionsProps = {
-  initialData: FormData
-  configurations: BaseConfiguration<FormData>[]
+  initialData: Record<string, any>
+  configurations: BaseConfiguration[]
   isRunning: boolean
   controlFoldOptions?: number
   schema: ZodSchema
-  onSubmit: (data: FormData) => void
+  onSubmit: (data: Record<string, any>) => void
 }
 
 const Options = ({
@@ -104,7 +99,7 @@ const Options = ({
       {!fold && (
         <div className='flex flex-col gap-3 border-t border-divider-subtle px-4 py-3'>
           {configurations.map((config, index) => {
-            const FieldComponent = BaseField<FormData>({
+            const FieldComponent = BaseField({
               initialData,
               config,
             })
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/hooks.ts
index 777c568356..42f90ab75a 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/hooks.ts
+++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/hooks.ts
@@ -1,7 +1,6 @@
 import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
 import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
 import { useTranslation } from 'react-i18next'
-import type { FormData } from '../base/options'
 import { z } from 'zod'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
@@ -9,7 +8,7 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 export const useConfigurations = () => {
     const { t } = useTranslation()
-    const configurations: BaseConfiguration<FormData>[] = [
+    const configurations: BaseConfiguration[] = [
       {
         type: BaseFieldType.textInput,
         variable: 'url',
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/hooks.ts
index f1b2562427..9f57726c04 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/hooks.ts
+++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/hooks.ts
@@ -1,7 +1,6 @@
 import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
 import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
 import { useTranslation } from 'react-i18next'
-import type { FormData } from '../base/options'
 import { z } from 'zod'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
@@ -9,7 +8,7 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 export const useConfigurations = () => {
     const { t } = useTranslation()
-    const configurations: BaseConfiguration<FormData>[] = [
+    const configurations: BaseConfiguration[] = [
       {
         type: BaseFieldType.textInput,
         variable: 'url',
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/hooks.ts
index 777c568356..42f90ab75a 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/hooks.ts
+++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/hooks.ts
@@ -1,7 +1,6 @@
 import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
 import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
 import { useTranslation } from 'react-i18next'
-import type { FormData } from '../base/options'
 import { z } from 'zod'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
@@ -9,7 +8,7 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 export const useConfigurations = () => {
     const { t } = useTranslation()
-    const configurations: BaseConfiguration<FormData>[] = [
+    const configurations: BaseConfiguration[] = [
       {
         type: BaseFieldType.textInput,
         variable: 'url',
diff --git a/web/models/pipeline.ts b/web/models/pipeline.ts
index 0128da841e..d61cc059f4 100644
--- a/web/models/pipeline.ts
+++ b/web/models/pipeline.ts
@@ -1,3 +1,4 @@
+import type { InputVarType } from '@/app/components/workflow/types'
 import type { DSLImportMode, DSLImportStatus } from './app'
 import type { ChunkingMode, IconInfo } from './datasets'
 import type { Dependency } from '@/app/components/plugins/types'
@@ -69,3 +70,17 @@ export type ImportPipelineDSLResponse = {
   error: string
   leaked_dependencies: Dependency[]
 }
+
+export type Variables = {
+  type: InputVarType
+  label: string
+  description: string
+  variable: string
+  max_length: number
+  required: boolean
+  options?: string[]
+}
+
+export type PipelineProcessingParamsResponse = {
+  variables: Variables[]
+}
diff --git a/web/service/use-pipeline.ts b/web/service/use-pipeline.ts
index 13ada282ad..b6a00610b2 100644
--- a/web/service/use-pipeline.ts
+++ b/web/service/use-pipeline.ts
@@ -6,6 +6,7 @@ import type {
   ExportPipelineDSLResponse,
   ImportPipelineDSLRequest,
   ImportPipelineDSLResponse,
+  PipelineProcessingParamsResponse,
   PipelineTemplateByIdResponse,
   PipelineTemplateListParams,
   PipelineTemplateListResponse,
@@ -85,3 +86,12 @@ export const useImportPipelineDSL = (
     ...mutationOptions,
   })
 }
+
+export const usePipelineProcessingParams = (pipelineId: string) => {
+  return useQuery<PipelineProcessingParamsResponse>({
+    queryKey: [NAME_SPACE, 'pipeline-processing-params', pipelineId],
+    queryFn: () => {
+      return get<PipelineProcessingParamsResponse>(`/rag/pipeline/${pipelineId}/workflows/processing/parameters`)
+    },
+  })
+}