Merge branch 'feat/rag-2' of https://github.com/langgenius/dify into feat/rag-2

api/core/rag/index_processor/processor/parent_child_index_processor.py

@@ -1,5 +1,6 @@
 """Paragraph index processor."""
 
+import json
 import uuid
 from collections.abc import Mapping
 from typing import Any, Optional
@@ -16,7 +17,7 @@ from core.rag.index_processor.index_processor_base import BaseIndexProcessor
 from core.rag.models.document import ChildDocument, Document, ParentChildStructureChunk
 from extensions.ext_database import db
 from libs import helper
-from models.dataset import ChildChunk, Dataset, DocumentSegment
+from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentSegment
 from models.dataset import Document as DatasetDocument
 from services.entities.knowledge_entities.knowledge_entities import ParentMode, Rule
 
@@ -228,13 +229,31 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
             doc = Document(page_content=parent_child.parent_content, metadata=metadata, children=child_documents)
             documents.append(doc)
         if documents:
+            # update document parent mode
+            dataset_process_rule = DatasetProcessRule(
+                dataset_id=dataset.id,
+                mode="hierarchical",
+                rules=json.dumps({
+                    "parent_mode": parent_childs.parent_mode,
+                }),
+                created_by=document.created_by,
+            )
+            db.session.add(dataset_process_rule)
+            db.session.flush()
+            document.dataset_process_rule_id = dataset_process_rule.id
+            db.session.commit()
             # save node to document segment
             doc_store = DatasetDocumentStore(dataset=dataset, user_id=document.created_by, document_id=document.id)
             # add document segments
             doc_store.add_documents(docs=documents, save_child=True)
             if dataset.indexing_technique == "high_quality":
-                vector = Vector(dataset)
+                all_child_documents = []
-                vector.create(documents)
+                for doc in documents:
+                    if doc.children:
+                        all_child_documents.extend(doc.children)
+                if all_child_documents:
+                    vector = Vector(dataset)
+                    vector.create(all_child_documents)
 
     def format_preview(self, chunks: Mapping[str, Any]) -> Mapping[str, Any]:
         parent_childs = ParentChildStructureChunk(**chunks)

api/core/rag/models/document.py

@@ -58,6 +58,7 @@ class ParentChildStructureChunk(BaseModel):
     """
 
     parent_child_chunks: list[ParentChildChunk]
+    parent_mode: str = "paragraph"
 
 
 class QAChunk(BaseModel):

web/app/components/base/prompt-editor/index.tsx

@@ -1,6 +1,6 @@
 'use client'
 
-import type { FC } from 'react'
+import type { FC, ReactNode } from 'react'
 import { useEffect } from 'react'
 import type {
   EditorState,
@@ -66,7 +66,7 @@ export type PromptEditorProps = {
   compact?: boolean
   wrapperClassName?: string
   className?: string
-  placeholder?: string | JSX.Element
+  placeholder?: string | ReactNode
   placeholderClassName?: string
   style?: React.CSSProperties
   value?: string

web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx

@@ -34,6 +34,7 @@ type WorkflowVariableBlockComponentProps = {
   workflowNodesMap: WorkflowNodesMap
   environmentVariables?: Var[]
   conversationVariables?: Var[]
+  ragVariables?: Var[]
   getVarType?: (payload: {
     nodeId: string,
     valueSelector: ValueSelector,
@@ -47,6 +48,7 @@ const WorkflowVariableBlockComponent = ({
   getVarType,
   environmentVariables,
   conversationVariables,
+  ragVariables,
 }: WorkflowVariableBlockComponentProps) => {
   const { t } = useTranslation()
   const [editor] = useLexicalComposerContext()
@@ -66,7 +68,6 @@ const WorkflowVariableBlockComponent = ({
   const isEnv = isENV(variables)
   const isChatVar = isConversationVar(variables)
   const isException = isExceptionVariable(varName, node?.type)
-
   let variableValid = true
   if (isEnv) {
     if (environmentVariables)
@@ -77,7 +78,8 @@ const WorkflowVariableBlockComponent = ({
       variableValid = conversationVariables.some(v => v.variable === `${variables?.[0] ?? ''}.${variables?.[1] ?? ''}`)
   }
   else if (isRagVar) {
-    variableValid = variables[0] === 'rag'
+    if (ragVariables)
+      variableValid = ragVariables.some(v => v.variable === `${variables?.[0] ?? ''}.${variables?.[1] ?? ''}.${variables?.[2] ?? ''}`)
   }
   else {
     variableValid = !!node

web/app/components/base/prompt-editor/plugins/workflow-variable-block/node.tsx

@@ -13,6 +13,7 @@ export type SerializedNode = SerializedLexicalNode & {
   getVarType?: GetVarType
   environmentVariables?: Var[]
   conversationVariables?: Var[]
+  ragVariables?: Var[]
 }
 
 export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element> {
@@ -21,20 +22,21 @@ export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element>
   __getVarType?: GetVarType
   __environmentVariables?: Var[]
   __conversationVariables?: Var[]
+  __ragVariables?: Var[]
 
   static getType(): string {
     return 'workflow-variable-block'
   }
 
   static clone(node: WorkflowVariableBlockNode): WorkflowVariableBlockNode {
-    return new WorkflowVariableBlockNode(node.__variables, node.__workflowNodesMap, node.__getVarType, node.__key, node.__environmentVariables, node.__conversationVariables)
+    return new WorkflowVariableBlockNode(node.__variables, node.__workflowNodesMap, node.__getVarType, node.__key, node.__environmentVariables, node.__conversationVariables, node.__ragVariables)
   }
 
   isInline(): boolean {
     return true
   }
 
-  constructor(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType: any, key?: NodeKey, environmentVariables?: Var[], conversationVariables?: Var[]) {
+  constructor(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType: any, key?: NodeKey, environmentVariables?: Var[], conversationVariables?: Var[], ragVariables?: Var[]) {
     super(key)
 
     this.__variables = variables
@@ -42,6 +44,7 @@ export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element>
     this.__getVarType = getVarType
     this.__environmentVariables = environmentVariables
     this.__conversationVariables = conversationVariables
+    this.__ragVariables = ragVariables
   }
 
   createDOM(): HTMLElement {
@@ -63,12 +66,13 @@ export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element>
         getVarType={this.__getVarType!}
         environmentVariables={this.__environmentVariables}
         conversationVariables={this.__conversationVariables}
+        ragVariables={this.__ragVariables}
       />
     )
   }
 
   static importJSON(serializedNode: SerializedNode): WorkflowVariableBlockNode {
-    const node = $createWorkflowVariableBlockNode(serializedNode.variables, serializedNode.workflowNodesMap, serializedNode.getVarType, serializedNode.environmentVariables, serializedNode.conversationVariables)
+    const node = $createWorkflowVariableBlockNode(serializedNode.variables, serializedNode.workflowNodesMap, serializedNode.getVarType, serializedNode.environmentVariables, serializedNode.conversationVariables, serializedNode.ragVariables)
 
     return node
   }
@@ -82,6 +86,7 @@ export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element>
       getVarType: this.getVarType(),
       environmentVariables: this.getEnvironmentVariables(),
       conversationVariables: this.getConversationVariables(),
+      ragVariables: this.getRagVariables(),
     }
   }
 
@@ -110,12 +115,17 @@ export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element>
     return self.__conversationVariables
   }
 
+  getRagVariables(): any {
+    const self = this.getLatest()
+    return self.__ragVariables
+  }
+
   getTextContent(): string {
     return `{{#${this.getVariables().join('.')}#}}`
   }
 }
-export function $createWorkflowVariableBlockNode(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType?: GetVarType, environmentVariables?: Var[], conversationVariables?: Var[]): WorkflowVariableBlockNode {
+export function $createWorkflowVariableBlockNode(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType?: GetVarType, environmentVariables?: Var[], conversationVariables?: Var[], ragVariables?: Var[]): WorkflowVariableBlockNode {
-  return new WorkflowVariableBlockNode(variables, workflowNodesMap, getVarType, undefined, environmentVariables, conversationVariables)
+  return new WorkflowVariableBlockNode(variables, workflowNodesMap, getVarType, undefined, environmentVariables, conversationVariables, ragVariables)
 }
 
 export function $isWorkflowVariableBlockNode(

web/app/components/base/prompt-editor/plugins/workflow-variable-block/workflow-variable-block-replacement-block.tsx

@@ -32,7 +32,7 @@ const WorkflowVariableBlockReplacementBlock = ({
       onInsert()
 
     const nodePathString = textNode.getTextContent().slice(3, -3)
-    return $applyNodeReplacement($createWorkflowVariableBlockNode(nodePathString.split('.'), workflowNodesMap, getVarType, variables?.find(o => o.nodeId === 'env')?.vars || [], variables?.find(o => o.nodeId === 'conversation')?.vars || []))
+    return $applyNodeReplacement($createWorkflowVariableBlockNode(nodePathString.split('.'), workflowNodesMap, getVarType, variables?.find(o => o.nodeId === 'env')?.vars || [], variables?.find(o => o.nodeId === 'conversation')?.vars || [], variables?.find(o => o.nodeId === 'rag')?.vars || []))
   }, [onInsert, workflowNodesMap, getVarType, variables])
 
   const getMatch = useCallback((text: string) => {

web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx

@@ -45,7 +45,6 @@ import Tooltip from '@/app/components/base/tooltip'
 import { isExceptionVariable } from '@/app/components/workflow/utils'
 import VarFullPathPanel from './var-full-path-panel'
 import { noop } from 'lodash-es'
-import { useStore as useWorkflowStore } from '@/app/components/workflow/store'
 import { useFetchDynamicOptions } from '@/service/use-plugins'
 import type { Tool } from '@/app/components/tools/types'
 import { VariableIconWithColor } from '@/app/components/workflow/nodes/_base/components/variable/variable-label'
@@ -129,7 +128,6 @@ const VarReferencePicker: FC<Props> = ({
   })
 
   const node = nodes.find(n => n.id === nodeId)
-  const ragPipelineVariables = useWorkflowStore(s => s.ragPipelineVariables)
   const isInIteration = !!(node?.data as any).isInIteration
   const iterationNode = isInIteration ? nodes.find(n => n.id === node?.parentId) : null
 

web/app/components/workflow/nodes/_base/hooks/use-available-var-list.ts

@@ -75,7 +75,10 @@ const useAvailableVarList = (nodeId: string, {
   return {
     availableVars,
     availableNodes,
-    availableNodesWithParent: availableNodes,
+    availableNodesWithParent: [
+      ...availableNodes,
+      ...(isDataSourceNode ? [currNode] : []),
+    ],
   }
 }
 

web/app/components/workflow/nodes/data-source/panel.tsx

@@ -1,8 +1,6 @@
 import type { FC } from 'react'
 import {
-  useCallback,
   useMemo,
-  useState,
 } from 'react'
 import { useTranslation } from 'react-i18next'
 import { memo } from 'react'
@@ -24,10 +22,8 @@ import {
   WEBSITE_CRAWL_OUTPUT,
 } from './constants'
 import { useStore } from '@/app/components/workflow/store'
-import InputVarList from '@/app/components/workflow/nodes/tool/components/input-var-list'
 import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema'
-import type { Var } from '@/app/components/workflow/types'
+import ToolForm from '../tool/components/tool-form'
-import { VarType } from '@/app/components/workflow/types'
 
 const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
   const { t } = useTranslation()
@@ -52,18 +48,9 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
   const formSchemas = useMemo(() => {
     return currentDataSourceItem ? toolParametersToFormSchemas(currentDataSourceItem.parameters) : []
   }, [currentDataSourceItem])
-  const [currVarIndex, setCurrVarIndex] = useState(-1)
-  const currVarType = formSchemas[currVarIndex]?._type
-  const handleOnVarOpen = useCallback((index: number) => {
-    setCurrVarIndex(index)
-  }, [])
 
-  const filterVar = useCallback((varPayload: Var) => {
+  const pipelineId = useStore(s => s.pipelineId)
-    if (currVarType)
+  const setShowInputFieldPanel = useStore(s => s.setShowInputFieldPanel)
-      return varPayload.type === currVarType
-
-    return varPayload.type !== VarType.arrayFile
-  }, [currVarType])
 
   return (
     <div >
@@ -80,16 +67,19 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
               supportCollapse: true,
             }}
           >
-            <InputVarList
+            {formSchemas.length > 0 && (
-              readOnly={nodesReadOnly}
+              <ToolForm
-              nodeId={id}
+                readOnly={nodesReadOnly}
-              schema={formSchemas as any}
+                nodeId={id}
-              filterVar={filterVar}
+                schema={formSchemas as any}
-              value={datasource_parameters}
+                value={datasource_parameters}
-              onChange={handleParametersChange}
+                onChange={handleParametersChange}
-              isSupportConstantValue
+                currentProvider={currentDataSource}
-              onOpen={handleOnVarOpen}
+                currentTool={currentDataSourceItem}
-            />
+                showManageInputField={!!pipelineId}
+                onManageInputField={() => setShowInputFieldPanel?.(true)}
+              />
+            )}
           </BoxGroupField>
         )
       }