opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/rag-2' of https://github.com/langgenius/dify into feat/rag-2

This commit is contained in:
twwu 2025-08-06 10:33:50 +08:00
parent ec6fabb222 6cae1a2872
commit 6954926df9
9 changed files with 65 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
api/core/rag/index_processor/processor/parent_child_index_processor.py
View File

@ -1,5 +1,6 @@
"""Paragraph index processor."""
import json
import uuid
from collections.abc import Mapping
from typing import Any, Optional
@ -16,7 +17,7 @@ from core.rag.index_processor.index_processor_base import BaseIndexProcessor
from core.rag.models.document import ChildDocument, Document, ParentChildStructureChunk
from extensions.ext_database import db
from libs import helper
from models.dataset import ChildChunk, Dataset, DocumentSegment
from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentSegment
from models.dataset import Document as DatasetDocument
from services.entities.knowledge_entities.knowledge_entities import ParentMode, Rule
@ -228,13 +229,31 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
doc = Document(page_content=parent_child.parent_content, metadata=metadata, children=child_documents)
documents.append(doc)
if documents:
# update document parent mode
dataset_process_rule = DatasetProcessRule(
dataset_id=dataset.id,
mode="hierarchical",
rules=json.dumps({
"parent_mode": parent_childs.parent_mode,
}),
created_by=document.created_by,
)
db.session.add(dataset_process_rule)
db.session.flush()
document.dataset_process_rule_id = dataset_process_rule.id
db.session.commit()
# save node to document segment
doc_store = DatasetDocumentStore(dataset=dataset, user_id=document.created_by, document_id=document.id)
# add document segments
doc_store.add_documents(docs=documents, save_child=True)
if dataset.indexing_technique == "high_quality":
vector = Vector(dataset)
vector.create(documents)
all_child_documents = []
for doc in documents:
if doc.children:
all_child_documents.extend(doc.children)
if all_child_documents:
vector = Vector(dataset)
vector.create(all_child_documents)
def format_preview(self, chunks: Mapping[str, Any]) -> Mapping[str, Any]:
parent_childs = ParentChildStructureChunk(**chunks)

1
api/core/rag/models/document.py
View File

@ -58,6 +58,7 @@ class ParentChildStructureChunk(BaseModel):
"""
parent_child_chunks: list[ParentChildChunk]
parent_mode: str = "paragraph"
class QAChunk(BaseModel):

4
web/app/components/base/prompt-editor/index.tsx
View File

@ -1,6 +1,6 @@
'use client'
import type { FC } from 'react'
import type { FC, ReactNode } from 'react'
import { useEffect } from 'react'
import type {
EditorState,
@ -66,7 +66,7 @@ export type PromptEditorProps = {
compact?: boolean
wrapperClassName?: string
className?: string
placeholder?: string | JSX.Element
placeholder?: string | ReactNode
placeholderClassName?: string
style?: React.CSSProperties
value?: string

6
web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx
View File

@ -34,6 +34,7 @@ type WorkflowVariableBlockComponentProps = {
workflowNodesMap: WorkflowNodesMap
environmentVariables?: Var[]
conversationVariables?: Var[]
ragVariables?: Var[]
getVarType?: (payload: {
nodeId: string,
valueSelector: ValueSelector,
@ -47,6 +48,7 @@ const WorkflowVariableBlockComponent = ({
getVarType,
environmentVariables,
conversationVariables,
ragVariables,
}: WorkflowVariableBlockComponentProps) => {
const { t } = useTranslation()
const [editor] = useLexicalComposerContext()
@ -66,7 +68,6 @@ const WorkflowVariableBlockComponent = ({
const isEnv = isENV(variables)
const isChatVar = isConversationVar(variables)
const isException = isExceptionVariable(varName, node?.type)
let variableValid = true
if (isEnv) {
if (environmentVariables)
@ -77,7 +78,8 @@ const WorkflowVariableBlockComponent = ({
variableValid = conversationVariables.some(v => v.variable === `${variables?.[0] ?? ''}.${variables?.[1] ?? ''}`)
}
else if (isRagVar) {
variableValid = variables[0] === 'rag'
if (ragVariables)
variableValid = ragVariables.some(v => v.variable === `${variables?.[0] ?? ''}.${variables?.[1] ?? ''}.${variables?.[2] ?? ''}`)
}
else {
variableValid = !!node

20
web/app/components/base/prompt-editor/plugins/workflow-variable-block/node.tsx
View File

@ -13,6 +13,7 @@ export type SerializedNode = SerializedLexicalNode & {
getVarType?: GetVarType
environmentVariables?: Var[]
conversationVariables?: Var[]
ragVariables?: Var[]
}
export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element> {
@ -21,20 +22,21 @@ export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element>
__getVarType?: GetVarType
__environmentVariables?: Var[]
__conversationVariables?: Var[]
__ragVariables?: Var[]
static getType(): string {
return 'workflow-variable-block'
}
static clone(node: WorkflowVariableBlockNode): WorkflowVariableBlockNode {
return new WorkflowVariableBlockNode(node.__variables, node.__workflowNodesMap, node.__getVarType, node.__key, node.__environmentVariables, node.__conversationVariables)
return new WorkflowVariableBlockNode(node.__variables, node.__workflowNodesMap, node.__getVarType, node.__key, node.__environmentVariables, node.__conversationVariables, node.__ragVariables)
}
isInline(): boolean {
return true
}
constructor(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType: any, key?: NodeKey, environmentVariables?: Var[], conversationVariables?: Var[]) {
constructor(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType: any, key?: NodeKey, environmentVariables?: Var[], conversationVariables?: Var[], ragVariables?: Var[]) {
super(key)
this.__variables = variables
@ -42,6 +44,7 @@ export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element>
this.__getVarType = getVarType
this.__environmentVariables = environmentVariables
this.__conversationVariables = conversationVariables
this.__ragVariables = ragVariables
}
createDOM(): HTMLElement {
@ -63,12 +66,13 @@ export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element>
getVarType={this.__getVarType!}
environmentVariables={this.__environmentVariables}
conversationVariables={this.__conversationVariables}
ragVariables={this.__ragVariables}
/>
)
}
static importJSON(serializedNode: SerializedNode): WorkflowVariableBlockNode {
const node = $createWorkflowVariableBlockNode(serializedNode.variables, serializedNode.workflowNodesMap, serializedNode.getVarType, serializedNode.environmentVariables, serializedNode.conversationVariables)
const node = $createWorkflowVariableBlockNode(serializedNode.variables, serializedNode.workflowNodesMap, serializedNode.getVarType, serializedNode.environmentVariables, serializedNode.conversationVariables, serializedNode.ragVariables)
return node
}
@ -82,6 +86,7 @@ export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element>
getVarType: this.getVarType(),
environmentVariables: this.getEnvironmentVariables(),
conversationVariables: this.getConversationVariables(),
ragVariables: this.getRagVariables(),
}
}
@ -110,12 +115,17 @@ export class WorkflowVariableBlockNode extends DecoratorNode<React.JSX.Element>
return self.__conversationVariables
}
getRagVariables(): any {
const self = this.getLatest()
return self.__ragVariables
}
getTextContent(): string {
return `{{#${this.getVariables().join('.')}#}}`
}
}
export function $createWorkflowVariableBlockNode(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType?: GetVarType, environmentVariables?: Var[], conversationVariables?: Var[]): WorkflowVariableBlockNode {
return new WorkflowVariableBlockNode(variables, workflowNodesMap, getVarType, undefined, environmentVariables, conversationVariables)
export function $createWorkflowVariableBlockNode(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType?: GetVarType, environmentVariables?: Var[], conversationVariables?: Var[], ragVariables?: Var[]): WorkflowVariableBlockNode {
return new WorkflowVariableBlockNode(variables, workflowNodesMap, getVarType, undefined, environmentVariables, conversationVariables, ragVariables)
}
export function $isWorkflowVariableBlockNode(

2
web/app/components/base/prompt-editor/plugins/workflow-variable-block/workflow-variable-block-replacement-block.tsx
View File

@ -32,7 +32,7 @@ const WorkflowVariableBlockReplacementBlock = ({
onInsert()
const nodePathString = textNode.getTextContent().slice(3, -3)
return $applyNodeReplacement($createWorkflowVariableBlockNode(nodePathString.split('.'), workflowNodesMap, getVarType, variables?.find(o => o.nodeId === 'env')?.vars || [], variables?.find(o => o.nodeId === 'conversation')?.vars || []))
return $applyNodeReplacement($createWorkflowVariableBlockNode(nodePathString.split('.'), workflowNodesMap, getVarType, variables?.find(o => o.nodeId === 'env')?.vars || [], variables?.find(o => o.nodeId === 'conversation')?.vars || [], variables?.find(o => o.nodeId === 'rag')?.vars || []))
}, [onInsert, workflowNodesMap, getVarType, variables])
const getMatch = useCallback((text: string) => {

2
web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx
View File

@ -45,7 +45,6 @@ import Tooltip from '@/app/components/base/tooltip'
import { isExceptionVariable } from '@/app/components/workflow/utils'
import VarFullPathPanel from './var-full-path-panel'
import { noop } from 'lodash-es'
import { useStore as useWorkflowStore } from '@/app/components/workflow/store'
import { useFetchDynamicOptions } from '@/service/use-plugins'
import type { Tool } from '@/app/components/tools/types'
import { VariableIconWithColor } from '@/app/components/workflow/nodes/_base/components/variable/variable-label'
@ -129,7 +128,6 @@ const VarReferencePicker: FC<Props> = ({
})
const node = nodes.find(n => n.id === nodeId)
const ragPipelineVariables = useWorkflowStore(s => s.ragPipelineVariables)
const isInIteration = !!(node?.data as any).isInIteration
const iterationNode = isInIteration ? nodes.find(n => n.id === node?.parentId) : null

5
web/app/components/workflow/nodes/_base/hooks/use-available-var-list.ts
View File

@ -75,7 +75,10 @@ const useAvailableVarList = (nodeId: string, {
return {
availableVars,
availableNodes,
availableNodesWithParent: availableNodes,
availableNodesWithParent: [
...availableNodes,
...(isDataSourceNode ? [currNode] : []),
],
}
}

42
web/app/components/workflow/nodes/data-source/panel.tsx
View File

@ -1,8 +1,6 @@
import type { FC } from 'react'
import {
useCallback,
useMemo,
useState,
} from 'react'
import { useTranslation } from 'react-i18next'
import { memo } from 'react'
@ -24,10 +22,8 @@ import {
WEBSITE_CRAWL_OUTPUT,
} from './constants'
import { useStore } from '@/app/components/workflow/store'
import InputVarList from '@/app/components/workflow/nodes/tool/components/input-var-list'
import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema'
import type { Var } from '@/app/components/workflow/types'
import { VarType } from '@/app/components/workflow/types'
import ToolForm from '../tool/components/tool-form'
const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
const { t } = useTranslation()
@ -52,18 +48,9 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
const formSchemas = useMemo(() => {
return currentDataSourceItem ? toolParametersToFormSchemas(currentDataSourceItem.parameters) : []
}, [currentDataSourceItem])
const [currVarIndex, setCurrVarIndex] = useState(-1)
const currVarType = formSchemas[currVarIndex]?._type
const handleOnVarOpen = useCallback((index: number) => {
setCurrVarIndex(index)
}, [])
const filterVar = useCallback((varPayload: Var) => {
if (currVarType)
return varPayload.type === currVarType
return varPayload.type !== VarType.arrayFile
}, [currVarType])
const pipelineId = useStore(s => s.pipelineId)
const setShowInputFieldPanel = useStore(s => s.setShowInputFieldPanel)
return (
<div >
@ -80,16 +67,19 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
supportCollapse: true,
}}
>
<InputVarList
readOnly={nodesReadOnly}
nodeId={id}
schema={formSchemas as any}
filterVar={filterVar}
value={datasource_parameters}
onChange={handleParametersChange}
isSupportConstantValue
onOpen={handleOnVarOpen}
/>
{formSchemas.length > 0 && (
<ToolForm
readOnly={nodesReadOnly}
nodeId={id}
schema={formSchemas as any}
value={datasource_parameters}
onChange={handleParametersChange}
currentProvider={currentDataSource}
currentTool={currentDataSourceItem}
showManageInputField={!!pipelineId}
onManageInputField={() => setShowInputFieldPanel?.(true)}
/>
)}
</BoxGroupField>
)
}