opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feat/rag-2' into feat/rag-2

This commit is contained in:
jyong 2025-07-11 18:46:29 +08:00
parent 9c4356e9a1 0fdb1fedb0
commit ed77877db1
10 changed files with 133 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
web/app/components/app-sidebar/navLink.tsx
View File

@ -1,5 +1,5 @@
'use client'
import React from 'react'
import { useSelectedLayoutSegment } from 'next/navigation'
import Link from 'next/link'
import classNames from '@/utils/classnames'
@ -22,13 +22,13 @@ export type NavLinkProps = {
disabled?: boolean
}
export default function NavLink({
const NavLink = ({
name,
href,
iconMap,
mode = 'expand',
disabled = false,
}: NavLinkProps) {
}: NavLinkProps) => {
const segment = useSelectedLayoutSegment()
const formattedSegment = (() => {
let res = segment?.toLowerCase()
@ -90,3 +90,5 @@ export default function NavLink({
</Link>
)
}
export default React.memo(NavLink)

34
web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx
View File

@ -11,6 +11,7 @@ import Toast from '@/app/components/base/toast'
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
import { useShallow } from 'zustand/react/shallow'
type OnlineDocumentsProps = {
isInPipeline?: boolean
@ -24,11 +25,17 @@ const OnlineDocuments = ({
nodeData,
}: OnlineDocumentsProps) => {
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const documentsData = useDataSourceStoreWithSelector(state => state.documentsData)
const searchValue = useDataSourceStoreWithSelector(state => state.searchValue)
const selectedPagesId = useDataSourceStoreWithSelector(state => state.selectedPagesId)
const currentWorkspaceId = useDataSourceStoreWithSelector(state => state.currentWorkspaceId)
const currentNodeIdRef = useDataSourceStoreWithSelector(state => state.currentNodeIdRef)
const {
documentsData,
searchValue,
selectedPagesId,
currentWorkspaceId,
} = useDataSourceStoreWithSelector(useShallow(state => ({
documentsData: state.documentsData,
searchValue: state.searchValue,
selectedPagesId: state.selectedPagesId,
currentWorkspaceId: state.currentWorkspaceId,
})))
const dataSourceStore = useDataSourceStore()
const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
@ -75,15 +82,16 @@ const OnlineDocuments = ({
}, [dataSourceStore, datasourceNodeRunURL])
useEffect(() => {
const {
setDocumentsData,
setCurrentWorkspaceId,
setSearchValue,
setSelectedPagesId,
setOnlineDocuments,
setCurrentDocument,
currentNodeIdRef,
} = dataSourceStore.getState()
if (nodeId !== currentNodeIdRef.current) {
const {
setDocumentsData,
setCurrentWorkspaceId,
setSearchValue,
setSelectedPagesId,
setOnlineDocuments,
setCurrentDocument,
} = dataSourceStore.getState()
setDocumentsData([])
setCurrentWorkspaceId('')
setSearchValue('')

35
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
View File

@ -11,6 +11,7 @@ import Toast from '@/app/components/base/toast'
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
import { convertOnlineDriveData } from './utils'
import produce from 'immer'
import { useShallow } from 'zustand/react/shallow'
type OnlineDriveProps = {
nodeId: string
@ -24,12 +25,19 @@ const OnlineDrive = ({
isInPipeline = false,
}: OnlineDriveProps) => {
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const prefix = useDataSourceStoreWithSelector(state => state.prefix)
const keywords = useDataSourceStoreWithSelector(state => state.keywords)
const bucket = useDataSourceStoreWithSelector(state => state.bucket)
const selectedFileKeys = useDataSourceStoreWithSelector(state => state.selectedFileKeys)
const fileList = useDataSourceStoreWithSelector(state => state.fileList)
const currentNodeIdRef = useDataSourceStoreWithSelector(state => state.currentNodeIdRef)
const {
prefix,
keywords,
bucket,
selectedFileKeys,
fileList,
} = useDataSourceStoreWithSelector(useShallow(state => ({
prefix: state.prefix,
keywords: state.keywords,
bucket: state.bucket,
selectedFileKeys: state.selectedFileKeys,
fileList: state.fileList,
})))
const dataSourceStore = useDataSourceStore()
const [isLoading, setIsLoading] = useState(false)
@ -83,14 +91,15 @@ const OnlineDrive = ({
}, [datasourceNodeRunURL, dataSourceStore])
useEffect(() => {
const {
setFileList,
setBucket,
setPrefix,
setKeywords,
setSelectedFileKeys,
currentNodeIdRef,
} = dataSourceStore.getState()
if (nodeId !== currentNodeIdRef.current) {
const {
setFileList,
setBucket,
setPrefix,
setKeywords,
setSelectedFileKeys,
} = dataSourceStore.getState()
setFileList([])
setBucket('')
setPrefix([])

20
web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx
View File

@ -22,6 +22,7 @@ import type {
} from '@/types/pipeline'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
import { useShallow } from 'zustand/react/shallow'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
@ -42,10 +43,17 @@ const WebsiteCrawl = ({
const [crawledNum, setCrawledNum] = useState(0)
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const crawlResult = useDataSourceStoreWithSelector(state => state.crawlResult)
const step = useDataSourceStoreWithSelector(state => state.step)
const checkedCrawlResult = useDataSourceStoreWithSelector(state => state.websitePages)
const previewIndex = useDataSourceStoreWithSelector(state => state.previewIndex)
const {
crawlResult,
step,
checkedCrawlResult,
previewIndex,
} = useDataSourceStoreWithSelector(useShallow(state => ({
crawlResult: state.crawlResult,
step: state.step,
checkedCrawlResult: state.websitePages,
previewIndex: state.previewIndex,
})))
const dataSourceStore = useDataSourceStore()
const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams)
@ -132,7 +140,7 @@ const WebsiteCrawl = ({
time_consuming: time_consuming ?? 0,
}
setCrawlResult(crawlResultData)
handleCheckedCrawlResultChange(crawlData || []) // default select the crawl result
handleCheckedCrawlResultChange(isInPipeline ? [crawlData[0]] : crawlData) // default select the crawl result
setCrawlErrorMessage('')
setStep(CrawlStep.finished)
},
@ -142,7 +150,7 @@ const WebsiteCrawl = ({
},
},
)
}, [dataSourceStore, datasourceNodeRunURL, handleCheckedCrawlResultChange, t])
}, [dataSourceStore, datasourceNodeRunURL, handleCheckedCrawlResultChange, isInPipeline, t])
const handleSubmit = useCallback((value: Record<string, any>) => {
handleRun(value)

43
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
View File

@ -6,6 +6,7 @@ import { BlockEnum, type Node } from '@/app/components/workflow/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { useDataSourceStore, useDataSourceStoreWithSelector } from './data-source/store'
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace } from '@/models/common'
import { useShallow } from 'zustand/react/shallow'
export const useAddDocumentsSteps = () => {
const { t } = useTranslation()
@ -62,8 +63,13 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
}
export const useLocalFile = () => {
const fileList = useDataSourceStoreWithSelector(state => state.localFileList)
const currentLocalFile = useDataSourceStoreWithSelector(state => state.currentLocalFile)
const {
localFileList: fileList,
currentLocalFile,
} = useDataSourceStoreWithSelector(useShallow(state => ({
localFileList: state.localFileList,
currentLocalFile: state.currentLocalFile,
})))
const dataSourceStore = useDataSourceStore()
const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList])
@ -82,10 +88,17 @@ export const useLocalFile = () => {
}
export const useOnlineDocuments = () => {
const documentsData = useDataSourceStoreWithSelector(state => state.documentsData)
const currentWorkspaceId = useDataSourceStoreWithSelector(state => state.currentWorkspaceId)
const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments)
const currentDocument = useDataSourceStoreWithSelector(state => state.currentDocument)
const {
documentsData,
currentWorkspaceId,
onlineDocuments,
currentDocument,
} = useDataSourceStoreWithSelector(useShallow(state => ({
documentsData: state.documentsData,
currentWorkspaceId: state.currentWorkspaceId,
onlineDocuments: state.onlineDocuments,
currentDocument: state.currentDocument,
})))
const dataSourceStore = useDataSourceStore()
const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId)
@ -119,8 +132,13 @@ export const useOnlineDocuments = () => {
}
export const useWebsiteCrawl = () => {
const websitePages = useDataSourceStoreWithSelector(state => state.websitePages)
const currentWebsite = useDataSourceStoreWithSelector(state => state.currentWebsite)
const {
websitePages,
currentWebsite,
} = useDataSourceStoreWithSelector(useShallow(state => ({
websitePages: state.websitePages,
currentWebsite: state.currentWebsite,
})))
const dataSourceStore = useDataSourceStore()
const hideWebsitePreview = useCallback(() => {
@ -137,8 +155,13 @@ export const useWebsiteCrawl = () => {
}
export const useOnlineDrive = () => {
const fileList = useDataSourceStoreWithSelector(state => state.fileList)
const selectedFileKeys = useDataSourceStoreWithSelector(state => state.selectedFileKeys)
const {
fileList,
selectedFileKeys,
} = useDataSourceStoreWithSelector(useShallow(state => ({
fileList: state.fileList,
selectedFileKeys: state.selectedFileKeys,
})))
const selectedOnlineDriveFileList = useMemo(() => {
return selectedFileKeys.map(key => fileList.find(item => item.key === key)!)

5
web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx
View File

@ -16,6 +16,7 @@ import Badge from '@/app/components/base/badge'
import Button from '@/app/components/base/button'
import type { OnlineDriveFile } from '@/models/pipeline'
import { DatasourceType } from '@/models/pipeline'
import { getFileExtension } from '../data-source/online-drive/file-list/list/utils'
type ChunkPreviewProps = {
dataSourceType: DatasourceType
@ -122,7 +123,7 @@ const ChunkPreview = ({
onlineDriveFiles.map(file => ({
id: file.key,
name: file.displayName,
extension: 'md',
extension: getFileExtension(previewOnlineDriveFile?.displayName),
}))
}
onChange={(selected) => {
@ -134,7 +135,7 @@ const ChunkPreview = ({
{
id: previewOnlineDriveFile?.key || '',
name: previewOnlineDriveFile?.displayName || '',
extension: 'md',
extension: getFileExtension(previewOnlineDriveFile?.displayName),
}
}
/>

16
web/app/components/rag-pipeline/components/panel/test-run/index.tsx
View File

@ -17,13 +17,21 @@ import Header from './header'
import FooterTips from './footer-tips'
import DataSourceProvider from '@/app/components/datasets/documents/create-from-pipeline/data-source/store/provider'
import { useDataSourceStore, useDataSourceStoreWithSelector } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store'
import { useShallow } from 'zustand/react/shallow'
const TestRunPanel = () => {
const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel)
const fileList = useDataSourceStoreWithSelector(state => state.localFileList)
const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments)
const websitePages = useDataSourceStoreWithSelector(state => state.websitePages)
const selectedFileKeys = useDataSourceStoreWithSelector(state => state.selectedFileKeys)
const {
localFileList: fileList,
onlineDocuments,
websitePages,
selectedFileKeys,
} = useDataSourceStoreWithSelector(useShallow(state => ({
localFileList: state.localFileList,
onlineDocuments: state.onlineDocuments,
websitePages: state.websitePages,
selectedFileKeys: state.selectedFileKeys,
})))
const dataSourceStore = useDataSourceStore()
const [datasource, setDatasource] = useState<Datasource>()

6
web/app/components/rag-pipeline/components/rag-pipeline-children.tsx
View File

@ -4,6 +4,7 @@ import {
} from 'react'
import { useStore } from '../../workflow/store'
import InputField from './input-field'
import PluginDependency from '../../workflow/plugin-dependency'
import RagPipelinePanel from './panel'
import RagPipelineHeader from './rag-pipeline-header'
import type { EnvironmentVariable } from '@/app/components/workflow/types'
@ -26,8 +27,8 @@ const RagPipelineChildren = () => {
handlePaneContextmenuCancel,
} = usePanelInteractions()
const {
exportCheck,
handleExportDSL,
exportCheck,
handleExportDSL,
} = useDSL()
eventEmitter?.useSubscription((v: any) => {
@ -37,6 +38,7 @@ const RagPipelineChildren = () => {
return (
<>
<PluginDependency />
{
showImportDSLModal && (
<UpdateDSLModal

2
web/app/components/workflow-app/components/workflow-children.tsx
View File

@ -6,6 +6,7 @@ import type { EnvironmentVariable } from '@/app/components/workflow/types'
import { DSL_EXPORT_CHECK } from '@/app/components/workflow/constants'
import { useStore } from '@/app/components/workflow/store'
import Features from '@/app/components/workflow/features'
import PluginDependency from '../../workflow/plugin-dependency'
import UpdateDSLModal from '@/app/components/workflow/update-dsl-modal'
import DSLExportConfirmModal from '@/app/components/workflow/dsl-export-confirm-modal'
import {
@ -37,6 +38,7 @@ const WorkflowChildren = () => {
return (
<>
<PluginDependency />
{
showFeaturesPanel && <Features />
}

28
web/app/components/workflow/nodes/_base/components/workflow-panel/index.tsx
View File

@ -120,6 +120,7 @@ const BasePanel: FC<BasePanelProps> = ({
return
if (workflowCanvasWidth - 400 <= nodePanelWidth + otherPanelWidth)
debounceUpdate(workflowCanvasWidth - 400 - otherPanelWidth)
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [nodePanelWidth, otherPanelWidth, workflowCanvasWidth, updateNodePanelWidth])
const { handleNodeSelect } = useNodesInteractions()
@ -151,11 +152,11 @@ const BasePanel: FC<BasePanelProps> = ({
const [isPaused, setIsPaused] = useState(false)
useEffect(() => {
if(data._singleRunningStatus === NodeRunningStatus.Running) {
if (data._singleRunningStatus === NodeRunningStatus.Running) {
hasClickRunning.current = true
setIsPaused(false)
}
else if(data._isSingleRun && data._singleRunningStatus === undefined && hasClickRunning) {
else if (data._isSingleRun && data._singleRunningStatus === undefined && hasClickRunning) {
setIsPaused(true)
hasClickRunning.current = false
}
@ -216,9 +217,9 @@ const BasePanel: FC<BasePanelProps> = ({
return {}
})()
if(logParams.showSpecialResultPanel) {
if (logParams.showSpecialResultPanel) {
return (
<div className={cn(
<div className={cn(
'relative mr-1 h-full',
)}>
<div
@ -269,10 +270,15 @@ const BasePanel: FC<BasePanelProps> = ({
}
return (
<div className={cn(
'relative mr-1 h-full',
showMessageLogModal && '!absolute -top-[5px] right-[416px] z-0 !mr-0 w-[384px] overflow-hidden rounded-2xl border-[0.5px] border-components-panel-border shadow-lg transition-all',
)}>
<div
className={cn(
'relative mr-1 h-full',
showMessageLogModal && 'absolute z-0 mr-2 w-[400px] overflow-hidden rounded-2xl border-[0.5px] border-components-panel-border shadow-lg transition-all',
)}
style={{
right: !showMessageLogModal ? '0' : `${otherPanelWidth}px`,
}}
>
<div
ref={triggerRef}
className='absolute -left-1 top-0 flex h-full w-1 cursor-col-resize resize-x items-center justify-center'>
@ -280,7 +286,7 @@ const BasePanel: FC<BasePanelProps> = ({
</div>
<div
ref={containerRef}
className={cn('flex h-full flex-col rounded-2xl border-[0.5px] border-components-panel-border bg-components-panel-bg shadow-lg', showSingleRunPanel ? 'overflow-hidden' : 'overflow-y-auto')}
className={cn('flex h-full flex-col rounded-2xl border-[0.5px] border-components-panel-border bg-components-panel-bg shadow-lg transition-[width] ease-linear', showSingleRunPanel ? 'overflow-hidden' : 'overflow-y-auto')}
style={{
width: `${nodePanelWidth}px`,
}}
@ -308,7 +314,7 @@ const BasePanel: FC<BasePanelProps> = ({
<div
className='mr-1 flex h-6 w-6 cursor-pointer items-center justify-center rounded-md hover:bg-state-base-hover'
onClick={() => {
if(isSingleRunning) {
if (isSingleRunning) {
handleNodeDataUpdate({
id,
data: {
@ -324,7 +330,7 @@ const BasePanel: FC<BasePanelProps> = ({
>
{
isSingleRunning ? <Stop className='h-4 w-4 text-text-tertiary' />
: <RiPlayLargeLine className='h-4 w-4 text-text-tertiary' />
: <RiPlayLargeLine className='h-4 w-4 text-text-tertiary' />
}
</div>
</Tooltip>