opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feat/rag-2' into feat/rag-2

This commit is contained in:
jyong 2025-08-01 13:59:19 +08:00
parent 79f30daf03 e7e531dc06
commit 383ee368e6
17 changed files with 177 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
web/app/components/base/notion-page-selector/base.tsx
View File

@ -19,6 +19,7 @@ type NotionPageSelectorProps = {
onPreview?: (selectedPage: NotionPage) => void
datasetId?: string
credentialList: DataSourceCredential[]
onSelectCredential?: (credentialId: string) => void
}
const NotionPageSelector = ({
@ -29,10 +30,13 @@ const NotionPageSelector = ({
onPreview,
datasetId = '',
credentialList,
onSelectCredential,
}: NotionPageSelectorProps) => {
const [searchValue, setSearchValue] = useState('')
const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
const invalidPreImportNotionPages = useInvalidPreImportNotionPages()
const notionCredentials = useMemo((): NotionCredential[] => {
return credentialList.map((item) => {
return {
@ -47,8 +51,16 @@ const NotionPageSelector = ({
useEffect(() => {
const credential = notionCredentials.find(item => item.credentialId === currentCredential?.credentialId)
if (!credential)
if (!credential) {
const firstCredential = notionCredentials[0]
invalidPreImportNotionPages({ datasetId, credentialId: firstCredential.credentialId })
setCurrentCredential(notionCredentials[0])
onSelect([]) // Clear selected pages when changing credential
onSelectCredential?.(firstCredential.credentialId)
}
else {
onSelectCredential?.(credential?.credentialId || '')
}
}, [notionCredentials])
const {
@ -91,14 +103,13 @@ const NotionPageSelector = ({
setSearchValue(value)
}, [])
const invalidPreImportNotionPages = useInvalidPreImportNotionPages()
const handleSelectCredential = useCallback((credentialId: string) => {
const credential = notionCredentials.find(item => item.credentialId === credentialId)!
invalidPreImportNotionPages({ datasetId, credentialId: credential.credentialId })
setCurrentCredential(credential)
onSelect([]) // Clear selected pages when changing credential
}, [onSelect])
onSelectCredential?.(credential.credentialId)
}, [invalidPreImportNotionPages, onSelect, onSelectCredential])
const handleSelectPages = useCallback((newSelectedPagesId: Set<string>) => {
const selectedPages = Array.from(newSelectedPagesId).map(pageId => pagesMapAndSelectedPagesId[0][pageId])

20
web/app/components/datasets/create/index.tsx
View File

@ -44,15 +44,26 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
const [fileList, setFiles] = useState<FileItem[]>([])
const [result, setResult] = useState<createDocumentResponse | undefined>()
const [notionPages, setNotionPages] = useState<NotionPage[]>([])
const [notionCredentialId, setNotionCredentialId] = useState<string>('')
const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
const [crawlOptions, setCrawlOptions] = useState<CrawlOptions>(DEFAULT_CRAWL_OPTIONS)
const [websiteCrawlProvider, setWebsiteCrawlProvider] = useState<DataSourceProvider>(DataSourceProvider.fireCrawl)
const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
const {
data: dataSourceList,
isLoading: isLoadingAuthedDataSourceList,
isError: fetchingAuthedDataSourceListError,
} = useGetDefaultDataSourceListAuth()
const updateNotionPages = useCallback((value: NotionPage[]) => {
setNotionPages(value)
}, [])
const updateNotionCredentialId = useCallback((credentialId: string) => {
setNotionCredentialId(credentialId)
}, [])
const updateFileList = useCallback((preparedFiles: FileItem[]) => {
setFiles(preparedFiles)
}, [])
@ -88,12 +99,6 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
setStep(step + delta)
}, [step, setStep])
const {
data: dataSourceList,
isLoading: isLoadingAuthedDataSourceList,
isError: fetchingAuthedDataSourceListError,
} = useGetDefaultDataSourceListAuth()
if (fetchingAuthedDataSourceListError)
return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
@ -121,7 +126,9 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
updateFile={updateFile}
updateFileList={updateFileList}
notionPages={notionPages}
notionCredentialId={notionCredentialId}
updateNotionPages={updateNotionPages}
updateNotionCredentialId={updateNotionCredentialId}
onStepChange={nextStep}
websitePages={websitePages}
updateWebsitePages={setWebsitePages}
@ -140,6 +147,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
dataSourceType={dataSourceType}
files={fileList.map(file => file.file)}
notionPages={notionPages}
notionCredentialId={notionCredentialId}
websitePages={websitePages}
websiteCrawlProvider={websiteCrawlProvider}
websiteCrawlJobId={websiteCrawlJobId}

4
web/app/components/datasets/create/notion-page-preview/index.tsx
View File

@ -11,11 +11,13 @@ import { fetchNotionPagePreview } from '@/service/datasets'
type IProps = {
currentPage?: NotionPage
notionCredentialId: string
hidePreview: () => void
}
const NotionPagePreview = ({
currentPage,
notionCredentialId,
hidePreview,
}: IProps) => {
const { t } = useTranslation()
@ -29,7 +31,7 @@ const NotionPagePreview = ({
const res = await fetchNotionPagePreview({
workspaceID: currentPage.workspace_id,
pageID: currentPage.page_id,
pageType: currentPage.type,
credentialID: notionCredentialId,
})
setPreviewContent(res.content)
setLoading(false)

13
web/app/components/datasets/create/step-one/index.tsx
View File

@ -32,7 +32,9 @@ type IStepOneProps = {
updateFileList: (files: FileItem[]) => void
updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void
notionPages?: NotionPage[]
notionCredentialId: string
updateNotionPages: (value: NotionPage[]) => void
updateNotionCredentialId: (credentialId: string) => void
onStepChange: () => void
changeType: (type: DataSourceType) => void
websitePages?: CrawlResultItem[]
@ -55,7 +57,9 @@ const StepOne = ({
updateFileList,
updateFile,
notionPages = [],
notionCredentialId,
updateNotionPages,
updateNotionCredentialId,
websitePages = [],
updateWebsitePages,
onWebsiteCrawlProviderChange,
@ -253,6 +257,7 @@ const StepOne = ({
onSelect={updateNotionPages}
onPreview={updateCurrentPage}
credentialList={notionCredentialList}
onSelectCredential={updateNotionCredentialId}
datasetId={datasetId}
/>
</div>
@ -317,7 +322,13 @@ const StepOne = ({
</div>
<div className='h-full w-1/2 overflow-y-auto'>
{currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
{currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
{currentNotionPage && (
<NotionPagePreview
currentPage={currentNotionPage}
hidePreview={hideNotionPagePreview}
notionCredentialId={notionCredentialId}
/>
)}
{currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
</div>
</div>

3
web/app/components/datasets/create/step-two/index.tsx
View File

@ -79,6 +79,7 @@ type StepTwoProps = {
dataSourceType: DataSourceType
files: CustomFile[]
notionPages?: NotionPage[]
notionCredentialId: string
websitePages?: CrawlResultItem[]
crawlOptions?: CrawlOptions
websiteCrawlProvider?: DataSourceProvider
@ -134,6 +135,7 @@ const StepTwo = ({
dataSourceType: inCreatePageDataSourceType,
files,
notionPages = [],
notionCredentialId,
websitePages = [],
crawlOptions,
websiteCrawlProvider = DataSourceProvider.fireCrawl,
@ -282,6 +284,7 @@ const StepTwo = ({
indexingTechnique: getIndexing_technique() as any,
processRule: getProcessRule(),
dataset_id: datasetId || '',
credential_id: notionCredentialId,
})
const websiteIndexingEstimateQuery = useFetchFileIndexingEstimateForWeb({

32
web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx
View File

@ -1,9 +1,9 @@
'use client'
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import React, { useCallback, useEffect, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import type { CrawlResultItem } from '@/models/datasets'
import { CrawlStep } from '@/models/datasets'
import Header from '@/app/components/datasets/create/website/base/header'
import Header from '../base/header'
import Options from './base/options'
import Crawling from './base/crawling'
import ErrorMessage from './base/error-message'
@ -24,6 +24,8 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-so
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
import { useShallow } from 'zustand/react/shallow'
import { useModalContextSelector } from '@/context/modal-context'
import { CredentialTypeEnum } from '@/app/components/plugins/plugin-auth'
import { noop } from 'lodash-es'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
@ -157,14 +159,6 @@ const WebsiteCrawl = ({
handleRun(value)
}, [handleRun])
const headerInfo = useMemo(() => {
return {
title: nodeData.title,
docTitle: 'How to use?',
docLink: 'https://docs.dify.ai',
}
}, [nodeData])
const handleSetting = useCallback(() => {
setShowAccountSettingModal({
payload: 'data-source',
@ -174,9 +168,23 @@ const WebsiteCrawl = ({
return (
<div className='flex flex-col'>
<Header
isInPipeline
// todo: delete mock data
docTitle='How to use?'
docLink='https://docs.dify.ai'
onClickConfiguration={handleSetting}
{...headerInfo}
pluginName={nodeData.datasource_label}
currentCredentialId={'12345678'}
onCredentialChange={noop}
credentials={[{
avatar_url: 'https://cloud.dify.ai/logo/logo.svg',
credential: {
credentials: '......',
},
id: '12345678',
is_default: true,
name: 'test123',
type: CredentialTypeEnum.API_KEY,
}]}
/>
<div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle'>
<Options

2
web/app/components/rag-pipeline/components/input-field/footer-tip.tsx
View File

@ -3,7 +3,7 @@ import { RiDragDropLine } from '@remixicon/react'
const FooterTip = () => {
return (
<div className='flex items-center justify-center gap-x-2 py-4 text-text-quaternary'>
<div className='flex shrink-0 items-center justify-center gap-x-2 py-4 text-text-quaternary'>
<RiDragDropLine className='size-4' />
<span className='system-xs-regular'>Drag to adjust grouping</span>
</div>

136
web/app/components/rag-pipeline/components/input-field/index.tsx
View File

@ -109,80 +109,80 @@ const InputFieldDialog = ({
<DialogWrapper
show={!!showInputFieldDialog}
onClose={closePanel}
outerWrapperClassName='overflow-y-auto'
panelWrapperClassName='justify-start'
className='grow rounded-l-2xl border-y-[0.5px] border-l-[0.5px]'
>
<div className='flex grow flex-col'>
<div className='flex items-center p-4 pb-0'>
<div className='system-xl-semibold grow'>
{t('datasetPipeline.inputFieldPanel.title')}
</div>
<Button
variant={'ghost'}
size='small'
className={cn(
'shrink-0 gap-x-px px-1.5',
previewPanelOpen && 'bg-state-accent-active text-text-accent',
)}
onClick={togglePreviewPanel}
>
<RiEyeLine className='size-3.5' />
<span className='px-[3px]'>{t('datasetPipeline.operations.preview')}</span>
</Button>
<Divider type='vertical' className='mx-1 h-3' />
<button
type='button'
className='flex size-6 shrink-0 items-center justify-center p-0.5'
onClick={closePanel}
>
<RiCloseLine className='size-4 text-text-tertiary' />
</button>
<div className='flex shrink-0 items-center p-4 pb-0'>
<div className='system-xl-semibold grow'>
{t('datasetPipeline.inputFieldPanel.title')}
</div>
<div className='system-sm-regular px-4 pb-2 pt-1 text-text-tertiary'>
{t('datasetPipeline.inputFieldPanel.description')}
</div>
<div className='flex grow flex-col overflow-y-auto'>
{/* Unique Inputs for Each Entrance */}
<div className='flex h-6 items-center gap-x-0.5 px-4 pt-2'>
<span className='system-sm-semibold-uppercase text-text-secondary'>
{t('datasetPipeline.inputFieldPanel.uniqueInputs.title')}
</span>
<Tooltip
popupContent={t('datasetPipeline.inputFieldPanel.uniqueInputs.tooltip')}
popupClassName='max-w-[240px]'
/>
</div>
<div className='flex flex-col gap-y-1 py-1'>
{
Object.keys(datasourceNodeDataMap).map((key) => {
const inputFields = inputFieldsMap.current[key] || []
return (
<FieldList
key={key}
nodeId={key}
LabelRightContent={<Datasource nodeData={datasourceNodeDataMap[key]} />}
inputFields={inputFields}
readonly={readonly}
labelClassName='pt-1 pb-1'
handleInputFieldsChange={updateInputFields}
allVariableNames={allVariableNames}
/>
)
})
}
</div>
{/* Global Inputs */}
<FieldList
nodeId='shared'
LabelRightContent={<GlobalInputs />}
inputFields={inputFieldsMap.current.shared || []}
readonly={readonly}
labelClassName='pt-2 pb-1'
handleInputFieldsChange={updateInputFields}
allVariableNames={allVariableNames}
<Button
variant={'ghost'}
size='small'
className={cn(
'shrink-0 gap-x-px px-1.5',
previewPanelOpen && 'bg-state-accent-active text-text-accent',
)}
onClick={togglePreviewPanel}
>
<RiEyeLine className='size-3.5' />
<span className='px-[3px]'>{t('datasetPipeline.operations.preview')}</span>
</Button>
<Divider type='vertical' className='mx-1 h-3' />
<button
type='button'
className='flex size-6 shrink-0 items-center justify-center p-0.5'
onClick={closePanel}
>
<RiCloseLine className='size-4 text-text-tertiary' />
</button>
</div>
<div className='system-sm-regular shrink-0 px-4 pb-2 pt-1 text-text-tertiary'>
{t('datasetPipeline.inputFieldPanel.description')}
</div>
<div className='flex grow flex-col overflow-hidden'>
{/* Unique Inputs for Each Entrance */}
<div className='flex h-6 items-center gap-x-0.5 px-4 pt-2'>
<span className='system-sm-semibold-uppercase text-text-secondary'>
{t('datasetPipeline.inputFieldPanel.uniqueInputs.title')}
</span>
<Tooltip
popupContent={t('datasetPipeline.inputFieldPanel.uniqueInputs.tooltip')}
popupClassName='max-w-[240px]'
/>
</div>
<FooterTip />
<div className='flex flex-col gap-y-1 py-1'>
{
Object.keys(datasourceNodeDataMap).map((key) => {
const inputFields = inputFieldsMap.current[key] || []
return (
<FieldList
key={key}
nodeId={key}
LabelRightContent={<Datasource nodeData={datasourceNodeDataMap[key]} />}
inputFields={inputFields}
readonly={readonly}
labelClassName='pt-1 pb-1'
handleInputFieldsChange={updateInputFields}
allVariableNames={allVariableNames}
/>
)
})
}
</div>
{/* Global Inputs */}
<FieldList
nodeId='shared'
LabelRightContent={<GlobalInputs />}
inputFields={inputFieldsMap.current.shared || []}
readonly={readonly}
labelClassName='pt-2 pb-1'
handleInputFieldsChange={updateInputFields}
allVariableNames={allVariableNames}
/>
</div>
<FooterTip />
</DialogWrapper>
{previewPanelOpen && (
<PreviewPanel

1
web/app/components/rag-pipeline/components/input-field/preview/index.tsx
View File

@ -24,6 +24,7 @@ const PreviewPanel = ({
<DialogWrapper
show={show}
onClose={onClose}
outerWrapperClassName='overflow-y-auto'
panelWrapperClassName='pr-[424px] justify-start'
className='w-[480px] grow rounded-2xl border-[0.5px] bg-components-panel-bg'
>

6
web/app/components/workflow/nodes/_base/components/form-input-item.tsx
View File

@ -31,6 +31,8 @@ type Props = {
inPanel?: boolean
currentTool?: Tool
currentProvider?: ToolWithProvider
showManageInputField?: boolean
onManageInputField?: () => void
}
const FormInputItem: FC<Props> = ({
@ -42,6 +44,8 @@ const FormInputItem: FC<Props> = ({
inPanel,
currentTool,
currentProvider,
showManageInputField,
onManageInputField,
}) => {
const language = useLanguage()
@ -192,6 +196,8 @@ const FormInputItem: FC<Props> = ({
onChange={handleValueChange}
nodesOutputVars={availableVars}
availableNodes={availableNodesWithParent}
showManageInputField={showManageInputField}
onManageInputField={onManageInputField}
/>
)}
{isNumber && isConstant && (

4
web/app/components/workflow/nodes/_base/components/prompt/editor.tsx
View File

@ -146,6 +146,8 @@ const Editor: FC<Props> = ({
}
const getVarType = useWorkflowVariableType()
const pipelineId = useStore(s => s.pipelineId)
const setShowInputFieldDialog = useStore(s => s.setShowInputFieldDialog)
return (
<Wrap className={cn(className, wrapClassName)} style={wrapStyle} isInNode isExpand={isExpand}>
@ -270,6 +272,8 @@ const Editor: FC<Props> = ({
}
return acc
}, {} as any),
showManageInputField: !!pipelineId,
onManageInputField: () => setShowInputFieldDialog?.(true),
}}
onChange={onChange}
onBlur={setBlur}

6
web/app/components/workflow/nodes/tool/components/mixed-variable-text-input/index.tsx
View File

@ -17,6 +17,8 @@ type MixedVariableTextInputProps = {
availableNodes?: Node[]
value?: string
onChange?: (text: string) => void
showManageInputField?: boolean
onManageInputField?: () => void
}
const MixedVariableTextInput = ({
readOnly = false,
@ -24,6 +26,8 @@ const MixedVariableTextInput = ({
availableNodes = [],
value = '',
onChange,
showManageInputField,
onManageInputField,
}: MixedVariableTextInputProps) => {
const { t } = useTranslation()
return (
@ -52,6 +56,8 @@ const MixedVariableTextInput = ({
}
return acc
}, {} as any),
showManageInputField,
onManageInputField,
}}
placeholder={<Placeholder />}
onChange={onChange}

6
web/app/components/workflow/nodes/tool/components/tool-form/index.tsx
View File

@ -16,6 +16,8 @@ type Props = {
inPanel?: boolean
currentTool?: Tool
currentProvider?: ToolWithProvider
showManageInputField?: boolean
onManageInputField?: () => void
}
const ToolForm: FC<Props> = ({
@ -27,6 +29,8 @@ const ToolForm: FC<Props> = ({
inPanel,
currentTool,
currentProvider,
showManageInputField,
onManageInputField,
}) => {
return (
<div className='space-y-1'>
@ -42,6 +46,8 @@ const ToolForm: FC<Props> = ({
inPanel={inPanel}
currentTool={currentTool}
currentProvider={currentProvider}
showManageInputField={showManageInputField}
onManageInputField={onManageInputField}
/>
))
}

6
web/app/components/workflow/nodes/tool/components/tool-form/item.tsx
View File

@ -24,6 +24,8 @@ type Props = {
inPanel?: boolean
currentTool?: Tool
currentProvider?: ToolWithProvider
showManageInputField?: boolean
onManageInputField?: () => void
}
const ToolFormItem: FC<Props> = ({
@ -35,6 +37,8 @@ const ToolFormItem: FC<Props> = ({
inPanel,
currentTool,
currentProvider,
showManageInputField,
onManageInputField,
}) => {
const language = useLanguage()
const { name, label, type, required, tooltip, input_schema } = schema
@ -89,6 +93,8 @@ const ToolFormItem: FC<Props> = ({
inPanel={inPanel}
currentTool={currentTool}
currentProvider={currentProvider}
showManageInputField={showManageInputField}
onManageInputField={onManageInputField}
/>
{isShowSchema && (

5
web/app/components/workflow/nodes/tool/panel.tsx
View File

@ -11,6 +11,7 @@ import Loading from '@/app/components/base/loading'
import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars'
import StructureOutputItem from '@/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/show'
import { Type } from '../llm/types'
import { useStore } from '@/app/components/workflow/store'
const i18nPrefix = 'workflow.nodes.tool'
@ -37,6 +38,8 @@ const Panel: FC<NodePanelProps<ToolNodeType>> = ({
} = useConfig(id, data)
const [collapsed, setCollapsed] = React.useState(false)
const pipelineId = useStore(s => s.pipelineId)
const setShowInputFieldDialog = useStore(s => s.setShowInputFieldDialog)
if (isLoading) {
return <div className='flex h-[200px] items-center justify-center'>
@ -61,6 +64,8 @@ const Panel: FC<NodePanelProps<ToolNodeType>> = ({
onChange={setInputVar}
currentProvider={currCollection}
currentTool={currTool}
showManageInputField={!!pipelineId}
onManageInputField={() => setShowInputFieldDialog?.(true)}
/>
</Field>
)}

8
web/service/datasets.ts
View File

@ -183,8 +183,12 @@ export const fetchFileIndexingEstimate: Fetcher<FileIndexingEstimateResponse, In
return post<FileIndexingEstimateResponse>('/datasets/indexing-estimate', { body })
}
export const fetchNotionPagePreview: Fetcher<{ content: string }, { workspaceID: string; pageID: string; pageType: string }> = ({ workspaceID, pageID, pageType }) => {
return get<{ content: string }>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`)
export const fetchNotionPagePreview: Fetcher<{ content: string }, { workspaceID: string; pageID: string; credentialID: string; }> = ({ workspaceID, pageID, credentialID }) => {
return get<{ content: string }>(`notion/workspaces/${workspaceID}/pages/${pageID}/preview`, {
params: {
credential_id: credentialID,
},
})
}
export const fetchApiKeysList: Fetcher<ApiKeysListResponse, { url: string; params: Record<string, any> }> = ({ url, params }) => {

1
web/service/knowledge/use-create-dataset.ts
View File

@ -119,6 +119,7 @@ export const useFetchFileIndexingEstimateForFile = (
type GetFileIndexingEstimateParamsOptionNotion = GetFileIndexingEstimateParamsOptionBase & {
dataSourceType: DataSourceType.NOTION
notionPages: NotionPage[]
credential_id: string
}
const getFileIndexingEstimateParamsForNotion = ({