opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feat/rag-2' into feat/rag-2

This commit is contained in:
jyong 2025-07-31 15:21:18 +08:00
parent 97136ca8f0 195bf6621a
commit 9915364740
5 changed files with 110 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
web/app/components/base/notion-page-selector/base.tsx
View File

@ -6,9 +6,10 @@ import PageSelector from './page-selector'
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
import { useModalContextSelector } from '@/context/modal-context'
import NotionConnector from '../notion-connector'
import { usePreImportNotionPages } from '@/service/knowledge/use-import'
import { useInvalidPreImportNotionPages, usePreImportNotionPages } from '@/service/knowledge/use-import'
import Header from '../../datasets/create/website/base/header'
import type { DataSourceCredential } from '../../header/account-setting/data-source-page-new/types'
import Loading from '../loading'
type NotionPageSelectorProps = {
value?: string[]
@ -50,12 +51,16 @@ const NotionPageSelector = ({
setCurrentCredential(notionCredentials[0])
}, [notionCredentials])
const { data } = usePreImportNotionPages({ datasetId, credentialId: currentCredential.credentialId || '' })
const {
data: notionsPages,
isFetching: isFetchingNotionPages,
isError: isFetchingNotionPagesError,
} = usePreImportNotionPages({ datasetId, credentialId: currentCredential.credentialId || '' })
const getPagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
const pagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
const selectedPagesId = new Set<string>()
const boundPagesId = new Set<string>()
const notionWorkspaces = data?.notion_info || []
const notionWorkspaces = notionsPages?.notion_info || []
const pagesMap = notionWorkspaces.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
next.pages.forEach((page) => {
if (page.is_bound) {
@ -71,41 +76,47 @@ const NotionPageSelector = ({
return prev
}, {})
return [pagesMap, selectedPagesId, boundPagesId]
}, [data?.notion_info])
}, [notionsPages?.notion_info])
const defaultSelectedPagesId = [...Array.from(getPagesMapAndSelectedPagesId[1]), ...(value || [])]
const defaultSelectedPagesId = useMemo(() => {
return [...Array.from(pagesMapAndSelectedPagesId[1]), ...(value || [])]
}, [pagesMapAndSelectedPagesId, value])
const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
useEffect(() => {
setSelectedPagesId(new Set(defaultSelectedPagesId))
}, [data])
}, [defaultSelectedPagesId])
const handleSearchValueChange = useCallback((value: string) => {
setSearchValue(value)
}, [])
const invalidPreImportNotionPages = useInvalidPreImportNotionPages()
const handleSelectCredential = useCallback((credentialId: string) => {
const credential = notionCredentials.find(item => item.credentialId === credentialId)!
invalidPreImportNotionPages({ datasetId, credentialId: credential.credentialId })
setCurrentCredential(credential)
}, [])
onSelect([]) // Clear selected pages when changing credential
}, [onSelect])
const handleSelectPages = useCallback((newSelectedPagesId: Set<string>) => {
const selectedPages = Array.from(newSelectedPagesId).map(pageId => getPagesMapAndSelectedPagesId[0][pageId])
const selectedPages = Array.from(newSelectedPagesId).map(pageId => pagesMapAndSelectedPagesId[0][pageId])
setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
onSelect(selectedPages)
}, [getPagesMapAndSelectedPagesId, onSelect])
}, [pagesMapAndSelectedPagesId, onSelect])
const handlePreviewPage = useCallback((previewPageId: string) => {
if (onPreview)
onPreview(getPagesMapAndSelectedPagesId[0][previewPageId])
}, [getPagesMapAndSelectedPagesId, onSelect, onPreview])
onPreview(pagesMapAndSelectedPagesId[0][previewPageId])
}, [pagesMapAndSelectedPagesId, onPreview])
const handleConfigureNotion = useCallback(() => {
setShowAccountSettingModal({ payload: 'data-source' })
}, [setShowAccountSettingModal])
if (!data) {
if (isFetchingNotionPagesError) {
return (
<NotionConnector
onSetting={handleConfigureNotion}
@ -137,17 +148,23 @@ const NotionPageSelector = ({
/>
</div>
<div className='overflow-hidden rounded-b-xl'>
<PageSelector
value={selectedPagesId}
disabledValue={getPagesMapAndSelectedPagesId[2]}
searchValue={searchValue}
list={data.notion_info?.[0].pages || []}
pagesMap={getPagesMapAndSelectedPagesId[0]}
onSelect={handleSelectPages}
canPreview={canPreview}
previewPageId={previewPageId}
onPreview={handlePreviewPage}
/>
{isFetchingNotionPages ? (
<div className='flex h-[296px] items-center justify-center'>
<Loading />
</div>
) : (
<PageSelector
value={selectedPagesId}
disabledValue={pagesMapAndSelectedPagesId[2]}
searchValue={searchValue}
list={notionsPages!.notion_info?.[0].pages || []}
pagesMap={pagesMapAndSelectedPagesId[0]}
onSelect={handleSelectPages}
canPreview={canPreview}
previewPageId={previewPageId}
onPreview={handlePreviewPage}
/>
)}
</div>
</div>
</div>

55
web/app/components/base/notion-page-selector/notion-page-selector-modal/index.tsx
View File

@ -1,4 +1,4 @@
import { useState } from 'react'
import { useCallback, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { XMarkIcon } from '@heroicons/react/24/outline'
import NotionPageSelector from '../base'
@ -7,6 +7,9 @@ import type { NotionPage } from '@/models/common'
import cn from '@/utils/classnames'
import Modal from '@/app/components/base/modal'
import { noop } from 'lodash-es'
import { useGetDefaultDataSourceListAuth } from '@/service/use-datasource'
import NotionConnector from '../../notion-connector'
import { useModalContextSelector } from '@/context/modal-context'
type NotionPageSelectorModalProps = {
isShow: boolean
@ -21,17 +24,39 @@ const NotionPageSelectorModal = ({
datasetId,
}: NotionPageSelectorModalProps) => {
const { t } = useTranslation()
const setShowAccountSettingModal = useModalContextSelector(state => state.setShowAccountSettingModal)
const [selectedPages, setSelectedPages] = useState<NotionPage[]>([])
const handleClose = () => {
const { data: dataSourceList } = useGetDefaultDataSourceListAuth()
const handleClose = useCallback(() => {
onClose()
}
const handleSelectPage = (newSelectedPages: NotionPage[]) => {
}, [onClose])
const handleSelectPage = useCallback((newSelectedPages: NotionPage[]) => {
setSelectedPages(newSelectedPages)
}
const handleSave = () => {
}, [])
const handleSave = useCallback(() => {
onSave(selectedPages)
}
}, [onSave])
const handleOpenSetting = useCallback(() => {
setShowAccountSettingModal({ payload: 'data-source' })
}, [setShowAccountSettingModal])
const authedDataSourceList = dataSourceList?.result || []
const isNotionAuthed = useMemo(() => {
if (!authedDataSourceList) return false
const notionSource = authedDataSourceList.find(item => item.provider === 'notion_datasource')
if (!notionSource) return false
return notionSource.credentials_list.length > 0
}, [authedDataSourceList])
const notionCredentialList = useMemo(() => {
return authedDataSourceList.find(item => item.provider === 'notion_datasource')?.credentials_list || []
}, [authedDataSourceList])
return (
<Modal
@ -40,18 +65,22 @@ const NotionPageSelectorModal = ({
onClose={noop}
>
<div className='mb-6 flex h-8 items-center justify-between'>
<div className='text-xl font-semibold text-gray-900'>{t('common.dataSource.notion.selector.addPages')}</div>
<div className='text-xl font-semibold text-text-primary'>{t('common.dataSource.notion.selector.addPages')}</div>
<div
className='-mr-2 flex h-8 w-8 cursor-pointer items-center justify-center'
onClick={handleClose}>
<XMarkIcon className='h-4 w-4' />
</div>
</div>
<NotionPageSelector
onSelect={handleSelectPage}
canPreview={false}
datasetId={datasetId}
/>
{!isNotionAuthed && <NotionConnector onSetting={handleOpenSetting} />}
{isNotionAuthed && (
<NotionPageSelector
credentialList={notionCredentialList}
onSelect={handleSelectPage}
canPreview={false}
datasetId={datasetId}
/>
)}
<div className='mt-8 flex justify-end'>
<div className={s.operate} onClick={handleClose}>{t('common.operation.cancel')}</div>
<div className={cn(s.operate, s['operate-save'])} onClick={handleSave}>{t('common.operation.save')}</div>

10
web/app/components/base/notion-page-selector/page-selector/index.tsx
View File

@ -1,4 +1,4 @@
import { memo, useMemo, useState } from 'react'
import { memo, useEffect, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { FixedSizeList as List, areEqual } from 'react-window'
import type { ListChildComponentProps } from 'react-window'
@ -194,11 +194,10 @@ const PageSelector = ({
onPreview,
}: PageSelectorProps) => {
const { t } = useTranslation()
const [prevDataList, setPrevDataList] = useState(list)
const [dataList, setDataList] = useState<NotionPageItem[]>([])
const [localPreviewPageId, setLocalPreviewPageId] = useState('')
if (prevDataList !== list) {
setPrevDataList(list)
useEffect(() => {
setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => {
return {
...item,
@ -206,7 +205,8 @@ const PageSelector = ({
depth: 0,
}
}))
}
}, [list])
const searchDataList = list.filter((item) => {
return item.page_name.includes(searchValue)
}).map((item) => {

4
web/app/components/datasets/create/index.tsx
View File

@ -10,7 +10,7 @@ import { TopBar } from './top-bar'
import { DataSourceType } from '@/models/datasets'
import type { CrawlOptions, CrawlResultItem, FileItem, createDocumentResponse } from '@/models/datasets'
import { DataSourceProvider, type NotionPage } from '@/models/common'
import { useModalContext } from '@/context/modal-context'
import { useModalContextSelector } from '@/context/modal-context'
import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
import { useGetDefaultDataSourceListAuth } from '@/service/use-datasource'
import produce from 'immer'
@ -33,7 +33,7 @@ const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
const { t } = useTranslation()
const { setShowAccountSettingModal } = useModalContext()
const setShowAccountSettingModal = useModalContextSelector(state => state.setShowAccountSettingModal)
const datasetDetail = useDatasetDetailContextWithSelector(state => state.dataset)
const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)

24
web/service/knowledge/use-import.ts
View File

@ -1,18 +1,20 @@
import { useQuery } from '@tanstack/react-query'
import { useQuery, useQueryClient } from '@tanstack/react-query'
import { get } from '../base'
import type { DataSourceNotionWorkspace } from '@/models/common'
type PreImportNotionPagesParams = {
credentialId: string
datasetId: string
credentialId: string
}
const PRE_IMPORT_NOTION_PAGES_QUERY_KEY = 'notion-pre-import-pages'
export const usePreImportNotionPages = ({
credentialId,
datasetId,
credentialId,
}: PreImportNotionPagesParams) => {
return useQuery({
queryKey: ['notion-pre-import-pages', credentialId, datasetId],
queryKey: [PRE_IMPORT_NOTION_PAGES_QUERY_KEY, datasetId, credentialId],
queryFn: async () => {
return get<{ notion_info: DataSourceNotionWorkspace[] }>('/notion/pre-import/pages', {
params: {
@ -24,3 +26,17 @@ export const usePreImportNotionPages = ({
retry: 0,
})
}
export const useInvalidPreImportNotionPages = () => {
const queryClient = useQueryClient()
return ({
datasetId,
credentialId,
}: PreImportNotionPagesParams) => {
queryClient.invalidateQueries(
{
queryKey: [PRE_IMPORT_NOTION_PAGES_QUERY_KEY, datasetId, credentialId],
},
)
}
}