diff --git a/web/app/components/base/notion-page-selector/base.tsx b/web/app/components/base/notion-page-selector/base.tsx index 1f9ddeaebd..9315605cdf 100644 --- a/web/app/components/base/notion-page-selector/base.tsx +++ b/web/app/components/base/notion-page-selector/base.tsx @@ -21,6 +21,7 @@ type NotionPageSelectorProps = { datasetId?: string credentialList: DataSourceCredential[] onSelectCredential?: (credentialId: string) => void + supportBatchUpload?: boolean } const NotionPageSelector = ({ @@ -32,6 +33,7 @@ const NotionPageSelector = ({ datasetId = '', credentialList, onSelectCredential, + supportBatchUpload = false, }: NotionPageSelectorProps) => { const [searchValue, setSearchValue] = useState('') const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal) @@ -110,7 +112,7 @@ const NotionPageSelector = ({ setCurrentCredential(credential) onSelect([]) // Clear selected pages when changing credential onSelectCredential?.(credential.credentialId) - }, [invalidPreImportNotionPages, onSelect, onSelectCredential]) + }, [datasetId, invalidPreImportNotionPages, notionCredentials, onSelect, onSelectCredential]) const handleSelectPages = useCallback((newSelectedPagesId: Set) => { const selectedPages = Array.from(newSelectedPagesId).map(pageId => pagesMapAndSelectedPagesId[0][pageId]) @@ -175,6 +177,7 @@ const NotionPageSelector = ({ canPreview={canPreview} previewPageId={previewPageId} onPreview={handlePreviewPage} + isMultipleChoice={supportBatchUpload} /> )} diff --git a/web/app/components/base/notion-page-selector/credential-selector/index.tsx b/web/app/components/base/notion-page-selector/credential-selector/index.tsx index f0ec399544..360a38ba8f 100644 --- a/web/app/components/base/notion-page-selector/credential-selector/index.tsx +++ b/web/app/components/base/notion-page-selector/credential-selector/index.tsx @@ -1,9 +1,8 @@ 'use client' -import { useTranslation } from 'react-i18next' import React, { Fragment, useMemo } from 'react' import { Menu, MenuButton, MenuItem, MenuItems, Transition } from '@headlessui/react' import { RiArrowDownSLine } from '@remixicon/react' -import NotionIcon from '../../notion-icon' +import { CredentialIcon } from '@/app/components/datasets/common/credential-icon' export type NotionCredential = { credentialId: string @@ -23,14 +22,10 @@ const CredentialSelector = ({ items, onSelect, }: CredentialSelectorProps) => { - const { t } = useTranslation() const currentCredential = items.find(item => item.credentialId === value)! const getDisplayName = (item: NotionCredential) => { - return item.workspaceName || t('datasetPipeline.credentialSelector.name', { - credentialName: item.credentialName, - pluginName: 'Notion', - }) + return item.workspaceName || item.credentialName } const currentDisplayName = useMemo(() => { @@ -43,10 +38,11 @@ const CredentialSelector = ({ ({ open }) => ( <> -
onSelect(item.credentialId)} > -
@@ -18,6 +19,7 @@ type PageSelectorProps = { canPreview?: boolean previewPageId?: string onPreview?: (selectedPageId: string) => void + isMultipleChoice?: boolean } type NotionPageTreeItem = { children: Set @@ -80,6 +82,7 @@ const ItemComponent = ({ index, style, data }: ListChildComponentProps<{ searchValue: string previewPageId: string pagesMap: DataSourceNotionPageMap + isMultipleChoice?: boolean }>) => { const { t } = useTranslation() const { @@ -94,6 +97,7 @@ const ItemComponent = ({ index, style, data }: ListChildComponentProps<{ searchValue, previewPageId, pagesMap, + isMultipleChoice, } = data const current = dataList[index] const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[current.page_id] @@ -134,16 +138,24 @@ const ItemComponent = ({ index, style, data }: ListChildComponentProps<{ previewPageId === current.page_id && 'bg-state-base-hover')} style={{ ...style, top: style.top as number + 8, left: 8, right: 8, width: 'calc(100% - 16px)' }} > - { - if (disabled) - return - handleCheck(index) - }} - /> + {isMultipleChoice ? ( + { + handleCheck(index) + }} + />) : ( + { + handleCheck(index) + }} + /> + )} {!searchValue && renderArrow()} { const { t } = useTranslation() const [dataList, setDataList] = useState([]) @@ -265,7 +278,7 @@ const PageSelector = ({ const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[pageId] if (copyValue.has(pageId)) { - if (!searchValue) { + if (!searchValue && isMultipleChoice) { for (const item of currentWithChildrenAndDescendants.descendants) copyValue.delete(item) } @@ -273,12 +286,18 @@ const PageSelector = ({ copyValue.delete(pageId) } else { - if (!searchValue) { + if (!searchValue && isMultipleChoice) { for (const item of currentWithChildrenAndDescendants.descendants) copyValue.add(item) } - - copyValue.add(pageId) + // Single choice mode, clear previous selection + if (!isMultipleChoice && copyValue.size > 0) { + copyValue.clear() + copyValue.add(pageId) + } + else { + copyValue.add(pageId) + } } onSelect(new Set(copyValue)) @@ -322,6 +341,7 @@ const PageSelector = ({ searchValue, previewPageId: currentPreviewPageId, pagesMap, + isMultipleChoice, }} > {Item} diff --git a/web/app/components/datasets/common/credential-icon.tsx b/web/app/components/datasets/common/credential-icon.tsx index 5a25963f3b..d4e6fd69ac 100644 --- a/web/app/components/datasets/common/credential-icon.tsx +++ b/web/app/components/datasets/common/credential-icon.tsx @@ -2,7 +2,7 @@ import cn from '@/utils/classnames' import React, { useCallback, useMemo, useState } from 'react' type CredentialIconProps = { - avatar_url?: string + avatarUrl?: string name: string size?: number className?: string @@ -16,12 +16,12 @@ const ICON_BG_COLORS = [ ] export const CredentialIcon: React.FC = ({ - avatar_url, + avatarUrl, name, size = 20, className = '', }) => { - const [showAvatar, setShowAvatar] = useState(!!avatar_url && avatar_url !== 'default') + const [showAvatar, setShowAvatar] = useState(!!avatarUrl && avatarUrl !== 'default') const firstLetter = useMemo(() => name.charAt(0).toUpperCase(), [name]) const bgColor = useMemo(() => ICON_BG_COLORS[firstLetter.charCodeAt(0) % ICON_BG_COLORS.length], [firstLetter]) @@ -29,17 +29,20 @@ export const CredentialIcon: React.FC = ({ setShowAvatar(false) }, []) - if (avatar_url && avatar_url !== 'default' && showAvatar) { + if (avatarUrl && avatarUrl !== 'default' && showAvatar) { return (
diff --git a/web/app/components/datasets/create/file-uploader/index.tsx b/web/app/components/datasets/create/file-uploader/index.tsx index abe2564ad2..700a5f7680 100644 --- a/web/app/components/datasets/create/file-uploader/index.tsx +++ b/web/app/components/datasets/create/file-uploader/index.tsx @@ -25,7 +25,7 @@ type IFileUploaderProps = { onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void onFileListUpdate?: (files: FileItem[]) => void onPreview: (file: File) => void - notSupportBatchUpload?: boolean + supportBatchUpload?: boolean } const FileUploader = ({ @@ -35,7 +35,7 @@ const FileUploader = ({ onFileUpdate, onFileListUpdate, onPreview, - notSupportBatchUpload, + supportBatchUpload = false, }: IFileUploaderProps) => { const { t } = useTranslation() const { notify } = useContext(ToastContext) @@ -44,7 +44,7 @@ const FileUploader = ({ const dropRef = useRef(null) const dragRef = useRef(null) const fileUploader = useRef(null) - const hideUpload = notSupportBatchUpload && fileList.length > 0 + const hideUpload = !supportBatchUpload && fileList.length > 0 const { data: fileUploadConfigResponse } = useFileUploadConfig() const { data: supportFileTypesResponse } = useFileSupportTypes() @@ -68,9 +68,9 @@ const FileUploader = ({ const ACCEPTS = supportTypes.map((ext: string) => `.${ext}`) const fileUploadConfig = useMemo(() => ({ file_size_limit: fileUploadConfigResponse?.file_size_limit ?? 15, - batch_count_limit: fileUploadConfigResponse?.batch_count_limit ?? 5, - file_upload_limit: fileUploadConfigResponse?.file_upload_limit ?? 5, - }), [fileUploadConfigResponse]) + batch_count_limit: supportBatchUpload ? (fileUploadConfigResponse?.batch_count_limit ?? 5) : 1, + file_upload_limit: supportBatchUpload ? (fileUploadConfigResponse?.file_upload_limit ?? 5) : 1, + }), [fileUploadConfigResponse, supportBatchUpload]) const fileListRef = useRef([]) @@ -254,12 +254,12 @@ const FileUploader = ({ }), ) let files = nested.flat() - if (notSupportBatchUpload) files = files.slice(0, 1) + if (!supportBatchUpload) files = files.slice(0, 1) files = files.slice(0, fileUploadConfig.batch_count_limit) const valid = files.filter(isValid) initialUpload(valid) }, - [initialUpload, isValid, notSupportBatchUpload, traverseFileEntry, fileUploadConfig], + [initialUpload, isValid, supportBatchUpload, traverseFileEntry, fileUploadConfig], ) const selectHandle = () => { if (fileUploader.current) @@ -303,7 +303,7 @@ const FileUploader = ({ id="fileUploader" className="hidden" type="file" - multiple={!notSupportBatchUpload} + multiple={supportBatchUpload} accept={ACCEPTS.join(',')} onChange={fileChangeHandle} /> @@ -317,7 +317,7 @@ const FileUploader = ({ - {notSupportBatchUpload ? t('datasetCreation.stepOne.uploader.buttonSingleFile') : t('datasetCreation.stepOne.uploader.button')} + {supportBatchUpload ? t('datasetCreation.stepOne.uploader.button') : t('datasetCreation.stepOne.uploader.buttonSingleFile')} {supportTypes.length > 0 && ( )} @@ -326,7 +326,7 @@ const FileUploader = ({
{t('datasetCreation.stepOne.uploader.tip', { size: fileUploadConfig.file_size_limit, supportTypes: supportTypesShowNames, - batchCount: notSupportBatchUpload ? 1 : fileUploadConfig.batch_count_limit, + batchCount: fileUploadConfig.batch_count_limit, totalCount: fileUploadConfig.file_upload_limit, })}
{dragging &&
} diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx index cab1637661..f2768be470 100644 --- a/web/app/components/datasets/create/step-one/index.tsx +++ b/web/app/components/datasets/create/step-one/index.tsx @@ -110,7 +110,7 @@ const StepOne = ({ const hasNotin = notionPages.length > 0 const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace const isShowVectorSpaceFull = (allFileLoaded || hasNotin) && isVectorSpaceFull && enableBilling - const notSupportBatchUpload = enableBilling && plan.type === 'sandbox' + const supportBatchUpload = !enableBilling || plan.type !== 'sandbox' const nextDisabled = useMemo(() => { if (!files.length) return true @@ -229,7 +229,7 @@ const StepOne = ({ onFileListUpdate={updateFileList} onFileUpdate={updateFile} onPreview={updateCurrentFile} - notSupportBatchUpload={notSupportBatchUpload} + supportBatchUpload={supportBatchUpload} /> {isShowVectorSpaceFull && (
@@ -259,6 +259,7 @@ const StepOne = ({ credentialList={notionCredentialList} onSelectCredential={updateNotionCredentialId} datasetId={datasetId} + supportBatchUpload={supportBatchUpload} />
{isShowVectorSpaceFull && ( diff --git a/web/app/components/datasets/create/website/base/crawled-result-item.tsx b/web/app/components/datasets/create/website/base/crawled-result-item.tsx index 8ea316f62a..51e043c35a 100644 --- a/web/app/components/datasets/create/website/base/crawled-result-item.tsx +++ b/web/app/components/datasets/create/website/base/crawled-result-item.tsx @@ -6,6 +6,7 @@ import cn from '@/utils/classnames' import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets' import Checkbox from '@/app/components/base/checkbox' import Button from '@/app/components/base/button' +import Radio from '@/app/components/base/radio/ui' type Props = { payload: CrawlResultItemType @@ -13,6 +14,7 @@ type Props = { isPreview: boolean onCheckChange: (checked: boolean) => void onPreview: () => void + isMultipleChoice: boolean } const CrawledResultItem: FC = ({ @@ -21,6 +23,7 @@ const CrawledResultItem: FC = ({ isChecked, onCheckChange, onPreview, + isMultipleChoice, }) => { const { t } = useTranslation() @@ -31,7 +34,21 @@ const CrawledResultItem: FC = ({
- + { + isMultipleChoice ? ( + + ) : ( + + ) + }
void onPreview: (payload: CrawlResultItem) => void usedTime: number + isMultipleChoice: boolean } const CrawledResult: FC = ({ @@ -25,6 +26,7 @@ const CrawledResult: FC = ({ onSelectedChange, onPreview, usedTime, + isMultipleChoice, }) => { const { t } = useTranslation() @@ -40,13 +42,17 @@ const CrawledResult: FC = ({ const handleItemCheckChange = useCallback((item: CrawlResultItem) => { return (checked: boolean) => { - if (checked) - onSelectedChange([...checkedList, item]) - - else + if (checked) { + if (isMultipleChoice) + onSelectedChange([...checkedList, item]) + else + onSelectedChange([item]) + } + else { onSelectedChange(checkedList.filter(checkedItem => checkedItem.source_url !== item.source_url)) + } } - }, [checkedList, onSelectedChange]) + }, [checkedList, isMultipleChoice, onSelectedChange]) const [previewIndex, setPreviewIndex] = React.useState(-1) const handlePreview = useCallback((index: number) => { @@ -59,11 +65,13 @@ const CrawledResult: FC = ({ return (
- + {isMultipleChoice && ( + + )}
{t(`${I18N_PREFIX}.scrapTimeInfo`, { total: list.length, @@ -80,6 +88,7 @@ const CrawledResult: FC = ({ payload={item} isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)} onCheckChange={handleItemCheckChange(item)} + isMultipleChoice={isMultipleChoice} /> ))}
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 51c2c7d505..1ef934308a 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -26,6 +26,7 @@ type Props = { onJobIdChange: (jobId: string) => void crawlOptions: CrawlOptions onCrawlOptionsChange: (payload: CrawlOptions) => void + supportBatchUpload: boolean } enum Step { @@ -41,6 +42,7 @@ const FireCrawl: FC = ({ onJobIdChange, crawlOptions, onCrawlOptionsChange, + supportBatchUpload, }) => { const { t } = useTranslation() const [step, setStep] = useState(Step.init) @@ -171,7 +173,7 @@ const FireCrawl: FC = ({ content: item.markdown, })) setCrawlResult(data) - onCheckedCrawlResultChange(data.data || []) // default select the crawl result + onCheckedCrawlResultChange(supportBatchUpload ? (data.data || []) : (data.data?.slice(0, 1) || [])) // default select the crawl result setCrawlErrorMessage('') } } @@ -182,7 +184,7 @@ const FireCrawl: FC = ({ finally { setStep(Step.finished) } - }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished, onCheckedCrawlResultChange]) + }, [checkValid, crawlOptions, onJobIdChange, waitForCrawlFinished, t, onCheckedCrawlResultChange, supportBatchUpload]) return (
@@ -221,6 +223,7 @@ const FireCrawl: FC = ({ onSelectedChange={onCheckedCrawlResultChange} onPreview={onPreview} usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0} + isMultipleChoice={supportBatchUpload} /> }
diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx index ee7ace6815..15324f642e 100644 --- a/web/app/components/datasets/create/website/index.tsx +++ b/web/app/components/datasets/create/website/index.tsx @@ -24,6 +24,7 @@ type Props = { crawlOptions: CrawlOptions onCrawlOptionsChange: (payload: CrawlOptions) => void authedDataSourceList: DataSourceAuth[] + supportBatchUpload?: boolean } const Website: FC = ({ @@ -35,6 +36,7 @@ const Website: FC = ({ crawlOptions, onCrawlOptionsChange, authedDataSourceList, + supportBatchUpload = false, }) => { const { t } = useTranslation() const { setShowAccountSettingModal } = useModalContext() @@ -116,6 +118,7 @@ const Website: FC = ({ onJobIdChange={onJobIdChange} crawlOptions={crawlOptions} onCrawlOptionsChange={onCrawlOptionsChange} + supportBatchUpload={supportBatchUpload} /> )} {source && selectedProvider === DataSourceProvider.waterCrawl && ( @@ -126,6 +129,7 @@ const Website: FC = ({ onJobIdChange={onJobIdChange} crawlOptions={crawlOptions} onCrawlOptionsChange={onCrawlOptionsChange} + supportBatchUpload={supportBatchUpload} /> )} {source && selectedProvider === DataSourceProvider.jinaReader && ( @@ -136,6 +140,7 @@ const Website: FC = ({ onJobIdChange={onJobIdChange} crawlOptions={crawlOptions} onCrawlOptionsChange={onCrawlOptionsChange} + supportBatchUpload={supportBatchUpload} /> )} {!source && ( diff --git a/web/app/components/datasets/create/website/jina-reader/index.tsx b/web/app/components/datasets/create/website/jina-reader/index.tsx index b6e6177af2..b2189b3e5c 100644 --- a/web/app/components/datasets/create/website/jina-reader/index.tsx +++ b/web/app/components/datasets/create/website/jina-reader/index.tsx @@ -26,6 +26,7 @@ type Props = { onJobIdChange: (jobId: string) => void crawlOptions: CrawlOptions onCrawlOptionsChange: (payload: CrawlOptions) => void + supportBatchUpload: boolean } enum Step { @@ -41,6 +42,7 @@ const JinaReader: FC = ({ onJobIdChange, crawlOptions, onCrawlOptionsChange, + supportBatchUpload, }) => { const { t } = useTranslation() const [step, setStep] = useState(Step.init) @@ -157,7 +159,7 @@ const JinaReader: FC = ({ total: 1, data: [{ title, - content, + markdown: content, description, source_url: url, }], @@ -176,7 +178,7 @@ const JinaReader: FC = ({ } else { setCrawlResult(data) - onCheckedCrawlResultChange(data.data || []) // default select the crawl result + onCheckedCrawlResultChange(supportBatchUpload ? (data.data || []) : (data.data?.slice(0, 1) || [])) // default select the crawl result setCrawlErrorMessage('') } } @@ -188,7 +190,7 @@ const JinaReader: FC = ({ finally { setStep(Step.finished) } - }, [checkValid, crawlOptions, onCheckedCrawlResultChange, onJobIdChange, t, waitForCrawlFinished]) + }, [checkValid, crawlOptions, onCheckedCrawlResultChange, onJobIdChange, supportBatchUpload, t, waitForCrawlFinished]) return (
@@ -227,6 +229,7 @@ const JinaReader: FC = ({ onSelectedChange={onCheckedCrawlResultChange} onPreview={onPreview} usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0} + isMultipleChoice={supportBatchUpload} /> }
diff --git a/web/app/components/datasets/create/website/preview.tsx b/web/app/components/datasets/create/website/preview.tsx index d148c87196..f43dc83589 100644 --- a/web/app/components/datasets/create/website/preview.tsx +++ b/web/app/components/datasets/create/website/preview.tsx @@ -32,7 +32,7 @@ const WebsitePreview = ({
{payload.source_url}
-
{payload.content}
+
{payload.markdown}
) diff --git a/web/app/components/datasets/create/website/watercrawl/index.tsx b/web/app/components/datasets/create/website/watercrawl/index.tsx index 67a3e53feb..bf0048b788 100644 --- a/web/app/components/datasets/create/website/watercrawl/index.tsx +++ b/web/app/components/datasets/create/website/watercrawl/index.tsx @@ -26,6 +26,7 @@ type Props = { onJobIdChange: (jobId: string) => void crawlOptions: CrawlOptions onCrawlOptionsChange: (payload: CrawlOptions) => void + supportBatchUpload: boolean } enum Step { @@ -41,6 +42,7 @@ const WaterCrawl: FC = ({ onJobIdChange, crawlOptions, onCrawlOptionsChange, + supportBatchUpload, }) => { const { t } = useTranslation() const [step, setStep] = useState(Step.init) @@ -132,7 +134,7 @@ const WaterCrawl: FC = ({ }, } } - }, [crawlOptions.limit]) + }, [crawlOptions.limit, onCheckedCrawlResultChange]) const handleRun = useCallback(async (url: string) => { const { isValid, errorMsg } = checkValid(url) @@ -163,7 +165,7 @@ const WaterCrawl: FC = ({ } else { setCrawlResult(data) - onCheckedCrawlResultChange(data.data || []) // default select the crawl result + onCheckedCrawlResultChange(supportBatchUpload ? (data.data || []) : (data.data?.slice(0, 1) || [])) // default select the crawl result setCrawlErrorMessage('') } } @@ -174,7 +176,7 @@ const WaterCrawl: FC = ({ finally { setStep(Step.finished) } - }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished]) + }, [checkValid, crawlOptions, onCheckedCrawlResultChange, onJobIdChange, supportBatchUpload, t, waitForCrawlFinished]) return (
@@ -213,6 +215,7 @@ const WaterCrawl: FC = ({ onSelectedChange={onCheckedCrawlResultChange} onPreview={onPreview} usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0} + isMultipleChoice={supportBatchUpload} /> }
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/index.tsx index 0de3879969..0e588e4e1d 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/index.tsx @@ -10,14 +10,12 @@ import Trigger from './trigger' import List from './list' export type CredentialSelectorProps = { - pluginName: string currentCredentialId: string onCredentialChange: (credentialId: string) => void credentials: Array } const CredentialSelector = ({ - pluginName, currentCredentialId, onCredentialChange, credentials, @@ -50,7 +48,6 @@ const CredentialSelector = ({ @@ -58,7 +55,6 @@ const CredentialSelector = ({ diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/item.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/item.tsx index ab8de51fb1..9c8368e299 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/item.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/item.tsx @@ -2,22 +2,18 @@ import { CredentialIcon } from '@/app/components/datasets/common/credential-icon import type { DataSourceCredential } from '@/types/pipeline' import { RiCheckLine } from '@remixicon/react' import React, { useCallback } from 'react' -import { useTranslation } from 'react-i18next' type ItemProps = { credential: DataSourceCredential - pluginName: string isSelected: boolean onCredentialChange: (credentialId: string) => void } const Item = ({ credential, - pluginName, isSelected, onCredentialChange, }: ItemProps) => { - const { t } = useTranslation() const { avatar_url, name } = credential const handleCredentialChange = useCallback(() => { @@ -30,15 +26,12 @@ const Item = ({ onClick={handleCredentialChange} > - {t('datasetPipeline.credentialSelector.name', { - credentialName: name, - pluginName, - })} + {name} { isSelected && ( diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/list.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/list.tsx index b161a80309..cdcb2b5af5 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/list.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/list.tsx @@ -5,14 +5,12 @@ import Item from './item' type ListProps = { currentCredentialId: string credentials: Array - pluginName: string onCredentialChange: (credentialId: string) => void } const List = ({ currentCredentialId, credentials, - pluginName, onCredentialChange, }: ListProps) => { return ( @@ -24,7 +22,6 @@ const List = ({ diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/trigger.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/trigger.tsx index 88f47384f3..dc328ef87f 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/trigger.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/base/credential-selector/trigger.tsx @@ -1,23 +1,18 @@ import React from 'react' import type { DataSourceCredential } from '@/types/pipeline' -import { useTranslation } from 'react-i18next' import { RiArrowDownSLine } from '@remixicon/react' import cn from '@/utils/classnames' import { CredentialIcon } from '@/app/components/datasets/common/credential-icon' type TriggerProps = { currentCredential: DataSourceCredential | undefined - pluginName: string isOpen: boolean } const Trigger = ({ currentCredential, - pluginName, isOpen, }: TriggerProps) => { - const { t } = useTranslation() - const { avatar_url, name = '', @@ -31,16 +26,13 @@ const Trigger = ({ )} >
- {t('datasetPipeline.credentialSelector.name', { - credentialName: name, - pluginName, - })} + {name}
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/base/header.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/base/header.tsx index ef8932ba24..b826e53d93 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/base/header.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/base/header.tsx @@ -11,12 +11,14 @@ type HeaderProps = { docTitle: string docLink: string onClickConfiguration?: () => void + pluginName: string } & CredentialSelectorProps const Header = ({ docTitle, docLink, onClickConfiguration, + pluginName, ...rest }: HeaderProps) => { const { t } = useTranslation() @@ -29,7 +31,7 @@ const Header = ({ />
) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/base/crawled-result-item.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/base/crawled-result-item.tsx index 753b32c396..bdfcddfd77 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/base/crawled-result-item.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/base/crawled-result-item.tsx @@ -46,6 +46,7 @@ const CrawledResultItem = ({ /> ) : ( diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx index 648f6a5d93..513ac8edd9 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx @@ -33,14 +33,16 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website' export type WebsiteCrawlProps = { nodeId: string nodeData: DataSourceNodeType - isInPipeline?: boolean onCredentialChange: (credentialId: string) => void + isInPipeline?: boolean + supportBatchUpload?: boolean } const WebsiteCrawl = ({ nodeId, nodeData, isInPipeline = false, + supportBatchUpload = false, onCredentialChange, }: WebsiteCrawlProps) => { const { t } = useTranslation() @@ -122,7 +124,7 @@ const WebsiteCrawl = ({ time_consuming: time_consuming ?? 0, } setCrawlResult(crawlResultData) - handleCheckedCrawlResultChange(isInPipeline ? [crawlData[0]] : crawlData) // default select the crawl result + handleCheckedCrawlResultChange(supportBatchUpload ? crawlData : crawlData.slice(0, 1)) // default select the crawl result setCrawlErrorMessage('') setStep(CrawlStep.finished) }, @@ -132,7 +134,7 @@ const WebsiteCrawl = ({ }, }, ) - }, [dataSourceStore, datasourceNodeRunURL, handleCheckedCrawlResultChange, isInPipeline, t]) + }, [dataSourceStore, datasourceNodeRunURL, handleCheckedCrawlResultChange, supportBatchUpload, t]) const handleSubmit = useCallback((value: Record) => { handleRun(value) @@ -149,7 +151,7 @@ const WebsiteCrawl = ({ setTotalNum(0) setCrawlErrorMessage('') onCredentialChange(credentialId) - }, [dataSourceStore, onCredentialChange]) + }, [onCredentialChange]) return (
@@ -195,7 +197,7 @@ const WebsiteCrawl = ({ previewIndex={previewIndex} onPreview={handlePreview} showPreview={!isInPipeline} - isMultipleChoice={!isInPipeline} // only support single choice in test run + isMultipleChoice={supportBatchUpload} // only support single choice in test run /> )}
diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index 77b77700ca..1d9232403a 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -102,7 +102,7 @@ const CreateFormPipeline = () => { return onlineDriveFileList.length > 0 && isVectorSpaceFull && enableBilling return false }, [allFileLoaded, datasource, datasourceType, enableBilling, isVectorSpaceFull, onlineDocuments.length, onlineDriveFileList.length, websitePages.length]) - const notSupportBatchUpload = enableBilling && plan.type === 'sandbox' + const supportBatchUpload = !enableBilling || plan.type !== 'sandbox' const nextBtnDisabled = useMemo(() => { if (!datasource) return true @@ -125,15 +125,16 @@ const CreateFormPipeline = () => { const showSelect = useMemo(() => { if (datasourceType === DatasourceType.onlineDocument) { const pagesCount = currentWorkspace?.pages.length ?? 0 - return pagesCount > 0 + return supportBatchUpload && pagesCount > 0 } if (datasourceType === DatasourceType.onlineDrive) { const isBucketList = onlineDriveFileList.some(file => file.type === 'bucket') - return !isBucketList && onlineDriveFileList.filter((item) => { + return supportBatchUpload && !isBucketList && onlineDriveFileList.filter((item) => { return item.type !== 'bucket' }).length > 0 } - }, [currentWorkspace?.pages.length, datasourceType, onlineDriveFileList]) + return false + }, [currentWorkspace?.pages.length, datasourceType, supportBatchUpload, onlineDriveFileList]) const totalOptions = useMemo(() => { if (datasourceType === DatasourceType.onlineDocument) @@ -395,7 +396,7 @@ const CreateFormPipeline = () => { clearWebsiteCrawlData() else if (dataSource.nodeData.provider_type === DatasourceType.onlineDrive) clearOnlineDriveData() - }, []) + }, [clearOnlineDocumentData, clearOnlineDriveData, clearWebsiteCrawlData]) const handleSwitchDataSource = useCallback((dataSource: Datasource) => { const { @@ -406,13 +407,13 @@ const CreateFormPipeline = () => { setCurrentCredentialId('') currentNodeIdRef.current = dataSource.nodeId setDatasource(dataSource) - }, [dataSourceStore]) + }, [clearDataSourceData, dataSourceStore]) const handleCredentialChange = useCallback((credentialId: string) => { const { setCurrentCredentialId } = dataSourceStore.getState() clearDataSourceData(datasource!) setCurrentCredentialId(credentialId) - }, [dataSourceStore, datasource]) + }, [clearDataSourceData, dataSourceStore, datasource]) if (isFetchingPipelineInfo) { return ( @@ -443,7 +444,7 @@ const CreateFormPipeline = () => { {datasourceType === DatasourceType.localFile && ( )} {datasourceType === DatasourceType.onlineDocument && ( @@ -451,6 +452,7 @@ const CreateFormPipeline = () => { nodeId={datasource!.nodeId} nodeData={datasource!.nodeData} onCredentialChange={handleCredentialChange} + supportBatchUpload={supportBatchUpload} /> )} {datasourceType === DatasourceType.websiteCrawl && ( @@ -458,6 +460,7 @@ const CreateFormPipeline = () => { nodeId={datasource!.nodeId} nodeData={datasource!.nodeData} onCredentialChange={handleCredentialChange} + supportBatchUpload={supportBatchUpload} /> )} {datasourceType === DatasourceType.onlineDrive && ( @@ -465,6 +468,7 @@ const CreateFormPipeline = () => { nodeId={datasource!.nodeId} nodeData={datasource!.nodeData} onCredentialChange={handleCredentialChange} + supportBatchUpload={supportBatchUpload} /> )} {isShowVectorSpaceFull && ( diff --git a/web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx b/web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx index bae4deb86e..ce7a5da24c 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx @@ -27,7 +27,7 @@ const WebsitePreview = ({ {currentWebsite.source_url} · · - {`${formatNumberAbbreviated(currentWebsite.content.length)} ${t('datasetPipeline.addDocuments.characters')}`} + {`${formatNumberAbbreviated(currentWebsite.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`}
- {currentWebsite.content} + {currentWebsite.markdown}
) diff --git a/web/app/components/datasets/documents/detail/settings/document-settings.tsx b/web/app/components/datasets/documents/detail/settings/document-settings.tsx index 3bcb8ef3aa..16c90c925f 100644 --- a/web/app/components/datasets/documents/detail/settings/document-settings.tsx +++ b/web/app/components/datasets/documents/detail/settings/document-settings.tsx @@ -113,7 +113,7 @@ const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => { return [{ title: websiteInfo.title, source_url: websiteInfo.source_url, - content: websiteInfo.content, + markdown: websiteInfo.content, description: websiteInfo.description, }] }, [websiteInfo]) diff --git a/web/app/components/datasets/documents/detail/settings/pipeline-settings/index.tsx b/web/app/components/datasets/documents/detail/settings/pipeline-settings/index.tsx index 1ab47be445..0381222415 100644 --- a/web/app/components/datasets/documents/detail/settings/pipeline-settings/index.tsx +++ b/web/app/components/datasets/documents/detail/settings/pipeline-settings/index.tsx @@ -55,7 +55,7 @@ const PipelineSettings = ({ if (lastRunData?.datasource_type === DatasourceType.websiteCrawl) { const { content, description, source_url, title } = lastRunData.datasource_info websitePages.push({ - content, + markdown: content, description, source_url, title, @@ -135,7 +135,7 @@ const PipelineSettings = ({ push(`/datasets/${datasetId}/documents`) }, }) - }, [datasetId, invalidDocumentDetail, invalidDocumentList, lastRunData, pipelineId, push, runPublishedPipeline]) + }, [datasetId, documentId, invalidDocumentDetail, invalidDocumentList, lastRunData, pipelineId, push, runPublishedPipeline]) const onClickProcess = useCallback(() => { isPreview.current = false diff --git a/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx index eb73599314..c659d8669a 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx @@ -131,7 +131,7 @@ const Preparation = () => { clearWebsiteCrawlData() else if (dataSource.nodeData.provider_type === DatasourceType.onlineDrive) clearOnlineDriveData() - }, []) + }, [clearOnlineDocumentData, clearOnlineDriveData, clearWebsiteCrawlData]) const handleSwitchDataSource = useCallback((dataSource: Datasource) => { const { @@ -142,13 +142,13 @@ const Preparation = () => { setCurrentCredentialId('') currentNodeIdRef.current = dataSource.nodeId setDatasource(dataSource) - }, [dataSourceStore]) + }, [clearDataSourceData, dataSourceStore]) const handleCredentialChange = useCallback((credentialId: string) => { const { setCurrentCredentialId } = dataSourceStore.getState() clearDataSourceData(datasource!) setCurrentCredentialId(credentialId) - }, [dataSourceStore, datasource]) + }, [clearDataSourceData, dataSourceStore, datasource]) return ( <> @@ -164,7 +164,7 @@ const Preparation = () => { {datasourceType === DatasourceType.localFile && ( )} {datasourceType === DatasourceType.onlineDocument && ( @@ -173,6 +173,7 @@ const Preparation = () => { nodeData={datasource!.nodeData} isInPipeline onCredentialChange={handleCredentialChange} + supportBatchUpload={false} /> )} {datasourceType === DatasourceType.websiteCrawl && ( @@ -181,6 +182,7 @@ const Preparation = () => { nodeData={datasource!.nodeData} isInPipeline onCredentialChange={handleCredentialChange} + supportBatchUpload={false} /> )} {datasourceType === DatasourceType.onlineDrive && ( @@ -189,6 +191,7 @@ const Preparation = () => { nodeData={datasource!.nodeData} isInPipeline onCredentialChange={handleCredentialChange} + supportBatchUpload={false} /> )}
diff --git a/web/app/components/workflow/nodes/data-source/before-run-form.tsx b/web/app/components/workflow/nodes/data-source/before-run-form.tsx index 764599b4cb..521fdfb087 100644 --- a/web/app/components/workflow/nodes/data-source/before-run-form.tsx +++ b/web/app/components/workflow/nodes/data-source/before-run-form.tsx @@ -43,13 +43,13 @@ const BeforeRunForm: FC = (props) => { clearWebsiteCrawlData() else if (datasourceType === DatasourceType.onlineDrive) clearOnlineDriveData() - }, [datasourceType]) + }, [clearOnlineDocumentData, clearOnlineDriveData, clearWebsiteCrawlData, datasourceType]) const handleCredentialChange = useCallback((credentialId: string) => { const { setCurrentCredentialId } = dataSourceStore.getState() clearDataSourceData() setCurrentCredentialId(credentialId) - }, [dataSourceStore]) + }, [clearDataSourceData, dataSourceStore]) return ( = (props) => { {datasourceType === DatasourceType.localFile && ( )} {datasourceType === DatasourceType.onlineDocument && ( @@ -69,6 +69,7 @@ const BeforeRunForm: FC = (props) => { nodeData={datasourceNodeData} isInPipeline onCredentialChange={handleCredentialChange} + supportBatchUpload={false} /> )} {datasourceType === DatasourceType.websiteCrawl && ( @@ -77,6 +78,7 @@ const BeforeRunForm: FC = (props) => { nodeData={datasourceNodeData} isInPipeline onCredentialChange={handleCredentialChange} + supportBatchUpload={false} /> )} {datasourceType === DatasourceType.onlineDrive && ( @@ -85,6 +87,7 @@ const BeforeRunForm: FC = (props) => { nodeData={datasourceNodeData} isInPipeline onCredentialChange={handleCredentialChange} + supportBatchUpload={false} /> )}
diff --git a/web/i18n/en-US/dataset-pipeline.ts b/web/i18n/en-US/dataset-pipeline.ts index c83d358eec..29237e844a 100644 --- a/web/i18n/en-US/dataset-pipeline.ts +++ b/web/i18n/en-US/dataset-pipeline.ts @@ -145,9 +145,6 @@ const translation = { emptySearchResult: 'No items were found', resetKeywords: 'Reset keywords', }, - credentialSelector: { - name: '{{credentialName}}\'s {{pluginName}}', - }, configurationTip: 'Configure {{pluginName}}', conversion: { title: 'Convert to Knowledge Pipeline', diff --git a/web/i18n/ja-JP/dataset-pipeline.ts b/web/i18n/ja-JP/dataset-pipeline.ts index 0dddb25356..5091c17807 100644 --- a/web/i18n/ja-JP/dataset-pipeline.ts +++ b/web/i18n/ja-JP/dataset-pipeline.ts @@ -137,9 +137,6 @@ const translation = { emptySearchResult: 'アイテムは見つかりませんでした', resetKeywords: 'キーワードをリセットする', }, - credentialSelector: { - name: '{{credentialName}}の{{pluginName}}', - }, configurationTip: '{{pluginName}}を設定', conversion: { confirm: { diff --git a/web/i18n/zh-Hans/dataset-pipeline.ts b/web/i18n/zh-Hans/dataset-pipeline.ts index 7fbe8a0532..0e23d7a1e0 100644 --- a/web/i18n/zh-Hans/dataset-pipeline.ts +++ b/web/i18n/zh-Hans/dataset-pipeline.ts @@ -145,9 +145,6 @@ const translation = { emptySearchResult: '未找到任何项目', resetKeywords: '重置关键词', }, - credentialSelector: { - name: '{{credentialName}} 的 {{pluginName}}', - }, configurationTip: '配置 {{pluginName}}', conversion: { title: '转换为知识流水线', diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 574897a9b4..fe4c568e46 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -156,7 +156,7 @@ export type CrawlOptions = { export type CrawlResultItem = { title: string - content: string + markdown: string description: string source_url: string }