diff --git a/web/app/components/base/icons/assets/vender/line/mapsAndTravel/globe-01.svg b/web/app/components/base/icons/assets/vender/line/mapsAndTravel/globe-01.svg deleted file mode 100644 index 88898bb6e1..0000000000 --- a/web/app/components/base/icons/assets/vender/line/mapsAndTravel/globe-01.svg +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/web/app/components/base/icons/src/vender/line/mapsAndTravel/Globe01.json b/web/app/components/base/icons/src/vender/line/mapsAndTravel/Globe01.json deleted file mode 100644 index 9ccee71b02..0000000000 --- a/web/app/components/base/icons/src/vender/line/mapsAndTravel/Globe01.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "icon": { - "type": "element", - "isRootNode": true, - "name": "svg", - "attributes": { - "width": "16", - "height": "16", - "viewBox": "0 0 16 16", - "fill": "none", - "xmlns": "http://www.w3.org/2000/svg" - }, - "children": [ - { - "type": "element", - "name": "g", - "attributes": { - "id": "globe-01", - "clip-path": "url(#clip0_8902_1914)" - }, - "children": [ - { - "type": "element", - "name": "path", - "attributes": { - "id": "Icon", - "d": "M1.33325 7.99998H14.6666M1.33325 7.99998C1.33325 11.6819 4.31802 14.6666 7.99992 14.6666M1.33325 7.99998C1.33325 4.31808 4.31802 1.33331 7.99992 1.33331M14.6666 7.99998C14.6666 11.6819 11.6818 14.6666 7.99992 14.6666M14.6666 7.99998C14.6666 4.31808 11.6818 1.33331 7.99992 1.33331M7.99992 1.33331C9.66744 3.15888 10.6151 5.528 10.6666 7.99998C10.6151 10.472 9.66744 12.8411 7.99992 14.6666M7.99992 1.33331C6.3324 3.15888 5.38475 5.528 5.33325 7.99998C5.38475 10.472 6.3324 12.8411 7.99992 14.6666", - "stroke": "currentColor", - "stroke-width": "1.25", - "stroke-linecap": "round", - "stroke-linejoin": "round" - }, - "children": [] - } - ] - }, - { - "type": "element", - "name": "defs", - "attributes": {}, - "children": [ - { - "type": "element", - "name": "clipPath", - "attributes": { - "id": "clip0_8902_1914" - }, - "children": [ - { - "type": "element", - "name": "rect", - "attributes": { - "width": "16", - "height": "16", - "fill": "white" - }, - "children": [] - } - ] - } - ] - } - ] - }, - "name": "Globe01" -} diff --git a/web/app/components/base/icons/src/vender/line/mapsAndTravel/Globe01.tsx b/web/app/components/base/icons/src/vender/line/mapsAndTravel/Globe01.tsx deleted file mode 100644 index 445fde6304..0000000000 --- a/web/app/components/base/icons/src/vender/line/mapsAndTravel/Globe01.tsx +++ /dev/null @@ -1,20 +0,0 @@ -// GENERATE BY script -// DON NOT EDIT IT MANUALLY - -import * as React from 'react' -import data from './Globe01.json' -import IconBase from '@/app/components/base/icons/IconBase' -import type { IconData } from '@/app/components/base/icons/IconBase' - -const Icon = ( - { - ref, - ...props - }: React.SVGProps & { - ref?: React.RefObject>; - }, -) => - -Icon.displayName = 'Globe01' - -export default Icon diff --git a/web/app/components/base/icons/src/vender/line/mapsAndTravel/index.ts b/web/app/components/base/icons/src/vender/line/mapsAndTravel/index.ts index 8f3cf15258..82cb320ff2 100644 --- a/web/app/components/base/icons/src/vender/line/mapsAndTravel/index.ts +++ b/web/app/components/base/icons/src/vender/line/mapsAndTravel/index.ts @@ -1,2 +1 @@ -export { default as Globe01 } from './Globe01' export { default as Route } from './Route' diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index c9bb641287..75c31acd1b 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -209,10 +209,11 @@ const CreateFormPipeline = () => { } if (datasourceType === DatasourceType.onlineDrive) { const { bucket } = dataSourceStore.getState() - const { id, type } = previewOnlineDriveFileRef.current! + const { id, type, name } = previewOnlineDriveFileRef.current! datasourceInfoList.push({ bucket, id, + name, type, credential_id: currentCredentialId, }) @@ -292,6 +293,7 @@ const CreateFormPipeline = () => { datasourceInfoList.push({ bucket, id: file?.id, + name: file?.name, type: file?.type, credential_id: currentCredentialId, }) diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index ded538e007..ff45a8ea83 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -6,10 +6,10 @@ import { ArrowDownIcon } from '@heroicons/react/24/outline' import { pick, uniq } from 'lodash-es' import { RiEditLine, + RiGlobalLine, } from '@remixicon/react' import { useRouter } from 'next/navigation' import { useTranslation } from 'react-i18next' -import { Globe01 } from '../../base/icons/src/vender/line/mapsAndTravel' import ChunkingModeLabel from '../common/chunking-mode-label' import FileTypeIcon from '../../base/file-uploader/file-type-icon' import s from './style.module.css' @@ -23,6 +23,7 @@ import { asyncRunSafe } from '@/utils' import { formatNumber } from '@/utils/format' import NotionIcon from '@/app/components/base/notion-icon' import ProgressBar from '@/app/components/base/progress-bar' +import type { LegacyDataSourceInfo, LocalFileInfo, OnlineDocumentInfo, OnlineDriveInfo } from '@/models/datasets' import { ChunkingMode, DataSourceType, DocumentActionType, type SimpleDocumentDetail } from '@/models/datasets' import type { CommonResponse } from '@/models/common' import useTimestamp from '@/hooks/use-timestamp' @@ -36,6 +37,7 @@ import useBatchEditDocumentMetadata from '../metadata/hooks/use-batch-edit-docum import EditMetadataBatchModal from '@/app/components/datasets/metadata/edit-metadata-batch/modal' import StatusItem from './status-item' import Operations from './operations' +import { DatasourceType } from '@/models/pipeline' export const renderTdValue = (value: string | number | null, isEmptyStyle = false) => { return ( @@ -249,6 +251,46 @@ const DocumentList: FC = ({ } } + const getFileExtension = useCallback((fileName: string): string => { + if (!fileName) + return '' + const parts = fileName.split('.') + if (parts.length <= 1 || (parts[0] === '' && parts.length === 2)) + return '' + + return parts[parts.length - 1].toLowerCase() + }, []) + + const isCreateFromRAGPipeline = useCallback((createFrom: string) => { + return createFrom === 'rag-pipeline' + }, []) + + /** + * Calculate the data source type + * DataSourceType: FILE, NOTION, WEB (legacy) + * DatasourceType: localFile, onlineDocument, websiteCrawl, onlineDrive (new) + */ + const isLocalFile = useCallback((dataSourceType: DataSourceType | DatasourceType, createFrom: string) => { + if (createFrom === 'rag-pipeline') + return dataSourceType === DatasourceType.localFile + return dataSourceType === DataSourceType.FILE + }, []) + const isOnlineDocument = useCallback((dataSourceType: DataSourceType | DatasourceType, createFrom: string) => { + if (createFrom === 'rag-pipeline') + return dataSourceType === DatasourceType.onlineDocument + return dataSourceType === DataSourceType.NOTION + }, []) + const isWebsiteCrawl = useCallback((dataSourceType: DataSourceType | DatasourceType, createFrom: string) => { + if (createFrom === 'rag-pipeline') + return dataSourceType === DatasourceType.websiteCrawl + return dataSourceType === DataSourceType.WEB + }, []) + const isOnlineDrive = useCallback((dataSourceType: DataSourceType | DatasourceType, createFrom: string) => { + if (createFrom === 'rag-pipeline') + return dataSourceType === DatasourceType.onlineDrive + return false + }, []) + return (
@@ -287,7 +329,8 @@ const DocumentList: FC = ({ {localDocs.map((doc, index) => { - const isFile = doc.data_source_type === DataSourceType.FILE + const isFile = isLocalFile(doc.data_source_type, doc.created_from) + const createFromRAGPipeline = isCreateFromRAGPipeline(doc.created_from) const fileType = isFile ? doc.data_source_detail_dict?.upload_file?.extension : '' return = ({
-
- {doc?.data_source_type === DataSourceType.NOTION && } - {doc?.data_source_type === DataSourceType.FILE && } - {doc?.data_source_type === DataSourceType.WEB && } +
+ {isOnlineDocument(doc.data_source_type, doc.created_from) && ( + + )} + {isLocalFile(doc.data_source_type, doc.created_from) && ( + + )} + {isOnlineDrive(doc.data_source_type, doc.created_from) && ( + + )} + {isWebsiteCrawl(doc.data_source_type, doc.created_from) && ( + + )}
= ({ { - (['indexing', 'splitting', 'parsing', 'cleaning'].includes(doc.indexing_status) && doc?.data_source_type === DataSourceType.NOTION) + (['indexing', 'splitting', 'parsing', 'cleaning'].includes(doc.indexing_status) + && isOnlineDocument(doc.data_source_type, doc.created_from)) ? : } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx index 0f869c2621..81f6ed5271 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx @@ -109,6 +109,7 @@ const Preparation = () => { datasourceInfoList.push({ bucket, id: file?.id, + name: file?.name, type: file?.type, credential_id: credentialId, }) diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 8132acb83f..9c17980865 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -1,5 +1,5 @@ import type { DataSourceNotionPage, DataSourceProvider } from './common' -import type { AppIconType, AppMode, RetrievalConfig } from '@/types/app' +import type { AppIconType, AppMode, RetrievalConfig, TransferMethod } from '@/types/app' import type { Tag } from '@/app/components/base/tag-management/constant' import type { IndexingType } from '@/app/components/datasets/create/step-two' import type { MetadataFilteringVariableType } from '@/app/components/workflow/nodes/knowledge-retrieval/types' @@ -308,7 +308,7 @@ export const DisplayStatusList = [ export type DocumentDisplayStatus = typeof DisplayStatusList[number] -export type DataSourceInfo = { +export type LegacyDataSourceInfo = { upload_file: { id: string name: string @@ -327,6 +327,47 @@ export type DataSourceInfo = { credential_id?: string } +export type LocalFileInfo = { + extension: string + mime_type: string + name: string + related_id: string + size: number + transfer_method: TransferMethod + url: string +} + +export type WebsiteCrawlInfo = { + content: string + credential_id: string + description: string + source_url: string + title: string +} + +export type OnlineDocumentInfo = { + credential_id: string + workspace_id: string + page: { + last_edited_time: string + page_icon: DataSourceNotionPage['page_icon'] + page_id: string + page_name: string + parent_id: string + type: string + }, +} + +export type OnlineDriveInfo = { + bucket: string + credential_id: string + id: string + name: string + type: 'file' | 'folder' +} + +export type DataSourceInfo = LegacyDataSourceInfo | LocalFileInfo | OnlineDocumentInfo | WebsiteCrawlInfo + export type InitialDocumentDetail = { id: string batch: string @@ -350,7 +391,6 @@ export type InitialDocumentDetail = { export type SimpleDocumentDetail = InitialDocumentDetail & { enabled: boolean word_count: number - is_qa: boolean // TODO waiting for backend to add this field error?: string | null archived: boolean updated_at: number @@ -363,6 +403,7 @@ export type SimpleDocumentDetail = InitialDocumentDetail & { } } doc_metadata?: MetadataItemWithValue[] + created_from: string } export type DocumentListResponse = {