{!datasetId && (
<>
-
-
{t('datasetCreation.stepThree.creationTitle')}
-
{t('datasetCreation.stepThree.creationContent')}
-
{t('datasetCreation.stepThree.label')}
-
{datasetName || creationCache?.dataset?.name}
+
+
{t('datasetCreation.stepThree.creationTitle')}
+
{t('datasetCreation.stepThree.creationContent')}
+
+
+
+
{t('datasetCreation.stepThree.label')}
+
{datasetName || creationCache?.dataset?.name}
+
+
-
+
)}
{datasetId && (
-
-
{t('datasetCreation.stepThree.additionTitle')}
-
{`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`}
+
+
{t('datasetCreation.stepThree.additionTitle')}
+
{`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`}
)}
- {!isMobile &&
-
-
-
{t('datasetCreation.stepThree.sideTipTitle')}
-
{t('datasetCreation.stepThree.sideTipContent')}
+ {!isMobile && (
+
+
+
+
+
+
{t('datasetCreation.stepThree.sideTipTitle')}
+
{t('datasetCreation.stepThree.sideTipContent')}
+
-
}
+ )}
)
}
diff --git a/web/app/components/datasets/create/step-two/index.module.css b/web/app/components/datasets/create/step-two/index.module.css
index f89d6d67ea..178cbeba85 100644
--- a/web/app/components/datasets/create/step-two/index.module.css
+++ b/web/app/components/datasets/create/step-two/index.module.css
@@ -13,18 +13,6 @@
z-index: 10;
}
-.form {
- @apply px-16 pb-8;
-}
-
-.form .label {
- @apply pt-6 pb-2 flex items-center;
- font-weight: 500;
- font-size: 16px;
- line-height: 24px;
- color: #344054;
-}
-
.segmentationItem {
min-height: 68px;
}
@@ -75,6 +63,10 @@
cursor: pointer;
}
+.disabled {
+ cursor: not-allowed !important;
+}
+
.indexItem.disabled:hover {
background-color: #fcfcfd;
border-color: #f2f4f7;
@@ -87,8 +79,7 @@
}
.radioItem {
- @apply relative mb-2 rounded-xl border border-gray-100 cursor-pointer;
- background-color: #fcfcfd;
+ @apply relative mb-2 rounded-xl border border-components-option-card-option-border cursor-pointer bg-components-option-card-option-bg;
}
.radioItem.segmentationItem.custom {
@@ -146,7 +137,7 @@
}
.typeIcon.economical {
- background-image: url(../assets/piggy-bank-01.svg);
+ background-image: url(../assets/piggy-bank-mod.svg);
}
.radioItem .radio {
@@ -247,7 +238,7 @@
}
.ruleItem {
- @apply flex items-center;
+ @apply flex items-center py-1.5;
}
.formFooter {
@@ -394,19 +385,6 @@
max-width: 524px;
}
-.previewHeader {
- position: sticky;
- top: 0;
- left: 0;
- padding-top: 42px;
- background-color: #fff;
- font-weight: 600;
- font-size: 18px;
- line-height: 28px;
- color: #101828;
- z-index: 10;
-}
-
/*
* `fixed` must under `previewHeader` because of style override would not work
*/
@@ -432,4 +410,4 @@
font-size: 12px;
line-height: 18px;
}
-}
\ No newline at end of file
+}
diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx
index af7ec2136f..302544e152 100644
--- a/web/app/components/datasets/create/step-two/index.tsx
+++ b/web/app/components/datasets/create/step-two/index.tsx
@@ -1,65 +1,79 @@
'use client'
-import React, { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
+import type { FC, PropsWithChildren } from 'react'
+import React, { useCallback, useEffect, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
-import { useBoolean } from 'ahooks'
-import { XMarkIcon } from '@heroicons/react/20/solid'
-import { RocketLaunchIcon } from '@heroicons/react/24/outline'
import {
- RiCloseLine,
+ RiAlertFill,
+ RiArrowLeftLine,
+ RiSearchEyeLine,
} from '@remixicon/react'
import Link from 'next/link'
-import { groupBy } from 'lodash-es'
-import PreviewItem, { PreviewType } from './preview-item'
-import LanguageSelect from './language-select'
+import Image from 'next/image'
+import { useHover } from 'ahooks'
+import SettingCog from '../assets/setting-gear-mod.svg'
+import OrangeEffect from '../assets/option-card-effect-orange.svg'
+import FamilyMod from '../assets/family-mod.svg'
+import Note from '../assets/note-mod.svg'
+import FileList from '../assets/file-list-3-fill.svg'
+import { indexMethodIcon } from '../icons'
+import { PreviewContainer } from '../../preview/container'
+import { ChunkContainer, QAPreview } from '../../chunk'
+import { PreviewHeader } from '../../preview/header'
+import { FormattedText } from '../../formatted-text/formatted'
+import { PreviewSlice } from '../../formatted-text/flavours/preview-slice'
+import PreviewDocumentPicker from '../../common/document-picker/preview-document-picker'
import s from './index.module.css'
import unescape from './unescape'
import escape from './escape'
+import { OptionCard } from './option-card'
+import LanguageSelect from './language-select'
+import { DelimiterInput, MaxLengthInput, OverlapInput } from './inputs'
import cn from '@/utils/classnames'
-import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
-import {
- createDocument,
- createFirstDocument,
- fetchFileIndexingEstimate as didFetchFileIndexingEstimate,
- fetchDefaultProcessRule,
-} from '@/service/datasets'
+import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
+
import Button from '@/app/components/base/button'
-import Input from '@/app/components/base/input'
-import Loading from '@/app/components/base/loading'
import FloatRightContainer from '@/app/components/base/float-right-container'
import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
import type { RetrievalConfig } from '@/types/app'
import { ensureRerankModelSelected, isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
import Toast from '@/app/components/base/toast'
-import { formatNumber } from '@/utils/format'
import type { NotionPage } from '@/models/common'
import { DataSourceProvider } from '@/models/common'
-import { DataSourceType, DocForm } from '@/models/datasets'
-import NotionIcon from '@/app/components/base/notion-icon'
-import Switch from '@/app/components/base/switch'
-import { MessageChatSquare } from '@/app/components/base/icons/src/public/common'
+import { ChuckingMode, DataSourceType } from '@/models/datasets'
import { useDatasetDetailContext } from '@/context/dataset-detail'
import I18n from '@/context/i18n'
-import { IS_CE_EDITION } from '@/config'
import { RETRIEVE_METHOD } from '@/types/app'
import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
-import Tooltip from '@/app/components/base/tooltip'
import { useDefaultModel, useModelList, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
import { LanguagesSupported } from '@/i18n/language'
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
-import { Globe01 } from '@/app/components/base/icons/src/vender/line/mapsAndTravel'
+import Checkbox from '@/app/components/base/checkbox'
+import RadioCard from '@/app/components/base/radio-card'
+import { IS_CE_EDITION } from '@/config'
+import Divider from '@/app/components/base/divider'
+import { getNotionInfo, getWebsiteInfo, useCreateDocument, useCreateFirstDocument, useFetchDefaultProcessRule, useFetchFileIndexingEstimateForFile, useFetchFileIndexingEstimateForNotion, useFetchFileIndexingEstimateForWeb } from '@/service/knowledge/use-create-dataset'
+import Badge from '@/app/components/base/badge'
+import { SkeletonContanier, SkeletonPoint, SkeletonRectangle, SkeletonRow } from '@/app/components/base/skeleton'
+import Tooltip from '@/app/components/base/tooltip'
+import CustomDialog from '@/app/components/base/dialog'
+import { PortalToFollowElem, PortalToFollowElemContent, PortalToFollowElemTrigger } from '@/app/components/base/portal-to-follow-elem'
+
+const TextLabel: FC
= (props) => {
+ return
+}
-type ValueOf = T[keyof T]
type StepTwoProps = {
isSetting?: boolean
documentDetail?: FullDocumentDetail
isAPIKeySet: boolean
onSetting: () => void
datasetId?: string
- indexingType?: ValueOf
+ indexingType?: IndexingType
+ retrievalMethod?: string
dataSourceType: DataSourceType
files: CustomFile[]
notionPages?: NotionPage[]
@@ -69,21 +83,48 @@ type StepTwoProps = {
websiteCrawlJobId?: string
onStepChange?: (delta: number) => void
updateIndexingTypeCache?: (type: string) => void
+ updateRetrievalMethodCache?: (method: string) => void
updateResultCache?: (res: createDocumentResponse) => void
onSave?: () => void
onCancel?: () => void
}
-enum SegmentType {
+export enum SegmentType {
AUTO = 'automatic',
CUSTOM = 'custom',
}
-enum IndexingType {
+export enum IndexingType {
QUALIFIED = 'high_quality',
ECONOMICAL = 'economy',
}
const DEFAULT_SEGMENT_IDENTIFIER = '\\n\\n'
+const DEFAULT_MAXMIMUM_CHUNK_LENGTH = 500
+const DEFAULT_OVERLAP = 50
+
+type ParentChildConfig = {
+ chunkForContext: ParentMode
+ parent: {
+ delimiter: string
+ maxLength: number
+ }
+ child: {
+ delimiter: string
+ maxLength: number
+ }
+}
+
+const defaultParentChildConfig: ParentChildConfig = {
+ chunkForContext: 'paragraph',
+ parent: {
+ delimiter: '\\n\\n',
+ maxLength: 500,
+ },
+ child: {
+ delimiter: '\\n',
+ maxLength: 200,
+ },
+}
const StepTwo = ({
isSetting,
@@ -104,6 +145,7 @@ const StepTwo = ({
updateResultCache,
onSave,
onCancel,
+ updateRetrievalMethodCache,
}: StepTwoProps) => {
const { t } = useTranslation()
const { locale } = useContext(I18n)
@@ -113,64 +155,157 @@ const StepTwo = ({
const { dataset: currentDataset, mutateDatasetRes } = useDatasetDetailContext()
const isInCreatePage = !datasetId || (datasetId && !currentDataset?.data_source_type)
const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : currentDataset?.data_source_type
- const scrollRef = useRef(null)
- const [scrolled, setScrolled] = useState(false)
- const previewScrollRef = useRef(null)
- const [previewScrolled, setPreviewScrolled] = useState(false)
- const [segmentationType, setSegmentationType] = useState(SegmentType.AUTO)
+ const [segmentationType, setSegmentationType] = useState(SegmentType.CUSTOM)
const [segmentIdentifier, doSetSegmentIdentifier] = useState(DEFAULT_SEGMENT_IDENTIFIER)
- const setSegmentIdentifier = useCallback((value: string) => {
- doSetSegmentIdentifier(value ? escape(value) : DEFAULT_SEGMENT_IDENTIFIER)
+ const setSegmentIdentifier = useCallback((value: string, canEmpty?: boolean) => {
+ doSetSegmentIdentifier(value ? escape(value) : (canEmpty ? '' : DEFAULT_SEGMENT_IDENTIFIER))
}, [])
- const [maxChunkLength, setMaxChunkLength] = useState(4000) // default chunk length
+ const [maxChunkLength, setMaxChunkLength] = useState(DEFAULT_MAXMIMUM_CHUNK_LENGTH) // default chunk length
const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(4000)
- const [overlap, setOverlap] = useState(50)
+ const [overlap, setOverlap] = useState(DEFAULT_OVERLAP)
const [rules, setRules] = useState([])
const [defaultConfig, setDefaultConfig] = useState()
const hasSetIndexType = !!indexingType
- const [indexType, setIndexType] = useState>(
+ const [indexType, setIndexType] = useState(
(indexingType
|| isAPIKeySet)
? IndexingType.QUALIFIED
: IndexingType.ECONOMICAL,
)
- const [isLanguageSelectDisabled, setIsLanguageSelectDisabled] = useState(false)
- const [docForm, setDocForm] = useState(
- (datasetId && documentDetail) ? documentDetail.doc_form : DocForm.TEXT,
+
+ const [previewFile, setPreviewFile] = useState(
+ (datasetId && documentDetail)
+ ? documentDetail.file
+ : files[0],
)
+ const [previewNotionPage, setPreviewNotionPage] = useState(
+ (datasetId && documentDetail)
+ ? documentDetail.notion_page
+ : notionPages[0],
+ )
+
+ const [previewWebsitePage, setPreviewWebsitePage] = useState(
+ (datasetId && documentDetail)
+ ? documentDetail.website_page
+ : websitePages[0],
+ )
+
+ // QA Related
+ const [isLanguageSelectDisabled, _setIsLanguageSelectDisabled] = useState(false)
+ const [isQAConfirmDialogOpen, setIsQAConfirmDialogOpen] = useState(false)
+ const [docForm, setDocForm] = useState(
+ (datasetId && documentDetail) ? documentDetail.doc_form as ChuckingMode : ChuckingMode.text,
+ )
+ const handleChangeDocform = (value: ChuckingMode) => {
+ if (value === ChuckingMode.qa && indexType === IndexingType.ECONOMICAL) {
+ setIsQAConfirmDialogOpen(true)
+ return
+ }
+ if (value === ChuckingMode.parentChild && indexType === IndexingType.ECONOMICAL)
+ setIndexType(IndexingType.QUALIFIED)
+ setDocForm(value)
+ // eslint-disable-next-line ts/no-use-before-define
+ currentEstimateMutation.reset()
+ }
+
const [docLanguage, setDocLanguage] = useState(
(datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese'),
)
- const [QATipHide, setQATipHide] = useState(false)
- const [previewSwitched, setPreviewSwitched] = useState(false)
- const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean()
- const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState(null)
- const [automaticFileIndexingEstimate, setAutomaticFileIndexingEstimate] = useState(null)
- const fileIndexingEstimate = (() => {
- return segmentationType === SegmentType.AUTO ? automaticFileIndexingEstimate : customFileIndexingEstimate
- })()
- const [isCreating, setIsCreating] = useState(false)
+ const [parentChildConfig, setParentChildConfig] = useState(defaultParentChildConfig)
- const scrollHandle = (e: Event) => {
- if ((e.target as HTMLDivElement).scrollTop > 0)
- setScrolled(true)
+ const getIndexing_technique = () => indexingType || indexType
- else
- setScrolled(false)
+ const getProcessRule = (): ProcessRule => {
+ if (docForm === ChuckingMode.parentChild) {
+ return {
+ rules: {
+ pre_processing_rules: rules,
+ segmentation: {
+ separator: unescape(
+ parentChildConfig.parent.delimiter,
+ ),
+ max_tokens: parentChildConfig.parent.maxLength,
+ },
+ parent_mode: parentChildConfig.chunkForContext,
+ subchunk_segmentation: {
+ separator: unescape(parentChildConfig.child.delimiter),
+ max_tokens: parentChildConfig.child.maxLength,
+ },
+ }, // api will check this. It will be removed after api refactored.
+ mode: 'hierarchical',
+ } as ProcessRule
+ }
+ return {
+ rules: {
+ pre_processing_rules: rules,
+ segmentation: {
+ separator: unescape(segmentIdentifier),
+ max_tokens: maxChunkLength,
+ chunk_overlap: overlap,
+ },
+ }, // api will check this. It will be removed after api refactored.
+ mode: segmentationType,
+ } as ProcessRule
}
- const previewScrollHandle = (e: Event) => {
- if ((e.target as HTMLDivElement).scrollTop > 0)
- setPreviewScrolled(true)
+ const fileIndexingEstimateQuery = useFetchFileIndexingEstimateForFile({
+ docForm,
+ docLanguage,
+ dataSourceType: DataSourceType.FILE,
+ files: previewFile
+ ? [files.find(file => file.name === previewFile.name)!]
+ : files,
+ indexingTechnique: getIndexing_technique() as any,
+ processRule: getProcessRule(),
+ dataset_id: datasetId!,
+ })
+ const notionIndexingEstimateQuery = useFetchFileIndexingEstimateForNotion({
+ docForm,
+ docLanguage,
+ dataSourceType: DataSourceType.NOTION,
+ notionPages: [previewNotionPage],
+ indexingTechnique: getIndexing_technique() as any,
+ processRule: getProcessRule(),
+ dataset_id: datasetId || '',
+ })
- else
- setPreviewScrolled(false)
- }
- const getFileName = (name: string) => {
- const arr = name.split('.')
- return arr.slice(0, -1).join('.')
- }
+ const websiteIndexingEstimateQuery = useFetchFileIndexingEstimateForWeb({
+ docForm,
+ docLanguage,
+ dataSourceType: DataSourceType.WEB,
+ websitePages: [previewWebsitePage],
+ crawlOptions,
+ websiteCrawlProvider,
+ websiteCrawlJobId,
+ indexingTechnique: getIndexing_technique() as any,
+ processRule: getProcessRule(),
+ dataset_id: datasetId || '',
+ })
+
+ const currentEstimateMutation = dataSourceType === DataSourceType.FILE
+ ? fileIndexingEstimateQuery
+ : dataSourceType === DataSourceType.NOTION
+ ? notionIndexingEstimateQuery
+ : websiteIndexingEstimateQuery
+
+ const fetchEstimate = useCallback(() => {
+ if (dataSourceType === DataSourceType.FILE)
+ fileIndexingEstimateQuery.mutate()
+
+ if (dataSourceType === DataSourceType.NOTION)
+ notionIndexingEstimateQuery.mutate()
+
+ if (dataSourceType === DataSourceType.WEB)
+ websiteIndexingEstimateQuery.mutate()
+ }, [dataSourceType, fileIndexingEstimateQuery, notionIndexingEstimateQuery, websiteIndexingEstimateQuery])
+
+ const estimate
+ = dataSourceType === DataSourceType.FILE
+ ? fileIndexingEstimateQuery.data
+ : dataSourceType === DataSourceType.NOTION
+ ? notionIndexingEstimateQuery.data
+ : websiteIndexingEstimateQuery.data
const getRuleName = (key: string) => {
if (key === 'remove_extra_spaces')
@@ -198,128 +333,20 @@ const StepTwo = ({
if (defaultConfig) {
setSegmentIdentifier(defaultConfig.segmentation.separator)
setMaxChunkLength(defaultConfig.segmentation.max_tokens)
- setOverlap(defaultConfig.segmentation.chunk_overlap)
+ setOverlap(defaultConfig.segmentation.chunk_overlap!)
setRules(defaultConfig.pre_processing_rules)
}
+ setParentChildConfig(defaultParentChildConfig)
}
- const fetchFileIndexingEstimate = async (docForm = DocForm.TEXT, language?: string) => {
- // eslint-disable-next-line ts/no-use-before-define
- const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams(docForm, language)!)
- if (segmentationType === SegmentType.CUSTOM)
- setCustomFileIndexingEstimate(res)
- else
- setAutomaticFileIndexingEstimate(res)
- }
-
- const confirmChangeCustomConfig = () => {
- if (segmentationType === SegmentType.CUSTOM && maxChunkLength > limitMaxChunkLength) {
- Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: limitMaxChunkLength }) })
+ const updatePreview = () => {
+ if (segmentationType === SegmentType.CUSTOM && maxChunkLength > 4000) {
+ Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') })
return
}
- setCustomFileIndexingEstimate(null)
- setShowPreview()
- fetchFileIndexingEstimate()
- setPreviewSwitched(false)
+ fetchEstimate()
}
- const getIndexing_technique = () => indexingType || indexType
-
- const getProcessRule = () => {
- const processRule: ProcessRule = {
- rules: {} as any, // api will check this. It will be removed after api refactored.
- mode: segmentationType,
- }
- if (segmentationType === SegmentType.CUSTOM) {
- const ruleObj = {
- pre_processing_rules: rules,
- segmentation: {
- separator: unescape(segmentIdentifier),
- max_tokens: maxChunkLength,
- chunk_overlap: overlap,
- },
- }
- processRule.rules = ruleObj
- }
- return processRule
- }
-
- const getNotionInfo = () => {
- const workspacesMap = groupBy(notionPages, 'workspace_id')
- const workspaces = Object.keys(workspacesMap).map((workspaceId) => {
- return {
- workspaceId,
- pages: workspacesMap[workspaceId],
- }
- })
- return workspaces.map((workspace) => {
- return {
- workspace_id: workspace.workspaceId,
- pages: workspace.pages.map((page) => {
- const { page_id, page_name, page_icon, type } = page
- return {
- page_id,
- page_name,
- page_icon,
- type,
- }
- }),
- }
- }) as NotionInfo[]
- }
-
- const getWebsiteInfo = () => {
- return {
- provider: websiteCrawlProvider,
- job_id: websiteCrawlJobId,
- urls: websitePages.map(page => page.source_url),
- only_main_content: crawlOptions?.only_main_content,
- }
- }
-
- const getFileIndexingEstimateParams = (docForm: DocForm, language?: string): IndexingEstimateParams | undefined => {
- if (dataSourceType === DataSourceType.FILE) {
- return {
- info_list: {
- data_source_type: dataSourceType,
- file_info_list: {
- file_ids: files.map(file => file.id) as string[],
- },
- },
- indexing_technique: getIndexing_technique() as string,
- process_rule: getProcessRule(),
- doc_form: docForm,
- doc_language: language || docLanguage,
- dataset_id: datasetId as string,
- }
- }
- if (dataSourceType === DataSourceType.NOTION) {
- return {
- info_list: {
- data_source_type: dataSourceType,
- notion_info_list: getNotionInfo(),
- },
- indexing_technique: getIndexing_technique() as string,
- process_rule: getProcessRule(),
- doc_form: docForm,
- doc_language: language || docLanguage,
- dataset_id: datasetId as string,
- }
- }
- if (dataSourceType === DataSourceType.WEB) {
- return {
- info_list: {
- data_source_type: dataSourceType,
- website_info_list: getWebsiteInfo(),
- },
- indexing_technique: getIndexing_technique() as string,
- process_rule: getProcessRule(),
- doc_form: docForm,
- doc_language: language || docLanguage,
- dataset_id: datasetId as string,
- }
- }
- }
const {
modelList: rerankModelList,
defaultModel: rerankDefaultModel,
@@ -403,29 +430,36 @@ const StepTwo = ({
}
}
if (dataSourceType === DataSourceType.NOTION)
- params.data_source.info_list.notion_info_list = getNotionInfo()
+ params.data_source.info_list.notion_info_list = getNotionInfo(notionPages)
- if (dataSourceType === DataSourceType.WEB)
- params.data_source.info_list.website_info_list = getWebsiteInfo()
+ if (dataSourceType === DataSourceType.WEB) {
+ params.data_source.info_list.website_info_list = getWebsiteInfo({
+ websiteCrawlProvider,
+ websiteCrawlJobId,
+ websitePages,
+ })
+ }
}
return params
}
- const getRules = async () => {
- try {
- const res = await fetchDefaultProcessRule({ url: '/datasets/process-rule' })
- const separator = res.rules.segmentation.separator
+ const fetchDefaultProcessRuleMutation = useFetchDefaultProcessRule({
+ onSuccess(data) {
+ const separator = data.rules.segmentation.separator
setSegmentIdentifier(separator)
- setMaxChunkLength(res.rules.segmentation.max_tokens)
- setLimitMaxChunkLength(res.limits.indexing_max_segmentation_tokens_length)
- setOverlap(res.rules.segmentation.chunk_overlap)
- setRules(res.rules.pre_processing_rules)
- setDefaultConfig(res.rules)
- }
- catch (err) {
- console.log(err)
- }
- }
+ setMaxChunkLength(data.rules.segmentation.max_tokens)
+ setOverlap(data.rules.segmentation.chunk_overlap!)
+ setRules(data.rules.pre_processing_rules)
+ setDefaultConfig(data.rules)
+ setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length)
+ },
+ onError(error) {
+ Toast.notify({
+ type: 'error',
+ message: `${error}`,
+ })
+ },
+ })
const getRulesFromDetail = () => {
if (documentDetail) {
@@ -435,7 +469,7 @@ const StepTwo = ({
const overlap = rules.segmentation.chunk_overlap
setSegmentIdentifier(separator)
setMaxChunkLength(max)
- setOverlap(overlap)
+ setOverlap(overlap!)
setRules(rules.pre_processing_rules)
setDefaultConfig(rules)
}
@@ -443,119 +477,81 @@ const StepTwo = ({
const getDefaultMode = () => {
if (documentDetail)
+ // @ts-expect-error fix after api refactored
setSegmentationType(documentDetail.dataset_process_rule.mode)
}
- const createHandle = async () => {
- if (isCreating)
- return
- setIsCreating(true)
- try {
- let res
- const params = getCreationParams()
- if (!params)
- return false
-
- setIsCreating(true)
- if (!datasetId) {
- res = await createFirstDocument({
- body: params as CreateDocumentReq,
- })
- updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
- updateResultCache && updateResultCache(res)
- }
- else {
- res = await createDocument({
- datasetId,
- body: params as CreateDocumentReq,
- })
- updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
- updateResultCache && updateResultCache(res)
- }
- if (mutateDatasetRes)
- mutateDatasetRes()
- onStepChange && onStepChange(+1)
- isSetting && onSave && onSave()
- }
- catch (err) {
+ const createFirstDocumentMutation = useCreateFirstDocument({
+ onError(error) {
Toast.notify({
type: 'error',
- message: `${err}`,
+ message: `${error}`,
+ })
+ },
+ })
+ const createDocumentMutation = useCreateDocument(datasetId!, {
+ onError(error) {
+ Toast.notify({
+ type: 'error',
+ message: `${error}`,
+ })
+ },
+ })
+
+ const isCreating = createFirstDocumentMutation.isPending || createDocumentMutation.isPending
+
+ const createHandle = async () => {
+ const params = getCreationParams()
+ if (!params)
+ return false
+
+ if (!datasetId) {
+ await createFirstDocumentMutation.mutateAsync(
+ params,
+ {
+ onSuccess(data) {
+ updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
+ updateResultCache && updateResultCache(data)
+ // eslint-disable-next-line ts/no-use-before-define
+ updateRetrievalMethodCache && updateRetrievalMethodCache(retrievalConfig.search_method as string)
+ },
+ },
+ )
+ }
+ else {
+ await createDocumentMutation.mutateAsync(params, {
+ onSuccess(data) {
+ updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
+ updateResultCache && updateResultCache(data)
+ },
})
}
- finally {
- setIsCreating(false)
- }
- }
-
- const handleSwitch = (state: boolean) => {
- if (state)
- setDocForm(DocForm.QA)
- else
- setDocForm(DocForm.TEXT)
- }
-
- const previewSwitch = async (language?: string) => {
- setPreviewSwitched(true)
- setIsLanguageSelectDisabled(true)
- if (segmentationType === SegmentType.AUTO)
- setAutomaticFileIndexingEstimate(null)
- else
- setCustomFileIndexingEstimate(null)
- try {
- await fetchFileIndexingEstimate(DocForm.QA, language)
- }
- finally {
- setIsLanguageSelectDisabled(false)
- }
- }
-
- const handleSelect = (language: string) => {
- setDocLanguage(language)
- // Switch language, re-cutter
- if (docForm === DocForm.QA && previewSwitched)
- previewSwitch(language)
+ if (mutateDatasetRes)
+ mutateDatasetRes()
+ onStepChange && onStepChange(+1)
+ isSetting && onSave && onSave()
}
const changeToEconomicalType = () => {
- if (!hasSetIndexType) {
+ if (docForm !== ChuckingMode.text)
+ return
+
+ if (!hasSetIndexType)
setIndexType(IndexingType.ECONOMICAL)
- setDocForm(DocForm.TEXT)
- }
}
useEffect(() => {
// fetch rules
if (!isSetting) {
- getRules()
+ fetchDefaultProcessRuleMutation.mutate('/datasets/process-rule')
}
else {
getRulesFromDetail()
getDefaultMode()
}
+ // eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
- useEffect(() => {
- scrollRef.current?.addEventListener('scroll', scrollHandle)
- return () => {
- scrollRef.current?.removeEventListener('scroll', scrollHandle)
- }
- }, [])
-
- useLayoutEffect(() => {
- if (showPreview) {
- previewScrollRef.current?.addEventListener('scroll', previewScrollHandle)
- return () => {
- previewScrollRef.current?.removeEventListener('scroll', previewScrollHandle)
- }
- }
- }, [showPreview])
-
- useEffect(() => {
- if (indexingType === IndexingType.ECONOMICAL && docForm === DocForm.QA)
- setDocForm(DocForm.TEXT)
- }, [indexingType, docForm])
-
useEffect(() => {
// get indexing type by props
if (indexingType)
@@ -565,20 +561,6 @@ const StepTwo = ({
setIndexType(isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL)
}, [isAPIKeySet, indexingType, datasetId])
- useEffect(() => {
- if (segmentationType === SegmentType.AUTO) {
- setAutomaticFileIndexingEstimate(null)
- !isMobile && setShowPreview()
- fetchFileIndexingEstimate()
- setPreviewSwitched(false)
- }
- else {
- hidePreview()
- setCustomFileIndexingEstimate(null)
- setPreviewSwitched(false)
- }
- }, [segmentationType, indexType])
-
const [retrievalConfig, setRetrievalConfig] = useState(currentDataset?.retrieval_model_dict || {
search_method: RETRIEVE_METHOD.semantic,
reranking_enable: false,
@@ -591,433 +573,558 @@ const StepTwo = ({
score_threshold: 0.5,
} as RetrievalConfig)
+ const economyDomRef = useRef(null)
+ const isHoveringEconomy = useHover(economyDomRef)
+
return (
-
-
-
{t('datasetCreation.steps.two')}
- {(isMobile || !showPreview) && (
-
- )}
-
-
-
{t('datasetCreation.stepTwo.segmentation')}
-
-
setSegmentationType(SegmentType.AUTO)}
- >
-
-
-
-
{t('datasetCreation.stepTwo.auto')}
-
{t('datasetCreation.stepTwo.autoDescription')}
-
-
-
setSegmentationType(SegmentType.CUSTOM)}
- >
-
-
-
-
{t('datasetCreation.stepTwo.custom')}
-
{t('datasetCreation.stepTwo.customDescription')}
-
- {segmentationType === SegmentType.CUSTOM && (
-
-
-
-
- {t('datasetCreation.stepTwo.separator')}
-
- {t('datasetCreation.stepTwo.separatorTip')}
-
- }
- />
-
-
setSegmentIdentifier(e.target.value)}
- />
-
-
-
-
-
{t('datasetCreation.stepTwo.maxLength')}
-
setMaxChunkLength(Number.parseInt(e.target.value.replace(/^0+/, ''), 10))}
- />
-
-
-
-
-
- {t('datasetCreation.stepTwo.overlap')}
-
- {t('datasetCreation.stepTwo.overlapTip')}
-
- }
- />
-
-
setOverlap(Number.parseInt(e.target.value.replace(/^0+/, ''), 10))}
- />
-
-
-
-
-
{t('datasetCreation.stepTwo.rules')}
- {rules.map(rule => (
-
- ruleChangeHandle(rule.id)} className="w-4 h-4 rounded border-gray-300 text-blue-700 focus:ring-blue-700" />
-
-
- ))}
-
-
-
-
-
-
-
- )}
-
-
-
{t('datasetCreation.stepTwo.indexMode')}
-
-
- {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && (
-
{
- if (isAPIKeySet)
- setIndexType(IndexingType.QUALIFIED)
- }}
- >
-
- {!hasSetIndexType &&
}
-
-
- {t('datasetCreation.stepTwo.qualified')}
- {!hasSetIndexType && {t('datasetCreation.stepTwo.recommend')}}
-
-
{t('datasetCreation.stepTwo.qualifiedTip')}
-
- {!isAPIKeySet && (
-
- {t('datasetCreation.stepTwo.warning')}
- {t('datasetCreation.stepTwo.click')}
-
- )}
-
- )}
-
- {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && (
-
-
- {!hasSetIndexType &&
}
-
-
{t('datasetCreation.stepTwo.economical')}
-
{t('datasetCreation.stepTwo.economicalTip')}
-
-
- )}
-
- {hasSetIndexType && indexType === IndexingType.ECONOMICAL && (
-
- {t('datasetCreation.stepTwo.indexSettingTip')}
- {t('datasetCreation.stepTwo.datasetSettingLink')}
-
- )}
- {IS_CE_EDITION && indexType === IndexingType.QUALIFIED && (
-
-
-
-
-
-
-
{t('datasetCreation.stepTwo.QATitle')}
-
- {t('datasetCreation.stepTwo.QALanguage')}
-
-
-
-
-
-
-
- {docForm === DocForm.QA && !QATipHide && (
-
- {t('datasetCreation.stepTwo.QATip')}
- setQATipHide(true)} />
-
- )}
-
- )}
- {/* Embedding model */}
- {indexType === IndexingType.QUALIFIED && (
-
-
{t('datasetSettings.form.embeddingModel')}
-
{
- setEmbeddingModel(model)
- }}
- />
- {!!datasetId && (
-
- {t('datasetCreation.stepTwo.indexSettingTip')}
- {t('datasetCreation.stepTwo.datasetSettingLink')}
-
- )}
-
- )}
- {/* Retrieval Method Config */}
-
- {!datasetId
- ? (
-
-
{t('datasetSettings.form.retrievalSetting.title')}
-
-
- )
- : (
-
-
{t('datasetSettings.form.retrievalSetting.title')}
-
- )}
-
-
- {
- getIndexing_technique() === IndexingType.QUALIFIED
- ? (
-
- )
- : (
-
- )
- }
-
-
-
-
-
- {dataSourceType === DataSourceType.FILE && (
- <>
-
{t('datasetCreation.stepTwo.fileSource')}
-
-
- {getFileName(files[0].name || '')}
- {files.length > 1 && (
-
- {t('datasetCreation.stepTwo.other')}
- {files.length - 1}
- {t('datasetCreation.stepTwo.fileUnit')}
-
- )}
-
-
- )}
- {dataSourceType === DataSourceType.NOTION && (
- <>
-
{t('datasetCreation.stepTwo.notionSource')}
-
-
- {notionPages[0]?.page_name}
- {notionPages.length > 1 && (
-
- {t('datasetCreation.stepTwo.other')}
- {notionPages.length - 1}
- {t('datasetCreation.stepTwo.notionUnit')}
-
- )}
-
-
- )}
- {dataSourceType === DataSourceType.WEB && (
- <>
-
{t('datasetCreation.stepTwo.websiteSource')}
-
-
- {websitePages[0].source_url}
- {websitePages.length > 1 && (
-
- {t('datasetCreation.stepTwo.other')}
- {websitePages.length - 1}
- {t('datasetCreation.stepTwo.webpageUnit')}
-
- )}
-
-
- )}
-
-
-
-
{t('datasetCreation.stepTwo.estimateSegment')}
-
- {
- fileIndexingEstimate
- ? (
-
{formatNumber(fileIndexingEstimate.total_segments)}
- )
- : (
-
{t('datasetCreation.stepTwo.calculating')}
- )
- }
-
-
-
- {!isSetting
- ? (
-
-
-
-
-
+
+
{t('datasetCreation.stepTwo.segmentation')}
+ {(!datasetId || [ChuckingMode.text, ChuckingMode.qa].includes(currentDataset!.doc_form))
+ &&
}
+ activeHeaderClassName='bg-dataset-option-card-blue-gradient'
+ description={t('datasetCreation.stepTwo.generalTip')}
+ isActive={
+ [ChuckingMode.text, ChuckingMode.qa].includes(
+ datasetId ? currentDataset!.doc_form : docForm,
)
- : (
-
-
-
+ }
+ onSwitched={() =>
+ handleChangeDocform(ChuckingMode.text)
+ }
+ actions={
+ <>
+
+
+
+ }
+ noHighlight={Boolean(datasetId)}
+ >
+
+
+ setSegmentIdentifier(e.target.value, true)}
+ />
+
+
+
+
+
+
+ {t('datasetCreation.stepTwo.rules')}
+
+
- )}
-
+
+ {rules.map(rule => (
+
{
+ ruleChangeHandle(rule.id)
+ }}>
+
+
+
+ ))}
+ {IS_CE_EDITION && <>
+
+
{
+ if (docForm === ChuckingMode.qa)
+ handleChangeDocform(ChuckingMode.text)
+ else
+ handleChangeDocform(ChuckingMode.qa)
+ }}
+ />
+
+
+
+
+
+
+
+ {docForm === ChuckingMode.qa && (
+
+
+
+ {t('datasetCreation.stepTwo.QATip')}
+
+
+ )}
+ }
+
+
+
+ }
+ {
+ (!datasetId || currentDataset!.doc_form === ChuckingMode.parentChild)
+ &&
}
+ effectImg={OrangeEffect.src}
+ activeHeaderClassName='bg-dataset-option-card-orange-gradient'
+ description={t('datasetCreation.stepTwo.parentChildTip')}
+ isActive={
+ datasetId ? currentDataset!.doc_form === ChuckingMode.parentChild : docForm === ChuckingMode.parentChild
+ }
+ onSwitched={() => handleChangeDocform(ChuckingMode.parentChild)}
+ actions={
+ <>
+
+
+
+ }
+ noHighlight={Boolean(datasetId)}
+ >
+
+
+
+
+ {t('datasetCreation.stepTwo.parentChunkForContext')}
+
+
+
+
}
+ title={t('datasetCreation.stepTwo.paragraph')}
+ description={t('datasetCreation.stepTwo.paragraphTip')}
+ isChosen={parentChildConfig.chunkForContext === 'paragraph'}
+ onChosen={() => setParentChildConfig(
+ {
+ ...parentChildConfig,
+ chunkForContext: 'paragraph',
+ },
+ )}
+ chosenConfig={
+
+ setParentChildConfig({
+ ...parentChildConfig,
+ parent: {
+ ...parentChildConfig.parent,
+ delimiter: e.target.value ? escape(e.target.value) : '',
+ },
+ })}
+ />
+ setParentChildConfig({
+ ...parentChildConfig,
+ parent: {
+ ...parentChildConfig.parent,
+ maxLength: value,
+ },
+ })}
+ />
+
+ }
+ />
+
}
+ title={t('datasetCreation.stepTwo.fullDoc')}
+ description={t('datasetCreation.stepTwo.fullDocTip')}
+ onChosen={() => setParentChildConfig(
+ {
+ ...parentChildConfig,
+ chunkForContext: 'full-doc',
+ },
+ )}
+ isChosen={parentChildConfig.chunkForContext === 'full-doc'}
+ />
+
+
+
+
+
+ {t('datasetCreation.stepTwo.childChunkForRetrieval')}
+
+
+
+
+ setParentChildConfig({
+ ...parentChildConfig,
+ child: {
+ ...parentChildConfig.child,
+ delimiter: e.target.value ? escape(e.target.value) : '',
+ },
+ })}
+ />
+ setParentChildConfig({
+ ...parentChildConfig,
+ child: {
+ ...parentChildConfig.child,
+ maxLength: value,
+ },
+ })}
+ />
+
+
+
+
+
+ {t('datasetCreation.stepTwo.rules')}
+
+
+
+
+ {rules.map(rule => (
+
{
+ ruleChangeHandle(rule.id)
+ }}>
+
+
+
+ ))}
+
+
+
+ }
+
+
{t('datasetCreation.stepTwo.indexMode')}
+
+ {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && (
+
+ {t('datasetCreation.stepTwo.qualified')}
+ {!hasSetIndexType
+ && {t('datasetCreation.stepTwo.recommend')}}
+
+ {!hasSetIndexType && }
+
+ }
+ description={t('datasetCreation.stepTwo.qualifiedTip')}
+ icon={}
+ isActive={!hasSetIndexType && indexType === IndexingType.QUALIFIED}
+ disabled={!isAPIKeySet || hasSetIndexType}
+ onSwitched={() => {
+ if (isAPIKeySet)
+ setIndexType(IndexingType.QUALIFIED)
+ }}
+ />
+ )}
+
+ {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && (
+ <>
+ setIsQAConfirmDialogOpen(false)} className='w-[432px]'>
+
+
+
+
+
+
+
+
+ }
+ isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL}
+ disabled={!isAPIKeySet || hasSetIndexType || docForm !== ChuckingMode.text}
+ ref={economyDomRef}
+ onSwitched={() => {
+ if (isAPIKeySet && docForm === ChuckingMode.text)
+ setIndexType(IndexingType.ECONOMICAL)
+ }}
+ />
+
+
+
+ {
+ docForm === ChuckingMode.qa
+ ? t('datasetCreation.stepTwo.notAvailableForQA')
+ : t('datasetCreation.stepTwo.notAvailableForParentChild')
+ }
+
+
+
+ )}
-
-
- {showPreview &&
-
-
-
-
{t('datasetCreation.stepTwo.previewTitle')}
- {docForm === DocForm.QA && !previewSwitched && (
-
- )}
-
-
-
-
-
- {docForm === DocForm.QA && !previewSwitched && (
-
- {t('datasetCreation.stepTwo.previewSwitchTipStart')}
- {t('datasetCreation.stepTwo.previewSwitchTipEnd')}
-
- )}
-
-
- {previewSwitched && docForm === DocForm.QA && fileIndexingEstimate?.qa_preview && (
- <>
- {fileIndexingEstimate?.qa_preview.map((item, index) => (
-
- ))}
-
- )}
- {(docForm === DocForm.TEXT || !previewSwitched) && fileIndexingEstimate?.preview && (
- <>
- {fileIndexingEstimate?.preview.map((item, index) => (
-
- ))}
-
- )}
- {previewSwitched && docForm === DocForm.QA && !fileIndexingEstimate?.qa_preview && (
-
-
-
- )}
- {!previewSwitched && !fileIndexingEstimate?.preview && (
-
-
-
- )}
-
-
-
-
-
{t('datasetCreation.stepTwo.sideTipTitle')}
-
-
{t('datasetCreation.stepTwo.sideTipP1')}
-
{t('datasetCreation.stepTwo.sideTipP2')}
-
{t('datasetCreation.stepTwo.sideTipP3')}
-
{t('datasetCreation.stepTwo.sideTipP4')}
-
-
+ {hasSetIndexType && indexType === IndexingType.ECONOMICAL && (
+
+ {t('datasetCreation.stepTwo.indexSettingTip')}
+ {t('datasetCreation.stepTwo.datasetSettingLink')}
)}
+ {/* Embedding model */}
+ {indexType === IndexingType.QUALIFIED && (
+
+
{t('datasetSettings.form.embeddingModel')}
+
{
+ setEmbeddingModel(model)
+ }}
+ />
+ {!!datasetId && (
+
+ {t('datasetCreation.stepTwo.indexSettingTip')}
+ {t('datasetCreation.stepTwo.datasetSettingLink')}
+
+ )}
+
+ )}
+
+ {/* Retrieval Method Config */}
+
+ {!datasetId
+ ? (
+
+
{t('datasetSettings.form.retrievalSetting.title')}
+
+
+ )
+ : (
+
+
{t('datasetSettings.form.retrievalSetting.title')}
+
+ )}
+
+
+ {
+ getIndexing_technique() === IndexingType.QUALIFIED
+ ? (
+
+ )
+ : (
+
+ )
+ }
+
+
+
+ {!isSetting
+ ? (
+
+
+
+
+ )
+ : (
+
+
+
+
+ )}
+
+ {dataSourceType === DataSourceType.FILE
+ &&
>}
+ onChange={(selected) => {
+ currentEstimateMutation.reset()
+ setPreviewFile(selected)
+ currentEstimateMutation.mutate()
+ }}
+ value={previewFile}
+ />
+ }
+ {dataSourceType === DataSourceType.NOTION
+ && ({
+ id: page.page_id,
+ name: page.page_name,
+ extension: 'md',
+ }))
+ }
+ onChange={(selected) => {
+ currentEstimateMutation.reset()
+ const selectedPage = notionPages.find(page => page.page_id === selected.id)
+ setPreviewNotionPage(selectedPage!)
+ currentEstimateMutation.mutate()
+ }}
+ value={{
+ id: previewNotionPage?.page_id || '',
+ name: previewNotionPage?.page_name || '',
+ extension: 'md',
+ }}
+ />
+ }
+ {dataSourceType === DataSourceType.WEB
+ && ({
+ id: page.source_url,
+ name: page.title,
+ extension: 'md',
+ }))
+ }
+ onChange={(selected) => {
+ currentEstimateMutation.reset()
+ const selectedPage = websitePages.find(page => page.source_url === selected.id)
+ setPreviewWebsitePage(selectedPage!)
+ currentEstimateMutation.mutate()
+ }}
+ value={
+ {
+ id: previewWebsitePage?.source_url || '',
+ name: previewWebsitePage?.title || '',
+ extension: 'md',
+ }
+ }
+ />
+ }
+
+
+
+
+
+ {t('datasetCreation.stepTwo.previewChunkTip')}
+
+
+
+ {Array.from({ length: 10 }, (_, i) => (
+
+
+
+
+
+
+
+
+
+
+ ))}
+
+ )}
+
)
diff --git a/web/app/components/datasets/create/step-two/inputs.tsx b/web/app/components/datasets/create/step-two/inputs.tsx
new file mode 100644
index 0000000000..3d38a256f1
--- /dev/null
+++ b/web/app/components/datasets/create/step-two/inputs.tsx
@@ -0,0 +1,77 @@
+import type { FC, PropsWithChildren, ReactNode } from 'react'
+import { useTranslation } from 'react-i18next'
+import type { InputProps } from '@/app/components/base/input'
+import Input from '@/app/components/base/input'
+import Tooltip from '@/app/components/base/tooltip'
+import type { InputNumberProps } from '@/app/components/base/input-number'
+import { InputNumber } from '@/app/components/base/input-number'
+
+const TextLabel: FC
= (props) => {
+ return
+}
+
+const FormField: FC> = (props) => {
+ return
+ {props.label}
+ {props.children}
+
+}
+
+export const DelimiterInput: FC = (props) => {
+ const { t } = useTranslation()
+ return
+ {t('datasetCreation.stepTwo.separator')}
+
+ {t('datasetCreation.stepTwo.separatorTip')}
+