diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 51b5c15178..cc10a1d9d5 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -2,7 +2,7 @@ import type { FC, PropsWithChildren } from 'react' import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations' import type { NotionPage } from '@/models/common' -import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, createDocumentResponse, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules } from '@/models/datasets' +import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, createDocumentResponse, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets' import type { RetrievalConfig } from '@/types/app' import { RiAlertFill, @@ -12,7 +12,7 @@ import { import { noop } from 'es-toolkit/function' import Image from 'next/image' import Link from 'next/link' -import { useCallback, useEffect, useMemo, useState } from 'react' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { trackEvent } from '@/app/components/base/amplitude' import Badge from '@/app/components/base/badge' @@ -29,8 +29,8 @@ import Toast from '@/app/components/base/toast' import Tooltip from '@/app/components/base/tooltip' import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model' import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config' -import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config' +import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' import { useDefaultModel, useModelList, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector' @@ -51,6 +51,7 @@ import { PreviewSlice } from '../../formatted-text/flavours/preview-slice' import { FormattedText } from '../../formatted-text/formatted' import PreviewContainer from '../../preview/container' import { PreviewHeader } from '../../preview/header' +import SummaryIndexSetting from '../../settings/summary-index-setting' import { checkShowMultiModalTip } from '../../settings/utils' import FileList from '../assets/file-list-3-fill.svg' import Note from '../assets/note-mod.svg' @@ -174,6 +175,18 @@ const StepTwo = ({ const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(MAXIMUM_CHUNK_TOKEN_LENGTH) const [overlap, setOverlap] = useState(DEFAULT_OVERLAP) const [rules, setRules] = useState([]) + const [summaryIndexSetting, setSummaryIndexSetting] = useState(currentDataset?.summary_index_setting) + const summaryIndexSettingRef = useRef(currentDataset?.summary_index_setting) + const handleSummaryIndexSettingChange = useCallback((payload?: SummaryIndexSettingType, reset?: boolean) => { + if (reset) { + setSummaryIndexSetting(payload) + summaryIndexSettingRef.current = payload + } + else { + setSummaryIndexSetting({ ...summaryIndexSettingRef.current, ...payload }) + summaryIndexSettingRef.current = { ...summaryIndexSettingRef.current, ...payload } + } + }, [setSummaryIndexSetting, summaryIndexSettingRef]) const [defaultConfig, setDefaultConfig] = useState() const hasSetIndexType = !!indexingType const [indexType, setIndexType] = useState(() => { @@ -250,6 +263,7 @@ const StepTwo = ({ }, }, mode: 'hierarchical', + summary_index_setting: summaryIndexSetting, } as ProcessRule } return { @@ -262,6 +276,7 @@ const StepTwo = ({ }, }, // api will check this. It will be removed after api refactored. mode: segmentationType, + summary_index_setting: summaryIndexSetting, } as ProcessRule } @@ -352,6 +367,7 @@ const StepTwo = ({ setMaxChunkLength(defaultConfig.segmentation.max_tokens) setOverlap(defaultConfig.segmentation.chunk_overlap!) setRules(defaultConfig.pre_processing_rules) + handleSummaryIndexSettingChange(currentDataset?.summary_index_setting, true) } setParentChildConfig(defaultParentChildConfig) } @@ -492,6 +508,7 @@ const StepTwo = ({ setMaxChunkLength(data.rules.segmentation.max_tokens) setOverlap(data.rules.segmentation.chunk_overlap!) setRules(data.rules.pre_processing_rules) + handleSummaryIndexSettingChange(data.summary_index_setting, true) setDefaultConfig(data.rules) setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length) }, @@ -509,6 +526,7 @@ const StepTwo = ({ setMaxChunkLength(max) setOverlap(overlap!) setRules(rules.pre_processing_rules) + handleSummaryIndexSettingChange(documentDetail.summary_index_setting) setDefaultConfig(rules) if (isHierarchicalDocument) { @@ -728,6 +746,13 @@ const StepTwo = ({ )} )} +
+ +
@@ -879,6 +904,13 @@ const StepTwo = ({ ))} +
+ +
diff --git a/web/app/components/datasets/settings/form/index.tsx b/web/app/components/datasets/settings/form/index.tsx index 206c264a88..afc78b7844 100644 --- a/web/app/components/datasets/settings/form/index.tsx +++ b/web/app/components/datasets/settings/form/index.tsx @@ -2,7 +2,7 @@ import type { AppIconSelection } from '@/app/components/base/app-icon-picker' import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations' import type { Member } from '@/models/common' -import type { IconInfo } from '@/models/datasets' +import type { IconInfo, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets' import type { AppIconType, RetrievalConfig } from '@/types/app' import { RiAlertFill } from '@remixicon/react' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' @@ -33,6 +33,7 @@ import RetrievalSettings from '../../external-knowledge-base/create/RetrievalSet import ChunkStructure from '../chunk-structure' import IndexMethod from '../index-method' import PermissionSelector from '../permission-selector' +import SummaryIndexSetting from '../summary-index-setting' import { checkShowMultiModalTip } from '../utils' const rowClass = 'flex gap-x-1' @@ -76,6 +77,12 @@ const Form = () => { model: '', }, ) + const [summaryIndexSetting, setSummaryIndexSetting] = useState(currentDataset?.summary_index_setting) + const summaryIndexSettingRef = useRef(currentDataset?.summary_index_setting) + const handleSummaryIndexSettingChange = useCallback((payload: SummaryIndexSettingType) => { + setSummaryIndexSetting({ ...summaryIndexSettingRef.current, ...payload }) + summaryIndexSettingRef.current = { ...summaryIndexSettingRef.current, ...payload } + }, []) const { data: rerankModelList } = useModelList(ModelTypeEnum.rerank) const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding) const { data: membersData } = useMembers() @@ -167,6 +174,7 @@ const Form = () => { }, }), keyword_number: keywordNumber, + summary_index_setting: summaryIndexSetting, }, } as any if (permission === DatasetPermission.partialMembers) { @@ -348,6 +356,21 @@ const Form = () => { )} + { + indexMethod === IndexingType.QUALIFIED && ( + <> + + + + ) + } {/* Retrieval Method Config */} {currentDataset?.provider === 'external' ? ( diff --git a/web/app/components/datasets/settings/summary-index-setting.tsx b/web/app/components/datasets/settings/summary-index-setting.tsx new file mode 100644 index 0000000000..b79f8ffe0c --- /dev/null +++ b/web/app/components/datasets/settings/summary-index-setting.tsx @@ -0,0 +1,228 @@ +import type { ChangeEvent } from 'react' +import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations' +import type { SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets' +import { + memo, + useCallback, + useMemo, +} from 'react' +import { useTranslation } from 'react-i18next' +import Switch from '@/app/components/base/switch' +import Textarea from '@/app/components/base/textarea' +import Tooltip from '@/app/components/base/tooltip' +import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks' +import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector' + +type SummaryIndexSettingProps = { + entry?: 'knowledge-base' | 'dataset-settings' | 'create-document' + summaryIndexSetting?: SummaryIndexSettingType + onSummaryIndexSettingChange?: (payload: SummaryIndexSettingType) => void + readonly?: boolean +} +const SummaryIndexSetting = ({ + entry = 'knowledge-base', + summaryIndexSetting, + onSummaryIndexSettingChange, + readonly = false, +}: SummaryIndexSettingProps) => { + const { t } = useTranslation() + const { + data: textGenerationModelList, + } = useModelList(ModelTypeEnum.textGeneration) + const summaryIndexModelConfig = useMemo(() => { + if (!summaryIndexSetting?.model_name || !summaryIndexSetting?.model_provider_name) + return undefined + + return { + providerName: summaryIndexSetting?.model_provider_name, + modelName: summaryIndexSetting?.model_name, + } + }, [summaryIndexSetting?.model_name, summaryIndexSetting?.model_provider_name]) + + const handleSummaryIndexEnableChange = useCallback((value: boolean) => { + onSummaryIndexSettingChange?.({ + enable: value, + }) + }, [onSummaryIndexSettingChange]) + + const handleSummaryIndexModelChange = useCallback((model: DefaultModel) => { + onSummaryIndexSettingChange?.({ + model_provider_name: model.provider, + model_name: model.model, + }) + }, [onSummaryIndexSettingChange]) + + const handleSummaryIndexPromptChange = useCallback((e: ChangeEvent) => { + onSummaryIndexSettingChange?.({ + summary_prompt: e.target.value, + }) + }, [onSummaryIndexSettingChange]) + + if (entry === 'knowledge-base') { + return ( +
+
+
+ {t('form.summaryAutoGen', { ns: 'datasetSettings' })} + + +
+ +
+ { + summaryIndexSetting?.enable && ( +
+
+ {t('form.summaryModel', { ns: 'datasetSettings' })} +
+ +
+ {t('form.summaryInstructions', { ns: 'datasetSettings' })} +
+