feat: summary index

This commit is contained in:
zxhlyh 2026-01-12 13:38:18 +08:00
parent cd1af04dee
commit 13eec13a14
9 changed files with 344 additions and 4 deletions

View File

@ -2,7 +2,7 @@
import type { FC, PropsWithChildren } from 'react'
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { NotionPage } from '@/models/common'
import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, createDocumentResponse, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules } from '@/models/datasets'
import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, createDocumentResponse, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
import type { RetrievalConfig } from '@/types/app'
import {
RiAlertFill,
@ -12,7 +12,7 @@ import {
import { noop } from 'es-toolkit/function'
import Image from 'next/image'
import Link from 'next/link'
import { useCallback, useEffect, useMemo, useState } from 'react'
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { trackEvent } from '@/app/components/base/amplitude'
import Badge from '@/app/components/base/badge'
@ -29,8 +29,8 @@ import Toast from '@/app/components/base/toast'
import Tooltip from '@/app/components/base/tooltip'
import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { useDefaultModel, useModelList, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
@ -51,6 +51,7 @@ import { PreviewSlice } from '../../formatted-text/flavours/preview-slice'
import { FormattedText } from '../../formatted-text/formatted'
import PreviewContainer from '../../preview/container'
import { PreviewHeader } from '../../preview/header'
import SummaryIndexSetting from '../../settings/summary-index-setting'
import { checkShowMultiModalTip } from '../../settings/utils'
import FileList from '../assets/file-list-3-fill.svg'
import Note from '../assets/note-mod.svg'
@ -174,6 +175,18 @@ const StepTwo = ({
const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(MAXIMUM_CHUNK_TOKEN_LENGTH)
const [overlap, setOverlap] = useState(DEFAULT_OVERLAP)
const [rules, setRules] = useState<PreProcessingRule[]>([])
const [summaryIndexSetting, setSummaryIndexSetting] = useState<SummaryIndexSettingType | undefined>(currentDataset?.summary_index_setting)
const summaryIndexSettingRef = useRef<SummaryIndexSettingType | undefined>(currentDataset?.summary_index_setting)
const handleSummaryIndexSettingChange = useCallback((payload?: SummaryIndexSettingType, reset?: boolean) => {
if (reset) {
setSummaryIndexSetting(payload)
summaryIndexSettingRef.current = payload
}
else {
setSummaryIndexSetting({ ...summaryIndexSettingRef.current, ...payload })
summaryIndexSettingRef.current = { ...summaryIndexSettingRef.current, ...payload }
}
}, [setSummaryIndexSetting, summaryIndexSettingRef])
const [defaultConfig, setDefaultConfig] = useState<Rules>()
const hasSetIndexType = !!indexingType
const [indexType, setIndexType] = useState<IndexingType>(() => {
@ -250,6 +263,7 @@ const StepTwo = ({
},
},
mode: 'hierarchical',
summary_index_setting: summaryIndexSetting,
} as ProcessRule
}
return {
@ -262,6 +276,7 @@ const StepTwo = ({
},
}, // api will check this. It will be removed after api refactored.
mode: segmentationType,
summary_index_setting: summaryIndexSetting,
} as ProcessRule
}
@ -352,6 +367,7 @@ const StepTwo = ({
setMaxChunkLength(defaultConfig.segmentation.max_tokens)
setOverlap(defaultConfig.segmentation.chunk_overlap!)
setRules(defaultConfig.pre_processing_rules)
handleSummaryIndexSettingChange(currentDataset?.summary_index_setting, true)
}
setParentChildConfig(defaultParentChildConfig)
}
@ -492,6 +508,7 @@ const StepTwo = ({
setMaxChunkLength(data.rules.segmentation.max_tokens)
setOverlap(data.rules.segmentation.chunk_overlap!)
setRules(data.rules.pre_processing_rules)
handleSummaryIndexSettingChange(data.summary_index_setting, true)
setDefaultConfig(data.rules)
setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length)
},
@ -509,6 +526,7 @@ const StepTwo = ({
setMaxChunkLength(max)
setOverlap(overlap!)
setRules(rules.pre_processing_rules)
handleSummaryIndexSettingChange(documentDetail.summary_index_setting)
setDefaultConfig(rules)
if (isHierarchicalDocument) {
@ -728,6 +746,13 @@ const StepTwo = ({
)}
</>
)}
<div className="mt-4">
<SummaryIndexSetting
entry="create-document"
summaryIndexSetting={summaryIndexSetting}
onSummaryIndexSettingChange={handleSummaryIndexSettingChange}
/>
</div>
</div>
</div>
</div>
@ -879,6 +904,13 @@ const StepTwo = ({
</div>
))}
</div>
<div className="mt-4">
<SummaryIndexSetting
entry="create-document"
summaryIndexSetting={summaryIndexSetting}
onSummaryIndexSettingChange={handleSummaryIndexSettingChange}
/>
</div>
</div>
</div>
</OptionCard>

View File

@ -2,7 +2,7 @@
import type { AppIconSelection } from '@/app/components/base/app-icon-picker'
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { Member } from '@/models/common'
import type { IconInfo } from '@/models/datasets'
import type { IconInfo, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
import type { AppIconType, RetrievalConfig } from '@/types/app'
import { RiAlertFill } from '@remixicon/react'
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
@ -33,6 +33,7 @@ import RetrievalSettings from '../../external-knowledge-base/create/RetrievalSet
import ChunkStructure from '../chunk-structure'
import IndexMethod from '../index-method'
import PermissionSelector from '../permission-selector'
import SummaryIndexSetting from '../summary-index-setting'
import { checkShowMultiModalTip } from '../utils'
const rowClass = 'flex gap-x-1'
@ -76,6 +77,12 @@ const Form = () => {
model: '',
},
)
const [summaryIndexSetting, setSummaryIndexSetting] = useState(currentDataset?.summary_index_setting)
const summaryIndexSettingRef = useRef(currentDataset?.summary_index_setting)
const handleSummaryIndexSettingChange = useCallback((payload: SummaryIndexSettingType) => {
setSummaryIndexSetting({ ...summaryIndexSettingRef.current, ...payload })
summaryIndexSettingRef.current = { ...summaryIndexSettingRef.current, ...payload }
}, [])
const { data: rerankModelList } = useModelList(ModelTypeEnum.rerank)
const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding)
const { data: membersData } = useMembers()
@ -167,6 +174,7 @@ const Form = () => {
},
}),
keyword_number: keywordNumber,
summary_index_setting: summaryIndexSetting,
},
} as any
if (permission === DatasetPermission.partialMembers) {
@ -348,6 +356,21 @@ const Form = () => {
</div>
</div>
)}
{
indexMethod === IndexingType.QUALIFIED && (
<>
<Divider
type="horizontal"
className="my-1 h-px bg-divider-subtle"
/>
<SummaryIndexSetting
entry="dataset-settings"
summaryIndexSetting={summaryIndexSetting}
onSummaryIndexSettingChange={handleSummaryIndexSettingChange}
/>
</>
)
}
{/* Retrieval Method Config */}
{currentDataset?.provider === 'external'
? (

View File

@ -0,0 +1,228 @@
import type { ChangeEvent } from 'react'
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
import {
memo,
useCallback,
useMemo,
} from 'react'
import { useTranslation } from 'react-i18next'
import Switch from '@/app/components/base/switch'
import Textarea from '@/app/components/base/textarea'
import Tooltip from '@/app/components/base/tooltip'
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
type SummaryIndexSettingProps = {
entry?: 'knowledge-base' | 'dataset-settings' | 'create-document'
summaryIndexSetting?: SummaryIndexSettingType
onSummaryIndexSettingChange?: (payload: SummaryIndexSettingType) => void
readonly?: boolean
}
const SummaryIndexSetting = ({
entry = 'knowledge-base',
summaryIndexSetting,
onSummaryIndexSettingChange,
readonly = false,
}: SummaryIndexSettingProps) => {
const { t } = useTranslation()
const {
data: textGenerationModelList,
} = useModelList(ModelTypeEnum.textGeneration)
const summaryIndexModelConfig = useMemo(() => {
if (!summaryIndexSetting?.model_name || !summaryIndexSetting?.model_provider_name)
return undefined
return {
providerName: summaryIndexSetting?.model_provider_name,
modelName: summaryIndexSetting?.model_name,
}
}, [summaryIndexSetting?.model_name, summaryIndexSetting?.model_provider_name])
const handleSummaryIndexEnableChange = useCallback((value: boolean) => {
onSummaryIndexSettingChange?.({
enable: value,
})
}, [onSummaryIndexSettingChange])
const handleSummaryIndexModelChange = useCallback((model: DefaultModel) => {
onSummaryIndexSettingChange?.({
model_provider_name: model.provider,
model_name: model.model,
})
}, [onSummaryIndexSettingChange])
const handleSummaryIndexPromptChange = useCallback((e: ChangeEvent<HTMLTextAreaElement>) => {
onSummaryIndexSettingChange?.({
summary_prompt: e.target.value,
})
}, [onSummaryIndexSettingChange])
if (entry === 'knowledge-base') {
return (
<div>
<div className="flex h-6 items-center justify-between">
<div className="system-sm-semibold-uppercase flex items-center text-text-secondary">
{t('form.summaryAutoGen', { ns: 'datasetSettings' })}
<Tooltip
triggerClassName="ml-1 h-4 w-4 shrink-0"
popupContent={t('form.summaryAutoGenTip', { ns: 'datasetSettings' })}
>
</Tooltip>
</div>
<Switch
defaultValue={summaryIndexSetting?.enable ?? false}
onChange={handleSummaryIndexEnableChange}
size="md"
/>
</div>
{
summaryIndexSetting?.enable && (
<div>
<div className="system-xs-medium-uppercase mb-1.5 mt-2 flex h-6 items-center text-text-tertiary">
{t('form.summaryModel', { ns: 'datasetSettings' })}
</div>
<ModelSelector
defaultModel={summaryIndexModelConfig && { provider: summaryIndexModelConfig.providerName, model: summaryIndexModelConfig.modelName }}
modelList={textGenerationModelList}
onSelect={handleSummaryIndexModelChange}
readonly={readonly}
showDeprecatedWarnIcon
/>
<div className="system-xs-medium-uppercase mt-3 flex h-6 items-center text-text-tertiary">
{t('form.summaryInstructions', { ns: 'datasetSettings' })}
</div>
<Textarea
value={summaryIndexSetting?.summary_prompt ?? ''}
onChange={handleSummaryIndexPromptChange}
disabled={readonly}
placeholder={t('form.summaryInstructionsPlaceholder', { ns: 'datasetSettings' })}
/>
</div>
)
}
</div>
)
}
if (entry === 'dataset-settings') {
return (
<div className="space-y-4">
<div className="flex gap-x-1">
<div className="flex h-7 w-[180px] shrink-0 items-center pt-1">
<div className="system-sm-semibold text-text-secondary">
{t('form.summaryAutoGen', { ns: 'datasetSettings' })}
</div>
</div>
<div className="py-1.5">
<div className="system-sm-semibold flex items-center text-text-secondary">
<Switch
className="mr-2"
defaultValue={summaryIndexSetting?.enable ?? false}
onChange={handleSummaryIndexEnableChange}
size="md"
/>
{
summaryIndexSetting?.enable ? t('list.status.enabled', { ns: 'datasetDocuments' }) : t('list.status.disabled', { ns: 'datasetDocuments' })
}
</div>
<div className="system-sm-regular mt-2 text-text-tertiary">
{
summaryIndexSetting?.enable && t('form.summaryAutoGenTip', { ns: 'datasetSettings' })
}
{
!summaryIndexSetting?.enable && t('form.summaryAutoGenEnableTip', { ns: 'datasetSettings' })
}
</div>
</div>
</div>
{
summaryIndexSetting?.enable && (
<>
<div className="flex gap-x-1">
<div className="flex h-7 w-[180px] shrink-0 items-center pt-1">
<div className="system-sm-medium text-text-tertiary">
{t('form.summaryModel', { ns: 'datasetSettings' })}
</div>
</div>
<div className="grow">
<ModelSelector
defaultModel={summaryIndexModelConfig && { provider: summaryIndexModelConfig.providerName, model: summaryIndexModelConfig.modelName }}
modelList={textGenerationModelList}
onSelect={handleSummaryIndexModelChange}
readonly={readonly}
showDeprecatedWarnIcon
triggerClassName="h-8"
/>
</div>
</div>
<div className="flex">
<div className="flex h-7 w-[180px] shrink-0 items-center pt-1">
<div className="system-sm-medium text-text-tertiary">
{t('form.summaryInstructions', { ns: 'datasetSettings' })}
</div>
</div>
<div className="grow">
<Textarea
value={summaryIndexSetting?.summary_prompt ?? ''}
onChange={handleSummaryIndexPromptChange}
disabled={readonly}
placeholder={t('form.summaryInstructionsPlaceholder', { ns: 'datasetSettings' })}
/>
</div>
</div>
</>
)
}
</div>
)
}
return (
<div className="space-y-3">
<div className="flex h-6 items-center">
<Switch
className="mr-2"
defaultValue={summaryIndexSetting?.enable ?? false}
onChange={handleSummaryIndexEnableChange}
size="md"
/>
<div className="system-sm-semibold text-text-secondary">
{t('form.summaryAutoGen', { ns: 'datasetSettings' })}
</div>
</div>
{
summaryIndexSetting?.enable && (
<>
<div>
<div className="system-sm-medium mb-1.5 flex h-6 items-center text-text-secondary">
{t('form.summaryModel', { ns: 'datasetSettings' })}
</div>
<ModelSelector
defaultModel={summaryIndexModelConfig && { provider: summaryIndexModelConfig.providerName, model: summaryIndexModelConfig.modelName }}
modelList={textGenerationModelList}
onSelect={handleSummaryIndexModelChange}
readonly={readonly}
showDeprecatedWarnIcon
triggerClassName="h-8"
/>
</div>
<div>
<div className="system-sm-medium mb-1.5 flex h-6 items-center text-text-secondary">
{t('form.summaryInstructions', { ns: 'datasetSettings' })}
</div>
<Textarea
value={summaryIndexSetting?.summary_prompt ?? ''}
onChange={handleSummaryIndexPromptChange}
disabled={readonly}
placeholder={t('form.summaryInstructionsPlaceholder', { ns: 'datasetSettings' })}
/>
</div>
</>
)
}
</div>
)
}
export default memo(SummaryIndexSetting)

View File

@ -1,6 +1,7 @@
import type {
KnowledgeBaseNodeType,
RerankingModel,
SummaryIndexSetting,
} from '../types'
import type { ValueSelector } from '@/app/components/workflow/types'
import { produce } from 'immer'
@ -246,6 +247,16 @@ export const useConfig = (id: string) => {
})
}, [handleNodeDataUpdate])
const handleSummaryIndexSettingChange = useCallback((summaryIndexSetting: SummaryIndexSetting) => {
const nodeData = getNodeData()
handleNodeDataUpdate({
summary_index_setting: {
...nodeData?.data.summary_index_setting,
...summaryIndexSetting,
},
})
}, [handleNodeDataUpdate, getNodeData])
return {
handleChunkStructureChange,
handleIndexMethodChange,
@ -260,5 +271,6 @@ export const useConfig = (id: string) => {
handleScoreThresholdChange,
handleScoreThresholdEnabledChange,
handleInputVariableChange,
handleSummaryIndexSettingChange,
}
}

View File

@ -7,6 +7,7 @@ import {
useMemo,
} from 'react'
import { useTranslation } from 'react-i18next'
import SummaryIndexSetting from '@/app/components/datasets/settings/summary-index-setting'
import { checkShowMultiModalTip } from '@/app/components/datasets/settings/utils'
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
@ -51,6 +52,7 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
handleScoreThresholdChange,
handleScoreThresholdEnabledChange,
handleInputVariableChange,
handleSummaryIndexSettingChange,
} = useConfig(id)
const filterVar = useCallback((variable: Var) => {
@ -167,6 +169,20 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
<div className="pt-1">
<Split className="h-[1px]" />
</div>
{
data.indexing_technique === IndexMethodEnum.QUALIFIED && (
<>
<SummaryIndexSetting
summaryIndexSetting={data.summary_index_setting}
onSummaryIndexSettingChange={handleSummaryIndexSettingChange}
readonly={nodesReadOnly}
/>
<div className="pt-1">
<Split className="h-[1px]" />
</div>
</>
)
}
<RetrievalSetting
indexMethod={data.indexing_technique}
searchMethod={data.retrieval_model.search_method}

View File

@ -42,6 +42,12 @@ export type RetrievalSetting = {
score_threshold: number
reranking_mode?: RerankingModeEnum
}
export type SummaryIndexSetting = {
enable?: boolean
model_name?: string
model_provider_name?: string
summary_prompt?: string
}
export type KnowledgeBaseNodeType = CommonNodeType & {
index_chunk_variable_selector: string[]
chunk_structure?: ChunkStructureEnum
@ -52,4 +58,5 @@ export type KnowledgeBaseNodeType = CommonNodeType & {
retrieval_model: RetrievalSetting
_embeddingModelList?: Model[]
_rerankModelList?: Model[]
summary_index_setting?: SummaryIndexSetting
}

View File

@ -39,6 +39,12 @@
"form.retrievalSettings": "Retrieval Settings",
"form.save": "Save",
"form.searchModel": "Search model",
"form.summaryAutoGen": "Summary Auto-Gen",
"form.summaryAutoGenEnableTip": "Once enabled, summaries will be generated automatically for newly added documents. Existing documents can still be summarized manually.",
"form.summaryAutoGenTip": "Summaries are automatically generated for newly added documents. Existing documents can still be summarized manually.",
"form.summaryInstructions": "Instructions",
"form.summaryInstructionsPlaceholder": "Describe the rules or style for auto-generated summaries…",
"form.summaryModel": "Summary Model",
"form.upgradeHighQualityTip": "Once upgrading to High Quality mode, reverting to Economical mode is not available",
"title": "Knowledge settings"
}

View File

@ -39,6 +39,12 @@
"form.retrievalSettings": "检索设置",
"form.save": "保存",
"form.searchModel": "搜索模型",
"form.summaryAutoGen": "摘要自动生成",
"form.summaryAutoGenEnableTip": "启用后,将自动为新添加的文档生成摘要。已有的文档仍可以手动摘要。",
"form.summaryAutoGenTip": "将自动为新添加的文档生成摘要。已有的文档仍可以手动摘要。",
"form.summaryInstructions": "指令",
"form.summaryInstructionsPlaceholder": "描述自动生成摘要的规则或风格…",
"form.summaryModel": "摘要模型",
"form.upgradeHighQualityTip": "一旦升级为高质量模式,将无法切换回经济模式。",
"title": "知识库设置"
}

View File

@ -42,6 +42,13 @@ export type IconInfo = {
icon_url?: string
}
export type SummaryIndexSetting = {
enable?: boolean
model_name?: string
model_provider_name?: string
summary_prompt?: string
}
export type DataSet = {
id: string
name: string
@ -88,6 +95,7 @@ export type DataSet = {
runtime_mode: 'rag_pipeline' | 'general'
enable_api: boolean // Indicates if the service API is enabled
is_multimodal: boolean // Indicates if the dataset supports multimodal
summary_index_setting?: SummaryIndexSetting
}
export type ExternalAPIItem = {
@ -262,6 +270,7 @@ export type ProcessRuleResponse = {
mode: ProcessMode
rules: Rules
limits: Limits
summary_index_setting?: SummaryIndexSetting
}
export type Rules = {
@ -467,6 +476,7 @@ export type NotionPage = {
export type ProcessRule = {
mode: ProcessMode
rules: Rules
summary_index_setting?: SummaryIndexSetting
}
export type createDocumentResponse = {