diff --git a/web/app/components/base/icons/assets/vender/knowledge/search-lines-sparkle.svg b/web/app/components/base/icons/assets/vender/knowledge/search-lines-sparkle.svg new file mode 100644 index 0000000000..1eb2781715 --- /dev/null +++ b/web/app/components/base/icons/assets/vender/knowledge/search-lines-sparkle.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/web/app/components/base/icons/src/vender/knowledge/SearchLinesSparkle.json b/web/app/components/base/icons/src/vender/knowledge/SearchLinesSparkle.json new file mode 100644 index 0000000000..7fa195092d --- /dev/null +++ b/web/app/components/base/icons/src/vender/knowledge/SearchLinesSparkle.json @@ -0,0 +1,53 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "16", + "height": "16", + "viewBox": "0 0 16 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M12 7.33337V2.66671H4.00002V13.3334H8.00002C8.36821 13.3334 8.66669 13.6319 8.66669 14C8.66669 14.3682 8.36821 14.6667 8.00002 14.6667H3.33335C2.96516 14.6667 2.66669 14.3682 2.66669 14V2.00004C2.66669 1.63185 2.96516 1.33337 3.33335 1.33337H12.6667C13.0349 1.33337 13.3334 1.63185 13.3334 2.00004V7.33337C13.3334 7.70156 13.0349 8.00004 12.6667 8.00004C12.2985 8.00004 12 7.70156 12 7.33337Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M10 4.00004C10.3682 4.00004 10.6667 4.29852 10.6667 4.66671C10.6667 5.0349 10.3682 5.33337 10 5.33337H6.00002C5.63183 5.33337 5.33335 5.0349 5.33335 4.66671C5.33335 4.29852 5.63183 4.00004 6.00002 4.00004H10Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M8.00002 6.66671C8.36821 6.66671 8.66669 6.96518 8.66669 7.33337C8.66669 7.70156 8.36821 8.00004 8.00002 8.00004H6.00002C5.63183 8.00004 5.33335 7.70156 5.33335 7.33337C5.33335 6.96518 5.63183 6.66671 6.00002 6.66671H8.00002Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M12.827 10.7902L12.3624 9.58224C12.3048 9.43231 12.1607 9.33337 12 9.33337C11.8394 9.33337 11.6953 9.43231 11.6376 9.58224L11.173 10.7902C11.1054 10.9662 10.9662 11.1054 10.7902 11.173L9.58222 11.6376C9.43229 11.6953 9.33335 11.8394 9.33335 12C9.33335 12.1607 9.43229 12.3048 9.58222 12.3624L10.7902 12.827C10.9662 12.8947 11.1054 13.0338 11.173 13.2099L11.6376 14.4178C11.6953 14.5678 11.8394 14.6667 12 14.6667C12.1607 14.6667 12.3048 14.5678 12.3624 14.4178L12.827 13.2099C12.8947 13.0338 13.0338 12.8947 13.2099 12.827L14.4178 12.3624C14.5678 12.3048 14.6667 12.1607 14.6667 12C14.6667 11.8394 14.5678 11.6953 14.4178 11.6376L13.2099 11.173C13.0338 11.1054 12.8947 10.9662 12.827 10.7902Z", + "fill": "currentColor" + }, + "children": [] + } + ] + }, + "name": "SearchLinesSparkle" +} diff --git a/web/app/components/base/icons/src/vender/knowledge/SearchLinesSparkle.tsx b/web/app/components/base/icons/src/vender/knowledge/SearchLinesSparkle.tsx new file mode 100644 index 0000000000..3ae90b5fa1 --- /dev/null +++ b/web/app/components/base/icons/src/vender/knowledge/SearchLinesSparkle.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import type { IconData } from '@/app/components/base/icons/IconBase' +import * as React from 'react' +import IconBase from '@/app/components/base/icons/IconBase' +import data from './SearchLinesSparkle.json' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject> + }, +) => + +Icon.displayName = 'SearchLinesSparkle' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/knowledge/index.ts b/web/app/components/base/icons/src/vender/knowledge/index.ts index 7239511af3..44055c4975 100644 --- a/web/app/components/base/icons/src/vender/knowledge/index.ts +++ b/web/app/components/base/icons/src/vender/knowledge/index.ts @@ -11,5 +11,6 @@ export { default as HighQuality } from './HighQuality' export { default as HybridSearch } from './HybridSearch' export { default as ParentChildChunk } from './ParentChildChunk' export { default as QuestionAndAnswer } from './QuestionAndAnswer' +export { default as SearchLinesSparkle } from './SearchLinesSparkle' export { default as SearchMenu } from './SearchMenu' export { default as VectorSearch } from './VectorSearch' diff --git a/web/app/components/datasets/create/step-two/components/general-chunking-options.tsx b/web/app/components/datasets/create/step-two/components/general-chunking-options.tsx index 5140c902f5..84d742d734 100644 --- a/web/app/components/datasets/create/step-two/components/general-chunking-options.tsx +++ b/web/app/components/datasets/create/step-two/components/general-chunking-options.tsx @@ -1,7 +1,7 @@ 'use client' import type { FC } from 'react' -import type { PreProcessingRule } from '@/models/datasets' +import type { PreProcessingRule, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets' import { RiAlertFill, RiSearchEyeLine, @@ -12,6 +12,7 @@ import Button from '@/app/components/base/button' import Checkbox from '@/app/components/base/checkbox' import Divider from '@/app/components/base/divider' import Tooltip from '@/app/components/base/tooltip' +import SummaryIndexSetting from '@/app/components/datasets/settings/summary-index-setting' import { IS_CE_EDITION } from '@/config' import { ChunkingMode } from '@/models/datasets' import SettingCog from '../../assets/setting-gear-mod.svg' @@ -52,6 +53,9 @@ type GeneralChunkingOptionsProps = { onReset: () => void // Locale locale: string + showSummaryIndexSetting?: boolean + summaryIndexSetting?: SummaryIndexSettingType + onSummaryIndexSettingChange?: (payload: SummaryIndexSettingType) => void } export const GeneralChunkingOptions: FC = ({ @@ -74,6 +78,9 @@ export const GeneralChunkingOptions: FC = ({ onPreview, onReset, locale, + showSummaryIndexSetting, + summaryIndexSetting, + onSummaryIndexSettingChange, }) => { const { t } = useTranslation() @@ -146,6 +153,17 @@ export const GeneralChunkingOptions: FC = ({ ))} + { + showSummaryIndexSetting && ( +
+ +
+ ) + } {IS_CE_EDITION && ( <> diff --git a/web/app/components/datasets/create/step-two/components/parent-child-options.tsx b/web/app/components/datasets/create/step-two/components/parent-child-options.tsx index e46aa5817b..22b88037e1 100644 --- a/web/app/components/datasets/create/step-two/components/parent-child-options.tsx +++ b/web/app/components/datasets/create/step-two/components/parent-child-options.tsx @@ -2,7 +2,7 @@ import type { FC } from 'react' import type { ParentChildConfig } from '../hooks' -import type { ParentMode, PreProcessingRule } from '@/models/datasets' +import type { ParentMode, PreProcessingRule, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets' import { RiSearchEyeLine } from '@remixicon/react' import Image from 'next/image' import { useTranslation } from 'react-i18next' @@ -11,6 +11,7 @@ import Checkbox from '@/app/components/base/checkbox' import Divider from '@/app/components/base/divider' import { ParentChildChunk } from '@/app/components/base/icons/src/vender/knowledge' import RadioCard from '@/app/components/base/radio-card' +import SummaryIndexSetting from '@/app/components/datasets/settings/summary-index-setting' import { ChunkingMode } from '@/models/datasets' import FileList from '../../assets/file-list-3-fill.svg' import Note from '../../assets/note-mod.svg' @@ -31,6 +32,8 @@ type ParentChildOptionsProps = { // State parentChildConfig: ParentChildConfig rules: PreProcessingRule[] + summaryIndexSetting?: SummaryIndexSettingType + onSummaryIndexSettingChange?: (payload: SummaryIndexSettingType) => void currentDocForm: ChunkingMode // Flags isActive: boolean @@ -46,11 +49,13 @@ type ParentChildOptionsProps = { onRuleToggle: (id: string) => void onPreview: () => void onReset: () => void + showSummaryIndexSetting?: boolean } export const ParentChildOptions: FC = ({ parentChildConfig, rules, + summaryIndexSetting, currentDocForm: _currentDocForm, isActive, isInUpload, @@ -62,8 +67,10 @@ export const ParentChildOptions: FC = ({ onChildDelimiterChange, onChildMaxLengthChange, onRuleToggle, + onSummaryIndexSettingChange, onPreview, onReset, + showSummaryIndexSetting, }) => { const { t } = useTranslation() @@ -183,6 +190,17 @@ export const ParentChildOptions: FC = ({ ))} + { + showSummaryIndexSetting && ( +
+ +
+ ) + } diff --git a/web/app/components/datasets/create/step-two/components/preview-panel.tsx b/web/app/components/datasets/create/step-two/components/preview-panel.tsx index 4f25cee5bd..5cb33f2d6d 100644 --- a/web/app/components/datasets/create/step-two/components/preview-panel.tsx +++ b/web/app/components/datasets/create/step-two/components/preview-panel.tsx @@ -14,6 +14,7 @@ import { ChunkingMode } from '@/models/datasets' import { cn } from '@/utils/classnames' import { ChunkContainer, QAPreview } from '../../../chunk' import PreviewDocumentPicker from '../../../common/document-picker/preview-document-picker' +import SummaryLabel from '../../../documents/detail/completed/common/summary-label' import { PreviewSlice } from '../../../formatted-text/flavours/preview-slice' import { FormattedText } from '../../../formatted-text/formatted' import PreviewContainer from '../../../preview/container' @@ -99,6 +100,7 @@ export const PreviewPanel: FC = ({ characterCount={item.content.length} > {item.content} + {item.summary && } )) )} @@ -131,6 +133,7 @@ export const PreviewPanel: FC = ({ ) })} + {item.summary && } ) }) diff --git a/web/app/components/datasets/create/step-two/hooks/use-document-creation.ts b/web/app/components/datasets/create/step-two/hooks/use-document-creation.ts index fd132b38ef..eaa51e393a 100644 --- a/web/app/components/datasets/create/step-two/hooks/use-document-creation.ts +++ b/web/app/components/datasets/create/step-two/hooks/use-document-creation.ts @@ -9,6 +9,7 @@ import type { CustomFile, FullDocumentDetail, ProcessRule, + SummaryIndexSetting as SummaryIndexSettingType, } from '@/models/datasets' import type { RetrievalConfig, RETRIEVE_METHOD } from '@/types/app' import { useCallback } from 'react' @@ -141,6 +142,7 @@ export const useDocumentCreation = (options: UseDocumentCreationOptions) => { retrievalConfig: RetrievalConfig, embeddingModel: DefaultModel, indexingTechnique: string, + summaryIndexSetting?: SummaryIndexSettingType, ): CreateDocumentReq | null => { if (isSetting) { return { @@ -148,6 +150,7 @@ export const useDocumentCreation = (options: UseDocumentCreationOptions) => { doc_form: currentDocForm, doc_language: docLanguage, process_rule: processRule, + summary_index_setting: summaryIndexSetting, retrieval_model: retrievalConfig, embedding_model: embeddingModel.model, embedding_model_provider: embeddingModel.provider, @@ -164,6 +167,7 @@ export const useDocumentCreation = (options: UseDocumentCreationOptions) => { }, indexing_technique: indexingTechnique, process_rule: processRule, + summary_index_setting: summaryIndexSetting, doc_form: currentDocForm, doc_language: docLanguage, retrieval_model: retrievalConfig, diff --git a/web/app/components/datasets/create/step-two/hooks/use-segmentation-state.ts b/web/app/components/datasets/create/step-two/hooks/use-segmentation-state.ts index 69cc089b4f..41c34a9a96 100644 --- a/web/app/components/datasets/create/step-two/hooks/use-segmentation-state.ts +++ b/web/app/components/datasets/create/step-two/hooks/use-segmentation-state.ts @@ -1,5 +1,5 @@ -import type { ParentMode, PreProcessingRule, ProcessRule, Rules } from '@/models/datasets' -import { useCallback, useState } from 'react' +import type { ParentMode, PreProcessingRule, ProcessRule, Rules, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets' +import { useCallback, useRef, useState } from 'react' import { ChunkingMode, ProcessMode } from '@/models/datasets' import escape from './escape' import unescape from './unescape' @@ -39,10 +39,11 @@ export const defaultParentChildConfig: ParentChildConfig = { export type UseSegmentationStateOptions = { initialSegmentationType?: ProcessMode + initialSummaryIndexSetting?: SummaryIndexSettingType } export const useSegmentationState = (options: UseSegmentationStateOptions = {}) => { - const { initialSegmentationType } = options + const { initialSegmentationType, initialSummaryIndexSetting } = options // Segmentation type (general or parent-child) const [segmentationType, setSegmentationType] = useState( @@ -58,6 +59,12 @@ export const useSegmentationState = (options: UseSegmentationStateOptions = {}) // Pre-processing rules const [rules, setRules] = useState([]) const [defaultConfig, setDefaultConfig] = useState() + const [summaryIndexSetting, setSummaryIndexSetting] = useState(initialSummaryIndexSetting) + const summaryIndexSettingRef = useRef(initialSummaryIndexSetting) + const handleSummaryIndexSettingChange = useCallback((payload: SummaryIndexSettingType) => { + setSummaryIndexSetting({ ...summaryIndexSettingRef.current, ...payload }) + summaryIndexSettingRef.current = { ...summaryIndexSettingRef.current, ...payload } + }, []) // Parent-child config const [parentChildConfig, setParentChildConfig] = useState(defaultParentChildConfig) @@ -134,6 +141,7 @@ export const useSegmentationState = (options: UseSegmentationStateOptions = {}) }, }, mode: 'hierarchical', + summary_index_setting: summaryIndexSettingRef.current, } as ProcessRule } @@ -147,6 +155,7 @@ export const useSegmentationState = (options: UseSegmentationStateOptions = {}) }, }, mode: segmentationType, + summary_index_setting: summaryIndexSettingRef.current, } as ProcessRule }, [rules, parentChildConfig, segmentIdentifier, maxChunkLength, overlap, segmentationType]) @@ -204,6 +213,8 @@ export const useSegmentationState = (options: UseSegmentationStateOptions = {}) defaultConfig, setDefaultConfig, toggleRule, + summaryIndexSetting, + handleSummaryIndexSettingChange, // Parent-child config parentChildConfig, diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index b4d2c5f6e9..a77d829488 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -65,7 +65,9 @@ const StepTwo: FC = ({ // Custom hooks const segmentation = useSegmentationState({ initialSegmentationType: currentDataset?.doc_form === ChunkingMode.parentChild ? ProcessMode.parentChild : ProcessMode.general, + initialSummaryIndexSetting: currentDataset?.summary_index_setting, }) + const showSummaryIndexSetting = !currentDataset const indexing = useIndexingConfig({ initialIndexType: propsIndexingType, initialEmbeddingModel: currentDataset?.embedding_model ? { provider: currentDataset.embedding_model_provider, model: currentDataset.embedding_model } : undefined, @@ -156,7 +158,7 @@ const StepTwo: FC = ({ }) if (!isValid) return - const params = creation.buildCreationParams(currentDocForm, docLanguage, segmentation.getProcessRule(currentDocForm), indexing.retrievalConfig, indexing.embeddingModel, indexing.getIndexingTechnique()) + const params = creation.buildCreationParams(currentDocForm, docLanguage, segmentation.getProcessRule(currentDocForm), indexing.retrievalConfig, indexing.embeddingModel, indexing.getIndexingTechnique(), segmentation.summaryIndexSetting) if (!params) return await creation.executeCreation(params, indexing.indexType, indexing.retrievalConfig) @@ -217,6 +219,9 @@ const StepTwo: FC = ({ onPreview={updatePreview} onReset={segmentation.resetToDefaults} locale={locale} + showSummaryIndexSetting={showSummaryIndexSetting} + summaryIndexSetting={segmentation.summaryIndexSetting} + onSummaryIndexSettingChange={segmentation.handleSummaryIndexSettingChange} /> )} {showParentChildOption && ( @@ -236,6 +241,9 @@ const StepTwo: FC = ({ onRuleToggle={segmentation.toggleRule} onPreview={updatePreview} onReset={segmentation.resetToDefaults} + showSummaryIndexSetting={showSummaryIndexSetting} + summaryIndexSetting={segmentation.summaryIndexSetting} + onSummaryIndexSettingChange={segmentation.handleSummaryIndexSettingChange} /> )} diff --git a/web/app/components/datasets/documents/components/list.tsx b/web/app/components/datasets/documents/components/list.tsx index 01d4afb646..f63d6d987e 100644 --- a/web/app/components/datasets/documents/components/list.tsx +++ b/web/app/components/datasets/documents/components/list.tsx @@ -30,12 +30,13 @@ import { useDatasetDetailContextWithSelector as useDatasetDetailContext } from ' import useTimestamp from '@/hooks/use-timestamp' import { ChunkingMode, DataSourceType, DocumentActionType } from '@/models/datasets' import { DatasourceType } from '@/models/pipeline' -import { useDocumentArchive, useDocumentBatchRetryIndex, useDocumentDelete, useDocumentDisable, useDocumentDownloadZip, useDocumentEnable } from '@/service/knowledge/use-document' +import { useDocumentArchive, useDocumentBatchRetryIndex, useDocumentDelete, useDocumentDisable, useDocumentDownloadZip, useDocumentEnable, useDocumentSummary } from '@/service/knowledge/use-document' import { asyncRunSafe } from '@/utils' import { cn } from '@/utils/classnames' import { downloadBlob } from '@/utils/download' import { formatNumber } from '@/utils/format' import BatchAction from '../detail/completed/common/batch-action' +import SummaryStatus from '../detail/completed/common/summary-status' import StatusItem from '../status-item' import s from '../style.module.css' import Operations from './operations' @@ -219,6 +220,7 @@ const DocumentList: FC = ({ onSelectedIdChange(uniq([...selectedIds, ...localDocs.map(doc => doc.id)])) }, [isAllSelected, localDocs, onSelectedIdChange, selectedIds]) const { mutateAsync: archiveDocument } = useDocumentArchive() + const { mutateAsync: generateSummary } = useDocumentSummary() const { mutateAsync: enableDocument } = useDocumentEnable() const { mutateAsync: disableDocument } = useDocumentDisable() const { mutateAsync: deleteDocument } = useDocumentDelete() @@ -232,6 +234,9 @@ const DocumentList: FC = ({ case DocumentActionType.archive: opApi = archiveDocument break + case DocumentActionType.summary: + opApi = generateSummary + break case DocumentActionType.enable: opApi = enableDocument break @@ -444,6 +449,13 @@ const DocumentList: FC = ({ > {doc.name} + { + doc.summary_index_status && ( +
+ +
+ ) + }
= ({ className="absolute bottom-16 left-0 z-20" selectedIds={selectedIds} onArchive={handleAction(DocumentActionType.archive)} + onBatchSummary={handleAction(DocumentActionType.summary)} onBatchEnable={handleAction(DocumentActionType.enable)} onBatchDisable={handleAction(DocumentActionType.disable)} onBatchDownload={downloadableSelectedIds.length > 0 ? handleBatchDownload : undefined} diff --git a/web/app/components/datasets/documents/components/operations.tsx b/web/app/components/datasets/documents/components/operations.tsx index ee638c5e12..d3dcc23121 100644 --- a/web/app/components/datasets/documents/components/operations.tsx +++ b/web/app/components/datasets/documents/components/operations.tsx @@ -21,6 +21,7 @@ import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' import Confirm from '@/app/components/base/confirm' import Divider from '@/app/components/base/divider' +import { SearchLinesSparkle } from '@/app/components/base/icons/src/vender/knowledge' import CustomPopover from '@/app/components/base/popover' import Switch from '@/app/components/base/switch' import { ToastContext } from '@/app/components/base/toast' @@ -34,6 +35,7 @@ import { useDocumentEnable, useDocumentPause, useDocumentResume, + useDocumentSummary, useDocumentUnArchive, useSyncDocument, useSyncWebsite, @@ -87,6 +89,7 @@ const Operations = ({ const { mutateAsync: downloadDocument, isPending: isDownloading } = useDocumentDownload() const { mutateAsync: syncDocument } = useSyncDocument() const { mutateAsync: syncWebsite } = useSyncWebsite() + const { mutateAsync: generateSummary } = useDocumentSummary() const { mutateAsync: pauseDocument } = useDocumentPause() const { mutateAsync: resumeDocument } = useDocumentResume() const isListScene = scene === 'list' @@ -112,6 +115,9 @@ const Operations = ({ else opApi = syncWebsite break + case 'summary': + opApi = generateSummary + break case 'pause': opApi = pauseDocument break @@ -257,6 +263,10 @@ const Operations = ({ {t('list.action.sync', { ns: 'datasetDocuments' })}
)} +
onOperate('summary')}> + + {t('list.action.summary', { ns: 'datasetDocuments' })} +
)} diff --git a/web/app/components/datasets/documents/detail/completed/common/batch-action.tsx b/web/app/components/datasets/documents/detail/completed/common/batch-action.tsx index 2de72d9ff6..486ba2ffdf 100644 --- a/web/app/components/datasets/documents/detail/completed/common/batch-action.tsx +++ b/web/app/components/datasets/documents/detail/completed/common/batch-action.tsx @@ -6,6 +6,7 @@ import { useTranslation } from 'react-i18next' import Button from '@/app/components/base/button' import Confirm from '@/app/components/base/confirm' import Divider from '@/app/components/base/divider' +import { SearchLinesSparkle } from '@/app/components/base/icons/src/vender/knowledge' import { cn } from '@/utils/classnames' const i18nPrefix = 'batchAction' @@ -16,6 +17,7 @@ type IBatchActionProps = { onBatchDisable: () => void onBatchDownload?: () => void onBatchDelete: () => Promise + onBatchSummary?: () => void onArchive?: () => void onEditMetadata?: () => void onBatchReIndex?: () => void @@ -27,6 +29,7 @@ const BatchAction: FC = ({ selectedIds, onBatchEnable, onBatchDisable, + onBatchSummary, onBatchDownload, onArchive, onBatchDelete, @@ -84,7 +87,16 @@ const BatchAction: FC = ({ {t('metadata.metadata', { ns: 'dataset' })} )} - + {onBatchSummary && ( + + )} {onArchive && (