mirror of https://github.com/langgenius/dify.git
Merge branch 'feat/parent-child-retrieval' of https://github.com/langgenius/dify into feat/parent-child-retrieval
This commit is contained in:
twwu
commit
c9114d171e
|
|
@ -20,6 +20,8 @@ import { PreviewContainer } from '../../preview/container'
|
|||
import { ChunkContainer, QAPreview } from '../../chunk'
|
||||
import { PreviewHeader } from '../../preview/header'
|
||||
import DocumentPicker from '../../common/document-picker'
|
||||
import { FormattedText } from '../../formatted-text/formatted'
|
||||
import { PreviewSlice } from '../../formatted-text/flavours/preview-slice'
|
||||
import s from './index.module.css'
|
||||
import unescape from './unescape'
|
||||
import escape from './escape'
|
||||
|
|
@ -27,7 +29,7 @@ import { OptionCard } from './option-card'
|
|||
import LanguageSelect from './language-select'
|
||||
import { DelimiterInput, MaxLengthInput, OverlapInput } from './inputs'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FullDocumentDetail, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
|
||||
import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
|
||||
|
||||
import Button from '@/app/components/base/button'
|
||||
import FloatRightContainer from '@/app/components/base/float-right-container'
|
||||
|
|
@ -38,7 +40,7 @@ import { ensureRerankModelSelected, isReRankModelSelected } from '@/app/componen
|
|||
import Toast from '@/app/components/base/toast'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import { DataSourceProvider } from '@/models/common'
|
||||
import { DataSourceType, DocForm } from '@/models/datasets'
|
||||
import { ChuckingMode, DataSourceType } from '@/models/datasets'
|
||||
import { useDatasetDetailContext } from '@/context/dataset-detail'
|
||||
import I18n from '@/context/i18n'
|
||||
import { RETRIEVE_METHOD } from '@/types/app'
|
||||
|
|
@ -96,7 +98,7 @@ export enum IndexingType {
|
|||
const DEFAULT_SEGMENT_IDENTIFIER = '\\n\\n'
|
||||
|
||||
type ParentChildConfig = {
|
||||
chunkForContext: 'paragraph' | 'full_doc'
|
||||
chunkForContext: ParentMode
|
||||
parent: {
|
||||
delimiter: string
|
||||
maxLength: number
|
||||
|
|
@ -111,11 +113,11 @@ const defaultParentChildConfig: ParentChildConfig = {
|
|||
chunkForContext: 'paragraph',
|
||||
parent: {
|
||||
delimiter: '\\n\\n',
|
||||
maxLength: 4000,
|
||||
maxLength: 500,
|
||||
},
|
||||
child: {
|
||||
delimiter: '\\n\\n',
|
||||
maxLength: 4000,
|
||||
maxLength: 200,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -148,7 +150,7 @@ const StepTwo = ({
|
|||
const { dataset: currentDataset, mutateDatasetRes } = useDatasetDetailContext()
|
||||
const isInCreatePage = !datasetId || (datasetId && !currentDataset?.data_source_type)
|
||||
const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : currentDataset?.data_source_type
|
||||
const [segmentationType, setSegmentationType] = useState<SegmentType>(SegmentType.AUTO)
|
||||
const [segmentationType, setSegmentationType] = useState<SegmentType>(SegmentType.CUSTOM)
|
||||
const [segmentIdentifier, doSetSegmentIdentifier] = useState(DEFAULT_SEGMENT_IDENTIFIER)
|
||||
const setSegmentIdentifier = useCallback((value: string) => {
|
||||
doSetSegmentIdentifier(value ? escape(value) : DEFAULT_SEGMENT_IDENTIFIER)
|
||||
|
|
@ -168,9 +170,14 @@ const StepTwo = ({
|
|||
|
||||
// QA Related
|
||||
const [isLanguageSelectDisabled, setIsLanguageSelectDisabled] = useState(false)
|
||||
const [docForm, setDocForm] = useState<DocForm | string>(
|
||||
(datasetId && documentDetail) ? documentDetail.doc_form : DocForm.TEXT,
|
||||
const [docForm, setDocForm] = useState<ChuckingMode>(
|
||||
(datasetId && documentDetail) ? documentDetail.doc_form as ChuckingMode : ChuckingMode.text,
|
||||
)
|
||||
const handleChangeDocform = (value: ChuckingMode) => {
|
||||
setDocForm(value)
|
||||
// eslint-disable-next-line @typescript-eslint/no-use-before-define
|
||||
currentEstimateMutation.reset()
|
||||
}
|
||||
|
||||
const [docLanguage, setDocLanguage] = useState<string>(
|
||||
(datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese'),
|
||||
|
|
@ -180,28 +187,42 @@ const StepTwo = ({
|
|||
|
||||
const getIndexing_technique = () => indexingType || indexType
|
||||
|
||||
const getProcessRule = () => {
|
||||
const processRule: ProcessRule = {
|
||||
rules: {} as any, // api will check this. It will be removed after api refactored.
|
||||
mode: segmentationType,
|
||||
const getProcessRule = (): ProcessRule => {
|
||||
if (docForm === ChuckingMode.parentChild) {
|
||||
return {
|
||||
rules: {
|
||||
pre_processing_rules: rules,
|
||||
segmentation: {
|
||||
separator: unescape(
|
||||
parentChildConfig.parent.delimiter,
|
||||
),
|
||||
max_tokens: parentChildConfig.parent.maxLength,
|
||||
chunk_overlap: overlap,
|
||||
},
|
||||
parent_mode: parentChildConfig.chunkForContext,
|
||||
subchunk_segmentation: {
|
||||
separator: parentChildConfig.child.delimiter,
|
||||
max_tokens: parentChildConfig.child.maxLength,
|
||||
},
|
||||
}, // api will check this. It will be removed after api refactored.
|
||||
mode: 'hierarchical',
|
||||
} as ProcessRule
|
||||
}
|
||||
if (segmentationType === SegmentType.CUSTOM) {
|
||||
const ruleObj = {
|
||||
return {
|
||||
rules: {
|
||||
pre_processing_rules: rules,
|
||||
segmentation: {
|
||||
separator: unescape(segmentIdentifier),
|
||||
max_tokens: maxChunkLength,
|
||||
chunk_overlap: overlap,
|
||||
},
|
||||
}
|
||||
// @ts-expect-error will be removed after api refactored.
|
||||
processRule.rules = ruleObj
|
||||
}
|
||||
return processRule
|
||||
}, // api will check this. It will be removed after api refactored.
|
||||
mode: segmentationType,
|
||||
} as ProcessRule
|
||||
}
|
||||
|
||||
const fileIndexingEstimateQuery = useFetchFileIndexingEstimateForFile({
|
||||
docForm: docForm as DocForm,
|
||||
docForm,
|
||||
docLanguage,
|
||||
dataSourceType: DataSourceType.FILE,
|
||||
files,
|
||||
|
|
@ -210,7 +231,7 @@ const StepTwo = ({
|
|||
dataset_id: datasetId!,
|
||||
})
|
||||
const notionIndexingEstimateQuery = useFetchFileIndexingEstimateForNotion({
|
||||
docForm: docForm as DocForm,
|
||||
docForm,
|
||||
docLanguage,
|
||||
dataSourceType: DataSourceType.NOTION,
|
||||
notionPages,
|
||||
|
|
@ -220,7 +241,7 @@ const StepTwo = ({
|
|||
})
|
||||
|
||||
const websiteIndexingEstimateQuery = useFetchFileIndexingEstimateForWeb({
|
||||
docForm: docForm as DocForm,
|
||||
docForm,
|
||||
docLanguage,
|
||||
dataSourceType: DataSourceType.WEB,
|
||||
websitePages,
|
||||
|
|
@ -481,29 +502,11 @@ const StepTwo = ({
|
|||
isSetting && onSave && onSave()
|
||||
}
|
||||
|
||||
const handleDocformSwitch = (isQAMode: boolean) => {
|
||||
if (isQAMode)
|
||||
setDocForm(DocForm.QA)
|
||||
else
|
||||
setDocForm(DocForm.TEXT)
|
||||
}
|
||||
|
||||
const previewSwitch = () => {
|
||||
setIsLanguageSelectDisabled(true)
|
||||
fetchEstimate()
|
||||
}
|
||||
|
||||
const handleSelect = (language: string) => {
|
||||
setDocLanguage(language)
|
||||
// Switch language, re-cutter
|
||||
if (docForm === DocForm.QA)
|
||||
previewSwitch()
|
||||
}
|
||||
|
||||
const changeToEconomicalType = () => {
|
||||
if (!hasSetIndexType) {
|
||||
setIndexType(IndexingType.ECONOMICAL)
|
||||
setDocForm(DocForm.TEXT)
|
||||
if (docForm === ChuckingMode.qa)
|
||||
handleChangeDocform(ChuckingMode.text)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -519,11 +522,6 @@ const StepTwo = ({
|
|||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (indexingType === IndexingType.ECONOMICAL && docForm === DocForm.QA)
|
||||
setDocForm(DocForm.TEXT)
|
||||
}, [indexingType, docForm])
|
||||
|
||||
useEffect(() => {
|
||||
// get indexing type by props
|
||||
if (indexingType)
|
||||
|
|
@ -557,8 +555,8 @@ const StepTwo = ({
|
|||
icon={<Image src={SettingCog} alt={t('datasetCreation.stepTwo.general')} />}
|
||||
activeHeaderClassName='bg-gradient-to-r from-[#EFF0F9] to-[#F9FAFB]'
|
||||
description={t('datasetCreation.stepTwo.generalTip')}
|
||||
isActive={SegmentType.AUTO === segmentationType}
|
||||
onClick={() => setSegmentationType(SegmentType.AUTO)}
|
||||
isActive={docForm === ChuckingMode.qa || docForm === ChuckingMode.text}
|
||||
onSelect={() => handleChangeDocform(ChuckingMode.text)}
|
||||
actions={
|
||||
<>
|
||||
<Button variant={'secondary-accent'} onClick={() => updatePreview()}>
|
||||
|
|
@ -607,12 +605,12 @@ const StepTwo = ({
|
|||
{IS_CE_EDITION && <>
|
||||
<div className='flex items-center'>
|
||||
<Checkbox
|
||||
checked={docForm === DocForm.QA}
|
||||
checked={docForm === ChuckingMode.qa}
|
||||
onCheck={() => {
|
||||
if (docForm === DocForm.QA)
|
||||
setDocForm(DocForm.TEXT)
|
||||
if (docForm === ChuckingMode.qa)
|
||||
handleChangeDocform(ChuckingMode.text)
|
||||
else
|
||||
setDocForm(DocForm.QA)
|
||||
handleChangeDocform(ChuckingMode.qa)
|
||||
}}
|
||||
className='mr-2'
|
||||
/>
|
||||
|
|
@ -630,7 +628,7 @@ const StepTwo = ({
|
|||
<Tooltip popupContent={t('datasetCreation.stepTwo.QATip')} />
|
||||
</div>
|
||||
</div>
|
||||
{docForm === DocForm.QA && (
|
||||
{docForm === ChuckingMode.qa && (
|
||||
<div
|
||||
style={{
|
||||
background: 'linear-gradient(92deg, rgba(247, 144, 9, 0.1) 0%, rgba(255, 255, 255, 0.00) 100%)',
|
||||
|
|
@ -652,8 +650,8 @@ const StepTwo = ({
|
|||
effectImg={OrangeEffect.src}
|
||||
activeHeaderClassName='bg-gradient-to-r from-[#F9F1EE] to-[#F9FAFB]'
|
||||
description={t('datasetCreation.stepTwo.parentChildTip')}
|
||||
isActive={SegmentType.CUSTOM === segmentationType}
|
||||
onClick={() => setSegmentationType(SegmentType.CUSTOM)}
|
||||
isActive={docForm === ChuckingMode.parentChild}
|
||||
onSelected={() => handleChangeDocform(ChuckingMode.parentChild)}
|
||||
actions={
|
||||
<>
|
||||
<Button variant={'secondary-accent'} onClick={() => updatePreview()}>
|
||||
|
|
@ -714,10 +712,10 @@ const StepTwo = ({
|
|||
onChosen={() => setParentChildConfig(
|
||||
{
|
||||
...parentChildConfig,
|
||||
chunkForContext: 'full_doc',
|
||||
chunkForContext: 'full-doc',
|
||||
},
|
||||
)}
|
||||
isChosen={parentChildConfig.chunkForContext === 'full_doc'}
|
||||
isChosen={parentChildConfig.chunkForContext === 'full-doc'}
|
||||
/>
|
||||
</div>
|
||||
|
||||
|
|
@ -924,24 +922,51 @@ const StepTwo = ({
|
|||
<Badge text='276 Estimated chunks' />
|
||||
</div>
|
||||
</PreviewHeader>}
|
||||
className={cn(s.previewWrap, isMobile && s.isMobile, 'relative h-full overflow-y-scroll space-y-4')}
|
||||
className={cn(s.previewWrap, isMobile && s.isMobile, 'relative h-full overflow-y-scroll')}
|
||||
mainClassName='space-y-6'
|
||||
>
|
||||
{docForm === DocForm.QA && estimate?.qa_preview && (
|
||||
{docForm === ChuckingMode.qa && estimate?.qa_preview && (
|
||||
estimate?.qa_preview.map(item => (
|
||||
<QAPreview key={item.question} qa={item} />
|
||||
))
|
||||
)}
|
||||
{docForm === DocForm.TEXT && estimate?.preview && (
|
||||
{docForm === ChuckingMode.text && estimate?.preview && (
|
||||
estimate?.preview.map((item, index) => (
|
||||
<ChunkContainer
|
||||
key={item}
|
||||
key={item.content}
|
||||
label={`Chunk-${index + 1}`}
|
||||
characterCount={item.length}
|
||||
characterCount={item.content.length}
|
||||
>
|
||||
{item}
|
||||
{item.content}
|
||||
</ChunkContainer>
|
||||
))
|
||||
)}
|
||||
{docForm === ChuckingMode.parentChild && currentEstimateMutation.data?.preview && (
|
||||
estimate?.preview?.map((item, index) => {
|
||||
const indexForLabel = index + 1
|
||||
return (
|
||||
<ChunkContainer
|
||||
key={item.content}
|
||||
label={`Chunk-${indexForLabel}`}
|
||||
characterCount={item.content.length}
|
||||
>
|
||||
<FormattedText>
|
||||
{item.child_chunks.map((child, index) => {
|
||||
const indexForLabel = index + 1
|
||||
return (
|
||||
<PreviewSlice
|
||||
key={child}
|
||||
label={`C-${indexForLabel}`}
|
||||
text={child}
|
||||
tooltip={`Child-chunk-${indexForLabel} · ${child.length} Characters`}
|
||||
/>
|
||||
)
|
||||
})}
|
||||
</FormattedText>
|
||||
</ChunkContainer>
|
||||
)
|
||||
})
|
||||
)}
|
||||
{currentEstimateMutation.isIdle && (
|
||||
<div className='h-full w-full flex items-center justify-center'>
|
||||
<div className='flex flex-col items-center justify-center gap-3'>
|
||||
|
|
|
|||
|
|
@ -50,10 +50,11 @@ type OptionCardProps = {
|
|||
isActive?: boolean
|
||||
actions?: ReactNode
|
||||
effectImg?: string
|
||||
onSelected?: () => void
|
||||
} & Omit<ComponentProps<'div'>, 'title'>
|
||||
|
||||
export const OptionCard: FC<OptionCardProps> = (props) => {
|
||||
const { icon, className, title, description, isActive, children, actions, activeHeaderClassName, style, effectImg, ...rest } = props
|
||||
const { icon, className, title, description, isActive, children, actions, activeHeaderClassName, style, effectImg, onSelected, onClick, ...rest } = props
|
||||
return <div
|
||||
className={classNames(
|
||||
'rounded-xl',
|
||||
|
|
@ -64,6 +65,11 @@ export const OptionCard: FC<OptionCardProps> = (props) => {
|
|||
...style,
|
||||
borderWidth: 1.5,
|
||||
}}
|
||||
onClick={(e) => {
|
||||
if (!isActive)
|
||||
onSelected?.()
|
||||
onClick?.(e)
|
||||
}}
|
||||
{...rest}>
|
||||
<OptionCardHeader
|
||||
icon={icon}
|
||||
|
|
|
|||
|
|
@ -1,14 +1,19 @@
|
|||
import React, { type FC } from 'react'
|
||||
import { RiCheckboxCircleLine, RiCloseCircleLine, RiDeleteBinLine } from '@remixicon/react'
|
||||
import { RiArchive2Line, RiCheckboxCircleLine, RiCloseCircleLine, RiDeleteBinLine } from '@remixicon/react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useBoolean } from 'ahooks'
|
||||
import Divider from '@/app/components/base/divider'
|
||||
import classNames from '@/utils/classnames'
|
||||
import Confirm from '@/app/components/base/confirm'
|
||||
|
||||
const i18nPrefix = 'dataset.batchAction'
|
||||
type IBatchActionProps = {
|
||||
className?: string
|
||||
selectedIds: string[]
|
||||
onBatchEnable: () => Promise<void>
|
||||
onBatchDisable: () => Promise<void>
|
||||
onBatchDelete: () => Promise<void>
|
||||
onBatchEnable: () => void
|
||||
onBatchDisable: () => void
|
||||
onBatchDelete: () => void
|
||||
onArchive?: () => void
|
||||
onCancel: () => void
|
||||
}
|
||||
|
||||
|
|
@ -17,9 +22,15 @@ const BatchAction: FC<IBatchActionProps> = ({
|
|||
selectedIds,
|
||||
onBatchEnable,
|
||||
onBatchDisable,
|
||||
onArchive,
|
||||
onBatchDelete,
|
||||
onCancel,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
const [isShowDeleteConfirm, {
|
||||
setTrue: showDeleteConfirm,
|
||||
setFalse: hideDeleteConfirm,
|
||||
}] = useBoolean(false)
|
||||
return (
|
||||
<div className={classNames('w-full flex justify-center gap-x-2', className)}>
|
||||
<div className='flex items-center gap-x-1 p-1 rounded-[10px] bg-components-actionbar-bg-accent border border-components-actionbar-border-accent shadow-xl shadow-shadow-shadow-5 backdrop-blur-[5px]'>
|
||||
|
|
@ -27,32 +38,53 @@ const BatchAction: FC<IBatchActionProps> = ({
|
|||
<span className='w-5 h-5 flex items-center justify-center px-1 py-0.5 bg-text-accent rounded-md text-text-primary-on-surface text-xs font-medium'>
|
||||
{selectedIds.length}
|
||||
</span>
|
||||
<span className='text-text-accent text-[13px] font-semibold leading-[16px]'>Selected</span>
|
||||
<span className='text-text-accent text-[13px] font-semibold leading-[16px]'>{t(`${i18nPrefix}.selected`)}</span>
|
||||
</div>
|
||||
<Divider type='vertical' className='mx-0.5 h-3.5 bg-divider-regular' />
|
||||
<div className='flex items-center gap-x-0.5 px-3 py-2'>
|
||||
<RiCheckboxCircleLine className='w-4 h-4 text-components-button-ghost-text' />
|
||||
<button className='px-0.5 text-components-button-ghost-text text-[13px] font-medium leading-[16px]' onClick={onBatchEnable}>
|
||||
Enable
|
||||
{t(`${i18nPrefix}.enable`)}
|
||||
</button>
|
||||
</div>
|
||||
<div className='flex items-center gap-x-0.5 px-3 py-2'>
|
||||
<RiCloseCircleLine className='w-4 h-4 text-components-button-ghost-text' />
|
||||
<button className='px-0.5 text-components-button-ghost-text text-[13px] font-medium leading-[16px]' onClick={onBatchDisable}>
|
||||
Disable
|
||||
{t(`${i18nPrefix}.disable`)}
|
||||
</button>
|
||||
</div>
|
||||
{onArchive && (
|
||||
<div className='flex items-center gap-x-0.5 px-3 py-2'>
|
||||
<RiArchive2Line className='w-4 h-4 text-components-button-ghost-text' />
|
||||
<button className='px-0.5 text-components-button-ghost-text text-[13px] font-medium leading-[16px]' onClick={onArchive}>
|
||||
{t(`${i18nPrefix}.archive`)}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
<div className='flex items-center gap-x-0.5 px-3 py-2'>
|
||||
<RiDeleteBinLine className='w-4 h-4 text-components-button-destructive-ghost-text' />
|
||||
<button className='px-0.5 text-components-button-destructive-ghost-text text-[13px] font-medium leading-[16px]' onClick={onBatchDelete}>
|
||||
Delete
|
||||
<button className='px-0.5 text-components-button-destructive-ghost-text text-[13px] font-medium leading-[16px]' onClick={showDeleteConfirm}>
|
||||
{t(`${i18nPrefix}.delete`)}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<Divider type='vertical' className='mx-0.5 h-3.5 bg-divider-regular' />
|
||||
<button className='px-3.5 py-2 text-components-button-ghost-text text-[13px] font-medium leading-[16px]' onClick={onCancel}>
|
||||
Cancel
|
||||
{t(`${i18nPrefix}.cancel`)}
|
||||
</button>
|
||||
</div>
|
||||
{
|
||||
isShowDeleteConfirm && (
|
||||
<Confirm
|
||||
isShow
|
||||
title={t('datasetDocuments.list.delete.title')}
|
||||
content={t('datasetDocuments.list.delete.content')}
|
||||
confirmText={t('common.operation.sure')}
|
||||
onConfirm={onBatchDelete}
|
||||
onCancel={hideDeleteConfirm}
|
||||
/>
|
||||
)
|
||||
}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,27 +21,28 @@ import { Globe01 } from '../../base/icons/src/vender/line/mapsAndTravel'
|
|||
import ChunkingModeLabel from '../common/chunking-mode-label'
|
||||
import s from './style.module.css'
|
||||
import RenameModal from './rename-modal'
|
||||
import BatchAction from './detail/completed/batch-action'
|
||||
import cn from '@/utils/classnames'
|
||||
import Switch from '@/app/components/base/switch'
|
||||
import Divider from '@/app/components/base/divider'
|
||||
import Popover from '@/app/components/base/popover'
|
||||
import Confirm from '@/app/components/base/confirm'
|
||||
import Tooltip from '@/app/components/base/tooltip'
|
||||
import { ToastContext } from '@/app/components/base/toast'
|
||||
import Toast, { ToastContext } from '@/app/components/base/toast'
|
||||
import type { ColorMap, IndicatorProps } from '@/app/components/header/indicator'
|
||||
import Indicator from '@/app/components/header/indicator'
|
||||
import { asyncRunSafe } from '@/utils'
|
||||
import { formatNumber } from '@/utils/format'
|
||||
import { archiveDocument, deleteDocument, disableDocument, enableDocument, syncDocument, syncWebsite, unArchiveDocument } from '@/service/datasets'
|
||||
import NotionIcon from '@/app/components/base/notion-icon'
|
||||
import ProgressBar from '@/app/components/base/progress-bar'
|
||||
import { ChuckingMode, DataSourceType, type DocumentDisplayStatus, type SimpleDocumentDetail } from '@/models/datasets'
|
||||
import { ChuckingMode, DataSourceType, DocumentActionType, type DocumentDisplayStatus, type SimpleDocumentDetail } from '@/models/datasets'
|
||||
import type { CommonResponse } from '@/models/common'
|
||||
import useTimestamp from '@/hooks/use-timestamp'
|
||||
import { useDatasetDetailContextWithSelector as useDatasetDetailContext } from '@/context/dataset-detail'
|
||||
import type { Props as PaginationProps } from '@/app/components/base/pagination'
|
||||
import Pagination from '@/app/components/base/pagination'
|
||||
import Checkbox from '@/app/components/base/checkbox'
|
||||
import { useDocumentArchive, useDocumentDelete, useDocumentDisable, useDocumentEnable, useDocumentUnArchive, useSyncDocument, useSyncWebsite } from '@/service/knowledge/use-document'
|
||||
|
||||
export const useIndexStatus = () => {
|
||||
const { t } = useTranslation()
|
||||
|
|
@ -87,6 +88,9 @@ export const StatusItem: FC<{
|
|||
const { enabled = false, archived = false, id = '' } = detail || {}
|
||||
const { notify } = useContext(ToastContext)
|
||||
const { t } = useTranslation()
|
||||
const { mutateAsync: enableDocument } = useDocumentEnable()
|
||||
const { mutateAsync: disableDocument } = useDocumentDisable()
|
||||
const { mutateAsync: deleteDocument } = useDocumentDelete()
|
||||
|
||||
const onOperate = async (operationName: OperationName) => {
|
||||
let opApi = deleteDocument
|
||||
|
|
@ -99,11 +103,11 @@ export const StatusItem: FC<{
|
|||
break
|
||||
}
|
||||
const [e] = await asyncRunSafe<CommonResponse>(opApi({ datasetId, documentId: id }) as Promise<CommonResponse>)
|
||||
if (!e)
|
||||
if (!e) {
|
||||
notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') })
|
||||
else
|
||||
notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') })
|
||||
onUpdate?.(operationName)
|
||||
onUpdate?.(operationName)
|
||||
}
|
||||
else { notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') }) }
|
||||
}
|
||||
|
||||
const { run: handleSwitch } = useDebounceFn((operationName: OperationName) => {
|
||||
|
|
@ -179,7 +183,13 @@ export const OperationAction: FC<{
|
|||
const { notify } = useContext(ToastContext)
|
||||
const { t } = useTranslation()
|
||||
const router = useRouter()
|
||||
|
||||
const { mutateAsync: archiveDocument } = useDocumentArchive()
|
||||
const { mutateAsync: unArchiveDocument } = useDocumentUnArchive()
|
||||
const { mutateAsync: enableDocument } = useDocumentEnable()
|
||||
const { mutateAsync: disableDocument } = useDocumentDisable()
|
||||
const { mutateAsync: deleteDocument } = useDocumentDelete()
|
||||
const { mutateAsync: syncDocument } = useSyncDocument()
|
||||
const { mutateAsync: syncWebsite } = useSyncWebsite()
|
||||
const isListScene = scene === 'list'
|
||||
|
||||
const onOperate = async (operationName: OperationName) => {
|
||||
|
|
@ -200,10 +210,8 @@ export const OperationAction: FC<{
|
|||
case 'sync':
|
||||
if (data_source_type === 'notion_import')
|
||||
opApi = syncDocument
|
||||
|
||||
else
|
||||
opApi = syncWebsite
|
||||
|
||||
break
|
||||
default:
|
||||
opApi = deleteDocument
|
||||
|
|
@ -211,13 +219,13 @@ export const OperationAction: FC<{
|
|||
break
|
||||
}
|
||||
const [e] = await asyncRunSafe<CommonResponse>(opApi({ datasetId, documentId: id }) as Promise<CommonResponse>)
|
||||
if (!e)
|
||||
if (!e) {
|
||||
notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') })
|
||||
else
|
||||
notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') })
|
||||
onUpdate(operationName)
|
||||
}
|
||||
else { notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') }) }
|
||||
if (operationName === 'delete')
|
||||
setDeleting(false)
|
||||
onUpdate(operationName)
|
||||
}
|
||||
|
||||
const { run: handleSwitch } = useDebounceFn((operationName: OperationName) => {
|
||||
|
|
@ -454,6 +462,37 @@ const DocumentList: FC<IDocumentListProps> = ({
|
|||
else
|
||||
onSelectedIdChange(uniq([...selectedIds, ...localDocs.map(doc => doc.id)]))
|
||||
}, [isAllSelected, localDocs, onSelectedIdChange, selectedIds])
|
||||
const { mutateAsync: archiveDocument } = useDocumentArchive()
|
||||
const { mutateAsync: enableDocument } = useDocumentEnable()
|
||||
const { mutateAsync: disableDocument } = useDocumentDisable()
|
||||
const { mutateAsync: deleteDocument } = useDocumentDelete()
|
||||
|
||||
const handleAction = (actionName: DocumentActionType) => {
|
||||
return async () => {
|
||||
let opApi = deleteDocument
|
||||
switch (actionName) {
|
||||
case DocumentActionType.archive:
|
||||
opApi = archiveDocument
|
||||
break
|
||||
case DocumentActionType.enable:
|
||||
opApi = enableDocument
|
||||
break
|
||||
case DocumentActionType.disable:
|
||||
opApi = disableDocument
|
||||
break
|
||||
default:
|
||||
opApi = deleteDocument
|
||||
break
|
||||
}
|
||||
const [e] = await asyncRunSafe<CommonResponse>(opApi({ datasetId, documentIds: selectedIds }) as Promise<CommonResponse>)
|
||||
|
||||
if (!e) {
|
||||
Toast.notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') })
|
||||
onUpdate()
|
||||
}
|
||||
else { Toast.notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') }) }
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className='relative w-full h-full overflow-x-auto'>
|
||||
|
|
@ -576,6 +615,19 @@ const DocumentList: FC<IDocumentListProps> = ({
|
|||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
{(selectedIds.length > 0) && (
|
||||
<BatchAction
|
||||
className='absolute left-0 bottom-16 z-20'
|
||||
selectedIds={selectedIds}
|
||||
onArchive={handleAction(DocumentActionType.archive)}
|
||||
onBatchEnable={handleAction(DocumentActionType.enable)}
|
||||
onBatchDisable={handleAction(DocumentActionType.disable)}
|
||||
onBatchDelete={handleAction(DocumentActionType.delete)}
|
||||
onCancel={() => {
|
||||
onSelectedIdChange([])
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
{/* Show Pagination only if the total is more than the limit */}
|
||||
{pagination.total && pagination.total > (pagination.limit || 10) && (
|
||||
<Pagination
|
||||
|
|
|
|||
|
|
@ -4,10 +4,11 @@ import classNames from '@/utils/classnames'
|
|||
|
||||
export type PreviewContainerProps = ComponentProps<'div'> & {
|
||||
header: ReactNode
|
||||
mainClassName?: string
|
||||
}
|
||||
|
||||
export const PreviewContainer: FC<PreviewContainerProps> = forwardRef((props, ref) => {
|
||||
const { children, className, header, ...rest } = props
|
||||
const { children, className, header, mainClassName, ...rest } = props
|
||||
return <div
|
||||
{...rest}
|
||||
ref={ref}
|
||||
|
|
@ -19,7 +20,7 @@ export const PreviewContainer: FC<PreviewContainerProps> = forwardRef((props, re
|
|||
<header className='py-4 pl-5 pr-3 border-b border-divider-subtle'>
|
||||
{header}
|
||||
</header>
|
||||
<main className='py-5 px-6 w-full h-full'>
|
||||
<main className={classNames('py-5 px-6 w-full h-full', mainClassName)}>
|
||||
{children}
|
||||
</main>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -150,6 +150,14 @@ const translation = {
|
|||
nTo1RetrievalLegacy: 'N-to-1 retrieval will be officially deprecated from September. It is recommended to use the latest Multi-path retrieval to obtain better results. ',
|
||||
nTo1RetrievalLegacyLink: 'Learn more',
|
||||
nTo1RetrievalLegacyLinkText: ' N-to-1 retrieval will be officially deprecated in September.',
|
||||
batchAction: {
|
||||
selected: 'Selected',
|
||||
enable: 'Enable',
|
||||
disable: 'Disable',
|
||||
archive: 'Archive',
|
||||
delete: 'Delete',
|
||||
cancel: 'Cancel',
|
||||
},
|
||||
}
|
||||
|
||||
export default translation
|
||||
|
|
|
|||
|
|
@ -150,6 +150,14 @@ const translation = {
|
|||
nTo1RetrievalLegacy: '9 月 1 日起我们将不再提供此能力，推荐使用最新的多路召回获得更好的检索效果。',
|
||||
nTo1RetrievalLegacyLink: '了解更多',
|
||||
nTo1RetrievalLegacyLinkText: '9 月 1 日起我们将不再提供此能力。',
|
||||
batchAction: {
|
||||
selected: '已选择',
|
||||
enable: '启用',
|
||||
disable: '禁用',
|
||||
archive: '归档',
|
||||
delete: '删除',
|
||||
cancel: '取消',
|
||||
},
|
||||
}
|
||||
|
||||
export default translation
|
||||
|
|
|
|||
|
|
@ -151,7 +151,7 @@ export type IndexingEstimateResponse = {
|
|||
total_price: number
|
||||
currency: string
|
||||
total_segments: number
|
||||
preview: string[]
|
||||
preview: Array<{ content: string; child_chunks: string[] }>
|
||||
qa_preview?: QA[]
|
||||
}
|
||||
|
||||
|
|
@ -304,7 +304,7 @@ export type DocumentListResponse = {
|
|||
export type DocumentReq = {
|
||||
original_document_id?: string
|
||||
indexing_technique?: string
|
||||
doc_form: 'text_model' | 'qa_model'
|
||||
doc_form: ChuckingMode
|
||||
doc_language: string
|
||||
process_rule: ProcessRule
|
||||
}
|
||||
|
|
@ -346,7 +346,7 @@ export type NotionPage = {
|
|||
}
|
||||
|
||||
export type ProcessRule = {
|
||||
mode: string
|
||||
mode: ChildChunkType | 'hierarchical'
|
||||
rules: Rules
|
||||
}
|
||||
|
||||
|
|
@ -623,3 +623,23 @@ export type ChildSegmentResponse = {
|
|||
page: number
|
||||
limit: number
|
||||
}
|
||||
|
||||
export type UpdateDocumentParams = {
|
||||
datasetId: string
|
||||
documentId: string
|
||||
}
|
||||
|
||||
// Used in api url
|
||||
export enum DocumentActionType {
|
||||
enable = 'enable',
|
||||
disable = 'disable',
|
||||
archive = 'archive',
|
||||
unArchive = 'un_archive',
|
||||
delete = 'delete',
|
||||
}
|
||||
|
||||
export type UpdateDocumentBatchParams = {
|
||||
datasetId: string
|
||||
documentId?: string
|
||||
documentIds?: string[] | string
|
||||
}
|
||||
|
|
|
|||
|
|
@ -171,34 +171,6 @@ export const resumeDocIndexing: Fetcher<CommonResponse, CommonDocReq> = ({ datas
|
|||
return patch<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/processing/resume`)
|
||||
}
|
||||
|
||||
export const deleteDocument: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
|
||||
return del<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}`)
|
||||
}
|
||||
|
||||
export const archiveDocument: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
|
||||
return patch<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/status/archive`)
|
||||
}
|
||||
|
||||
export const unArchiveDocument: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
|
||||
return patch<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/status/un_archive`)
|
||||
}
|
||||
|
||||
export const enableDocument: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
|
||||
return patch<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/status/enable`)
|
||||
}
|
||||
|
||||
export const disableDocument: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
|
||||
return patch<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/status/disable`)
|
||||
}
|
||||
|
||||
export const syncDocument: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
|
||||
return get<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/notion/sync`)
|
||||
}
|
||||
|
||||
export const syncWebsite: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
|
||||
return get<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/website-sync`)
|
||||
}
|
||||
|
||||
export const preImportNotionPages: Fetcher<{ notion_info: DataSourceNotionWorkspace[] }, { url: string; datasetId?: string }> = ({ url, datasetId }) => {
|
||||
return get<{ notion_info: DataSourceNotionWorkspace[] }>(url, { params: { dataset_id: datasetId } })
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import type { MutationOptions } from '@tanstack/react-query'
|
|||
import { useMutation } from '@tanstack/react-query'
|
||||
import { createDocument, createFirstDocument, fetchDefaultProcessRule, fetchFileIndexingEstimate } from '../datasets'
|
||||
import { type IndexingType } from '@/app/components/datasets/create/step-two'
|
||||
import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, DataSourceType, DocForm, FileIndexingEstimateResponse, IndexingEstimateParams, NotionInfo, ProcessRule, ProcessRuleResponse, createDocumentResponse } from '@/models/datasets'
|
||||
import type { ChuckingMode, CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, DataSourceType, FileIndexingEstimateResponse, IndexingEstimateParams, NotionInfo, ProcessRule, ProcessRuleResponse, createDocumentResponse } from '@/models/datasets'
|
||||
import type { DataSourceProvider, NotionPage } from '@/models/common'
|
||||
|
||||
export const getNotionInfo = (
|
||||
|
|
@ -50,7 +50,7 @@ export const getWebsiteInfo = (
|
|||
}
|
||||
|
||||
type GetFileIndexingEstimateParamsOptionBase = {
|
||||
docForm: DocForm
|
||||
docForm: ChuckingMode
|
||||
docLanguage: string
|
||||
indexingTechnique: IndexingType
|
||||
processRule: ProcessRule
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
import {
|
||||
useMutation,
|
||||
useQuery,
|
||||
} from '@tanstack/react-query'
|
||||
import { get } from '../base'
|
||||
import type { SimpleDocumentDetail } from '@/models/datasets'
|
||||
import { del, get, patch } from '../base'
|
||||
import type { SimpleDocumentDetail, UpdateDocumentBatchParams } from '@/models/datasets'
|
||||
import { DocumentActionType } from '@/models/datasets'
|
||||
import type { CommonResponse } from '@/models/common'
|
||||
|
||||
const NAME_SPACE = 'knowledge/document'
|
||||
|
||||
|
|
@ -23,3 +26,56 @@ export const useDocumentList = (payload: {
|
|||
}),
|
||||
})
|
||||
}
|
||||
|
||||
const toBatchDocumentsIdParams = (documentIds: string[] | string) => {
|
||||
const ids = Array.isArray(documentIds) ? documentIds : [documentIds]
|
||||
return ids.map(id => `document_id=${id}`).join('&')
|
||||
}
|
||||
|
||||
export const useDocumentBatchAction = (action: DocumentActionType) => {
|
||||
return useMutation({
|
||||
mutationFn: ({ datasetId, documentIds, documentId }: UpdateDocumentBatchParams) => {
|
||||
return patch<CommonResponse>(`/datasets/${datasetId}/documents/status/${action}?${toBatchDocumentsIdParams(documentId || documentIds!)}`)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export const useDocumentEnable = () => {
|
||||
return useDocumentBatchAction(DocumentActionType.enable)
|
||||
}
|
||||
|
||||
export const useDocumentDisable = () => {
|
||||
return useDocumentBatchAction(DocumentActionType.disable)
|
||||
}
|
||||
|
||||
export const useDocumentArchive = () => {
|
||||
return useDocumentBatchAction(DocumentActionType.archive)
|
||||
}
|
||||
|
||||
export const useDocumentUnArchive = () => {
|
||||
return useDocumentBatchAction(DocumentActionType.unArchive)
|
||||
}
|
||||
|
||||
export const useDocumentDelete = () => {
|
||||
return useMutation({
|
||||
mutationFn: ({ datasetId, documentIds, documentId }: UpdateDocumentBatchParams) => {
|
||||
return del<CommonResponse>(`/datasets/${datasetId}/documents?${toBatchDocumentsIdParams(documentId || documentIds!)}`)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export const useSyncDocument = () => {
|
||||
return useMutation({
|
||||
mutationFn: ({ datasetId, documentId }: UpdateDocumentBatchParams) => {
|
||||
return get<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/notion/sync`)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export const useSyncWebsite = () => {
|
||||
return useMutation({
|
||||
mutationFn: ({ datasetId, documentId }: UpdateDocumentBatchParams) => {
|
||||
return get<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/website-sync`)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue