opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/parent-child-retrieval' of https://github.com/langgenius/dify into feat/parent-child-retrieval

This commit is contained in:
twwu 2024-12-11 18:18:06 +08:00
parent 51f6a87aef d88bb70427
commit db01c5c89d
18 changed files with 627 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
web/app/components/base/file-uploader/file-type-icon.tsx
View File

@ -82,8 +82,8 @@ const FileTypeIcon = ({
size = 'sm',
className,
}: FileTypeIconProps) => {
const Icon = FILE_TYPE_ICON_MAP[type].component || FileAppearanceTypeEnum.custom
const color = FILE_TYPE_ICON_MAP[type].color
const Icon = FILE_TYPE_ICON_MAP[type]?.component || FileAppearanceTypeEnum.document
const color = FILE_TYPE_ICON_MAP[type]?.color || FILE_TYPE_ICON_MAP[FileAppearanceTypeEnum.document].color
return <Icon className={cn('shrink-0', SizeMap[size], color, className)} />
}

2
web/app/components/datasets/create/step-two/index.module.css
View File

@ -76,7 +76,7 @@
}
.disabled {
cursor: not-allowed;
cursor: not-allowed !important;
}
.indexItem.disabled:hover {

528
web/app/components/datasets/create/step-two/index.tsx
View File

@ -1,6 +1,6 @@
'use client'
import type { FC, PropsWithChildren } from 'react'
import React, { useCallback, useEffect, useState } from 'react'
import React, { useCallback, useEffect, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
import {
@ -10,6 +10,7 @@ import {
} from '@remixicon/react'
import Link from 'next/link'
import Image from 'next/image'
import { useHover } from 'ahooks'
import SettingCog from '../assets/setting-gear-mod.svg'
import OrangeEffect from '../assets/option-card-effect-orange.svg'
import FamilyMod from '../assets/family-mod.svg'
@ -58,6 +59,8 @@ import { getNotionInfo, getWebsiteInfo, useCreateDocument, useCreateFirstDocumen
import Badge from '@/app/components/base/badge'
import { SkeletonContanier, SkeletonPoint, SkeletonRectangle, SkeletonRow } from '@/app/components/base/skeleton'
import Tooltip from '@/app/components/base/tooltip'
import CustomDialog from '@/app/components/base/dialog'
import { PortalToFollowElem, PortalToFollowElemContent, PortalToFollowElemTrigger } from '@/app/components/base/portal-to-follow-elem'
const TextLabel: FC<PropsWithChildren> = (props) => {
return <label className='text-text-secondary text-xs font-semibold leading-none'>{props.children}</label>
@ -175,16 +178,21 @@ const StepTwo = ({
)
// QA Related
const [isLanguageSelectDisabled, setIsLanguageSelectDisabled] = useState(false)
const [isLanguageSelectDisabled, _setIsLanguageSelectDisabled] = useState(false)
const [isQAConfirmDialogOpen, setIsQAConfirmDialogOpen] = useState(false)
const [docForm, setDocForm] = useState<ChuckingMode>(
(datasetId && documentDetail) ? documentDetail.doc_form as ChuckingMode : ChuckingMode.text,
)
const handleChangeDocform = (value: ChuckingMode) => {
if (value === ChuckingMode.qa && indexType === IndexingType.ECONOMICAL) {
setIsQAConfirmDialogOpen(true)
return
}
if (value === ChuckingMode.parentChild && indexType === IndexingType.ECONOMICAL)
setIndexType(IndexingType.QUALIFIED)
setDocForm(value)
// eslint-disable-next-line @typescript-eslint/no-use-before-define
currentEstimateMutation.reset()
if (value === ChuckingMode.parentChild)
setIndexType(IndexingType.QUALIFIED)
}
const [docLanguage, setDocLanguage] = useState<string>(
@ -513,14 +521,11 @@ const StepTwo = ({
}
const changeToEconomicalType = () => {
if (docForm === ChuckingMode.parentChild)
if (docForm !== ChuckingMode.text)
return
if (!hasSetIndexType) {
if (!hasSetIndexType)
setIndexType(IndexingType.ECONOMICAL)
if (docForm === ChuckingMode.qa)
handleChangeDocform(ChuckingMode.text)
}
}
useEffect(() => {
@ -556,6 +561,9 @@ const StepTwo = ({
score_threshold: 0.5,
} as RetrievalConfig)
const economyDomRef = useRef<HTMLDivElement>(null)
const isHoveringEconomy = useHover(economyDomRef)
return (
<div className='flex w-full max-h-full h-full overflow-y-auto'>
<div className='relative h-full w-full overflow-y-scroll'>
@ -563,226 +571,231 @@ const StepTwo = ({
<div className={s.label}>{t('datasetCreation.stepTwo.segmentation')}</div>
<div className='max-w-[640px]'>
<div className='space-y-4'>
<OptionCard
title={t('datasetCreation.stepTwo.general')}
icon={<Image src={SettingCog} alt={t('datasetCreation.stepTwo.general')} />}
activeHeaderClassName='bg-gradient-to-r from-[#EFF0F9] to-[#F9FAFB]'
description={t('datasetCreation.stepTwo.generalTip')}
isActive={
[ChuckingMode.text, ChuckingMode.qa].includes(docForm)
}
onSwitched={() =>
handleChangeDocform(ChuckingMode.text)
}
actions={
<>
<Button variant={'secondary-accent'} onClick={() => updatePreview()}>
<RiSearchEyeLine className='h-4 w-4 mr-1.5' />
{t('datasetCreation.stepTwo.previewChunk')}
</Button>
<Button variant={'ghost'} onClick={resetRules}>
{t('datasetCreation.stepTwo.reset')}
</Button>
</>
}
>
<div className='space-y-4'>
<div className='flex gap-3'>
<DelimiterInput
value={segmentIdentifier}
onChange={e => setSegmentIdentifier(e.target.value)}
/>
<MaxLengthInput
value={maxChunkLength}
onChange={setMaxChunkLength}
/>
<OverlapInput
value={overlap}
min={1}
onChange={setOverlap}
/>
</div>
<div className='space-y-2'>
<div className='w-full flex flex-col'>
<TextLabel>{t('datasetCreation.stepTwo.rules')}</TextLabel>
<div className='mt-4 space-y-2'>
{rules.map(rule => (
<div key={rule.id} className={s.ruleItem} onClick={() => {
ruleChangeHandle(rule.id)
}}>
<Checkbox
checked={rule.enabled}
/>
<label className="ml-2 text-sm font-normal cursor-pointer text-gray-800">{getRuleName(rule.id)}</label>
</div>
))}
</div>
</div>
</div>
{IS_CE_EDITION && <>
<div className='flex items-center'>
<Checkbox
checked={docForm === ChuckingMode.qa}
onCheck={() => {
if (docForm === ChuckingMode.qa)
handleChangeDocform(ChuckingMode.text)
else
handleChangeDocform(ChuckingMode.qa)
}}
className='mr-2'
/>
<div className='flex items-center gap-1'>
<TextLabel>
{t('datasetCreation.stepTwo.QALanguage')}
</TextLabel>
<div className='z-50 relative'>
<LanguageSelect
currentLanguage={docLanguage || locale}
onSelect={setDocLanguage}
disabled={isLanguageSelectDisabled}
/>
</div>
<Tooltip popupContent={t('datasetCreation.stepTwo.QATip')} />
</div>
</div>
{docForm === ChuckingMode.qa && (
<div
style={{
background: 'linear-gradient(92deg, rgba(247, 144, 9, 0.1) 0%, rgba(255, 255, 255, 0.00) 100%)',
}}
className='h-10 flex items-center gap-2 rounded-xl border-components-panel-border border shadow-shadow-shadow-3 px-3 text-xs'
>
<RiAlertFill className='size-4 text-text-warning-secondary' />
<span className='text-sm font-medium text-text-primary'>
{t('datasetCreation.stepTwo.QATip')}
</span>
</div>
)}
</>}
</div>
</OptionCard>
<OptionCard
title={t('datasetCreation.stepTwo.parentChild')}
icon={<Image src={FamilyMod} alt={t('datasetCreation.stepTwo.parentChild')} />}
effectImg={OrangeEffect.src}
activeHeaderClassName='bg-gradient-to-r from-[#F9F1EE] to-[#F9FAFB]'
description={t('datasetCreation.stepTwo.parentChildTip')}
isActive={docForm === ChuckingMode.parentChild}
onSwitched={() => handleChangeDocform(ChuckingMode.parentChild)}
actions={
<>
<Button variant={'secondary-accent'} onClick={() => updatePreview()}>
<RiSearchEyeLine className='h-4 w-4 mr-1.5' />
{t('datasetCreation.stepTwo.previewChunk')}
</Button>
<Button variant={'ghost'} onClick={resetRules}>
{t('datasetCreation.stepTwo.reset')}
</Button>
</>
}
>
<div className='space-y-4'>
<div className='space-y-2'>
<TextLabel>
{t('datasetCreation.stepTwo.parentChunkForContext')}
</TextLabel>
<RadioCard
icon={<Image src={Note} alt='' />}
title={t('datasetCreation.stepTwo.paragraph')}
description={t('datasetCreation.stepTwo.paragraphTip')}
isChosen={parentChildConfig.chunkForContext === 'paragraph'}
onChosen={() => setParentChildConfig(
{
...parentChildConfig,
chunkForContext: 'paragraph',
},
)}
chosenConfig={
<div className='flex gap-2'>
<DelimiterInput
value={parentChildConfig.parent.delimiter}
onChange={e => setParentChildConfig({
...parentChildConfig,
parent: {
...parentChildConfig.parent,
delimiter: e.target.value,
},
})}
/>
<MaxLengthInput
value={parentChildConfig.parent.maxLength}
onChange={value => setParentChildConfig({
...parentChildConfig,
parent: {
...parentChildConfig.parent,
maxLength: value,
},
})}
/>
</div>
}
/>
<RadioCard
icon={<Image src={FileList} alt='' />}
title={t('datasetCreation.stepTwo.fullDoc')}
description={t('datasetCreation.stepTwo.fullDocTip')}
onChosen={() => setParentChildConfig(
{
...parentChildConfig,
chunkForContext: 'full-doc',
},
)}
isChosen={parentChildConfig.chunkForContext === 'full-doc'}
/>
</div>
{(!datasetId || [ChuckingMode.text, ChuckingMode.qa].includes(docForm))
&& <OptionCard
title={t('datasetCreation.stepTwo.general')}
icon={<Image src={SettingCog} alt={t('datasetCreation.stepTwo.general')} />}
activeHeaderClassName='bg-gradient-to-r from-[#EFF0F9] to-[#F9FAFB]'
description={t('datasetCreation.stepTwo.generalTip')}
isActive={
[ChuckingMode.text, ChuckingMode.qa].includes(docForm)
}
onSwitched={() =>
handleChangeDocform(ChuckingMode.text)
}
actions={
<>
<Button variant={'secondary-accent'} onClick={() => updatePreview()}>
<RiSearchEyeLine className='h-4 w-4 mr-1.5' />
{t('datasetCreation.stepTwo.previewChunk')}
</Button>
<Button variant={'ghost'} onClick={resetRules}>
{t('datasetCreation.stepTwo.reset')}
</Button>
</>
}
noHighlight={Boolean(datasetId)}
>
<div className='space-y-4'>
<TextLabel>
{t('datasetCreation.stepTwo.childChunkForRetrieval')}
</TextLabel>
<div className='flex gap-3 mt-2'>
<div className='flex gap-3'>
<DelimiterInput
value={parentChildConfig.child.delimiter}
onChange={e => setParentChildConfig({
...parentChildConfig,
child: {
...parentChildConfig.child,
delimiter: e.target.value,
},
})}
value={segmentIdentifier}
onChange={e => setSegmentIdentifier(e.target.value)}
/>
<MaxLengthInput
value={parentChildConfig.child.maxLength}
onChange={value => setParentChildConfig({
...parentChildConfig,
child: {
...parentChildConfig.child,
maxLength: value,
value={maxChunkLength}
onChange={setMaxChunkLength}
/>
<OverlapInput
value={overlap}
min={1}
onChange={setOverlap}
/>
</div>
<div className='space-y-2'>
<div className='w-full flex flex-col'>
<TextLabel>{t('datasetCreation.stepTwo.rules')}</TextLabel>
<div className='mt-4 space-y-2'>
{rules.map(rule => (
<div key={rule.id} className={s.ruleItem} onClick={() => {
ruleChangeHandle(rule.id)
}}>
<Checkbox
checked={rule.enabled}
/>
<label className="ml-2 text-sm font-normal cursor-pointer text-gray-800">{getRuleName(rule.id)}</label>
</div>
))}
</div>
</div>
</div>
{IS_CE_EDITION && <>
<div className='flex items-center'>
<Checkbox
checked={docForm === ChuckingMode.qa}
onCheck={() => {
if (docForm === ChuckingMode.qa)
handleChangeDocform(ChuckingMode.text)
else
handleChangeDocform(ChuckingMode.qa)
}}
className='mr-2'
/>
<div className='flex items-center gap-1'>
<TextLabel>
{t('datasetCreation.stepTwo.QALanguage')}
</TextLabel>
<div className='z-50 relative'>
<LanguageSelect
currentLanguage={docLanguage || locale}
onSelect={setDocLanguage}
disabled={isLanguageSelectDisabled}
/>
</div>
<Tooltip popupContent={t('datasetCreation.stepTwo.QATip')} />
</div>
</div>
{docForm === ChuckingMode.qa && (
<div
style={{
background: 'linear-gradient(92deg, rgba(247, 144, 9, 0.1) 0%, rgba(255, 255, 255, 0.00) 100%)',
}}
className='h-10 flex items-center gap-2 rounded-xl border-components-panel-border border shadow-shadow-shadow-3 px-3 text-xs'
>
<RiAlertFill className='size-4 text-text-warning-secondary' />
<span className='text-sm font-medium text-text-primary'>
{t('datasetCreation.stepTwo.QATip')}
</span>
</div>
)}
</>}
</div>
</OptionCard>}
{
(!datasetId || docForm === ChuckingMode.parentChild)
&& <OptionCard
title={t('datasetCreation.stepTwo.parentChild')}
icon={<Image src={FamilyMod} alt={t('datasetCreation.stepTwo.parentChild')} />}
effectImg={OrangeEffect.src}
activeHeaderClassName='bg-gradient-to-r from-[#F9F1EE] to-[#F9FAFB]'
description={t('datasetCreation.stepTwo.parentChildTip')}
isActive={docForm === ChuckingMode.parentChild}
onSwitched={() => handleChangeDocform(ChuckingMode.parentChild)}
actions={
<>
<Button variant={'secondary-accent'} onClick={() => updatePreview()}>
<RiSearchEyeLine className='h-4 w-4 mr-1.5' />
{t('datasetCreation.stepTwo.previewChunk')}
</Button>
<Button variant={'ghost'} onClick={resetRules}>
{t('datasetCreation.stepTwo.reset')}
</Button>
</>
}
noHighlight={Boolean(datasetId)}
>
<div className='space-y-4'>
<div className='space-y-2'>
<TextLabel>
{t('datasetCreation.stepTwo.parentChunkForContext')}
</TextLabel>
<RadioCard
icon={<Image src={Note} alt='' />}
title={t('datasetCreation.stepTwo.paragraph')}
description={t('datasetCreation.stepTwo.paragraphTip')}
isChosen={parentChildConfig.chunkForContext === 'paragraph'}
onChosen={() => setParentChildConfig(
{
...parentChildConfig,
chunkForContext: 'paragraph',
},
})}
)}
chosenConfig={
<div className='flex gap-2'>
<DelimiterInput
value={parentChildConfig.parent.delimiter}
onChange={e => setParentChildConfig({
...parentChildConfig,
parent: {
...parentChildConfig.parent,
delimiter: e.target.value,
},
})}
/>
<MaxLengthInput
value={parentChildConfig.parent.maxLength}
onChange={value => setParentChildConfig({
...parentChildConfig,
parent: {
...parentChildConfig.parent,
maxLength: value,
},
})}
/>
</div>
}
/>
<RadioCard
icon={<Image src={FileList} alt='' />}
title={t('datasetCreation.stepTwo.fullDoc')}
description={t('datasetCreation.stepTwo.fullDocTip')}
onChosen={() => setParentChildConfig(
{
...parentChildConfig,
chunkForContext: 'full-doc',
},
)}
isChosen={parentChildConfig.chunkForContext === 'full-doc'}
/>
</div>
<div className='space-y-2'>
<div className='space-y-4'>
<TextLabel>
{t('datasetCreation.stepTwo.rules')}
{t('datasetCreation.stepTwo.childChunkForRetrieval')}
</TextLabel>
<div className='space-y-2 mt-2'>
{rules.map(rule => (
<div key={rule.id} className={s.ruleItem} onClick={() => {
ruleChangeHandle(rule.id)
}}>
<Checkbox
checked={rule.enabled}
/>
<label className="ml-2 text-sm font-normal cursor-pointer text-gray-800">{getRuleName(rule.id)}</label>
</div>
))}
<div className='flex gap-3 mt-2'>
<DelimiterInput
value={parentChildConfig.child.delimiter}
onChange={e => setParentChildConfig({
...parentChildConfig,
child: {
...parentChildConfig.child,
delimiter: e.target.value,
},
})}
/>
<MaxLengthInput
value={parentChildConfig.child.maxLength}
onChange={value => setParentChildConfig({
...parentChildConfig,
child: {
...parentChildConfig.child,
maxLength: value,
},
})}
/>
</div>
<div className='space-y-2'>
<TextLabel>
{t('datasetCreation.stepTwo.rules')}
</TextLabel>
<div className='space-y-2 mt-2'>
{rules.map(rule => (
<div key={rule.id} className={s.ruleItem} onClick={() => {
ruleChangeHandle(rule.id)
}}>
<Checkbox
checked={rule.enabled}
/>
<label className="ml-2 text-sm font-normal cursor-pointer text-gray-800">{getRuleName(rule.id)}</label>
</div>
))}
</div>
</div>
</div>
</div>
</div>
</OptionCard>
</OptionCard>}
</div>
</div>
<Divider className='my-5' />
@ -825,26 +838,69 @@ const StepTwo = ({
)}
{(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && (
<div
className={cn(
s.radioItem,
s.indexItem,
!hasSetIndexType && indexType === IndexingType.ECONOMICAL && s.active,
hasSetIndexType && s.disabled,
hasSetIndexType && '!w-full !min-h-[96px]',
docForm === ChuckingMode.parentChild && s.disabled,
)}
onClick={changeToEconomicalType}
<PortalToFollowElem
open={
isHoveringEconomy && docForm !== ChuckingMode.text
}
placement={'top'}
>
<div className='h-8 p-1.5 bg-white rounded-lg border border-components-panel-border-subtle justify-center items-center inline-flex absolute left-5 top-[18px]'>
<Image src={indexMethodIcon.economical} alt='Economical Icon' width={20} height={20} />
</div>
{!hasSetIndexType && <span className={cn(s.radio)} />}
<div className={s.typeHeader}>
<div className={s.title}>{t('datasetCreation.stepTwo.economical')}</div>
<div className={s.tip}>{t('datasetCreation.stepTwo.economicalTip')}</div>
</div>
</div>
<PortalToFollowElemTrigger>
<div
className={cn(
s.radioItem,
s.indexItem,
!hasSetIndexType && indexType === IndexingType.ECONOMICAL && s.active,
hasSetIndexType && s.disabled,
hasSetIndexType && '!w-full !min-h-[96px]',
docForm !== ChuckingMode.text && s.disabled,
)}
onClick={changeToEconomicalType}
ref={economyDomRef}
>
<CustomDialog show={isQAConfirmDialogOpen} onClose={() => setIsQAConfirmDialogOpen(false)} className='w-[432px]'>
<header className='pt-6 mb-4'>
<h2 className='text-lg font-semibold'>
{t('datasetCreation.stepTwo.qaSwitchHighQualityTipTitle')}
</h2>
<p className='font-normal text-sm mt-2'>
{t('datasetCreation.stepTwo.qaSwitchHighQualityTipContent')}
</p>
</header>
<div className='flex gap-2 pb-6'>
<Button className='ml-auto' onClick={() => {
setIsQAConfirmDialogOpen(false)
}}>
{t('datasetCreation.stepTwo.cancel')}
</Button>
<Button variant={'primary'} onClick={() => {
setIsQAConfirmDialogOpen(false)
setIndexType(IndexingType.QUALIFIED)
setDocForm(ChuckingMode.qa)
}}>
{t('datasetCreation.stepTwo.switch')}
</Button>
</div>
</CustomDialog>
<div className='h-8 p-1.5 bg-white rounded-lg border border-components-panel-border-subtle justify-center items-center inline-flex absolute left-5 top-[18px]'>
<Image src={indexMethodIcon.economical} alt='Economical Icon' width={20} height={20} />
</div>
{!hasSetIndexType && <span className={cn(s.radio)} />}
<div className={s.typeHeader}>
<div className={s.title}>{t('datasetCreation.stepTwo.economical')}</div>
<div className={s.tip}>{t('datasetCreation.stepTwo.economicalTip')}</div>
</div>
</div>
</PortalToFollowElemTrigger>
<PortalToFollowElemContent>
<div className='p-3 bg-white text-xs font-medium text-text-secondary rounded-lg shadow-lg'>
{
docForm === ChuckingMode.qa
? t('datasetCreation.stepTwo.notAvailableForQA')
: t('datasetCreation.stepTwo.notAvailableForParentChild')
}
</div>
</PortalToFollowElemContent>
</PortalToFollowElem>
)}
</div>
{hasSetIndexType && indexType === IndexingType.ECONOMICAL && (
@ -937,7 +993,7 @@ const StepTwo = ({
>
<div className='flex items-center gap-2'>
<PreviewDocumentPicker
files={files.map(file => ({ name: file.name!, id: file.id!, extension: 'pdf' }))}
files={files as Array<Required<CustomFile>>}
onChange={(selected) => {
currentEstimateMutation.reset()
setPreviewFile(selected)

7
web/app/components/datasets/create/step-two/option-card.tsx
View File

@ -51,14 +51,15 @@ type OptionCardProps = {
actions?: ReactNode
effectImg?: string
onSwitched?: () => void
noHighlight?: boolean
} & Omit<ComponentProps<'div'>, 'title'>
export const OptionCard: FC<OptionCardProps> = (props) => {
const { icon, className, title, description, isActive, children, actions, activeHeaderClassName, style, effectImg, onSwitched, onClick, ...rest } = props
const { icon, className, title, description, isActive, children, actions, activeHeaderClassName, style, effectImg, onSwitched, onClick, noHighlight, ...rest } = props
return <div
className={classNames(
'rounded-xl',
isActive ? 'border-components-option-card-option-selected-border bg-components-panel-bg' : 'border-components-option-card-option-border bg-components-option-card-option-bg',
(isActive && !noHighlight) ? 'border-components-option-card-option-selected-border bg-components-panel-bg' : 'border-components-option-card-option-border bg-components-option-card-option-bg',
className,
)}
style={{
@ -75,7 +76,7 @@ export const OptionCard: FC<OptionCardProps> = (props) => {
icon={icon}
title={title}
description={description}
isActive={isActive}
isActive={isActive && !noHighlight}
activeClassName={activeHeaderClassName}
effectImg={effectImg}
/>

5
web/app/components/datasets/documents/detail/completed/common/tag.tsx
View File

@ -1,8 +1,9 @@
import React from 'react'
import cn from '@/utils/classnames'
const Tag = ({ text }: { text: string }) => {
const Tag = ({ text, className }: { text: string; className?: string }) => {
return (
<div className='inline-flex items-center gap-x-0.5'>
<div className={cn('inline-flex items-center gap-x-0.5', className)}>
<span className='text-text-quaternary text-xs font-medium'>#</span>
<span className='text-text-tertiary text-xs'>{text}</span>
</div>

37
web/app/components/datasets/documents/detail/completed/index.tsx
View File

@ -42,21 +42,22 @@ type SegmentListContextValue = {
const SegmentListContext = createContext({
isCollapsed: true,
toggleCollapsed: () => {},
toggleCollapsed: () => { },
fullScreen: false,
toggleFullScreen: () => {},
toggleFullScreen: () => { },
})
export const useSegmentListContext = (selector: (value: SegmentListContextValue) => any) => {
return useContextSelector(SegmentListContext, selector)
}
export const SegmentIndexTag: FC<{ positionId?: string | number; label?: string; className?: string }> = React.memo(({ positionId, label, className }) => {
export const SegmentIndexTag: FC<{ positionId?: string | number; label?: string; className?: string; isParentChildRetrieval?: boolean }> = React.memo(({ positionId, label, className, isParentChildRetrieval }) => {
const prefix = `${isParentChildRetrieval ? 'Parent-' : ''}Chunk`
const localPositionId = useMemo(() => {
const positionIdStr = String(positionId)
if (positionIdStr.length >= 3)
return `Chunk-${positionId}`
return `Chunk-${positionIdStr.padStart(2, '0')}`
return `${prefix}-${positionId}`
return `${prefix}-${positionIdStr.padStart(2, '0')}`
}, [positionId])
return (
<div className={cn('flex items-center', className)}>
@ -179,7 +180,7 @@ const Completed: FC<ICompletedProps> = ({
setSegments([])
setSelectedSegmentIds([])
invalidSegmentList()
// eslint-disable-next-line react-hooks/exhaustive-deps
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
const onClickCard = (detail: SegmentDetailModel, isEditMode = false) => {
@ -209,7 +210,7 @@ const Completed: FC<ICompletedProps> = ({
notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') })
},
})
// eslint-disable-next-line react-hooks/exhaustive-deps
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [datasetId, documentId, selectedSegmentIds, segments])
const { mutateAsync: deleteSegment } = useDeleteSegment()
@ -225,7 +226,7 @@ const Completed: FC<ICompletedProps> = ({
notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') })
},
})
// eslint-disable-next-line react-hooks/exhaustive-deps
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [datasetId, documentId, selectedSegmentIds])
const handleUpdateSegment = async (
@ -337,7 +338,7 @@ const Completed: FC<ICompletedProps> = ({
resetList()
currentPage !== totalPages && setCurrentPage(totalPages)
}
// eslint-disable-next-line react-hooks/exhaustive-deps
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [segmentListData, limit, currentPage])
return (
@ -388,7 +389,7 @@ const Completed: FC<ICompletedProps> = ({
/>
<ChildSegmentList
childChunks={childSegments}
handleInputChange={() => {}}
handleInputChange={() => { }}
enabled={!archived}
/>
</div>
@ -443,14 +444,14 @@ const Completed: FC<ICompletedProps> = ({
</FullScreenDrawer>
{/* Batch Action Buttons */}
{selectedSegmentIds.length > 0
&& <BatchAction
className='absolute left-0 bottom-16 z-20'
selectedIds={selectedSegmentIds}
onBatchEnable={onChangeSwitch.bind(null, true, '')}
onBatchDisable={onChangeSwitch.bind(null, false, '')}
onBatchDelete={onDelete.bind(null, '')}
onCancel={onCancelBatchOperation}
/>}
&& <BatchAction
className='absolute left-0 bottom-16 z-20'
selectedIds={selectedSegmentIds}
onBatchEnable={onChangeSwitch.bind(null, true, '')}
onBatchDisable={onChangeSwitch.bind(null, false, '')}
onBatchDelete={onDelete.bind(null, '')}
onCancel={onCancelBatchOperation}
/>}
</SegmentListContext.Provider>
)
}

23
web/app/components/datasets/hit-testing/assets/test-data.ts
View File

@ -1,4 +1,6 @@
export const generalResultData = [
import type { HitTesting } from '@/models/datasets'
export const generalResultData: HitTesting[] = [
{
segment: {
id: 'b621b153-f8a7-4e85-bd3d-07feaf61bd9e',
@ -40,8 +42,19 @@ export const generalResultData = [
doc_type: null,
},
},
child_chunks: null,
score: 0.8771945,
child_chunks: [
{
id: '1',
score: 0.8771945,
content: 'It is quite natural for academics who are continuously told to “publish or perish” to want to always create something from scratch that is their own fresh creation.',
},
{
id: '2',
score: 0.5,
content: 'It is quite natural for ',
},
],
score: 0.99,
tsne_position: null,
},
{
@ -86,7 +99,7 @@ export const generalResultData = [
},
},
child_chunks: null,
score: 0.8642928,
score: 1,
tsne_position: null,
},
{
@ -131,7 +144,7 @@ export const generalResultData = [
},
},
child_chunks: null,
score: 0.80618876,
score: 0.2,
tsne_position: null,
},
]

30
web/app/components/datasets/hit-testing/components/child-chunks-item.tsx Normal file
View File

@ -0,0 +1,30 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import { SliceContent } from '../../formatted-text/flavours/shared'
import Score from './score'
import type { HitTestingChildChunk } from '@/models/datasets'
type Props = {
payload: HitTestingChildChunk
isShowAll: boolean
}
const ChildChunks: FC<Props> = ({
payload,
isShowAll,
}) => {
const { id, score, content } = payload
return (
<div
className={!isShowAll ? 'line-clamp-2' : ''}
>
<div className='inline-flex items-center relative top-[-3px]'>
<div className='flex items-center h-[24px] bg-state-accent-solid system-2xs-semibold-uppercase text-text-primary-on-surface px-1'>C-{id}</div>
<Score value={score} besideChunkName />
</div>
<SliceContent className='bg-state-accent-hover group-hover:bg-state-accent-hover text-text-secondary font-normal'>{content}</SliceContent>
</div>
)
}
export default React.memo(ChildChunks)

89
web/app/components/datasets/hit-testing/components/chunk-detail-modal.tsx Normal file
View File

@ -0,0 +1,89 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import { useTranslation } from 'react-i18next'
import { SegmentIndexTag } from '../../documents/detail/completed'
import Dot from '../../documents/detail/completed/common/dot'
import Score from './score'
import ChildChunksItem from './child-chunks-item'
import Modal from '@/app/components/base/modal'
import type { HitTesting } from '@/models/datasets'
import FileIcon from '@/app/components/base/file-uploader/file-type-icon'
import type { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader/types'
import cn from '@/utils/classnames'
import Tag from '@/app/components/datasets/documents/detail/completed/common/tag'
const i18nPrefix = 'datasetHitTesting'
type Props = {
payload: HitTesting
onHide: () => void
}
const ChunkDetailModal: FC<Props> = ({
payload,
onHide,
}) => {
const { t } = useTranslation()
const { segment, score, child_chunks } = payload
const { position, content, keywords, document } = segment
const isParentChildRetrieval = !!(child_chunks && child_chunks.length > 0)
const extension = document.name.split('.').slice(-1)[0] as FileAppearanceTypeEnum
const maxHeighClassName = 'max-h-[min(752px,_80vh)] overflow-y-auto'
return (
<Modal
title={t(`${i18nPrefix}.chunkDetail`)}
isShow
closable
onClose={onHide}
className={cn(isParentChildRetrieval ? '!min-w-[1200px]' : '!min-w-[720px]')}
>
<div className='mt-4 flex pb-6'>
<div>
{/* Meta info */}
<div className='flex justify-between items-center'>
<div className='grow flex items-center space-x-2'>
<SegmentIndexTag
isParentChildRetrieval={isParentChildRetrieval}
positionId={position}
className={cn('w-fit group-hover:opacity-100')}
/>
<Dot />
<div className='grow flex items-center space-x-1'>
<FileIcon type={extension} size='sm' />
<span className='grow w-0 truncate text-text-secondary text-[13px] font-normal'>{document.name}</span>
</div>
</div>
<Score value={score} />
</div>
<div className={cn('mt-2 body-md-regular text-text-secondary', maxHeighClassName)}>
{content}
</div>
{!isParentChildRetrieval && keywords && keywords.length > 0 && (
<div className='mt-6'>
<div className='font-medium text-xs text-text-tertiary uppercase'>{t(`${i18nPrefix}.keyword`)}</div>
<div className='mt-1 flex flex-wrap'>
{keywords.map(keyword => (
<Tag key={keyword} text={keyword} className='mr-2' />
))}
</div>
</div>
)}
</div>
{isParentChildRetrieval && (
<div className='shrink-0 w-[552px] px-6'>
<div className='system-xs-semibold-uppercase text-text-secondary'>{t(`${i18nPrefix}.hitChunks`, { num: child_chunks.length })}</div>
<div className={cn('mt-1 space-y-2', maxHeighClassName)}>
{child_chunks.map(item => (
<ChildChunksItem key={item.id} payload={item} isShowAll />
))}
</div>
</div>
)}
</div>
</Modal>
)
}
export default React.memo(ChunkDetailModal)

94
web/app/components/datasets/hit-testing/components/result-item.tsx
View File

@ -2,9 +2,20 @@
import type { FC } from 'react'
import React from 'react'
import { useTranslation } from 'react-i18next'
import { RiArrowDownSLine, RiArrowRightSLine, RiArrowRightUpLine } from '@remixicon/react'
import { useBoolean } from 'ahooks'
import { SegmentIndexTag } from '../../documents/detail/completed/common/segment-index-tag'
import Dot from '../../documents/detail/completed/common/dot'
import Score from './score'
import ChildChunkItem from './child-chunks-item'
import ChunkDetailModal from './chunk-detail-modal'
import type { HitTesting } from '@/models/datasets'
import cn from '@/utils/classnames'
import FileIcon from '@/app/components/base/file-uploader/file-type-icon'
import type { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader/types'
import Tag from '@/app/components/datasets/documents/detail/completed/common/tag'
const i18nPrefix = 'datasetHitTesting'
type Props = {
payload: HitTesting
}
@ -13,21 +24,88 @@ const ResultItem: FC<Props> = ({
payload,
}) => {
const { t } = useTranslation()
const { segment } = payload
const { position, word_count } = segment
const { segment, score, child_chunks } = payload
const { position, word_count, content, keywords, document } = segment
const isParentChildRetrieval = !!(child_chunks && child_chunks.length > 0)
const extension = document.name.split('.').slice(-1)[0] as FileAppearanceTypeEnum
const [isFold, {
toggle: toggleFold,
}] = useBoolean(false)
const Icon = isFold ? RiArrowRightSLine : RiArrowDownSLine
const [isShowDetailModal, {
setTrue: showDetailModal,
setFalse: hideDetailModal,
}] = useBoolean(false)
return (
<div>
<div className='flex justify-between items-center'>
<div className='pt-3 bg-chat-bubble-bg rounded-xl hover:shadow-lg'>
{/* Meta info */}
<div className='flex justify-between items-center px-3'>
<div className='flex items-center space-x-2'>
<SegmentIndexTag positionId={position} className={cn('w-fit group-hover:opacity-100')} />
<div className='text-xs font-medium text-text-quaternary'>·</div>
<SegmentIndexTag
isParentChildRetrieval={isParentChildRetrieval}
positionId={position}
className={cn('w-fit group-hover:opacity-100')}
/>
<Dot />
<div className='system-xs-medium text-text-tertiary'>{word_count} {t('datasetDocuments.segment.characters')}</div>
</div>
{/* Score */}
<Score value={score} />
</div>
</div>
{/* Main */}
<div className='mt-1 px-3'>
<div className='line-clamp-2 body-md-regular'>{content}</div>
{isParentChildRetrieval && (
<div className='mt-1'>
<div className={cn('inline-flex items-center h-6 space-x-0.5 text-text-secondary select-none rounded-lg cursor-pointer', isFold && 'pl-1 bg-[linear-gradient(90deg,_rgba(200,_206,_218,_0.20)_0%,_rgba(200,_206,_218,_0.04)_100%)]')} onClick={toggleFold}>
<Icon className={cn('w-4 h-4', isFold && 'opacity-50')} />
<div className='text-xs font-semibold uppercase'>{t(`${i18nPrefix}.hitChunks`, { num: child_chunks.length })}</div>
</div>
{!isFold && (
<div className='space-y-2'>
{child_chunks.map(item => (
<div key={item.id} className='ml-[7px] pl-[7px] border-l-[2px] border-text-accent-secondary'>
<ChildChunkItem payload={item} isShowAll={false} />
</div>
))}
</div>
)}
</div>
)}
{!isParentChildRetrieval && keywords && keywords.length > 0 && (
<div className='mt-2 flex flex-wrap'>
{keywords.map(keyword => (
<Tag key={keyword} text={keyword} className='mr-2' />
))}
</div>
)}
</div>
{/* Foot */}
<div className='mt-3 flex justify-between items-center h-10 pl-3 pr-2 border-t border-divider-subtle'>
<div className='grow flex items-center space-x-1'>
<FileIcon type={extension} size='sm' />
<span className='grow w-0 truncate text-text-secondary text-[13px] font-normal'>{document.name}</span>
</div>
<div
className='flex items-center space-x-1 cursor-pointer text-text-tertiary'
onClick={showDetailModal}
>
<div className='text-xs uppercase'>{t(`${i18nPrefix}.open`)}</div>
<RiArrowRightUpLine className='size-3.5' />
</div>
</div>
{
isShowDetailModal && (
<ChunkDetailModal
payload={payload}
onHide={hideDetailModal}
/>
)
}
</div >
)
}
export default React.memo(ResultItem)

25
web/app/components/datasets/hit-testing/components/score.tsx Normal file
View File

@ -0,0 +1,25 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import cn from '@/utils/classnames'
type Props = {
value: number
besideChunkName?: boolean
}
const Score: FC<Props> = ({
value,
besideChunkName,
}) => {
return (
<div className={cn('relative items-center px-[5px] border border-components-progress-bar-border overflow-hidden', besideChunkName ? 'border-l-0 h-[25px]' : 'h-[20px] rounded-md')}>
<div className={cn('absolute top-0 left-0 h-full bg-util-colors-blue-brand-blue-brand-100 border-r-[1.5px] border-components-progress-brand-progress', value === 1 && 'border-r-0')} style={{ width: `${value * 100}%` }} />
<div className={cn('relative flex items-center h-full space-x-0.5 text-util-colors-blue-brand-blue-brand-700')}>
<div className='system-2xs-medium-uppercase'>score</div>
<div className='system-xs-semibold'>{value.toFixed(2)}</div>
</div>
</div>
)
}
export default React.memo(Score)

87
web/app/components/datasets/hit-testing/index.tsx
View File

@ -11,7 +11,7 @@ import Textarea from './textarea'
import s from './style.module.css'
import HitDetail from './hit-detail'
import ModifyRetrievalModal from './modify-retrieval-modal'
import { generalResultData } from './assets/test-data'
import ResultItem from './components/result-item'
import cn from '@/utils/classnames'
import type { ExternalKnowledgeBaseHitTestingResponse, ExternalKnowledgeBaseHitTesting as ExternalKnowledgeBaseHitTestingType, HitTestingResponse, HitTesting as HitTestingType } from '@/models/datasets'
import Loading from '@/app/components/base/loading'
@ -24,6 +24,8 @@ import DatasetDetailContext from '@/context/dataset-detail'
import type { RetrievalConfig } from '@/types/app'
import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
import useTimestamp from '@/hooks/use-timestamp'
import docStyle from '@/app/components/datasets/documents/detail/completed/style.module.css'
const limit = 10
type Props = {
@ -64,55 +66,35 @@ const HitTesting: FC<Props> = ({ datasetId }: Props) => {
const total = recordsRes?.total || 0
const onClickCard = (detail: HitTestingType) => {
setCurrParagraph({ paraInfo: detail, showModal: true })
}
const onClickExternalCard = (detail: ExternalKnowledgeBaseHitTestingType) => {
setExternalCurrParagraph({ paraInfo: detail, showModal: true })
}
const { dataset: currentDataset } = useContext(DatasetDetailContext)
const isExternal = currentDataset?.provider === 'external'
const [retrievalConfig, setRetrievalConfig] = useState(currentDataset?.retrieval_model_dict as RetrievalConfig)
const [isShowModifyRetrievalModal, setIsShowModifyRetrievalModal] = useState(false)
const [isShowRightPanel, { setTrue: showRightPanel, setFalse: hideRightPanel, set: setShowRightPanel }] = useBoolean(!isMobile)
const renderHitResults = (results: any[], onClickCard: (record: any) => void) => (
<>
<div className='text-gray-600 font-semibold mb-4'>{t('datasetHitTesting.hit.title')}</div>
<div className='overflow-auto flex-1'>
<div className={s.cardWrapper}>
{results.map((record, idx) => (
<SegmentCard
key={idx}
loading={false}
refSource={{
title: record.title,
uri: record.metadata ? record.metadata['x-amz-bedrock-kb-source-uri'] : '',
}}
isExternal={isExternal}
detail={record.segment}
contentExternal={record.content}
score={record.score}
scene='hitTesting'
className='h-[216px] mb-4'
onClick={() => onClickCard(record)}
/>
))}
</div>
const renderHitResults = (results: any[]) => (
<div className='h-full flex flex-col py-3 px-4 rounded-t-2xl bg-background-body'>
<div className='shrink-0 pl-2 text-text-primary font-semibold leading-6 mb-2'>
{t('datasetHitTesting.hit.title', { num: results.length })}
</div>
</>
<div className='grow overflow-y-auto space-y-2'>
{results.map((record, idx) => (
<ResultItem
key={idx}
payload={record}
/>
))}
</div>
</div>
)
const renderEmptyState = () => (
// for test
<div></div>
// <div className='h-full flex flex-col justify-center items-center'>
// <div className={cn(docStyle.commonIcon, docStyle.targetIcon, '!bg-gray-200 !h-14 !w-14')} />
// <div className='text-gray-300 text-[13px] mt-3'>
// {t('datasetHitTesting.hit.emptyTip')}
// </div>
// </div>
<div className='h-full flex flex-col justify-center items-center py-3 px-4 rounded-t-2xl bg-background-body'>
<div className={cn(docStyle.commonIcon, docStyle.targetIcon, '!bg-text-quaternary !h-14 !w-14')} />
<div className='text-text-quaternary text-[13px] mt-3'>
{t('datasetHitTesting.hit.emptyTip')}
</div>
</div>
)
useEffect(() => {
@ -190,30 +172,23 @@ const HitTesting: FC<Props> = ({ datasetId }: Props) => {
)}
</div>
<FloatRightContainer panelClassname='!justify-start !overflow-y-auto' showClose isMobile={isMobile} isOpen={isShowRightPanel} onClose={hideRightPanel} footer={null}>
<div className={cn(s.rightDiv, 'p-0 sm:px-8 sm:pt-[42px] sm:pb-[26px]')}>
{renderHitResults(generalResultData, onClickCard)}
<div className={cn(s.rightDiv, 'pt-3')}>
{/* {renderHitResults(generalResultData)} */}
{submitLoading
? <div className={s.cardWrapper}>
<SegmentCard
loading={true}
scene='hitTesting'
className='h-[216px]'
/>
<SegmentCard
loading={true}
scene='hitTesting'
className='h-[216px]'
/>
</div>
? <SegmentCard
loading={true}
scene='hitTesting'
className='h-[216px]'
/>
: (
(() => {
if (!hitResult?.records.length && !externalHitResult?.records.length)
return renderEmptyState()
if (hitResult?.records.length)
return renderHitResults(hitResult.records, onClickCard)
return renderHitResults(hitResult.records)
return renderHitResults(externalHitResult?.records || [], onClickExternalCard)
return renderHitResults(externalHitResult?.records || [])
})()
)
}

2
web/app/components/datasets/hit-testing/style.module.css
View File

@ -5,7 +5,7 @@
@apply flex-1 h-full;
}
.leftDiv {
@apply border-r border-gray-100 px-6 py-3 flex flex-col;
@apply px-6 py-3 flex flex-col;
}
.rightDiv {
@apply flex flex-col;

5
web/i18n/en-US/dataset-creation.ts
View File

@ -166,6 +166,11 @@ const translation = {
datasetSettingLink: 'Knowledge settings.',
previewChunkTip: 'Click the \'Preview Chunk\' button on the left to load the preview',
previewChunkCount: '{{count}} Estimated chunks',
switch: 'Switch',
qaSwitchHighQualityTipTitle: 'Q&A Format Requires High-quality Indexing Method',
qaSwitchHighQualityTipContent: 'Currently, only high-quality index method supports Q&A format chunking. Would you like to switch to high-quality mode?',
notAvailableForParentChild: 'Not available for Parent-child Index',
notAvailableForQA: 'Not available for Q&A Index',
},
stepThree: {
creationTitle: '🎉 Knowledge created',

6
web/i18n/en-US/dataset-hit-testing.ts
View File

@ -19,12 +19,16 @@ const translation = {
testing: 'Testing',
},
hit: {
title: 'RETRIEVAL PARAGRAPHS',
title: '{{num}} Retrieved Chunks',
emptyTip: 'Retrieval Testing results will show here',
},
noRecentTip: 'No recent query results here',
viewChart: 'View VECTOR CHART',
viewDetail: 'View Detail',
chunkDetail: 'Chunk Detail',
hitChunks: 'Hit {{num}} child chunks',
open: 'Open',
keyword: 'Keywords',
}
export default translation

5
web/i18n/zh-Hans/dataset-creation.ts
View File

@ -166,6 +166,11 @@ const translation = {
datasetSettingLink: '知识库设置。',
previewChunkTip: '点击左侧的“预览块”按钮来加载预览',
previewChunkCount: '{{count}} 预估块',
switch: '切换',
qaSwitchHighQualityTipTitle: 'Q&A 格式需要高质量的索引方法',
qaSwitchHighQualityTipContent: '目前，只有高质量的索引方法支持 Q&A 格式分块。您要切换到高质量模式吗？',
notAvailableForParentChild: '不支持父子索引',
notAvailableForQA: '不支持 Q&A 索引',
},
stepThree: {
creationTitle: '🎉 知识库已创建',

6
web/i18n/zh-Hans/dataset-hit-testing.ts
View File

@ -19,12 +19,16 @@ const translation = {
testing: '测试',
},
hit: {
title: '召回段落',
title: '{{num}} 个召回段落',
emptyTip: '召回测试结果将展示在这里',
},
noRecentTip: '最近无查询结果',
viewChart: '查看向量图表',
viewDetail: '查看详情',
chunkDetail: '段落详情',
hitChunks: '命中 {{num}} 个子段落',
open: '打开',
keyword: '关键词',
}
export default translation

6
web/models/datasets.ts
View File

@ -479,10 +479,16 @@ export type HitTestingRecord = {
created_at: number
}
export type HitTestingChildChunk = {
id: string
content: string
score: number
}
export type HitTesting = {
segment: Segment
score: number
tsne_position: TsnePosition
child_chunks?: HitTestingChildChunk[] | null
}
export type ExternalKnowledgeBaseHitTesting = {