opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/parent-child-retrieval' of https://github.com/langgenius/dify into feat/parent-child-retrieval

This commit is contained in:
twwu 2024-12-18 15:03:12 +08:00
parent 8d9a2d6d99 ade6dd4b6f
commit 7185fd2d5a
15 changed files with 446 additions and 447 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
web/app/components/base/param-item/index.tsx
View File

@ -24,30 +24,29 @@ type Props = {
const ParamItem: FC<Props> = ({ className, id, name, noTooltip, tip, step = 0.1, min = 0, max, value, enable, onChange, hasSwitch, onSwitchChange }) => {
return (
<div className={className}>
<div className="flex items-center h-8 justify-between">
<div className="flex items-center justify-between">
<div className="flex items-center">
{hasSwitch && (
<Switch
size='md'
className='mr-2'
defaultValue={enable}
onChange={async (val) => {
onSwitchChange?.(id, val)
}}
/>
)}
<span className="mx-1 text-text-secondary text-[13px] leading-[18px] font-medium">{name}</span>
<span className="mr-1 text-text-secondary system-sm-semibold">{name}</span>
{!noTooltip && (
<Tooltip
triggerClassName='w-4 h-4 shrink-0'
popupContent={<div className="w-[200px]">{tip}</div>}
/>
)}
</div>
<div className="flex items-center"></div>
</div>
<div className="mt-2 flex items-center">
<div className="mr-4 flex shrink-0 items-center">
<div className="mt-1 flex items-center">
<div className="mr-3 flex shrink-0 items-center">
<InputNumber
disabled={!enable}
type="number"
@ -62,7 +61,7 @@ const ParamItem: FC<Props> = ({ className, id, name, noTooltip, tip, step = 0.1,
className='w-[72px]'
/>
</div>
<div className="flex items-center h-7 grow">
<div className="flex items-center grow">
<Slider
className='w-full'
disabled={!enable}

21
web/app/components/base/radio-card/index.tsx
View File

@ -31,21 +31,21 @@ const RadioCard: FC<Props> = ({
return (
<div
className={cn(
'border border-components-option-card-option-border bg-components-option-card-option-bg rounded-xl hover:shadow-xs cursor-pointer',
isChosen && 'bg-components-option-card-option-selected-bg border-components-panel-border shadow-xs',
'relative p-3 border-[0.5px] border-components-option-card-option-border bg-components-option-card-option-bg rounded-xl cursor-pointer',
isChosen && 'border-[1.5px] bg-components-option-card-option-selected-bg',
className,
)}
>
<div className='flex py-3 pl-3 pr-4' onClick={onChosen}>
<div className={cn(iconBgClassName, 'mr-3 shrink-0 flex w-8 justify-center h-8 items-center rounded-lg')}>
<div className='flex gap-x-2' onClick={onChosen}>
<div className={cn(iconBgClassName, 'shrink-0 flex size-8 justify-center items-center rounded-lg shadow-md')}>
{icon}
</div>
<div className='grow'>
<div className='leading-5 text-sm font-medium text-text-secondary'>{title}</div>
<div className='leading-[18px] text-xs font-normal text-text-tertiary'>{description}</div>
<div className='system-sm-semibold text-text-secondary mb-1'>{title}</div>
<div className='system-sm-regular text-text-tertiary'>{description}</div>
</div>
{!noRadio && (
<div className='shrink-0 flex items-center h-8'>
<div className='absolute top-3 right-3'>
<div className={cn(
'w-4 h-4 border border-components-radio-border bg-components-radio-bg shadow-xs rounded-full',
isChosen && 'border-[5px] border-components-radio-border-checked',
@ -54,8 +54,11 @@ const RadioCard: FC<Props> = ({
)}
</div>
{((isChosen && chosenConfig) || noRadio) && (
<div className={cn(chosenConfigWrapClassName, 'p-3')}>
{chosenConfig}
<div className='flex gap-x-2 mt-2'>
<div className='size-8 shrink-0'></div>
<div className={cn(chosenConfigWrapClassName, 'grow')}>
{chosenConfig}
</div>
</div>
)}
</div>

2
web/app/components/datasets/common/retrieval-method-config/index.tsx
View File

@ -104,7 +104,7 @@ const RetrievalMethodConfig: FC<Props> = ({
title={
<div className='flex items-center space-x-1'>
<div>{t('dataset.retrieval.hybrid_search.title')}</div>
<Badge text={t('dataset.retrieval.hybrid_search.recommend')!} className='border-text-accent-secondary text-text-accent-secondary ml-2' uppercase />
<Badge text={t('dataset.retrieval.hybrid_search.recommend')!} className='border-text-accent-secondary text-text-accent-secondary ml-1 h-[18px]' uppercase />
</div>
}
description={t('dataset.retrieval.hybrid_search.description')} isActive={

6
web/app/components/datasets/common/retrieval-param-config/index.tsx
View File

@ -120,7 +120,7 @@ const RetrievalParamConfig: FC<Props> = ({
<div>
{!isEconomical && !isHybridSearch && (
<div>
<div className='flex h-8 items-center text-[13px] font-medium text-gray-900 space-x-2'>
<div className='flex items-center space-x-2 mb-2'>
{canToggleRerankModalEnable && (
<div
className='flex items-center'
@ -140,7 +140,7 @@ const RetrievalParamConfig: FC<Props> = ({
</div>
)}
<div className='flex items-center'>
<span className='mr-0.5 text-text-secondary'>{t('common.modelProvider.rerankModel.key')}</span>
<span className='mr-0.5 system-sm-semibold text-text-secondary'>{t('common.modelProvider.rerankModel.key')}</span>
<Tooltip
popupContent={
<div className="w-[200px]">{t('common.modelProvider.rerankModel.tip')}</div>
@ -167,7 +167,7 @@ const RetrievalParamConfig: FC<Props> = ({
)}
{
!isHybridSearch && (
<div className={cn(!isEconomical && 'mt-4', 'flex space-between space-x-6')}>
<div className={cn(!isEconomical && 'mt-4', 'flex space-between space-x-4')}>
<TopKItem
className='grow'
value={value.top_k}

13
web/app/components/datasets/create/step-two/index.module.css
View File

@ -13,17 +13,6 @@
z-index: 10;
}
.form {
@apply px-16 py-8;
}
.form .label {
@apply pb-2 flex items-center text-text-secondary;
font-weight: 500;
font-size: 16px;
line-height: 24px;
}
.segmentationItem {
min-height: 68px;
}
@ -249,7 +238,7 @@
}
.ruleItem {
@apply flex items-center;
@apply flex items-center py-1.5;
}
.formFooter {

783
web/app/components/datasets/create/step-two/index.tsx
View File

@ -63,7 +63,7 @@ import CustomDialog from '@/app/components/base/dialog'
import { PortalToFollowElem, PortalToFollowElemContent, PortalToFollowElemTrigger } from '@/app/components/base/portal-to-follow-elem'
const TextLabel: FC<PropsWithChildren> = (props) => {
return <label className='text-text-secondary text-xs font-semibold leading-none'>{props.children}</label>
return <label className='text-text-secondary system-sm-semibold'>{props.children}</label>
}
type StepTwoProps = {
@ -157,8 +157,8 @@ const StepTwo = ({
const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : currentDataset?.data_source_type
const [segmentationType, setSegmentationType] = useState<SegmentType>(SegmentType.CUSTOM)
const [segmentIdentifier, doSetSegmentIdentifier] = useState(DEFAULT_SEGMENT_IDENTIFIER)
const setSegmentIdentifier = useCallback((value: string) => {
doSetSegmentIdentifier(value ? escape(value) : DEFAULT_SEGMENT_IDENTIFIER)
const setSegmentIdentifier = useCallback((value: string, canEmpty?: boolean) => {
doSetSegmentIdentifier(value ? escape(value) : (canEmpty ? '' : DEFAULT_SEGMENT_IDENTIFIER))
}, [])
const [maxChunkLength, setMaxChunkLength] = useState(DEFAULT_MAXMIMUM_CHUNK_LENGTH) // default chunk length
const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(4000)
@ -226,11 +226,10 @@ const StepTwo = ({
parentChildConfig.parent.delimiter,
),
max_tokens: parentChildConfig.parent.maxLength,
chunk_overlap: overlap,
},
parent_mode: parentChildConfig.chunkForContext,
subchunk_segmentation: {
separator: parentChildConfig.child.delimiter,
separator: unescape(parentChildConfig.child.delimiter),
max_tokens: parentChildConfig.child.maxLength,
},
}, // api will check this. It will be removed after api refactored.
@ -550,7 +549,7 @@ const StepTwo = ({
getRulesFromDetail()
getDefaultMode()
}
// eslint-disable-next-line react-hooks/exhaustive-deps
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
useEffect(() => {
@ -578,407 +577,411 @@ const StepTwo = ({
const isHoveringEconomy = useHover(economyDomRef)
return (
<div className='flex w-full max-h-full h-full overflow-y-auto'>
<div className='relative h-full w-full overflow-y-scroll'>
<div className={cn(s.form, isMobile && '!px-4')}>
<div className={s.label}>{t('datasetCreation.stepTwo.segmentation')}</div>
<div className='max-w-[640px]'>
<div className='space-y-4'>
{(!datasetId || [ChuckingMode.text, ChuckingMode.qa].includes(currentDataset!.doc_form))
&& <OptionCard
title={t('datasetCreation.stepTwo.general')}
icon={<Image src={SettingCog} alt={t('datasetCreation.stepTwo.general')} />}
activeHeaderClassName='bg-dataset-option-card-blue-gradient'
description={t('datasetCreation.stepTwo.generalTip')}
isActive={
[ChuckingMode.text, ChuckingMode.qa].includes(
datasetId ? currentDataset!.doc_form : docForm,
)
}
onSwitched={() =>
handleChangeDocform(ChuckingMode.text)
}
actions={
<>
<Button variant={'secondary-accent'} onClick={() => updatePreview()}>
<RiSearchEyeLine className='h-4 w-4 mr-1.5' />
{t('datasetCreation.stepTwo.previewChunk')}
</Button>
<Button variant={'ghost'} onClick={resetRules}>
{t('datasetCreation.stepTwo.reset')}
</Button>
</>
}
noHighlight={Boolean(datasetId)}
>
<div className='space-y-4'>
<div className='flex gap-3'>
<DelimiterInput
value={segmentIdentifier}
onChange={e => setSegmentIdentifier(e.target.value)}
/>
<MaxLengthInput
value={maxChunkLength}
onChange={setMaxChunkLength}
/>
<OverlapInput
value={overlap}
min={1}
onChange={setOverlap}
/>
</div>
<div className='space-y-2'>
<div className='w-full flex flex-col'>
<TextLabel>{t('datasetCreation.stepTwo.rules')}</TextLabel>
<div className='mt-4 space-y-2'>
{rules.map(rule => (
<div key={rule.id} className={s.ruleItem} onClick={() => {
ruleChangeHandle(rule.id)
}}>
<Checkbox
checked={rule.enabled}
/>
<label className="ml-2 text-sm font-normal cursor-pointer text-text-secondary">{getRuleName(rule.id)}</label>
</div>
))}
</div>
</div>
</div>
{IS_CE_EDITION && <>
<div className='flex items-center'>
<Checkbox
checked={docForm === ChuckingMode.qa}
onCheck={() => {
if (docForm === ChuckingMode.qa)
handleChangeDocform(ChuckingMode.text)
else
handleChangeDocform(ChuckingMode.qa)
}}
className='mr-2'
/>
<div className='flex items-center gap-1'>
<TextLabel>
{t('datasetCreation.stepTwo.QALanguage')}
</TextLabel>
<div className='z-50 relative'>
<LanguageSelect
currentLanguage={docLanguage || locale}
onSelect={setDocLanguage}
disabled={isLanguageSelectDisabled}
/>
</div>
<Tooltip popupContent={t('datasetCreation.stepTwo.QATip')} />
</div>
</div>
{docForm === ChuckingMode.qa && (
<div
style={{
background: 'linear-gradient(92deg, rgba(247, 144, 9, 0.1) 0%, rgba(255, 255, 255, 0.00) 100%)',
}}
className='h-10 flex items-center gap-2 rounded-xl border-components-panel-border border shadow-shadow-shadow-3 px-3 text-xs'
>
<RiAlertFill className='size-4 text-text-warning-secondary' />
<span className='text-sm font-medium text-text-primary'>
{t('datasetCreation.stepTwo.QATip')}
</span>
</div>
)}
</>}
</div>
</OptionCard>}
{
(!datasetId || currentDataset!.doc_form === ChuckingMode.parentChild)
&& <OptionCard
title={t('datasetCreation.stepTwo.parentChild')}
icon={<Image src={FamilyMod} alt={t('datasetCreation.stepTwo.parentChild')} />}
effectImg={OrangeEffect.src}
activeHeaderClassName='bg-dataset-option-card-orange-gradient'
description={t('datasetCreation.stepTwo.parentChildTip')}
isActive={
datasetId ? currentDataset!.doc_form === ChuckingMode.parentChild : docForm === ChuckingMode.parentChild
}
onSwitched={() => handleChangeDocform(ChuckingMode.parentChild)}
actions={
<>
<Button variant={'secondary-accent'} onClick={() => updatePreview()}>
<RiSearchEyeLine className='h-4 w-4 mr-1.5' />
{t('datasetCreation.stepTwo.previewChunk')}
</Button>
<Button variant={'ghost'} onClick={resetRules}>
{t('datasetCreation.stepTwo.reset')}
</Button>
</>
}
noHighlight={Boolean(datasetId)}
>
<div className='space-y-4'>
<div className='space-y-2'>
<TextLabel>
{t('datasetCreation.stepTwo.parentChunkForContext')}
</TextLabel>
<RadioCard
icon={<Image src={Note} alt='' />}
title={t('datasetCreation.stepTwo.paragraph')}
description={t('datasetCreation.stepTwo.paragraphTip')}
isChosen={parentChildConfig.chunkForContext === 'paragraph'}
onChosen={() => setParentChildConfig(
{
...parentChildConfig,
chunkForContext: 'paragraph',
},
)}
chosenConfig={
<div className='flex gap-2'>
<DelimiterInput
value={parentChildConfig.parent.delimiter}
onChange={e => setParentChildConfig({
...parentChildConfig,
parent: {
...parentChildConfig.parent,
delimiter: e.target.value,
},
})}
/>
<MaxLengthInput
value={parentChildConfig.parent.maxLength}
onChange={value => setParentChildConfig({
...parentChildConfig,
parent: {
...parentChildConfig.parent,
maxLength: value,
},
})}
/>
</div>
}
/>
<RadioCard
icon={<Image src={FileList} alt='' />}
title={t('datasetCreation.stepTwo.fullDoc')}
description={t('datasetCreation.stepTwo.fullDocTip')}
onChosen={() => setParentChildConfig(
{
...parentChildConfig,
chunkForContext: 'full-doc',
},
)}
isChosen={parentChildConfig.chunkForContext === 'full-doc'}
/>
</div>
<div className='space-y-4'>
<TextLabel>
{t('datasetCreation.stepTwo.childChunkForRetrieval')}
</TextLabel>
<div className='flex gap-3 mt-2'>
<DelimiterInput
value={parentChildConfig.child.delimiter}
onChange={e => setParentChildConfig({
...parentChildConfig,
child: {
...parentChildConfig.child,
delimiter: e.target.value,
},
})}
/>
<MaxLengthInput
value={parentChildConfig.child.maxLength}
onChange={value => setParentChildConfig({
...parentChildConfig,
child: {
...parentChildConfig.child,
maxLength: value,
},
})}
/>
</div>
<div className='space-y-2'>
<TextLabel>
{t('datasetCreation.stepTwo.rules')}
</TextLabel>
<div className='space-y-2 mt-2'>
{rules.map(rule => (
<div key={rule.id} className={s.ruleItem} onClick={() => {
ruleChangeHandle(rule.id)
}}>
<Checkbox
checked={rule.enabled}
/>
<label className="ml-2 text-sm font-normal cursor-pointer text-text-secondary">{getRuleName(rule.id)}</label>
</div>
))}
</div>
</div>
</div>
</div>
</OptionCard>}
</div>
</div>
<Divider className='my-5' />
<div className={s.label}>{t('datasetCreation.stepTwo.indexMode')}</div>
<div className='max-w-[640px]'>
<div className='flex items-center gap-3 flex-wrap sm:flex-nowrap'>
{(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && (
<OptionCard
title={<p className='flex items-center'>
{t('datasetCreation.stepTwo.qualified')}
{!hasSetIndexType && <Badge className='ml-1' uppercase>{t('datasetCreation.stepTwo.recommend')}</Badge>}
<span className='ml-auto'>
{!hasSetIndexType && <span className={cn(s.radio)} />}
</span>
</p>}
description={t('datasetCreation.stepTwo.qualifiedTip')}
icon={<Image src={indexMethodIcon.high_quality} alt='' />}
isActive={!hasSetIndexType && indexType === IndexingType.QUALIFIED}
disabled={!isAPIKeySet || hasSetIndexType}
onSwitched={() => {
if (isAPIKeySet)
setIndexType(IndexingType.QUALIFIED)
}}
<div className='flex w-full max-h-full h-full'>
<div className={cn('relative h-full w-1/2 py-6 overflow-y-auto', isMobile ? 'px-4' : 'px-12')}>
<div className={'system-md-semibold mb-1'}>{t('datasetCreation.stepTwo.segmentation')}</div>
{(!datasetId || [ChuckingMode.text, ChuckingMode.qa].includes(currentDataset!.doc_form))
&& <OptionCard
className='bg-background-section mb-2'
title={t('datasetCreation.stepTwo.general')}
icon={<Image width={20} height={20} src={SettingCog} alt={t('datasetCreation.stepTwo.general')} />}
activeHeaderClassName='bg-dataset-option-card-blue-gradient'
description={t('datasetCreation.stepTwo.generalTip')}
isActive={
[ChuckingMode.text, ChuckingMode.qa].includes(
datasetId ? currentDataset!.doc_form : docForm,
)
}
onSwitched={() =>
handleChangeDocform(ChuckingMode.text)
}
actions={
<>
<Button variant={'secondary-accent'} onClick={() => updatePreview()}>
<RiSearchEyeLine className='h-4 w-4 mr-0.5' />
{t('datasetCreation.stepTwo.previewChunk')}
</Button>
<Button variant={'ghost'} onClick={resetRules}>
{t('datasetCreation.stepTwo.reset')}
</Button>
</>
}
noHighlight={Boolean(datasetId)}
>
<div className='flex flex-col gap-y-4'>
<div className='flex gap-3'>
<DelimiterInput
value={segmentIdentifier}
onChange={e => setSegmentIdentifier(e.target.value, true)}
/>
)}
{(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && (
<>
<CustomDialog show={isQAConfirmDialogOpen} onClose={() => setIsQAConfirmDialogOpen(false)} className='w-[432px]'>
<header className='pt-6 mb-4'>
<h2 className='text-lg font-semibold'>
{t('datasetCreation.stepTwo.qaSwitchHighQualityTipTitle')}
</h2>
<p className='font-normal text-sm mt-2'>
{t('datasetCreation.stepTwo.qaSwitchHighQualityTipContent')}
</p>
</header>
<div className='flex gap-2 pb-6'>
<Button className='ml-auto' onClick={() => {
setIsQAConfirmDialogOpen(false)
}}>
{t('datasetCreation.stepTwo.cancel')}
</Button>
<Button variant={'primary'} onClick={() => {
setIsQAConfirmDialogOpen(false)
setIndexType(IndexingType.QUALIFIED)
setDocForm(ChuckingMode.qa)
}}>
{t('datasetCreation.stepTwo.switch')}
</Button>
<MaxLengthInput
value={maxChunkLength}
onChange={setMaxChunkLength}
/>
<OverlapInput
value={overlap}
min={1}
onChange={setOverlap}
/>
</div>
<div className='w-full flex flex-col'>
<div className='flex items-center gap-x-2'>
<div className='inline-flex shrink-0'>
<TextLabel>{t('datasetCreation.stepTwo.rules')}</TextLabel>
</div>
<Divider className='grow' bgStyle='gradient' />
</div>
<div className='mt-1'>
{rules.map(rule => (
<div key={rule.id} className={s.ruleItem} onClick={() => {
ruleChangeHandle(rule.id)
}}>
<Checkbox
checked={rule.enabled}
/>
<label className="ml-2 system-sm-regular cursor-pointer text-text-secondary">{getRuleName(rule.id)}</label>
</div>
</CustomDialog>
<PortalToFollowElem
open={
isHoveringEconomy && docForm !== ChuckingMode.text
}
placement={'top'}
>
<PortalToFollowElemTrigger>
<OptionCard
title={t('datasetCreation.stepTwo.economical')}
description={t('datasetCreation.stepTwo.economicalTip')}
icon={<Image src={indexMethodIcon.economical} alt='' />}
isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL}
disabled={!isAPIKeySet || hasSetIndexType || docForm !== ChuckingMode.text}
ref={economyDomRef}
onSwitched={() => {
if (isAPIKeySet && docForm === ChuckingMode.text)
setIndexType(IndexingType.ECONOMICAL)
))}
{IS_CE_EDITION && <>
<div className='flex items-center'>
<Checkbox
checked={docForm === ChuckingMode.qa}
onCheck={() => {
if (docForm === ChuckingMode.qa)
handleChangeDocform(ChuckingMode.text)
else
handleChangeDocform(ChuckingMode.qa)
}}
/>
</PortalToFollowElemTrigger>
<PortalToFollowElemContent>
<div className='p-3 bg-components-tooltip-bg border-components-panel-border text-xs font-medium text-text-secondary rounded-lg shadow-lg'>
{
docForm === ChuckingMode.qa
? t('datasetCreation.stepTwo.notAvailableForQA')
: t('datasetCreation.stepTwo.notAvailableForParentChild')
}
<div className='flex items-center gap-1'>
<label className="ml-2 system-sm-regular cursor-pointer text-text-secondary">
{t('datasetCreation.stepTwo.useQALanguage')}
</label>
<div className='z-50 relative'>
<LanguageSelect
currentLanguage={docLanguage || locale}
onSelect={setDocLanguage}
disabled={isLanguageSelectDisabled}
/>
</div>
<Tooltip popupContent={t('datasetCreation.stepTwo.QATip')} />
</div>
</PortalToFollowElemContent>
</PortalToFollowElem>
</>)}
</div>
{docForm === ChuckingMode.qa && (
<div
style={{
background: 'linear-gradient(92deg, rgba(247, 144, 9, 0.1) 0%, rgba(255, 255, 255, 0.00) 100%)',
}}
className='h-10 mt-2 flex items-center gap-2 rounded-xl backdrop-blur-[5px] border-components-panel-border border shadow-xs px-3 text-xs'
>
<RiAlertFill className='size-4 text-text-warning-secondary' />
<span className='system-xs-medium text-text-primary'>
{t('datasetCreation.stepTwo.QATip')}
</span>
</div>
)}
</>}
</div>
</div>
</div>
{hasSetIndexType && indexType === IndexingType.ECONOMICAL && (
</OptionCard>}
{
(!datasetId || currentDataset!.doc_form === ChuckingMode.parentChild)
&& <OptionCard
title={t('datasetCreation.stepTwo.parentChild')}
icon={<Image width={20} height={20} src={FamilyMod} alt={t('datasetCreation.stepTwo.parentChild')} />}
effectImg={OrangeEffect.src}
activeHeaderClassName='bg-dataset-option-card-orange-gradient'
description={t('datasetCreation.stepTwo.parentChildTip')}
isActive={
datasetId ? currentDataset!.doc_form === ChuckingMode.parentChild : docForm === ChuckingMode.parentChild
}
onSwitched={() => handleChangeDocform(ChuckingMode.parentChild)}
actions={
<>
<Button variant={'secondary-accent'} onClick={() => updatePreview()}>
<RiSearchEyeLine className='h-4 w-4 mr-0.5' />
{t('datasetCreation.stepTwo.previewChunk')}
</Button>
<Button variant={'ghost'} onClick={resetRules}>
{t('datasetCreation.stepTwo.reset')}
</Button>
</>
}
noHighlight={Boolean(datasetId)}
>
<div className='flex flex-col gap-4'>
<div>
<div className='flex items-center gap-x-2'>
<div className='inline-flex shrink-0'>
<TextLabel>{t('datasetCreation.stepTwo.parentChunkForContext')}</TextLabel>
</div>
<Divider className='grow' bgStyle='gradient' />
</div>
<RadioCard className='mt-1'
icon={<Image src={Note} alt='' />}
title={t('datasetCreation.stepTwo.paragraph')}
description={t('datasetCreation.stepTwo.paragraphTip')}
isChosen={parentChildConfig.chunkForContext === 'paragraph'}
onChosen={() => setParentChildConfig(
{
...parentChildConfig,
chunkForContext: 'paragraph',
},
)}
chosenConfig={
<div className='flex gap-3'>
<DelimiterInput
value={parentChildConfig.parent.delimiter}
onChange={e => setParentChildConfig({
...parentChildConfig,
parent: {
...parentChildConfig.parent,
delimiter: e.target.value ? escape(e.target.value) : '',
},
})}
/>
<MaxLengthInput
value={parentChildConfig.parent.maxLength}
onChange={value => setParentChildConfig({
...parentChildConfig,
parent: {
...parentChildConfig.parent,
maxLength: value,
},
})}
/>
</div>
}
/>
<RadioCard className='mt-2'
icon={<Image src={FileList} alt='' />}
title={t('datasetCreation.stepTwo.fullDoc')}
description={t('datasetCreation.stepTwo.fullDocTip')}
onChosen={() => setParentChildConfig(
{
...parentChildConfig,
chunkForContext: 'full-doc',
},
)}
isChosen={parentChildConfig.chunkForContext === 'full-doc'}
/>
</div>
<div>
<div className='flex items-center gap-x-2'>
<div className='inline-flex shrink-0'>
<TextLabel>{t('datasetCreation.stepTwo.childChunkForRetrieval')}</TextLabel>
</div>
<Divider className='grow' bgStyle='gradient' />
</div>
<div className='flex gap-3 mt-1'>
<DelimiterInput
value={parentChildConfig.child.delimiter}
onChange={e => setParentChildConfig({
...parentChildConfig,
child: {
...parentChildConfig.child,
delimiter: e.target.value ? escape(e.target.value) : '',
},
})}
/>
<MaxLengthInput
value={parentChildConfig.child.maxLength}
onChange={value => setParentChildConfig({
...parentChildConfig,
child: {
...parentChildConfig.child,
maxLength: value,
},
})}
/>
</div>
</div>
<div>
<div className='flex items-center gap-x-2'>
<div className='inline-flex shrink-0'>
<TextLabel>{t('datasetCreation.stepTwo.rules')}</TextLabel>
</div>
<Divider className='grow' bgStyle='gradient' />
</div>
<div className='mt-1'>
{rules.map(rule => (
<div key={rule.id} className={s.ruleItem} onClick={() => {
ruleChangeHandle(rule.id)
}}>
<Checkbox
checked={rule.enabled}
/>
<label className="ml-2 system-sm-regular cursor-pointer text-text-secondary">{getRuleName(rule.id)}</label>
</div>
))}
</div>
</div>
</div>
</OptionCard>}
<Divider className='my-5' />
<div className={'system-md-semibold mb-1'}>{t('datasetCreation.stepTwo.indexMode')}</div>
<div className='flex items-center gap-2 flex-wrap sm:flex-nowrap'>
{(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && (
<OptionCard
title={<p className='flex items-center'>
{t('datasetCreation.stepTwo.qualified')}
{!hasSetIndexType
&& <Badge className={cn('ml-1 h-[18px]', (!hasSetIndexType && indexType === IndexingType.QUALIFIED) ? 'border-text-accent-secondary text-text-accent-secondary' : '')} uppercase>{t('datasetCreation.stepTwo.recommend')}</Badge>}
<span className='ml-auto'>
{!hasSetIndexType && <span className={cn(s.radio)} />}
</span>
</p>}
description={t('datasetCreation.stepTwo.qualifiedTip')}
icon={<Image src={indexMethodIcon.high_quality} alt='' />}
isActive={!hasSetIndexType && indexType === IndexingType.QUALIFIED}
disabled={!isAPIKeySet || hasSetIndexType}
onSwitched={() => {
if (isAPIKeySet)
setIndexType(IndexingType.QUALIFIED)
}}
/>
)}
{(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && (
<>
<CustomDialog show={isQAConfirmDialogOpen} onClose={() => setIsQAConfirmDialogOpen(false)} className='w-[432px]'>
<header className='pt-6 mb-4'>
<h2 className='text-lg font-semibold'>
{t('datasetCreation.stepTwo.qaSwitchHighQualityTipTitle')}
</h2>
<p className='font-normal text-sm mt-2'>
{t('datasetCreation.stepTwo.qaSwitchHighQualityTipContent')}
</p>
</header>
<div className='flex gap-2 pb-6'>
<Button className='ml-auto' onClick={() => {
setIsQAConfirmDialogOpen(false)
}}>
{t('datasetCreation.stepTwo.cancel')}
</Button>
<Button variant={'primary'} onClick={() => {
setIsQAConfirmDialogOpen(false)
setIndexType(IndexingType.QUALIFIED)
setDocForm(ChuckingMode.qa)
}}>
{t('datasetCreation.stepTwo.switch')}
</Button>
</div>
</CustomDialog>
<PortalToFollowElem
open={
isHoveringEconomy && docForm !== ChuckingMode.text
}
placement={'top'}
>
<PortalToFollowElemTrigger>
<OptionCard
title={t('datasetCreation.stepTwo.economical')}
description={t('datasetCreation.stepTwo.economicalTip')}
icon={<Image src={indexMethodIcon.economical} alt='' />}
isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL}
disabled={!isAPIKeySet || hasSetIndexType || docForm !== ChuckingMode.text}
ref={economyDomRef}
onSwitched={() => {
if (isAPIKeySet && docForm === ChuckingMode.text)
setIndexType(IndexingType.ECONOMICAL)
}}
/>
</PortalToFollowElemTrigger>
<PortalToFollowElemContent>
<div className='p-3 bg-components-tooltip-bg border-components-panel-border text-xs font-medium text-text-secondary rounded-lg shadow-lg'>
{
docForm === ChuckingMode.qa
? t('datasetCreation.stepTwo.notAvailableForQA')
: t('datasetCreation.stepTwo.notAvailableForParentChild')
}
</div>
</PortalToFollowElemContent>
</PortalToFollowElem>
</>)}
</div>
{hasSetIndexType && indexType === IndexingType.ECONOMICAL && (
<div className='mt-2 text-xs text-gray-500 font-medium'>
{t('datasetCreation.stepTwo.indexSettingTip')}
<Link className='text-text-accent' href={`/datasets/${datasetId}/settings`}>{t('datasetCreation.stepTwo.datasetSettingLink')}</Link>
</div>
)}
{/* Embedding model */}
{indexType === IndexingType.QUALIFIED && (
<div className='mt-5'>
<div className={cn('system-md-semibold mb-1', datasetId && 'flex justify-between items-center')}>{t('datasetSettings.form.embeddingModel')}</div>
<ModelSelector
readonly={!!datasetId}
defaultModel={embeddingModel}
modelList={embeddingModelList}
onSelect={(model: DefaultModel) => {
setEmbeddingModel(model)
}}
/>
{!!datasetId && (
<div className='mt-2 text-xs text-gray-500 font-medium'>
{t('datasetCreation.stepTwo.indexSettingTip')}
<Link className='text-text-accent' href={`/datasets/${datasetId}/settings`}>{t('datasetCreation.stepTwo.datasetSettingLink')}</Link>
</div>
)}
{/* Embedding model */}
{indexType === IndexingType.QUALIFIED && (
<div className='mt-6 my-2'>
<div className={cn(s.label, datasetId && 'flex justify-between items-center')}>{t('datasetSettings.form.embeddingModel')}</div>
<ModelSelector
readonly={!!datasetId}
defaultModel={embeddingModel}
modelList={embeddingModelList}
onSelect={(model: DefaultModel) => {
setEmbeddingModel(model)
}}
/>
{!!datasetId && (
<div className='mt-2 text-xs text-gray-500 font-medium'>
{t('datasetCreation.stepTwo.indexSettingTip')}
<Link className='text-text-accent' href={`/datasets/${datasetId}/settings`}>{t('datasetCreation.stepTwo.datasetSettingLink')}</Link>
</div>
)}
</div>
)}
<Divider className='my-5' />
{/* Retrieval Method Config */}
<div>
{!datasetId
? (
<div className={'mb-1'}>
<div className='system-md-semibold mb-0.5'>{t('datasetSettings.form.retrievalSetting.title')}</div>
<div className='body-xs-regular text-text-tertiary'>
<a target='_blank' rel='noopener noreferrer' href='https://docs.dify.ai/guides/knowledge-base/create-knowledge-and-upload-documents#id-4-retrieval-settings' className='text-text-accent'>{t('datasetSettings.form.retrievalSetting.learnMore')}</a>
{t('datasetSettings.form.retrievalSetting.longDescription')}
</div>
</div>
)
: (
<div className={cn('system-md-semibold mb-0.5', 'flex justify-between items-center')}>
<div>{t('datasetSettings.form.retrievalSetting.title')}</div>
</div>
)}
<Divider className='my-5' />
{/* Retrieval Method Config */}
<div>
{!datasetId
<div className=''>
{
getIndexing_technique() === IndexingType.QUALIFIED
? (
<div className={s.label}>
<div className='shrink-0 mr-4'>{t('datasetSettings.form.retrievalSetting.title')}</div>
<div className='leading-[18px] text-xs font-normal text-gray-500'>
<a target='_blank' rel='noopener noreferrer' href='https://docs.dify.ai/guides/knowledge-base/create-knowledge-and-upload-documents#id-4-retrieval-settings' className='text-text-accent'>{t('datasetSettings.form.retrievalSetting.learnMore')}</a>
{t('datasetSettings.form.retrievalSetting.longDescription')}
</div>
</div>
<RetrievalMethodConfig
value={retrievalConfig}
onChange={setRetrievalConfig}
/>
)
: (
<div className={cn(s.label, 'flex justify-between items-center')}>
<div>{t('datasetSettings.form.retrievalSetting.title')}</div>
</div>
)}
<div className='max-w-[640px]'>
{
getIndexing_technique() === IndexingType.QUALIFIED
? (
<RetrievalMethodConfig
value={retrievalConfig}
onChange={setRetrievalConfig}
/>
)
: (
<EconomicalRetrievalMethodConfig
value={retrievalConfig}
onChange={setRetrievalConfig}
/>
)
}
</div>
</div>
{!isSetting
? (
<div className='flex items-center mt-8 py-2'>
<Button onClick={() => onStepChange && onStepChange(-1)}>
<RiArrowLeftLine className='w-4 h-4 mr-1' />
{t('datasetCreation.stepTwo.previousStep')}
</Button>
<Button className='ml-auto' loading={isCreating} variant='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.nextStep')}</Button>
</div>
)
: (
<div className='flex items-center mt-8 py-2'>
<Button loading={isCreating} variant='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.save')}</Button>
<Button className='ml-2' onClick={onCancel}>{t('datasetCreation.stepTwo.cancel')}</Button>
</div>
)}
<EconomicalRetrievalMethodConfig
value={retrievalConfig}
onChange={setRetrievalConfig}
/>
)
}
</div>
</div>
{!isSetting
? (
<div className='flex items-center mt-8 py-2'>
<Button onClick={() => onStepChange && onStepChange(-1)}>
<RiArrowLeftLine className='w-4 h-4 mr-1' />
{t('datasetCreation.stepTwo.previousStep')}
</Button>
<Button className='ml-auto' loading={isCreating} variant='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.nextStep')}</Button>
</div>
)
: (
<div className='flex items-center mt-8 py-2'>
<Button loading={isCreating} variant='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.save')}</Button>
<Button className='ml-2' onClick={onCancel}>{t('datasetCreation.stepTwo.cancel')}</Button>
</div>
)}
</div>
<FloatRightContainer isMobile={isMobile} isOpen={true} onClose={() => { }} footer={null}>
<PreviewContainer
@ -1049,7 +1052,7 @@ const StepTwo = ({
}) as string} />
</div>
</PreviewHeader>}
className={cn(s.previewWrap, isMobile && s.isMobile, 'relative h-full overflow-y-scroll')}
className={cn('flex shrink-0 w-1/2 relative h-full overflow-y-scroll', isMobile && 'w-full max-w-[524px]')}
mainClassName='space-y-6'
>
{docForm === ChuckingMode.qa && estimate?.qa_preview && (

10
web/app/components/datasets/create/step-two/inputs.tsx
View File

@ -19,8 +19,8 @@ const FormField: FC<PropsWithChildren<{ label: ReactNode }>> = (props) => {
export const DelimiterInput: FC<InputProps> = (props) => {
const { t } = useTranslation()
return <FormField label={<div className='flex'>
{t('datasetCreation.stepTwo.separator')}
return <FormField label={<div className='flex items-center mb-1'>
<span className='system-sm-semibold mr-0.5'>{t('datasetCreation.stepTwo.separator')}</span>
<Tooltip
popupContent={
<div className='max-w-[200px]'>
@ -40,7 +40,7 @@ export const DelimiterInput: FC<InputProps> = (props) => {
export const MaxLengthInput: FC<InputNumberProps> = (props) => {
const { t } = useTranslation()
return <FormField label={<div className='h-[14px]'>
return <FormField label={<div className='system-sm-semibold mb-1'>
{t('datasetCreation.stepTwo.maxLength')}
</div>}>
<InputNumber
@ -56,8 +56,8 @@ export const MaxLengthInput: FC<InputNumberProps> = (props) => {
export const OverlapInput: FC<InputNumberProps> = (props) => {
const { t } = useTranslation()
return <FormField label={<div className='flex'>
{t('datasetCreation.stepTwo.overlap')}
return <FormField label={<div className='flex items-center mb-1'>
<span className='system-sm-semibold'>{t('datasetCreation.stepTwo.overlap')}</span>
<Tooltip
popupContent={
<div className='max-w-[200px]'>

21
web/app/components/datasets/create/step-two/option-card.tsx
View File

@ -25,16 +25,18 @@ export const OptionCardHeader: FC<OptionCardHeaderProps> = (props) => {
)}>
<div className='size-14 flex items-center justify-center relative overflow-hidden'>
{isActive && effectImg && <Image src={effectImg} className='absolute top-0 left-0 w-full h-full' alt='' width={56} height={56} />}
<div className='size-8 rounded-lg border p-1.5 shadow border-components-panel-border-subtle justify-center flex bg-background-default-dodge'>
{icon}
<div className='p-1'>
<div className='size-8 rounded-lg border p-1.5 shadow-md border-components-panel-border-subtle justify-center flex bg-background-default-dodge'>
{icon}
</div>
</div>
</div>
<TriangleArrow
className='absolute left-4 -bottom-1.5 text-components-panel-bg'
/>
<div className='flex-1 space-y-1 py-3 pr-4'>
<div className='text-text-secondary text-sm font-semibold leading-tight'>{title}</div>
<div className='text-text-tertiary text-xs font-normal leading-none'>{description}</div>
<div className='flex-1 space-y-0.5 py-3'>
<div className='text-text-secondary system-md-semibold'>{title}</div>
<div className='text-text-tertiary system-xs-regular'>{description}</div>
</div>
</div>
}
@ -57,16 +59,15 @@ export const OptionCard: FC<OptionCardProps> = forwardRef((props, ref) => {
const { icon, className, title, description, isActive, children, actions, activeHeaderClassName, style, effectImg, onSwitched, onClick, noHighlight, disabled, ...rest } = props
return <div
className={classNames(
'rounded-xl border bg-components-option-card-option-bg',
'rounded-xl bg-components-option-card-option-bg shadow-xs',
(isActive && !noHighlight)
? 'border-components-option-card-option-selected-border'
: 'border-components-option-card-option-border',
? 'border-[1.5px] border-components-option-card-option-selected-border'
: 'border border-components-option-card-option-border',
disabled && 'opacity-50',
className,
)}
style={{
...style,
borderWidth: 1.5,
}}
onClick={(e) => {
if (!isActive)
@ -85,7 +86,7 @@ export const OptionCard: FC<OptionCardProps> = forwardRef((props, ref) => {
effectImg={effectImg}
/>
{/** Body */}
{isActive && (children || actions) && <div className='p-3'>
{isActive && (children || actions) && <div className='py-3 px-4'>
{children}
{actions && <div className='flex gap-2 mt-4'>
{actions}

4
web/app/components/datasets/create/top-bar/index.tsx
View File

@ -20,7 +20,9 @@ export const Topbar: FC<TopbarProps> = (props) => {
const { t } = useTranslation()
return <div className={classNames('flex h-[52px] items-center justify-between relative border-b border-b-divider-subtle', className)}>
<Link href={'/datasets'} className="h-12 pl-2 pr-6 py-2 justify-start items-center gap-1 inline-flex">
<RiArrowLeftLine className='size-4 mr-2 text-text-primary' />
<div className='p-2'>
<RiArrowLeftLine className='size-4 text-text-primary' />
</div>
<p className="text-text-primary system-sm-semibold-uppercase">
{t('datasetCreation.steps.header.creation')}
</p>

6
web/app/components/datasets/documents/index.tsx
View File

@ -23,7 +23,7 @@ import type { CreateDocumentReq } from '@/models/datasets'
import { DataSourceType } from '@/models/datasets'
import IndexFailed from '@/app/components/datasets/common/document-status-with-action/index-failed'
import { useProviderContext } from '@/context/provider-context'
import cn from '@/utils/classnames'
const FolderPlusIcon = ({ className }: React.SVGProps<SVGElement>) => {
return <svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg" className={className ?? ''}>
<path d="M10.8332 5.83333L9.90355 3.9741C9.63601 3.439 9.50222 3.17144 9.30265 2.97597C9.12615 2.80311 8.91344 2.67164 8.6799 2.59109C8.41581 2.5 8.11668 2.5 7.51841 2.5H4.33317C3.39975 2.5 2.93304 2.5 2.57652 2.68166C2.26292 2.84144 2.00795 3.09641 1.84816 3.41002C1.6665 3.76654 1.6665 4.23325 1.6665 5.16667V5.83333M1.6665 5.83333H14.3332C15.7333 5.83333 16.4334 5.83333 16.9681 6.10582C17.4386 6.3455 17.821 6.72795 18.0607 7.19836C18.3332 7.73314 18.3332 8.4332 18.3332 9.83333V13.5C18.3332 14.9001 18.3332 15.6002 18.0607 16.135C17.821 16.6054 17.4386 16.9878 16.9681 17.2275C16.4334 17.5 15.7333 17.5 14.3332 17.5H5.6665C4.26637 17.5 3.56631 17.5 3.03153 17.2275C2.56112 16.9878 2.17867 16.6054 1.93899 16.135C1.6665 15.6002 1.6665 14.9001 1.6665 13.5V5.83333ZM9.99984 14.1667V9.16667M7.49984 11.6667H12.4998" stroke="#667085" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" />
@ -239,10 +239,10 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
<IndexFailed datasetId={datasetId} />
{embeddingAvailable && (
<Button variant='primary' onClick={routeToDocCreate} className='shrink-0'>
<PlusIcon className='h-4 w-4 mr-2 stroke-current' />
<PlusIcon className={cn('h-4 w-4 mr-2 stroke-current')} />
{isDataSourceNotion && t('datasetDocuments.list.addPages')}
{isDataSourceWeb && t('datasetDocuments.list.addUrl')}
{isDataSourceFile && t('datasetDocuments.list.addFile')}
{(!dataset?.data_source_type || isDataSourceFile) && t('datasetDocuments.list.addFile')}
</Button>
)}
</div>

6
web/app/components/datasets/hit-testing/components/child-chunks-item.tsx
View File

@ -19,11 +19,11 @@ const ChildChunks: FC<Props> = ({
<div
className={!isShowAll ? 'line-clamp-2' : ''}
>
<div className='inline-flex items-center relative top-[-3px]'>
<div className='flex items-center h-[24px] bg-state-accent-solid system-2xs-semibold-uppercase text-text-primary-on-surface px-1'>C-{position}</div>
<div className='inline-flex items-center relative top-[-2px]'>
<div className='flex items-center h-[20.5px] bg-state-accent-solid system-2xs-semibold-uppercase text-text-primary-on-surface px-1'>C-{position}</div>
<Score value={score} besideChunkName />
</div>
<SliceContent className='bg-state-accent-hover group-hover:bg-state-accent-hover text-text-secondary font-normal'>{content}</SliceContent>
<SliceContent className='py-0.5 bg-state-accent-hover group-hover:bg-state-accent-hover text-sm text-text-secondary font-normal'>{content}</SliceContent>
</div>
)
}

4
web/app/components/datasets/hit-testing/components/chunk-detail-modal.tsx
View File

@ -38,8 +38,8 @@ const ChunkDetailModal: FC<Props> = ({
onClose={onHide}
className={cn(isParentChildRetrieval ? '!min-w-[1200px]' : '!min-w-[720px]')}
>
<div className='mt-4 flex pb-6'>
<div>
<div className='mt-4 flex'>
<div className='w-full'>
{/* Meta info */}
<div className='flex justify-between items-center'>
<div className='grow flex items-center space-x-2'>

2
web/app/components/datasets/hit-testing/components/score.tsx
View File

@ -13,7 +13,7 @@ const Score: FC<Props> = ({
besideChunkName,
}) => {
return (
<div className={cn('relative items-center px-[5px] border border-components-progress-bar-border overflow-hidden', besideChunkName ? 'border-l-0 h-[25px]' : 'h-[20px] rounded-md')}>
<div className={cn('relative items-center px-[5px] border border-components-progress-bar-border overflow-hidden', besideChunkName ? 'border-l-0 h-[20.5px]' : 'h-[20px] rounded-md')}>
<div className={cn('absolute top-0 left-0 h-full bg-util-colors-blue-brand-blue-brand-100 border-r-[1.5px] border-components-progress-brand-progress', value === 1 && 'border-r-0')} style={{ width: `${value * 100}%` }} />
<div className={cn('relative flex items-center h-full space-x-0.5 text-util-colors-blue-brand-blue-brand-700')}>
<div className='system-2xs-medium-uppercase'>score</div>

1
web/i18n/en-US/dataset-creation.ts
View File

@ -136,6 +136,7 @@ const translation = {
QATitle: 'Segmenting in Question & Answer format',
QATip: 'Enable this option will consume more tokens',
QALanguage: 'Segment using',
useQALanguage: 'Chunk using Q&A format in',
estimateCost: 'Estimation',
estimateSegment: 'Estimated chunks',
segmentCount: 'chunks',

1
web/i18n/zh-Hans/dataset-creation.ts
View File

@ -136,6 +136,7 @@ const translation = {
QATitle: '采用 Q&A 分段模式',
QATip: '开启后将会消耗额外的 token',
QALanguage: '分段使用',
useQALanguage: '使用 Q&A 分段，语言',
estimateCost: '执行嵌入预估消耗',
estimateSegment: '预估分段数',
segmentCount: '段',