From 5248fcca561da176cca4135975c39a0fb739e21c Mon Sep 17 00:00:00 2001
From: twwu <twwu@dify.ai>
Date: Fri, 27 Jun 2025 15:56:38 +0800
Subject: [PATCH] feat: implement support for single and multiple choice in
 crawled result items

---
 web/app/components/base/radio/ui.tsx          |  19 +-
 .../online-documents/page-selector/index.tsx  | 332 ++++++++++++++++++
 .../base/crawled-result-item.tsx              |  23 +-
 .../website-crawl/base/crawled-result.tsx     |  22 +-
 .../website-crawl/base/crawler.tsx            |   1 +
 5 files changed, 381 insertions(+), 16 deletions(-)
 create mode 100644 web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx

diff --git a/web/app/components/base/radio/ui.tsx b/web/app/components/base/radio/ui.tsx
index 178262d0b9..19778c7cc3 100644
--- a/web/app/components/base/radio/ui.tsx
+++ b/web/app/components/base/radio/ui.tsx
@@ -5,14 +5,29 @@ import cn from '@/utils/classnames'
 
 type Props = {
   isChecked: boolean
+  disabled?: boolean
+  onCheck?: () => void
 }
 
 const RadioUI: FC<Props> = ({
   isChecked,
+  disabled = false,
+  onCheck,
 }) => {
   return (
-    <div className={cn(isChecked ? 'border-[5px] border-components-radio-border-checked' : 'border-[2px] border-components-radio-border', 'h-4 w-4  rounded-full')}>
-    </div>
+    <div
+      className={cn(
+        'size-4 rounded-full',
+        isChecked && !disabled && 'border-[5px] border-components-radio-border-checked hover:border-components-radio-border-checked-hover',
+        !isChecked && !disabled && 'border border-components-radio-border hover:border-components-radio-border-hover',
+        isChecked && disabled && 'border-[5px] border-components-radio-border-checked-disabled',
+        !isChecked && disabled && 'border border-components-radio-border-disabled bg-components-radio-bg-disabled',
+        !disabled && 'bg-components-radio-bg shadow-xs shadow-shadow-shadow-3 hover:bg-components-radio-bg-hover',
+      )}
+      onClick={() => {
+        onCheck?.()
+      }}
+    />
   )
 }
 export default React.memo(RadioUI)
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx
new file mode 100644
index 0000000000..37710201c7
--- /dev/null
+++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx
@@ -0,0 +1,332 @@
+import { memo, useMemo, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import { FixedSizeList as List, areEqual } from 'react-window'
+import type { ListChildComponentProps } from 'react-window'
+import { RiArrowDownSLine, RiArrowRightSLine } from '@remixicon/react'
+import Checkbox from '@/app/components/base/checkbox'
+import NotionIcon from '@/app/components/base/notion-icon'
+import cn from '@/utils/classnames'
+import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
+
+type PageSelectorProps = {
+  value: Set<string>
+  disabledValue: Set<string>
+  searchValue: string
+  pagesMap: DataSourceNotionPageMap
+  list: DataSourceNotionPage[]
+  onSelect: (selectedPagesId: Set<string>) => void
+  canPreview?: boolean
+  previewPageId?: string
+  onPreview?: (selectedPageId: string) => void
+}
+type NotionPageTreeItem = {
+  children: Set<string>
+  descendants: Set<string>
+  depth: number
+  ancestors: string[]
+} & DataSourceNotionPage
+type NotionPageTreeMap = Record<string, NotionPageTreeItem>
+type NotionPageItem = {
+  expand: boolean
+  depth: number
+} & DataSourceNotionPage
+
+const recursivePushInParentDescendants = (
+  pagesMap: DataSourceNotionPageMap,
+  listTreeMap: NotionPageTreeMap,
+  current: NotionPageTreeItem,
+  leafItem: NotionPageTreeItem,
+) => {
+  const parentId = current.parent_id
+  const pageId = current.page_id
+
+  if (!parentId || !pageId)
+    return
+
+  if (parentId !== 'root' && pagesMap[parentId]) {
+    if (!listTreeMap[parentId]) {
+      const children = new Set([pageId])
+      const descendants = new Set([pageId, leafItem.page_id])
+      listTreeMap[parentId] = {
+        ...pagesMap[parentId],
+        children,
+        descendants,
+        depth: 0,
+        ancestors: [],
+      }
+    }
+    else {
+      listTreeMap[parentId].children.add(pageId)
+      listTreeMap[parentId].descendants.add(pageId)
+      listTreeMap[parentId].descendants.add(leafItem.page_id)
+    }
+    leafItem.depth++
+    leafItem.ancestors.unshift(listTreeMap[parentId].page_name)
+
+    if (listTreeMap[parentId].parent_id !== 'root')
+      recursivePushInParentDescendants(pagesMap, listTreeMap, listTreeMap[parentId], leafItem)
+  }
+}
+
+const ItemComponent = ({ index, style, data }: ListChildComponentProps<{
+  dataList: NotionPageItem[]
+  handleToggle: (index: number) => void
+  checkedIds: Set<string>
+  disabledCheckedIds: Set<string>
+  handleCheck: (index: number) => void
+  canPreview?: boolean
+  handlePreview: (index: number) => void
+  listMapWithChildrenAndDescendants: NotionPageTreeMap
+  searchValue: string
+  previewPageId: string
+  pagesMap: DataSourceNotionPageMap
+}>) => {
+  const { t } = useTranslation()
+  const {
+    dataList,
+    handleToggle,
+    checkedIds,
+    disabledCheckedIds,
+    handleCheck,
+    canPreview,
+    handlePreview,
+    listMapWithChildrenAndDescendants,
+    searchValue,
+    previewPageId,
+    pagesMap,
+  } = data
+  const current = dataList[index]
+  const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[current.page_id]
+  const hasChild = currentWithChildrenAndDescendants.descendants.size > 0
+  const ancestors = currentWithChildrenAndDescendants.ancestors
+  const breadCrumbs = ancestors.length ? [...ancestors, current.page_name] : [current.page_name]
+  const disabled = disabledCheckedIds.has(current.page_id)
+
+  const renderArrow = () => {
+    if (hasChild) {
+      return (
+        <div
+          className='mr-1 flex h-5 w-5 shrink-0 items-center justify-center rounded-md hover:bg-components-button-ghost-bg-hover'
+          style={{ marginLeft: current.depth * 8 }}
+          onClick={() => handleToggle(index)}
+        >
+          {
+            current.expand
+              ? <RiArrowDownSLine className='h-4 w-4 text-text-tertiary' />
+              : <RiArrowRightSLine className='h-4 w-4 text-text-tertiary' />
+          }
+        </div>
+      )
+    }
+    if (current.parent_id === 'root' || !pagesMap[current.parent_id]) {
+      return (
+        <div></div>
+      )
+    }
+    return (
+      <div className='mr-1 h-5 w-5 shrink-0' style={{ marginLeft: current.depth * 8 }} />
+    )
+  }
+
+  return (
+    <div
+      className={cn('group flex cursor-pointer items-center rounded-md pl-2 pr-[2px] hover:bg-state-base-hover',
+        previewPageId === current.page_id && 'bg-state-base-hover')}
+      style={{ ...style, top: style.top as number + 8, left: 8, right: 8, width: 'calc(100% - 16px)' }}
+    >
+      <Checkbox
+        className='mr-2 shrink-0'
+        checked={checkedIds.has(current.page_id)}
+        disabled={disabled}
+        onCheck={() => {
+          if (disabled)
+            return
+          handleCheck(index)
+        }}
+      />
+      {!searchValue && renderArrow()}
+      <NotionIcon
+        className='mr-1 shrink-0'
+        type='page'
+        src={current.page_icon}
+      />
+      <div
+        className='grow truncate text-[13px] font-medium leading-4 text-text-secondary'
+        title={current.page_name}
+      >
+        {current.page_name}
+      </div>
+      {
+        canPreview && (
+          <div
+            className='ml-1 hidden h-6 shrink-0 cursor-pointer items-center rounded-md border-[0.5px] border-components-button-secondary-border bg-components-button-secondary-bg px-2 text-xs
+            font-medium leading-4 text-components-button-secondary-text shadow-xs shadow-shadow-shadow-3 backdrop-blur-[10px]
+            hover:border-components-button-secondary-border-hover hover:bg-components-button-secondary-bg-hover group-hover:flex'
+            onClick={() => handlePreview(index)}>
+            {t('common.dataSource.notion.selector.preview')}
+          </div>
+        )
+      }
+      {
+        searchValue && (
+          <div
+            className='ml-1 max-w-[120px] shrink-0 truncate text-xs text-text-quaternary'
+            title={breadCrumbs.join(' / ')}
+          >
+            {breadCrumbs.join(' / ')}
+          </div>
+        )
+      }
+    </div>
+  )
+}
+const Item = memo(ItemComponent, areEqual)
+
+const PageSelector = ({
+  value,
+  disabledValue,
+  searchValue,
+  pagesMap,
+  list,
+  onSelect,
+  canPreview = true,
+  previewPageId,
+  onPreview,
+}: PageSelectorProps) => {
+  const { t } = useTranslation()
+  const [prevDataList, setPrevDataList] = useState(list)
+  const [dataList, setDataList] = useState<NotionPageItem[]>([])
+  const [localPreviewPageId, setLocalPreviewPageId] = useState('')
+  if (prevDataList !== list) {
+    setPrevDataList(list)
+    setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => {
+      return {
+        ...item,
+        expand: false,
+        depth: 0,
+      }
+    }))
+  }
+  const searchDataList = list.filter((item) => {
+    return item.page_name.includes(searchValue)
+  }).map((item) => {
+    return {
+      ...item,
+      expand: false,
+      depth: 0,
+    }
+  })
+  const currentDataList = searchValue ? searchDataList : dataList
+  const currentPreviewPageId = previewPageId === undefined ? localPreviewPageId : previewPageId
+
+  const listMapWithChildrenAndDescendants = useMemo(() => {
+    return list.reduce((prev: NotionPageTreeMap, next: DataSourceNotionPage) => {
+      const pageId = next.page_id
+      if (!prev[pageId])
+        prev[pageId] = { ...next, children: new Set(), descendants: new Set(), depth: 0, ancestors: [] }
+
+      recursivePushInParentDescendants(pagesMap, prev, prev[pageId], prev[pageId])
+      return prev
+    }, {})
+  }, [list, pagesMap])
+
+  const handleToggle = (index: number) => {
+    const current = dataList[index]
+    const pageId = current.page_id
+    const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[pageId]
+    const descendantsIds = Array.from(currentWithChildrenAndDescendants.descendants)
+    const childrenIds = Array.from(currentWithChildrenAndDescendants.children)
+    let newDataList = []
+
+    if (current.expand) {
+      current.expand = false
+
+      newDataList = [...dataList.filter(item => !descendantsIds.includes(item.page_id))]
+    }
+    else {
+      current.expand = true
+
+      newDataList = [
+        ...dataList.slice(0, index + 1),
+        ...childrenIds.map(item => ({
+          ...pagesMap[item],
+          expand: false,
+          depth: listMapWithChildrenAndDescendants[item].depth,
+        })),
+        ...dataList.slice(index + 1)]
+    }
+    setDataList(newDataList)
+  }
+
+  const copyValue = new Set([...value])
+  const handleCheck = (index: number) => {
+    const current = currentDataList[index]
+    const pageId = current.page_id
+    const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[pageId]
+
+    if (copyValue.has(pageId)) {
+      if (!searchValue) {
+        for (const item of currentWithChildrenAndDescendants.descendants)
+          copyValue.delete(item)
+      }
+
+      copyValue.delete(pageId)
+    }
+    else {
+      if (!searchValue) {
+        for (const item of currentWithChildrenAndDescendants.descendants)
+          copyValue.add(item)
+      }
+
+      copyValue.add(pageId)
+    }
+
+    onSelect(new Set([...copyValue]))
+  }
+
+  const handlePreview = (index: number) => {
+    const current = currentDataList[index]
+    const pageId = current.page_id
+
+    setLocalPreviewPageId(pageId)
+
+    if (onPreview)
+      onPreview(pageId)
+  }
+
+  if (!currentDataList.length) {
+    return (
+      <div className='flex h-[296px] items-center justify-center text-[13px] text-text-tertiary'>
+        {t('common.dataSource.notion.selector.noSearchResult')}
+      </div>
+    )
+  }
+
+  return (
+    <List
+      className='py-2'
+      height={296}
+      itemCount={currentDataList.length}
+      itemSize={28}
+      width='100%'
+      itemKey={(index, data) => data.dataList[index].page_id}
+      itemData={{
+        dataList: currentDataList,
+        handleToggle,
+        checkedIds: value,
+        disabledCheckedIds: disabledValue,
+        handleCheck,
+        canPreview,
+        handlePreview,
+        listMapWithChildrenAndDescendants,
+        searchValue,
+        previewPageId: currentPreviewPageId,
+        pagesMap,
+      }}
+    >
+      {Item}
+    </List>
+  )
+}
+
+export default PageSelector
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawled-result-item.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawled-result-item.tsx
index 577521b6d3..2e80085cfe 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawled-result-item.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawled-result-item.tsx
@@ -5,6 +5,7 @@ import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
 import Checkbox from '@/app/components/base/checkbox'
 import Button from '@/app/components/base/button'
 import { useTranslation } from 'react-i18next'
+import Radio from '@/app/components/base/radio/ui'
 
 type CrawledResultItemProps = {
   payload: CrawlResultItemType
@@ -13,6 +14,7 @@ type CrawledResultItemProps = {
   isPreview: boolean
   showPreview: boolean
   onPreview: () => void
+  supportMultipleChoice?: boolean
 }
 
 const CrawledResultItem = ({
@@ -22,22 +24,33 @@ const CrawledResultItem = ({
   isPreview,
   onPreview,
   showPreview,
+  supportMultipleChoice = true,
 }: CrawledResultItemProps) => {
   const { t } = useTranslation()
 
   const handleCheckChange = useCallback(() => {
     onCheckChange(!isChecked)
   }, [isChecked, onCheckChange])
+
   return (
     <div className={cn(
       'relative flex cursor-pointer gap-x-2 rounded-lg p-2',
       isPreview ? 'bg-state-base-active' : 'group hover:bg-state-base-hover',
     )}>
-      <Checkbox
-        className='shrink-0'
-        checked={isChecked}
-        onCheck={handleCheckChange}
-      />
+      {
+        supportMultipleChoice ? (
+          <Checkbox
+            className='shrink-0'
+            checked={isChecked}
+            onCheck={handleCheckChange}
+          />
+        ) : (
+          <Radio
+            isChecked={isChecked}
+            onCheck={handleCheckChange}
+          />
+        )
+      }
       <div className='flex min-w-0 grow flex-col gap-y-0.5'>
         <div
           className='system-sm-medium truncate text-text-secondary'
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawled-result.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawled-result.tsx
index 465206b4dc..9be584d935 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawled-result.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawled-result.tsx
@@ -16,6 +16,7 @@ type CrawledResultProps = {
   onSelectedChange: (selected: CrawlResultItem[]) => void
   onPreview?: (payload: CrawlResultItem, index: number) => void
   usedTime: number
+  supportMultipleChoice?: boolean
 }
 
 const CrawledResult = ({
@@ -26,6 +27,7 @@ const CrawledResult = ({
   onSelectedChange,
   usedTime,
   onPreview,
+  supportMultipleChoice = true,
 }: CrawledResultProps) => {
   const { t } = useTranslation()
 
@@ -42,12 +44,11 @@ const CrawledResult = ({
   const handleItemCheckChange = useCallback((item: CrawlResultItem) => {
     return (checked: boolean) => {
       if (checked)
-        onSelectedChange([...checkedList, item])
-
+        supportMultipleChoice ? onSelectedChange([...checkedList, item]) : onSelectedChange([item])
       else
         onSelectedChange(checkedList.filter(checkedItem => checkedItem.source_url !== item.source_url))
     }
-  }, [checkedList, onSelectedChange])
+  }, [checkedList, onSelectedChange, supportMultipleChoice])
 
   const handlePreview = useCallback((index: number) => {
     if (!onPreview) return
@@ -63,12 +64,14 @@ const CrawledResult = ({
         })}
       </div>
       <div className='overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg'>
-        <div className='flex items-center px-4 py-2'>
-          <CheckboxWithLabel
-            isChecked={isCheckAll}
-            onChange={handleCheckedAll} label={isCheckAll ? t(`${I18N_PREFIX}.resetAll`) : t(`${I18N_PREFIX}.selectAll`)}
-          />
-        </div>
+        {supportMultipleChoice && (
+          <div className='flex items-center px-4 py-2'>
+            <CheckboxWithLabel
+              isChecked={isCheckAll}
+              onChange={handleCheckedAll} label={isCheckAll ? t(`${I18N_PREFIX}.resetAll`) : t(`${I18N_PREFIX}.selectAll`)}
+            />
+          </div>
+        )}
         <div className='flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2'>
           {list.map((item, index) => (
             <CrawledResultItem
@@ -79,6 +82,7 @@ const CrawledResult = ({
               isPreview={index === previewIndex}
               onPreview={handlePreview.bind(null, index)}
               showPreview={!!onPreview}
+              supportMultipleChoice={supportMultipleChoice}
             />
           ))}
         </div>
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx
index 278c22918a..2b72742bb0 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx
@@ -169,6 +169,7 @@ const Crawler = ({
               usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
               previewIndex={previewIndex}
               onPreview={onPreview}
+              supportMultipleChoice={false} // only support single choice in test run
             />
           )}
         </div>