feat(rag-pipeline): enhance result preview with chunk formatting and add configuration for preview chunk limit

web/app/components/rag-pipeline/components/chunk-card-list/index.tsx

@@ -24,7 +24,7 @@ const QAItem = (props: QAItemProps) => {
   </div>
 }
 
-enum ChunkType {
+export enum ChunkType {
   General = 'general',
   Paragraph = 'paragraph',
   FullDoc = 'full-doc',
@@ -145,6 +145,7 @@ export const ChunkCardList = (props: ChunkCardListProps) => {
 
         return (
           <ChunkCard
+            key={`${chunkType}-${index}`}
             type={chunkType}
             content={isParentChildMode ? (seg as ParentChildChunk).child_contents : (seg as string | QAChunk)}
             wordCount={wordCount}

web/app/components/rag-pipeline/components/panel/test-run/result/result-preview/index.tsx

@@ -5,6 +5,8 @@ import { RiLoader2Line } from '@remixicon/react'
 import React, { useMemo } from 'react'
 import { useTranslation } from 'react-i18next'
 import { ChunkCardList } from '../../../../chunk-card-list'
+import { RAG_PIPELINE_PREVIEW_CHUNK_NUM } from '@/config'
+import { formatPreviewChunks } from './utils'
 
 type ResultTextProps = {
   isRunning?: boolean
@@ -30,6 +32,10 @@ const ResultPreview = ({
     return knowledgeIndexNode?.inputs?.chunks
   }, [outputs, tracing])
 
+  const previewChunks = useMemo(() => {
+    return formatPreviewChunks(chunkInfo, outputs)
+  }, [chunkInfo, outputs])
+
   return (
     <>
       {isRunning && !outputs && (
@@ -46,13 +52,13 @@ const ResultPreview = ({
           </Button>
         </div>
       )}
-      {outputs && (
+      {outputs && previewChunks && (
         <div className='flex grow flex-col bg-background-body p-1'>
-          {!!chunkInfo && <ChunkCardList chunkInfo={chunkInfo} />}
+          <ChunkCardList chunkInfo={previewChunks} />
           <div className='system-xs-regular mt-1 flex items-center gap-x-2 text-text-tertiary'>
             <div className='h-px flex-1 bg-gradient-to-r from-background-gradient-mask-transparent to-divider-regular' />
-            <span className='shrink-0truncate' title={t('pipeline.result.resultPreview.footerTip', { count: 20 })}>
-              {t('pipeline.result.resultPreview.footerTip', { count: 20 })}
+            <span className='shrink-0truncate' title={t('pipeline.result.resultPreview.footerTip', { count: RAG_PIPELINE_PREVIEW_CHUNK_NUM })}>
+              {t('pipeline.result.resultPreview.footerTip', { count: RAG_PIPELINE_PREVIEW_CHUNK_NUM })}
             </span>
             <div className='h-px flex-1 bg-gradient-to-l from-background-gradient-mask-transparent to-divider-regular' />
           </div>

web/app/components/rag-pipeline/components/panel/test-run/result/result-preview/utils.ts

@@ -0,0 +1,95 @@
+import { RAG_PIPELINE_PREVIEW_CHUNK_NUM } from '@/config'
+import { type ChunkInfo, ChunkType } from '../../../../chunk-card-list'
+
+const formatGeneralChunks = (outputs: any) => {
+  if (!outputs) return undefined
+  const chunkInfo: ChunkInfo = {
+    general_chunks: [],
+  }
+  const chunks = outputs.preview as string[]
+  chunks.slice(0, RAG_PIPELINE_PREVIEW_CHUNK_NUM).forEach((chunk) => {
+    chunkInfo.general_chunks?.push(chunk)
+  })
+
+  return chunkInfo
+}
+
+type ParentChildChunkPreview = {
+  content: string
+  child_chunks: string[]
+}
+
+const formatParentChildChunks = (outputs: any, chunkType: ChunkType) => {
+  if (!outputs) return undefined
+  const chunkInfo: ChunkInfo = {
+    parent_child_chunks: [],
+    parent_mode: chunkType,
+  }
+  const chunks = outputs.preview as ParentChildChunkPreview[]
+  if (chunkType === ChunkType.Paragraph) {
+    chunks.slice(0, RAG_PIPELINE_PREVIEW_CHUNK_NUM).forEach((chunk) => {
+      chunkInfo.parent_child_chunks?.push({
+        parent_content: chunk.content,
+        child_contents: chunk.child_chunks,
+        parent_mode: chunkType,
+      })
+    })
+    return chunkInfo
+  }
+  else {
+    chunks.forEach((chunk) => {
+      chunkInfo.parent_child_chunks?.push({
+        parent_content: chunk.content,
+        child_contents: chunk.child_chunks.slice(0, RAG_PIPELINE_PREVIEW_CHUNK_NUM),
+        parent_mode: chunkType,
+      })
+    })
+  }
+
+  return chunkInfo
+}
+
+type QAChunkPreview = {
+  question: string
+  answer: string
+}
+
+const formatQAChunks = (outputs: any) => {
+  if (!outputs) return undefined
+  const chunkInfo: ChunkInfo = {
+    qa_chunks: [],
+  }
+  const chunks = outputs.qa_preview as QAChunkPreview[]
+  chunks.slice(0, RAG_PIPELINE_PREVIEW_CHUNK_NUM).forEach((chunk) => {
+    chunkInfo.qa_chunks?.push({
+      ...chunk,
+    })
+  })
+
+  return chunkInfo
+}
+
+export const formatPreviewChunks = (chunkInfo: ChunkInfo, outputs: any): ChunkInfo | undefined => {
+  if (!chunkInfo) return undefined
+
+  let chunkType = ChunkType.General
+  if (chunkInfo?.general_chunks)
+    chunkType = ChunkType.General
+
+  if (chunkInfo?.parent_child_chunks)
+    chunkType = chunkInfo.parent_mode as ChunkType
+
+  if (chunkInfo?.qa_chunks)
+    chunkType = ChunkType.QA
+
+  if (chunkType === ChunkType.General)
+    return formatGeneralChunks(outputs)
+
+  if (chunkType === ChunkType.Paragraph || chunkType === ChunkType.FullDoc)
+    return formatParentChildChunks(outputs, chunkType)
+
+  if (chunkType === ChunkType.QA)
+    return formatQAChunks(outputs)
+
+  return undefined
+}

web/config/index.ts

@@ -295,3 +295,5 @@ export const ENABLE_WEBSITE_WATERCRAWL = getBooleanConfig(process.env.NEXT_PUBLI
 export const VALUE_SELECTOR_DELIMITER = '@@@'
 
 export const validPassword = /^(?=.*[a-zA-Z])(?=.*\d)\S{8,}$/
+
+export const RAG_PIPELINE_PREVIEW_CHUNK_NUM = 20