feat: ui component finish for chunk preview

web/app/components/datasets/assets/selection-mod-nocolor.svg

@@ -0,0 +1,13 @@
+<svg width="10" height="10" viewBox="0 0 10 10" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="Group">
+<path id="Vector" d="M2.5 10H0V7.5H2.5V10Z" fill="#676F83"/>
+<path id="Vector_2" d="M6.25 6.25H3.75V3.75H6.25V6.25Z" fill="#676F83"/>
+<path id="Vector_3" d="M2.5 6.25H0V3.75H2.5V6.25Z" fill="#676F83"/>
+<path id="Vector_4" d="M6.25 2.5H3.75V0H6.25V2.5Z" fill="#676F83"/>
+<path id="Vector_5" d="M2.5 2.5H0V0H2.5V2.5Z" fill="#676F83"/>
+<path id="Vector_6" d="M10 2.5H7.5V0H10V2.5Z" fill="#676F83"/>
+<path id="Vector_7" d="M9.58332 7.91663H7.91666V9.58329H9.58332V7.91663Z" fill="#676F83"/>
+<path id="Vector_8" d="M9.58332 4.16663H7.91666V5.83329H9.58332V4.16663Z" fill="#676F83"/>
+<path id="Vector_9" d="M5.83332 7.91663H4.16666V9.58329H5.83332V7.91663Z" fill="#676F83"/>
+</g>
+</svg>

web/app/components/datasets/chunk.tsx

@@ -0,0 +1,55 @@
+import type { FC, PropsWithChildren } from 'react'
+import Image from 'next/image'
+import SelectionMod from './assets/selection-mod-nocolor.svg'
+import type { QA } from '@/models/datasets'
+
+export type ChunkLabelProps = {
+  label: string
+  characterCount: number
+}
+
+export const ChunkLabel: FC<ChunkLabelProps> = (props) => {
+  const { label, characterCount } = props
+  return <div className='flex items-center text-text-tertiary text-xs font-medium'>
+    <Image src={SelectionMod} alt="Selection Mod" width={10} height={10} />
+    <p className='flex gap-2 ml-0.5'><span>
+      {label}
+    </span>
+    <span>
+      ·
+    </span>
+    <span>
+      {`${characterCount} characters`}
+    </span></p>
+  </div>
+}
+
+export type ChunkContainerProps = ChunkLabelProps & PropsWithChildren
+
+export const ChunkContainer: FC<ChunkContainerProps> = (props) => {
+  const { label, characterCount, children } = props
+  return <div className='space-y-2'>
+    <ChunkLabel label={label} characterCount={characterCount} />
+    <p className='text-text-secondary text-sm tracking-[-0.0005em]'>
+      {children}
+    </p>
+  </div>
+}
+
+export type QAPreviewProps = {
+  qa: QA
+}
+
+export const QAPreview: FC<QAPreviewProps> = (props) => {
+  const { qa } = props
+  return <div className='space-y-2'>
+    <div className='flex gap-1 items-start'>
+      <label className='text-text-tertiary text-[13px] font-medium'>Q</label>
+      <p className='text-text-secondary tracking-[-0.0005em]'>{qa.question}</p>
+    </div>
+    <div className='flex gap-1 items-start'>
+      <label className='text-text-tertiary text-[13px] font-medium'>A</label>
+      <p className='text-text-secondary tracking-[-0.0005em]'>{qa.answer}</p>
+    </div>
+  </div>
+}

web/app/components/datasets/create/step-two/index.module.css

@@ -394,19 +394,6 @@
   max-width: 524px;
 }
 
-.previewHeader {
-  position: sticky;
-  top: 0;
-  left: 0;
-  padding-top: 42px;
-  background-color: #fff;
-  font-weight: 600;
-  font-size: 18px;
-  line-height: 28px;
-  color: #101828;
-  z-index: 10;
-}
-
 /* 
  * `fixed` must under `previewHeader` because of style override would not work
  */

web/app/components/datasets/create/step-two/index.tsx

@@ -16,13 +16,15 @@ import FamilyMod from '../assets/family-mod.svg'
 import Note from '../assets/note-mod.svg'
 import FileList from '../assets/file-list-3-fill.svg'
 import { indexMethodIcon } from '../icons'
+import { PreviewContainer } from '../../preview/container'
+import { ChunkContainer, QAPreview } from '../../chunk'
+import { PreviewHeader } from '../../preview/header'
 import s from './index.module.css'
 import unescape from './unescape'
 import escape from './escape'
 import { OptionCard } from './option-card'
 import LanguageSelect from './language-select'
 import { DelimiterInput, MaxLengthInput, OverlapInput } from './inputs'
-import PreviewItem, { PreviewType } from './preview-item'
 import cn from '@/utils/classnames'
 import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FullDocumentDetail, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
 
@@ -58,14 +60,13 @@ const TextLabel: FC<PropsWithChildren> = (props) => {
   return <label className='text-text-secondary text-xs font-semibold leading-none'>{props.children}</label>
 }
 
-type ValueOf<T> = T[keyof T]
 type StepTwoProps = {
   isSetting?: boolean
   documentDetail?: FullDocumentDetail
   isAPIKeySet: boolean
   onSetting: () => void
   datasetId?: string
-  indexingType?: ValueOf<IndexingType>
+  indexingType?: IndexingType
   retrievalMethod?: string
   dataSourceType: DataSourceType
   files: CustomFile[]
@@ -156,7 +157,7 @@ const StepTwo = ({
   const [rules, setRules] = useState<PreProcessingRule[]>([])
   const [defaultConfig, setDefaultConfig] = useState<Rules>()
   const hasSetIndexType = !!indexingType
-  const [indexType, setIndexType] = useState<ValueOf<IndexingType>>(
+  const [indexType, setIndexType] = useState<IndexingType>(
     (indexingType
       || isAPIKeySet)
       ? IndexingType.QUALIFIED
@@ -906,52 +907,40 @@ const StepTwo = ({
         </div>
       </div>
       <FloatRightContainer isMobile={isMobile} isOpen={true} onClose={() => { }} footer={null}>
-        <div
+        <PreviewContainer
-          className={cn(s.previewWrap, isMobile && s.isMobile, 'relative h-full overflow-y-scroll border-l border-[#F2F4F7]')}
+          header={<PreviewHeader
+            title='Preview'
+          >
+          </PreviewHeader>}
+          className={cn(s.previewWrap, isMobile && s.isMobile, 'relative h-full overflow-y-scroll space-y-4')}
         >
-          <div className={cn(s.previewHeader)}>
+          {qaPreviewSwitched && docForm === DocForm.QA && estimate?.qa_preview && (
-            <div className='flex items-center justify-between px-8'>
+            estimate?.qa_preview.map(item => (
-              <div className='grow flex items-center'>
+              <QAPreview key={item.question} qa={item} />
-                <div>{t('datasetCreation.stepTwo.previewTitle')}</div>
+            ))
-                {docForm === DocForm.QA && !qaPreviewSwitched && (
+          )}
-                  <Button className='ml-2' variant='secondary-accent' onClick={() => previewSwitch()}>{t('datasetCreation.stepTwo.previewButton')}</Button>
+          {(docForm === DocForm.TEXT || !qaPreviewSwitched) && estimate?.preview && (
-                )}
+            estimate?.preview.map((item, index) => (
-              </div>
+              <ChunkContainer
+                key={item}
+                label={`Chunk-${index + 1}`}
+                characterCount={item.length}
+              >
+                {item}
+              </ChunkContainer>
+            ))
+          )}
+          {qaPreviewSwitched && docForm === DocForm.QA && !estimate?.qa_preview && (
+            <div className='flex items-center justify-center h-[200px]'>
+              <Loading type='area' />
             </div>
-            {docForm === DocForm.QA && !qaPreviewSwitched && (
+          )}
-              <div className='px-8 pr-12 text-xs text-gray-500'>
+          {!qaPreviewSwitched && !estimate?.preview && (
-                <span>{t('datasetCreation.stepTwo.previewSwitchTipStart')}</span>
+            <div className='flex items-center justify-center h-[200px]'>
-                <span className='text-amber-600'>{t('datasetCreation.stepTwo.previewSwitchTipEnd')}</span>
+              <Loading type='area' />
-              </div>
+            </div>
-            )}
+          )}
-          </div>
+        </PreviewContainer>
-          <div className='my-4 px-8 space-y-4'>
-            {qaPreviewSwitched && docForm === DocForm.QA && estimate?.qa_preview && (
-              <>
-                {estimate?.qa_preview.map((item, index) => (
-                  <PreviewItem type={PreviewType.QA} key={item.question} qa={item} index={index + 1} />
-                ))}
-              </>
-            )}
-            {(docForm === DocForm.TEXT || !qaPreviewSwitched) && estimate?.preview && (
-              <>
-                {estimate?.preview.map((item, index) => (
-                  <PreviewItem type={PreviewType.TEXT} key={item} content={item} index={index + 1} />
-                ))}
-              </>
-            )}
-            {qaPreviewSwitched && docForm === DocForm.QA && !estimate?.qa_preview && (
-              <div className='flex items-center justify-center h-[200px]'>
-                <Loading type='area' />
-              </div>
-            )}
-            {!qaPreviewSwitched && !estimate?.preview && (
-              <div className='flex items-center justify-center h-[200px]'>
-                <Loading type='area' />
-              </div>
-            )}
-          </div>
-        </div>
       </FloatRightContainer>
     </div>
   )

web/app/components/datasets/formatted-text/flavours/shared.tsx

@@ -34,7 +34,7 @@ export const SliceContent: FC<SliceContentProps> = forwardRef((props, ref) => {
   const { className, children, ...rest } = props
   return <span {...rest} ref={ref} className={classNames(
     baseStyle,
-    'px-1 bg-state-base-hover group-hover:bg-state-accent-hover-alt group-hover:text-text-primary',
+    'px-1 bg-state-base-hover group-hover:bg-state-accent-hover-alt group-hover:text-text-primary leading-7',
     className,
   )}>
     {children}

web/app/dev-preview/page.tsx

@@ -1,5 +1,6 @@
 'use client'
 
+import { useState } from 'react'
 import { FormattedText } from '../components/datasets/formatted-text/formatted'
 import { PreviewSlice } from '../components/datasets/formatted-text/flavours/preview-slice'
 import { PreviewContainer } from '../components/datasets/preview/container'
@@ -8,9 +9,22 @@ import FileIcon from '../components/base/file-icon'
 import { ChevronDown } from '../components/base/icons/src/vender/solid/arrows'
 import Badge from '../components/base/badge'
 import { DividerWithLabel } from '../components/base/divider/with-label'
+import Button from '../components/base/button'
+import { ChunkContainer, QAPreview } from '../components/datasets/chunk'
+import classNames from '@/utils/classnames'
 
 export default function Page() {
+  const [parentChild, setParentChild] = useState(false)
+  const [vertical, setVertical] = useState(false)
+  const [qa, setQa] = useState(false)
   return <div className='p-4'>
+    <div className='flex gap-2 my-4'>
+      <Button onClick={() => setParentChild(!parentChild)}>
+        Parent-Child
+      </Button>
+      <Button onClick={() => setVertical(!vertical)}>Vertical</Button>
+      <Button onClick={() => setQa(!qa)}>QA</Button>
+    </div>
     <PreviewContainer header={
       <PreviewHeader title='Preview'>
         <div className='flex items-center'>
@@ -23,13 +37,39 @@ export default function Page() {
         </div>
       </PreviewHeader>
     }>
-      <FormattedText>
+      <div className='space-y-6'>{parentChild
-        <PreviewSlice label='C-1' text='lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.' tooltip={'Child-chunk-2 · 268 Characters'} />
+        ? Array.from({ length: 4 }, (_, i) => {
-        <PreviewSlice label='C-1' text='lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.' tooltip={'Child-chunk-2 · 268 Characters'} />
+          return <ChunkContainer
-        <PreviewSlice label='C-1' text='lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.' tooltip={'Child-chunk-2 · 268 Characters'} />
+            label='Parent-Chunk-01'
-        <PreviewSlice label='C-1' text='lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.' tooltip={'Child-chunk-2 · 268 Characters'} />
+            characterCount={521}
-        <PreviewSlice label='C-1' text='lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.' tooltip={'Child-chunk-2 · 268 Characters'} />
+            key={i}
-      </FormattedText>
+          >
+            <FormattedText className={classNames(
+              'w-full',
+              vertical && 'flex flex-col gap-2',
+            )}>
+              {Array.from({ length: 4 }, (_, i) => {
+                return <PreviewSlice
+                  key={i}
+                  label='C-1'
+                  text='lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.' tooltip={'Child-chunk-2 · 268 Characters'} />
+              })}
+            </FormattedText>
+          </ChunkContainer>
+        })
+        : Array.from({ length: 2 }, (_, i) => {
+          return <ChunkContainer label='Chunk-01' characterCount={521} key={i}>
+            {
+              qa
+                ? <QAPreview qa={{
+                  question: 'What is the author\'s unconventional approach to writing this book, and how does it challenge the traditional academic mindset of \'publish or perish\'?',
+                  answer: 'It is quite natural for academics who are continuously told to “publish or perish” to want to always create something from scratch that is their own fresh creation. This book is an experiment in not starting from scratch, but instead “re-mixing” the book titled Think Python: How to Think Like a Computer Scientist written by Allen B. Downey, Jeff Elkner and others.',
+                }} />
+                : 'In December of 2009, I was preparing to teach SI502 - Networked Programming at the University of Michigan for the fifth semester in a row and decided it was time to write a Python textbook that focused on exploring data instead of understanding algorithms and abstractions. My goal in SI502 is to teach people life-long data handling skills using Python. Few of my students were planning to be professional computer programmers. Instead, they planned be librarians, managers, lawyers, biologists, economists, etc. who happened to want to skillfully use technology in their chosen field.'
+            }
+          </ChunkContainer>
+        })
+      }</div>
       <DividerWithLabel label='Display previews of up to 10 paragraphs' />
     </PreviewContainer>
   </div>

web/models/datasets.ts

@@ -330,6 +330,7 @@ export type NotionPage = {
 }
 
 export type ProcessRule = {
+  processRule: { pre_processing_rules: PreProcessingRule[]; segmentation: { separator: string; max_tokens: number; chunk_overlap: number } }
   mode: string
   rules: Rules
 }