Merge branch 'feat/rag-pipeline' into deploy/rag-dev

api/core/repositories/sqlalchemy_workflow_execution_repository.py

@@ -6,7 +6,7 @@ import json
 import logging
 from typing import Optional, Union
 
-from sqlalchemy import select
+from sqlalchemy import func, select
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import sessionmaker
 
@@ -151,11 +151,11 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):
             existing = session.scalar(select(WorkflowRun).where(WorkflowRun.id == domain_model.id_))
             if not existing:
                 # For new records, get the next sequence number
-                stmt = select(WorkflowRun.sequence_number).where(
+                stmt = select(func.max(WorkflowRun.sequence_number)).where(
                     WorkflowRun.app_id == self._app_id,
                     WorkflowRun.tenant_id == self._tenant_id,
                 )
-                max_sequence = session.scalar(stmt.order_by(WorkflowRun.sequence_number.desc()))
+                max_sequence = session.scalar(stmt)
                 db_model.sequence_number = (max_sequence or 0) + 1
             else:
                 # For updates, keep the existing sequence number

web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx

@@ -27,12 +27,13 @@ import { Variable02 } from '@/app/components/base/icons/src/vender/solid/develop
 import { BubbleX, Env } from '@/app/components/base/icons/src/vender/line/others'
 import { VarBlockIcon } from '@/app/components/workflow/block-icon'
 import { Line3 } from '@/app/components/base/icons/src/public/common'
-import { isConversationVar, isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils'
+import { isConversationVar, isENV, isRagVariableVar, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils'
 import Tooltip from '@/app/components/base/tooltip'
 import { isExceptionVariable } from '@/app/components/workflow/utils'
 import VarFullPathPanel from '@/app/components/workflow/nodes/_base/components/variable/var-full-path-panel'
 import { Type } from '@/app/components/workflow/nodes/llm/types'
 import type { ValueSelector } from '@/app/components/workflow/types'
+import { InputField } from '@/app/components/base/icons/src/vender/pipeline'
 
 type WorkflowVariableBlockComponentProps = {
   nodeKey: string
@@ -54,7 +55,8 @@ const WorkflowVariableBlockComponent = ({
   const [editor] = useLexicalComposerContext()
   const [ref, isSelected] = useSelectOrDelete(nodeKey, DELETE_WORKFLOW_VARIABLE_BLOCK_COMMAND)
   const variablesLength = variables.length
-  const isShowAPart = variablesLength > 2
+  const isRagVar = isRagVariableVar(variables)
+  const isShowAPart = variablesLength > 2 && !isRagVar
   const varName = (
     () => {
       const isSystem = isSystemVar(variables)
@@ -113,7 +115,7 @@ const WorkflowVariableBlockComponent = ({
       className={cn(
         'group/wrap relative mx-0.5 flex h-[18px] select-none items-center rounded-[5px] border pl-0.5 pr-[3px] hover:border-state-accent-solid hover:bg-state-accent-hover',
         isSelected ? ' border-state-accent-solid bg-state-accent-hover' : ' border-components-panel-border-subtle bg-components-badge-white-to-dark',
-        !node && !isEnv && !isChatVar && '!border-state-destructive-solid !bg-state-destructive-hover',
+        !node && !isEnv && !isChatVar && !isRagVar && '!border-state-destructive-solid !bg-state-destructive-hover',
       )}
       onClick={(e) => {
         e.stopPropagation()
@@ -121,7 +123,7 @@ const WorkflowVariableBlockComponent = ({
       }}
       ref={ref}
     >
-      {!isEnv && !isChatVar && (
+      {!isEnv && !isChatVar && !isRagVar && (
         <div className='flex items-center'>
           {
             node?.type && (
@@ -146,17 +148,19 @@ const WorkflowVariableBlockComponent = ({
       )}
 
       <div className='flex items-center text-text-accent'>
-        {!isEnv && !isChatVar && <Variable02 className={cn('h-3.5 w-3.5 shrink-0', isException && 'text-text-warning')} />}
+        {!isEnv && !isChatVar && !isRagVar && <Variable02 className={cn('h-3.5 w-3.5 shrink-0', isException && 'text-text-warning')} />}
         {isEnv && <Env className='h-3.5 w-3.5 shrink-0 text-util-colors-violet-violet-600' />}
         {isChatVar && <BubbleX className='h-3.5 w-3.5 text-util-colors-teal-teal-700' />}
+        {isRagVar && <InputField className='h-3.5 w-3.5 text-text-accent' />}
         <div className={cn(
           'ml-0.5 shrink-0 truncate text-xs font-medium',
           isEnv && 'text-util-colors-violet-violet-600',
           isChatVar && 'text-util-colors-teal-teal-700',
           isException && 'text-text-warning',
+          isRagVar && 'text-text-accent',
         )} title={varName}>{varName}</div>
         {
-          !node && !isEnv && !isChatVar && (
+          !node && !isEnv && !isChatVar && !isRagVar && (
             <RiErrorWarningFill className='ml-0.5 h-3 w-3 text-text-destructive' />
           )
         }
@@ -164,7 +168,7 @@ const WorkflowVariableBlockComponent = ({
     </div>
   )
 
-  if (!node && !isEnv && !isChatVar) {
+  if (!node && !isEnv && !isChatVar && !isRagVar) {
     return (
       <Tooltip popupContent={t('workflow.errorMsg.invalidVariable')}>
         {Item}

web/app/components/datasets/create-from-pipeline/list/template-card/details/index.tsx

@@ -50,7 +50,10 @@ const Details = ({
   return (
     <div className='flex h-full'>
       <div className='flex grow items-center justify-center p-3 pr-0'>
-        <WorkflowPreview {...pipelineTemplateInfo.graph} />
+        <WorkflowPreview
+          {...pipelineTemplateInfo.graph}
+          className='overflow-hidden rounded-2xl'
+        />
       </div>
       <div className='relative flex w-[360px] shrink-0 flex-col'>
         <button

web/app/components/datasets/create-from-pipeline/list/template-card/index.tsx

@@ -189,7 +189,7 @@ const TemplateCard = ({
         <Modal
           isShow={showDetailModal}
           onClose={closeDetailsModal}
-          className='h-[calc(100vh-64px)] max-w-[1680px] p-0'
+          className='h-[calc(100vh-64px)] max-w-[1680px] rounded-3xl p-0'
         >
           <Details
             id={pipeline.id}

web/app/components/datasets/create/step-two/index.tsx

@@ -17,7 +17,7 @@ import FamilyMod from '../assets/family-mod.svg'
 import Note from '../assets/note-mod.svg'
 import FileList from '../assets/file-list-3-fill.svg'
 import { indexMethodIcon } from '../icons'
-import { PreviewContainer } from '../../preview/container'
+import PreviewContainer from '../../preview/container'
 import { ChunkContainer, QAPreview } from '../../chunk'
 import { PreviewHeader } from '../../preview/header'
 import { FormattedText } from '../../formatted-text/formatted'

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -18,7 +18,7 @@ import Loading from '@/app/components/base/loading'
 import type { Node } from '@/app/components/workflow/types'
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
 import FilePreview from './preview/file-preview'
-import NotionPagePreview from './preview/notion-page-preview'
+import OnlineDocumentPreview from './preview/online-document-preview'
 import WebsitePreview from './preview/web-preview'
 import ProcessDocuments from './process-documents'
 import ChunkPreview from './preview/chunk-preview'
@@ -223,9 +223,9 @@ const CreateFormPipeline = () => {
 
   return (
     <div
-      className='relative flex h-[calc(100vh-56px)] overflow-x-auto rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
+      className='relative flex h-[calc(100vh-56px)] w-full min-w-[1024px] overflow-x-auto rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
     >
-      <div className='flex h-full min-w-[760px] flex-1 flex-col px-14'>
+      <div className='flex h-full flex-1 flex-col px-14'>
         <LeftHeader
           steps={steps}
           title={t('datasetPipeline.addDocuments.title')}
@@ -309,16 +309,16 @@ const CreateFormPipeline = () => {
       {/* Preview */}
       {
         currentStep === 1 && (
-          <div className='flex h-full w-[752px] shrink-0 pl-2 pt-2'>
+          <div className='flex h-full flex-1 pl-2 pt-2'>
             {currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
-            {currentDocuments && <NotionPagePreview currentPage={currentDocuments} hidePreview={hideOnlineDocumentPreview} />}
+            {currentDocuments && <OnlineDocumentPreview currentPage={currentDocuments} hidePreview={hideOnlineDocumentPreview} />}
             {currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
           </div>
         )
       }
       {
         currentStep === 2 && (
-          <div className='flex h-full w-[752px] shrink-0 pl-2 pt-2'>
+          <div className='flex h-full flex-1 pl-2 pt-2'>
             <ChunkPreview
               dataSourceType={datasource!.type}
               files={fileList.map(file => file.file)}

web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx

@@ -1,6 +1,6 @@
 import React, { useState } from 'react'
 import { useTranslation } from 'react-i18next'
-import { PreviewContainer } from '../../../preview/container'
+import PreviewContainer from '../../../preview/container'
 import { PreviewHeader } from '../../../preview/header'
 import type { CrawlResultItem, CustomFile, DocumentItem, FileIndexingEstimateResponse } from '@/models/datasets'
 import { ChunkingMode } from '@/models/datasets'
@@ -122,7 +122,7 @@ const ChunkPreview = ({
       className='relative flex h-full w-full shrink-0'
       mainClassName='space-y-6'
     >
-      {currentDocForm === ChunkingMode.qa && estimateData?.qa_preview && (
+      {!isPending && currentDocForm === ChunkingMode.qa && estimateData?.qa_preview && (
         estimateData?.qa_preview.map((item, index) => (
           <ChunkContainer
             key={item.question}
@@ -133,7 +133,7 @@ const ChunkPreview = ({
           </ChunkContainer>
         ))
       )}
-      {currentDocForm === ChunkingMode.text && estimateData?.preview && (
+      {!isPending && currentDocForm === ChunkingMode.text && estimateData?.preview && (
         estimateData?.preview.map((item, index) => (
           <ChunkContainer
             key={item.content}
@@ -144,7 +144,7 @@ const ChunkPreview = ({
           </ChunkContainer>
         ))
       )}
-      {currentDocForm === ChunkingMode.parentChild && estimateData?.preview && (
+      {!isPending && currentDocForm === ChunkingMode.parentChild && estimateData?.preview && (
         estimateData?.preview?.map((item, index) => {
           const indexForLabel = index + 1
           // const childChunks = parentChildConfig.chunkForContext === 'full-doc'
@@ -189,7 +189,7 @@ const ChunkPreview = ({
         </div>
       )}
       {isPending && (
-        <div className='space-y-6'>
+        <div className='h-full w-full space-y-6 overflow-hidden'>
           {Array.from({ length: 10 }, (_, i) => (
             <SkeletonContainer key={i}>
               <SkeletonRow>

web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx

@@ -40,7 +40,7 @@ const FilePreview = ({
   }
 
   return (
-    <div className='h-full w-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
+    <div className='flex h-full w-full flex-col rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
       <div className='flex gap-x-2 border-b border-divider-subtle pb-3 pl-6 pr-4 pt-4'>
         <div className='flex grow flex-col gap-y-1'>
           <div className='system-2xs-semibold-uppercase text-text-accent'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
@@ -70,9 +70,15 @@ const FilePreview = ({
           <RiCloseLine className='size-[18px]' />
         </button>
       </div>
-      {isFetching && <Loading />}
+      {isFetching && (
+        <div className='grow'>
+          <Loading />
+        </div>
+      )}
       {!isFetching && fileData && (
-        <div className='body-md-regular overflow-hidden px-6 py-5 text-text-secondary'>{fileData.content}</div>
+        <div className='body-md-regular grow overflow-hidden px-6 py-5 text-text-secondary'>
+          {fileData.content}
+        </div>
       )}
     </div>
   )

web/app/components/datasets/documents/create-from-pipeline/preview/online-document-preview.tsx

@@ -8,17 +8,18 @@ import { formatNumberAbbreviated } from '@/utils/format'
 import Loading from './loading'
 import { Notion } from '@/app/components/base/icons/src/public/common'
 
-type NotionPagePreviewProps = {
+type OnlineDocumentPreviewProps = {
   currentPage: NotionPage
   hidePreview: () => void
 }
 
-const NotionPagePreview = ({
+const OnlineDocumentPreview = ({
   currentPage,
   hidePreview,
-}: NotionPagePreviewProps) => {
+}: OnlineDocumentPreviewProps) => {
   const { t } = useTranslation()
 
+  // todo: replace with a generic hook for previewing online documents
   const { data: notionPageData, isFetching } = usePreviewNotionPage({
     workspaceID: currentPage.workspace_id,
     pageID: currentPage.page_id,
@@ -26,8 +27,8 @@ const NotionPagePreview = ({
   })
 
   return (
-    <div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
+    <div className='flex h-full w-full flex-col rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
-      <div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'>
+      <div className='flex gap-x-2 border-b border-divider-subtle pb-3 pl-6 pr-4 pt-4'>
         <div className='flex grow flex-col gap-y-1'>
           <div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
           <div className='title-md-semi-bold text-tex-primary'>{currentPage?.page_name}</div>
@@ -52,14 +53,18 @@ const NotionPagePreview = ({
           <RiCloseLine className='size-[18px]' />
         </button>
       </div>
-      <div className='px-6 py-5'>
+      {isFetching && (
-        {isFetching && <Loading />}
+        <div className='grow'>
-        {!isFetching && notionPageData && (
+          <Loading />
-          <div className='body-md-regular overflow-hidden text-text-secondary'>{notionPageData.content}</div>
+        </div>
-        )}
+      )}
-      </div>
+      {!isFetching && notionPageData && (
+        <div className='body-md-regular grow overflow-hidden px-6 py-5 text-text-secondary'>
+          {notionPageData.content}
+        </div>
+      )}
     </div>
   )
 }
 
-export default NotionPagePreview
+export default OnlineDocumentPreview

web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx

@@ -17,8 +17,8 @@ const WebsitePreview = ({
   const { t } = useTranslation()
 
   return (
-    <div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
+    <div className='flex h-full w-full flex-col rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
-      <div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'>
+      <div className='flex gap-x-2 border-b border-divider-subtle pb-3 pl-6 pr-4 pt-4'>
         <div className='flex grow flex-col gap-y-1'>
           <div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
           <div className='title-md-semi-bold text-tex-primary'>{payload.title}</div>
@@ -38,8 +38,8 @@ const WebsitePreview = ({
           <RiCloseLine className='size-[18px]' />
         </button>
       </div>
-      <div className='px-6 py-5'>
+      <div className='body-md-regular grow overflow-hidden px-6 py-5 text-text-secondary'>
-        <div className='body-md-regular overflow-hidden text-text-secondary'>{payload.markdown}</div>
+        {payload.markdown}
       </div>
     </div>
   )

web/app/components/datasets/documents/detail/settings/pipeline-settings/index.tsx

@@ -78,7 +78,7 @@ const PipelineSettings = ({
 
   return (
     <div
-      className='relative flex h-[calc(100vh-56px)] overflow-x-auto rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
+      className='relative flex h-[calc(100vh-56px)] min-w-[1024px] overflow-x-auto rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
     >
       <div className='flex h-full flex-1 flex-col px-14'>
         <LeftHeader title={t('datasetPipeline.documentSettings.title')} />
@@ -93,7 +93,7 @@ const PipelineSettings = ({
         </div>
       </div>
       {/* Preview */}
-      <div className='flex h-full flex-1 shrink-0 pl-2 pt-2'>
+      <div className='flex h-full flex-1 pl-2 pt-2'>
         <ChunkPreview
           dataSourceType={documentDetail!.data_source_type as DatasourceType}
           // @ts-expect-error mock data // todo: remove mock data

web/app/components/datasets/preview/container.tsx

@@ -1,29 +1,32 @@
 import type { ComponentProps, FC, ReactNode } from 'react'
-import { forwardRef } from 'react'
 import classNames from '@/utils/classnames'
 
 export type PreviewContainerProps = ComponentProps<'div'> & {
   header: ReactNode
   mainClassName?: string
+  ref?: React.Ref<HTMLDivElement>
 }
 
-export const PreviewContainer: FC<PreviewContainerProps> = forwardRef((props, ref) => {
+const PreviewContainer: FC<PreviewContainerProps> = (props) => {
-  const { children, className, header, mainClassName, ...rest } = props
+  const { children, className, header, mainClassName, ref, ...rest } = props
   return <div className={className}>
     <div
       {...rest}
       ref={ref}
       className={classNames(
-        'flex flex-col w-full h-full overflow-y-auto rounded-l-xl border-t-[0.5px] border-l-[0.5px] border-components-panel-border bg-background-default-lighter shadow shadow-shadow-shadow-5',
+        'flex flex-col w-full h-full rounded-tl-xl border-t-[0.5px] border-l-[0.5px] border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5',
       )}
     >
       <header className='border-b border-divider-subtle pb-3 pl-5 pr-4 pt-4'>
         {header}
       </header>
-      <main className={classNames('py-5 px-6 w-full h-full', mainClassName)}>
+      <main className={classNames('py-5 px-6 w-full grow overflow-y-auto', mainClassName)}>
         {children}
       </main>
     </div>
   </div>
-})
+}
+
 PreviewContainer.displayName = 'PreviewContainer'
+
+export default PreviewContainer

web/app/components/workflow/workflow-preview/index.tsx

@@ -60,11 +60,13 @@ type WorkflowPreviewProps = {
   nodes: Node[]
   edges: Edge[]
   viewport: Viewport
+  className?: string
 }
 const WorkflowPreview = ({
   nodes,
   edges,
   viewport,
+  className,
 }: WorkflowPreviewProps) => {
   const [nodesData, setNodesData] = useState(initialNodes(nodes, edges))
   const [edgesData, setEdgesData] = useState(initialEdges(edges, nodes))
@@ -83,6 +85,7 @@ const WorkflowPreview = ({
       id='workflow-container'
       className={cn(
         'relative h-full w-full',
+        className,
       )}
     >
       <>
@@ -125,7 +128,7 @@ const WorkflowPreview = ({
         <Background
           gap={[14, 14]}
           size={2}
-          className="bg-workflow-canvas-workflow-bg"
+          className='bg-workflow-canvas-workflow-bg'
           color='var(--color-workflow-canvas-workflow-dot-color)'
         />
       </ReactFlow>

web/config/index.ts

@@ -275,7 +275,7 @@ Thought: {{agent_scratchpad}}
   `,
 }
 
-export const VAR_REGEX = /\{\{(#[a-zA-Z0-9_-]{1,50}(\.[a-zA-Z_]\w{0,29}){1,10}#)\}\}/gi
+export const VAR_REGEX = /\{\{(#[a-zA-Z0-9_-]{1,50}(\.[a-zA-Z_]?\w{0,29}){1,10}(\.[a-zA-Z0-9_-]{1,50})?#)\}\}/gi
 
 export const resetReg = () => VAR_REGEX.lastIndex = 0