diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index 3eb2f0445d..c435ca108c 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -274,7 +274,6 @@ const CreateFormPipeline = () => { {datasource?.type === DatasourceType.websiteCrawl && ( { onCheckedCrawlResultChange={setWebsitePages} onJobIdChange={setWebsiteCrawlJobId} onPreview={updateCurrentWebsite} + usingPublished /> )} {isShowVectorSpaceFull && ( diff --git a/web/app/components/rag-pipeline/components/input-field/index.tsx b/web/app/components/rag-pipeline/components/input-field/index.tsx index e481e5c924..c4947a68a7 100644 --- a/web/app/components/rag-pipeline/components/input-field/index.tsx +++ b/web/app/components/rag-pipeline/components/input-field/index.tsx @@ -56,16 +56,17 @@ const InputFieldDialog = ({ const { doSyncWorkflowDraft } = useNodesSyncDraft() - useUnmount(() => { - doSyncWorkflowDraft() - }) - - const { run: syncWorkflowDraft } = useDebounceFn(() => { - doSyncWorkflowDraft() + const { run: syncWorkflowDraft, cancel: cancelSyncWorkflowDraft } = useDebounceFn(async () => { + await doSyncWorkflowDraft() }, { wait: 500, }) + useUnmount(() => { + cancelSyncWorkflowDraft() + doSyncWorkflowDraft() + }) + const datasourceNodeDataMap = useMemo(() => { const datasourceNodeDataMap: Record = {} const datasourceNodes: Node[] = nodes.filter(node => node.data.type === BlockEnum.DataSource) @@ -76,7 +77,7 @@ const InputFieldDialog = ({ return datasourceNodeDataMap }, [nodes]) - const updateInputFields = useCallback((key: string, value: InputVar[]) => { + const updateInputFields = useCallback(async (key: string, value: InputVar[]) => { inputFieldsMap.current[key] = value const newRagPipelineVariables: RAGPipelineVariables = [] Object.keys(inputFieldsMap.current).forEach((key) => { @@ -89,7 +90,7 @@ const InputFieldDialog = ({ }) }) setRagPipelineVariables?.(newRagPipelineVariables) - syncWorkflowDraft() + await syncWorkflowDraft() }, [setRagPipelineVariables, syncWorkflowDraft]) const closePanel = useCallback(() => { diff --git a/web/app/components/rag-pipeline/components/input-field/preview/data-source.tsx b/web/app/components/rag-pipeline/components/input-field/preview/data-source.tsx index 82c676d548..aa7c686784 100644 --- a/web/app/components/rag-pipeline/components/input-field/preview/data-source.tsx +++ b/web/app/components/rag-pipeline/components/input-field/preview/data-source.tsx @@ -3,17 +3,24 @@ import { useTranslation } from 'react-i18next' import DataSourceOptions from '../../panel/test-run/data-source-options' import Form from './form' import type { Datasource } from '../../panel/test-run/types' +import { useStore } from '@/app/components/workflow/store' +import { useDraftPipelinePreProcessingParams } from '@/service/use-pipeline' type DatasourceProps = { onSelect: (dataSource: Datasource) => void - datasourceNodeId: string + dataSourceNodeId: string } const DataSource = ({ onSelect: setDatasource, - datasourceNodeId, + dataSourceNodeId, }: DatasourceProps) => { const { t } = useTranslation() + const pipelineId = useStore(state => state.pipelineId) + const { data: paramsConfig } = useDraftPipelinePreProcessingParams({ + pipeline_id: pipelineId!, + node_id: dataSourceNodeId, + }, !!pipelineId && !!dataSourceNodeId) return (
@@ -23,10 +30,10 @@ const DataSource = ({
-
+
) } diff --git a/web/app/components/rag-pipeline/components/input-field/preview/index.tsx b/web/app/components/rag-pipeline/components/input-field/preview/index.tsx index 692306a642..d3ca95cb56 100644 --- a/web/app/components/rag-pipeline/components/input-field/preview/index.tsx +++ b/web/app/components/rag-pipeline/components/input-field/preview/index.tsx @@ -40,13 +40,15 @@ const PreviewPanel = ({ + {/* Data source form Preview */}
+ {/* Process documents form Preview */} ) diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx index 9c00aa1d7d..afbba09594 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx @@ -4,12 +4,12 @@ import OptionCard from './option-card' import type { Datasource } from '../types' type DataSourceOptionsProps = { - datasourceNodeId: string + dataSourceNodeId: string onSelect: (option: Datasource) => void } const DataSourceOptions = ({ - datasourceNodeId, + dataSourceNodeId, onSelect, }: DataSourceOptionsProps) => { const { datasources, options } = useDatasourceOptions() @@ -34,7 +34,7 @@ const DataSourceOptions = ({ key={option.value} label={option.label} nodeData={option.data} - selected={datasourceNodeId === option.value} + selected={dataSourceNodeId === option.value} onClick={handelSelect.bind(null, option.value)} /> ))} diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx index e3f3edac0a..4c598bfc10 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx @@ -1,5 +1,5 @@ 'use client' -import React, { useCallback, useEffect, useState } from 'react' +import React, { useCallback, useEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import type { CrawlResultItem } from '@/models/datasets' import Header from '@/app/components/datasets/create/website/base/header' @@ -7,15 +7,17 @@ import Options from './options' import Crawling from './crawling' import ErrorMessage from './error-message' import CrawledResult from './crawled-result' -import type { RAGPipelineVariables } from '@/models/pipeline' -import { useDatasourceNodeRun } from '@/service/use-pipeline' -import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' +import { + useDatasourceNodeRun, + useDraftPipelinePreProcessingParams, + usePublishedPipelineProcessingParams, +} from '@/service/use-pipeline' +import { useStore } from '@/app/components/workflow/store' const I18N_PREFIX = 'datasetCreation.stepOne.website' type CrawlerProps = { nodeId: string - variables: RAGPipelineVariables checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onJobIdChange: (jobId: string) => void @@ -25,6 +27,7 @@ type CrawlerProps = { docLink: string } onPreview?: (payload: CrawlResultItem) => void + usingPublished?: boolean } enum Step { @@ -35,17 +38,23 @@ enum Step { const Crawler = ({ nodeId, - variables, checkedCrawlResult, headerInfo, onCheckedCrawlResultChange, onJobIdChange, onPreview, + usingPublished = false, }: CrawlerProps) => { const { t } = useTranslation() const [step, setStep] = useState(Step.init) const [controlFoldOptions, setControlFoldOptions] = useState(0) - const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) + const pipelineId = useStore(s => s.pipelineId) + + const usePreProcessingParams = useRef(usingPublished ? usePublishedPipelineProcessingParams : useDraftPipelinePreProcessingParams) + const { data: paramsConfig } = usePreProcessingParams.current({ + pipeline_id: pipelineId!, + node_id: nodeId, + }, !!pipelineId && !!nodeId) useEffect(() => { if (step !== Step.init) @@ -95,7 +104,7 @@ const Crawler = ({ />
{ diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx index 181322256f..50be76e524 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx @@ -1,12 +1,10 @@ 'use client' import React from 'react' import type { CrawlResultItem } from '@/models/datasets' -import type { RAGPipelineVariables } from '@/models/pipeline' import Crawler from './base/crawler' type WebsiteCrawlProps = { nodeId: string - variables: RAGPipelineVariables checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onJobIdChange: (jobId: string) => void @@ -16,26 +14,27 @@ type WebsiteCrawlProps = { docLink: string } onPreview?: (payload: CrawlResultItem) => void + usingPublished?: boolean } const WebsiteCrawl = ({ nodeId, - variables, checkedCrawlResult, headerInfo, onCheckedCrawlResultChange, onJobIdChange, onPreview, + usingPublished, }: WebsiteCrawlProps) => { return ( ) } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index 84b48c3653..97c0a4002f 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -117,7 +117,7 @@ const TestRunPanel = () => { <>
{datasource?.type === DatasourceType.localFile && ( @@ -139,7 +139,6 @@ const TestRunPanel = () => { {datasource?.type === DatasourceType.websiteCrawl && ( { const { pipeline_id, node_id } = params return useQuery({ - queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id], + queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id, node_id], queryFn: () => { return get(`/rag/pipelines/${pipeline_id}/workflows/draft/processing/parameters`, { params: { @@ -157,7 +157,7 @@ export const useDraftPipelineProcessingParams = (params: PipelineProcessingParam export const usePublishedPipelineProcessingParams = (params: PipelineProcessingParamsRequest) => { const { pipeline_id, node_id } = params return useQuery({ - queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id], + queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id, node_id], queryFn: () => { return get(`/rag/pipelines/${pipeline_id}/workflows/published/processing/parameters`, { params: { @@ -165,6 +165,7 @@ export const usePublishedPipelineProcessingParams = (params: PipelineProcessingP }, }) }, + staleTime: 0, }) } @@ -254,7 +255,7 @@ export const useUpdateDataSourceCredentials = ( export const useDraftPipelinePreProcessingParams = (params: PipelinePreProcessingParamsRequest, enabled = true) => { const { pipeline_id, node_id } = params return useQuery({ - queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id], + queryKey: [NAME_SPACE, 'pipeline-pre-processing-params', pipeline_id, node_id], queryFn: () => { return get(`/rag/pipelines/${pipeline_id}/workflows/draft/pre-processing/parameters`, { params: { @@ -270,7 +271,7 @@ export const useDraftPipelinePreProcessingParams = (params: PipelinePreProcessin export const usePublishedPipelinePreProcessingParams = (params: PipelinePreProcessingParamsRequest, enabled = true) => { const { pipeline_id, node_id } = params return useQuery({ - queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id], + queryKey: [NAME_SPACE, 'pipeline-pre-processing-params', pipeline_id, node_id], queryFn: () => { return get(`/rag/pipelines/${pipeline_id}/workflows/published/processing/parameters`, { params: { @@ -278,6 +279,7 @@ export const usePublishedPipelinePreProcessingParams = (params: PipelinePreProce }, }) }, + staleTime: 0, enabled, }) }