From a33ac77a2206b7e3cc992c26b4953fde825fddb4 Mon Sep 17 00:00:00 2001 From: Coding On Star <447357187@qq.com> Date: Thu, 15 Jan 2026 10:33:48 +0800 Subject: [PATCH] feat: implement document creation pipeline with multi-step wizard and datasource management (#30843) Co-authored-by: CodingOnStar --- .../create-from-dsl-modal/index.spec.tsx | 2028 +++++++++++++ .../create-from-pipeline/hooks/index.ts | 5 + .../hooks/use-add-documents-steps.ts | 41 + .../hooks/use-datasource-actions.ts | 321 ++ .../hooks/use-datasource-options.ts | 27 + .../use-datasource-store.ts} | 76 +- .../hooks/use-datasource-ui-state.ts | 132 + .../create-from-pipeline/index.spec.tsx | 2698 +++++++++++++++++ .../documents/create-from-pipeline/index.tsx | 683 +---- .../create-from-pipeline/steps/index.ts | 3 + .../steps/preview-panel.tsx | 112 + .../steps/step-one-content.tsx | 110 + .../steps/step-three-content.tsx | 23 + .../steps/step-two-content.tsx | 38 + .../utils/datasource-info-builder.ts | 63 + 15 files changed, 5783 insertions(+), 577 deletions(-) create mode 100644 web/app/components/datasets/create-from-pipeline/create-options/create-from-dsl-modal/index.spec.tsx create mode 100644 web/app/components/datasets/documents/create-from-pipeline/hooks/index.ts create mode 100644 web/app/components/datasets/documents/create-from-pipeline/hooks/use-add-documents-steps.ts create mode 100644 web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-actions.ts create mode 100644 web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-options.ts rename web/app/components/datasets/documents/create-from-pipeline/{hooks.ts => hooks/use-datasource-store.ts} (70%) create mode 100644 web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-ui-state.ts create mode 100644 web/app/components/datasets/documents/create-from-pipeline/index.spec.tsx create mode 100644 web/app/components/datasets/documents/create-from-pipeline/steps/index.ts create mode 100644 web/app/components/datasets/documents/create-from-pipeline/steps/preview-panel.tsx create mode 100644 web/app/components/datasets/documents/create-from-pipeline/steps/step-one-content.tsx create mode 100644 web/app/components/datasets/documents/create-from-pipeline/steps/step-three-content.tsx create mode 100644 web/app/components/datasets/documents/create-from-pipeline/steps/step-two-content.tsx create mode 100644 web/app/components/datasets/documents/create-from-pipeline/utils/datasource-info-builder.ts diff --git a/web/app/components/datasets/create-from-pipeline/create-options/create-from-dsl-modal/index.spec.tsx b/web/app/components/datasets/create-from-pipeline/create-options/create-from-dsl-modal/index.spec.tsx new file mode 100644 index 0000000000..8ae7a41e72 --- /dev/null +++ b/web/app/components/datasets/create-from-pipeline/create-options/create-from-dsl-modal/index.spec.tsx @@ -0,0 +1,2028 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react' +import DSLConfirmModal from './dsl-confirm-modal' +import Header from './header' +import CreateFromDSLModal, { CreateFromDSLModalTab } from './index' +import Tab from './tab' +import TabItem from './tab/item' +import Uploader from './uploader' + +// Mock next/navigation +const mockPush = vi.fn() +vi.mock('next/navigation', () => ({ + useRouter: () => ({ + push: mockPush, + }), +})) + +// Mock service hooks +const mockImportDSL = vi.fn() +const mockImportDSLConfirm = vi.fn() + +vi.mock('@/service/use-pipeline', () => ({ + useImportPipelineDSL: () => ({ + mutateAsync: mockImportDSL, + }), + useImportPipelineDSLConfirm: () => ({ + mutateAsync: mockImportDSLConfirm, + }), +})) + +// Mock plugin dependencies hook +const mockHandleCheckPluginDependencies = vi.fn() + +vi.mock('@/app/components/workflow/plugin-dependency/hooks', () => ({ + usePluginDependencies: () => ({ + handleCheckPluginDependencies: mockHandleCheckPluginDependencies, + }), +})) + +// Mock toast context +const mockNotify = vi.fn() + +vi.mock('use-context-selector', async () => { + const actual = await vi.importActual('use-context-selector') + return { + ...actual, + useContext: vi.fn(() => ({ notify: mockNotify })), + } +}) + +// Test data builders +const createMockFile = (name = 'test.pipeline'): File => { + return new File(['test content'], name, { type: 'application/octet-stream' }) +} + +const createImportDSLResponse = (overrides = {}) => ({ + id: 'import-123', + status: 'completed' as const, + pipeline_id: 'pipeline-456', + dataset_id: 'dataset-789', + current_dsl_version: '1.0.0', + imported_dsl_version: '1.0.0', + ...overrides, +}) + +// Helper function to create QueryClient wrapper +const createWrapper = () => { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }) + return ({ children }: { children: React.ReactNode }) => ( + + {children} + + ) +} + +describe('CreateFromDSLModal', () => { + beforeEach(() => { + vi.clearAllMocks() + mockImportDSL.mockReset() + mockImportDSLConfirm.mockReset() + mockPush.mockReset() + mockNotify.mockReset() + mockHandleCheckPluginDependencies.mockReset() + }) + + // ============================================ + // Rendering Tests + // ============================================ + describe('Rendering', () => { + it('should render without crashing when show is true', () => { + render( + , + { wrapper: createWrapper() }, + ) + + expect(screen.getByText('app.importFromDSL')).toBeInTheDocument() + }) + + it('should not render modal content when show is false', () => { + render( + , + { wrapper: createWrapper() }, + ) + + // Modal with show=false should not display its content visibly + const modal = screen.queryByText('app.importFromDSL') + expect(modal).toBeNull() + }) + + it('should render file tab by default', () => { + render( + , + { wrapper: createWrapper() }, + ) + + expect(screen.getByText('app.importFromDSLFile')).toBeInTheDocument() + expect(screen.getByText('app.importFromDSLUrl')).toBeInTheDocument() + }) + + it('should render cancel and import buttons', () => { + render( + , + { wrapper: createWrapper() }, + ) + + expect(screen.getByText('app.newApp.Cancel')).toBeInTheDocument() + expect(screen.getByText('app.newApp.import')).toBeInTheDocument() + }) + + it('should render uploader when file tab is active', () => { + render( + , + { wrapper: createWrapper() }, + ) + + expect(screen.getByText('app.dslUploader.button')).toBeInTheDocument() + }) + + it('should render URL input when URL tab is active', () => { + render( + , + { wrapper: createWrapper() }, + ) + + expect(screen.getByText('DSL URL')).toBeInTheDocument() + expect(screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder')).toBeInTheDocument() + }) + }) + + // ============================================ + // Props Testing + // ============================================ + describe('Props', () => { + it('should use FROM_FILE as default activeTab', () => { + render( + , + { wrapper: createWrapper() }, + ) + + // File tab content should be visible + expect(screen.getByText('app.dslUploader.button')).toBeInTheDocument() + }) + + it('should use provided activeTab prop', () => { + render( + , + { wrapper: createWrapper() }, + ) + + expect(screen.getByText('DSL URL')).toBeInTheDocument() + }) + + it('should use provided dslUrl prop', () => { + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + expect(input).toHaveValue('https://example.com/test.pipeline') + }) + + it('should call onClose when cancel button is clicked', () => { + const onClose = vi.fn() + render( + , + { wrapper: createWrapper() }, + ) + + fireEvent.click(screen.getByText('app.newApp.Cancel')) + expect(onClose).toHaveBeenCalled() + }) + }) + + // ============================================ + // State Management Tests + // ============================================ + describe('State Management', () => { + it('should switch between tabs', () => { + render( + , + { wrapper: createWrapper() }, + ) + + // Initially file tab is active + expect(screen.getByText('app.dslUploader.button')).toBeInTheDocument() + + // Click URL tab + fireEvent.click(screen.getByText('app.importFromDSLUrl')) + + // URL input should be visible + expect(screen.getByText('DSL URL')).toBeInTheDocument() + }) + + it('should update URL value when typing', () => { + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + expect(input).toHaveValue('https://example.com/test.pipeline') + }) + + it('should have disabled import button when no file is selected in file tab', () => { + render( + , + { wrapper: createWrapper() }, + ) + + const importButton = screen.getByText('app.newApp.import').closest('button') + expect(importButton).toBeDisabled() + }) + + it('should have disabled import button when no URL is entered in URL tab', () => { + render( + , + { wrapper: createWrapper() }, + ) + + const importButton = screen.getByText('app.newApp.import').closest('button') + expect(importButton).toBeDisabled() + }) + + it('should enable import button when URL is entered', () => { + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button') + expect(importButton).not.toBeDisabled() + }) + }) + + // ============================================ + // API Call Tests + // ============================================ + describe('API Calls', () => { + it('should call importDSL with URL mode when URL tab is active', async () => { + mockImportDSL.mockResolvedValue(createImportDSLResponse()) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + await waitFor(() => { + expect(mockImportDSL).toHaveBeenCalledWith({ + mode: 'yaml-url', + yaml_url: 'https://example.com/test.pipeline', + }) + }) + }) + + it('should handle successful import with COMPLETED status', async () => { + const onSuccess = vi.fn() + const onClose = vi.fn() + mockImportDSL.mockResolvedValue(createImportDSLResponse({ status: 'completed' })) + mockHandleCheckPluginDependencies.mockResolvedValue(undefined) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + await waitFor(() => { + expect(onSuccess).toHaveBeenCalled() + expect(onClose).toHaveBeenCalled() + expect(mockNotify).toHaveBeenCalledWith(expect.objectContaining({ + type: 'success', + })) + expect(mockPush).toHaveBeenCalledWith('/datasets/dataset-789/pipeline') + }) + }) + + it('should handle import with COMPLETED_WITH_WARNINGS status', async () => { + const onSuccess = vi.fn() + const onClose = vi.fn() + mockImportDSL.mockResolvedValue(createImportDSLResponse({ status: 'completed-with-warnings' })) + mockHandleCheckPluginDependencies.mockResolvedValue(undefined) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + await waitFor(() => { + expect(mockNotify).toHaveBeenCalledWith(expect.objectContaining({ + type: 'warning', + })) + }) + }) + + it('should handle import with PENDING status and show error modal', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + const onClose = vi.fn() + mockImportDSL.mockResolvedValue(createImportDSLResponse({ + status: 'pending', + imported_dsl_version: '0.9.0', + current_dsl_version: '1.0.0', + })) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + await waitFor(() => { + expect(onClose).toHaveBeenCalled() + }) + + // Advance timer to show error modal + await act(async () => { + vi.advanceTimersByTime(400) + }) + + await waitFor(() => { + expect(screen.getByText('app.newApp.appCreateDSLErrorTitle')).toBeInTheDocument() + }) + + vi.useRealTimers() + }) + + it('should handle API error', async () => { + mockImportDSL.mockResolvedValue(null) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + await waitFor(() => { + expect(mockNotify).toHaveBeenCalledWith(expect.objectContaining({ + type: 'error', + })) + }) + }) + + it('should handle FAILED status', async () => { + mockImportDSL.mockResolvedValue(createImportDSLResponse({ status: 'failed' })) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + await waitFor(() => { + expect(mockNotify).toHaveBeenCalledWith(expect.objectContaining({ + type: 'error', + })) + }) + }) + + it('should check plugin dependencies after successful import', async () => { + mockImportDSL.mockResolvedValue(createImportDSLResponse({ + status: 'completed', + pipeline_id: 'pipeline-123', + })) + mockHandleCheckPluginDependencies.mockResolvedValue(undefined) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + await waitFor(() => { + expect(mockHandleCheckPluginDependencies).toHaveBeenCalledWith('pipeline-123', true) + }) + }) + }) + + // ============================================ + // Event Handler Tests + // ============================================ + describe('Event Handlers', () => { + it('should call onClose when header close button is clicked', () => { + const onClose = vi.fn() + render( + , + { wrapper: createWrapper() }, + ) + + // Find and click the close icon in header + const closeIcon = document.querySelector('[class*="cursor-pointer"]') + + if (closeIcon) { + fireEvent.click(closeIcon) + expect(onClose).toHaveBeenCalled() + } + }) + + it('should close modal on ESC key press', () => { + const onClose = vi.fn() + render( + , + { wrapper: createWrapper() }, + ) + + // Trigger ESC key event - ahooks useKeyPress listens for 'esc' which maps to Escape key + // Need to dispatch on window/document with the correct event properties + const escEvent = new KeyboardEvent('keydown', { + key: 'Escape', + code: 'Escape', + keyCode: 27, + bubbles: true, + }) + document.dispatchEvent(escEvent) + + expect(onClose).toHaveBeenCalled() + }) + + it('should not close on ESC when error modal is shown', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + const onClose = vi.fn() + mockImportDSL.mockResolvedValue(createImportDSLResponse({ status: 'pending' })) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + await waitFor(() => { + expect(onClose).toHaveBeenCalled() + }) + + // Clear previous calls + onClose.mockClear() + + // Show error modal + await act(async () => { + vi.advanceTimersByTime(400) + }) + + // Now ESC should not close main modal because error modal is shown + const escEvent = new KeyboardEvent('keydown', { + key: 'Escape', + code: 'Escape', + keyCode: 27, + bubbles: true, + }) + document.dispatchEvent(escEvent) + + // onClose should not be called again when error modal is shown + expect(onClose).not.toHaveBeenCalled() + + vi.useRealTimers() + }) + + it('should prevent duplicate submissions', async () => { + mockImportDSL.mockImplementation(() => new Promise(resolve => + setTimeout(() => resolve(createImportDSLResponse()), 1000), + )) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + + // Click multiple times rapidly + fireEvent.click(importButton) + fireEvent.click(importButton) + fireEvent.click(importButton) + + // Should only be called once due to isCreatingRef + await waitFor(() => { + expect(mockImportDSL).toHaveBeenCalledTimes(1) + }) + }) + }) + + // ============================================ + // Memoization Tests + // ============================================ + describe('Memoization', () => { + it('should correctly compute buttonDisabled based on currentTab and file/URL', () => { + render( + , + { wrapper: createWrapper() }, + ) + + // File tab with no file - disabled + let importButton = screen.getByText('app.newApp.import').closest('button') + expect(importButton).toBeDisabled() + + // Switch to URL tab by clicking on it + fireEvent.click(screen.getByText('app.importFromDSLUrl')) + + // Still disabled (no URL) + importButton = screen.getByText('app.newApp.import').closest('button') + expect(importButton).toBeDisabled() + + // Add URL value - should enable + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com' } }) + + importButton = screen.getByText('app.newApp.import').closest('button') + expect(importButton).not.toBeDisabled() + }) + }) + + // ============================================ + // Edge Cases Tests + // ============================================ + describe('Edge Cases', () => { + it('should handle empty URL gracefully', () => { + render( + , + { wrapper: createWrapper() }, + ) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + // Should not call API with empty URL + expect(mockImportDSL).not.toHaveBeenCalled() + }) + + it('should handle undefined onSuccess gracefully', async () => { + mockImportDSL.mockResolvedValue(createImportDSLResponse()) + mockHandleCheckPluginDependencies.mockResolvedValue(undefined) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + + // Should not throw + fireEvent.click(importButton) + + await waitFor(() => { + expect(mockPush).toHaveBeenCalled() + }) + }) + + it('should handle response without pipeline_id', async () => { + mockImportDSL.mockResolvedValue(createImportDSLResponse({ + status: 'completed', + pipeline_id: null, + })) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + await waitFor(() => { + // Should not call handleCheckPluginDependencies when pipeline_id is null + expect(mockHandleCheckPluginDependencies).not.toHaveBeenCalled() + }) + }) + + it('should handle empty file in file tab gracefully', () => { + render( + , + { wrapper: createWrapper() }, + ) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + // Should not call API with no file + expect(mockImportDSL).not.toHaveBeenCalled() + }) + + it('should return early in onCreate when file tab has no file (direct trigger)', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + + // Test the early return branch by force-triggering the button even when disabled + render( + , + { wrapper: createWrapper() }, + ) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + + // Remove disabled attribute temporarily to test the early return + importButton.removeAttribute('disabled') + + // Dispatch a native click event to bypass any React disabled checks + const clickEvent = new MouseEvent('click', { bubbles: true, cancelable: true }) + importButton.dispatchEvent(clickEvent) + + // Wait for debounce to trigger + await act(async () => { + vi.advanceTimersByTime(400) + }) + + // Should not call API due to early return in onCreate + expect(mockImportDSL).not.toHaveBeenCalled() + + vi.useRealTimers() + }) + + it('should return early in onCreate when URL tab has no URL (direct trigger)', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + + render( + , + { wrapper: createWrapper() }, + ) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + + // Remove disabled attribute to test the early return + importButton.removeAttribute('disabled') + + // Dispatch a native click event + const clickEvent = new MouseEvent('click', { bubbles: true, cancelable: true }) + importButton.dispatchEvent(clickEvent) + + // Wait for debounce + await act(async () => { + vi.advanceTimersByTime(400) + }) + + // Should not call API due to early return + expect(mockImportDSL).not.toHaveBeenCalled() + + vi.useRealTimers() + }) + }) + + // ============================================ + // File Import Tests (covers readFile, handleFile, file mode import) + // ============================================ + describe('File Import', () => { + it('should read file content when file is selected', async () => { + mockImportDSL.mockResolvedValue(createImportDSLResponse()) + mockHandleCheckPluginDependencies.mockResolvedValue(undefined) + + render( + , + { wrapper: createWrapper() }, + ) + + // Create a mock file with content + const fileContent = 'test yaml content' + const mockFile = new File([fileContent], 'test.pipeline', { type: 'application/octet-stream' }) + + // Get the file input and simulate file selection + const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement + Object.defineProperty(fileInput, 'files', { + value: [mockFile], + configurable: true, + }) + fireEvent.change(fileInput) + + // Wait for FileReader to complete + await waitFor(() => { + const importButton = screen.getByText('app.newApp.import').closest('button') + expect(importButton).not.toBeDisabled() + }) + + // Click import button + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + // Verify API was called with file content + await waitFor(() => { + expect(mockImportDSL).toHaveBeenCalledWith({ + mode: 'yaml-content', + yaml_content: fileContent, + }) + }) + }) + + it('should clear file content when file is removed', async () => { + render( + , + { wrapper: createWrapper() }, + ) + + // First add a file + const mockFile = new File(['content'], 'test.pipeline', { type: 'application/octet-stream' }) + const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement + Object.defineProperty(fileInput, 'files', { + value: [mockFile], + configurable: true, + }) + fireEvent.change(fileInput) + + // Wait for file to be displayed + await waitFor(() => { + expect(screen.getByText('test.pipeline')).toBeInTheDocument() + }) + + // Now remove the file by clicking delete button (inside ActionButton) + const actionButton = document.querySelector('[class*="group-hover"]') + const deleteButton = actionButton?.querySelector('button') + if (deleteButton) { + fireEvent.click(deleteButton) + // File should be removed - uploader prompt should show again + await waitFor(() => { + expect(screen.getByText('app.dslUploader.button')).toBeInTheDocument() + }) + } + }) + }) + + // ============================================ + // DSL Confirm Flow Tests (covers onDSLConfirm) + // ============================================ + describe('DSL Confirm Flow', () => { + it('should handle DSL confirm success', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + const onSuccess = vi.fn() + const onClose = vi.fn() + + mockImportDSL.mockResolvedValue(createImportDSLResponse({ + id: 'import-123', + status: 'pending', + imported_dsl_version: '0.9.0', + current_dsl_version: '1.0.0', + })) + + mockImportDSLConfirm.mockResolvedValue({ + status: 'completed', + pipeline_id: 'pipeline-456', + dataset_id: 'dataset-789', + }) + + mockHandleCheckPluginDependencies.mockResolvedValue(undefined) + + render( + , + { wrapper: createWrapper() }, + ) + + // Enter URL and submit + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + // Wait for pending status handling + await waitFor(() => { + expect(onClose).toHaveBeenCalled() + }) + + // Advance timer to show error modal + await act(async () => { + vi.advanceTimersByTime(400) + }) + + // Click confirm button in error modal + await waitFor(() => { + expect(screen.getByText('app.newApp.Confirm')).toBeInTheDocument() + }) + + fireEvent.click(screen.getByText('app.newApp.Confirm')) + + // Verify confirm was called + await waitFor(() => { + expect(mockImportDSLConfirm).toHaveBeenCalledWith('import-123') + }) + + // Verify success handling + await waitFor(() => { + expect(mockNotify).toHaveBeenCalledWith(expect.objectContaining({ + type: 'success', + })) + }) + + vi.useRealTimers() + }) + + it('should handle DSL confirm with no importId', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + + mockImportDSL.mockResolvedValue(createImportDSLResponse({ + id: '', // Empty id + status: 'pending', + })) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + await act(async () => { + vi.advanceTimersByTime(400) + }) + + // Click confirm - should return early since importId is empty + await waitFor(() => { + expect(screen.getByText('app.newApp.Confirm')).toBeInTheDocument() + }) + + fireEvent.click(screen.getByText('app.newApp.Confirm')) + + // Confirm should not be called since importId is empty string (falsy) + expect(mockImportDSLConfirm).not.toHaveBeenCalled() + + vi.useRealTimers() + }) + + it('should handle DSL confirm API error', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + + mockImportDSL.mockResolvedValue(createImportDSLResponse({ + id: 'import-123', + status: 'pending', + })) + + mockImportDSLConfirm.mockResolvedValue(null) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + fireEvent.click(screen.getByText('app.newApp.import').closest('button')!) + + await act(async () => { + vi.advanceTimersByTime(400) + }) + + await waitFor(() => { + expect(screen.getByText('app.newApp.Confirm')).toBeInTheDocument() + }) + + fireEvent.click(screen.getByText('app.newApp.Confirm')) + + await waitFor(() => { + expect(mockNotify).toHaveBeenCalledWith(expect.objectContaining({ + type: 'error', + })) + }) + + vi.useRealTimers() + }) + + it('should handle DSL confirm with FAILED status', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + + mockImportDSL.mockResolvedValue(createImportDSLResponse({ + id: 'import-123', + status: 'pending', + })) + + mockImportDSLConfirm.mockResolvedValue({ + status: 'failed', + pipeline_id: 'pipeline-456', + dataset_id: 'dataset-789', + }) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + fireEvent.click(screen.getByText('app.newApp.import').closest('button')!) + + await act(async () => { + vi.advanceTimersByTime(400) + }) + + await waitFor(() => { + expect(screen.getByText('app.newApp.Confirm')).toBeInTheDocument() + }) + + fireEvent.click(screen.getByText('app.newApp.Confirm')) + + await waitFor(() => { + expect(mockNotify).toHaveBeenCalledWith(expect.objectContaining({ + type: 'error', + })) + }) + + vi.useRealTimers() + }) + + it('should close error modal when cancel is clicked', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + + mockImportDSL.mockResolvedValue(createImportDSLResponse({ + status: 'pending', + })) + + render( + , + { wrapper: createWrapper() }, + ) + + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/test.pipeline' } }) + + fireEvent.click(screen.getByText('app.newApp.import').closest('button')!) + + await act(async () => { + vi.advanceTimersByTime(400) + }) + + // Error modal should be visible + await waitFor(() => { + expect(screen.getByText('app.newApp.appCreateDSLErrorTitle')).toBeInTheDocument() + }) + + // There are two Cancel buttons now (one in main modal footer, one in error modal) + // Find the Cancel button in the error modal context + const cancelButtons = screen.getAllByText('app.newApp.Cancel') + // Click the last Cancel button (the one in the error modal) + fireEvent.click(cancelButtons[cancelButtons.length - 1]) + + vi.useRealTimers() + }) + }) +}) + +// ============================================ +// Header Component Tests +// ============================================ +describe('Header', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + describe('Rendering', () => { + it('should render title', () => { + render(
) + expect(screen.getByText('app.importFromDSL')).toBeInTheDocument() + }) + + it('should render close icon', () => { + render(
) + // Check for close icon container + const closeButton = document.querySelector('[class*="cursor-pointer"]') + expect(closeButton).toBeInTheDocument() + }) + }) + + describe('Event Handlers', () => { + it('should call onClose when close icon is clicked', () => { + const onClose = vi.fn() + render(
) + + const closeButton = document.querySelector('[class*="cursor-pointer"]')! + fireEvent.click(closeButton) + + expect(onClose).toHaveBeenCalled() + }) + }) +}) + +// ============================================ +// Tab Component Tests +// ============================================ +describe('Tab', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + describe('Rendering', () => { + it('should render both tabs', () => { + render( + , + ) + + expect(screen.getByText('app.importFromDSLFile')).toBeInTheDocument() + expect(screen.getByText('app.importFromDSLUrl')).toBeInTheDocument() + }) + }) + + describe('Event Handlers', () => { + it('should call setCurrentTab when clicking file tab', () => { + const setCurrentTab = vi.fn() + render( + , + ) + + fireEvent.click(screen.getByText('app.importFromDSLFile')) + // Tab uses bind() which passes the key as first argument and event as second + expect(setCurrentTab).toHaveBeenCalled() + expect(setCurrentTab.mock.calls[0][0]).toBe(CreateFromDSLModalTab.FROM_FILE) + }) + + it('should call setCurrentTab when clicking URL tab', () => { + const setCurrentTab = vi.fn() + render( + , + ) + + fireEvent.click(screen.getByText('app.importFromDSLUrl')) + // Tab uses bind() which passes the key as first argument and event as second + expect(setCurrentTab).toHaveBeenCalled() + expect(setCurrentTab.mock.calls[0][0]).toBe(CreateFromDSLModalTab.FROM_URL) + }) + }) +}) + +// ============================================ +// Tab Item Component Tests +// ============================================ +describe('TabItem', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + describe('Rendering', () => { + it('should render label', () => { + render( + , + ) + + expect(screen.getByText('Test Tab')).toBeInTheDocument() + }) + + it('should render active indicator when active', () => { + render( + , + ) + + // Active indicator is the bottom border div + const indicator = document.querySelector('[class*="bg-util-colors-blue"]') + expect(indicator).toBeInTheDocument() + }) + + it('should not render active indicator when inactive', () => { + render( + , + ) + + const indicator = document.querySelector('[class*="bg-util-colors-blue"]') + expect(indicator).toBeNull() + }) + + it('should have active text color when active', () => { + render( + , + ) + + const item = screen.getByText('Test Tab') + expect(item.className).toContain('text-text-primary') + }) + + it('should have inactive text color when inactive', () => { + render( + , + ) + + const item = screen.getByText('Test Tab') + expect(item.className).toContain('text-text-tertiary') + }) + }) + + describe('Event Handlers', () => { + it('should call onClick when clicked', () => { + const onClick = vi.fn() + render( + , + ) + + fireEvent.click(screen.getByText('Test Tab')) + expect(onClick).toHaveBeenCalled() + }) + }) +}) + +// ============================================ +// Uploader Component Tests +// ============================================ +describe('Uploader', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + describe('Rendering', () => { + it('should render upload prompt when no file', () => { + render( + , + ) + + expect(screen.getByText('app.dslUploader.button')).toBeInTheDocument() + expect(screen.getByText('app.dslUploader.browse')).toBeInTheDocument() + }) + + it('should render file info when file is selected', () => { + const mockFile = createMockFile('test.pipeline') + + render( + , + ) + + expect(screen.getByText('test.pipeline')).toBeInTheDocument() + expect(screen.getByText('PIPELINE')).toBeInTheDocument() + }) + + it('should apply custom className', () => { + const { container } = render( + , + ) + + expect(container.firstChild).toHaveClass('custom-class') + }) + }) + + describe('Event Handlers', () => { + it('should call updateFile when browse link is clicked and file is selected', async () => { + const updateFile = vi.fn() + render( + , + ) + + // Get the hidden input + const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement + + // Create a mock file + const mockFile = createMockFile() + + // Simulate file selection + Object.defineProperty(fileInput, 'files', { + value: [mockFile], + }) + + fireEvent.change(fileInput) + + expect(updateFile).toHaveBeenCalledWith(mockFile) + }) + + it('should call updateFile with undefined when delete button is clicked', () => { + const updateFile = vi.fn() + const mockFile = createMockFile() + + render( + , + ) + + // Find and click delete button - the button contains the delete icon + const deleteButton = document.querySelector('button') + if (deleteButton) { + fireEvent.click(deleteButton) + expect(updateFile).toHaveBeenCalledWith() + } + }) + + it('should handle browse click', () => { + const updateFile = vi.fn() + render( + , + ) + + const browseLink = screen.getByText('app.dslUploader.browse') + const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement + + // Mock click on input + const clickSpy = vi.spyOn(fileInput, 'click') + + fireEvent.click(browseLink) + + expect(clickSpy).toHaveBeenCalled() + }) + }) + + describe('Drag and Drop', () => { + it('should show drag state when dragging over', () => { + render( + , + ) + + const dropArea = document.querySelector('[class*="border-dashed"]')! + + // The drag state is triggered when dragEnter fires on something other than the dragRef + // In the component, setDragging(true) happens when e.target !== dragRef.current + fireEvent.dragEnter(dropArea, { + dataTransfer: { files: [] }, + }) + + // The class should be present since dropArea is not dragRef + expect(dropArea.className).toContain('border-components-dropzone') + }) + + it('should handle dragOver event', () => { + render( + , + ) + + const dashedArea = document.querySelector('[class*="border-dashed"]') + const dropArea = dashedArea?.parentElement + if (!dropArea) + return + + // DragOver should prevent default and stop propagation + const dragOverEvent = new Event('dragover', { bubbles: true, cancelable: true }) + dropArea.dispatchEvent(dragOverEvent) + + // Event should be handled without errors + expect(dropArea).toBeInTheDocument() + }) + + it('should handle dragLeave event and reset dragging state when target is dragRef', async () => { + render( + , + ) + + const dropArea = document.querySelector('[class*="border-dashed"]')! + const dropAreaParent = dropArea.parentElement + + if (!dropAreaParent) + return + + // First trigger dragEnter to set dragging state + fireEvent.dragEnter(dropArea, { + dataTransfer: { files: [] }, + }) + + // Verify dragging state is set - the accent class appears when dragging + await waitFor(() => { + expect(dropArea.className).toContain('border-components-dropzone-border-accent') + }) + + // The dragRef div appears when dragging is true + const dragRefDiv = document.querySelector('[class*="absolute left-0 top-0"]') + expect(dragRefDiv).toBeInTheDocument() + + // When dragLeave happens on the dragRef element, setDragging(false) is called + if (dragRefDiv) { + // Fire dragleave directly on the dragRef element + fireEvent.dragLeave(dragRefDiv) + + // After dragLeave on dragRef, dragging should be false and accent class removed + await waitFor(() => { + expect(dropArea.className).not.toContain('border-components-dropzone-border-accent') + }) + } + }) + + it('should not reset dragging when dragLeave target is not dragRef', async () => { + render( + , + ) + + const dropArea = document.querySelector('[class*="border-dashed"]')! + const dropAreaParent = dropArea.parentElement + + if (!dropAreaParent) + return + + // First trigger dragEnter to set dragging state + fireEvent.dragEnter(dropArea, { + dataTransfer: { files: [] }, + }) + + // Verify dragging state is set + await waitFor(() => { + expect(dropArea.className).toContain('border-components-dropzone-border-accent') + }) + + // Trigger dragLeave on the drop area (not dragRef) - should NOT reset dragging + fireEvent.dragLeave(dropArea, { + dataTransfer: { files: [] }, + }) + + // Dragging should still be true (accent class still present) + // because target is not dragRef + expect(dropArea.className).toContain('border-components-dropzone') + }) + + it('should handle file drop', async () => { + const updateFile = vi.fn() + render( + , + ) + + const dashedArea = document.querySelector('[class*="border-dashed"]') + const dropArea = dashedArea?.parentElement + if (!dropArea) + return + + const mockFile = createMockFile() + + fireEvent.drop(dropArea, { + dataTransfer: { + files: [mockFile], + }, + }) + + expect(updateFile).toHaveBeenCalledWith(mockFile) + }) + + it('should reject multiple files', async () => { + const updateFile = vi.fn() + render( + , + ) + + const dashedArea = document.querySelector('[class*="border-dashed"]') + const dropArea = dashedArea?.parentElement + if (!dropArea) + return + + const mockFile1 = createMockFile('file1.pipeline') + const mockFile2 = createMockFile('file2.pipeline') + + fireEvent.drop(dropArea, { + dataTransfer: { + files: [mockFile1, mockFile2], + }, + }) + + expect(updateFile).not.toHaveBeenCalled() + expect(mockNotify).toHaveBeenCalledWith(expect.objectContaining({ + type: 'error', + })) + }) + }) + + describe('Edge Cases', () => { + it('should handle drop event without dataTransfer', () => { + const updateFile = vi.fn() + render( + , + ) + + const dashedArea = document.querySelector('[class*="border-dashed"]') + const dropArea = dashedArea?.parentElement + if (!dropArea) + return + + fireEvent.drop(dropArea, { + dataTransfer: null, + }) + + expect(updateFile).not.toHaveBeenCalled() + }) + + it('should handle file cancel in selectHandle and restore original file', () => { + const updateFile = vi.fn() + + render( + , + ) + + // Get the file input + const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement + expect(fileInput).toBeInTheDocument() + + // Spy on input click before triggering selectHandle + const clickSpy = vi.spyOn(fileInput, 'click').mockImplementation(() => { + // After click, oncancel should be set + }) + + // Click browse link to trigger selectHandle + const browseLink = screen.getByText('app.dslUploader.browse') + fireEvent.click(browseLink) + + // selectHandle should have triggered click on input + expect(clickSpy).toHaveBeenCalled() + + // After selectHandle runs, oncancel should be set + // Trigger cancel - should restore original file (undefined in this case) + if (fileInput.oncancel) { + fileInput.oncancel(new Event('cancel')) + // updateFile should be called with undefined (the original file) + expect(updateFile).toHaveBeenCalledWith(undefined) + } + + clickSpy.mockRestore() + }) + + it('should not set dragging when target equals dragRef', () => { + render( + , + ) + + const dropArea = document.querySelector('[class*="border-dashed"]')! + + // First trigger drag to show dragRef div + fireEvent.dragEnter(dropArea, { + dataTransfer: { files: [] }, + }) + + // Now the dragRef div should exist + const dragRefDiv = document.querySelector('[class*="absolute left-0 top-0"]') + + // When dragEnter happens on dragRef itself, setDragging should NOT be called + if (dragRefDiv) { + const dropAreaParent = dropArea.parentElement + if (dropAreaParent) { + // Trigger dragEnter with target = dragRef - this should NOT set dragging + const dragEnterEvent = new Event('dragenter', { bubbles: true }) + Object.defineProperty(dragEnterEvent, 'target', { value: dragRefDiv }) + dropAreaParent.dispatchEvent(dragEnterEvent) + } + } + }) + + it('should handle removeFile when file input exists', () => { + const updateFile = vi.fn() + const mockFile = createMockFile() + + render( + , + ) + + // Find and click delete button + const deleteButton = document.querySelector('button') + expect(deleteButton).toBeInTheDocument() + + if (deleteButton) { + fireEvent.click(deleteButton) + // updateFile should be called without arguments + expect(updateFile).toHaveBeenCalledWith() + } + + // Verify file input value was cleared + const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement + expect(fileInput.value).toBe('') + }) + }) +}) + +// ============================================ +// DSLConfirmModal Component Tests +// ============================================ +describe('DSLConfirmModal', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + describe('Rendering', () => { + it('should render title', () => { + render( + , + ) + + expect(screen.getByText('app.newApp.appCreateDSLErrorTitle')).toBeInTheDocument() + }) + + it('should render version information', () => { + render( + , + ) + + expect(screen.getByText('0.9.0')).toBeInTheDocument() + expect(screen.getByText('1.0.0')).toBeInTheDocument() + }) + + it('should render cancel and confirm buttons', () => { + render( + , + ) + + expect(screen.getByText('app.newApp.Cancel')).toBeInTheDocument() + expect(screen.getByText('app.newApp.Confirm')).toBeInTheDocument() + }) + + it('should render with default empty versions', () => { + render( + , + ) + + // Should not crash with default empty strings + expect(screen.getByText('app.newApp.appCreateDSLErrorTitle')).toBeInTheDocument() + }) + + it('should disable confirm button when confirmDisabled is true', () => { + render( + , + ) + + const confirmButton = screen.getByText('app.newApp.Confirm').closest('button') + expect(confirmButton).toBeDisabled() + }) + }) + + describe('Event Handlers', () => { + it('should call onCancel when cancel button is clicked', () => { + const onCancel = vi.fn() + render( + , + ) + + fireEvent.click(screen.getByText('app.newApp.Cancel')) + expect(onCancel).toHaveBeenCalled() + }) + + it('should call onConfirm when confirm button is clicked', () => { + const onConfirm = vi.fn() + render( + , + ) + + fireEvent.click(screen.getByText('app.newApp.Confirm')) + expect(onConfirm).toHaveBeenCalled() + }) + + it('should bind onClose to onCancel via arrow function', () => { + // This test verifies that the Modal's onClose prop calls onCancel + // The implementation is: onClose={() => onCancel()} + const onCancel = vi.fn() + render( + , + ) + + // Trigger the cancel button which also calls onCancel + // This confirms onCancel is properly wired up + fireEvent.click(screen.getByText('app.newApp.Cancel')) + expect(onCancel).toHaveBeenCalledTimes(1) + }) + + it('should call onCancel when modal is closed via escape key', () => { + const onCancel = vi.fn() + render( + , + ) + + // Pressing Escape triggers Modal's onClose which calls onCancel + const escEvent = new KeyboardEvent('keydown', { + key: 'Escape', + code: 'Escape', + keyCode: 27, + bubbles: true, + }) + document.dispatchEvent(escEvent) + + // onCancel should be called via the onClose={() => onCancel()} callback + expect(onCancel).toHaveBeenCalled() + }) + }) + + describe('Props', () => { + it('should use default versions when not provided', () => { + render( + , + ) + + // Component should render without crashing + expect(screen.getByText('app.newApp.appCreateDSLErrorTitle')).toBeInTheDocument() + }) + + it('should use default confirmDisabled when not provided', () => { + render( + , + ) + + const confirmButton = screen.getByText('app.newApp.Confirm').closest('button') + expect(confirmButton).not.toBeDisabled() + }) + }) +}) + +// ============================================ +// Integration Tests +// ============================================ +describe('CreateFromDSLModal Integration', () => { + beforeEach(() => { + vi.clearAllMocks() + mockImportDSL.mockReset() + mockImportDSLConfirm.mockReset() + mockPush.mockReset() + mockNotify.mockReset() + mockHandleCheckPluginDependencies.mockReset() + }) + + it('should complete full import flow with URL', async () => { + const onSuccess = vi.fn() + const onClose = vi.fn() + mockImportDSL.mockResolvedValue(createImportDSLResponse()) + mockHandleCheckPluginDependencies.mockResolvedValue(undefined) + + render( + , + { wrapper: createWrapper() }, + ) + + // Switch to URL tab + fireEvent.click(screen.getByText('app.importFromDSLUrl')) + + // Enter URL + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/pipeline.yaml' } }) + + // Click import + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + // Verify API was called + await waitFor(() => { + expect(mockImportDSL).toHaveBeenCalled() + }) + + // Verify success callbacks after API completes + await waitFor(() => { + expect(onSuccess).toHaveBeenCalled() + }) + + await waitFor(() => { + expect(mockPush).toHaveBeenCalledWith('/datasets/dataset-789/pipeline') + }) + }) + + it('should handle version mismatch flow - shows error modal', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + const onClose = vi.fn() + mockImportDSL.mockResolvedValue(createImportDSLResponse({ + status: 'pending', + imported_dsl_version: '0.8.0', + current_dsl_version: '1.0.0', + })) + + render( + , + { wrapper: createWrapper() }, + ) + + // Enter URL + const input = screen.getByPlaceholderText('app.importFromDSLUrlPlaceholder') + fireEvent.change(input, { target: { value: 'https://example.com/old-pipeline.yaml' } }) + + // Click import + const importButton = screen.getByText('app.newApp.import').closest('button')! + fireEvent.click(importButton) + + // Wait for API call + await waitFor(() => { + expect(mockImportDSL).toHaveBeenCalled() + }) + + // Wait for onClose to be called + await waitFor(() => { + expect(onClose).toHaveBeenCalled() + }) + + // Advance timer to show error modal + await act(async () => { + vi.advanceTimersByTime(400) + }) + + // Verify error modal is shown + await waitFor(() => { + expect(screen.getByText('app.newApp.appCreateDSLErrorTitle')).toBeInTheDocument() + }) + + vi.useRealTimers() + }) +}) diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks/index.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks/index.ts new file mode 100644 index 0000000000..0faf3c52f7 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks/index.ts @@ -0,0 +1,5 @@ +export { useAddDocumentsSteps } from './use-add-documents-steps' +export { useDatasourceActions } from './use-datasource-actions' +export { useDatasourceOptions } from './use-datasource-options' +export { useLocalFile, useOnlineDocument, useOnlineDrive, useWebsiteCrawl } from './use-datasource-store' +export { useDatasourceUIState } from './use-datasource-ui-state' diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks/use-add-documents-steps.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks/use-add-documents-steps.ts new file mode 100644 index 0000000000..eba0f5a8c8 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks/use-add-documents-steps.ts @@ -0,0 +1,41 @@ +import { useCallback, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { AddDocumentsStep } from '../types' + +/** + * Hook for managing add documents wizard steps + */ +export const useAddDocumentsSteps = () => { + const { t } = useTranslation() + const [currentStep, setCurrentStep] = useState(1) + + const handleNextStep = useCallback(() => { + setCurrentStep(preStep => preStep + 1) + }, []) + + const handleBackStep = useCallback(() => { + setCurrentStep(preStep => preStep - 1) + }, []) + + const steps = [ + { + label: t('addDocuments.steps.chooseDatasource', { ns: 'datasetPipeline' }), + value: AddDocumentsStep.dataSource, + }, + { + label: t('addDocuments.steps.processDocuments', { ns: 'datasetPipeline' }), + value: AddDocumentsStep.processDocuments, + }, + { + label: t('addDocuments.steps.processingDocuments', { ns: 'datasetPipeline' }), + value: AddDocumentsStep.processingDocuments, + }, + ] + + return { + steps, + currentStep, + handleNextStep, + handleBackStep, + } +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-actions.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-actions.ts new file mode 100644 index 0000000000..66bd325c33 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-actions.ts @@ -0,0 +1,321 @@ +import type { StoreApi } from 'zustand' +import type { DataSourceShape } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store' +import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' +import type { DataSourceNotionPageMap, NotionPage } from '@/models/common' +import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEstimateResponse } from '@/models/datasets' +import type { + OnlineDriveFile, + PublishedPipelineRunPreviewResponse, + PublishedPipelineRunResponse, +} from '@/models/pipeline' +import { useCallback, useRef } from 'react' +import { trackEvent } from '@/app/components/base/amplitude' +import { DatasourceType } from '@/models/pipeline' +import { useRunPublishedPipeline } from '@/service/use-pipeline' +import { + buildLocalFileDatasourceInfo, + buildOnlineDocumentDatasourceInfo, + buildOnlineDriveDatasourceInfo, + buildWebsiteCrawlDatasourceInfo, +} from '../utils/datasource-info-builder' + +type DatasourceActionsParams = { + datasource: Datasource | undefined + datasourceType: string | undefined + pipelineId: string | undefined + dataSourceStore: StoreApi + setEstimateData: (data: FileIndexingEstimateResponse | undefined) => void + setBatchId: (id: string) => void + setDocuments: (docs: PublishedPipelineRunResponse['documents']) => void + handleNextStep: () => void + PagesMapAndSelectedPagesId: DataSourceNotionPageMap + currentWorkspacePages: { page_id: string }[] | undefined + clearOnlineDocumentData: () => void + clearWebsiteCrawlData: () => void + clearOnlineDriveData: () => void + setDatasource: (ds: Datasource) => void +} + +/** + * Hook for datasource-related actions (preview, process, etc.) + */ +export const useDatasourceActions = ({ + datasource, + datasourceType, + pipelineId, + dataSourceStore, + setEstimateData, + setBatchId, + setDocuments, + handleNextStep, + PagesMapAndSelectedPagesId, + currentWorkspacePages, + clearOnlineDocumentData, + clearWebsiteCrawlData, + clearOnlineDriveData, + setDatasource, +}: DatasourceActionsParams) => { + const isPreview = useRef(false) + const formRef = useRef<{ submit: () => void } | null>(null) + + const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline() + + // Build datasource info for preview (single item) + const buildPreviewDatasourceInfo = useCallback(() => { + const { + previewLocalFileRef, + previewOnlineDocumentRef, + previewWebsitePageRef, + previewOnlineDriveFileRef, + currentCredentialId, + bucket, + } = dataSourceStore.getState() + + const datasourceInfoList: Record[] = [] + + if (datasourceType === DatasourceType.localFile && previewLocalFileRef.current) { + datasourceInfoList.push(buildLocalFileDatasourceInfo( + previewLocalFileRef.current as File, + currentCredentialId, + )) + } + + if (datasourceType === DatasourceType.onlineDocument && previewOnlineDocumentRef.current) { + datasourceInfoList.push(buildOnlineDocumentDatasourceInfo( + previewOnlineDocumentRef.current, + currentCredentialId, + )) + } + + if (datasourceType === DatasourceType.websiteCrawl && previewWebsitePageRef.current) { + datasourceInfoList.push(buildWebsiteCrawlDatasourceInfo( + previewWebsitePageRef.current, + currentCredentialId, + )) + } + + if (datasourceType === DatasourceType.onlineDrive && previewOnlineDriveFileRef.current) { + datasourceInfoList.push(buildOnlineDriveDatasourceInfo( + previewOnlineDriveFileRef.current, + bucket, + currentCredentialId, + )) + } + + return datasourceInfoList + }, [dataSourceStore, datasourceType]) + + // Build datasource info for processing (all items) + const buildProcessDatasourceInfo = useCallback(() => { + const { + currentCredentialId, + localFileList, + onlineDocuments, + websitePages, + bucket, + selectedFileIds, + onlineDriveFileList, + } = dataSourceStore.getState() + + const datasourceInfoList: Record[] = [] + + if (datasourceType === DatasourceType.localFile) { + localFileList.forEach((file) => { + datasourceInfoList.push(buildLocalFileDatasourceInfo(file.file, currentCredentialId)) + }) + } + + if (datasourceType === DatasourceType.onlineDocument) { + onlineDocuments.forEach((page) => { + datasourceInfoList.push(buildOnlineDocumentDatasourceInfo(page, currentCredentialId)) + }) + } + + if (datasourceType === DatasourceType.websiteCrawl) { + websitePages.forEach((page) => { + datasourceInfoList.push(buildWebsiteCrawlDatasourceInfo(page, currentCredentialId)) + }) + } + + if (datasourceType === DatasourceType.onlineDrive) { + selectedFileIds.forEach((id) => { + const file = onlineDriveFileList.find(f => f.id === id) + if (file) + datasourceInfoList.push(buildOnlineDriveDatasourceInfo(file, bucket, currentCredentialId)) + }) + } + + return datasourceInfoList + }, [dataSourceStore, datasourceType]) + + // Handle chunk preview + const handlePreviewChunks = useCallback(async (data: Record) => { + if (!datasource || !pipelineId) + return + + const datasourceInfoList = buildPreviewDatasourceInfo() + await runPublishedPipeline({ + pipeline_id: pipelineId, + inputs: data, + start_node_id: datasource.nodeId, + datasource_type: datasourceType as DatasourceType, + datasource_info_list: datasourceInfoList, + is_preview: true, + }, { + onSuccess: (res) => { + setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs) + }, + }) + }, [datasource, pipelineId, datasourceType, buildPreviewDatasourceInfo, runPublishedPipeline, setEstimateData]) + + // Handle document processing + const handleProcess = useCallback(async (data: Record) => { + if (!datasource || !pipelineId) + return + + const datasourceInfoList = buildProcessDatasourceInfo() + await runPublishedPipeline({ + pipeline_id: pipelineId, + inputs: data, + start_node_id: datasource.nodeId, + datasource_type: datasourceType as DatasourceType, + datasource_info_list: datasourceInfoList, + is_preview: false, + }, { + onSuccess: (res) => { + setBatchId((res as PublishedPipelineRunResponse).batch || '') + setDocuments((res as PublishedPipelineRunResponse).documents || []) + handleNextStep() + trackEvent('dataset_document_added', { + data_source_type: datasourceType, + indexing_technique: 'pipeline', + }) + }, + }) + }, [datasource, pipelineId, datasourceType, buildProcessDatasourceInfo, runPublishedPipeline, setBatchId, setDocuments, handleNextStep]) + + // Form submission handlers + const onClickProcess = useCallback(() => { + isPreview.current = false + formRef.current?.submit() + }, []) + + const onClickPreview = useCallback(() => { + isPreview.current = true + formRef.current?.submit() + }, []) + + const handleSubmit = useCallback((data: Record) => { + if (isPreview.current) + handlePreviewChunks(data) + else + handleProcess(data) + }, [handlePreviewChunks, handleProcess]) + + // Preview change handlers + const handlePreviewFileChange = useCallback((file: DocumentItem) => { + const { previewLocalFileRef } = dataSourceStore.getState() + previewLocalFileRef.current = file + onClickPreview() + }, [dataSourceStore, onClickPreview]) + + const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => { + const { previewOnlineDocumentRef } = dataSourceStore.getState() + previewOnlineDocumentRef.current = page + onClickPreview() + }, [dataSourceStore, onClickPreview]) + + const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => { + const { previewWebsitePageRef } = dataSourceStore.getState() + previewWebsitePageRef.current = website + onClickPreview() + }, [dataSourceStore, onClickPreview]) + + const handlePreviewOnlineDriveFileChange = useCallback((file: OnlineDriveFile) => { + const { previewOnlineDriveFileRef } = dataSourceStore.getState() + previewOnlineDriveFileRef.current = file + onClickPreview() + }, [dataSourceStore, onClickPreview]) + + // Select all handler + const handleSelectAll = useCallback(() => { + const { + onlineDocuments, + onlineDriveFileList, + selectedFileIds, + setOnlineDocuments, + setSelectedFileIds, + setSelectedPagesId, + } = dataSourceStore.getState() + + if (datasourceType === DatasourceType.onlineDocument) { + const allIds = currentWorkspacePages?.map(page => page.page_id) || [] + if (onlineDocuments.length < allIds.length) { + const selectedPages = Array.from(allIds).map(pageId => PagesMapAndSelectedPagesId[pageId]) + setOnlineDocuments(selectedPages) + setSelectedPagesId(new Set(allIds)) + } + else { + setOnlineDocuments([]) + setSelectedPagesId(new Set()) + } + } + + if (datasourceType === DatasourceType.onlineDrive) { + const allKeys = onlineDriveFileList.filter(item => item.type !== 'bucket').map(file => file.id) + if (selectedFileIds.length < allKeys.length) + setSelectedFileIds(allKeys) + else + setSelectedFileIds([]) + } + }, [PagesMapAndSelectedPagesId, currentWorkspacePages, dataSourceStore, datasourceType]) + + // Clear datasource data based on type + const clearDataSourceData = useCallback((dataSource: Datasource) => { + const providerType = dataSource.nodeData.provider_type + const clearFunctions: Record void> = { + [DatasourceType.onlineDocument]: clearOnlineDocumentData, + [DatasourceType.websiteCrawl]: clearWebsiteCrawlData, + [DatasourceType.onlineDrive]: clearOnlineDriveData, + [DatasourceType.localFile]: () => {}, + } + clearFunctions[providerType]?.() + }, [clearOnlineDocumentData, clearOnlineDriveData, clearWebsiteCrawlData]) + + // Switch datasource handler + const handleSwitchDataSource = useCallback((dataSource: Datasource) => { + const { + setCurrentCredentialId, + currentNodeIdRef, + } = dataSourceStore.getState() + clearDataSourceData(dataSource) + setCurrentCredentialId('') + currentNodeIdRef.current = dataSource.nodeId + setDatasource(dataSource) + }, [clearDataSourceData, dataSourceStore, setDatasource]) + + // Credential change handler + const handleCredentialChange = useCallback((credentialId: string) => { + const { setCurrentCredentialId } = dataSourceStore.getState() + if (datasource) + clearDataSourceData(datasource) + setCurrentCredentialId(credentialId) + }, [clearDataSourceData, dataSourceStore, datasource]) + + return { + isPreview, + formRef, + isIdle, + isPending, + onClickProcess, + onClickPreview, + handleSubmit, + handlePreviewFileChange, + handlePreviewOnlineDocumentChange, + handlePreviewWebsiteChange, + handlePreviewOnlineDriveFileChange, + handleSelectAll, + handleSwitchDataSource, + handleCredentialChange, + } +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-options.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-options.ts new file mode 100644 index 0000000000..a8b233faba --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-options.ts @@ -0,0 +1,27 @@ +import type { DataSourceOption } from '@/app/components/rag-pipeline/components/panel/test-run/types' +import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' +import type { Node } from '@/app/components/workflow/types' +import { useMemo } from 'react' +import { BlockEnum } from '@/app/components/workflow/types' + +/** + * Hook for getting datasource options from pipeline nodes + */ +export const useDatasourceOptions = (pipelineNodes: Node[]) => { + const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource) + + const options = useMemo(() => { + const options: DataSourceOption[] = [] + datasourceNodes.forEach((node) => { + const label = node.data.title + options.push({ + label, + value: node.id, + data: node.data, + }) + }) + return options + }, [datasourceNodes]) + + return options +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-store.ts similarity index 70% rename from web/app/components/datasets/documents/create-from-pipeline/hooks.ts rename to web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-store.ts index 68d79de031..da620de154 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-store.ts @@ -1,69 +1,12 @@ -import type { DataSourceOption } from '@/app/components/rag-pipeline/components/panel/test-run/types' -import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' -import type { Node } from '@/app/components/workflow/types' import type { DataSourceNotionPageMap, DataSourceNotionWorkspace } from '@/models/common' -import { useCallback, useMemo, useState } from 'react' -import { useTranslation } from 'react-i18next' +import { useCallback, useMemo } from 'react' import { useShallow } from 'zustand/react/shallow' -import { BlockEnum } from '@/app/components/workflow/types' import { CrawlStep } from '@/models/datasets' -import { useDataSourceStore, useDataSourceStoreWithSelector } from './data-source/store' -import { AddDocumentsStep } from './types' - -export const useAddDocumentsSteps = () => { - const { t } = useTranslation() - const [currentStep, setCurrentStep] = useState(1) - - const handleNextStep = useCallback(() => { - setCurrentStep(preStep => preStep + 1) - }, []) - - const handleBackStep = useCallback(() => { - setCurrentStep(preStep => preStep - 1) - }, []) - - const steps = [ - { - label: t('addDocuments.steps.chooseDatasource', { ns: 'datasetPipeline' }), - value: AddDocumentsStep.dataSource, - }, - { - label: t('addDocuments.steps.processDocuments', { ns: 'datasetPipeline' }), - value: AddDocumentsStep.processDocuments, - }, - { - label: t('addDocuments.steps.processingDocuments', { ns: 'datasetPipeline' }), - value: AddDocumentsStep.processingDocuments, - }, - ] - - return { - steps, - currentStep, - handleNextStep, - handleBackStep, - } -} - -export const useDatasourceOptions = (pipelineNodes: Node[]) => { - const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource) - - const options = useMemo(() => { - const options: DataSourceOption[] = [] - datasourceNodes.forEach((node) => { - const label = node.data.title - options.push({ - label, - value: node.id, - data: node.data, - }) - }) - return options - }, [datasourceNodes]) - - return options -} +import { useDataSourceStore, useDataSourceStoreWithSelector } from '../data-source/store' +/** + * Hook for local file datasource store operations + */ export const useLocalFile = () => { const { localFileList, @@ -89,6 +32,9 @@ export const useLocalFile = () => { } } +/** + * Hook for online document datasource store operations + */ export const useOnlineDocument = () => { const { documentsData, @@ -147,6 +93,9 @@ export const useOnlineDocument = () => { } } +/** + * Hook for website crawl datasource store operations + */ export const useWebsiteCrawl = () => { const { websitePages, @@ -186,6 +135,9 @@ export const useWebsiteCrawl = () => { } } +/** + * Hook for online drive datasource store operations + */ export const useOnlineDrive = () => { const { onlineDriveFileList, diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-ui-state.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-ui-state.ts new file mode 100644 index 0000000000..e398f90a48 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks/use-datasource-ui-state.ts @@ -0,0 +1,132 @@ +import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' +import type { OnlineDriveFile } from '@/models/pipeline' +import { useMemo } from 'react' +import { useTranslation } from 'react-i18next' +import { DatasourceType } from '@/models/pipeline' + +type DatasourceUIStateParams = { + datasource: Datasource | undefined + allFileLoaded: boolean + localFileListLength: number + onlineDocumentsLength: number + websitePagesLength: number + selectedFileIdsLength: number + onlineDriveFileList: OnlineDriveFile[] + isVectorSpaceFull: boolean + enableBilling: boolean + currentWorkspacePagesLength: number + fileUploadConfig: { file_size_limit: number, batch_count_limit: number } +} + +/** + * Hook for computing datasource UI state based on datasource type + */ +export const useDatasourceUIState = ({ + datasource, + allFileLoaded, + localFileListLength, + onlineDocumentsLength, + websitePagesLength, + selectedFileIdsLength, + onlineDriveFileList, + isVectorSpaceFull, + enableBilling, + currentWorkspacePagesLength, + fileUploadConfig, +}: DatasourceUIStateParams) => { + const { t } = useTranslation() + const datasourceType = datasource?.nodeData.provider_type + + const isShowVectorSpaceFull = useMemo(() => { + if (!datasource || !datasourceType) + return false + + // Lookup table for vector space full condition check + const vectorSpaceFullConditions: Record = { + [DatasourceType.localFile]: allFileLoaded, + [DatasourceType.onlineDocument]: onlineDocumentsLength > 0, + [DatasourceType.websiteCrawl]: websitePagesLength > 0, + [DatasourceType.onlineDrive]: onlineDriveFileList.length > 0, + } + + const condition = vectorSpaceFullConditions[datasourceType] + return condition && isVectorSpaceFull && enableBilling + }, [datasource, datasourceType, allFileLoaded, onlineDocumentsLength, websitePagesLength, onlineDriveFileList.length, isVectorSpaceFull, enableBilling]) + + // Lookup table for next button disabled conditions + const nextBtnDisabled = useMemo(() => { + if (!datasource || !datasourceType) + return true + + const disabledConditions: Record = { + [DatasourceType.localFile]: isShowVectorSpaceFull || localFileListLength === 0 || !allFileLoaded, + [DatasourceType.onlineDocument]: isShowVectorSpaceFull || onlineDocumentsLength === 0, + [DatasourceType.websiteCrawl]: isShowVectorSpaceFull || websitePagesLength === 0, + [DatasourceType.onlineDrive]: isShowVectorSpaceFull || selectedFileIdsLength === 0, + } + + return disabledConditions[datasourceType] ?? true + }, [datasource, datasourceType, isShowVectorSpaceFull, localFileListLength, allFileLoaded, onlineDocumentsLength, websitePagesLength, selectedFileIdsLength]) + + // Check if select all should be shown + const showSelect = useMemo(() => { + if (datasourceType === DatasourceType.onlineDocument) + return currentWorkspacePagesLength > 0 + + if (datasourceType === DatasourceType.onlineDrive) { + const nonBucketItems = onlineDriveFileList.filter(item => item.type !== 'bucket') + const isBucketList = onlineDriveFileList.some(file => file.type === 'bucket') + return !isBucketList && nonBucketItems.length > 0 + } + + return false + }, [currentWorkspacePagesLength, datasourceType, onlineDriveFileList]) + + // Total selectable options count + const totalOptions = useMemo(() => { + if (datasourceType === DatasourceType.onlineDocument) + return currentWorkspacePagesLength + + if (datasourceType === DatasourceType.onlineDrive) + return onlineDriveFileList.filter(item => item.type !== 'bucket').length + + return undefined + }, [currentWorkspacePagesLength, datasourceType, onlineDriveFileList]) + + // Selected options count + const selectedOptions = useMemo(() => { + if (datasourceType === DatasourceType.onlineDocument) + return onlineDocumentsLength + + if (datasourceType === DatasourceType.onlineDrive) + return selectedFileIdsLength + + return undefined + }, [datasourceType, onlineDocumentsLength, selectedFileIdsLength]) + + // Tip message for selection + const tip = useMemo(() => { + if (datasourceType === DatasourceType.onlineDocument) + return t('addDocuments.selectOnlineDocumentTip', { ns: 'datasetPipeline', count: 50 }) + + if (datasourceType === DatasourceType.onlineDrive) { + return t('addDocuments.selectOnlineDriveTip', { + ns: 'datasetPipeline', + count: fileUploadConfig.batch_count_limit, + fileSize: fileUploadConfig.file_size_limit, + }) + } + + return '' + }, [datasourceType, fileUploadConfig.batch_count_limit, fileUploadConfig.file_size_limit, t]) + + return { + datasourceType, + isShowVectorSpaceFull, + nextBtnDisabled, + showSelect, + totalOptions, + selectedOptions, + tip, + } +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.spec.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.spec.tsx new file mode 100644 index 0000000000..c43678def0 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/index.spec.tsx @@ -0,0 +1,2698 @@ +import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' +import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' +import type { Node } from '@/app/components/workflow/types' +import type { NotionPage } from '@/models/common' +import type { CrawlResultItem, CustomFile, DocumentItem, FileItem } from '@/models/datasets' +import type { InitialDocumentDetail, OnlineDriveFile } from '@/models/pipeline' +import { act, fireEvent, render, renderHook, screen } from '@testing-library/react' +import * as React from 'react' +import { BlockEnum } from '@/app/components/workflow/types' +import { DatasourceType } from '@/models/pipeline' +import { TransferMethod } from '@/types/app' +import { + useAddDocumentsSteps, + useDatasourceActions, + useDatasourceOptions, + useDatasourceUIState, + useLocalFile, + useOnlineDocument, + useOnlineDrive, + useWebsiteCrawl, +} from './hooks' +import { StepOneContent, StepThreeContent, StepTwoContent } from './steps' +import { StepOnePreview, StepTwoPreview } from './steps/preview-panel' +import { + buildLocalFileDatasourceInfo, + buildOnlineDocumentDatasourceInfo, + buildOnlineDriveDatasourceInfo, + buildWebsiteCrawlDatasourceInfo, +} from './utils/datasource-info-builder' + +// ========================================== +// Mock External Dependencies Only +// ========================================== + +// Mock context providers +const mockPlan = { + usage: { vectorSpace: 50 }, + total: { vectorSpace: 100 }, + type: 'professional', +} + +vi.mock('@/context/provider-context', () => ({ + useProviderContextSelector: (selector: (state: { plan: typeof mockPlan, enableBilling: boolean }) => unknown) => + selector({ plan: mockPlan, enableBilling: true }), +})) + +vi.mock('@/context/dataset-detail', () => ({ + useDatasetDetailContextWithSelector: (selector: (state: { dataset: { pipeline_id: string } }) => unknown) => + selector({ dataset: { pipeline_id: 'test-pipeline-id' } }), +})) + +// Mock API services +const mockRunPublishedPipeline = vi.fn() +vi.mock('@/service/use-pipeline', () => ({ + usePublishedPipelineInfo: () => ({ + data: { + graph: { + nodes: [ + { + id: 'node-1', + data: { + type: 'data-source', + title: 'Local File', + provider_type: DatasourceType.localFile, + plugin_id: 'plugin-1', + fileExtensions: ['.txt', '.pdf'], + }, + }, + ], + }, + }, + isFetching: false, + }), + useRunPublishedPipeline: () => ({ + mutateAsync: mockRunPublishedPipeline, + isIdle: true, + isPending: false, + }), +})) + +vi.mock('@/service/use-common', () => ({ + useFileUploadConfig: () => ({ + data: { + file_size_limit: 15, + batch_count_limit: 5, + }, + }), +})) + +// Mock amplitude tracking +vi.mock('@/app/components/base/amplitude', () => ({ + trackEvent: vi.fn(), +})) + +// Mock next/navigation +vi.mock('next/navigation', () => ({ + useParams: () => ({ datasetId: 'test-dataset-id' }), + useRouter: () => ({ + push: vi.fn(), + replace: vi.fn(), + back: vi.fn(), + }), + usePathname: () => '/datasets/test-dataset-id/documents/create-from-pipeline', +})) + +// Mock next/link +vi.mock('next/link', () => ({ + default: ({ children, href }: { children: React.ReactNode, href: string }) => ( + {children} + ), +})) + +// Mock billing components (external dependencies) +vi.mock('@/app/components/billing/vector-space-full', () => ({ + default: () =>
Vector Space Full
, +})) + +vi.mock('@/app/components/billing/plan-upgrade-modal', () => ({ + default: ({ show, onClose }: { show: boolean, onClose: () => void }) => ( + show + ? ( +
+ +
+ ) + : null + ), +})) + +vi.mock('@/app/components/datasets/create/step-one/upgrade-card', () => ({ + default: () =>
Upgrade Card
, +})) + +// Mock zustand store +// eslint-disable-next-line ts/no-explicit-any +type MockDataSourceStore = any + +const mockStoreState = { + localFileList: [] as FileItem[], + currentLocalFile: undefined as CustomFile | undefined, + setCurrentLocalFile: vi.fn(), + documentsData: [] as { workspace_id: string, pages: { page_id: string }[] }[], + onlineDocuments: [] as (NotionPage & { workspace_id: string })[], + currentDocument: undefined as (NotionPage & { workspace_id: string }) | undefined, + setDocumentsData: vi.fn(), + setSearchValue: vi.fn(), + setSelectedPagesId: vi.fn(), + setOnlineDocuments: vi.fn(), + setCurrentDocument: vi.fn(), + websitePages: [] as CrawlResultItem[], + currentWebsite: undefined as CrawlResultItem | undefined, + setCurrentWebsite: vi.fn(), + setPreviewIndex: vi.fn(), + setStep: vi.fn(), + setCrawlResult: vi.fn(), + setWebsitePages: vi.fn(), + onlineDriveFileList: [] as OnlineDriveFile[], + selectedFileIds: [] as string[], + setOnlineDriveFileList: vi.fn(), + setBucket: vi.fn(), + setPrefix: vi.fn(), + setKeywords: vi.fn(), + setSelectedFileIds: vi.fn(), + previewLocalFileRef: { current: undefined }, + previewOnlineDocumentRef: { current: undefined }, + previewWebsitePageRef: { current: undefined }, + previewOnlineDriveFileRef: { current: undefined }, + currentCredentialId: '', + setCurrentCredentialId: vi.fn(), + currentNodeIdRef: { current: '' }, + bucket: '', +} + +vi.mock('./data-source/store', () => ({ + useDataSourceStore: () => ({ + getState: () => mockStoreState, + }), + useDataSourceStoreWithSelector: (selector: (state: typeof mockStoreState) => unknown) => selector(mockStoreState), +})) + +vi.mock('./data-source/store/provider', () => ({ + default: ({ children }: { children: React.ReactNode }) => <>{children}, +})) + +// ========================================== +// Test Data Factories +// ========================================== + +const createMockDatasource = (overrides?: Partial): Datasource => ({ + nodeId: 'node-1', + nodeData: { + type: 'data-source', + title: 'Local File', + desc: '', + provider_type: DatasourceType.localFile, + plugin_id: 'plugin-1', + provider_name: 'local', + datasource_name: 'local-file', + datasource_label: 'Local File', + fileExtensions: ['.txt', '.pdf'], + } as unknown as DataSourceNodeType, + ...overrides, +}) + +const createMockFile = (overrides?: Partial): CustomFile => ({ + id: 'file-1', + name: 'test.txt', + type: 'text/plain', + size: 1024, + extension: '.txt', + mime_type: 'text/plain', + ...overrides, +} as CustomFile) + +const createMockFileItem = (overrides?: Partial): FileItem => ({ + file: createMockFile(), + progress: 100, + ...overrides, +} as FileItem) + +const createMockNotionPage = (overrides?: Partial): NotionPage & { workspace_id: string } => ({ + page_id: 'page-1', + page_name: 'Test Page', + page_icon: null, + type: 'page', + workspace_id: 'workspace-1', + ...overrides, +} as NotionPage & { workspace_id: string }) + +const createMockCrawlResult = (overrides?: Partial): CrawlResultItem => ({ + source_url: 'https://example.com', + title: 'Test Page', + markdown: '# Test', + description: 'A test page', + ...overrides, +} as CrawlResultItem) + +const createMockOnlineDriveFile = (overrides?: Partial): OnlineDriveFile => ({ + id: 'drive-file-1', + name: 'test-file.pdf', + type: 'file', + ...overrides, +} as OnlineDriveFile) + +// ========================================== +// Hook Tests - useAddDocumentsSteps +// ========================================== +describe('useAddDocumentsSteps', () => { + it('should initialize with step 1', () => { + const { result } = renderHook(() => useAddDocumentsSteps()) + expect(result.current.currentStep).toBe(1) + }) + + it('should return 3 steps', () => { + const { result } = renderHook(() => useAddDocumentsSteps()) + expect(result.current.steps).toHaveLength(3) + }) + + it('should increment step when handleNextStep is called', () => { + const { result } = renderHook(() => useAddDocumentsSteps()) + + act(() => { + result.current.handleNextStep() + }) + + expect(result.current.currentStep).toBe(2) + }) + + it('should decrement step when handleBackStep is called', () => { + const { result } = renderHook(() => useAddDocumentsSteps()) + + act(() => { + result.current.handleNextStep() + result.current.handleBackStep() + }) + + expect(result.current.currentStep).toBe(1) + }) + + it('should maintain callback reference stability (handleNextStep)', () => { + const { result, rerender } = renderHook(() => useAddDocumentsSteps()) + const firstRef = result.current.handleNextStep + rerender() + expect(result.current.handleNextStep).toBe(firstRef) + }) + + it('should maintain callback reference stability (handleBackStep)', () => { + const { result, rerender } = renderHook(() => useAddDocumentsSteps()) + const firstRef = result.current.handleBackStep + rerender() + expect(result.current.handleBackStep).toBe(firstRef) + }) +}) + +// ========================================== +// Hook Tests - useDatasourceUIState +// ========================================== +describe('useDatasourceUIState', () => { + const defaultParams = { + datasource: undefined as Datasource | undefined, + allFileLoaded: false, + localFileListLength: 0, + onlineDocumentsLength: 0, + websitePagesLength: 0, + selectedFileIdsLength: 0, + onlineDriveFileList: [] as OnlineDriveFile[], + isVectorSpaceFull: false, + enableBilling: true, + currentWorkspacePagesLength: 0, + fileUploadConfig: { file_size_limit: 15, batch_count_limit: 5 }, + } + + describe('nextBtnDisabled', () => { + it('should return true when no datasource is selected', () => { + const { result } = renderHook(() => useDatasourceUIState(defaultParams)) + expect(result.current.nextBtnDisabled).toBe(true) + }) + + it('should return true for localFile when no files are loaded', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource(), + allFileLoaded: false, + localFileListLength: 0, + })) + expect(result.current.nextBtnDisabled).toBe(true) + }) + + it('should return false for localFile when files are loaded', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource(), + allFileLoaded: true, + localFileListLength: 1, + })) + expect(result.current.nextBtnDisabled).toBe(false) + }) + + it('should return true for onlineDocument when no documents are selected', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDocument, + }, + }), + onlineDocumentsLength: 0, + })) + expect(result.current.nextBtnDisabled).toBe(true) + }) + + it('should return false for onlineDocument when documents are selected', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDocument, + }, + }), + onlineDocumentsLength: 1, + })) + expect(result.current.nextBtnDisabled).toBe(false) + }) + }) + + describe('isShowVectorSpaceFull', () => { + it('should return false when vector space is not full', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource(), + allFileLoaded: true, + isVectorSpaceFull: false, + })) + expect(result.current.isShowVectorSpaceFull).toBe(false) + }) + + it('should return true when vector space is full and billing is enabled', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource(), + allFileLoaded: true, + isVectorSpaceFull: true, + enableBilling: true, + })) + expect(result.current.isShowVectorSpaceFull).toBe(true) + }) + + it('should return false when vector space is full but billing is disabled', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource(), + allFileLoaded: true, + isVectorSpaceFull: true, + enableBilling: false, + })) + expect(result.current.isShowVectorSpaceFull).toBe(false) + }) + }) + + describe('showSelect', () => { + it('should return false for localFile datasource', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource(), + })) + expect(result.current.showSelect).toBe(false) + }) + + it('should return true for onlineDocument when pages exist', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDocument, + }, + }), + currentWorkspacePagesLength: 5, + })) + expect(result.current.showSelect).toBe(true) + }) + + it('should return true for onlineDrive when non-bucket files exist', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDrive, + }, + }), + onlineDriveFileList: [createMockOnlineDriveFile()], + })) + expect(result.current.showSelect).toBe(true) + }) + + it('should return false for onlineDrive when only buckets exist', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDrive, + }, + }), + onlineDriveFileList: [createMockOnlineDriveFile({ type: 'bucket' as OnlineDriveFile['type'] })], + })) + expect(result.current.showSelect).toBe(false) + }) + }) + + describe('tip', () => { + it('should return empty string for localFile', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource(), + })) + expect(result.current.tip).toBe('') + }) + + it('should return translation key for onlineDocument', () => { + const { result } = renderHook(() => useDatasourceUIState({ + ...defaultParams, + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDocument, + }, + }), + })) + expect(result.current.tip).toContain('datasetPipeline.addDocuments.selectOnlineDocumentTip') + }) + }) +}) + +// ========================================== +// Utility Functions Tests - datasource-info-builder +// ========================================== +describe('datasource-info-builder', () => { + describe('buildLocalFileDatasourceInfo', () => { + it('should build correct info for local file', () => { + const file = createMockFile() + const result = buildLocalFileDatasourceInfo(file, 'cred-1') + + expect(result).toEqual({ + related_id: 'file-1', + name: 'test.txt', + type: 'text/plain', + size: 1024, + extension: '.txt', + mime_type: 'text/plain', + url: '', + transfer_method: TransferMethod.local_file, + credential_id: 'cred-1', + }) + }) + + it('should handle file with undefined id', () => { + const file = createMockFile({ id: undefined }) + const result = buildLocalFileDatasourceInfo(file, 'cred-1') + expect(result.related_id).toBeUndefined() + }) + }) + + describe('buildOnlineDocumentDatasourceInfo', () => { + it('should build correct info for online document', () => { + const page = createMockNotionPage() + const result = buildOnlineDocumentDatasourceInfo(page, 'cred-1') + + expect(result.workspace_id).toBe('workspace-1') + expect(result.credential_id).toBe('cred-1') + expect(result.page).toBeDefined() + expect((result.page as NotionPage).page_id).toBe('page-1') + }) + + it('should exclude workspace_id from page object', () => { + const page = createMockNotionPage() + const result = buildOnlineDocumentDatasourceInfo(page, 'cred-1') + + expect((result.page as Record).workspace_id).toBeUndefined() + }) + }) + + describe('buildWebsiteCrawlDatasourceInfo', () => { + it('should build correct info for website crawl', () => { + const page = createMockCrawlResult() + const result = buildWebsiteCrawlDatasourceInfo(page, 'cred-1') + + expect(result.source_url).toBe('https://example.com') + expect(result.credential_id).toBe('cred-1') + }) + + it('should spread all page properties', () => { + const page = createMockCrawlResult({ title: 'Custom Title' }) + const result = buildWebsiteCrawlDatasourceInfo(page, 'cred-1') + + expect(result.title).toBe('Custom Title') + }) + }) + + describe('buildOnlineDriveDatasourceInfo', () => { + it('should build correct info for online drive', () => { + const file = createMockOnlineDriveFile() + const result = buildOnlineDriveDatasourceInfo(file, 'my-bucket', 'cred-1') + + expect(result).toEqual({ + bucket: 'my-bucket', + id: 'drive-file-1', + name: 'test-file.pdf', + type: 'file', + credential_id: 'cred-1', + }) + }) + }) +}) + +// ========================================== +// Step Components Tests (with real components) +// ========================================== +describe('StepOneContent', () => { + const defaultProps = { + datasource: undefined as Datasource | undefined, + datasourceType: undefined as string | undefined, + pipelineNodes: [] as Node[], + supportBatchUpload: true, + localFileListLength: 0, + isShowVectorSpaceFull: false, + showSelect: false, + totalOptions: undefined as number | undefined, + selectedOptions: undefined as number | undefined, + tip: '', + nextBtnDisabled: true, + onSelectDataSource: vi.fn(), + onCredentialChange: vi.fn(), + onSelectAll: vi.fn(), + onNextStep: vi.fn(), + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('should render VectorSpaceFull when isShowVectorSpaceFull is true', () => { + render() + expect(screen.getByTestId('vector-space-full')).toBeInTheDocument() + }) + + it('should not render VectorSpaceFull when isShowVectorSpaceFull is false', () => { + render() + expect(screen.queryByTestId('vector-space-full')).not.toBeInTheDocument() + }) + + it('should render UpgradeCard when conditions are met', () => { + render( + , + ) + expect(screen.getByTestId('upgrade-card')).toBeInTheDocument() + }) + + it('should not render UpgradeCard when supportBatchUpload is true', () => { + render( + , + ) + expect(screen.queryByTestId('upgrade-card')).not.toBeInTheDocument() + }) + + it('should call onNextStep when next button is clicked', () => { + const onNextStep = vi.fn() + render() + + // Find button with translation key text (using regex for flexibility) + const nextButton = screen.getByRole('button', { name: /datasetCreation.stepOne.button/i }) + fireEvent.click(nextButton) + + expect(onNextStep).toHaveBeenCalled() + }) + + it('should disable next button when nextBtnDisabled is true', () => { + render() + + const nextButton = screen.getByRole('button', { name: /datasetCreation.stepOne.button/i }) + expect(nextButton).toBeDisabled() + }) +}) + +describe('StepTwoContent', () => { + // Mock ProcessDocuments since it has complex dependencies + vi.mock('./process-documents', () => ({ + default: React.forwardRef(({ dataSourceNodeId, isRunning, onProcess, onPreview, onSubmit, onBack }: { + dataSourceNodeId: string + isRunning: boolean + onProcess: () => void + onPreview: () => void + onSubmit: (data: Record) => void + onBack: () => void + }, ref: React.Ref<{ submit: () => void }>) => { + React.useImperativeHandle(ref, () => ({ + submit: () => onSubmit({ test: 'data' }), + })) + return ( +
+ {dataSourceNodeId} + {isRunning.toString()} + + + +
+ ) + }), + })) + + const defaultProps = { + formRef: { current: null } as unknown as React.RefObject<{ submit: () => void }>, + dataSourceNodeId: 'node-1', + isRunning: false, + onProcess: vi.fn(), + onPreview: vi.fn(), + onSubmit: vi.fn(), + onBack: vi.fn(), + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('should render ProcessDocuments component', () => { + render() + expect(screen.getByTestId('process-documents')).toBeInTheDocument() + }) + + it('should pass dataSourceNodeId to ProcessDocuments', () => { + render() + expect(screen.getByTestId('datasource-node-id')).toHaveTextContent('custom-node') + }) + + it('should pass isRunning to ProcessDocuments', () => { + render() + expect(screen.getByTestId('is-running')).toHaveTextContent('true') + }) + + it('should call onProcess when process button is clicked', () => { + const onProcess = vi.fn() + render() + + fireEvent.click(screen.getByTestId('process-btn')) + + expect(onProcess).toHaveBeenCalled() + }) + + it('should call onBack when back button is clicked', () => { + const onBack = vi.fn() + render() + + fireEvent.click(screen.getByTestId('back-btn')) + + expect(onBack).toHaveBeenCalled() + }) +}) + +describe('StepThreeContent', () => { + // Mock Processing since it has complex dependencies + vi.mock('./processing', () => ({ + default: ({ batchId, documents }: { batchId: string, documents: unknown[] }) => ( +
+ {batchId} + {documents.length} +
+ ), + })) + + it('should render Processing component', () => { + render() + expect(screen.getByTestId('processing')).toBeInTheDocument() + }) + + it('should pass batchId to Processing', () => { + render() + expect(screen.getByTestId('batch-id')).toHaveTextContent('batch-123') + }) + + it('should pass documents count to Processing', () => { + const documents = [{ id: '1' }, { id: '2' }] + render() + expect(screen.getByTestId('documents-count')).toHaveTextContent('2') + }) +}) + +// ========================================== +// Preview Panel Tests +// ========================================== +describe('StepOnePreview', () => { + // Mock preview components + vi.mock('./preview/file-preview', () => ({ + default: ({ file, hidePreview }: { file: CustomFile, hidePreview: () => void }) => ( +
+ {file.name} + +
+ ), + })) + + vi.mock('./preview/online-document-preview', () => ({ + default: ({ datasourceNodeId, currentPage, hidePreview }: { + datasourceNodeId: string + currentPage: NotionPage & { workspace_id: string } + hidePreview: () => void + }) => ( +
+ {datasourceNodeId} + {currentPage.page_id} + +
+ ), + })) + + vi.mock('./preview/web-preview', () => ({ + default: ({ currentWebsite, hidePreview }: { currentWebsite: CrawlResultItem, hidePreview: () => void }) => ( +
+ {currentWebsite.source_url} + +
+ ), + })) + + const defaultProps = { + datasource: undefined as Datasource | undefined, + currentLocalFile: undefined as CustomFile | undefined, + currentDocument: undefined as (NotionPage & { workspace_id: string }) | undefined, + currentWebsite: undefined as CrawlResultItem | undefined, + hidePreviewLocalFile: vi.fn(), + hidePreviewOnlineDocument: vi.fn(), + hideWebsitePreview: vi.fn(), + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('should not render any preview when no file is selected', () => { + const { container } = render() + expect(container.querySelector('[data-testid="file-preview"]')).not.toBeInTheDocument() + expect(container.querySelector('[data-testid="online-document-preview"]')).not.toBeInTheDocument() + expect(container.querySelector('[data-testid="web-preview"]')).not.toBeInTheDocument() + }) + + it('should render FilePreview when currentLocalFile is set', () => { + render( + , + ) + expect(screen.getByTestId('file-preview')).toBeInTheDocument() + expect(screen.getByTestId('file-name')).toHaveTextContent('test.txt') + }) + + it('should render OnlineDocumentPreview when currentDocument is set', () => { + render( + , + ) + expect(screen.getByTestId('online-document-preview')).toBeInTheDocument() + }) + + it('should render WebsitePreview when currentWebsite is set', () => { + render( + , + ) + expect(screen.getByTestId('web-preview')).toBeInTheDocument() + }) + + it('should call hidePreviewLocalFile when hide button is clicked', () => { + const hidePreviewLocalFile = vi.fn() + render( + , + ) + + fireEvent.click(screen.getByTestId('hide-preview')) + + expect(hidePreviewLocalFile).toHaveBeenCalled() + }) +}) + +describe('StepTwoPreview', () => { + // Mock ChunkPreview + vi.mock('./preview/chunk-preview', () => ({ + default: ({ dataSourceType, isIdle, isPending, onPreview }: { + dataSourceType: string + isIdle: boolean + isPending: boolean + onPreview: () => void + }) => ( +
+ {dataSourceType} + {isIdle.toString()} + {isPending.toString()} + +
+ ), + })) + + const defaultProps = { + datasourceType: DatasourceType.localFile as string | undefined, + localFileList: [] as FileItem[], + onlineDocuments: [] as (NotionPage & { workspace_id: string })[], + websitePages: [] as CrawlResultItem[], + selectedOnlineDriveFileList: [] as OnlineDriveFile[], + isIdle: true, + isPendingPreview: false, + estimateData: undefined, + onPreview: vi.fn(), + handlePreviewFileChange: vi.fn(), + handlePreviewOnlineDocumentChange: vi.fn(), + handlePreviewWebsitePageChange: vi.fn(), + handlePreviewOnlineDriveFileChange: vi.fn(), + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('should render ChunkPreview component', () => { + render() + expect(screen.getByTestId('chunk-preview')).toBeInTheDocument() + }) + + it('should pass datasourceType to ChunkPreview', () => { + render() + expect(screen.getByTestId('datasource-type')).toHaveTextContent(DatasourceType.onlineDocument) + }) + + it('should pass isIdle to ChunkPreview', () => { + render() + expect(screen.getByTestId('is-idle')).toHaveTextContent('false') + }) + + it('should pass isPendingPreview to ChunkPreview', () => { + render() + expect(screen.getByTestId('is-pending')).toHaveTextContent('true') + }) + + it('should call onPreview when preview button is clicked', () => { + const onPreview = vi.fn() + render() + + fireEvent.click(screen.getByTestId('preview-btn')) + + expect(onPreview).toHaveBeenCalled() + }) +}) + +// ========================================== +// Edge Cases Tests +// ========================================== +describe('Edge Cases', () => { + describe('Empty States', () => { + it('should handle undefined datasource in useDatasourceUIState', () => { + const { result } = renderHook(() => useDatasourceUIState({ + datasource: undefined, + allFileLoaded: false, + localFileListLength: 0, + onlineDocumentsLength: 0, + websitePagesLength: 0, + selectedFileIdsLength: 0, + onlineDriveFileList: [], + isVectorSpaceFull: false, + enableBilling: true, + currentWorkspacePagesLength: 0, + fileUploadConfig: { file_size_limit: 15, batch_count_limit: 5 }, + })) + + expect(result.current.datasourceType).toBeUndefined() + expect(result.current.nextBtnDisabled).toBe(true) + }) + }) + + describe('Boundary Conditions', () => { + it('should handle zero file size limit', () => { + const { result } = renderHook(() => useDatasourceUIState({ + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDrive, + }, + }), + allFileLoaded: false, + localFileListLength: 0, + onlineDocumentsLength: 0, + websitePagesLength: 0, + selectedFileIdsLength: 0, + onlineDriveFileList: [], + isVectorSpaceFull: false, + enableBilling: true, + currentWorkspacePagesLength: 0, + fileUploadConfig: { file_size_limit: 0, batch_count_limit: 0 }, + })) + + expect(result.current.tip).toContain('datasetPipeline.addDocuments.selectOnlineDriveTip') + }) + + it('should handle very large file counts', () => { + const { result } = renderHook(() => useDatasourceUIState({ + datasource: createMockDatasource(), + allFileLoaded: true, + localFileListLength: 10000, + onlineDocumentsLength: 0, + websitePagesLength: 0, + selectedFileIdsLength: 0, + onlineDriveFileList: [], + isVectorSpaceFull: false, + enableBilling: true, + currentWorkspacePagesLength: 0, + fileUploadConfig: { file_size_limit: 15, batch_count_limit: 5 }, + })) + + expect(result.current.nextBtnDisabled).toBe(false) + }) + }) + + describe('File with special characters', () => { + it('should handle file name with special characters', () => { + const file = createMockFile({ name: 'test<>&"\'file.txt' }) + const result = buildLocalFileDatasourceInfo(file, 'cred-1') + expect(result.name).toBe('test<>&"\'file.txt') + }) + + it('should handle unicode file names', () => { + const file = createMockFile({ name: '测试文件🚀.txt' }) + const result = buildLocalFileDatasourceInfo(file, 'cred-1') + expect(result.name).toBe('测试文件🚀.txt') + }) + }) +}) + +// ========================================== +// Component Memoization Tests +// ========================================== +describe('Component Memoization', () => { + it('StepOneContent should be memoized', async () => { + const StepOneContentModule = await import('./steps/step-one-content') + expect(StepOneContentModule.default.$$typeof).toBe(Symbol.for('react.memo')) + }) + + it('StepTwoContent should be memoized', async () => { + const StepTwoContentModule = await import('./steps/step-two-content') + expect(StepTwoContentModule.default.$$typeof).toBe(Symbol.for('react.memo')) + }) + + it('StepThreeContent should be memoized', async () => { + const StepThreeContentModule = await import('./steps/step-three-content') + expect(StepThreeContentModule.default.$$typeof).toBe(Symbol.for('react.memo')) + }) + + it('StepOnePreview should be memoized', () => { + expect(StepOnePreview.$$typeof).toBe(Symbol.for('react.memo')) + }) + + it('StepTwoPreview should be memoized', () => { + expect(StepTwoPreview.$$typeof).toBe(Symbol.for('react.memo')) + }) +}) + +// ========================================== +// Hook Callback Stability Tests +// ========================================== +describe('Hook Callback Stability', () => { + describe('useDatasourceUIState memoization', () => { + it('should maintain stable reference for datasourceType when dependencies unchanged', () => { + const params = { + datasource: createMockDatasource(), + allFileLoaded: true, + localFileListLength: 1, + onlineDocumentsLength: 0, + websitePagesLength: 0, + selectedFileIdsLength: 0, + onlineDriveFileList: [] as OnlineDriveFile[], + isVectorSpaceFull: false, + enableBilling: true, + currentWorkspacePagesLength: 0, + fileUploadConfig: { file_size_limit: 15, batch_count_limit: 5 }, + } + + const { result, rerender } = renderHook(() => useDatasourceUIState(params)) + const firstType = result.current.datasourceType + + rerender() + + expect(result.current.datasourceType).toBe(firstType) + }) + }) +}) + +// ========================================== +// Store Hooks Tests +// ========================================== +describe('Store Hooks', () => { + describe('useLocalFile', () => { + it('should return localFileList from store', () => { + mockStoreState.localFileList = [createMockFileItem()] + const { result } = renderHook(() => useLocalFile()) + expect(result.current.localFileList).toHaveLength(1) + }) + + it('should compute allFileLoaded correctly when all files have ids', () => { + mockStoreState.localFileList = [createMockFileItem()] + const { result } = renderHook(() => useLocalFile()) + expect(result.current.allFileLoaded).toBe(true) + }) + + it('should compute allFileLoaded as false when no files', () => { + mockStoreState.localFileList = [] + const { result } = renderHook(() => useLocalFile()) + expect(result.current.allFileLoaded).toBe(false) + }) + }) + + describe('useOnlineDocument', () => { + it('should return onlineDocuments from store', () => { + mockStoreState.onlineDocuments = [createMockNotionPage()] + const { result } = renderHook(() => useOnlineDocument()) + expect(result.current.onlineDocuments).toHaveLength(1) + }) + + it('should compute PagesMapAndSelectedPagesId correctly', () => { + mockStoreState.documentsData = [{ + workspace_id: 'ws-1', + pages: [{ page_id: 'page-1' }], + }] + const { result } = renderHook(() => useOnlineDocument()) + expect(result.current.PagesMapAndSelectedPagesId['page-1']).toBeDefined() + }) + }) + + describe('useWebsiteCrawl', () => { + it('should return websitePages from store', () => { + mockStoreState.websitePages = [createMockCrawlResult()] + const { result } = renderHook(() => useWebsiteCrawl()) + expect(result.current.websitePages).toHaveLength(1) + }) + }) + + describe('useOnlineDrive', () => { + it('should return onlineDriveFileList from store', () => { + mockStoreState.onlineDriveFileList = [createMockOnlineDriveFile()] + const { result } = renderHook(() => useOnlineDrive()) + expect(result.current.onlineDriveFileList).toHaveLength(1) + }) + + it('should compute selectedOnlineDriveFileList correctly', () => { + mockStoreState.onlineDriveFileList = [ + createMockOnlineDriveFile({ id: 'file-1' }), + createMockOnlineDriveFile({ id: 'file-2' }), + ] + mockStoreState.selectedFileIds = ['file-1'] + const { result } = renderHook(() => useOnlineDrive()) + expect(result.current.selectedOnlineDriveFileList).toHaveLength(1) + expect(result.current.selectedOnlineDriveFileList[0].id).toBe('file-1') + }) + }) +}) + +// ========================================== +// All Datasource Types Tests +// ========================================== +describe('All Datasource Types', () => { + const datasourceTypes = [ + { type: DatasourceType.localFile, name: 'Local File' }, + { type: DatasourceType.onlineDocument, name: 'Online Document' }, + { type: DatasourceType.websiteCrawl, name: 'Website Crawl' }, + { type: DatasourceType.onlineDrive, name: 'Online Drive' }, + ] + + describe.each(datasourceTypes)('$name datasource type', ({ type }) => { + it(`should handle ${type} in useDatasourceUIState`, () => { + const { result } = renderHook(() => useDatasourceUIState({ + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: type, + }, + }), + allFileLoaded: type === DatasourceType.localFile, + localFileListLength: type === DatasourceType.localFile ? 1 : 0, + onlineDocumentsLength: type === DatasourceType.onlineDocument ? 1 : 0, + websitePagesLength: type === DatasourceType.websiteCrawl ? 1 : 0, + selectedFileIdsLength: type === DatasourceType.onlineDrive ? 1 : 0, + onlineDriveFileList: type === DatasourceType.onlineDrive ? [createMockOnlineDriveFile()] : [], + isVectorSpaceFull: false, + enableBilling: true, + currentWorkspacePagesLength: type === DatasourceType.onlineDocument ? 1 : 0, + fileUploadConfig: { file_size_limit: 15, batch_count_limit: 5 }, + })) + + expect(result.current.datasourceType).toBe(type) + expect(result.current.nextBtnDisabled).toBe(false) + }) + }) +}) + +// ========================================== +// useDatasourceOptions Hook Tests +// ========================================== +describe('useDatasourceOptions', () => { + it('should return empty array when no pipeline nodes', () => { + const { result } = renderHook(() => useDatasourceOptions([])) + expect(result.current).toEqual([]) + }) + + it('should filter and map data source nodes', () => { + const mockNodes: Node[] = [ + { + id: 'node-1', + type: 'data-source', + position: { x: 0, y: 0 }, + data: { + type: BlockEnum.DataSource, + title: 'Local File Source', + provider_type: DatasourceType.localFile, + plugin_id: 'plugin-1', + } as DataSourceNodeType, + }, + { + id: 'node-2', + type: 'other', + position: { x: 0, y: 0 }, + data: { + type: BlockEnum.Start, + title: 'Start Node', + } as unknown as DataSourceNodeType, + }, + ] + + const { result } = renderHook(() => useDatasourceOptions(mockNodes)) + expect(result.current).toHaveLength(1) + expect(result.current[0].label).toBe('Local File Source') + expect(result.current[0].value).toBe('node-1') + }) + + it('should return multiple options for multiple data source nodes', () => { + const mockNodes: Node[] = [ + { + id: 'node-1', + type: 'data-source', + position: { x: 0, y: 0 }, + data: { + type: BlockEnum.DataSource, + title: 'Source 1', + provider_type: DatasourceType.localFile, + plugin_id: 'plugin-1', + } as DataSourceNodeType, + }, + { + id: 'node-2', + type: 'data-source', + position: { x: 0, y: 0 }, + data: { + type: BlockEnum.DataSource, + title: 'Source 2', + provider_type: DatasourceType.onlineDocument, + plugin_id: 'plugin-2', + } as DataSourceNodeType, + }, + ] + + const { result } = renderHook(() => useDatasourceOptions(mockNodes)) + expect(result.current).toHaveLength(2) + }) +}) + +// ========================================== +// useDatasourceActions Hook Tests +// ========================================== +describe('useDatasourceActions', () => { + const createMockDataSourceStore = () => ({ + getState: () => ({ + ...mockStoreState, + previewLocalFileRef: { current: createMockFile() }, + previewOnlineDocumentRef: { current: createMockNotionPage() }, + previewWebsitePageRef: { current: createMockCrawlResult() }, + previewOnlineDriveFileRef: { current: createMockOnlineDriveFile() }, + currentCredentialId: 'cred-1', + bucket: 'test-bucket', + localFileList: [createMockFileItem()], + onlineDocuments: [createMockNotionPage()], + websitePages: [createMockCrawlResult()], + selectedFileIds: ['file-1'], + onlineDriveFileList: [createMockOnlineDriveFile({ id: 'file-1' })], + setCurrentCredentialId: vi.fn(), + currentNodeIdRef: { current: '' }, + setOnlineDocuments: vi.fn(), + setSelectedFileIds: vi.fn(), + setSelectedPagesId: vi.fn(), + }), + }) + + const defaultParams = { + datasource: createMockDatasource(), + datasourceType: DatasourceType.localFile, + pipelineId: 'pipeline-1', + dataSourceStore: createMockDataSourceStore() as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: { 'page-1': createMockNotionPage() }, + currentWorkspacePages: [{ page_id: 'page-1' }], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('should return initial state and callbacks', () => { + const { result } = renderHook(() => useDatasourceActions(defaultParams)) + + expect(result.current.isPreview).toBeDefined() + expect(result.current.formRef).toBeDefined() + expect(result.current.isIdle).toBe(true) + expect(result.current.isPending).toBe(false) + expect(typeof result.current.onClickProcess).toBe('function') + expect(typeof result.current.onClickPreview).toBe('function') + expect(typeof result.current.handleSubmit).toBe('function') + }) + + it('should set isPreview to false when onClickProcess is called', () => { + const { result } = renderHook(() => useDatasourceActions(defaultParams)) + + act(() => { + result.current.onClickProcess() + }) + + expect(result.current.isPreview.current).toBe(false) + }) + + it('should set isPreview to true when onClickPreview is called', () => { + const { result } = renderHook(() => useDatasourceActions(defaultParams)) + + act(() => { + result.current.onClickPreview() + }) + + expect(result.current.isPreview.current).toBe(true) + }) + + it('should call handlePreviewFileChange and trigger preview', () => { + const { result } = renderHook(() => useDatasourceActions(defaultParams)) + const mockFile = { id: 'file-1', name: 'test.txt' } as unknown as DocumentItem + + act(() => { + result.current.handlePreviewFileChange(mockFile) + }) + + expect(result.current.isPreview.current).toBe(true) + }) + + it('should call handlePreviewOnlineDocumentChange and trigger preview', () => { + const { result } = renderHook(() => useDatasourceActions(defaultParams)) + const mockPage = createMockNotionPage() + + act(() => { + result.current.handlePreviewOnlineDocumentChange(mockPage) + }) + + expect(result.current.isPreview.current).toBe(true) + }) + + it('should call handlePreviewWebsiteChange and trigger preview', () => { + const { result } = renderHook(() => useDatasourceActions(defaultParams)) + const mockWebsite = createMockCrawlResult() + + act(() => { + result.current.handlePreviewWebsiteChange(mockWebsite) + }) + + expect(result.current.isPreview.current).toBe(true) + }) + + it('should call handlePreviewOnlineDriveFileChange and trigger preview', () => { + const { result } = renderHook(() => useDatasourceActions(defaultParams)) + const mockFile = createMockOnlineDriveFile() + + act(() => { + result.current.handlePreviewOnlineDriveFileChange(mockFile) + }) + + expect(result.current.isPreview.current).toBe(true) + }) + + it('should handle select all for online document', () => { + const params = { + ...defaultParams, + datasourceType: DatasourceType.onlineDocument, + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + onlineDocuments: [], + setOnlineDocuments: vi.fn(), + setSelectedPagesId: vi.fn(), + }), + } as MockDataSourceStore, + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.handleSelectAll() + }) + + // Verify the callback was executed (no error thrown) + expect(true).toBe(true) + }) + + it('should handle select all for online drive', () => { + const params = { + ...defaultParams, + datasourceType: DatasourceType.onlineDrive, + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + onlineDriveFileList: [createMockOnlineDriveFile({ id: 'file-1' })], + selectedFileIds: [], + setSelectedFileIds: vi.fn(), + }), + } as MockDataSourceStore, + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.handleSelectAll() + }) + + expect(true).toBe(true) + }) + + it('should handle switch data source', () => { + const setDatasource = vi.fn() + const params = { + ...defaultParams, + setDatasource, + } + + const { result } = renderHook(() => useDatasourceActions(params)) + const newDatasource = createMockDatasource({ nodeId: 'node-2' }) + + act(() => { + result.current.handleSwitchDataSource(newDatasource) + }) + + expect(setDatasource).toHaveBeenCalledWith(newDatasource) + }) + + it('should handle credential change', () => { + const { result } = renderHook(() => useDatasourceActions(defaultParams)) + + act(() => { + result.current.handleCredentialChange('new-cred-id') + }) + + // Should not throw error + expect(true).toBe(true) + }) + + it('should clear online document data when switching datasource', () => { + const clearOnlineDocumentData = vi.fn() + const params = { + ...defaultParams, + clearOnlineDocumentData, + } + + const { result } = renderHook(() => useDatasourceActions(params)) + const newDatasource = createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDocument, + }, + }) + + act(() => { + result.current.handleSwitchDataSource(newDatasource) + }) + + expect(clearOnlineDocumentData).toHaveBeenCalled() + }) + + it('should clear website crawl data when switching datasource', () => { + const clearWebsiteCrawlData = vi.fn() + const params = { + ...defaultParams, + clearWebsiteCrawlData, + } + + const { result } = renderHook(() => useDatasourceActions(params)) + const newDatasource = createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.websiteCrawl, + }, + }) + + act(() => { + result.current.handleSwitchDataSource(newDatasource) + }) + + expect(clearWebsiteCrawlData).toHaveBeenCalled() + }) + + it('should clear online drive data when switching datasource', () => { + const clearOnlineDriveData = vi.fn() + const params = { + ...defaultParams, + clearOnlineDriveData, + } + + const { result } = renderHook(() => useDatasourceActions(params)) + const newDatasource = createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDrive, + }, + }) + + act(() => { + result.current.handleSwitchDataSource(newDatasource) + }) + + expect(clearOnlineDriveData).toHaveBeenCalled() + }) +}) + +// ========================================== +// Store Hooks - Additional Coverage Tests +// ========================================== +describe('Store Hooks - Callbacks', () => { + beforeEach(() => { + vi.clearAllMocks() + // Reset mock store state + mockStoreState.localFileList = [] + mockStoreState.documentsData = [] + mockStoreState.onlineDocuments = [] + mockStoreState.websitePages = [] + mockStoreState.onlineDriveFileList = [] + mockStoreState.selectedFileIds = [] + }) + + describe('useLocalFile callbacks', () => { + it('should call hidePreviewLocalFile callback', () => { + const { result } = renderHook(() => useLocalFile()) + + act(() => { + result.current.hidePreviewLocalFile() + }) + + expect(mockStoreState.setCurrentLocalFile).toHaveBeenCalledWith(undefined) + }) + }) + + describe('useOnlineDocument callbacks', () => { + it('should return currentWorkspace from documentsData', () => { + mockStoreState.documentsData = [{ workspace_id: 'ws-1', pages: [] }] + const { result } = renderHook(() => useOnlineDocument()) + + expect(result.current.currentWorkspace).toBeDefined() + expect(result.current.currentWorkspace?.workspace_id).toBe('ws-1') + }) + + it('should call hidePreviewOnlineDocument callback', () => { + const { result } = renderHook(() => useOnlineDocument()) + + act(() => { + result.current.hidePreviewOnlineDocument() + }) + + expect(mockStoreState.setCurrentDocument).toHaveBeenCalledWith(undefined) + }) + + it('should call clearOnlineDocumentData callback', () => { + const { result } = renderHook(() => useOnlineDocument()) + + act(() => { + result.current.clearOnlineDocumentData() + }) + + expect(mockStoreState.setDocumentsData).toHaveBeenCalledWith([]) + expect(mockStoreState.setSearchValue).toHaveBeenCalledWith('') + expect(mockStoreState.setOnlineDocuments).toHaveBeenCalledWith([]) + expect(mockStoreState.setCurrentDocument).toHaveBeenCalledWith(undefined) + }) + }) + + describe('useWebsiteCrawl callbacks', () => { + it('should call hideWebsitePreview callback', () => { + const { result } = renderHook(() => useWebsiteCrawl()) + + act(() => { + result.current.hideWebsitePreview() + }) + + expect(mockStoreState.setCurrentWebsite).toHaveBeenCalledWith(undefined) + expect(mockStoreState.setPreviewIndex).toHaveBeenCalledWith(-1) + }) + + it('should call clearWebsiteCrawlData callback', () => { + const { result } = renderHook(() => useWebsiteCrawl()) + + act(() => { + result.current.clearWebsiteCrawlData() + }) + + expect(mockStoreState.setStep).toHaveBeenCalled() + expect(mockStoreState.setCrawlResult).toHaveBeenCalledWith(undefined) + expect(mockStoreState.setCurrentWebsite).toHaveBeenCalledWith(undefined) + expect(mockStoreState.setWebsitePages).toHaveBeenCalledWith([]) + expect(mockStoreState.setPreviewIndex).toHaveBeenCalledWith(-1) + }) + }) + + describe('useOnlineDrive callbacks', () => { + it('should call clearOnlineDriveData callback', () => { + const { result } = renderHook(() => useOnlineDrive()) + + act(() => { + result.current.clearOnlineDriveData() + }) + + expect(mockStoreState.setOnlineDriveFileList).toHaveBeenCalledWith([]) + expect(mockStoreState.setBucket).toHaveBeenCalledWith('') + expect(mockStoreState.setPrefix).toHaveBeenCalledWith([]) + expect(mockStoreState.setKeywords).toHaveBeenCalledWith('') + expect(mockStoreState.setSelectedFileIds).toHaveBeenCalledWith([]) + }) + }) +}) + +// ========================================== +// StepOneContent - All Datasource Types +// ========================================== +describe('StepOneContent - All Datasource Types', () => { + // Mock data source components + vi.mock('./data-source/local-file', () => ({ + default: () =>
Local File
, + })) + + vi.mock('./data-source/online-documents', () => ({ + default: () =>
Online Documents
, + })) + + vi.mock('./data-source/website-crawl', () => ({ + default: () =>
Website Crawl
, + })) + + vi.mock('./data-source/online-drive', () => ({ + default: () =>
Online Drive
, + })) + + const defaultProps = { + datasource: undefined as Datasource | undefined, + datasourceType: undefined as string | undefined, + pipelineNodes: [] as Node[], + supportBatchUpload: true, + localFileListLength: 0, + isShowVectorSpaceFull: false, + showSelect: false, + totalOptions: undefined as number | undefined, + selectedOptions: undefined as number | undefined, + tip: '', + nextBtnDisabled: true, + onSelectDataSource: vi.fn(), + onCredentialChange: vi.fn(), + onSelectAll: vi.fn(), + onNextStep: vi.fn(), + } + + it('should render OnlineDocuments when datasourceType is onlineDocument', () => { + render( + , + ) + expect(screen.getByTestId('online-documents-component')).toBeInTheDocument() + }) + + it('should render WebsiteCrawl when datasourceType is websiteCrawl', () => { + render( + , + ) + expect(screen.getByTestId('website-crawl-component')).toBeInTheDocument() + }) + + it('should render OnlineDrive when datasourceType is onlineDrive', () => { + render( + , + ) + expect(screen.getByTestId('online-drive-component')).toBeInTheDocument() + }) + + it('should render LocalFile when datasourceType is localFile', () => { + render( + , + ) + expect(screen.getByTestId('local-file-component')).toBeInTheDocument() + }) +}) + +// ========================================== +// StepTwoPreview - with localFileList +// ========================================== +describe('StepTwoPreview - File List Mapping', () => { + it('should correctly map localFileList to localFiles', () => { + const fileList = [ + createMockFileItem({ file: createMockFile({ id: 'f1', name: 'file1.txt' }) }), + createMockFileItem({ file: createMockFile({ id: 'f2', name: 'file2.txt' }) }), + ] + + render( + , + ) + + // ChunkPreview should be rendered + expect(screen.getByTestId('chunk-preview')).toBeInTheDocument() + }) +}) + +// ========================================== +// useDatasourceActions - Additional Coverage +// ========================================== +describe('useDatasourceActions - Async Functions', () => { + beforeEach(() => { + vi.clearAllMocks() + mockRunPublishedPipeline.mockReset() + }) + + const createMockDataSourceStoreForAsync = (datasourceType: string) => ({ + getState: () => ({ + previewLocalFileRef: { current: datasourceType === DatasourceType.localFile ? createMockFile() : undefined }, + previewOnlineDocumentRef: { current: datasourceType === DatasourceType.onlineDocument ? createMockNotionPage() : undefined }, + previewWebsitePageRef: { current: datasourceType === DatasourceType.websiteCrawl ? createMockCrawlResult() : undefined }, + previewOnlineDriveFileRef: { current: datasourceType === DatasourceType.onlineDrive ? createMockOnlineDriveFile() : undefined }, + currentCredentialId: 'cred-1', + bucket: 'test-bucket', + localFileList: [createMockFileItem()], + onlineDocuments: [createMockNotionPage()], + websitePages: [createMockCrawlResult()], + selectedFileIds: ['file-1'], + onlineDriveFileList: [createMockOnlineDriveFile({ id: 'file-1' })], + setCurrentCredentialId: vi.fn(), + currentNodeIdRef: { current: '' }, + setOnlineDocuments: vi.fn(), + setSelectedFileIds: vi.fn(), + setSelectedPagesId: vi.fn(), + }), + }) + + it('should call handleSubmit with preview mode', () => { + const setEstimateData = vi.fn() + const params = { + datasource: createMockDatasource(), + datasourceType: DatasourceType.localFile, + pipelineId: 'pipeline-1', + dataSourceStore: createMockDataSourceStoreForAsync(DatasourceType.localFile) as MockDataSourceStore, + setEstimateData, + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.onClickPreview() + result.current.handleSubmit({ test: 'data' }) + }) + + // Should have triggered preview + expect(result.current.isPreview.current).toBe(true) + }) + + it('should call handleSubmit with process mode', () => { + const setBatchId = vi.fn() + const setDocuments = vi.fn() + const handleNextStep = vi.fn() + const params = { + datasource: createMockDatasource(), + datasourceType: DatasourceType.localFile, + pipelineId: 'pipeline-1', + dataSourceStore: createMockDataSourceStoreForAsync(DatasourceType.localFile) as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId, + setDocuments, + handleNextStep, + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.onClickProcess() + result.current.handleSubmit({ test: 'data' }) + }) + + // Should have triggered process + expect(result.current.isPreview.current).toBe(false) + }) + + it('should not call API when datasource is undefined', () => { + const params = { + datasource: undefined, + datasourceType: DatasourceType.localFile, + pipelineId: 'pipeline-1', + dataSourceStore: createMockDataSourceStoreForAsync(DatasourceType.localFile) as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.handleSubmit({ test: 'data' }) + }) + + expect(mockRunPublishedPipeline).not.toHaveBeenCalled() + }) + + it('should not call API when pipelineId is undefined', () => { + const params = { + datasource: createMockDatasource(), + datasourceType: DatasourceType.localFile, + pipelineId: undefined, + dataSourceStore: createMockDataSourceStoreForAsync(DatasourceType.localFile) as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.handleSubmit({ test: 'data' }) + }) + + expect(mockRunPublishedPipeline).not.toHaveBeenCalled() + }) + + it('should build preview info for online document type', () => { + const params = { + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDocument, + }, + }), + datasourceType: DatasourceType.onlineDocument, + pipelineId: 'pipeline-1', + dataSourceStore: createMockDataSourceStoreForAsync(DatasourceType.onlineDocument) as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.onClickPreview() + result.current.handleSubmit({ test: 'data' }) + }) + + expect(result.current.isPreview.current).toBe(true) + }) + + it('should build preview info for website crawl type', () => { + const params = { + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.websiteCrawl, + }, + }), + datasourceType: DatasourceType.websiteCrawl, + pipelineId: 'pipeline-1', + dataSourceStore: createMockDataSourceStoreForAsync(DatasourceType.websiteCrawl) as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.onClickPreview() + result.current.handleSubmit({ test: 'data' }) + }) + + expect(result.current.isPreview.current).toBe(true) + }) + + it('should build preview info for online drive type', () => { + const params = { + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDrive, + }, + }), + datasourceType: DatasourceType.onlineDrive, + pipelineId: 'pipeline-1', + dataSourceStore: createMockDataSourceStoreForAsync(DatasourceType.onlineDrive) as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.onClickPreview() + result.current.handleSubmit({ test: 'data' }) + }) + + expect(result.current.isPreview.current).toBe(true) + }) + + it('should toggle select all for online document - deselect all when already selected', () => { + const setOnlineDocuments = vi.fn() + const setSelectedPagesId = vi.fn() + const params = { + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDocument, + }, + }), + datasourceType: DatasourceType.onlineDocument, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + onlineDocuments: [createMockNotionPage()], + setOnlineDocuments, + setSelectedPagesId, + }), + } as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: { 'page-1': createMockNotionPage() }, + currentWorkspacePages: [{ page_id: 'page-1' }], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.handleSelectAll() + }) + + // Should deselect all since documents.length >= allIds.length + expect(setOnlineDocuments).toHaveBeenCalledWith([]) + }) + + it('should toggle select all for online drive - deselect all when already selected', () => { + const setSelectedFileIds = vi.fn() + const params = { + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDrive, + }, + }), + datasourceType: DatasourceType.onlineDrive, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + onlineDriveFileList: [createMockOnlineDriveFile({ id: 'file-1' })], + selectedFileIds: ['file-1'], + setSelectedFileIds, + }), + } as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.handleSelectAll() + }) + + // Should deselect all since selectedFileIds.length >= allKeys.length + expect(setSelectedFileIds).toHaveBeenCalledWith([]) + }) + + it('should clear data when credential changes with datasource', () => { + const clearOnlineDocumentData = vi.fn() + const params = { + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDocument, + }, + }), + datasourceType: DatasourceType.onlineDocument, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + setCurrentCredentialId: vi.fn(), + }), + } as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData, + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.handleCredentialChange('new-cred') + }) + + expect(clearOnlineDocumentData).toHaveBeenCalled() + }) +}) + +// ========================================== +// useDatasourceActions - onSuccess Callbacks +// ========================================== +describe('useDatasourceActions - API Success Callbacks', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('should call setEstimateData on preview success', async () => { + const setEstimateData = vi.fn() + const mockResponse = { + data: { outputs: { chunks: 10, tokens: 100 } }, + } + + // Create a mock that calls onSuccess + const mockMutateAsync = vi.fn().mockImplementation((_params, options) => { + options?.onSuccess?.(mockResponse) + return Promise.resolve(mockResponse) + }) + + vi.mocked(mockRunPublishedPipeline).mockImplementation(mockMutateAsync) + + const params = { + datasource: createMockDatasource(), + datasourceType: DatasourceType.localFile, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + previewLocalFileRef: { current: createMockFile() }, + currentCredentialId: 'cred-1', + localFileList: [createMockFileItem()], + }), + } as MockDataSourceStore, + setEstimateData, + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + await act(async () => { + result.current.isPreview.current = true + await result.current.handleSubmit({ test: 'data' }) + }) + + expect(setEstimateData).toHaveBeenCalledWith(mockResponse.data.outputs) + }) + + it('should call setBatchId, setDocuments, handleNextStep on process success', async () => { + const setBatchId = vi.fn() + const setDocuments = vi.fn() + const handleNextStep = vi.fn() + const mockResponse = { + batch: 'batch-123', + documents: [{ id: 'doc-1' }], + } + + const mockMutateAsync = vi.fn().mockImplementation((_params, options) => { + options?.onSuccess?.(mockResponse) + return Promise.resolve(mockResponse) + }) + + vi.mocked(mockRunPublishedPipeline).mockImplementation(mockMutateAsync) + + const params = { + datasource: createMockDatasource(), + datasourceType: DatasourceType.localFile, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + previewLocalFileRef: { current: createMockFile() }, + currentCredentialId: 'cred-1', + localFileList: [createMockFileItem()], + }), + } as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId, + setDocuments, + handleNextStep, + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + await act(async () => { + result.current.isPreview.current = false + await result.current.handleSubmit({ test: 'data' }) + }) + + expect(setBatchId).toHaveBeenCalledWith('batch-123') + expect(setDocuments).toHaveBeenCalledWith([{ id: 'doc-1' }]) + expect(handleNextStep).toHaveBeenCalled() + }) + + it('should handle empty batch and documents in process response', async () => { + const setBatchId = vi.fn() + const setDocuments = vi.fn() + const handleNextStep = vi.fn() + const mockResponse = {} // Empty response + + const mockMutateAsync = vi.fn().mockImplementation((_params, options) => { + options?.onSuccess?.(mockResponse) + return Promise.resolve(mockResponse) + }) + + vi.mocked(mockRunPublishedPipeline).mockImplementation(mockMutateAsync) + + const params = { + datasource: createMockDatasource(), + datasourceType: DatasourceType.localFile, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + previewLocalFileRef: { current: createMockFile() }, + currentCredentialId: 'cred-1', + localFileList: [createMockFileItem()], + }), + } as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId, + setDocuments, + handleNextStep, + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + await act(async () => { + result.current.isPreview.current = false + await result.current.handleSubmit({ test: 'data' }) + }) + + expect(setBatchId).toHaveBeenCalledWith('') + expect(setDocuments).toHaveBeenCalledWith([]) + expect(handleNextStep).toHaveBeenCalled() + }) +}) + +// ========================================== +// useDatasourceActions - buildProcessDatasourceInfo Coverage +// ========================================== +describe('useDatasourceActions - Process Mode for All Datasource Types', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('should build process info for onlineDocument type', async () => { + const setBatchId = vi.fn() + const setDocuments = vi.fn() + const handleNextStep = vi.fn() + const mockResponse = { batch: 'batch-1', documents: [] } + + const mockMutateAsync = vi.fn().mockImplementation((_params, options) => { + options?.onSuccess?.(mockResponse) + return Promise.resolve(mockResponse) + }) + vi.mocked(mockRunPublishedPipeline).mockImplementation(mockMutateAsync) + + const params = { + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDocument, + }, + }), + datasourceType: DatasourceType.onlineDocument, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + currentCredentialId: 'cred-1', + onlineDocuments: [createMockNotionPage()], + }), + } as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId, + setDocuments, + handleNextStep, + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + await act(async () => { + result.current.isPreview.current = false + await result.current.handleSubmit({ test: 'data' }) + }) + + expect(mockMutateAsync).toHaveBeenCalled() + expect(setBatchId).toHaveBeenCalled() + }) + + it('should build process info for websiteCrawl type', async () => { + const setBatchId = vi.fn() + const setDocuments = vi.fn() + const handleNextStep = vi.fn() + const mockResponse = { batch: 'batch-1', documents: [] } + + const mockMutateAsync = vi.fn().mockImplementation((_params, options) => { + options?.onSuccess?.(mockResponse) + return Promise.resolve(mockResponse) + }) + vi.mocked(mockRunPublishedPipeline).mockImplementation(mockMutateAsync) + + const params = { + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.websiteCrawl, + }, + }), + datasourceType: DatasourceType.websiteCrawl, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + currentCredentialId: 'cred-1', + websitePages: [createMockCrawlResult()], + }), + } as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId, + setDocuments, + handleNextStep, + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + await act(async () => { + result.current.isPreview.current = false + await result.current.handleSubmit({ test: 'data' }) + }) + + expect(mockMutateAsync).toHaveBeenCalled() + expect(setBatchId).toHaveBeenCalled() + }) + + it('should build process info for onlineDrive type', async () => { + const setBatchId = vi.fn() + const setDocuments = vi.fn() + const handleNextStep = vi.fn() + const mockResponse = { batch: 'batch-1', documents: [] } + + const mockMutateAsync = vi.fn().mockImplementation((_params, options) => { + options?.onSuccess?.(mockResponse) + return Promise.resolve(mockResponse) + }) + vi.mocked(mockRunPublishedPipeline).mockImplementation(mockMutateAsync) + + const params = { + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDrive, + }, + }), + datasourceType: DatasourceType.onlineDrive, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + currentCredentialId: 'cred-1', + bucket: 'test-bucket', + selectedFileIds: ['file-1'], + onlineDriveFileList: [createMockOnlineDriveFile({ id: 'file-1' })], + }), + } as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId, + setDocuments, + handleNextStep, + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + await act(async () => { + result.current.isPreview.current = false + await result.current.handleSubmit({ test: 'data' }) + }) + + expect(mockMutateAsync).toHaveBeenCalled() + expect(setBatchId).toHaveBeenCalled() + }) + + it('should return early in preview mode when datasource is undefined', async () => { + const setEstimateData = vi.fn() + const mockMutateAsync = vi.fn() + vi.mocked(mockRunPublishedPipeline).mockImplementation(mockMutateAsync) + + const params = { + datasource: undefined, // undefined datasource + datasourceType: DatasourceType.localFile, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ ...mockStoreState }), + } as MockDataSourceStore, + setEstimateData, + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + await act(async () => { + result.current.isPreview.current = true + await result.current.handleSubmit({ test: 'data' }) + }) + + // Should not call API when datasource is undefined + expect(mockMutateAsync).not.toHaveBeenCalled() + expect(setEstimateData).not.toHaveBeenCalled() + }) + + it('should return early in preview mode when pipelineId is undefined', async () => { + const setEstimateData = vi.fn() + const mockMutateAsync = vi.fn() + vi.mocked(mockRunPublishedPipeline).mockImplementation(mockMutateAsync) + + const params = { + datasource: createMockDatasource(), + datasourceType: DatasourceType.localFile, + pipelineId: undefined, // undefined pipelineId + dataSourceStore: { + getState: () => ({ ...mockStoreState }), + } as MockDataSourceStore, + setEstimateData, + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + await act(async () => { + result.current.isPreview.current = true + await result.current.handleSubmit({ test: 'data' }) + }) + + // Should not call API when pipelineId is undefined + expect(mockMutateAsync).not.toHaveBeenCalled() + expect(setEstimateData).not.toHaveBeenCalled() + }) + + it('should skip file if not found in onlineDriveFileList', async () => { + const setBatchId = vi.fn() + const mockResponse = { batch: 'batch-1', documents: [] } + + const mockMutateAsync = vi.fn().mockImplementation((_params, options) => { + options?.onSuccess?.(mockResponse) + return Promise.resolve(mockResponse) + }) + vi.mocked(mockRunPublishedPipeline).mockImplementation(mockMutateAsync) + + const params = { + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDrive, + }, + }), + datasourceType: DatasourceType.onlineDrive, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + currentCredentialId: 'cred-1', + bucket: 'test-bucket', + selectedFileIds: ['non-existent-file'], + onlineDriveFileList: [createMockOnlineDriveFile({ id: 'file-1' })], + }), + } as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId, + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + await act(async () => { + result.current.isPreview.current = false + await result.current.handleSubmit({ test: 'data' }) + }) + + // Should still call API but with empty datasource_info_list + expect(mockMutateAsync).toHaveBeenCalled() + }) +}) + +// ========================================== +// useDatasourceActions - Edge Case Branches +// ========================================== +describe('useDatasourceActions - Edge Case Branches', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('should handle selectAll when currentWorkspacePages is undefined', () => { + const setOnlineDocuments = vi.fn() + const setSelectedPagesId = vi.fn() + + const params = { + datasource: createMockDatasource({ + nodeData: { + ...createMockDatasource().nodeData, + provider_type: DatasourceType.onlineDocument, + }, + }), + datasourceType: DatasourceType.onlineDocument, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + onlineDocuments: [], + setOnlineDocuments, + setSelectedPagesId, + }), + } as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: undefined, // undefined currentWorkspacePages + clearOnlineDocumentData: vi.fn(), + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.handleSelectAll() + }) + + // Should use empty array when currentWorkspacePages is undefined + // Since allIds.length is 0 and onlineDocuments.length is 0, it should deselect + expect(setOnlineDocuments).toHaveBeenCalledWith([]) + }) + + it('should not clear data when datasource is undefined in handleCredentialChange', () => { + const clearOnlineDocumentData = vi.fn() + + const params = { + datasource: undefined, // undefined datasource + datasourceType: DatasourceType.onlineDocument, + pipelineId: 'pipeline-1', + dataSourceStore: { + getState: () => ({ + ...mockStoreState, + setCurrentCredentialId: vi.fn(), + }), + } as MockDataSourceStore, + setEstimateData: vi.fn(), + setBatchId: vi.fn(), + setDocuments: vi.fn(), + handleNextStep: vi.fn(), + PagesMapAndSelectedPagesId: {}, + currentWorkspacePages: [], + clearOnlineDocumentData, + clearWebsiteCrawlData: vi.fn(), + clearOnlineDriveData: vi.fn(), + setDatasource: vi.fn(), + } + + const { result } = renderHook(() => useDatasourceActions(params)) + + act(() => { + result.current.handleCredentialChange('new-cred') + }) + + // Should not call clearOnlineDocumentData when datasource is undefined + expect(clearOnlineDocumentData).not.toHaveBeenCalled() + }) +}) + +// ========================================== +// Hooks Index Re-exports Test +// ========================================== +describe('Hooks Index Re-exports', () => { + it('should export useAddDocumentsSteps', async () => { + const hooksModule = await import('./hooks') + expect(hooksModule.useAddDocumentsSteps).toBeDefined() + }) + + it('should export useDatasourceActions', async () => { + const hooksModule = await import('./hooks') + expect(hooksModule.useDatasourceActions).toBeDefined() + }) + + it('should export useDatasourceOptions', async () => { + const hooksModule = await import('./hooks') + expect(hooksModule.useDatasourceOptions).toBeDefined() + }) + + it('should export useLocalFile', async () => { + const hooksModule = await import('./hooks') + expect(hooksModule.useLocalFile).toBeDefined() + }) + + it('should export useOnlineDocument', async () => { + const hooksModule = await import('./hooks') + expect(hooksModule.useOnlineDocument).toBeDefined() + }) + + it('should export useOnlineDrive', async () => { + const hooksModule = await import('./hooks') + expect(hooksModule.useOnlineDrive).toBeDefined() + }) + + it('should export useWebsiteCrawl', async () => { + const hooksModule = await import('./hooks') + expect(hooksModule.useWebsiteCrawl).toBeDefined() + }) + + it('should export useDatasourceUIState', async () => { + const hooksModule = await import('./hooks') + expect(hooksModule.useDatasourceUIState).toBeDefined() + }) +}) + +// ========================================== +// Steps Index Re-exports Test +// ========================================== +describe('Steps Index Re-exports', () => { + it('should export StepOneContent', async () => { + const stepsModule = await import('./steps') + expect(stepsModule.StepOneContent).toBeDefined() + }) + + it('should export StepTwoContent', async () => { + const stepsModule = await import('./steps') + expect(stepsModule.StepTwoContent).toBeDefined() + }) + + it('should export StepThreeContent', async () => { + const stepsModule = await import('./steps') + expect(stepsModule.StepThreeContent).toBeDefined() + }) +}) diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index 2b17f97baa..62c1b919fe 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -2,75 +2,71 @@ import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' import type { Node } from '@/app/components/workflow/types' -import type { NotionPage } from '@/models/common' -import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEstimateResponse } from '@/models/datasets' -import type { - InitialDocumentDetail, - OnlineDriveFile, - PublishedPipelineRunPreviewResponse, - PublishedPipelineRunResponse, -} from '@/models/pipeline' +import type { FileIndexingEstimateResponse } from '@/models/datasets' +import type { InitialDocumentDetail } from '@/models/pipeline' import { useBoolean } from 'ahooks' -import { useCallback, useMemo, useRef, useState } from 'react' +import { useCallback, useMemo, useState } from 'react' import { useTranslation } from 'react-i18next' -import { trackEvent } from '@/app/components/base/amplitude' -import Divider from '@/app/components/base/divider' import Loading from '@/app/components/base/loading' import PlanUpgradeModal from '@/app/components/billing/plan-upgrade-modal' -import VectorSpaceFull from '@/app/components/billing/vector-space-full' -import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file' -import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents' -import OnlineDrive from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-drive' -import WebsiteCrawl from '@/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl' import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' import { useProviderContextSelector } from '@/context/provider-context' import { DatasourceType } from '@/models/pipeline' import { useFileUploadConfig } from '@/service/use-common' -import { usePublishedPipelineInfo, useRunPublishedPipeline } from '@/service/use-pipeline' -import { TransferMethod } from '@/types/app' -import UpgradeCard from '../../create/step-one/upgrade-card' -import Actions from './actions' -import DataSourceOptions from './data-source-options' +import { usePublishedPipelineInfo } from '@/service/use-pipeline' import { useDataSourceStore } from './data-source/store' import DataSourceProvider from './data-source/store/provider' -import { useAddDocumentsSteps, useLocalFile, useOnlineDocument, useOnlineDrive, useWebsiteCrawl } from './hooks' +import { + useAddDocumentsSteps, + useDatasourceActions, + useDatasourceUIState, + useLocalFile, + useOnlineDocument, + useOnlineDrive, + useWebsiteCrawl, +} from './hooks' import LeftHeader from './left-header' -import ChunkPreview from './preview/chunk-preview' -import FilePreview from './preview/file-preview' -import OnlineDocumentPreview from './preview/online-document-preview' -import WebsitePreview from './preview/web-preview' -import ProcessDocuments from './process-documents' -import Processing from './processing' +import { StepOneContent, StepThreeContent, StepTwoContent } from './steps' +import { StepOnePreview, StepTwoPreview } from './steps/preview-panel' const CreateFormPipeline = () => { const { t } = useTranslation() const plan = useProviderContextSelector(state => state.plan) const enableBilling = useProviderContextSelector(state => state.enableBilling) const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) + const dataSourceStore = useDataSourceStore() + + // Core state const [datasource, setDatasource] = useState() const [estimateData, setEstimateData] = useState(undefined) const [batchId, setBatchId] = useState('') const [documents, setDocuments] = useState([]) - const dataSourceStore = useDataSourceStore() - - const isPreview = useRef(false) - const formRef = useRef(null) + // Data fetching const { data: pipelineInfo, isFetching: isFetchingPipelineInfo } = usePublishedPipelineInfo(pipelineId || '') const { data: fileUploadConfigResponse } = useFileUploadConfig() + const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? { + file_size_limit: 15, + batch_count_limit: 5, + }, [fileUploadConfigResponse]) + + // Steps management const { steps, currentStep, handleNextStep: doHandleNextStep, handleBackStep, } = useAddDocumentsSteps() + + // Datasource-specific hooks const { localFileList, allFileLoaded, currentLocalFile, hidePreviewLocalFile, } = useLocalFile() + const { currentWorkspace, onlineDocuments, @@ -79,12 +75,14 @@ const CreateFormPipeline = () => { hidePreviewOnlineDocument, clearOnlineDocumentData, } = useOnlineDocument() + const { websitePages, currentWebsite, hideWebsitePreview, clearWebsiteCrawlData, } = useWebsiteCrawl() + const { onlineDriveFileList, selectedFileIds, @@ -92,43 +90,50 @@ const CreateFormPipeline = () => { clearOnlineDriveData, } = useOnlineDrive() - const datasourceType = useMemo(() => datasource?.nodeData.provider_type, [datasource]) + // Computed values const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace - const isShowVectorSpaceFull = useMemo(() => { - if (!datasource) - return false - if (datasourceType === DatasourceType.localFile) - return allFileLoaded && isVectorSpaceFull && enableBilling - if (datasourceType === DatasourceType.onlineDocument) - return onlineDocuments.length > 0 && isVectorSpaceFull && enableBilling - if (datasourceType === DatasourceType.websiteCrawl) - return websitePages.length > 0 && isVectorSpaceFull && enableBilling - if (datasourceType === DatasourceType.onlineDrive) - return onlineDriveFileList.length > 0 && isVectorSpaceFull && enableBilling - return false - }, [allFileLoaded, datasource, datasourceType, enableBilling, isVectorSpaceFull, onlineDocuments.length, onlineDriveFileList.length, websitePages.length]) const supportBatchUpload = !enableBilling || plan.type !== 'sandbox' + // UI state + const { + datasourceType, + isShowVectorSpaceFull, + nextBtnDisabled, + showSelect, + totalOptions, + selectedOptions, + tip, + } = useDatasourceUIState({ + datasource, + allFileLoaded, + localFileListLength: localFileList.length, + onlineDocumentsLength: onlineDocuments.length, + websitePagesLength: websitePages.length, + selectedFileIdsLength: selectedFileIds.length, + onlineDriveFileList, + isVectorSpaceFull, + enableBilling, + currentWorkspacePagesLength: currentWorkspace?.pages.length ?? 0, + fileUploadConfig, + }) + + // Plan upgrade modal const [isShowPlanUpgradeModal, { setTrue: showPlanUpgradeModal, setFalse: hidePlanUpgradeModal, }] = useBoolean(false) + + // Next step with batch upload check const handleNextStep = useCallback(() => { if (!supportBatchUpload) { - let isMultiple = false - if (datasourceType === DatasourceType.localFile && localFileList.length > 1) - isMultiple = true - - if (datasourceType === DatasourceType.onlineDocument && onlineDocuments.length > 1) - isMultiple = true - - if (datasourceType === DatasourceType.websiteCrawl && websitePages.length > 1) - isMultiple = true - - if (datasourceType === DatasourceType.onlineDrive && selectedFileIds.length > 1) - isMultiple = true - - if (isMultiple) { + const multipleCheckMap: Record = { + [DatasourceType.localFile]: localFileList.length, + [DatasourceType.onlineDocument]: onlineDocuments.length, + [DatasourceType.websiteCrawl]: websitePages.length, + [DatasourceType.onlineDrive]: selectedFileIds.length, + } + const count = datasourceType ? multipleCheckMap[datasourceType] : 0 + if (count > 1) { showPlanUpgradeModal() return } @@ -136,334 +141,44 @@ const CreateFormPipeline = () => { doHandleNextStep() }, [datasourceType, doHandleNextStep, localFileList.length, onlineDocuments.length, selectedFileIds.length, showPlanUpgradeModal, supportBatchUpload, websitePages.length]) - const nextBtnDisabled = useMemo(() => { - if (!datasource) - return true - if (datasourceType === DatasourceType.localFile) - return isShowVectorSpaceFull || !localFileList.length || !allFileLoaded - if (datasourceType === DatasourceType.onlineDocument) - return isShowVectorSpaceFull || !onlineDocuments.length - if (datasourceType === DatasourceType.websiteCrawl) - return isShowVectorSpaceFull || !websitePages.length - if (datasourceType === DatasourceType.onlineDrive) - return isShowVectorSpaceFull || !selectedFileIds.length - return false - }, [datasource, datasourceType, isShowVectorSpaceFull, localFileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileIds.length]) + // Datasource actions + const { + isPreview, + formRef, + isIdle, + isPending, + onClickProcess, + onClickPreview, + handleSubmit, + handlePreviewFileChange, + handlePreviewOnlineDocumentChange, + handlePreviewWebsiteChange, + handlePreviewOnlineDriveFileChange, + handleSelectAll, + handleSwitchDataSource, + handleCredentialChange, + } = useDatasourceActions({ + datasource, + datasourceType, + pipelineId, + dataSourceStore, + setEstimateData, + setBatchId, + setDocuments, + handleNextStep, + PagesMapAndSelectedPagesId, + currentWorkspacePages: currentWorkspace?.pages, + clearOnlineDocumentData, + clearWebsiteCrawlData, + clearOnlineDriveData, + setDatasource, + }) - const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? { - file_size_limit: 15, - batch_count_limit: 5, - }, [fileUploadConfigResponse]) - - const showSelect = useMemo(() => { - if (datasourceType === DatasourceType.onlineDocument) { - const pagesCount = currentWorkspace?.pages.length ?? 0 - return pagesCount > 0 - } - if (datasourceType === DatasourceType.onlineDrive) { - const isBucketList = onlineDriveFileList.some(file => file.type === 'bucket') - return !isBucketList && onlineDriveFileList.filter((item) => { - return item.type !== 'bucket' - }).length > 0 - } - return false - }, [currentWorkspace?.pages.length, datasourceType, onlineDriveFileList]) - - const totalOptions = useMemo(() => { - if (datasourceType === DatasourceType.onlineDocument) - return currentWorkspace?.pages.length - if (datasourceType === DatasourceType.onlineDrive) { - return onlineDriveFileList.filter((item) => { - return item.type !== 'bucket' - }).length - } - }, [currentWorkspace?.pages.length, datasourceType, onlineDriveFileList]) - - const selectedOptions = useMemo(() => { - if (datasourceType === DatasourceType.onlineDocument) - return onlineDocuments.length - if (datasourceType === DatasourceType.onlineDrive) - return selectedFileIds.length - }, [datasourceType, onlineDocuments.length, selectedFileIds.length]) - - const tip = useMemo(() => { - if (datasourceType === DatasourceType.onlineDocument) - return t('addDocuments.selectOnlineDocumentTip', { ns: 'datasetPipeline', count: 50 }) - if (datasourceType === DatasourceType.onlineDrive) { - return t('addDocuments.selectOnlineDriveTip', { - ns: 'datasetPipeline', - count: fileUploadConfig.batch_count_limit, - fileSize: fileUploadConfig.file_size_limit, - }) - } - return '' - }, [datasourceType, fileUploadConfig.batch_count_limit, fileUploadConfig.file_size_limit, t]) - - const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline() - - const handlePreviewChunks = useCallback(async (data: Record) => { - if (!datasource) - return - const { - previewLocalFileRef, - previewOnlineDocumentRef, - previewWebsitePageRef, - previewOnlineDriveFileRef, - currentCredentialId, - } = dataSourceStore.getState() - const datasourceInfoList: Record[] = [] - if (datasourceType === DatasourceType.localFile) { - const { id, name, type, size, extension, mime_type } = previewLocalFileRef.current as File - const documentInfo = { - related_id: id, - name, - type, - size, - extension, - mime_type, - url: '', - transfer_method: TransferMethod.local_file, - credential_id: currentCredentialId, - } - datasourceInfoList.push(documentInfo) - } - if (datasourceType === DatasourceType.onlineDocument) { - const { workspace_id, ...rest } = previewOnlineDocumentRef.current! - const documentInfo = { - workspace_id, - page: rest, - credential_id: currentCredentialId, - } - datasourceInfoList.push(documentInfo) - } - if (datasourceType === DatasourceType.websiteCrawl) { - datasourceInfoList.push({ - ...previewWebsitePageRef.current!, - credential_id: currentCredentialId, - }) - } - if (datasourceType === DatasourceType.onlineDrive) { - const { bucket } = dataSourceStore.getState() - const { id, type, name } = previewOnlineDriveFileRef.current! - datasourceInfoList.push({ - bucket, - id, - name, - type, - credential_id: currentCredentialId, - }) - } - await runPublishedPipeline({ - pipeline_id: pipelineId!, - inputs: data, - start_node_id: datasource.nodeId, - datasource_type: datasourceType as DatasourceType, - datasource_info_list: datasourceInfoList, - is_preview: true, - }, { - onSuccess: (res) => { - setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs) - }, - }) - }, [datasource, datasourceType, runPublishedPipeline, pipelineId, dataSourceStore]) - - const handleProcess = useCallback(async (data: Record) => { - if (!datasource) - return - const { currentCredentialId } = dataSourceStore.getState() - const datasourceInfoList: Record[] = [] - if (datasourceType === DatasourceType.localFile) { - const { - localFileList, - } = dataSourceStore.getState() - localFileList.forEach((file) => { - const { id, name, type, size, extension, mime_type } = file.file - const documentInfo = { - related_id: id, - name, - type, - size, - extension, - mime_type, - url: '', - transfer_method: TransferMethod.local_file, - credential_id: currentCredentialId, - } - datasourceInfoList.push(documentInfo) - }) - } - if (datasourceType === DatasourceType.onlineDocument) { - const { - onlineDocuments, - } = dataSourceStore.getState() - onlineDocuments.forEach((page) => { - const { workspace_id, ...rest } = page - const documentInfo = { - workspace_id, - page: rest, - credential_id: currentCredentialId, - } - datasourceInfoList.push(documentInfo) - }) - } - if (datasourceType === DatasourceType.websiteCrawl) { - const { - websitePages, - } = dataSourceStore.getState() - websitePages.forEach((websitePage) => { - datasourceInfoList.push({ - ...websitePage, - credential_id: currentCredentialId, - }) - }) - } - if (datasourceType === DatasourceType.onlineDrive) { - const { - bucket, - selectedFileIds, - onlineDriveFileList, - } = dataSourceStore.getState() - selectedFileIds.forEach((id) => { - const file = onlineDriveFileList.find(file => file.id === id) - datasourceInfoList.push({ - bucket, - id: file?.id, - name: file?.name, - type: file?.type, - credential_id: currentCredentialId, - }) - }) - } - await runPublishedPipeline({ - pipeline_id: pipelineId!, - inputs: data, - start_node_id: datasource.nodeId, - datasource_type: datasourceType as DatasourceType, - datasource_info_list: datasourceInfoList, - is_preview: false, - }, { - onSuccess: (res) => { - setBatchId((res as PublishedPipelineRunResponse).batch || '') - setDocuments((res as PublishedPipelineRunResponse).documents || []) - handleNextStep() - trackEvent('dataset_document_added', { - data_source_type: datasourceType, - indexing_technique: 'pipeline', - }) - }, - }) - }, [dataSourceStore, datasource, datasourceType, handleNextStep, pipelineId, runPublishedPipeline]) - - const onClickProcess = useCallback(() => { - isPreview.current = false - formRef.current?.submit() - }, []) - - const onClickPreview = useCallback(() => { - isPreview.current = true - formRef.current?.submit() - }, []) - - const handleSubmit = useCallback((data: Record) => { - if (isPreview.current) - handlePreviewChunks(data) - else - handleProcess(data) - }, [handlePreviewChunks, handleProcess]) - - const handlePreviewFileChange = useCallback((file: DocumentItem) => { - const { previewLocalFileRef } = dataSourceStore.getState() - previewLocalFileRef.current = file - onClickPreview() - }, [dataSourceStore, onClickPreview]) - - const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => { - const { previewOnlineDocumentRef } = dataSourceStore.getState() - previewOnlineDocumentRef.current = page - onClickPreview() - }, [dataSourceStore, onClickPreview]) - - const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => { - const { previewWebsitePageRef } = dataSourceStore.getState() - previewWebsitePageRef.current = website - onClickPreview() - }, [dataSourceStore, onClickPreview]) - - const handlePreviewOnlineDriveFileChange = useCallback((file: OnlineDriveFile) => { - const { previewOnlineDriveFileRef } = dataSourceStore.getState() - previewOnlineDriveFileRef.current = file - onClickPreview() - }, [dataSourceStore, onClickPreview]) - - const handleSelectAll = useCallback(() => { - const { - onlineDocuments, - onlineDriveFileList, - selectedFileIds, - setOnlineDocuments, - setSelectedFileIds, - setSelectedPagesId, - } = dataSourceStore.getState() - if (datasourceType === DatasourceType.onlineDocument) { - const allIds = currentWorkspace?.pages.map(page => page.page_id) || [] - if (onlineDocuments.length < allIds.length) { - const selectedPages = Array.from(allIds).map(pageId => PagesMapAndSelectedPagesId[pageId]) - setOnlineDocuments(selectedPages) - setSelectedPagesId(new Set(allIds)) - } - else { - setOnlineDocuments([]) - setSelectedPagesId(new Set()) - } - } - if (datasourceType === DatasourceType.onlineDrive) { - const allKeys = onlineDriveFileList.filter((item) => { - return item.type !== 'bucket' - }).map(file => file.id) - if (selectedFileIds.length < allKeys.length) - setSelectedFileIds(allKeys) - else - setSelectedFileIds([]) - } - }, [PagesMapAndSelectedPagesId, currentWorkspace?.pages, dataSourceStore, datasourceType]) - - const clearDataSourceData = useCallback((dataSource: Datasource) => { - const providerType = dataSource.nodeData.provider_type - if (providerType === DatasourceType.onlineDocument) - clearOnlineDocumentData() - else if (providerType === DatasourceType.websiteCrawl) - clearWebsiteCrawlData() - else if (providerType === DatasourceType.onlineDrive) - clearOnlineDriveData() - }, [clearOnlineDocumentData, clearOnlineDriveData, clearWebsiteCrawlData]) - - const handleSwitchDataSource = useCallback((dataSource: Datasource) => { - const { - setCurrentCredentialId, - currentNodeIdRef, - } = dataSourceStore.getState() - clearDataSourceData(dataSource) - setCurrentCredentialId('') - currentNodeIdRef.current = dataSource.nodeId - setDatasource(dataSource) - }, [clearDataSourceData, dataSourceStore]) - - const handleCredentialChange = useCallback((credentialId: string) => { - const { setCurrentCredentialId } = dataSourceStore.getState() - clearDataSourceData(datasource!) - setCurrentCredentialId(credentialId) - }, [clearDataSourceData, dataSourceStore, datasource]) - - if (isFetchingPipelineInfo) { - return ( - - ) - } + if (isFetchingPipelineInfo) + return return ( -
+
{ currentStep={currentStep} />
- { - currentStep === 1 && ( -
- []} - /> - {datasourceType === DatasourceType.localFile && ( - - )} - {datasourceType === DatasourceType.onlineDocument && ( - - )} - {datasourceType === DatasourceType.websiteCrawl && ( - - )} - {datasourceType === DatasourceType.onlineDrive && ( - - )} - {isShowVectorSpaceFull && ( - - )} - - { - !supportBatchUpload && datasourceType === DatasourceType.localFile && localFileList.length > 0 && ( - <> - - - - ) - } -
- ) - } - { - currentStep === 2 && ( - - ) - } - { - currentStep === 3 && ( - - ) - } + {currentStep === 1 && ( + []} + supportBatchUpload={supportBatchUpload} + localFileListLength={localFileList.length} + isShowVectorSpaceFull={isShowVectorSpaceFull} + showSelect={showSelect} + totalOptions={totalOptions} + selectedOptions={selectedOptions} + tip={tip} + nextBtnDisabled={nextBtnDisabled} + onSelectDataSource={handleSwitchDataSource} + onCredentialChange={handleCredentialChange} + onSelectAll={handleSelectAll} + onNextStep={handleNextStep} + /> + )} + {currentStep === 2 && ( + + )} + {currentStep === 3 && ( + + )}
- {/* Preview */} - { - currentStep === 1 && ( -
-
- {currentLocalFile && ( - - )} - {currentDocument && ( - - )} - {currentWebsite && ( - - )} -
-
- ) - } - { - currentStep === 2 && ( -
-
- file.file)} - onlineDocuments={onlineDocuments} - websitePages={websitePages} - onlineDriveFiles={selectedOnlineDriveFileList} - isIdle={isIdle} - isPending={isPending && isPreview.current} - estimateData={estimateData} - onPreview={onClickPreview} - handlePreviewFileChange={handlePreviewFileChange} - handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange} - handlePreviewWebsitePageChange={handlePreviewWebsiteChange} - handlePreviewOnlineDriveFileChange={handlePreviewOnlineDriveFileChange} - /> -
-
- ) - } + + {/* Preview Panel */} + {currentStep === 1 && ( + + )} + {currentStep === 2 && ( + + )} + + {/* Plan Upgrade Modal */} {isShowPlanUpgradeModal && ( void + hidePreviewOnlineDocument: () => void + hideWebsitePreview: () => void +} + +export const StepOnePreview = memo(({ + datasource, + currentLocalFile, + currentDocument, + currentWebsite, + hidePreviewLocalFile, + hidePreviewOnlineDocument, + hideWebsitePreview, +}: StepOnePreviewProps) => { + return ( +
+
+ {currentLocalFile && ( + + )} + {currentDocument && ( + + )} + {currentWebsite && ( + + )} +
+
+ ) +}) +StepOnePreview.displayName = 'StepOnePreview' + +type StepTwoPreviewProps = { + datasourceType: string | undefined + localFileList: FileItem[] + onlineDocuments: (NotionPage & { workspace_id: string })[] + websitePages: CrawlResultItem[] + selectedOnlineDriveFileList: OnlineDriveFile[] + isIdle: boolean + isPendingPreview: boolean + estimateData: FileIndexingEstimateResponse | undefined + onPreview: () => void + handlePreviewFileChange: (file: DocumentItem) => void + handlePreviewOnlineDocumentChange: (page: NotionPage) => void + handlePreviewWebsitePageChange: (website: CrawlResultItem) => void + handlePreviewOnlineDriveFileChange: (file: OnlineDriveFile) => void +} + +export const StepTwoPreview = memo(({ + datasourceType, + localFileList, + onlineDocuments, + websitePages, + selectedOnlineDriveFileList, + isIdle, + isPendingPreview, + estimateData, + onPreview, + handlePreviewFileChange, + handlePreviewOnlineDocumentChange, + handlePreviewWebsitePageChange, + handlePreviewOnlineDriveFileChange, +}: StepTwoPreviewProps) => { + return ( +
+
+ file.file)} + onlineDocuments={onlineDocuments} + websitePages={websitePages} + onlineDriveFiles={selectedOnlineDriveFileList} + isIdle={isIdle} + isPending={isPendingPreview} + estimateData={estimateData} + onPreview={onPreview} + handlePreviewFileChange={handlePreviewFileChange} + handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange} + handlePreviewWebsitePageChange={handlePreviewWebsitePageChange} + handlePreviewOnlineDriveFileChange={handlePreviewOnlineDriveFileChange} + /> +
+
+ ) +}) +StepTwoPreview.displayName = 'StepTwoPreview' diff --git a/web/app/components/datasets/documents/create-from-pipeline/steps/step-one-content.tsx b/web/app/components/datasets/documents/create-from-pipeline/steps/step-one-content.tsx new file mode 100644 index 0000000000..8eed6d00b9 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/steps/step-one-content.tsx @@ -0,0 +1,110 @@ +'use client' +import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' +import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' +import type { Node } from '@/app/components/workflow/types' +import { memo } from 'react' +import Divider from '@/app/components/base/divider' +import VectorSpaceFull from '@/app/components/billing/vector-space-full' +import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file' +import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents' +import OnlineDrive from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-drive' +import WebsiteCrawl from '@/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl' +import { DatasourceType } from '@/models/pipeline' +import UpgradeCard from '../../../create/step-one/upgrade-card' +import Actions from '../actions' +import DataSourceOptions from '../data-source-options' + +type StepOneContentProps = { + datasource: Datasource | undefined + datasourceType: string | undefined + pipelineNodes: Node[] + supportBatchUpload: boolean + localFileListLength: number + isShowVectorSpaceFull: boolean + showSelect: boolean + totalOptions: number | undefined + selectedOptions: number | undefined + tip: string + nextBtnDisabled: boolean + onSelectDataSource: (dataSource: Datasource) => void + onCredentialChange: (credentialId: string) => void + onSelectAll: () => void + onNextStep: () => void +} + +const StepOneContent = ({ + datasource, + datasourceType, + pipelineNodes, + supportBatchUpload, + localFileListLength, + isShowVectorSpaceFull, + showSelect, + totalOptions, + selectedOptions, + tip, + nextBtnDisabled, + onSelectDataSource, + onCredentialChange, + onSelectAll, + onNextStep, +}: StepOneContentProps) => { + const showUpgradeCard = !supportBatchUpload + && datasourceType === DatasourceType.localFile + && localFileListLength > 0 + + return ( +
+ + {datasourceType === DatasourceType.localFile && ( + + )} + {datasourceType === DatasourceType.onlineDocument && ( + + )} + {datasourceType === DatasourceType.websiteCrawl && ( + + )} + {datasourceType === DatasourceType.onlineDrive && ( + + )} + {isShowVectorSpaceFull && } + + {showUpgradeCard && ( + <> + + + + )} +
+ ) +} + +export default memo(StepOneContent) diff --git a/web/app/components/datasets/documents/create-from-pipeline/steps/step-three-content.tsx b/web/app/components/datasets/documents/create-from-pipeline/steps/step-three-content.tsx new file mode 100644 index 0000000000..f4b15888a9 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/steps/step-three-content.tsx @@ -0,0 +1,23 @@ +'use client' +import type { InitialDocumentDetail } from '@/models/pipeline' +import { memo } from 'react' +import Processing from '../processing' + +type StepThreeContentProps = { + batchId: string + documents: InitialDocumentDetail[] +} + +const StepThreeContent = ({ + batchId, + documents, +}: StepThreeContentProps) => { + return ( + + ) +} + +export default memo(StepThreeContent) diff --git a/web/app/components/datasets/documents/create-from-pipeline/steps/step-two-content.tsx b/web/app/components/datasets/documents/create-from-pipeline/steps/step-two-content.tsx new file mode 100644 index 0000000000..ca95c9f354 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/steps/step-two-content.tsx @@ -0,0 +1,38 @@ +'use client' +import type { RefObject } from 'react' +import { memo } from 'react' +import ProcessDocuments from '../process-documents' + +type StepTwoContentProps = { + formRef: RefObject<{ submit: () => void } | null> + dataSourceNodeId: string + isRunning: boolean + onProcess: () => void + onPreview: () => void + onSubmit: (data: Record) => void + onBack: () => void +} + +const StepTwoContent = ({ + formRef, + dataSourceNodeId, + isRunning, + onProcess, + onPreview, + onSubmit, + onBack, +}: StepTwoContentProps) => { + return ( + + ) +} + +export default memo(StepTwoContent) diff --git a/web/app/components/datasets/documents/create-from-pipeline/utils/datasource-info-builder.ts b/web/app/components/datasets/documents/create-from-pipeline/utils/datasource-info-builder.ts new file mode 100644 index 0000000000..c9f4808bbc --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/utils/datasource-info-builder.ts @@ -0,0 +1,63 @@ +import type { NotionPage } from '@/models/common' +import type { CrawlResultItem, CustomFile as File } from '@/models/datasets' +import type { OnlineDriveFile } from '@/models/pipeline' +import { TransferMethod } from '@/types/app' + +/** + * Build datasource info for local files + */ +export const buildLocalFileDatasourceInfo = ( + file: File, + credentialId: string, +): Record => ({ + related_id: file.id, + name: file.name, + type: file.type, + size: file.size, + extension: file.extension, + mime_type: file.mime_type, + url: '', + transfer_method: TransferMethod.local_file, + credential_id: credentialId, +}) + +/** + * Build datasource info for online documents + */ +export const buildOnlineDocumentDatasourceInfo = ( + page: NotionPage & { workspace_id: string }, + credentialId: string, +): Record => { + const { workspace_id, ...rest } = page + return { + workspace_id, + page: rest, + credential_id: credentialId, + } +} + +/** + * Build datasource info for website crawl + */ +export const buildWebsiteCrawlDatasourceInfo = ( + page: CrawlResultItem, + credentialId: string, +): Record => ({ + ...page, + credential_id: credentialId, +}) + +/** + * Build datasource info for online drive + */ +export const buildOnlineDriveDatasourceInfo = ( + file: OnlineDriveFile, + bucket: string, + credentialId: string, +): Record => ({ + bucket, + id: file.id, + name: file.name, + type: file.type, + credential_id: credentialId, +})