'use client' import type { FC } from 'react' import type { CrawlOptions, CrawlResultItem } from '@/models/datasets' import { toast } from '@langgenius/dify-ui/toast' import * as React from 'react' import { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants' import { useIntegrationsSetting } from '@/app/components/header/account-setting/use-integrations-setting' import { checkWatercrawlTaskStatus, createWatercrawlTask } from '@/service/datasets' import { sleep } from '@/utils' import CrawledResult from '../base/crawled-result' import Crawling from '../base/crawling' import ErrorMessage from '../base/error-message' import Header from '../base/header' import OptionsWrap from '../base/options-wrap' import UrlInput from '../base/url-input' import Options from './options' const ERROR_I18N_PREFIX = 'errorMsg' const I18N_PREFIX = 'stepOne.website' type Props = Readonly<{ onPreview: (payload: CrawlResultItem) => void checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onJobIdChange: (jobId: string) => void crawlOptions: CrawlOptions onCrawlOptionsChange: (payload: CrawlOptions) => void }> const Step = { init: 'init', running: 'running', finished: 'finished', } as const type Step = (typeof Step)[keyof typeof Step] const STEP_CONTROL_FOLD_OPTIONS: Record = { [Step.init]: 0, [Step.running]: 1, [Step.finished]: 2, } const WaterCrawl: FC = ({ onPreview, checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, crawlOptions, onCrawlOptionsChange, }) => { const { t } = useTranslation() const [step, setStep] = useState(Step.init) const controlFoldOptions = STEP_CONTROL_FOLD_OPTIONS[step] const openIntegrationsSetting = useIntegrationsSetting() const handleSetting = useCallback(() => { openIntegrationsSetting({ payload: ACCOUNT_SETTING_TAB.DATA_SOURCE, }) }, [openIntegrationsSetting]) const checkValid = useCallback( (url: string) => { let errorMsg = '' if (!url) { errorMsg = t(($) => $[`${ERROR_I18N_PREFIX}.fieldRequired`], { ns: 'common', field: 'url', }) } if (!errorMsg && !(url.startsWith('http://') || url.startsWith('https://'))) errorMsg = t(($) => $[`${ERROR_I18N_PREFIX}.urlError`], { ns: 'common' }) if ( !errorMsg && (crawlOptions.limit === null || crawlOptions.limit === undefined || crawlOptions.limit === '') ) { errorMsg = t(($) => $[`${ERROR_I18N_PREFIX}.fieldRequired`], { ns: 'common', field: t(($) => $[`${I18N_PREFIX}.limit`], { ns: 'datasetCreation' }), }) } return { isValid: !errorMsg, errorMsg, } }, [crawlOptions, t], ) const isInit = step === Step.init const isCrawlFinished = step === Step.finished const isRunning = step === Step.running const [crawlResult, setCrawlResult] = useState< | { current: number total: number data: CrawlResultItem[] time_consuming: number | string } | undefined >(undefined) const [crawlErrorMessage, setCrawlErrorMessage] = useState('') const showError = isCrawlFinished && crawlErrorMessage const waitForCrawlFinished = useCallback( async (jobId: string): Promise => { try { const res = (await checkWatercrawlTaskStatus(jobId)) as any if (res.status === 'completed') { return { isError: false, data: { ...res, total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)), }, } } if (res.status === 'error' || !res.status) { // can't get the error message from the watercrawl api return { isError: true, errorMessage: res.message, data: { data: [], }, } } // update the progress setCrawlResult({ ...res, total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)), }) onCheckedCrawlResultChange(res.data || []) // default select the crawl result await sleep(2500) return await waitForCrawlFinished(jobId) } catch (error: unknown) { let errorMessage = '' const maybeErrorWithJson = error as { json?: () => Promise message?: unknown } | null if (maybeErrorWithJson?.json) { try { const errorBody = (await maybeErrorWithJson.json()) as { message?: unknown } | null if (typeof errorBody?.message === 'string') errorMessage = errorBody.message } catch {} } if (!errorMessage && typeof maybeErrorWithJson?.message === 'string') errorMessage = maybeErrorWithJson.message return { isError: true, errorMessage, data: { data: [], }, } } }, [crawlOptions.limit, onCheckedCrawlResultChange], ) const handleRun = useCallback( async (url: string) => { const { isValid, errorMsg } = checkValid(url) if (!isValid) { toast.error(errorMsg!) return } setStep(Step.running) try { const passToServerCrawlOptions: any = { ...crawlOptions, } if (crawlOptions.max_depth === '') delete passToServerCrawlOptions.max_depth const res = (await createWatercrawlTask({ url, options: passToServerCrawlOptions, })) as any const jobId = res.job_id onJobIdChange(jobId) const { isError, data, errorMessage } = await waitForCrawlFinished(jobId) if (isError) { setCrawlErrorMessage( errorMessage || t(($) => $[`${I18N_PREFIX}.unknownError`], { ns: 'datasetCreation' }), ) } else { setCrawlResult(data) onCheckedCrawlResultChange(data.data || []) // default select the crawl result setCrawlErrorMessage('') } } catch (e) { setCrawlErrorMessage(t(($) => $[`${I18N_PREFIX}.unknownError`], { ns: 'datasetCreation' })!) console.log(e) } finally { setStep(Step.finished) } }, [checkValid, crawlOptions, onCheckedCrawlResultChange, onJobIdChange, t, waitForCrawlFinished], ) return (
$[`${I18N_PREFIX}.watercrawlTitle`], { ns: 'datasetCreation' })} buttonText={t(($) => $[`${I18N_PREFIX}.configureWatercrawl`], { ns: 'datasetCreation' })} docTitle={t(($) => $[`${I18N_PREFIX}.watercrawlDoc`], { ns: 'datasetCreation' })} docLink="https://docs.watercrawl.dev/" />
{!isInit && (
{isRunning && ( )} {showError && ( $[`${I18N_PREFIX}.exceptionErrorTitle`], { ns: 'datasetCreation' })} errorMsg={crawlErrorMessage} /> )} {isCrawlFinished && !showError && ( )}
)}
) } export default React.memo(WaterCrawl)