'use client' import type { FC } from 'react' import type { CrawlOptions, CrawlResultItem } from '@/models/datasets' import { toast } from '@langgenius/dify-ui/toast' import * as React from 'react' import { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants' import { useIntegrationsSetting } from '@/app/components/header/account-setting/use-integrations-setting' import { checkJinaReaderTaskStatus, createJinaReaderTask } from '@/service/datasets' import { sleep } from '@/utils' import CrawledResult from '../base/crawled-result' import Crawling from '../base/crawling' import ErrorMessage from '../base/error-message' import Header from '../base/header' import OptionsWrap from '../base/options-wrap' import UrlInput from '../base/url-input' import Options from './options' const ERROR_I18N_PREFIX = 'errorMsg' const I18N_PREFIX = 'stepOne.website' type Props = Readonly<{ onPreview: (payload: CrawlResultItem) => void checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onJobIdChange: (jobId: string) => void crawlOptions: CrawlOptions onCrawlOptionsChange: (payload: CrawlOptions) => void }> const Step = { init: 'init', running: 'running', finished: 'finished', } as const type Step = (typeof Step)[keyof typeof Step] const JinaReader: FC = ({ onPreview, checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, crawlOptions, onCrawlOptionsChange, }) => { const { t } = useTranslation() const [step, setStep] = useState(Step.init) const [controlFoldOptions, setControlFoldOptions] = useState(0) const openIntegrationsSetting = useIntegrationsSetting() const handleSetting = useCallback(() => { openIntegrationsSetting({ payload: ACCOUNT_SETTING_TAB.DATA_SOURCE, }) }, [openIntegrationsSetting]) const checkValid = useCallback( (url: string) => { let errorMsg = '' if (!url) { errorMsg = t(($) => $[`${ERROR_I18N_PREFIX}.fieldRequired`], { ns: 'common', field: 'url', }) } if (!errorMsg && !(url.startsWith('http://') || url.startsWith('https://'))) errorMsg = t(($) => $[`${ERROR_I18N_PREFIX}.urlError`], { ns: 'common' }) if ( !errorMsg && (crawlOptions.limit === null || crawlOptions.limit === undefined || crawlOptions.limit === '') ) { errorMsg = t(($) => $[`${ERROR_I18N_PREFIX}.fieldRequired`], { ns: 'common', field: t(($) => $[`${I18N_PREFIX}.limit`], { ns: 'datasetCreation' }), }) } return { isValid: !errorMsg, errorMsg, } }, [crawlOptions, t], ) const isInit = step === Step.init const isCrawlFinished = step === Step.finished const isRunning = step === Step.running const [crawlResult, setCrawlResult] = useState< | { current: number total: number data: CrawlResultItem[] time_consuming: number | string } | undefined >(undefined) const [crawlErrorMessage, setCrawlErrorMessage] = useState('') const showError = isCrawlFinished && crawlErrorMessage const waitForCrawlFinished = useCallback( async (jobId: string) => { try { const res = (await checkJinaReaderTaskStatus(jobId)) as any if (res.status === 'completed') { return { isError: false, data: { ...res, total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)), }, } } if (res.status === 'failed' || !res.status) { return { isError: true, errorMessage: res.message, data: { data: [], }, } } // update the progress setCrawlResult({ ...res, total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)), }) onCheckedCrawlResultChange(res.data || []) // default select the crawl result await sleep(2500) return await waitForCrawlFinished(jobId) } catch (e: any) { const errorBody = await e.json() return { isError: true, errorMessage: errorBody.message, data: { data: [], }, } } }, [crawlOptions.limit, onCheckedCrawlResultChange], ) const handleRun = useCallback( async (url: string) => { const { isValid, errorMsg } = checkValid(url) if (!isValid) { toast.error(errorMsg!) return } setStep(Step.running) // fold the options panel when the step leaves `init`, // previously synced via a setState in an effect on `step` setControlFoldOptions(Date.now()) try { const startTime = Date.now() const res = (await createJinaReaderTask({ url, options: crawlOptions, })) as any if (res.data) { const { title, content, description, url } = res.data const data = { current: 1, total: 1, data: [ { title, markdown: content, description, source_url: url, }, ], time_consuming: (Date.now() - startTime) / 1000, } setCrawlResult(data) onCheckedCrawlResultChange(data.data || []) setCrawlErrorMessage('') } else if (res.job_id) { const jobId = res.job_id onJobIdChange(jobId) const { isError, data, errorMessage } = await waitForCrawlFinished(jobId) if (isError) { setCrawlErrorMessage( errorMessage || t(($) => $[`${I18N_PREFIX}.unknownError`], { ns: 'datasetCreation' }), ) } else { setCrawlResult(data) onCheckedCrawlResultChange(data.data || []) // default select the crawl result setCrawlErrorMessage('') } } } catch (e) { setCrawlErrorMessage(t(($) => $[`${I18N_PREFIX}.unknownError`], { ns: 'datasetCreation' })!) console.log(e) } finally { setStep(Step.finished) setControlFoldOptions(Date.now()) } }, [checkValid, crawlOptions, onCheckedCrawlResultChange, onJobIdChange, t, waitForCrawlFinished], ) return (
$[`${I18N_PREFIX}.jinaReaderTitle`], { ns: 'datasetCreation' })} buttonText={t(($) => $[`${I18N_PREFIX}.configureJinaReader`], { ns: 'datasetCreation' })} docTitle={t(($) => $[`${I18N_PREFIX}.jinaReaderDoc`], { ns: 'datasetCreation' })} docLink="https://jina.ai/reader" />
{!isInit && (
{isRunning && ( )} {showError && ( $[`${I18N_PREFIX}.exceptionErrorTitle`], { ns: 'datasetCreation' })} errorMsg={crawlErrorMessage} /> )} {isCrawlFinished && !showError && ( )}
)}
) } export default React.memo(JinaReader)