mirror of
https://github.com/langgenius/dify.git
synced 2026-09-05 08:48:10 +08:00
fix(web): discover source providers dynamically
This commit is contained in:
Stephen Zhou
parent
b2956fd544
commit
62cebcf64d
@ -495,6 +495,35 @@ const jinaDatasourceAuth: DatasourceProviderAuthListResponse['result'][number] =
|
||||
provider: 'jinareader',
|
||||
}
|
||||
|
||||
const customCrawlerDatasourcePlugin: DataSourceItem = {
|
||||
...firecrawlDatasourcePlugin,
|
||||
declaration: {
|
||||
...firecrawlDatasourcePlugin.declaration,
|
||||
datasources: [
|
||||
{
|
||||
description: { en_US: 'Acme crawler', zh_Hans: 'Acme crawler' },
|
||||
identity: {
|
||||
author: 'acme',
|
||||
label: { en_US: 'Acme Crawler', zh_Hans: 'Acme Crawler' },
|
||||
name: 'acme_crawler',
|
||||
provider: 'acme',
|
||||
},
|
||||
parameters: [],
|
||||
},
|
||||
],
|
||||
identity: {
|
||||
...firecrawlDatasourcePlugin.declaration.identity,
|
||||
author: 'acme',
|
||||
description: { en_US: 'Acme crawler', zh_Hans: 'Acme crawler' },
|
||||
label: { en_US: 'Acme Crawler', zh_Hans: 'Acme Crawler' },
|
||||
name: 'acme_crawler',
|
||||
},
|
||||
},
|
||||
plugin_id: 'acme/acme_crawler',
|
||||
plugin_unique_identifier: 'acme/acme_crawler:1.0.0@local',
|
||||
provider: 'acme',
|
||||
}
|
||||
|
||||
const connection = (
|
||||
status: 'provisioning' | 'active' | 'expired' | 'error' | 'revoked',
|
||||
version = 2,
|
||||
@ -698,10 +727,10 @@ describe('AddSourcePage', () => {
|
||||
expect(screen.getByRole('radio', { name: 'Jina Reader' })).toBeChecked()
|
||||
expect(providerHookOptionsMock.mock.lastCall?.[0]).toMatchObject({ enabled: true })
|
||||
expect(connectionHookOptionsMock.mock.lastCall?.[0]).toMatchObject({ enabled: true })
|
||||
expect(screen.getByText('dataset.newKnowledge.providerUnavailable')).toBeInTheDocument()
|
||||
expect(screen.getByText('workflow.nodes.common.pluginNotInstalled')).toBeInTheDocument()
|
||||
await user.click(
|
||||
screen.getByRole('button', {
|
||||
name: 'dataset.newKnowledge.configureProvider:{"provider":"Jina Reader"}',
|
||||
name: 'plugin.installPlugin',
|
||||
}),
|
||||
)
|
||||
expect(openMock).toHaveBeenCalledWith(
|
||||
@ -742,7 +771,7 @@ describe('AddSourcePage', () => {
|
||||
).not.toBeInTheDocument()
|
||||
|
||||
await user.click(screen.getByRole('radio', { name: 'Jina Reader' }))
|
||||
expect(screen.getByText('dataset.newKnowledge.providerUnavailable')).toBeInTheDocument()
|
||||
expect(screen.getByText('workflow.nodes.common.pluginNotInstalled')).toBeInTheDocument()
|
||||
expect(
|
||||
screen.queryByRole('textbox', { name: 'dataset.newKnowledge.rootUrl' }),
|
||||
).not.toBeInTheDocument()
|
||||
@ -1133,14 +1162,16 @@ describe('AddSourcePage', () => {
|
||||
expect(routerMock.push).toHaveBeenCalledWith('/datasets/new/space-1/sources')
|
||||
})
|
||||
|
||||
it('renders every designed website provider and the provider-management action', async () => {
|
||||
it('discovers installed website providers and keeps the provider-management action', async () => {
|
||||
const user = userEvent.setup()
|
||||
queryState.datasourcePlugins.data = [firecrawlDatasourcePlugin, customCrawlerDatasourcePlugin]
|
||||
render(<AddSourcePage knowledgeSpaceId="space-1" />)
|
||||
|
||||
expect(
|
||||
screen.getByRole('group', { name: 'dataset.newKnowledge.providerLabel' }),
|
||||
).toBeInTheDocument()
|
||||
expect(screen.getByRole('radio', { name: 'FakeCrawler' })).toBeEnabled()
|
||||
expect(screen.getByRole('radio', { name: 'Acme Crawler' })).toBeEnabled()
|
||||
expect(screen.queryByRole('radio', { name: 'FakeCrawler' })).not.toBeInTheDocument()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.moreProviders' }))
|
||||
expect(openMock).toHaveBeenCalledWith(
|
||||
'/integrations/data-source',
|
||||
|
||||
@ -552,6 +552,25 @@ describe('ConnectedSourceSetup', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('distinguishes an uninstalled provider from an installed provider without credentials', async () => {
|
||||
const user = userEvent.setup()
|
||||
renderSetup({
|
||||
...defaultDraft,
|
||||
provider: 'Confluence',
|
||||
})
|
||||
|
||||
expect(await screen.findByText('workflow.nodes.common.pluginNotInstalled')).toBeInTheDocument()
|
||||
expect(screen.queryByText('dataset.newKnowledge.notionNotConnected')).not.toBeInTheDocument()
|
||||
expect(clientMock.createConnection).not.toHaveBeenCalled()
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'plugin.installPlugin' }))
|
||||
expect(openMock).toHaveBeenCalledWith(
|
||||
'/integrations/data-source?package-ids=%5B%22langgenius%2Fconfluence_datasource%22%5D',
|
||||
'_blank',
|
||||
'noopener,noreferrer',
|
||||
)
|
||||
})
|
||||
|
||||
it('starts the selected Notion import and completes setup without waiting for indexing', async () => {
|
||||
const user = userEvent.setup()
|
||||
clientMock.listDatasourceAuth.mockResolvedValue({
|
||||
|
||||
@ -35,8 +35,8 @@ describe('New RAG routes', () => {
|
||||
).toBe('/datasets/new/space-1/sources/new?type=websiteCrawl&provider=Jina+Reader')
|
||||
})
|
||||
|
||||
it('falls back when a provider does not belong to the selected source type', () => {
|
||||
expect(createNewKnowledgeSourceDraft('onlineDrive', 'Confluence').provider).toBe('Google Drive')
|
||||
it('keeps a dynamically discovered provider supplied by the add-source entry point', () => {
|
||||
expect(createNewKnowledgeSourceDraft('onlineDrive', 'Acme Drive').provider).toBe('Acme Drive')
|
||||
})
|
||||
|
||||
it('defaults website crawls to a daily sync policy', () => {
|
||||
|
||||
@ -1,13 +1,9 @@
|
||||
'use client'
|
||||
|
||||
import type { DatasourceProviderAuthListResponse } from '@dify/contracts/api/console/auth/types.gen'
|
||||
import type {
|
||||
NewKnowledgeSourceDraft,
|
||||
NewKnowledgeSourceType,
|
||||
NewKnowledgeWebsiteProvider,
|
||||
} from './routes'
|
||||
import type { NewKnowledgeSourceDraft, NewKnowledgeSourceType } from './routes'
|
||||
import type { SourceConnection as Connection, SourceProvider as Provider } from './source-models'
|
||||
import type { DataSourceItem } from '@/app/components/workflow/block-selector/types'
|
||||
import type { InstalledSourceProviderOption, SourceProviderOption } from './source-provider-options'
|
||||
import { Button } from '@langgenius/dify-ui/button'
|
||||
import { Field, FieldControl, FieldDescription, FieldLabel } from '@langgenius/dify-ui/field'
|
||||
import { Fieldset, FieldsetLegend } from '@langgenius/dify-ui/fieldset'
|
||||
@ -43,7 +39,15 @@ import {
|
||||
sourceConnectionListFromApi,
|
||||
sourceProviderListFromApi,
|
||||
} from './source-models'
|
||||
import { SourceProviderRadioGroup, SourceTypeSelector } from './source-setup-fields'
|
||||
import {
|
||||
discoverSourceProviderOptions,
|
||||
sourceProviderOptionForDraft,
|
||||
} from './source-provider-options'
|
||||
import {
|
||||
SourceProviderNotInstalledCard,
|
||||
SourceProviderRadioGroup,
|
||||
SourceTypeSelector,
|
||||
} from './source-setup-fields'
|
||||
import { WebsiteCrawlPreview } from './website-crawl-preview'
|
||||
|
||||
type ProviderField = Provider['configuration'][number]
|
||||
@ -52,37 +56,6 @@ type SourceType = NewKnowledgeSourceType
|
||||
|
||||
const CONNECTION_PAGE_SIZE = 200
|
||||
const WEBSITE_SOURCE_PROVIDER_ID = 'plugin-daemon-website'
|
||||
const WEBSITE_PROVIDER_OPTIONS: Array<{
|
||||
aliases: string[]
|
||||
icon: string
|
||||
value: NewKnowledgeWebsiteProvider
|
||||
}> = [
|
||||
{
|
||||
aliases: ['firecrawl'],
|
||||
icon: 'i-custom-public-common-firecrawl',
|
||||
value: 'Firecrawl',
|
||||
},
|
||||
{
|
||||
aliases: ['jina', 'jina reader', 'jinareader'],
|
||||
icon: 'i-custom-public-llm-jina',
|
||||
value: 'Jina Reader',
|
||||
},
|
||||
{
|
||||
aliases: ['watercrawl'],
|
||||
icon: 'i-custom-public-knowledge-watercrawl',
|
||||
value: 'WaterCrawl',
|
||||
},
|
||||
{
|
||||
aliases: ['fakecrawler'],
|
||||
icon: 'i-ri-global-line text-text-accent',
|
||||
value: 'FakeCrawler',
|
||||
},
|
||||
]
|
||||
const WEBSITE_PROVIDER_PACKAGE_IDS: Partial<Record<NewKnowledgeWebsiteProvider, string>> = {
|
||||
Firecrawl: 'langgenius/firecrawl_datasource',
|
||||
'Jina Reader': 'langgenius/jina_datasource',
|
||||
WaterCrawl: 'watercrawl/watercrawl_datasource',
|
||||
}
|
||||
const MANAGED_PROVIDER_FIELD_NAMES = new Set([
|
||||
'credentialId',
|
||||
'datasource',
|
||||
@ -110,20 +83,10 @@ function fieldValue(value: string, type: ProviderField['type']) {
|
||||
return value.trim()
|
||||
}
|
||||
|
||||
function normalizeProviderName(value: string) {
|
||||
return value.toLocaleLowerCase().replace(/[^a-z0-9]+/g, '')
|
||||
}
|
||||
|
||||
function websiteProviderOption(provider: NewKnowledgeWebsiteProvider) {
|
||||
return WEBSITE_PROVIDER_OPTIONS.find((option) => option.value === provider)
|
||||
}
|
||||
|
||||
function websiteProviderIntegrationPath(provider?: NewKnowledgeWebsiteProvider) {
|
||||
function websiteProviderIntegrationPath(provider?: SourceProviderOption) {
|
||||
const base = buildIntegrationPath('data-source')
|
||||
if (!provider) return base
|
||||
const packageId = WEBSITE_PROVIDER_PACKAGE_IDS[provider]
|
||||
if (!packageId) return base
|
||||
const query = new URLSearchParams({ 'package-ids': JSON.stringify([packageId]) })
|
||||
const query = new URLSearchParams({ 'package-ids': JSON.stringify([provider.packageId]) })
|
||||
return `${base}?${query.toString()}`
|
||||
}
|
||||
|
||||
@ -131,38 +94,7 @@ function findWebsiteSourceProvider(providers: Provider[]) {
|
||||
return providers.find((provider) => provider.id === WEBSITE_SOURCE_PROVIDER_ID)
|
||||
}
|
||||
|
||||
function findWebsiteDatasourceProvider(
|
||||
datasourcePlugins: DataSourceItem[],
|
||||
providerName: NewKnowledgeWebsiteProvider,
|
||||
) {
|
||||
if (providerName === 'FakeCrawler') return undefined
|
||||
const option = websiteProviderOption(providerName)
|
||||
if (!option) return undefined
|
||||
const aliases = [option.value, ...option.aliases].map(normalizeProviderName)
|
||||
for (const plugin of datasourcePlugins) {
|
||||
if (plugin.declaration.provider_type !== 'website_crawl') continue
|
||||
const pluginIdentities = [
|
||||
plugin.declaration.identity.label.en_US,
|
||||
plugin.declaration.identity.name,
|
||||
plugin.plugin_id,
|
||||
plugin.provider,
|
||||
].map(normalizeProviderName)
|
||||
const datasource = plugin.declaration.datasources.find((candidate) => {
|
||||
const datasourceIdentities = [
|
||||
candidate.identity.label.en_US,
|
||||
candidate.identity.name,
|
||||
candidate.identity.provider,
|
||||
].map(normalizeProviderName)
|
||||
return [...pluginIdentities, ...datasourceIdentities].some((identity) =>
|
||||
aliases.some((alias) => identity.includes(alias) || alias.includes(identity)),
|
||||
)
|
||||
})
|
||||
if (datasource) return { datasource, plugin }
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
type WebsiteDatasourceProvider = NonNullable<ReturnType<typeof findWebsiteDatasourceProvider>>
|
||||
type WebsiteDatasourceProvider = InstalledSourceProviderOption
|
||||
|
||||
function findDatasourceAuth(
|
||||
providers: DatasourceProviderAuthListResponse['result'],
|
||||
@ -278,13 +210,15 @@ function getSupportedAuthKinds(provider: Provider, credentialId?: string) {
|
||||
function ProviderSelector({
|
||||
disabled = false,
|
||||
onMoreProviders,
|
||||
provider,
|
||||
options,
|
||||
providerKey,
|
||||
onChange,
|
||||
}: {
|
||||
disabled?: boolean
|
||||
onMoreProviders: () => void
|
||||
provider: NewKnowledgeWebsiteProvider
|
||||
onChange: (provider: NewKnowledgeWebsiteProvider) => void
|
||||
options: SourceProviderOption[]
|
||||
providerKey: string
|
||||
onChange: (providerKey: string) => void
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
|
||||
@ -307,19 +241,30 @@ function ProviderSelector({
|
||||
</Button>
|
||||
</div>
|
||||
<SourceProviderRadioGroup
|
||||
value={provider}
|
||||
value={providerKey}
|
||||
disabled={disabled}
|
||||
layout="grid-four"
|
||||
options={WEBSITE_PROVIDER_OPTIONS.map((option) => ({
|
||||
icon: <span aria-hidden className={`${option.icon} size-4`} />,
|
||||
value: option.value,
|
||||
layout="wrap"
|
||||
options={options.map((option) => ({
|
||||
icon: <WebsiteProviderIcon option={option} />,
|
||||
label: option.label,
|
||||
value: option.key,
|
||||
}))}
|
||||
size="small"
|
||||
onChange={onChange}
|
||||
/>
|
||||
</Fieldset>
|
||||
)
|
||||
}
|
||||
|
||||
function WebsiteProviderIcon({ option }: { option: SourceProviderOption }) {
|
||||
const icon = option.installed
|
||||
? (option.datasource.identity.icon ?? option.plugin.declaration.identity.icon)
|
||||
: undefined
|
||||
if (typeof icon === 'string' && icon)
|
||||
return <img aria-hidden alt="" className="size-4 shrink-0 object-contain" src={icon} />
|
||||
return <span aria-hidden className={`${option.fallbackIcon} size-4 shrink-0`} />
|
||||
}
|
||||
|
||||
function ProviderFieldControl({
|
||||
field,
|
||||
setValues,
|
||||
@ -421,7 +366,7 @@ function ConnectionForm({
|
||||
onDraftChange: (dirty: boolean) => void
|
||||
onReconcile: () => Promise<Connection | undefined>
|
||||
provider: Provider
|
||||
providerName: NewKnowledgeWebsiteProvider
|
||||
providerName: string
|
||||
credentialId?: string
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
@ -582,7 +527,7 @@ function ManagedProviderConnection({
|
||||
onConnected: (connection: Connection) => void
|
||||
onReconcile: () => Promise<Connection | undefined>
|
||||
provider: Provider
|
||||
providerName: NewKnowledgeWebsiteProvider
|
||||
providerName: string
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const { t: tCommon } = useTranslation('common')
|
||||
@ -687,6 +632,7 @@ function UnconfiguredProvider({
|
||||
onDraftChange,
|
||||
onReconcile,
|
||||
provider,
|
||||
providerOption,
|
||||
providerName,
|
||||
credentialId,
|
||||
}: {
|
||||
@ -697,7 +643,8 @@ function UnconfiguredProvider({
|
||||
onDraftChange: (dirty: boolean) => void
|
||||
onReconcile: () => Promise<Connection | undefined>
|
||||
provider: Provider
|
||||
providerName: NewKnowledgeWebsiteProvider
|
||||
providerOption: InstalledSourceProviderOption
|
||||
providerName: string
|
||||
credentialId?: string
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
@ -733,10 +680,7 @@ function UnconfiguredProvider({
|
||||
return (
|
||||
<div className="flex flex-col items-start gap-2.5 rounded-xl bg-background-section p-4">
|
||||
<span className="flex size-9 items-center justify-center rounded-lg border border-divider-subtle bg-background-default">
|
||||
<span
|
||||
aria-hidden
|
||||
className={`${websiteProviderOption(providerName)?.icon ?? 'i-ri-global-line'} size-4.5`}
|
||||
/>
|
||||
<WebsiteProviderIcon option={providerOption} />
|
||||
</span>
|
||||
<h3 className="system-sm-semibold text-text-primary">
|
||||
{t(($) => $['newKnowledge.providerNotConfigured'], {
|
||||
@ -993,12 +937,13 @@ export function AddSourcePage({
|
||||
}),
|
||||
)
|
||||
const provider = findWebsiteSourceProvider(providersQuery.data ?? [])
|
||||
const websiteProviderName =
|
||||
sourceDraft.sourceType === 'websiteCrawl' ? sourceDraft.provider : 'Firecrawl'
|
||||
const datasourceProvider = useMemo(
|
||||
() => findWebsiteDatasourceProvider(datasourcePluginsQuery.data ?? [], websiteProviderName),
|
||||
[datasourcePluginsQuery.data, websiteProviderName],
|
||||
const websiteProviderOptions = useMemo(
|
||||
() => discoverSourceProviderOptions('websiteCrawl', datasourcePluginsQuery.data ?? []),
|
||||
[datasourcePluginsQuery.data],
|
||||
)
|
||||
const websiteProviderOption = sourceProviderOptionForDraft(websiteProviderOptions, sourceDraft)
|
||||
const websiteProviderName = websiteProviderOption?.label ?? sourceDraft.provider
|
||||
const datasourceProvider = websiteProviderOption?.installed ? websiteProviderOption : undefined
|
||||
const datasourceProviders = datasourceAuthQuery.data?.result ?? []
|
||||
const datasourceCredential = findDatasourceCredential(datasourceProviders, datasourceProvider)
|
||||
const difyManagedProvider = provider ? isDifyManagedProvider(provider) : false
|
||||
@ -1368,12 +1313,21 @@ export function AddSourcePage({
|
||||
<>
|
||||
<ProviderSelector
|
||||
disabled={websiteSetupLocked}
|
||||
provider={sourceDraft.provider}
|
||||
options={websiteProviderOptions}
|
||||
providerKey={websiteProviderOption?.key ?? ''}
|
||||
onMoreProviders={() =>
|
||||
globalThis.open(websiteProviderIntegrationPath(), '_blank', 'noopener,noreferrer')
|
||||
}
|
||||
onChange={(provider) => {
|
||||
updateSourceDraft({ ...sourceDraft, provider })
|
||||
onChange={(providerKey) => {
|
||||
const nextProvider = websiteProviderOptions.find(
|
||||
(option) => option.key === providerKey,
|
||||
)
|
||||
if (!nextProvider) return
|
||||
updateSourceDraft({
|
||||
...sourceDraft,
|
||||
provider: nextProvider.label,
|
||||
providerKey: nextProvider.key,
|
||||
})
|
||||
}}
|
||||
/>
|
||||
{queryError ? (
|
||||
@ -1395,34 +1349,22 @@ export function AddSourcePage({
|
||||
{t(($) => $['newKnowledge.retryProviderLoad'])}
|
||||
</Button>
|
||||
</div>
|
||||
) : !provider ? (
|
||||
) : websiteProviderOption && !websiteProviderOption.installed ? (
|
||||
<SourceProviderNotInstalledCard
|
||||
icon={<WebsiteProviderIcon option={websiteProviderOption} />}
|
||||
provider={websiteProviderOption.label}
|
||||
onInstall={() =>
|
||||
globalThis.open(
|
||||
websiteProviderIntegrationPath(websiteProviderOption),
|
||||
'_blank',
|
||||
'noopener,noreferrer',
|
||||
)
|
||||
}
|
||||
/>
|
||||
) : !datasourceProvider || !provider ? (
|
||||
<div className="rounded-xl bg-background-section p-4 system-sm-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.providerUnavailable'])}
|
||||
</div>
|
||||
) : !datasourceProvider ? (
|
||||
<div className="rounded-xl bg-background-section p-4">
|
||||
<p className="system-sm-semibold text-text-primary">{websiteProviderName}</p>
|
||||
<p className="mt-1 system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.providerUnavailable'])}
|
||||
</p>
|
||||
{websiteProviderName !== 'FakeCrawler' && (
|
||||
<Button
|
||||
variant="primary"
|
||||
className="mt-3"
|
||||
onClick={() =>
|
||||
globalThis.open(
|
||||
websiteProviderIntegrationPath(websiteProviderName),
|
||||
'_blank',
|
||||
'noopener,noreferrer',
|
||||
)
|
||||
}
|
||||
>
|
||||
{t(($) => $['newKnowledge.configureProvider'], {
|
||||
provider: websiteProviderName,
|
||||
})}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
) : !provider.available || !supportsDirectConnection ? (
|
||||
<div className="rounded-xl bg-background-section p-4">
|
||||
<p className="system-sm-semibold text-text-primary">{websiteProviderName}</p>
|
||||
@ -1462,7 +1404,7 @@ export function AddSourcePage({
|
||||
onConnected={rememberConnection}
|
||||
onConfigureManagedProvider={() =>
|
||||
globalThis.open(
|
||||
websiteProviderIntegrationPath(websiteProviderName),
|
||||
websiteProviderIntegrationPath(websiteProviderOption),
|
||||
'_blank',
|
||||
'noopener,noreferrer',
|
||||
)
|
||||
@ -1470,6 +1412,7 @@ export function AddSourcePage({
|
||||
onDraftChange={setConnectionDraftDirty}
|
||||
onReconcile={reconcileConnection}
|
||||
provider={provider}
|
||||
providerOption={datasourceProvider}
|
||||
providerName={websiteProviderName}
|
||||
/>
|
||||
)}
|
||||
|
||||
@ -5,13 +5,12 @@ import type {
|
||||
KnowledgeFsSourcePageResponse,
|
||||
} from '@dify/contracts/api/console/knowledge-fs/types.gen'
|
||||
import type {
|
||||
NewKnowledgeOnlineDocumentsProvider,
|
||||
NewKnowledgeOnlineDocumentsSourceDraft,
|
||||
NewKnowledgeOnlineDriveProvider,
|
||||
NewKnowledgeOnlineDriveSourceDraft,
|
||||
NewKnowledgeSourceDraft,
|
||||
} from './routes'
|
||||
import type { Source, SourceConnection, SourceProvider } from './source-models'
|
||||
import type { InstalledSourceProviderOption, SourceProviderOption } from './source-provider-options'
|
||||
import type {
|
||||
DataSourceAuth,
|
||||
DataSourceCredential,
|
||||
@ -39,9 +38,15 @@ import {
|
||||
sourceProviderListFromApi,
|
||||
sourceSyncPolicyFromApi,
|
||||
} from './source-models'
|
||||
import {
|
||||
discoverSourceProviderOptions,
|
||||
normalizeSourceProviderName,
|
||||
sourceProviderOptionForDraft,
|
||||
} from './source-provider-options'
|
||||
import {
|
||||
SourceConnectionRequiredCard,
|
||||
SourceNameField,
|
||||
SourceProviderNotInstalledCard,
|
||||
SourceProviderRadioGroup,
|
||||
SourceSyncPolicyField,
|
||||
} from './source-setup-fields'
|
||||
@ -96,73 +101,25 @@ const CONNECTION_STATUS_PRIORITY: Record<SourceConnection['status'], number> = {
|
||||
revoked: 4,
|
||||
}
|
||||
|
||||
const providerOptions = {
|
||||
onlineDocuments: [
|
||||
{
|
||||
aliases: ['notion'],
|
||||
icon: 'i-custom-public-common-notion',
|
||||
label: 'Notion',
|
||||
},
|
||||
{
|
||||
aliases: ['google docs', 'googledocs', 'google drive', 'googledrive'],
|
||||
icon: 'i-ri-file-text-fill text-[#4d8bf5]',
|
||||
label: 'Google Docs',
|
||||
},
|
||||
{
|
||||
aliases: ['confluence'],
|
||||
icon: 'i-custom-public-common-confluence',
|
||||
label: 'Confluence',
|
||||
},
|
||||
],
|
||||
onlineDrive: [
|
||||
{
|
||||
aliases: ['google drive', 'googledrive'],
|
||||
icon: 'i-custom-public-common-google-drive',
|
||||
label: 'Google Drive',
|
||||
},
|
||||
{
|
||||
aliases: ['onedrive', 'microsoft onedrive'],
|
||||
icon: 'i-ri-cloud-line',
|
||||
label: 'OneDrive',
|
||||
},
|
||||
{
|
||||
aliases: ['amazon s3', 'amazons3', 's3'],
|
||||
icon: 'i-ri-box-3-line',
|
||||
label: 'Amazon S3',
|
||||
},
|
||||
],
|
||||
} as const
|
||||
|
||||
const providerPackageIds: Record<ConnectedSourceDraft['provider'], string | undefined> = {
|
||||
'Amazon S3': 'langgenius/aws_s3_storage',
|
||||
Confluence: 'langgenius/confluence_datasource',
|
||||
'Google Docs': 'langgenius/google_drive',
|
||||
'Google Drive': 'langgenius/google_drive',
|
||||
Notion: 'langgenius/notion_datasource',
|
||||
OneDrive: 'langgenius/onedrive_datasource',
|
||||
}
|
||||
|
||||
function normalizeProviderName(value: string) {
|
||||
return value.toLocaleLowerCase().replace(/[^a-z0-9]+/g, '')
|
||||
}
|
||||
|
||||
function usesDriveTransport(draft: ConnectedSourceDraft) {
|
||||
return draft.sourceType === 'onlineDrive' || draft.provider === 'Google Docs'
|
||||
}
|
||||
|
||||
function providerForDraft(providers: SourceProvider[], draft: ConnectedSourceDraft) {
|
||||
function providerForDraft(
|
||||
providers: SourceProvider[],
|
||||
draft: ConnectedSourceDraft,
|
||||
option?: SourceProviderOption,
|
||||
) {
|
||||
const capability = usesDriveTransport(draft) ? 'online-drive' : 'online-document'
|
||||
const option = providerOptions[draft.sourceType].find((item) => item.label === draft.provider)
|
||||
if (!option) return undefined
|
||||
const aliases = new Set(
|
||||
[option.label, ...option.aliases].map((candidate) => normalizeProviderName(candidate)),
|
||||
[draft.provider, option?.label ?? ''].map(normalizeSourceProviderName).filter(Boolean),
|
||||
)
|
||||
const capableProviders = providers.filter((provider) =>
|
||||
provider.capabilities.includes(capability),
|
||||
)
|
||||
return (
|
||||
capableProviders.find((provider) => {
|
||||
const names = [provider.id, provider.displayName].map(normalizeProviderName)
|
||||
const names = [provider.id, provider.displayName].map(normalizeSourceProviderName)
|
||||
return names.some(
|
||||
(name) =>
|
||||
aliases.has(name) ||
|
||||
@ -176,45 +133,19 @@ function providerForDraft(providers: SourceProvider[], draft: ConnectedSourceDra
|
||||
)
|
||||
}
|
||||
|
||||
function datasourceProviderForDraft(
|
||||
datasourcePlugins: DataSourceItem[],
|
||||
draft: ConnectedSourceDraft,
|
||||
) {
|
||||
const option = providerOptions[draft.sourceType].find((item) => item.label === draft.provider)
|
||||
if (!option) return undefined
|
||||
const aliases = [option.label, ...option.aliases].map(normalizeProviderName)
|
||||
const providerType = usesDriveTransport(draft) ? 'online_drive' : 'online_document'
|
||||
for (const plugin of datasourcePlugins) {
|
||||
if (plugin.declaration.provider_type !== providerType) continue
|
||||
const identities = [
|
||||
plugin.declaration.identity.label.en_US,
|
||||
plugin.declaration.identity.name,
|
||||
plugin.plugin_id,
|
||||
plugin.provider,
|
||||
].map(normalizeProviderName)
|
||||
const datasource = plugin.declaration.datasources.find((action) => {
|
||||
const actionIdentities = [
|
||||
action.identity.label.en_US,
|
||||
action.identity.name,
|
||||
action.identity.provider,
|
||||
].map(normalizeProviderName)
|
||||
return [...identities, ...actionIdentities].some((identity) =>
|
||||
aliases.some((alias) => identity.includes(alias) || alias.includes(identity)),
|
||||
)
|
||||
})
|
||||
if (datasource)
|
||||
return {
|
||||
datasource,
|
||||
plugin,
|
||||
function datasourceProviderForOption(option?: SourceProviderOption) {
|
||||
return option?.installed
|
||||
? {
|
||||
datasource: option.datasource,
|
||||
plugin: option.plugin,
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
: undefined
|
||||
}
|
||||
|
||||
type ProviderBrandIconValue = DataSourceItem['declaration']['identity']['icon']
|
||||
|
||||
function datasourceProviderIcon(
|
||||
datasourceProvider: ReturnType<typeof datasourceProviderForDraft>,
|
||||
datasourceProvider: ReturnType<typeof datasourceProviderForOption>,
|
||||
): ProviderBrandIconValue | undefined {
|
||||
return datasourceProvider?.plugin.declaration.identity.icon
|
||||
}
|
||||
@ -222,7 +153,7 @@ function datasourceProviderIcon(
|
||||
function credentialRegion(credential: DataSourceCredential | undefined) {
|
||||
const region = Object.entries(credential?.credential ?? {}).find(
|
||||
([key, value]) =>
|
||||
['awsregion', 'region', 'regionname'].includes(normalizeProviderName(key)) &&
|
||||
['awsregion', 'region', 'regionname'].includes(normalizeSourceProviderName(key)) &&
|
||||
typeof value === 'string' &&
|
||||
value.trim(),
|
||||
)?.[1]
|
||||
@ -231,7 +162,7 @@ function credentialRegion(credential: DataSourceCredential | undefined) {
|
||||
|
||||
function datasourceAuthForProvider(
|
||||
providers: DataSourceAuth[],
|
||||
datasourceProvider: ReturnType<typeof datasourceProviderForDraft>,
|
||||
datasourceProvider: ReturnType<typeof datasourceProviderForOption>,
|
||||
) {
|
||||
if (!datasourceProvider) return undefined
|
||||
return providers.find(
|
||||
@ -250,7 +181,7 @@ function preferredCredential(provider?: DataSourceAuth) {
|
||||
|
||||
function connectionMatchesDatasource(
|
||||
connection: SourceConnection,
|
||||
datasourceProvider: ReturnType<typeof datasourceProviderForDraft>,
|
||||
datasourceProvider: ReturnType<typeof datasourceProviderForOption>,
|
||||
credential: DataSourceCredential | undefined,
|
||||
) {
|
||||
if (!datasourceProvider) return false
|
||||
@ -266,7 +197,7 @@ function connectionMatchesDatasource(
|
||||
function findProviderConnection(
|
||||
connections: SourceConnection[],
|
||||
providerId: string | undefined,
|
||||
datasourceProvider: ReturnType<typeof datasourceProviderForDraft>,
|
||||
datasourceProvider: ReturnType<typeof datasourceProviderForOption>,
|
||||
credential: DataSourceCredential | undefined,
|
||||
) {
|
||||
if (!providerId || !datasourceProvider) return undefined
|
||||
@ -285,16 +216,15 @@ function findProviderConnection(
|
||||
)[0]
|
||||
}
|
||||
|
||||
function providerIntegrationPath(draft: ConnectedSourceDraft) {
|
||||
function providerIntegrationPath(option?: SourceProviderOption) {
|
||||
const base = buildIntegrationPath('data-source')
|
||||
const packageId = providerPackageIds[draft.provider]
|
||||
if (!packageId) return base
|
||||
const query = new URLSearchParams({ 'package-ids': JSON.stringify([packageId]) })
|
||||
if (!option) return base
|
||||
const query = new URLSearchParams({ 'package-ids': JSON.stringify([option.packageId]) })
|
||||
return `${base}?${query.toString()}`
|
||||
}
|
||||
|
||||
function providerScheme(providerName: string) {
|
||||
const normalized = normalizeProviderName(providerName)
|
||||
const normalized = normalizeSourceProviderName(providerName)
|
||||
if (normalized.includes('notion')) return 'notion'
|
||||
if (normalized.includes('googledocs')) return 'gdocs'
|
||||
if (normalized.includes('googledrive')) return 'gdrive'
|
||||
@ -312,7 +242,7 @@ function sourceUri(
|
||||
const scheme = providerScheme(`${draft.provider} ${provider.id} ${provider.displayName}`)
|
||||
if (scheme === 's3') {
|
||||
const bucket = Object.entries(connection.configuration).find(([key]) =>
|
||||
['bucket', 'bucketname'].includes(normalizeProviderName(key)),
|
||||
['bucket', 'bucketname'].includes(normalizeSourceProviderName(key)),
|
||||
)?.[1]
|
||||
if (typeof bucket === 'string' && bucket.trim()) return `s3://${bucket.trim()}`
|
||||
}
|
||||
@ -386,18 +316,17 @@ function requestStatus(error: unknown) {
|
||||
}
|
||||
|
||||
function ProviderSelector({
|
||||
datasourcePlugins,
|
||||
draft,
|
||||
options,
|
||||
providerKey,
|
||||
onChange,
|
||||
onMoreProviders,
|
||||
}: {
|
||||
datasourcePlugins: DataSourceItem[]
|
||||
draft: ConnectedSourceDraft
|
||||
onChange: (provider: string) => void
|
||||
options: SourceProviderOption[]
|
||||
providerKey: string
|
||||
onChange: (providerKey: string) => void
|
||||
onMoreProviders: () => void
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const options = providerOptions[draft.sourceType]
|
||||
return (
|
||||
<Fieldset>
|
||||
<div className="mb-2 flex items-center justify-between gap-3">
|
||||
@ -416,27 +345,22 @@ function ProviderSelector({
|
||||
</Button>
|
||||
</div>
|
||||
<SourceProviderRadioGroup
|
||||
value={draft.provider}
|
||||
value={providerKey}
|
||||
layout="wrap"
|
||||
options={options.map((option) => {
|
||||
const optionProvider = datasourceProviderForDraft(datasourcePlugins, {
|
||||
...draft,
|
||||
provider: option.label,
|
||||
} as ConnectedSourceDraft)
|
||||
return {
|
||||
icon: (
|
||||
<ProviderBrandIcon
|
||||
fallbackIcon={option.icon}
|
||||
icon={
|
||||
option.label === 'Google Docs'
|
||||
? undefined
|
||||
: datasourceProviderIcon(optionProvider)
|
||||
}
|
||||
/>
|
||||
),
|
||||
value: option.label,
|
||||
}
|
||||
})}
|
||||
options={options.map((option) => ({
|
||||
icon: (
|
||||
<ProviderBrandIcon
|
||||
fallbackIcon={option.fallbackIcon}
|
||||
icon={
|
||||
option.installed
|
||||
? (option.datasource.identity.icon ?? option.plugin.declaration.identity.icon)
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
),
|
||||
label: option.label,
|
||||
value: option.key,
|
||||
}))}
|
||||
size="small"
|
||||
onChange={onChange}
|
||||
/>
|
||||
@ -471,14 +395,15 @@ function ProviderBrandIcon({
|
||||
function OAuthConnectionCard({
|
||||
draft,
|
||||
icon,
|
||||
providerOption,
|
||||
onConnect,
|
||||
}: {
|
||||
draft: ConnectedSourceDraft
|
||||
icon?: ProviderBrandIconValue
|
||||
providerOption: InstalledSourceProviderOption
|
||||
onConnect: () => void
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const option = providerOptions[draft.sourceType].find((item) => item.label === draft.provider)
|
||||
return (
|
||||
<SourceConnectionRequiredCard
|
||||
actionLabel={t(($) => $['newKnowledge.connectProvider'], { provider: draft.provider })}
|
||||
@ -489,7 +414,7 @@ function OAuthConnectionCard({
|
||||
provider: draft.provider,
|
||||
})
|
||||
}
|
||||
icon={<ProviderBrandIcon fallbackIcon={option?.icon ?? 'i-ri-links-line'} icon={icon} />}
|
||||
icon={<ProviderBrandIcon fallbackIcon={providerOption.fallbackIcon} icon={icon} />}
|
||||
title={
|
||||
draft.provider === 'Notion'
|
||||
? t(($) => $['newKnowledge.notionNotConnected'])
|
||||
@ -1707,9 +1632,15 @@ export function ConnectedSourceSetup({
|
||||
retry: false,
|
||||
}),
|
||||
)
|
||||
const provider = providerForDraft(providersQuery.data ?? [], draft)
|
||||
const providerOptions = useMemo(
|
||||
() => discoverSourceProviderOptions(draft.sourceType, datasourcePluginsQuery.data ?? []),
|
||||
[datasourcePluginsQuery.data, draft.sourceType],
|
||||
)
|
||||
const providerOption = sourceProviderOptionForDraft(providerOptions, draft)
|
||||
const installedProviderOption = providerOption?.installed ? providerOption : undefined
|
||||
const provider = providerForDraft(providersQuery.data ?? [], draft, providerOption)
|
||||
const driveTransport = usesDriveTransport(draft)
|
||||
const datasourceProvider = datasourceProviderForDraft(datasourcePluginsQuery.data ?? [], draft)
|
||||
const datasourceProvider = datasourceProviderForOption(installedProviderOption)
|
||||
const datasourceAuth = datasourceAuthForProvider(
|
||||
datasourceAuthQuery.data?.result ?? [],
|
||||
datasourceProvider,
|
||||
@ -1866,23 +1797,27 @@ export function ConnectedSourceSetup({
|
||||
providersQuery.isPending,
|
||||
provisionConnection,
|
||||
])
|
||||
const selectProvider = (nextProvider: string) => {
|
||||
const selectProvider = (providerKey: string) => {
|
||||
const nextProvider = providerOptions.find((option) => option.key === providerKey)
|
||||
if (!nextProvider) return
|
||||
setConnectionOverride(undefined)
|
||||
setProvisionError(false)
|
||||
if (draft.sourceType === 'onlineDocuments') {
|
||||
onDraftChange({
|
||||
...draft,
|
||||
provider: nextProvider as NewKnowledgeOnlineDocumentsProvider,
|
||||
provider: nextProvider.label,
|
||||
providerKey: nextProvider.key,
|
||||
sourceName: '',
|
||||
})
|
||||
return
|
||||
}
|
||||
onDraftChange({
|
||||
...draft,
|
||||
provider: nextProvider as NewKnowledgeOnlineDriveProvider,
|
||||
provider: nextProvider.label,
|
||||
providerKey: nextProvider.key,
|
||||
sourceName: '',
|
||||
syncPolicy:
|
||||
nextProvider === 'Amazon S3' && draft.syncPolicy === 'provider'
|
||||
nextProvider.label === 'Amazon S3' && draft.syncPolicy === 'provider'
|
||||
? 'daily'
|
||||
: draft.syncPolicy,
|
||||
})
|
||||
@ -1890,8 +1825,8 @@ export function ConnectedSourceSetup({
|
||||
return (
|
||||
<div className="flex flex-col gap-4">
|
||||
<ProviderSelector
|
||||
datasourcePlugins={datasourcePluginsQuery.data ?? []}
|
||||
draft={draft}
|
||||
options={providerOptions}
|
||||
providerKey={providerOption?.key ?? ''}
|
||||
onChange={selectProvider}
|
||||
onMoreProviders={() =>
|
||||
globalThis.open(buildIntegrationPath('data-source'), '_blank', 'noopener,noreferrer')
|
||||
@ -1923,7 +1858,19 @@ export function ConnectedSourceSetup({
|
||||
{t(($) => $['newKnowledge.retryProviderLoad'])}
|
||||
</Button>
|
||||
</div>
|
||||
) : !provider ? (
|
||||
) : providerOption && !providerOption.installed ? (
|
||||
<SourceProviderNotInstalledCard
|
||||
icon={<ProviderBrandIcon fallbackIcon={providerOption.fallbackIcon} />}
|
||||
provider={providerOption.label}
|
||||
onInstall={() =>
|
||||
globalThis.open(
|
||||
providerIntegrationPath(providerOption),
|
||||
'_blank',
|
||||
'noopener,noreferrer',
|
||||
)
|
||||
}
|
||||
/>
|
||||
) : !installedProviderOption || !provider ? (
|
||||
<div className="rounded-xl bg-background-section p-4">
|
||||
<p className="system-sm-semibold text-text-primary">{draft.provider}</p>
|
||||
<p className="mt-1 system-xs-regular text-text-tertiary">
|
||||
@ -1977,7 +1924,11 @@ export function ConnectedSourceSetup({
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={() =>
|
||||
globalThis.open(providerIntegrationPath(draft), '_blank', 'noopener,noreferrer')
|
||||
globalThis.open(
|
||||
providerIntegrationPath(installedProviderOption),
|
||||
'_blank',
|
||||
'noopener,noreferrer',
|
||||
)
|
||||
}
|
||||
>
|
||||
{t(($) => $['newKnowledge.connectProvider'], { provider: draft.provider })}
|
||||
@ -2018,12 +1969,18 @@ export function ConnectedSourceSetup({
|
||||
<OAuthConnectionCard
|
||||
draft={draft}
|
||||
icon={datasourceProviderIcon(datasourceProvider)}
|
||||
providerOption={installedProviderOption}
|
||||
onConnect={() =>
|
||||
globalThis.open(providerIntegrationPath(draft), '_blank', 'noopener,noreferrer')
|
||||
globalThis.open(
|
||||
providerIntegrationPath(installedProviderOption),
|
||||
'_blank',
|
||||
'noopener,noreferrer',
|
||||
)
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{!connection &&
|
||||
Boolean(installedProviderOption) &&
|
||||
provider?.available &&
|
||||
!provisioningConnection &&
|
||||
!queryError &&
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
export type NewKnowledgeStartMode = 'empty' | 'source' | 'upload'
|
||||
export type NewKnowledgeSourceType = 'onlineDocuments' | 'onlineDrive' | 'websiteCrawl'
|
||||
type NewKnowledgeSyncPolicy = 'daily' | 'manual' | 'provider'
|
||||
export type NewKnowledgeWebsiteProvider = 'FakeCrawler' | 'Firecrawl' | 'Jina Reader' | 'WaterCrawl'
|
||||
export type NewKnowledgeOnlineDocumentsProvider = 'Confluence' | 'Google Docs' | 'Notion'
|
||||
export type NewKnowledgeOnlineDriveProvider = 'Amazon S3' | 'Google Drive' | 'OneDrive'
|
||||
export type NewKnowledgeWebsiteProvider = string
|
||||
export type NewKnowledgeOnlineDocumentsProvider = string
|
||||
export type NewKnowledgeOnlineDriveProvider = string
|
||||
export type NewKnowledgeSourceProvider =
|
||||
| NewKnowledgeOnlineDocumentsProvider
|
||||
| NewKnowledgeOnlineDriveProvider
|
||||
@ -12,6 +12,7 @@ export type NewKnowledgeSourceProvider =
|
||||
type NewKnowledgeSourceDraftBase = {
|
||||
sourceName: string
|
||||
syncPolicy: NewKnowledgeSyncPolicy
|
||||
providerKey?: string
|
||||
}
|
||||
|
||||
export type NewKnowledgeWebsiteSourceDraft = NewKnowledgeSourceDraftBase & {
|
||||
@ -39,6 +40,8 @@ export type NewKnowledgeSourceDraft =
|
||||
|
||||
export const NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH = 200
|
||||
export const NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH = 2048
|
||||
const NEW_KNOWLEDGE_PROVIDER_NAME_MAX_LENGTH = 200
|
||||
const NEW_KNOWLEDGE_PROVIDER_KEY_MAX_LENGTH = 1024
|
||||
const NEW_KNOWLEDGE_SOURCE_DRAFT_STORAGE_PREFIX = 'new-knowledge-source-draft:'
|
||||
|
||||
export function createNewKnowledgeSourceDraft(
|
||||
@ -47,18 +50,14 @@ export function createNewKnowledgeSourceDraft(
|
||||
): NewKnowledgeSourceDraft {
|
||||
if (sourceType === 'onlineDocuments')
|
||||
return {
|
||||
provider: ['Confluence', 'Google Docs', 'Notion'].includes(initialProvider ?? '')
|
||||
? (initialProvider as NewKnowledgeOnlineDocumentsProvider)
|
||||
: 'Notion',
|
||||
provider: initialProvider?.trim() || 'Notion',
|
||||
sourceName: '',
|
||||
sourceType,
|
||||
syncPolicy: 'provider',
|
||||
}
|
||||
if (sourceType === 'onlineDrive')
|
||||
return {
|
||||
provider: ['Amazon S3', 'Google Drive', 'OneDrive'].includes(initialProvider ?? '')
|
||||
? (initialProvider as NewKnowledgeOnlineDriveProvider)
|
||||
: 'Google Drive',
|
||||
provider: initialProvider?.trim() || 'Google Drive',
|
||||
sourceName: '',
|
||||
sourceType,
|
||||
syncPolicy: 'provider',
|
||||
@ -66,11 +65,7 @@ export function createNewKnowledgeSourceDraft(
|
||||
return {
|
||||
includeSubpages: true,
|
||||
maxPages: 100,
|
||||
provider: ['FakeCrawler', 'Firecrawl', 'Jina Reader', 'WaterCrawl'].includes(
|
||||
initialProvider ?? '',
|
||||
)
|
||||
? (initialProvider as NewKnowledgeWebsiteProvider)
|
||||
: 'Firecrawl',
|
||||
provider: initialProvider?.trim() || 'Firecrawl',
|
||||
rootUrl: '',
|
||||
sourceName: '',
|
||||
sourceType,
|
||||
@ -135,24 +130,29 @@ export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceD
|
||||
if (
|
||||
typeof candidate.sourceName !== 'string' ||
|
||||
candidate.sourceName.length > NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH ||
|
||||
typeof candidate.provider !== 'string' ||
|
||||
!candidate.provider.trim() ||
|
||||
candidate.provider.length > NEW_KNOWLEDGE_PROVIDER_NAME_MAX_LENGTH ||
|
||||
(candidate.providerKey !== undefined &&
|
||||
(typeof candidate.providerKey !== 'string' ||
|
||||
!candidate.providerKey ||
|
||||
candidate.providerKey.length > NEW_KNOWLEDGE_PROVIDER_KEY_MAX_LENGTH)) ||
|
||||
!syncPolicy
|
||||
)
|
||||
return undefined
|
||||
if (candidate.sourceType === 'onlineDocuments') {
|
||||
if (!['Confluence', 'Google Docs', 'Notion'].includes(String(candidate.provider)))
|
||||
return undefined
|
||||
return {
|
||||
provider: candidate.provider as NewKnowledgeOnlineDocumentsProvider,
|
||||
provider: candidate.provider,
|
||||
...(candidate.providerKey ? { providerKey: candidate.providerKey } : {}),
|
||||
sourceName: candidate.sourceName,
|
||||
sourceType: candidate.sourceType,
|
||||
syncPolicy,
|
||||
}
|
||||
}
|
||||
if (candidate.sourceType === 'onlineDrive') {
|
||||
if (!['Amazon S3', 'Google Drive', 'OneDrive'].includes(String(candidate.provider)))
|
||||
return undefined
|
||||
return {
|
||||
provider: candidate.provider as NewKnowledgeOnlineDriveProvider,
|
||||
provider: candidate.provider,
|
||||
...(candidate.providerKey ? { providerKey: candidate.providerKey } : {}),
|
||||
sourceName: candidate.sourceName,
|
||||
sourceType: candidate.sourceType,
|
||||
syncPolicy,
|
||||
@ -160,9 +160,6 @@ export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceD
|
||||
}
|
||||
if (
|
||||
(candidate.sourceType !== undefined && candidate.sourceType !== 'websiteCrawl') ||
|
||||
!['FakeCrawler', 'Firecrawl', 'Jina Reader', 'WaterCrawl'].includes(
|
||||
String(candidate.provider),
|
||||
) ||
|
||||
typeof candidate.includeSubpages !== 'boolean' ||
|
||||
typeof candidate.maxPages !== 'number' ||
|
||||
!Number.isInteger(candidate.maxPages) ||
|
||||
@ -175,7 +172,8 @@ export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceD
|
||||
return {
|
||||
includeSubpages: candidate.includeSubpages,
|
||||
maxPages: candidate.maxPages,
|
||||
provider: candidate.provider as NewKnowledgeWebsiteProvider,
|
||||
provider: candidate.provider,
|
||||
...(candidate.providerKey ? { providerKey: candidate.providerKey } : {}),
|
||||
rootUrl: candidate.rootUrl,
|
||||
sourceName: candidate.sourceName,
|
||||
sourceType: 'websiteCrawl',
|
||||
|
||||
266
web/features/new-rag/source-provider-options.ts
Normal file
266
web/features/new-rag/source-provider-options.ts
Normal file
@ -0,0 +1,266 @@
|
||||
import type { NewKnowledgeSourceDraft, NewKnowledgeSourceType } from './routes'
|
||||
import type { DataSourceItem } from '@/app/components/workflow/block-selector/types'
|
||||
|
||||
type Datasource = DataSourceItem['declaration']['datasources'][number]
|
||||
|
||||
type RecommendedProvider = {
|
||||
aliases: string[]
|
||||
fallbackIcon: string
|
||||
label: string
|
||||
packageId: string
|
||||
providerType: DataSourceItem['declaration']['provider_type']
|
||||
sourceType: NewKnowledgeSourceType
|
||||
}
|
||||
|
||||
export type InstalledSourceProviderOption = {
|
||||
datasource: Datasource
|
||||
fallbackIcon: string
|
||||
installed: true
|
||||
key: string
|
||||
label: string
|
||||
packageId: string
|
||||
plugin: DataSourceItem
|
||||
providerType: DataSourceItem['declaration']['provider_type']
|
||||
sourceType: NewKnowledgeSourceType
|
||||
}
|
||||
|
||||
export type UninstalledSourceProviderOption = {
|
||||
fallbackIcon: string
|
||||
installed: false
|
||||
key: string
|
||||
label: string
|
||||
packageId: string
|
||||
providerType: DataSourceItem['declaration']['provider_type']
|
||||
sourceType: NewKnowledgeSourceType
|
||||
}
|
||||
|
||||
export type SourceProviderOption = InstalledSourceProviderOption | UninstalledSourceProviderOption
|
||||
|
||||
const recommendedProviders: RecommendedProvider[] = [
|
||||
{
|
||||
aliases: ['firecrawl'],
|
||||
fallbackIcon: 'i-custom-public-common-firecrawl',
|
||||
label: 'Firecrawl',
|
||||
packageId: 'langgenius/firecrawl_datasource',
|
||||
providerType: 'website_crawl',
|
||||
sourceType: 'websiteCrawl',
|
||||
},
|
||||
{
|
||||
aliases: ['jina', 'jina reader', 'jinareader'],
|
||||
fallbackIcon: 'i-custom-public-llm-jina',
|
||||
label: 'Jina Reader',
|
||||
packageId: 'langgenius/jina_datasource',
|
||||
providerType: 'website_crawl',
|
||||
sourceType: 'websiteCrawl',
|
||||
},
|
||||
{
|
||||
aliases: ['watercrawl'],
|
||||
fallbackIcon: 'i-custom-public-knowledge-watercrawl',
|
||||
label: 'WaterCrawl',
|
||||
packageId: 'watercrawl/watercrawl_datasource',
|
||||
providerType: 'website_crawl',
|
||||
sourceType: 'websiteCrawl',
|
||||
},
|
||||
{
|
||||
aliases: ['notion'],
|
||||
fallbackIcon: 'i-custom-public-common-notion',
|
||||
label: 'Notion',
|
||||
packageId: 'langgenius/notion_datasource',
|
||||
providerType: 'online_document',
|
||||
sourceType: 'onlineDocuments',
|
||||
},
|
||||
{
|
||||
aliases: ['google docs', 'googledocs', 'google drive', 'googledrive'],
|
||||
fallbackIcon: 'i-ri-file-text-fill text-[#4d8bf5]',
|
||||
label: 'Google Docs',
|
||||
packageId: 'langgenius/google_drive',
|
||||
providerType: 'online_drive',
|
||||
sourceType: 'onlineDocuments',
|
||||
},
|
||||
{
|
||||
aliases: ['confluence'],
|
||||
fallbackIcon: 'i-custom-public-common-confluence',
|
||||
label: 'Confluence',
|
||||
packageId: 'langgenius/confluence_datasource',
|
||||
providerType: 'online_document',
|
||||
sourceType: 'onlineDocuments',
|
||||
},
|
||||
{
|
||||
aliases: ['google drive', 'googledrive'],
|
||||
fallbackIcon: 'i-custom-public-common-google-drive',
|
||||
label: 'Google Drive',
|
||||
packageId: 'langgenius/google_drive',
|
||||
providerType: 'online_drive',
|
||||
sourceType: 'onlineDrive',
|
||||
},
|
||||
{
|
||||
aliases: ['onedrive', 'microsoft onedrive'],
|
||||
fallbackIcon: 'i-ri-cloud-line',
|
||||
label: 'OneDrive',
|
||||
packageId: 'langgenius/onedrive_datasource',
|
||||
providerType: 'online_drive',
|
||||
sourceType: 'onlineDrive',
|
||||
},
|
||||
{
|
||||
aliases: ['amazon s3', 'amazons3', 's3'],
|
||||
fallbackIcon: 'i-ri-box-3-line',
|
||||
label: 'Amazon S3',
|
||||
packageId: 'langgenius/aws_s3_storage',
|
||||
providerType: 'online_drive',
|
||||
sourceType: 'onlineDrive',
|
||||
},
|
||||
]
|
||||
|
||||
export function normalizeSourceProviderName(value: string) {
|
||||
return value.toLocaleLowerCase().replace(/[^a-z0-9]+/g, '')
|
||||
}
|
||||
|
||||
function providerKey(
|
||||
sourceType: NewKnowledgeSourceType,
|
||||
plugin: DataSourceItem,
|
||||
datasource: Datasource,
|
||||
) {
|
||||
return `${sourceType}:${plugin.plugin_id}:${plugin.provider}:${datasource.identity.name}`
|
||||
}
|
||||
|
||||
function datasourceMatchesAliases(
|
||||
plugin: DataSourceItem,
|
||||
datasource: Datasource,
|
||||
aliases: string[],
|
||||
) {
|
||||
const normalizedAliases = aliases.map(normalizeSourceProviderName)
|
||||
return [
|
||||
plugin.declaration.identity.label.en_US,
|
||||
plugin.declaration.identity.name,
|
||||
plugin.provider,
|
||||
datasource.identity.label.en_US,
|
||||
datasource.identity.name,
|
||||
datasource.identity.provider,
|
||||
]
|
||||
.map(normalizeSourceProviderName)
|
||||
.some((identity) =>
|
||||
normalizedAliases.some((alias) => identity.includes(alias) || alias.includes(identity)),
|
||||
)
|
||||
}
|
||||
|
||||
function installedRecommendedProvider(
|
||||
definition: RecommendedProvider,
|
||||
datasourcePlugins: DataSourceItem[],
|
||||
): InstalledSourceProviderOption | undefined {
|
||||
const plugin = datasourcePlugins.find(
|
||||
(candidate) =>
|
||||
candidate.plugin_id === definition.packageId &&
|
||||
candidate.declaration.provider_type === definition.providerType,
|
||||
)
|
||||
if (!plugin) return undefined
|
||||
const datasource =
|
||||
plugin.declaration.datasources.find((candidate) =>
|
||||
datasourceMatchesAliases(plugin, candidate, definition.aliases),
|
||||
) ?? plugin.declaration.datasources[0]
|
||||
if (!datasource) return undefined
|
||||
return {
|
||||
datasource,
|
||||
fallbackIcon: definition.fallbackIcon,
|
||||
installed: true,
|
||||
key: providerKey(definition.sourceType, plugin, datasource),
|
||||
label: definition.label,
|
||||
packageId: definition.packageId,
|
||||
plugin,
|
||||
providerType: definition.providerType,
|
||||
sourceType: definition.sourceType,
|
||||
}
|
||||
}
|
||||
|
||||
function sourceTypeForProviderType(
|
||||
providerType: DataSourceItem['declaration']['provider_type'],
|
||||
): NewKnowledgeSourceType | undefined {
|
||||
if (providerType === 'website_crawl') return 'websiteCrawl'
|
||||
if (providerType === 'online_document') return 'onlineDocuments'
|
||||
if (providerType === 'online_drive') return 'onlineDrive'
|
||||
return undefined
|
||||
}
|
||||
|
||||
function fallbackIcon(sourceType: NewKnowledgeSourceType) {
|
||||
if (sourceType === 'websiteCrawl') return 'i-ri-global-line'
|
||||
if (sourceType === 'onlineDocuments') return 'i-ri-file-text-line'
|
||||
return 'i-ri-hard-drive-3-line'
|
||||
}
|
||||
|
||||
function uniqueLabel(label: string, pluginLabel: string, labels: Set<string>) {
|
||||
if (!labels.has(label)) return label
|
||||
const qualified = `${label} (${pluginLabel})`
|
||||
if (!labels.has(qualified)) return qualified
|
||||
let suffix = 2
|
||||
while (labels.has(`${qualified} ${suffix}`)) suffix += 1
|
||||
return `${qualified} ${suffix}`
|
||||
}
|
||||
|
||||
export function discoverSourceProviderOptions(
|
||||
sourceType: NewKnowledgeSourceType,
|
||||
datasourcePlugins: DataSourceItem[],
|
||||
): SourceProviderOption[] {
|
||||
const definitions = recommendedProviders.filter(
|
||||
(definition) => definition.sourceType === sourceType,
|
||||
)
|
||||
const recommended: SourceProviderOption[] = definitions.map((definition) => {
|
||||
const installed = installedRecommendedProvider(definition, datasourcePlugins)
|
||||
return (
|
||||
installed ?? {
|
||||
fallbackIcon: definition.fallbackIcon,
|
||||
installed: false as const,
|
||||
key: `${sourceType}:marketplace:${definition.packageId}`,
|
||||
label: definition.label,
|
||||
packageId: definition.packageId,
|
||||
providerType: definition.providerType,
|
||||
sourceType,
|
||||
}
|
||||
)
|
||||
})
|
||||
const consumedKeys = new Set(
|
||||
recommended.flatMap((option) => (option.installed ? [option.key] : [])),
|
||||
)
|
||||
const labels = new Set(recommended.map((option) => option.label))
|
||||
const discovered: InstalledSourceProviderOption[] = []
|
||||
|
||||
for (const plugin of datasourcePlugins) {
|
||||
const discoveredSourceType = sourceTypeForProviderType(plugin.declaration.provider_type)
|
||||
if (discoveredSourceType !== sourceType) continue
|
||||
for (const datasource of plugin.declaration.datasources) {
|
||||
const key = providerKey(sourceType, plugin, datasource)
|
||||
if (consumedKeys.has(key)) continue
|
||||
const rawLabel =
|
||||
datasource.identity.label.en_US ||
|
||||
plugin.declaration.identity.label.en_US ||
|
||||
datasource.identity.name
|
||||
const label = uniqueLabel(rawLabel, plugin.declaration.identity.label.en_US, labels)
|
||||
labels.add(label)
|
||||
discovered.push({
|
||||
datasource,
|
||||
fallbackIcon: fallbackIcon(sourceType),
|
||||
installed: true,
|
||||
key,
|
||||
label,
|
||||
packageId: plugin.plugin_id,
|
||||
plugin,
|
||||
providerType: plugin.declaration.provider_type,
|
||||
sourceType,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return [...recommended, ...discovered]
|
||||
}
|
||||
|
||||
export function sourceProviderOptionForDraft(
|
||||
options: SourceProviderOption[],
|
||||
draft: Pick<NewKnowledgeSourceDraft, 'provider' | 'providerKey'>,
|
||||
) {
|
||||
return (
|
||||
options.find((option) => option.key === draft.providerKey) ??
|
||||
options.find(
|
||||
(option) =>
|
||||
normalizeSourceProviderName(option.label) === normalizeSourceProviderName(draft.provider),
|
||||
) ??
|
||||
options[0]
|
||||
)
|
||||
}
|
||||
@ -95,7 +95,7 @@ export function SourceProviderRadioGroup<T extends string>({
|
||||
}: {
|
||||
disabled?: boolean
|
||||
layout: 'grid-four' | 'grid-three' | 'wrap'
|
||||
options: Array<{ disabled?: boolean; icon: ReactNode; value: T }>
|
||||
options: Array<{ disabled?: boolean; icon: ReactNode; label?: ReactNode; value: T }>
|
||||
size?: 'medium' | 'small'
|
||||
surface?: 'default' | 'transparent'
|
||||
value: T
|
||||
@ -127,7 +127,7 @@ export function SourceProviderRadioGroup<T extends string>({
|
||||
)}
|
||||
>
|
||||
{option.icon}
|
||||
<span className="truncate">{option.value}</span>
|
||||
<span className="truncate">{option.label ?? option.value}</span>
|
||||
</RadioItem>
|
||||
))}
|
||||
</RadioGroup>
|
||||
@ -274,3 +274,31 @@ export function SourceConnectionRequiredCard({
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
export function SourceProviderNotInstalledCard({
|
||||
icon,
|
||||
provider,
|
||||
onInstall,
|
||||
}: {
|
||||
icon: ReactNode
|
||||
provider: string
|
||||
onInstall: () => void
|
||||
}) {
|
||||
const { t: tPlugin } = useTranslation('plugin')
|
||||
const { t: tWorkflow } = useTranslation('workflow')
|
||||
|
||||
return (
|
||||
<section className="flex min-h-44 flex-col items-start gap-2.5 rounded-xl bg-background-section p-4">
|
||||
<span className="flex size-9 items-center justify-center rounded-lg border-[0.5px] border-divider-subtle bg-background-default">
|
||||
{icon}
|
||||
</span>
|
||||
<h3 className="system-sm-semibold text-text-primary">{provider}</h3>
|
||||
<p className="system-xs-regular text-text-tertiary">
|
||||
{tWorkflow(($) => $['nodes.common.pluginNotInstalled'])}
|
||||
</p>
|
||||
<Button type="button" variant="primary" className="mt-auto" onClick={onInstall}>
|
||||
{tPlugin(($) => $.installPlugin)}
|
||||
</Button>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user