From 62cebcf64dffc92fff3198d732eb6bb0dfcb94dd Mon Sep 17 00:00:00 2001 From: Stephen Zhou <38493346+hyoban@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:51:09 +0800 Subject: [PATCH] fix(web): discover source providers dynamically --- .../__tests__/add-source-page.spec.tsx | 41 ++- .../__tests__/connected-source-setup.spec.tsx | 19 ++ web/features/new-rag/__tests__/routes.spec.ts | 4 +- web/features/new-rag/add-source-page.tsx | 205 +++++--------- .../new-rag/connected-source-setup.tsx | 235 +++++++--------- web/features/new-rag/routes.ts | 46 ++- .../new-rag/source-provider-options.ts | 266 ++++++++++++++++++ web/features/new-rag/source-setup-fields.tsx | 32 ++- 8 files changed, 545 insertions(+), 303 deletions(-) create mode 100644 web/features/new-rag/source-provider-options.ts diff --git a/web/features/new-rag/__tests__/add-source-page.spec.tsx b/web/features/new-rag/__tests__/add-source-page.spec.tsx index 0530cf1a9aa..7f678e5cce3 100644 --- a/web/features/new-rag/__tests__/add-source-page.spec.tsx +++ b/web/features/new-rag/__tests__/add-source-page.spec.tsx @@ -495,6 +495,35 @@ const jinaDatasourceAuth: DatasourceProviderAuthListResponse['result'][number] = provider: 'jinareader', } +const customCrawlerDatasourcePlugin: DataSourceItem = { + ...firecrawlDatasourcePlugin, + declaration: { + ...firecrawlDatasourcePlugin.declaration, + datasources: [ + { + description: { en_US: 'Acme crawler', zh_Hans: 'Acme crawler' }, + identity: { + author: 'acme', + label: { en_US: 'Acme Crawler', zh_Hans: 'Acme Crawler' }, + name: 'acme_crawler', + provider: 'acme', + }, + parameters: [], + }, + ], + identity: { + ...firecrawlDatasourcePlugin.declaration.identity, + author: 'acme', + description: { en_US: 'Acme crawler', zh_Hans: 'Acme crawler' }, + label: { en_US: 'Acme Crawler', zh_Hans: 'Acme Crawler' }, + name: 'acme_crawler', + }, + }, + plugin_id: 'acme/acme_crawler', + plugin_unique_identifier: 'acme/acme_crawler:1.0.0@local', + provider: 'acme', +} + const connection = ( status: 'provisioning' | 'active' | 'expired' | 'error' | 'revoked', version = 2, @@ -698,10 +727,10 @@ describe('AddSourcePage', () => { expect(screen.getByRole('radio', { name: 'Jina Reader' })).toBeChecked() expect(providerHookOptionsMock.mock.lastCall?.[0]).toMatchObject({ enabled: true }) expect(connectionHookOptionsMock.mock.lastCall?.[0]).toMatchObject({ enabled: true }) - expect(screen.getByText('dataset.newKnowledge.providerUnavailable')).toBeInTheDocument() + expect(screen.getByText('workflow.nodes.common.pluginNotInstalled')).toBeInTheDocument() await user.click( screen.getByRole('button', { - name: 'dataset.newKnowledge.configureProvider:{"provider":"Jina Reader"}', + name: 'plugin.installPlugin', }), ) expect(openMock).toHaveBeenCalledWith( @@ -742,7 +771,7 @@ describe('AddSourcePage', () => { ).not.toBeInTheDocument() await user.click(screen.getByRole('radio', { name: 'Jina Reader' })) - expect(screen.getByText('dataset.newKnowledge.providerUnavailable')).toBeInTheDocument() + expect(screen.getByText('workflow.nodes.common.pluginNotInstalled')).toBeInTheDocument() expect( screen.queryByRole('textbox', { name: 'dataset.newKnowledge.rootUrl' }), ).not.toBeInTheDocument() @@ -1133,14 +1162,16 @@ describe('AddSourcePage', () => { expect(routerMock.push).toHaveBeenCalledWith('/datasets/new/space-1/sources') }) - it('renders every designed website provider and the provider-management action', async () => { + it('discovers installed website providers and keeps the provider-management action', async () => { const user = userEvent.setup() + queryState.datasourcePlugins.data = [firecrawlDatasourcePlugin, customCrawlerDatasourcePlugin] render() expect( screen.getByRole('group', { name: 'dataset.newKnowledge.providerLabel' }), ).toBeInTheDocument() - expect(screen.getByRole('radio', { name: 'FakeCrawler' })).toBeEnabled() + expect(screen.getByRole('radio', { name: 'Acme Crawler' })).toBeEnabled() + expect(screen.queryByRole('radio', { name: 'FakeCrawler' })).not.toBeInTheDocument() await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.moreProviders' })) expect(openMock).toHaveBeenCalledWith( '/integrations/data-source', diff --git a/web/features/new-rag/__tests__/connected-source-setup.spec.tsx b/web/features/new-rag/__tests__/connected-source-setup.spec.tsx index e1102578e62..7191c8afe0d 100644 --- a/web/features/new-rag/__tests__/connected-source-setup.spec.tsx +++ b/web/features/new-rag/__tests__/connected-source-setup.spec.tsx @@ -552,6 +552,25 @@ describe('ConnectedSourceSetup', () => { ) }) + it('distinguishes an uninstalled provider from an installed provider without credentials', async () => { + const user = userEvent.setup() + renderSetup({ + ...defaultDraft, + provider: 'Confluence', + }) + + expect(await screen.findByText('workflow.nodes.common.pluginNotInstalled')).toBeInTheDocument() + expect(screen.queryByText('dataset.newKnowledge.notionNotConnected')).not.toBeInTheDocument() + expect(clientMock.createConnection).not.toHaveBeenCalled() + + await user.click(screen.getByRole('button', { name: 'plugin.installPlugin' })) + expect(openMock).toHaveBeenCalledWith( + '/integrations/data-source?package-ids=%5B%22langgenius%2Fconfluence_datasource%22%5D', + '_blank', + 'noopener,noreferrer', + ) + }) + it('starts the selected Notion import and completes setup without waiting for indexing', async () => { const user = userEvent.setup() clientMock.listDatasourceAuth.mockResolvedValue({ diff --git a/web/features/new-rag/__tests__/routes.spec.ts b/web/features/new-rag/__tests__/routes.spec.ts index bdf2f1c886f..e4856fd909f 100644 --- a/web/features/new-rag/__tests__/routes.spec.ts +++ b/web/features/new-rag/__tests__/routes.spec.ts @@ -35,8 +35,8 @@ describe('New RAG routes', () => { ).toBe('/datasets/new/space-1/sources/new?type=websiteCrawl&provider=Jina+Reader') }) - it('falls back when a provider does not belong to the selected source type', () => { - expect(createNewKnowledgeSourceDraft('onlineDrive', 'Confluence').provider).toBe('Google Drive') + it('keeps a dynamically discovered provider supplied by the add-source entry point', () => { + expect(createNewKnowledgeSourceDraft('onlineDrive', 'Acme Drive').provider).toBe('Acme Drive') }) it('defaults website crawls to a daily sync policy', () => { diff --git a/web/features/new-rag/add-source-page.tsx b/web/features/new-rag/add-source-page.tsx index abce56752ba..1daadb5c645 100644 --- a/web/features/new-rag/add-source-page.tsx +++ b/web/features/new-rag/add-source-page.tsx @@ -1,13 +1,9 @@ 'use client' import type { DatasourceProviderAuthListResponse } from '@dify/contracts/api/console/auth/types.gen' -import type { - NewKnowledgeSourceDraft, - NewKnowledgeSourceType, - NewKnowledgeWebsiteProvider, -} from './routes' +import type { NewKnowledgeSourceDraft, NewKnowledgeSourceType } from './routes' import type { SourceConnection as Connection, SourceProvider as Provider } from './source-models' -import type { DataSourceItem } from '@/app/components/workflow/block-selector/types' +import type { InstalledSourceProviderOption, SourceProviderOption } from './source-provider-options' import { Button } from '@langgenius/dify-ui/button' import { Field, FieldControl, FieldDescription, FieldLabel } from '@langgenius/dify-ui/field' import { Fieldset, FieldsetLegend } from '@langgenius/dify-ui/fieldset' @@ -43,7 +39,15 @@ import { sourceConnectionListFromApi, sourceProviderListFromApi, } from './source-models' -import { SourceProviderRadioGroup, SourceTypeSelector } from './source-setup-fields' +import { + discoverSourceProviderOptions, + sourceProviderOptionForDraft, +} from './source-provider-options' +import { + SourceProviderNotInstalledCard, + SourceProviderRadioGroup, + SourceTypeSelector, +} from './source-setup-fields' import { WebsiteCrawlPreview } from './website-crawl-preview' type ProviderField = Provider['configuration'][number] @@ -52,37 +56,6 @@ type SourceType = NewKnowledgeSourceType const CONNECTION_PAGE_SIZE = 200 const WEBSITE_SOURCE_PROVIDER_ID = 'plugin-daemon-website' -const WEBSITE_PROVIDER_OPTIONS: Array<{ - aliases: string[] - icon: string - value: NewKnowledgeWebsiteProvider -}> = [ - { - aliases: ['firecrawl'], - icon: 'i-custom-public-common-firecrawl', - value: 'Firecrawl', - }, - { - aliases: ['jina', 'jina reader', 'jinareader'], - icon: 'i-custom-public-llm-jina', - value: 'Jina Reader', - }, - { - aliases: ['watercrawl'], - icon: 'i-custom-public-knowledge-watercrawl', - value: 'WaterCrawl', - }, - { - aliases: ['fakecrawler'], - icon: 'i-ri-global-line text-text-accent', - value: 'FakeCrawler', - }, -] -const WEBSITE_PROVIDER_PACKAGE_IDS: Partial> = { - Firecrawl: 'langgenius/firecrawl_datasource', - 'Jina Reader': 'langgenius/jina_datasource', - WaterCrawl: 'watercrawl/watercrawl_datasource', -} const MANAGED_PROVIDER_FIELD_NAMES = new Set([ 'credentialId', 'datasource', @@ -110,20 +83,10 @@ function fieldValue(value: string, type: ProviderField['type']) { return value.trim() } -function normalizeProviderName(value: string) { - return value.toLocaleLowerCase().replace(/[^a-z0-9]+/g, '') -} - -function websiteProviderOption(provider: NewKnowledgeWebsiteProvider) { - return WEBSITE_PROVIDER_OPTIONS.find((option) => option.value === provider) -} - -function websiteProviderIntegrationPath(provider?: NewKnowledgeWebsiteProvider) { +function websiteProviderIntegrationPath(provider?: SourceProviderOption) { const base = buildIntegrationPath('data-source') if (!provider) return base - const packageId = WEBSITE_PROVIDER_PACKAGE_IDS[provider] - if (!packageId) return base - const query = new URLSearchParams({ 'package-ids': JSON.stringify([packageId]) }) + const query = new URLSearchParams({ 'package-ids': JSON.stringify([provider.packageId]) }) return `${base}?${query.toString()}` } @@ -131,38 +94,7 @@ function findWebsiteSourceProvider(providers: Provider[]) { return providers.find((provider) => provider.id === WEBSITE_SOURCE_PROVIDER_ID) } -function findWebsiteDatasourceProvider( - datasourcePlugins: DataSourceItem[], - providerName: NewKnowledgeWebsiteProvider, -) { - if (providerName === 'FakeCrawler') return undefined - const option = websiteProviderOption(providerName) - if (!option) return undefined - const aliases = [option.value, ...option.aliases].map(normalizeProviderName) - for (const plugin of datasourcePlugins) { - if (plugin.declaration.provider_type !== 'website_crawl') continue - const pluginIdentities = [ - plugin.declaration.identity.label.en_US, - plugin.declaration.identity.name, - plugin.plugin_id, - plugin.provider, - ].map(normalizeProviderName) - const datasource = plugin.declaration.datasources.find((candidate) => { - const datasourceIdentities = [ - candidate.identity.label.en_US, - candidate.identity.name, - candidate.identity.provider, - ].map(normalizeProviderName) - return [...pluginIdentities, ...datasourceIdentities].some((identity) => - aliases.some((alias) => identity.includes(alias) || alias.includes(identity)), - ) - }) - if (datasource) return { datasource, plugin } - } - return undefined -} - -type WebsiteDatasourceProvider = NonNullable> +type WebsiteDatasourceProvider = InstalledSourceProviderOption function findDatasourceAuth( providers: DatasourceProviderAuthListResponse['result'], @@ -278,13 +210,15 @@ function getSupportedAuthKinds(provider: Provider, credentialId?: string) { function ProviderSelector({ disabled = false, onMoreProviders, - provider, + options, + providerKey, onChange, }: { disabled?: boolean onMoreProviders: () => void - provider: NewKnowledgeWebsiteProvider - onChange: (provider: NewKnowledgeWebsiteProvider) => void + options: SourceProviderOption[] + providerKey: string + onChange: (providerKey: string) => void }) { const { t } = useTranslation('dataset') @@ -307,19 +241,30 @@ function ProviderSelector({ ({ - icon: , - value: option.value, + layout="wrap" + options={options.map((option) => ({ + icon: , + label: option.label, + value: option.key, }))} + size="small" onChange={onChange} /> ) } +function WebsiteProviderIcon({ option }: { option: SourceProviderOption }) { + const icon = option.installed + ? (option.datasource.identity.icon ?? option.plugin.declaration.identity.icon) + : undefined + if (typeof icon === 'string' && icon) + return + return +} + function ProviderFieldControl({ field, setValues, @@ -421,7 +366,7 @@ function ConnectionForm({ onDraftChange: (dirty: boolean) => void onReconcile: () => Promise provider: Provider - providerName: NewKnowledgeWebsiteProvider + providerName: string credentialId?: string }) { const { t } = useTranslation('dataset') @@ -582,7 +527,7 @@ function ManagedProviderConnection({ onConnected: (connection: Connection) => void onReconcile: () => Promise provider: Provider - providerName: NewKnowledgeWebsiteProvider + providerName: string }) { const { t } = useTranslation('dataset') const { t: tCommon } = useTranslation('common') @@ -687,6 +632,7 @@ function UnconfiguredProvider({ onDraftChange, onReconcile, provider, + providerOption, providerName, credentialId, }: { @@ -697,7 +643,8 @@ function UnconfiguredProvider({ onDraftChange: (dirty: boolean) => void onReconcile: () => Promise provider: Provider - providerName: NewKnowledgeWebsiteProvider + providerOption: InstalledSourceProviderOption + providerName: string credentialId?: string }) { const { t } = useTranslation('dataset') @@ -733,10 +680,7 @@ function UnconfiguredProvider({ return (
- +

{t(($) => $['newKnowledge.providerNotConfigured'], { @@ -993,12 +937,13 @@ export function AddSourcePage({ }), ) const provider = findWebsiteSourceProvider(providersQuery.data ?? []) - const websiteProviderName = - sourceDraft.sourceType === 'websiteCrawl' ? sourceDraft.provider : 'Firecrawl' - const datasourceProvider = useMemo( - () => findWebsiteDatasourceProvider(datasourcePluginsQuery.data ?? [], websiteProviderName), - [datasourcePluginsQuery.data, websiteProviderName], + const websiteProviderOptions = useMemo( + () => discoverSourceProviderOptions('websiteCrawl', datasourcePluginsQuery.data ?? []), + [datasourcePluginsQuery.data], ) + const websiteProviderOption = sourceProviderOptionForDraft(websiteProviderOptions, sourceDraft) + const websiteProviderName = websiteProviderOption?.label ?? sourceDraft.provider + const datasourceProvider = websiteProviderOption?.installed ? websiteProviderOption : undefined const datasourceProviders = datasourceAuthQuery.data?.result ?? [] const datasourceCredential = findDatasourceCredential(datasourceProviders, datasourceProvider) const difyManagedProvider = provider ? isDifyManagedProvider(provider) : false @@ -1368,12 +1313,21 @@ export function AddSourcePage({ <> globalThis.open(websiteProviderIntegrationPath(), '_blank', 'noopener,noreferrer') } - onChange={(provider) => { - updateSourceDraft({ ...sourceDraft, provider }) + onChange={(providerKey) => { + const nextProvider = websiteProviderOptions.find( + (option) => option.key === providerKey, + ) + if (!nextProvider) return + updateSourceDraft({ + ...sourceDraft, + provider: nextProvider.label, + providerKey: nextProvider.key, + }) }} /> {queryError ? ( @@ -1395,34 +1349,22 @@ export function AddSourcePage({ {t(($) => $['newKnowledge.retryProviderLoad'])}

- ) : !provider ? ( + ) : websiteProviderOption && !websiteProviderOption.installed ? ( + } + provider={websiteProviderOption.label} + onInstall={() => + globalThis.open( + websiteProviderIntegrationPath(websiteProviderOption), + '_blank', + 'noopener,noreferrer', + ) + } + /> + ) : !datasourceProvider || !provider ? (
{t(($) => $['newKnowledge.providerUnavailable'])}
- ) : !datasourceProvider ? ( -
-

{websiteProviderName}

-

- {t(($) => $['newKnowledge.providerUnavailable'])} -

- {websiteProviderName !== 'FakeCrawler' && ( - - )} -
) : !provider.available || !supportsDirectConnection ? (

{websiteProviderName}

@@ -1462,7 +1404,7 @@ export function AddSourcePage({ onConnected={rememberConnection} onConfigureManagedProvider={() => globalThis.open( - websiteProviderIntegrationPath(websiteProviderName), + websiteProviderIntegrationPath(websiteProviderOption), '_blank', 'noopener,noreferrer', ) @@ -1470,6 +1412,7 @@ export function AddSourcePage({ onDraftChange={setConnectionDraftDirty} onReconcile={reconcileConnection} provider={provider} + providerOption={datasourceProvider} providerName={websiteProviderName} /> )} diff --git a/web/features/new-rag/connected-source-setup.tsx b/web/features/new-rag/connected-source-setup.tsx index bcd5d3d513c..25eacd3cbb2 100644 --- a/web/features/new-rag/connected-source-setup.tsx +++ b/web/features/new-rag/connected-source-setup.tsx @@ -5,13 +5,12 @@ import type { KnowledgeFsSourcePageResponse, } from '@dify/contracts/api/console/knowledge-fs/types.gen' import type { - NewKnowledgeOnlineDocumentsProvider, NewKnowledgeOnlineDocumentsSourceDraft, - NewKnowledgeOnlineDriveProvider, NewKnowledgeOnlineDriveSourceDraft, NewKnowledgeSourceDraft, } from './routes' import type { Source, SourceConnection, SourceProvider } from './source-models' +import type { InstalledSourceProviderOption, SourceProviderOption } from './source-provider-options' import type { DataSourceAuth, DataSourceCredential, @@ -39,9 +38,15 @@ import { sourceProviderListFromApi, sourceSyncPolicyFromApi, } from './source-models' +import { + discoverSourceProviderOptions, + normalizeSourceProviderName, + sourceProviderOptionForDraft, +} from './source-provider-options' import { SourceConnectionRequiredCard, SourceNameField, + SourceProviderNotInstalledCard, SourceProviderRadioGroup, SourceSyncPolicyField, } from './source-setup-fields' @@ -96,73 +101,25 @@ const CONNECTION_STATUS_PRIORITY: Record = { revoked: 4, } -const providerOptions = { - onlineDocuments: [ - { - aliases: ['notion'], - icon: 'i-custom-public-common-notion', - label: 'Notion', - }, - { - aliases: ['google docs', 'googledocs', 'google drive', 'googledrive'], - icon: 'i-ri-file-text-fill text-[#4d8bf5]', - label: 'Google Docs', - }, - { - aliases: ['confluence'], - icon: 'i-custom-public-common-confluence', - label: 'Confluence', - }, - ], - onlineDrive: [ - { - aliases: ['google drive', 'googledrive'], - icon: 'i-custom-public-common-google-drive', - label: 'Google Drive', - }, - { - aliases: ['onedrive', 'microsoft onedrive'], - icon: 'i-ri-cloud-line', - label: 'OneDrive', - }, - { - aliases: ['amazon s3', 'amazons3', 's3'], - icon: 'i-ri-box-3-line', - label: 'Amazon S3', - }, - ], -} as const - -const providerPackageIds: Record = { - 'Amazon S3': 'langgenius/aws_s3_storage', - Confluence: 'langgenius/confluence_datasource', - 'Google Docs': 'langgenius/google_drive', - 'Google Drive': 'langgenius/google_drive', - Notion: 'langgenius/notion_datasource', - OneDrive: 'langgenius/onedrive_datasource', -} - -function normalizeProviderName(value: string) { - return value.toLocaleLowerCase().replace(/[^a-z0-9]+/g, '') -} - function usesDriveTransport(draft: ConnectedSourceDraft) { return draft.sourceType === 'onlineDrive' || draft.provider === 'Google Docs' } -function providerForDraft(providers: SourceProvider[], draft: ConnectedSourceDraft) { +function providerForDraft( + providers: SourceProvider[], + draft: ConnectedSourceDraft, + option?: SourceProviderOption, +) { const capability = usesDriveTransport(draft) ? 'online-drive' : 'online-document' - const option = providerOptions[draft.sourceType].find((item) => item.label === draft.provider) - if (!option) return undefined const aliases = new Set( - [option.label, ...option.aliases].map((candidate) => normalizeProviderName(candidate)), + [draft.provider, option?.label ?? ''].map(normalizeSourceProviderName).filter(Boolean), ) const capableProviders = providers.filter((provider) => provider.capabilities.includes(capability), ) return ( capableProviders.find((provider) => { - const names = [provider.id, provider.displayName].map(normalizeProviderName) + const names = [provider.id, provider.displayName].map(normalizeSourceProviderName) return names.some( (name) => aliases.has(name) || @@ -176,45 +133,19 @@ function providerForDraft(providers: SourceProvider[], draft: ConnectedSourceDra ) } -function datasourceProviderForDraft( - datasourcePlugins: DataSourceItem[], - draft: ConnectedSourceDraft, -) { - const option = providerOptions[draft.sourceType].find((item) => item.label === draft.provider) - if (!option) return undefined - const aliases = [option.label, ...option.aliases].map(normalizeProviderName) - const providerType = usesDriveTransport(draft) ? 'online_drive' : 'online_document' - for (const plugin of datasourcePlugins) { - if (plugin.declaration.provider_type !== providerType) continue - const identities = [ - plugin.declaration.identity.label.en_US, - plugin.declaration.identity.name, - plugin.plugin_id, - plugin.provider, - ].map(normalizeProviderName) - const datasource = plugin.declaration.datasources.find((action) => { - const actionIdentities = [ - action.identity.label.en_US, - action.identity.name, - action.identity.provider, - ].map(normalizeProviderName) - return [...identities, ...actionIdentities].some((identity) => - aliases.some((alias) => identity.includes(alias) || alias.includes(identity)), - ) - }) - if (datasource) - return { - datasource, - plugin, +function datasourceProviderForOption(option?: SourceProviderOption) { + return option?.installed + ? { + datasource: option.datasource, + plugin: option.plugin, } - } - return undefined + : undefined } type ProviderBrandIconValue = DataSourceItem['declaration']['identity']['icon'] function datasourceProviderIcon( - datasourceProvider: ReturnType, + datasourceProvider: ReturnType, ): ProviderBrandIconValue | undefined { return datasourceProvider?.plugin.declaration.identity.icon } @@ -222,7 +153,7 @@ function datasourceProviderIcon( function credentialRegion(credential: DataSourceCredential | undefined) { const region = Object.entries(credential?.credential ?? {}).find( ([key, value]) => - ['awsregion', 'region', 'regionname'].includes(normalizeProviderName(key)) && + ['awsregion', 'region', 'regionname'].includes(normalizeSourceProviderName(key)) && typeof value === 'string' && value.trim(), )?.[1] @@ -231,7 +162,7 @@ function credentialRegion(credential: DataSourceCredential | undefined) { function datasourceAuthForProvider( providers: DataSourceAuth[], - datasourceProvider: ReturnType, + datasourceProvider: ReturnType, ) { if (!datasourceProvider) return undefined return providers.find( @@ -250,7 +181,7 @@ function preferredCredential(provider?: DataSourceAuth) { function connectionMatchesDatasource( connection: SourceConnection, - datasourceProvider: ReturnType, + datasourceProvider: ReturnType, credential: DataSourceCredential | undefined, ) { if (!datasourceProvider) return false @@ -266,7 +197,7 @@ function connectionMatchesDatasource( function findProviderConnection( connections: SourceConnection[], providerId: string | undefined, - datasourceProvider: ReturnType, + datasourceProvider: ReturnType, credential: DataSourceCredential | undefined, ) { if (!providerId || !datasourceProvider) return undefined @@ -285,16 +216,15 @@ function findProviderConnection( )[0] } -function providerIntegrationPath(draft: ConnectedSourceDraft) { +function providerIntegrationPath(option?: SourceProviderOption) { const base = buildIntegrationPath('data-source') - const packageId = providerPackageIds[draft.provider] - if (!packageId) return base - const query = new URLSearchParams({ 'package-ids': JSON.stringify([packageId]) }) + if (!option) return base + const query = new URLSearchParams({ 'package-ids': JSON.stringify([option.packageId]) }) return `${base}?${query.toString()}` } function providerScheme(providerName: string) { - const normalized = normalizeProviderName(providerName) + const normalized = normalizeSourceProviderName(providerName) if (normalized.includes('notion')) return 'notion' if (normalized.includes('googledocs')) return 'gdocs' if (normalized.includes('googledrive')) return 'gdrive' @@ -312,7 +242,7 @@ function sourceUri( const scheme = providerScheme(`${draft.provider} ${provider.id} ${provider.displayName}`) if (scheme === 's3') { const bucket = Object.entries(connection.configuration).find(([key]) => - ['bucket', 'bucketname'].includes(normalizeProviderName(key)), + ['bucket', 'bucketname'].includes(normalizeSourceProviderName(key)), )?.[1] if (typeof bucket === 'string' && bucket.trim()) return `s3://${bucket.trim()}` } @@ -386,18 +316,17 @@ function requestStatus(error: unknown) { } function ProviderSelector({ - datasourcePlugins, - draft, + options, + providerKey, onChange, onMoreProviders, }: { - datasourcePlugins: DataSourceItem[] - draft: ConnectedSourceDraft - onChange: (provider: string) => void + options: SourceProviderOption[] + providerKey: string + onChange: (providerKey: string) => void onMoreProviders: () => void }) { const { t } = useTranslation('dataset') - const options = providerOptions[draft.sourceType] return (
@@ -416,27 +345,22 @@ function ProviderSelector({
{ - const optionProvider = datasourceProviderForDraft(datasourcePlugins, { - ...draft, - provider: option.label, - } as ConnectedSourceDraft) - return { - icon: ( - - ), - value: option.label, - } - })} + options={options.map((option) => ({ + icon: ( + + ), + label: option.label, + value: option.key, + }))} size="small" onChange={onChange} /> @@ -471,14 +395,15 @@ function ProviderBrandIcon({ function OAuthConnectionCard({ draft, icon, + providerOption, onConnect, }: { draft: ConnectedSourceDraft icon?: ProviderBrandIconValue + providerOption: InstalledSourceProviderOption onConnect: () => void }) { const { t } = useTranslation('dataset') - const option = providerOptions[draft.sourceType].find((item) => item.label === draft.provider) return ( $['newKnowledge.connectProvider'], { provider: draft.provider })} @@ -489,7 +414,7 @@ function OAuthConnectionCard({ provider: draft.provider, }) } - icon={} + icon={} title={ draft.provider === 'Notion' ? t(($) => $['newKnowledge.notionNotConnected']) @@ -1707,9 +1632,15 @@ export function ConnectedSourceSetup({ retry: false, }), ) - const provider = providerForDraft(providersQuery.data ?? [], draft) + const providerOptions = useMemo( + () => discoverSourceProviderOptions(draft.sourceType, datasourcePluginsQuery.data ?? []), + [datasourcePluginsQuery.data, draft.sourceType], + ) + const providerOption = sourceProviderOptionForDraft(providerOptions, draft) + const installedProviderOption = providerOption?.installed ? providerOption : undefined + const provider = providerForDraft(providersQuery.data ?? [], draft, providerOption) const driveTransport = usesDriveTransport(draft) - const datasourceProvider = datasourceProviderForDraft(datasourcePluginsQuery.data ?? [], draft) + const datasourceProvider = datasourceProviderForOption(installedProviderOption) const datasourceAuth = datasourceAuthForProvider( datasourceAuthQuery.data?.result ?? [], datasourceProvider, @@ -1866,23 +1797,27 @@ export function ConnectedSourceSetup({ providersQuery.isPending, provisionConnection, ]) - const selectProvider = (nextProvider: string) => { + const selectProvider = (providerKey: string) => { + const nextProvider = providerOptions.find((option) => option.key === providerKey) + if (!nextProvider) return setConnectionOverride(undefined) setProvisionError(false) if (draft.sourceType === 'onlineDocuments') { onDraftChange({ ...draft, - provider: nextProvider as NewKnowledgeOnlineDocumentsProvider, + provider: nextProvider.label, + providerKey: nextProvider.key, sourceName: '', }) return } onDraftChange({ ...draft, - provider: nextProvider as NewKnowledgeOnlineDriveProvider, + provider: nextProvider.label, + providerKey: nextProvider.key, sourceName: '', syncPolicy: - nextProvider === 'Amazon S3' && draft.syncPolicy === 'provider' + nextProvider.label === 'Amazon S3' && draft.syncPolicy === 'provider' ? 'daily' : draft.syncPolicy, }) @@ -1890,8 +1825,8 @@ export function ConnectedSourceSetup({ return (
globalThis.open(buildIntegrationPath('data-source'), '_blank', 'noopener,noreferrer') @@ -1923,7 +1858,19 @@ export function ConnectedSourceSetup({ {t(($) => $['newKnowledge.retryProviderLoad'])}
- ) : !provider ? ( + ) : providerOption && !providerOption.installed ? ( + } + provider={providerOption.label} + onInstall={() => + globalThis.open( + providerIntegrationPath(providerOption), + '_blank', + 'noopener,noreferrer', + ) + } + /> + ) : !installedProviderOption || !provider ? (

{draft.provider}

@@ -1977,7 +1924,11 @@ export function ConnectedSourceSetup({ + + ) +}