From 357548ca07b9c470b5c9a777edef0769b5e47219 Mon Sep 17 00:00:00 2001 From: yyh <92089059+lyzno1@users.noreply.github.com> Date: Wed, 7 Jan 2026 10:25:52 +0800 Subject: [PATCH 01/21] chore: rename ralph-wiggum plugin to ralph-loop (#30664) Co-authored-by: Claude Opus 4.5 --- .claude/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/settings.json b/.claude/settings.json index c5c514b5f5..509dbe8447 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -4,6 +4,6 @@ "context7@claude-plugins-official": true, "typescript-lsp@claude-plugins-official": true, "pyright-lsp@claude-plugins-official": true, - "ralph-wiggum@claude-plugins-official": true + "ralph-loop@claude-plugins-official": true } } From e335cd0ef42ffafecd31880730a66b2e757677de Mon Sep 17 00:00:00 2001 From: Stephen Zhou <38493346+hyoban@users.noreply.github.com> Date: Wed, 7 Jan 2026 13:20:09 +0800 Subject: [PATCH 02/21] refactor(web): remove useMixedTranslation, better resource loading (#30630) Co-authored-by: Claude Opus 4.5 --- .../search-error-handling.test.ts | 8 +++ web/app/(commonLayout)/plugins/page.tsx | 4 +- .../chat/chat-with-history/hooks.spec.tsx | 2 +- .../base/chat/chat-with-history/hooks.tsx | 2 +- .../base/chat/embedded-chatbot/hooks.spec.tsx | 2 +- .../base/chat/embedded-chatbot/hooks.tsx | 2 +- .../actions/commands/account.tsx | 3 +- .../actions/commands/community.tsx | 3 +- .../goto-anything/actions/commands/docs.tsx | 5 +- .../goto-anything/actions/commands/forum.tsx | 3 +- .../actions/commands/language.tsx | 3 +- .../goto-anything/actions/commands/slash.tsx | 4 +- .../goto-anything/actions/commands/theme.tsx | 3 +- .../goto-anything/actions/commands/zen.tsx | 3 +- .../install-from-marketplace.tsx | 3 - .../account-setting/language-page/index.tsx | 6 +- .../install-from-marketplace.tsx | 3 - web/app/components/i18n-server.tsx | 22 ------ web/app/components/i18n.tsx | 45 ------------- .../plugins/base/deprecation-notice.tsx | 6 +- .../components/plugins/card/index.spec.tsx | 25 ------- web/app/components/plugins/card/index.tsx | 13 ++-- web/app/components/plugins/hooks.ts | 11 ++- .../plugins/marketplace/empty/index.spec.tsx | 45 ++----------- .../plugins/marketplace/empty/index.tsx | 6 +- .../components/plugins/marketplace/hooks.ts | 17 ----- .../plugins/marketplace/index.spec.tsx | 54 --------------- .../components/plugins/marketplace/index.tsx | 5 -- .../plugins/marketplace/list/card-wrapper.tsx | 18 ++--- .../plugins/marketplace/list/index.spec.tsx | 65 +++--------------- .../plugins/marketplace/list/index.tsx | 7 +- .../marketplace/list/list-with-collection.tsx | 9 +-- .../plugins/marketplace/list/list-wrapper.tsx | 10 +-- .../marketplace/plugin-type-switch.tsx | 6 +- .../marketplace/search-box/index.spec.tsx | 33 +-------- .../plugins/marketplace/search-box/index.tsx | 4 -- .../search-box/search-box-wrapper.tsx | 12 +--- .../marketplace/search-box/tags-filter.tsx | 9 +-- .../search-box/trigger/marketplace.tsx | 6 +- .../marketplace/sort-dropdown/index.spec.tsx | 43 +----------- .../marketplace/sort-dropdown/index.tsx | 11 +-- .../sticky-search-and-switch-wrapper.tsx | 5 +- .../components/plugins/plugin-item/index.tsx | 2 +- web/app/components/provider/i18n-server.tsx | 21 ++++++ web/app/components/provider/i18n.tsx | 24 +++++++ .../share/text-generation/index.tsx | 2 +- .../tools/marketplace/index.spec.tsx | 6 -- .../components/tools/marketplace/index.tsx | 5 +- web/app/layout.tsx | 15 +++-- web/context/i18n.ts | 6 +- web/i18n-config/client.ts | 35 ++++++++++ web/i18n-config/index.ts | 6 +- web/i18n-config/lib.client.ts | 2 +- web/i18n-config/lib.server.ts | 2 +- .../{i18next-config.ts => resources.ts} | 67 ++----------------- web/i18n-config/server.ts | 24 +++++-- web/i18n-config/settings.ts | 13 ++++ web/types/i18n.d.ts | 2 +- 58 files changed, 230 insertions(+), 548 deletions(-) delete mode 100644 web/app/components/i18n-server.tsx delete mode 100644 web/app/components/i18n.tsx create mode 100644 web/app/components/provider/i18n-server.tsx create mode 100644 web/app/components/provider/i18n.tsx create mode 100644 web/i18n-config/client.ts rename web/i18n-config/{i18next-config.ts => resources.ts} (59%) create mode 100644 web/i18n-config/settings.ts diff --git a/web/__tests__/goto-anything/search-error-handling.test.ts b/web/__tests__/goto-anything/search-error-handling.test.ts index 3a495834cd..42eb829583 100644 --- a/web/__tests__/goto-anything/search-error-handling.test.ts +++ b/web/__tests__/goto-anything/search-error-handling.test.ts @@ -14,6 +14,14 @@ import { fetchAppList } from '@/service/apps' import { postMarketplace } from '@/service/base' import { fetchDatasets } from '@/service/datasets' +// Mock react-i18next before importing modules that use it +vi.mock('react-i18next', () => ({ + getI18n: () => ({ + t: (key: string) => key, + language: 'en', + }), +})) + // Mock API functions vi.mock('@/service/base', () => ({ postMarketplace: vi.fn(), diff --git a/web/app/(commonLayout)/plugins/page.tsx b/web/app/(commonLayout)/plugins/page.tsx index 2df9cf23c4..81bda3a8a3 100644 --- a/web/app/(commonLayout)/plugins/page.tsx +++ b/web/app/(commonLayout)/plugins/page.tsx @@ -1,14 +1,12 @@ import Marketplace from '@/app/components/plugins/marketplace' import PluginPage from '@/app/components/plugins/plugin-page' import PluginsPanel from '@/app/components/plugins/plugin-page/plugins-panel' -import { getLocaleOnServer } from '@/i18n-config/server' const PluginList = async () => { - const locale = await getLocaleOnServer() return ( } - marketplace={} + marketplace={} /> ) } diff --git a/web/app/components/base/chat/chat-with-history/hooks.spec.tsx b/web/app/components/base/chat/chat-with-history/hooks.spec.tsx index 32ef133453..a6d51d8643 100644 --- a/web/app/components/base/chat/chat-with-history/hooks.spec.tsx +++ b/web/app/components/base/chat/chat-with-history/hooks.spec.tsx @@ -17,7 +17,7 @@ vi.mock('@/hooks/use-app-favicon', () => ({ useAppFavicon: vi.fn(), })) -vi.mock('@/i18n-config/i18next-config', () => ({ +vi.mock('@/i18n-config/client', () => ({ changeLanguage: vi.fn().mockResolvedValue(undefined), })) diff --git a/web/app/components/base/chat/chat-with-history/hooks.tsx b/web/app/components/base/chat/chat-with-history/hooks.tsx index 5ff8e61ff6..ed1981b530 100644 --- a/web/app/components/base/chat/chat-with-history/hooks.tsx +++ b/web/app/components/base/chat/chat-with-history/hooks.tsx @@ -25,7 +25,7 @@ import { useToastContext } from '@/app/components/base/toast' import { InputVarType } from '@/app/components/workflow/types' import { useWebAppStore } from '@/context/web-app-context' import { useAppFavicon } from '@/hooks/use-app-favicon' -import { changeLanguage } from '@/i18n-config/i18next-config' +import { changeLanguage } from '@/i18n-config/client' import { delConversation, pinConversation, diff --git a/web/app/components/base/chat/embedded-chatbot/hooks.spec.tsx b/web/app/components/base/chat/embedded-chatbot/hooks.spec.tsx index ca6a90c4d8..066fb8ebe9 100644 --- a/web/app/components/base/chat/embedded-chatbot/hooks.spec.tsx +++ b/web/app/components/base/chat/embedded-chatbot/hooks.spec.tsx @@ -13,7 +13,7 @@ import { shareQueryKeys } from '@/service/use-share' import { CONVERSATION_ID_INFO } from '../constants' import { useEmbeddedChatbot } from './hooks' -vi.mock('@/i18n-config/i18next-config', () => ({ +vi.mock('@/i18n-config/client', () => ({ changeLanguage: vi.fn().mockResolvedValue(undefined), })) diff --git a/web/app/components/base/chat/embedded-chatbot/hooks.tsx b/web/app/components/base/chat/embedded-chatbot/hooks.tsx index 803e905837..9028d10000 100644 --- a/web/app/components/base/chat/embedded-chatbot/hooks.tsx +++ b/web/app/components/base/chat/embedded-chatbot/hooks.tsx @@ -23,7 +23,7 @@ import { useToastContext } from '@/app/components/base/toast' import { addFileInfos, sortAgentSorts } from '@/app/components/tools/utils' import { InputVarType } from '@/app/components/workflow/types' import { useWebAppStore } from '@/context/web-app-context' -import { changeLanguage } from '@/i18n-config/i18next-config' +import { changeLanguage } from '@/i18n-config/client' import { updateFeedback } from '@/service/share' import { useInvalidateShareConversations, diff --git a/web/app/components/goto-anything/actions/commands/account.tsx b/web/app/components/goto-anything/actions/commands/account.tsx index 6465932a75..d1fa36b6f0 100644 --- a/web/app/components/goto-anything/actions/commands/account.tsx +++ b/web/app/components/goto-anything/actions/commands/account.tsx @@ -1,7 +1,7 @@ import type { SlashCommandHandler } from './types' import { RiUser3Line } from '@remixicon/react' import * as React from 'react' -import i18n from '@/i18n-config/i18next-config' +import { getI18n } from 'react-i18next' import { registerCommands, unregisterCommands } from './command-bus' // Account command dependency types - no external dependencies needed @@ -21,6 +21,7 @@ export const accountCommand: SlashCommandHandler = { }, async search(args: string, locale: string = 'en') { + const i18n = getI18n() return [{ id: 'account', title: i18n.t('account.account', { ns: 'common', lng: locale }), diff --git a/web/app/components/goto-anything/actions/commands/community.tsx b/web/app/components/goto-anything/actions/commands/community.tsx index fcd9a15000..685149402d 100644 --- a/web/app/components/goto-anything/actions/commands/community.tsx +++ b/web/app/components/goto-anything/actions/commands/community.tsx @@ -1,7 +1,7 @@ import type { SlashCommandHandler } from './types' import { RiDiscordLine } from '@remixicon/react' import * as React from 'react' -import i18n from '@/i18n-config/i18next-config' +import { getI18n } from 'react-i18next' import { registerCommands, unregisterCommands } from './command-bus' // Community command dependency types @@ -22,6 +22,7 @@ export const communityCommand: SlashCommandHandler = { }, async search(args: string, locale: string = 'en') { + const i18n = getI18n() return [{ id: 'community', title: i18n.t('userProfile.community', { ns: 'common', lng: locale }), diff --git a/web/app/components/goto-anything/actions/commands/docs.tsx b/web/app/components/goto-anything/actions/commands/docs.tsx index 9f09d32094..8b04e84157 100644 --- a/web/app/components/goto-anything/actions/commands/docs.tsx +++ b/web/app/components/goto-anything/actions/commands/docs.tsx @@ -1,8 +1,8 @@ import type { SlashCommandHandler } from './types' import { RiBookOpenLine } from '@remixicon/react' import * as React from 'react' +import { getI18n } from 'react-i18next' import { defaultDocBaseUrl } from '@/context/i18n' -import i18n from '@/i18n-config/i18next-config' import { getDocLanguage } from '@/i18n-config/language' import { registerCommands, unregisterCommands } from './command-bus' @@ -19,6 +19,7 @@ export const docsCommand: SlashCommandHandler = { // Direct execution function execute: () => { + const i18n = getI18n() const currentLocale = i18n.language const docLanguage = getDocLanguage(currentLocale) const url = `${defaultDocBaseUrl}/${docLanguage}` @@ -26,6 +27,7 @@ export const docsCommand: SlashCommandHandler = { }, async search(args: string, locale: string = 'en') { + const i18n = getI18n() return [{ id: 'doc', title: i18n.t('userProfile.helpCenter', { ns: 'common', lng: locale }), @@ -41,6 +43,7 @@ export const docsCommand: SlashCommandHandler = { }, register(_deps: DocDeps) { + const i18n = getI18n() registerCommands({ 'navigation.doc': async (_args) => { // Get the current language from i18n diff --git a/web/app/components/goto-anything/actions/commands/forum.tsx b/web/app/components/goto-anything/actions/commands/forum.tsx index e32632b4b5..36116ceb1f 100644 --- a/web/app/components/goto-anything/actions/commands/forum.tsx +++ b/web/app/components/goto-anything/actions/commands/forum.tsx @@ -1,7 +1,7 @@ import type { SlashCommandHandler } from './types' import { RiFeedbackLine } from '@remixicon/react' import * as React from 'react' -import i18n from '@/i18n-config/i18next-config' +import { getI18n } from 'react-i18next' import { registerCommands, unregisterCommands } from './command-bus' // Forum command dependency types @@ -22,6 +22,7 @@ export const forumCommand: SlashCommandHandler = { }, async search(args: string, locale: string = 'en') { + const i18n = getI18n() return [{ id: 'forum', title: i18n.t('userProfile.forum', { ns: 'common', lng: locale }), diff --git a/web/app/components/goto-anything/actions/commands/language.tsx b/web/app/components/goto-anything/actions/commands/language.tsx index df94fd49ce..f4bafc1d58 100644 --- a/web/app/components/goto-anything/actions/commands/language.tsx +++ b/web/app/components/goto-anything/actions/commands/language.tsx @@ -1,6 +1,6 @@ import type { CommandSearchResult } from '../types' import type { SlashCommandHandler } from './types' -import i18n from '@/i18n-config/i18next-config' +import { getI18n } from 'react-i18next' import { languages } from '@/i18n-config/language' import { registerCommands, unregisterCommands } from './command-bus' @@ -14,6 +14,7 @@ const buildLanguageCommands = (query: string): CommandSearchResult[] => { const list = languages.filter(item => item.supported && ( !q || item.name.toLowerCase().includes(q) || String(item.value).toLowerCase().includes(q) )) + const i18n = getI18n() return list.map(item => ({ id: `lang-${item.value}`, title: item.name, diff --git a/web/app/components/goto-anything/actions/commands/slash.tsx b/web/app/components/goto-anything/actions/commands/slash.tsx index ec0f333cd4..6aad67731f 100644 --- a/web/app/components/goto-anything/actions/commands/slash.tsx +++ b/web/app/components/goto-anything/actions/commands/slash.tsx @@ -2,8 +2,8 @@ import type { ActionItem } from '../types' import { useTheme } from 'next-themes' import { useEffect } from 'react' +import { getI18n } from 'react-i18next' import { setLocaleOnClient } from '@/i18n-config' -import i18n from '@/i18n-config/i18next-config' import { accountCommand } from './account' import { executeCommand } from './command-bus' import { communityCommand } from './community' @@ -14,6 +14,8 @@ import { slashCommandRegistry } from './registry' import { themeCommand } from './theme' import { zenCommand } from './zen' +const i18n = getI18n() + export const slashAction: ActionItem = { key: '/', shortcut: '/', diff --git a/web/app/components/goto-anything/actions/commands/theme.tsx b/web/app/components/goto-anything/actions/commands/theme.tsx index 335182af67..ba1416229d 100644 --- a/web/app/components/goto-anything/actions/commands/theme.tsx +++ b/web/app/components/goto-anything/actions/commands/theme.tsx @@ -2,7 +2,7 @@ import type { CommandSearchResult } from '../types' import type { SlashCommandHandler } from './types' import { RiComputerLine, RiMoonLine, RiSunLine } from '@remixicon/react' import * as React from 'react' -import i18n from '@/i18n-config/i18next-config' +import { getI18n } from 'react-i18next' import { registerCommands, unregisterCommands } from './command-bus' // Theme dependency types @@ -32,6 +32,7 @@ const THEME_ITEMS = [ ] as const const buildThemeCommands = (query: string, locale?: string): CommandSearchResult[] => { + const i18n = getI18n() const q = query.toLowerCase() const list = THEME_ITEMS.filter(item => !q diff --git a/web/app/components/goto-anything/actions/commands/zen.tsx b/web/app/components/goto-anything/actions/commands/zen.tsx index d6d9f1e5a2..1645e40fd9 100644 --- a/web/app/components/goto-anything/actions/commands/zen.tsx +++ b/web/app/components/goto-anything/actions/commands/zen.tsx @@ -1,8 +1,8 @@ import type { SlashCommandHandler } from './types' import { RiFullscreenLine } from '@remixicon/react' import * as React from 'react' +import { getI18n } from 'react-i18next' import { isInWorkflowPage } from '@/app/components/workflow/constants' -import i18n from '@/i18n-config/i18next-config' import { registerCommands, unregisterCommands } from './command-bus' // Zen command dependency types - no external dependencies needed @@ -32,6 +32,7 @@ export const zenCommand: SlashCommandHandler = { execute: toggleZenMode, async search(_args: string, locale: string = 'en') { + const i18n = getI18n() return [{ id: 'zen', title: i18n.t('gotoAnything.actions.zenTitle', { ns: 'app', lng: locale }) || 'Zen Mode', diff --git a/web/app/components/header/account-setting/data-source-page-new/install-from-marketplace.tsx b/web/app/components/header/account-setting/data-source-page-new/install-from-marketplace.tsx index 956352d6d3..f02e276f55 100644 --- a/web/app/components/header/account-setting/data-source-page-new/install-from-marketplace.tsx +++ b/web/app/components/header/account-setting/data-source-page-new/install-from-marketplace.tsx @@ -15,7 +15,6 @@ import Divider from '@/app/components/base/divider' import Loading from '@/app/components/base/loading' import List from '@/app/components/plugins/marketplace/list' import ProviderCard from '@/app/components/plugins/provider-card' -import { getLocaleOnClient } from '@/i18n-config' import { cn } from '@/utils/classnames' import { getMarketplaceUrl } from '@/utils/var' import { @@ -33,7 +32,6 @@ const InstallFromMarketplace = ({ const { t } = useTranslation() const { theme } = useTheme() const [collapse, setCollapse] = useState(false) - const locale = getLocaleOnClient() const { plugins: allPlugins, isLoading: isAllPluginsLoading, @@ -70,7 +68,6 @@ const InstallFromMarketplace = ({ marketplaceCollectionPluginsMap={{}} plugins={allPlugins} showInstallButton - locale={locale} cardContainerClassName="grid grid-cols-2 gap-2" cardRender={cardRender} emptyClassName="h-auto" diff --git a/web/app/components/header/account-setting/language-page/index.tsx b/web/app/components/header/account-setting/language-page/index.tsx index 5d888281e9..2a0604421f 100644 --- a/web/app/components/header/account-setting/language-page/index.tsx +++ b/web/app/components/header/account-setting/language-page/index.tsx @@ -2,13 +2,13 @@ import type { Item } from '@/app/components/base/select' import type { Locale } from '@/i18n-config' +import { useRouter } from 'next/navigation' import { useState } from 'react' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' import { SimpleSelect } from '@/app/components/base/select' import { ToastContext } from '@/app/components/base/toast' import { useAppContext } from '@/context/app-context' - import { useLocale } from '@/context/i18n' import { setLocaleOnClient } from '@/i18n-config' import { languages } from '@/i18n-config/language' @@ -25,6 +25,7 @@ export default function LanguagePage() { const { notify } = useContext(ToastContext) const [editing, setEditing] = useState(false) const { t } = useTranslation() + const router = useRouter() const handleSelectLanguage = async (item: Item) => { const url = '/account/interface-language' @@ -35,7 +36,8 @@ export default function LanguagePage() { await updateUserProfile({ url, body: { [bodyKey]: item.value } }) notify({ type: 'success', message: t('actionMsg.modifiedSuccessfully', { ns: 'common' }) }) - setLocaleOnClient(item.value.toString() as Locale) + setLocaleOnClient(item.value.toString() as Locale, false) + router.refresh() } catch (e) { notify({ type: 'error', message: (e as Error).message }) diff --git a/web/app/components/header/account-setting/model-provider-page/install-from-marketplace.tsx b/web/app/components/header/account-setting/model-provider-page/install-from-marketplace.tsx index 0e7506bf96..289146f2d2 100644 --- a/web/app/components/header/account-setting/model-provider-page/install-from-marketplace.tsx +++ b/web/app/components/header/account-setting/model-provider-page/install-from-marketplace.tsx @@ -14,7 +14,6 @@ import Divider from '@/app/components/base/divider' import Loading from '@/app/components/base/loading' import List from '@/app/components/plugins/marketplace/list' import ProviderCard from '@/app/components/plugins/provider-card' -import { getLocaleOnClient } from '@/i18n-config' import { cn } from '@/utils/classnames' import { getMarketplaceUrl } from '@/utils/var' import { @@ -32,7 +31,6 @@ const InstallFromMarketplace = ({ const { t } = useTranslation() const { theme } = useTheme() const [collapse, setCollapse] = useState(false) - const locale = getLocaleOnClient() const { plugins: allPlugins, isLoading: isAllPluginsLoading, @@ -69,7 +67,6 @@ const InstallFromMarketplace = ({ marketplaceCollectionPluginsMap={{}} plugins={allPlugins} showInstallButton - locale={locale} cardContainerClassName="grid grid-cols-2 gap-2" cardRender={cardRender} emptyClassName="h-auto" diff --git a/web/app/components/i18n-server.tsx b/web/app/components/i18n-server.tsx deleted file mode 100644 index 01dc5f0f13..0000000000 --- a/web/app/components/i18n-server.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import * as React from 'react' -import { getLocaleOnServer } from '@/i18n-config/server' -import { ToastProvider } from './base/toast' -import I18N from './i18n' - -export type II18NServerProps = { - children: React.ReactNode -} - -const I18NServer = async ({ - children, -}: II18NServerProps) => { - const locale = await getLocaleOnServer() - - return ( - - {children} - - ) -} - -export default I18NServer diff --git a/web/app/components/i18n.tsx b/web/app/components/i18n.tsx deleted file mode 100644 index e9af2face9..0000000000 --- a/web/app/components/i18n.tsx +++ /dev/null @@ -1,45 +0,0 @@ -'use client' - -import type { FC } from 'react' -import type { Locale } from '@/i18n-config' -import { usePrefetchQuery } from '@tanstack/react-query' -import { useHydrateAtoms } from 'jotai/utils' -import * as React from 'react' -import { useEffect, useState } from 'react' -import { localeAtom } from '@/context/i18n' -import { setLocaleOnClient } from '@/i18n-config' -import { getSystemFeatures } from '@/service/common' -import Loading from './base/loading' - -export type II18nProps = { - locale: Locale - children: React.ReactNode -} -const I18n: FC = ({ - locale, - children, -}) => { - useHydrateAtoms([[localeAtom, locale]]) - const [loading, setLoading] = useState(true) - - usePrefetchQuery({ - queryKey: ['systemFeatures'], - queryFn: getSystemFeatures, - }) - - useEffect(() => { - setLocaleOnClient(locale, false).then(() => { - setLoading(false) - }) - }, [locale]) - - if (loading) - return
- - return ( - <> - {children} - - ) -} -export default React.memo(I18n) diff --git a/web/app/components/plugins/base/deprecation-notice.tsx b/web/app/components/plugins/base/deprecation-notice.tsx index c2ddfa6975..513b27a2cf 100644 --- a/web/app/components/plugins/base/deprecation-notice.tsx +++ b/web/app/components/plugins/base/deprecation-notice.tsx @@ -1,4 +1,5 @@ import type { FC } from 'react' +import { useTranslation } from '#i18n' import { RiAlertFill } from '@remixicon/react' import { camelCase } from 'es-toolkit/string' import Link from 'next/link' @@ -6,14 +7,12 @@ import * as React from 'react' import { useMemo } from 'react' import { Trans } from 'react-i18next' import { cn } from '@/utils/classnames' -import { useMixedTranslation } from '../marketplace/hooks' type DeprecationNoticeProps = { status: 'deleted' | 'active' deprecatedReason: string alternativePluginId: string alternativePluginURL: string - locale?: string className?: string innerWrapperClassName?: string iconWrapperClassName?: string @@ -34,13 +33,12 @@ const DeprecationNotice: FC = ({ deprecatedReason, alternativePluginId, alternativePluginURL, - locale, className, innerWrapperClassName, iconWrapperClassName, textClassName, }) => { - const { t } = useMixedTranslation(locale) + const { t } = useTranslation() const deprecatedReasonKey = useMemo(() => { if (!deprecatedReason) diff --git a/web/app/components/plugins/card/index.spec.tsx b/web/app/components/plugins/card/index.spec.tsx index 4a3e5a587b..fd97534ec4 100644 --- a/web/app/components/plugins/card/index.spec.tsx +++ b/web/app/components/plugins/card/index.spec.tsx @@ -502,31 +502,6 @@ describe('Card', () => { }) }) - // ================================ - // Locale Tests - // ================================ - describe('Locale', () => { - it('should use locale from props when provided', () => { - const plugin = createMockPlugin({ - label: { 'en-US': 'English Title', 'zh-Hans': '中文标题' }, - }) - - render() - - expect(screen.getByText('中文标题')).toBeInTheDocument() - }) - - it('should fallback to default locale when prop locale not found', () => { - const plugin = createMockPlugin({ - label: { 'en-US': 'English Title' }, - }) - - render() - - expect(screen.getByText('English Title')).toBeInTheDocument() - }) - }) - // ================================ // Memoization Tests // ================================ diff --git a/web/app/components/plugins/card/index.tsx b/web/app/components/plugins/card/index.tsx index ada26801de..8578421116 100644 --- a/web/app/components/plugins/card/index.tsx +++ b/web/app/components/plugins/card/index.tsx @@ -1,15 +1,13 @@ 'use client' import type { Plugin } from '../types' -import type { Locale } from '@/i18n-config' +import { useTranslation } from '#i18n' import { RiAlertFill } from '@remixicon/react' import * as React from 'react' -import { useMixedTranslation } from '@/app/components/plugins/marketplace/hooks' import { useGetLanguage } from '@/context/i18n' import useTheme from '@/hooks/use-theme' import { renderI18nObject, } from '@/i18n-config' -import { getLanguage } from '@/i18n-config/language' import { Theme } from '@/types/app' import { cn } from '@/utils/classnames' import Partner from '../base/badges/partner' @@ -33,7 +31,6 @@ export type Props = { footer?: React.ReactNode isLoading?: boolean loadingFileName?: string - locale?: Locale limitedInstall?: boolean } @@ -48,13 +45,11 @@ const Card = ({ footer, isLoading = false, loadingFileName, - locale: localeFromProps, limitedInstall = false, }: Props) => { - const defaultLocale = useGetLanguage() - const locale = localeFromProps ? getLanguage(localeFromProps) : defaultLocale - const { t } = useMixedTranslation(localeFromProps) - const { categoriesMap } = useCategories(t, true) + const locale = useGetLanguage() + const { t } = useTranslation() + const { categoriesMap } = useCategories(true) const { category, type, name, org, label, brief, icon, icon_dark, verified, badges = [] } = payload const { theme } = useTheme() const iconSrc = theme === Theme.dark && icon_dark ? icon_dark : icon diff --git a/web/app/components/plugins/hooks.ts b/web/app/components/plugins/hooks.ts index 262935205b..65d073cc2f 100644 --- a/web/app/components/plugins/hooks.ts +++ b/web/app/components/plugins/hooks.ts @@ -1,4 +1,3 @@ -import type { TFunction } from 'i18next' import type { CategoryKey, TagKey } from './constants' import { useMemo } from 'react' import { useTranslation } from 'react-i18next' @@ -13,9 +12,8 @@ export type Tag = { label: string } -export const useTags = (translateFromOut?: TFunction) => { - const { t: translation } = useTranslation() - const t = translateFromOut || translation +export const useTags = () => { + const { t } = useTranslation() const tags = useMemo(() => { return tagKeys.map((tag) => { @@ -53,9 +51,8 @@ type Category = { label: string } -export const useCategories = (translateFromOut?: TFunction, isSingle?: boolean) => { - const { t: translation } = useTranslation() - const t = translateFromOut || translation +export const useCategories = (isSingle?: boolean) => { + const { t } = useTranslation() const categories = useMemo(() => { return categoryKeys.map((category) => { diff --git a/web/app/components/plugins/marketplace/empty/index.spec.tsx b/web/app/components/plugins/marketplace/empty/index.spec.tsx index 4cbc85a309..bc8e701dfc 100644 --- a/web/app/components/plugins/marketplace/empty/index.spec.tsx +++ b/web/app/components/plugins/marketplace/empty/index.spec.tsx @@ -7,9 +7,9 @@ import Line from './line' // Mock external dependencies only // ================================ -// Mock useMixedTranslation hook -vi.mock('../hooks', () => ({ - useMixedTranslation: (_locale?: string) => ({ +// Mock i18n translation hook +vi.mock('#i18n', () => ({ + useTranslation: () => ({ t: (key: string, options?: { ns?: string }) => { // Build full key with namespace prefix if provided const fullKey = options?.ns ? `${options.ns}.${key}` : key @@ -471,36 +471,6 @@ describe('Empty', () => { }) }) - // ================================ - // Locale Prop Tests - // ================================ - describe('Locale Prop', () => { - it('should pass locale to useMixedTranslation', () => { - render() - - // Translation should still work - expect(screen.getByText('No plugin found')).toBeInTheDocument() - }) - - it('should handle undefined locale', () => { - render() - - expect(screen.getByText('No plugin found')).toBeInTheDocument() - }) - - it('should handle en-US locale', () => { - render() - - expect(screen.getByText('No plugin found')).toBeInTheDocument() - }) - - it('should handle ja-JP locale', () => { - render() - - expect(screen.getByText('No plugin found')).toBeInTheDocument() - }) - }) - // ================================ // Placeholder Cards Layout Tests // ================================ @@ -634,7 +604,6 @@ describe('Empty', () => { text="Custom message" lightCard className="custom-wrapper" - locale="en-US" />, ) @@ -695,12 +664,6 @@ describe('Empty', () => { expect(container.querySelector('.only-class')).toBeInTheDocument() }) - it('should render with only locale prop', () => { - render() - - expect(screen.getByText('No plugin found')).toBeInTheDocument() - }) - it('should handle text with unicode characters', () => { render() @@ -813,7 +776,7 @@ describe('Empty and Line Integration', () => { }) it('should render complete Empty component structure', () => { - const { container } = render() + const { container } = render() // Container expect(container.querySelector('.test')).toBeInTheDocument() diff --git a/web/app/components/plugins/marketplace/empty/index.tsx b/web/app/components/plugins/marketplace/empty/index.tsx index 3c33d9b92a..6e5adff1b4 100644 --- a/web/app/components/plugins/marketplace/empty/index.tsx +++ b/web/app/components/plugins/marketplace/empty/index.tsx @@ -1,6 +1,6 @@ 'use client' +import { useTranslation } from '#i18n' import { Group } from '@/app/components/base/icons/src/vender/other' -import { useMixedTranslation } from '@/app/components/plugins/marketplace/hooks' import { cn } from '@/utils/classnames' import Line from './line' @@ -8,16 +8,14 @@ type Props = { text?: string lightCard?: boolean className?: string - locale?: string } const Empty = ({ text, lightCard, className, - locale, }: Props) => { - const { t } = useMixedTranslation(locale) + const { t } = useTranslation() return (
{ } } -/** - * ! Support zh-Hans, pt-BR, ja-JP and en-US for Marketplace page - * ! For other languages, use en-US as fallback - */ -export const useMixedTranslation = (localeFromOuter?: string) => { - let t = useTranslation().t - - if (localeFromOuter) - t = i18n.getFixedT(localeFromOuter) - - return { - t, - } -} - export const useMarketplaceContainerScroll = ( callback: () => void, scrollContainerId = 'marketplace-container', diff --git a/web/app/components/plugins/marketplace/index.spec.tsx b/web/app/components/plugins/marketplace/index.spec.tsx index 3073897ba1..b3b1d58dd4 100644 --- a/web/app/components/plugins/marketplace/index.spec.tsx +++ b/web/app/components/plugins/marketplace/index.spec.tsx @@ -11,7 +11,6 @@ import { PluginCategoryEnum } from '@/app/components/plugins/types' // Note: Import after mocks are set up import { DEFAULT_SORT, SCROLL_BOTTOM_THRESHOLD } from './constants' import { MarketplaceContext, MarketplaceContextProvider, useMarketplaceContext } from './context' -import { useMixedTranslation } from './hooks' import PluginTypeSwitch, { PLUGIN_TYPE_SEARCH_MAP } from './plugin-type-switch' import StickySearchAndSwitchWrapper from './sticky-search-and-switch-wrapper' import { @@ -602,48 +601,6 @@ describe('utils', () => { }) }) -// ================================ -// Hooks Tests -// ================================ -describe('hooks', () => { - describe('useMixedTranslation', () => { - it('should return translation function', () => { - const { result } = renderHook(() => useMixedTranslation()) - - expect(result.current.t).toBeDefined() - expect(typeof result.current.t).toBe('function') - }) - - it('should return translation key when no translation found', () => { - const { result } = renderHook(() => useMixedTranslation()) - - // The global mock returns key with namespace prefix - expect(result.current.t('category.all', { ns: 'plugin' })).toBe('plugin.category.all') - }) - - it('should use locale from outer when provided', () => { - const { result } = renderHook(() => useMixedTranslation('zh-Hans')) - - expect(result.current.t).toBeDefined() - }) - - it('should handle different locale values', () => { - const locales = ['en-US', 'zh-Hans', 'ja-JP', 'pt-BR'] - locales.forEach((locale) => { - const { result } = renderHook(() => useMixedTranslation(locale)) - expect(result.current.t).toBeDefined() - expect(typeof result.current.t).toBe('function') - }) - }) - - it('should use getFixedT when localeFromOuter is provided', () => { - const { result } = renderHook(() => useMixedTranslation('fr-FR')) - // The global mock returns key with namespace prefix - expect(result.current.t('search', { ns: 'plugin' })).toBe('plugin.search') - }) - }) -}) - // ================================ // useMarketplaceCollectionsAndPlugins Tests // ================================ @@ -2088,17 +2045,6 @@ describe('StickySearchAndSwitchWrapper', () => { }) describe('Props', () => { - it('should accept locale prop', () => { - render( - - - , - ) - - // Component should render without errors - expect(screen.getByTestId('portal-elem')).toBeInTheDocument() - }) - it('should accept showSearchParams prop', () => { render( diff --git a/web/app/components/plugins/marketplace/index.tsx b/web/app/components/plugins/marketplace/index.tsx index f9f7e86b9a..08d1bc833f 100644 --- a/web/app/components/plugins/marketplace/index.tsx +++ b/web/app/components/plugins/marketplace/index.tsx @@ -1,6 +1,5 @@ import type { MarketplaceCollection, SearchParams } from './types' import type { Plugin } from '@/app/components/plugins/types' -import type { Locale } from '@/i18n-config' import { TanstackQueryInitializer } from '@/context/query-client' import { MarketplaceContextProvider } from './context' import Description from './description' @@ -9,7 +8,6 @@ import StickySearchAndSwitchWrapper from './sticky-search-and-switch-wrapper' import { getMarketplaceCollectionsAndPlugins } from './utils' type MarketplaceProps = { - locale: Locale showInstallButton?: boolean shouldExclude?: boolean searchParams?: SearchParams @@ -18,7 +16,6 @@ type MarketplaceProps = { showSearchParams?: boolean } const Marketplace = async ({ - locale, showInstallButton = true, shouldExclude, searchParams, @@ -44,12 +41,10 @@ const Marketplace = async ({ > { - const { t } = useMixedTranslation(locale) + const { t } = useTranslation() const { theme } = useTheme() const [isShowInstallFromMarketplace, { setTrue: showInstallFromMarketplace, setFalse: hideInstallFromMarketplace, }] = useBoolean(false) - const localeFromLocale = useLocale() - const { getTagLabel } = useTags(t) + const locale = useLocale() + const { getTagLabel } = useTags() // Memoize marketplace link params to prevent unnecessary re-renders const marketplaceLinkParams = useMemo(() => ({ - language: localeFromLocale, + language: locale, theme, - }), [localeFromLocale, theme]) + }), [locale, theme]) // Memoize tag labels to prevent recreating array on every render const tagLabels = useMemo(() => @@ -52,7 +48,6 @@ const CardWrapperComponent = ({ ({ - useMixedTranslation: (_locale?: string) => ({ +// Mock i18n translation hook +vi.mock('#i18n', () => ({ + useTranslation: () => ({ t: (key: string, options?: { ns?: string, num?: number }) => { // Build full key with namespace prefix if provided const fullKey = options?.ns ? `${options.ns}.${key}` : key @@ -28,6 +27,7 @@ vi.mock('../hooks', () => ({ return translations[fullKey] || key }, }), + useLocale: () => 'en-US', })) // Mock useMarketplaceContext with controllable values @@ -148,15 +148,15 @@ vi.mock('@/app/components/plugins/install-plugin/install-from-marketplace', () = // Mock SortDropdown component vi.mock('../sort-dropdown', () => ({ - default: ({ locale }: { locale: Locale }) => ( -
Sort
+ default: () => ( +
Sort
), })) // Mock Empty component vi.mock('../empty', () => ({ - default: ({ className, locale }: { className?: string, locale?: string }) => ( -
+ default: ({ className }: { className?: string }) => ( +
No plugins found
), @@ -233,7 +233,6 @@ describe('List', () => { marketplaceCollectionPluginsMap: {} as Record, plugins: undefined, showInstallButton: false, - locale: 'en-US' as Locale, cardContainerClassName: '', cardRender: undefined, onMoreClick: undefined, @@ -351,18 +350,6 @@ describe('List', () => { expect(screen.getByTestId('empty-component')).toHaveClass('custom-empty-class') }) - it('should pass locale to Empty component', () => { - render( - , - ) - - expect(screen.getByTestId('empty-component')).toHaveAttribute('data-locale', 'zh-CN') - }) - it('should pass showInstallButton to CardWrapper', () => { const plugins = createMockPluginList(1) @@ -508,7 +495,6 @@ describe('ListWithCollection', () => { marketplaceCollections: [] as MarketplaceCollection[], marketplaceCollectionPluginsMap: {} as Record, showInstallButton: false, - locale: 'en-US' as Locale, cardContainerClassName: '', cardRender: undefined, onMoreClick: undefined, @@ -820,7 +806,6 @@ describe('ListWrapper', () => { marketplaceCollections: [] as MarketplaceCollection[], marketplaceCollectionPluginsMap: {} as Record, showInstallButton: false, - locale: 'en-US' as Locale, } beforeEach(() => { @@ -901,14 +886,6 @@ describe('ListWrapper', () => { expect(screen.queryByTestId('sort-dropdown')).not.toBeInTheDocument() }) - - it('should pass locale to SortDropdown', () => { - mockContextValues.plugins = createMockPluginList(1) - - render() - - expect(screen.getByTestId('sort-dropdown')).toHaveAttribute('data-locale', 'zh-CN') - }) }) // ================================ @@ -1169,7 +1146,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollections={[]} marketplaceCollectionPluginsMap={{}} plugins={[plugin]} - locale="en-US" />, ) @@ -1188,7 +1164,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollections={[]} marketplaceCollectionPluginsMap={{}} plugins={[plugin]} - locale="en-US" />, ) @@ -1209,7 +1184,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollections={[]} marketplaceCollectionPluginsMap={{}} plugins={plugins} - locale="en-US" />, ) @@ -1231,7 +1205,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollectionPluginsMap={{}} plugins={[plugin]} showInstallButton={true} - locale="en-US" />, ) @@ -1252,7 +1225,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollectionPluginsMap={{}} plugins={[plugin]} showInstallButton={true} - locale="en-US" />, ) @@ -1274,7 +1246,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollectionPluginsMap={{}} plugins={[plugin]} showInstallButton={true} - locale="en-US" />, ) @@ -1293,7 +1264,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollectionPluginsMap={{}} plugins={[plugin]} showInstallButton={true} - locale="en-US" />, ) @@ -1310,7 +1280,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollectionPluginsMap={{}} plugins={[plugin]} showInstallButton={true} - locale="en-US" />, ) @@ -1327,7 +1296,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollectionPluginsMap={{}} plugins={[plugin]} showInstallButton={true} - locale="en-US" />, ) @@ -1354,7 +1322,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollectionPluginsMap={{}} plugins={[plugin]} showInstallButton={false} - locale="en-US" />, ) @@ -1375,7 +1342,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollectionPluginsMap={{}} plugins={[plugin]} showInstallButton={false} - locale="en-US" />, ) @@ -1390,7 +1356,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollections={[]} marketplaceCollectionPluginsMap={{}} plugins={[plugin]} - locale="en-US" />, ) @@ -1414,7 +1379,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollections={[]} marketplaceCollectionPluginsMap={{}} plugins={[plugin]} - locale="en-US" />, ) @@ -1432,7 +1396,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollections={[]} marketplaceCollectionPluginsMap={{}} plugins={[plugin]} - locale="en-US" />, ) @@ -1450,7 +1413,6 @@ describe('CardWrapper (via List integration)', () => { marketplaceCollections={[]} marketplaceCollectionPluginsMap={{}} plugins={[plugin]} - locale="en-US" />, ) @@ -1482,7 +1444,6 @@ describe('Combined Workflows', () => { , ) @@ -1501,7 +1462,6 @@ describe('Combined Workflows', () => { , ) @@ -1521,7 +1481,6 @@ describe('Combined Workflows', () => { , ) @@ -1535,7 +1494,6 @@ describe('Combined Workflows', () => { , ) @@ -1551,7 +1509,6 @@ describe('Combined Workflows', () => { , ) @@ -1569,7 +1526,6 @@ describe('Combined Workflows', () => { , ) @@ -1601,7 +1557,6 @@ describe('Accessibility', () => { , ) @@ -1625,7 +1580,6 @@ describe('Accessibility', () => { marketplaceCollections={collections} marketplaceCollectionPluginsMap={pluginsMap} onMoreClick={onMoreClick} - locale="en-US" />, ) @@ -1642,7 +1596,6 @@ describe('Accessibility', () => { marketplaceCollections={[]} marketplaceCollectionPluginsMap={{}} plugins={plugins} - locale="en-US" />, ) @@ -1668,7 +1621,6 @@ describe('Performance', () => { marketplaceCollections={[]} marketplaceCollectionPluginsMap={{}} plugins={plugins} - locale="en-US" />, ) const endTime = performance.now() @@ -1689,7 +1641,6 @@ describe('Performance', () => { , ) const endTime = performance.now() diff --git a/web/app/components/plugins/marketplace/list/index.tsx b/web/app/components/plugins/marketplace/list/index.tsx index 54889b232f..80b33d0ffd 100644 --- a/web/app/components/plugins/marketplace/list/index.tsx +++ b/web/app/components/plugins/marketplace/list/index.tsx @@ -1,7 +1,6 @@ 'use client' import type { Plugin } from '../../types' import type { MarketplaceCollection } from '../types' -import type { Locale } from '@/i18n-config' import { cn } from '@/utils/classnames' import Empty from '../empty' import CardWrapper from './card-wrapper' @@ -12,7 +11,6 @@ type ListProps = { marketplaceCollectionPluginsMap: Record plugins?: Plugin[] showInstallButton?: boolean - locale: Locale cardContainerClassName?: string cardRender?: (plugin: Plugin) => React.JSX.Element | null onMoreClick?: () => void @@ -23,7 +21,6 @@ const List = ({ marketplaceCollectionPluginsMap, plugins, showInstallButton, - locale, cardContainerClassName, cardRender, onMoreClick, @@ -37,7 +34,6 @@ const List = ({ marketplaceCollections={marketplaceCollections} marketplaceCollectionPluginsMap={marketplaceCollectionPluginsMap} showInstallButton={showInstallButton} - locale={locale} cardContainerClassName={cardContainerClassName} cardRender={cardRender} onMoreClick={onMoreClick} @@ -61,7 +57,6 @@ const List = ({ key={`${plugin.org}/${plugin.name}`} plugin={plugin} showInstallButton={showInstallButton} - locale={locale} /> ) }) @@ -71,7 +66,7 @@ const List = ({ } { plugins && !plugins.length && ( - + ) } diff --git a/web/app/components/plugins/marketplace/list/list-with-collection.tsx b/web/app/components/plugins/marketplace/list/list-with-collection.tsx index 8830cc5ddf..c17715e71e 100644 --- a/web/app/components/plugins/marketplace/list/list-with-collection.tsx +++ b/web/app/components/plugins/marketplace/list/list-with-collection.tsx @@ -3,9 +3,8 @@ import type { MarketplaceCollection } from '../types' import type { SearchParamsFromCollection } from '@/app/components/plugins/marketplace/types' import type { Plugin } from '@/app/components/plugins/types' -import type { Locale } from '@/i18n-config' +import { useLocale, useTranslation } from '#i18n' import { RiArrowRightSLine } from '@remixicon/react' -import { useMixedTranslation } from '@/app/components/plugins/marketplace/hooks' import { getLanguage } from '@/i18n-config/language' import { cn } from '@/utils/classnames' import CardWrapper from './card-wrapper' @@ -14,7 +13,6 @@ type ListWithCollectionProps = { marketplaceCollections: MarketplaceCollection[] marketplaceCollectionPluginsMap: Record showInstallButton?: boolean - locale: Locale cardContainerClassName?: string cardRender?: (plugin: Plugin) => React.JSX.Element | null onMoreClick?: (searchParams?: SearchParamsFromCollection) => void @@ -23,12 +21,12 @@ const ListWithCollection = ({ marketplaceCollections, marketplaceCollectionPluginsMap, showInstallButton, - locale, cardContainerClassName, cardRender, onMoreClick, }: ListWithCollectionProps) => { - const { t } = useMixedTranslation(locale) + const { t } = useTranslation() + const locale = useLocale() return ( <> @@ -72,7 +70,6 @@ const ListWithCollection = ({ key={plugin.plugin_id} plugin={plugin} showInstallButton={showInstallButton} - locale={locale} /> ) }) diff --git a/web/app/components/plugins/marketplace/list/list-wrapper.tsx b/web/app/components/plugins/marketplace/list/list-wrapper.tsx index f8126eb34b..84fcf92daf 100644 --- a/web/app/components/plugins/marketplace/list/list-wrapper.tsx +++ b/web/app/components/plugins/marketplace/list/list-wrapper.tsx @@ -1,10 +1,9 @@ 'use client' import type { Plugin } from '../../types' import type { MarketplaceCollection } from '../types' -import type { Locale } from '@/i18n-config' +import { useTranslation } from '#i18n' import { useEffect } from 'react' import Loading from '@/app/components/base/loading' -import { useMixedTranslation } from '@/app/components/plugins/marketplace/hooks' import { useMarketplaceContext } from '../context' import SortDropdown from '../sort-dropdown' import List from './index' @@ -13,15 +12,13 @@ type ListWrapperProps = { marketplaceCollections: MarketplaceCollection[] marketplaceCollectionPluginsMap: Record showInstallButton?: boolean - locale: Locale } const ListWrapper = ({ marketplaceCollections, marketplaceCollectionPluginsMap, showInstallButton, - locale, }: ListWrapperProps) => { - const { t } = useMixedTranslation(locale) + const { t } = useTranslation() const plugins = useMarketplaceContext(v => v.plugins) const pluginsTotal = useMarketplaceContext(v => v.pluginsTotal) const marketplaceCollectionsFromClient = useMarketplaceContext(v => v.marketplaceCollectionsFromClient) @@ -55,7 +52,7 @@ const ListWrapper = ({
{t('marketplace.pluginsResult', { ns: 'plugin', num: pluginsTotal })}
- +
) } @@ -73,7 +70,6 @@ const ListWrapper = ({ marketplaceCollectionPluginsMap={marketplaceCollectionPluginsMapFromClient || marketplaceCollectionPluginsMap} plugins={plugins} showInstallButton={showInstallButton} - locale={locale} onMoreClick={handleMoreClick} /> ) diff --git a/web/app/components/plugins/marketplace/plugin-type-switch.tsx b/web/app/components/plugins/marketplace/plugin-type-switch.tsx index 2a89e6847e..b9572413ed 100644 --- a/web/app/components/plugins/marketplace/plugin-type-switch.tsx +++ b/web/app/components/plugins/marketplace/plugin-type-switch.tsx @@ -1,4 +1,5 @@ 'use client' +import { useTranslation } from '#i18n' import { RiArchive2Line, RiBrain2Line, @@ -12,7 +13,6 @@ import { Trigger as TriggerIcon } from '@/app/components/base/icons/src/vender/p import { cn } from '@/utils/classnames' import { PluginCategoryEnum } from '../types' import { useMarketplaceContext } from './context' -import { useMixedTranslation } from './hooks' export const PLUGIN_TYPE_SEARCH_MAP = { all: 'all', @@ -25,16 +25,14 @@ export const PLUGIN_TYPE_SEARCH_MAP = { bundle: 'bundle', } type PluginTypeSwitchProps = { - locale?: string className?: string showSearchParams?: boolean } const PluginTypeSwitch = ({ - locale, className, showSearchParams, }: PluginTypeSwitchProps) => { - const { t } = useMixedTranslation(locale) + const { t } = useTranslation() const activePluginType = useMarketplaceContext(s => s.activePluginType) const handleActivePluginTypeChange = useMarketplaceContext(s => s.handleActivePluginTypeChange) diff --git a/web/app/components/plugins/marketplace/search-box/index.spec.tsx b/web/app/components/plugins/marketplace/search-box/index.spec.tsx index 8c3131f6d1..3e9cc40be0 100644 --- a/web/app/components/plugins/marketplace/search-box/index.spec.tsx +++ b/web/app/components/plugins/marketplace/search-box/index.spec.tsx @@ -10,9 +10,9 @@ import ToolSelectorTrigger from './trigger/tool-selector' // Mock external dependencies only // ================================ -// Mock useMixedTranslation hook -vi.mock('../hooks', () => ({ - useMixedTranslation: (_locale?: string) => ({ +// Mock i18n translation hook +vi.mock('#i18n', () => ({ + useTranslation: () => ({ t: (key: string, options?: { ns?: string }) => { // Build full key with namespace prefix if provided const fullKey = options?.ns ? `${options.ns}.${key}` : key @@ -364,13 +364,6 @@ describe('SearchBox', () => { expect(container.querySelector('.custom-input-class')).toBeInTheDocument() }) - it('should pass locale to TagsFilter', () => { - render() - - // TagsFilter should be rendered with locale - expect(screen.getByTestId('portal-elem')).toBeInTheDocument() - }) - it('should handle empty placeholder', () => { render() @@ -449,12 +442,6 @@ describe('SearchBoxWrapper', () => { expect(screen.getByRole('textbox')).toBeInTheDocument() }) - it('should render with locale prop', () => { - render() - - expect(screen.getByRole('textbox')).toBeInTheDocument() - }) - it('should render in marketplace mode', () => { const { container } = render() @@ -500,13 +487,6 @@ describe('SearchBoxWrapper', () => { expect(screen.getByPlaceholderText('Search plugins')).toBeInTheDocument() }) - - it('should pass locale to useMixedTranslation', () => { - render() - - // Translation should still work - expect(screen.getByPlaceholderText('Search plugins')).toBeInTheDocument() - }) }) }) @@ -665,12 +645,6 @@ describe('MarketplaceTrigger', () => { }) describe('Props Variations', () => { - it('should handle locale prop', () => { - render() - - expect(screen.getByText('All Tags')).toBeInTheDocument() - }) - it('should handle empty tagsMap', () => { const { container } = render( , @@ -1251,7 +1225,6 @@ describe('Combined Workflows', () => { supportAddCustomTool onShowAddCustomCollectionModal={vi.fn()} placeholder="Search plugins" - locale="en-US" wrapperClassName="custom-wrapper" inputClassName="custom-input" autoFocus={false} diff --git a/web/app/components/plugins/marketplace/search-box/index.tsx b/web/app/components/plugins/marketplace/search-box/index.tsx index 05f98782b9..b6e1f8ee70 100644 --- a/web/app/components/plugins/marketplace/search-box/index.tsx +++ b/web/app/components/plugins/marketplace/search-box/index.tsx @@ -13,7 +13,6 @@ type SearchBoxProps = { tags: string[] onTagsChange: (tags: string[]) => void placeholder?: string - locale?: string supportAddCustomTool?: boolean usedInMarketplace?: boolean onShowAddCustomCollectionModal?: () => void @@ -28,7 +27,6 @@ const SearchBox = ({ tags, onTagsChange, placeholder = '', - locale, usedInMarketplace = false, supportAddCustomTool, onShowAddCustomCollectionModal, @@ -49,7 +47,6 @@ const SearchBox = ({ tags={tags} onTagsChange={onTagsChange} usedInMarketplace - locale={locale} />
@@ -109,7 +106,6 @@ const SearchBox = ({ ) diff --git a/web/app/components/plugins/marketplace/search-box/search-box-wrapper.tsx b/web/app/components/plugins/marketplace/search-box/search-box-wrapper.tsx index 1290c26210..d7fc004236 100644 --- a/web/app/components/plugins/marketplace/search-box/search-box-wrapper.tsx +++ b/web/app/components/plugins/marketplace/search-box/search-box-wrapper.tsx @@ -1,16 +1,11 @@ 'use client' +import { useTranslation } from '#i18n' import { useMarketplaceContext } from '../context' -import { useMixedTranslation } from '../hooks' import SearchBox from './index' -type SearchBoxWrapperProps = { - locale?: string -} -const SearchBoxWrapper = ({ - locale, -}: SearchBoxWrapperProps) => { - const { t } = useMixedTranslation(locale) +const SearchBoxWrapper = () => { + const { t } = useTranslation() const searchPluginText = useMarketplaceContext(v => v.searchPluginText) const handleSearchPluginTextChange = useMarketplaceContext(v => v.handleSearchPluginTextChange) const filterPluginTags = useMarketplaceContext(v => v.filterPluginTags) @@ -24,7 +19,6 @@ const SearchBoxWrapper = ({ onSearchChange={handleSearchPluginTextChange} tags={filterPluginTags} onTagsChange={handleFilterPluginTagsChange} - locale={locale} placeholder={t('searchPlugins', { ns: 'plugin' })} usedInMarketplace /> diff --git a/web/app/components/plugins/marketplace/search-box/tags-filter.tsx b/web/app/components/plugins/marketplace/search-box/tags-filter.tsx index df4d3eebab..9a8035e2e3 100644 --- a/web/app/components/plugins/marketplace/search-box/tags-filter.tsx +++ b/web/app/components/plugins/marketplace/search-box/tags-filter.tsx @@ -1,5 +1,6 @@ 'use client' +import { useTranslation } from '#i18n' import { useState } from 'react' import Checkbox from '@/app/components/base/checkbox' import Input from '@/app/components/base/input' @@ -9,7 +10,6 @@ import { PortalToFollowElemTrigger, } from '@/app/components/base/portal-to-follow-elem' import { useTags } from '@/app/components/plugins/hooks' -import { useMixedTranslation } from '@/app/components/plugins/marketplace/hooks' import MarketplaceTrigger from './trigger/marketplace' import ToolSelectorTrigger from './trigger/tool-selector' @@ -17,18 +17,16 @@ type TagsFilterProps = { tags: string[] onTagsChange: (tags: string[]) => void usedInMarketplace?: boolean - locale?: string } const TagsFilter = ({ tags, onTagsChange, usedInMarketplace = false, - locale, }: TagsFilterProps) => { - const { t } = useMixedTranslation(locale) + const { t } = useTranslation() const [open, setOpen] = useState(false) const [searchText, setSearchText] = useState('') - const { tags: options, tagsMap } = useTags(t) + const { tags: options, tagsMap } = useTags() const filteredOptions = options.filter(option => option.label.toLowerCase().includes(searchText.toLowerCase())) const handleCheck = (id: string) => { if (tags.includes(id)) @@ -59,7 +57,6 @@ const TagsFilter = ({ open={open} tags={tags} tagsMap={tagsMap} - locale={locale} onTagsChange={onTagsChange} /> ) diff --git a/web/app/components/plugins/marketplace/search-box/trigger/marketplace.tsx b/web/app/components/plugins/marketplace/search-box/trigger/marketplace.tsx index 2ba03bd2f2..e387d52d0e 100644 --- a/web/app/components/plugins/marketplace/search-box/trigger/marketplace.tsx +++ b/web/app/components/plugins/marketplace/search-box/trigger/marketplace.tsx @@ -1,15 +1,14 @@ import type { Tag } from '../../../hooks' +import { useTranslation } from '#i18n' import { RiArrowDownSLine, RiCloseCircleFill, RiFilter3Line } from '@remixicon/react' import * as React from 'react' import { cn } from '@/utils/classnames' -import { useMixedTranslation } from '../../hooks' type MarketplaceTriggerProps = { selectedTagsLength: number open: boolean tags: string[] tagsMap: Record - locale?: string onTagsChange: (tags: string[]) => void } @@ -18,10 +17,9 @@ const MarketplaceTrigger = ({ open, tags, tagsMap, - locale, onTagsChange, }: MarketplaceTriggerProps) => { - const { t } = useMixedTranslation(locale) + const { t } = useTranslation() return (
{ // Build full key with namespace prefix if provided const fullKey = options?.ns ? `${options.ns}.${key}` : key @@ -22,8 +22,8 @@ const mockTranslation = vi.fn((key: string, options?: { ns?: string }) => { return translations[fullKey] || key }) -vi.mock('../hooks', () => ({ - useMixedTranslation: (_locale?: string) => ({ +vi.mock('#i18n', () => ({ + useTranslation: () => ({ t: mockTranslation, }), })) @@ -145,36 +145,6 @@ describe('SortDropdown', () => { }) }) - // ================================ - // Props Testing - // ================================ - describe('Props', () => { - it('should accept locale prop', () => { - render() - - expect(screen.getByTestId('portal-wrapper')).toBeInTheDocument() - }) - - it('should call useMixedTranslation with provided locale', () => { - render() - - // Translation function should be called for labels - expect(mockTranslation).toHaveBeenCalledWith('marketplace.sortBy', { ns: 'plugin' }) - }) - - it('should render without locale prop (undefined)', () => { - render() - - expect(screen.getByText('Sort by')).toBeInTheDocument() - }) - - it('should render with empty string locale', () => { - render() - - expect(screen.getByText('Sort by')).toBeInTheDocument() - }) - }) - // ================================ // State Management Tests // ================================ @@ -618,13 +588,6 @@ describe('SortDropdown', () => { expect(mockTranslation).toHaveBeenCalledWith('marketplace.sortOption.newlyReleased', { ns: 'plugin' }) expect(mockTranslation).toHaveBeenCalledWith('marketplace.sortOption.firstReleased', { ns: 'plugin' }) }) - - it('should pass locale to useMixedTranslation', () => { - render() - - // Verify component renders with locale - expect(screen.getByTestId('portal-wrapper')).toBeInTheDocument() - }) }) // ================================ diff --git a/web/app/components/plugins/marketplace/sort-dropdown/index.tsx b/web/app/components/plugins/marketplace/sort-dropdown/index.tsx index a1f6631735..984b114d03 100644 --- a/web/app/components/plugins/marketplace/sort-dropdown/index.tsx +++ b/web/app/components/plugins/marketplace/sort-dropdown/index.tsx @@ -1,4 +1,5 @@ 'use client' +import { useTranslation } from '#i18n' import { RiArrowDownSLine, RiCheckLine, @@ -9,16 +10,10 @@ import { PortalToFollowElemContent, PortalToFollowElemTrigger, } from '@/app/components/base/portal-to-follow-elem' -import { useMixedTranslation } from '@/app/components/plugins/marketplace/hooks' import { useMarketplaceContext } from '../context' -type SortDropdownProps = { - locale?: string -} -const SortDropdown = ({ - locale, -}: SortDropdownProps) => { - const { t } = useMixedTranslation(locale) +const SortDropdown = () => { + const { t } = useTranslation() const options = [ { value: 'install_count', diff --git a/web/app/components/plugins/marketplace/sticky-search-and-switch-wrapper.tsx b/web/app/components/plugins/marketplace/sticky-search-and-switch-wrapper.tsx index 602a1e9af2..3d3530c83e 100644 --- a/web/app/components/plugins/marketplace/sticky-search-and-switch-wrapper.tsx +++ b/web/app/components/plugins/marketplace/sticky-search-and-switch-wrapper.tsx @@ -5,13 +5,11 @@ import PluginTypeSwitch from './plugin-type-switch' import SearchBoxWrapper from './search-box/search-box-wrapper' type StickySearchAndSwitchWrapperProps = { - locale?: string pluginTypeSwitchClassName?: string showSearchParams?: boolean } const StickySearchAndSwitchWrapper = ({ - locale, pluginTypeSwitchClassName, showSearchParams, }: StickySearchAndSwitchWrapperProps) => { @@ -25,9 +23,8 @@ const StickySearchAndSwitchWrapper = ({ pluginTypeSwitchClassName, )} > - +
diff --git a/web/app/components/plugins/plugin-item/index.tsx b/web/app/components/plugins/plugin-item/index.tsx index d287bd9e9a..3f658c63a8 100644 --- a/web/app/components/plugins/plugin-item/index.tsx +++ b/web/app/components/plugins/plugin-item/index.tsx @@ -44,7 +44,7 @@ const PluginItem: FC = ({ }) => { const { t } = useTranslation() const { theme } = useTheme() - const { categoriesMap } = useCategories(t, true) + const { categoriesMap } = useCategories(true) const currentPluginID = usePluginPageContext(v => v.currentPluginID) const setCurrentPluginID = usePluginPageContext(v => v.setCurrentPluginID) const { refreshPluginList } = useRefreshPluginList() diff --git a/web/app/components/provider/i18n-server.tsx b/web/app/components/provider/i18n-server.tsx new file mode 100644 index 0000000000..23391cf428 --- /dev/null +++ b/web/app/components/provider/i18n-server.tsx @@ -0,0 +1,21 @@ +import { getLocaleOnServer, getResources } from '@/i18n-config/server' + +import { I18nClientProvider } from './i18n' + +export async function I18nServerProvider({ + children, +}: { + children: React.ReactNode +}) { + const locale = await getLocaleOnServer() + const resource = await getResources(locale) + + return ( + + {children} + + ) +} diff --git a/web/app/components/provider/i18n.tsx b/web/app/components/provider/i18n.tsx new file mode 100644 index 0000000000..6441a09dd3 --- /dev/null +++ b/web/app/components/provider/i18n.tsx @@ -0,0 +1,24 @@ +'use client' + +import type { Resource } from 'i18next' +import type { Locale } from '@/i18n-config' +import { I18nextProvider } from 'react-i18next' +import { createI18nextInstance } from '@/i18n-config/client' + +export function I18nClientProvider({ + locale, + resource, + children, +}: { + locale: Locale + resource: Resource + children: React.ReactNode +}) { + const i18n = createI18nextInstance(locale, resource) + + return ( + + {children} + + ) +} diff --git a/web/app/components/share/text-generation/index.tsx b/web/app/components/share/text-generation/index.tsx index b9bb59664a..b793a03ce7 100644 --- a/web/app/components/share/text-generation/index.tsx +++ b/web/app/components/share/text-generation/index.tsx @@ -32,7 +32,7 @@ import { useWebAppStore } from '@/context/web-app-context' import { useAppFavicon } from '@/hooks/use-app-favicon' import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' import useDocumentTitle from '@/hooks/use-document-title' -import { changeLanguage } from '@/i18n-config/i18next-config' +import { changeLanguage } from '@/i18n-config/client' import { AccessMode } from '@/models/access-control' import { fetchSavedMessage as doFetchSavedMessage, removeMessage, saveMessage } from '@/service/share' import { Resolution, TransferMethod } from '@/types/app' diff --git a/web/app/components/tools/marketplace/index.spec.tsx b/web/app/components/tools/marketplace/index.spec.tsx index 354c717f2d..493d960e2a 100644 --- a/web/app/components/tools/marketplace/index.spec.tsx +++ b/web/app/components/tools/marketplace/index.spec.tsx @@ -19,7 +19,6 @@ vi.mock('@/app/components/plugins/marketplace/list', () => ({ marketplaceCollectionPluginsMap: Record plugins?: unknown[] showInstallButton?: boolean - locale: string }) => { listRenderSpy(props) return
@@ -42,10 +41,6 @@ vi.mock('@/utils/var', () => ({ getMarketplaceUrl: vi.fn(() => 'https://marketplace.test/market'), })) -vi.mock('@/i18n-config', () => ({ - getLocaleOnClient: () => 'en', -})) - vi.mock('next-themes', () => ({ useTheme: () => ({ theme: 'light' }), })) @@ -148,7 +143,6 @@ describe('Marketplace', () => { expect(screen.getByTestId('marketplace-list')).toBeInTheDocument() expect(listRenderSpy).toHaveBeenCalledWith(expect.objectContaining({ showInstallButton: true, - locale: 'en', })) }) }) diff --git a/web/app/components/tools/marketplace/index.tsx b/web/app/components/tools/marketplace/index.tsx index 476a47d8f6..3900a9e505 100644 --- a/web/app/components/tools/marketplace/index.tsx +++ b/web/app/components/tools/marketplace/index.tsx @@ -1,4 +1,5 @@ import type { useMarketplace } from './hooks' +import { useLocale } from '#i18n' import { RiArrowRightUpLine, RiArrowUpDoubleLine, @@ -7,7 +8,6 @@ import { useTheme } from 'next-themes' import { useTranslation } from 'react-i18next' import Loading from '@/app/components/base/loading' import List from '@/app/components/plugins/marketplace/list' -import { getLocaleOnClient } from '@/i18n-config' import { getMarketplaceUrl } from '@/utils/var' type MarketplaceProps = { @@ -24,7 +24,7 @@ const Marketplace = ({ showMarketplacePanel, marketplaceContext, }: MarketplaceProps) => { - const locale = getLocaleOnClient() + const locale = useLocale() const { t } = useTranslation() const { theme } = useTheme() const { @@ -104,7 +104,6 @@ const Marketplace = ({ marketplaceCollectionPluginsMap={marketplaceCollectionPluginsMap || {}} plugins={plugins} showInstallButton - locale={locale} /> ) } diff --git a/web/app/layout.tsx b/web/app/layout.tsx index 8fc5f8abcc..acd56e1da6 100644 --- a/web/app/layout.tsx +++ b/web/app/layout.tsx @@ -8,9 +8,10 @@ import { TanstackQueryInitializer } from '@/context/query-client' import { getLocaleOnServer } from '@/i18n-config/server' import { DatasetAttr } from '@/types/feature' import { cn } from '@/utils/classnames' +import { ToastProvider } from './components/base/toast' import BrowserInitializer from './components/browser-initializer' import { ReactScanLoader } from './components/devtools/react-scan/loader' -import I18nServer from './components/i18n-server' +import { I18nServerProvider } from './components/provider/i18n-server' import SentryInitializer from './components/sentry-initializer' import RoutePrefixHandle from './routePrefixHandle' import './styles/globals.css' @@ -104,11 +105,13 @@ const LocaleLayout = async ({ - - - {children} - - + + + + {children} + + + diff --git a/web/context/i18n.ts b/web/context/i18n.ts index e65049b506..d57fc5b984 100644 --- a/web/context/i18n.ts +++ b/web/context/i18n.ts @@ -1,10 +1,10 @@ import type { Locale } from '@/i18n-config/language' -import { atom, useAtomValue } from 'jotai' +import { useTranslation } from '#i18n' import { getDocLanguage, getLanguage, getPricingPageLanguage } from '@/i18n-config/language' -export const localeAtom = atom('en-US') export const useLocale = () => { - return useAtomValue(localeAtom) + const { i18n } = useTranslation() + return i18n.language as Locale } export const useGetLanguage = () => { diff --git a/web/i18n-config/client.ts b/web/i18n-config/client.ts new file mode 100644 index 0000000000..17d3dceae1 --- /dev/null +++ b/web/i18n-config/client.ts @@ -0,0 +1,35 @@ +'use client' +import type { Resource } from 'i18next' +import type { Locale } from '.' +import type { NamespaceCamelCase, NamespaceKebabCase } from './resources' +import { kebabCase } from 'es-toolkit/string' +import { createInstance } from 'i18next' +import resourcesToBackend from 'i18next-resources-to-backend' +import { getI18n, initReactI18next } from 'react-i18next' +import { getInitOptions } from './settings' + +export function createI18nextInstance(lng: Locale, resources: Resource) { + const instance = createInstance() + instance + .use(initReactI18next) + .use(resourcesToBackend(( + language: Locale, + namespace: NamespaceKebabCase | NamespaceCamelCase, + ) => { + const namespaceKebab = kebabCase(namespace) + return import(`../i18n/${language}/${namespaceKebab}.json`) + })) + .init({ + ...getInitOptions(), + lng, + resources, + }) + return instance +} + +export const changeLanguage = async (lng?: Locale) => { + if (!lng) + return + const i18n = getI18n() + await i18n.changeLanguage(lng) +} diff --git a/web/i18n-config/index.ts b/web/i18n-config/index.ts index bb73ef4b71..e24fd8533d 100644 --- a/web/i18n-config/index.ts +++ b/web/i18n-config/index.ts @@ -2,7 +2,7 @@ import type { Locale } from '@/i18n-config/language' import Cookies from 'js-cookie' import { LOCALE_COOKIE_NAME } from '@/config' -import { changeLanguage } from '@/i18n-config/i18next-config' +import { changeLanguage } from '@/i18n-config/client' import { LanguagesSupported } from '@/i18n-config/language' export const i18n = { @@ -19,10 +19,6 @@ export const setLocaleOnClient = async (locale: Locale, reloadPage = true) => { location.reload() } -export const getLocaleOnClient = (): Locale => { - return Cookies.get(LOCALE_COOKIE_NAME) as Locale || i18n.defaultLocale -} - export const renderI18nObject = (obj: Record, language: string) => { if (!obj) return '' diff --git a/web/i18n-config/lib.client.ts b/web/i18n-config/lib.client.ts index fc40384458..fffb4d95ae 100644 --- a/web/i18n-config/lib.client.ts +++ b/web/i18n-config/lib.client.ts @@ -1,6 +1,6 @@ 'use client' -import type { NamespaceCamelCase } from './i18next-config' +import type { NamespaceCamelCase } from './resources' import { useTranslation as useTranslationOriginal } from 'react-i18next' export function useTranslation(ns?: NamespaceCamelCase) { diff --git a/web/i18n-config/lib.server.ts b/web/i18n-config/lib.server.ts index 4969492cd4..4727ed482f 100644 --- a/web/i18n-config/lib.server.ts +++ b/web/i18n-config/lib.server.ts @@ -1,4 +1,4 @@ -import type { NamespaceCamelCase } from './i18next-config' +import type { NamespaceCamelCase } from './resources' import { use } from 'react' import { getLocaleOnServer, getTranslation } from './server' diff --git a/web/i18n-config/i18next-config.ts b/web/i18n-config/resources.ts similarity index 59% rename from web/i18n-config/i18next-config.ts rename to web/i18n-config/resources.ts index 0997485967..4bcfb98e14 100644 --- a/web/i18n-config/i18next-config.ts +++ b/web/i18n-config/resources.ts @@ -1,8 +1,4 @@ -'use client' -import type { Locale } from '.' -import { camelCase, kebabCase } from 'es-toolkit/string' -import i18n from 'i18next' -import { initReactI18next } from 'react-i18next' +import { kebabCase } from 'es-toolkit/string' import appAnnotation from '../i18n/en-US/app-annotation.json' import appApi from '../i18n/en-US/app-api.json' import appDebug from '../i18n/en-US/app-debug.json' @@ -35,7 +31,7 @@ import tools from '../i18n/en-US/tools.json' import workflow from '../i18n/en-US/workflow.json' // @keep-sorted -export const resources = { +const resources = { app, appAnnotation, appApi, @@ -82,60 +78,5 @@ export type Resources = typeof resources export type NamespaceCamelCase = keyof Resources export type NamespaceKebabCase = KebabCase -const requireSilent = async (lang: Locale, namespace: NamespaceKebabCase) => { - let res - try { - res = (await import(`../i18n/${lang}/${namespace}.json`)).default - } - catch { - res = (await import(`../i18n/en-US/${namespace}.json`)).default - } - - return res -} - -const NAMESPACES = Object.keys(resources).map(kebabCase) as NamespaceKebabCase[] - -// Load a single namespace for a language -export const loadNamespace = async (lang: Locale, ns: NamespaceKebabCase) => { - const camelNs = camelCase(ns) as NamespaceCamelCase - if (i18n.hasResourceBundle(lang, camelNs)) - return - - const resource = await requireSilent(lang, ns) - i18n.addResourceBundle(lang, camelNs, resource, true, true) -} - -// Load all namespaces for a language (used when switching language) -export const loadLangResources = async (lang: Locale) => { - await Promise.all( - NAMESPACES.map(ns => loadNamespace(lang, ns)), - ) -} - -// Initial resources: load en-US namespaces for fallback/default locale -const getInitialTranslations = () => { - return { - 'en-US': resources, - } -} - -if (!i18n.isInitialized) { - i18n.use(initReactI18next).init({ - lng: undefined, - fallbackLng: 'en-US', - resources: getInitialTranslations(), - defaultNS: 'common', - ns: Object.keys(resources), - keySeparator: false, - }) -} - -export const changeLanguage = async (lng?: Locale) => { - if (!lng) - return - await loadLangResources(lng) - await i18n.changeLanguage(lng) -} - -export default i18n +export const namespacesCamelCase = Object.keys(resources) as NamespaceCamelCase[] +export const namespacesKebabCase = namespacesCamelCase.map(ns => kebabCase(ns)) as NamespaceKebabCase[] diff --git a/web/i18n-config/server.ts b/web/i18n-config/server.ts index f5966149fd..403040c134 100644 --- a/web/i18n-config/server.ts +++ b/web/i18n-config/server.ts @@ -1,15 +1,19 @@ -import type { i18n as I18nInstance } from 'i18next' +import type { i18n as I18nInstance, Resource, ResourceLanguage } from 'i18next' import type { Locale } from '.' -import type { NamespaceCamelCase, NamespaceKebabCase } from './i18next-config' +import type { NamespaceCamelCase, NamespaceKebabCase } from './resources' import { match } from '@formatjs/intl-localematcher' import { kebabCase } from 'es-toolkit/compat' +import { camelCase } from 'es-toolkit/string' import { createInstance } from 'i18next' import resourcesToBackend from 'i18next-resources-to-backend' import Negotiator from 'negotiator' import { cookies, headers } from 'next/headers' +import { cache } from 'react' import { initReactI18next } from 'react-i18next/initReactI18next' import { serverOnlyContext } from '@/utils/server-only-context' import { i18n } from '.' +import { namespacesKebabCase } from './resources' +import { getInitOptions } from './settings' const [getLocaleCache, setLocaleCache] = serverOnlyContext(null) const [getI18nInstance, setI18nInstance] = serverOnlyContext(null) @@ -27,9 +31,8 @@ const getOrCreateI18next = async (lng: Locale) => { return import(`../i18n/${language}/${fileNamespace}.json`) })) .init({ + ...getInitOptions(), lng, - fallbackLng: 'en-US', - keySeparator: false, }) setI18nInstance(instance) return instance @@ -76,3 +79,16 @@ export const getLocaleOnServer = async (): Promise => { setLocaleCache(matchedLocale) return matchedLocale } + +export const getResources = cache(async (lng: Locale): Promise => { + const messages = {} as ResourceLanguage + + await Promise.all( + (namespacesKebabCase).map(async (ns) => { + const mod = await import(`../i18n/${lng}/${ns}.json`) + messages[camelCase(ns)] = mod.default + }), + ) + + return { [lng]: messages } +}) diff --git a/web/i18n-config/settings.ts b/web/i18n-config/settings.ts new file mode 100644 index 0000000000..1bf37ab21d --- /dev/null +++ b/web/i18n-config/settings.ts @@ -0,0 +1,13 @@ +import type { InitOptions } from 'i18next' +import { namespacesCamelCase } from './resources' + +export function getInitOptions(): InitOptions { + return { + // We do not have en for fallback + load: 'currentOnly', + fallbackLng: 'en-US', + partialBundledLanguages: true, + keySeparator: false, + ns: namespacesCamelCase, + } +} diff --git a/web/types/i18n.d.ts b/web/types/i18n.d.ts index 160d385730..9e20d5a55a 100644 --- a/web/types/i18n.d.ts +++ b/web/types/i18n.d.ts @@ -1,4 +1,4 @@ -import type { NamespaceCamelCase, Resources } from '../i18n-config/i18next-config' +import type { NamespaceCamelCase, Resources } from '../i18n-config/resources' import 'i18next' declare module 'i18next' { From 160b4d194bad9546cb3aff483b74dd0388bf3b1c Mon Sep 17 00:00:00 2001 From: yyh <92089059+lyzno1@users.noreply.github.com> Date: Wed, 7 Jan 2026 14:20:38 +0800 Subject: [PATCH 03/21] fix: signin page stuck on loading when refresh token valid but access token expired (#30675) Co-authored-by: Claude Opus 4.5 --- web/app/signin/normal-form.tsx | 4 +++- web/service/use-common.ts | 9 ++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/web/app/signin/normal-form.tsx b/web/app/signin/normal-form.tsx index bf4d80326b..be0feea6c1 100644 --- a/web/app/signin/normal-form.tsx +++ b/web/app/signin/normal-form.tsx @@ -28,7 +28,8 @@ const NormalForm = () => { const message = decodeURIComponent(searchParams.get('message') || '') const invite_token = decodeURIComponent(searchParams.get('invite_token') || '') const [isInitCheckLoading, setInitCheckLoading] = useState(true) - const isLoading = isCheckLoading || loginData?.logged_in || isInitCheckLoading + const [isRedirecting, setIsRedirecting] = useState(false) + const isLoading = isCheckLoading || isInitCheckLoading || isRedirecting const { systemFeatures } = useGlobalPublicStore() const [authType, updateAuthType] = useState<'code' | 'password'>('password') const [showORLine, setShowORLine] = useState(false) @@ -40,6 +41,7 @@ const NormalForm = () => { const init = useCallback(async () => { try { if (isLoggedIn) { + setIsRedirecting(true) const redirectUrl = resolvePostLoginRedirect(searchParams) router.replace(redirectUrl || '/apps') return diff --git a/web/service/use-common.ts b/web/service/use-common.ts index a1edb041c0..ca0845d95a 100644 --- a/web/service/use-common.ts +++ b/web/service/use-common.ts @@ -221,13 +221,12 @@ export const useIsLogin = () => { await get('/account/profile', {}, { silent: true, }) - } - catch (e: any) { - if (e.status === 401) - return { logged_in: false } return { logged_in: true } } - return { logged_in: true } + catch { + // Any error (401, 500, network error, etc.) means not logged in + return { logged_in: false } + } }, }) } From 666640f7d59ef787a6916d6c7b2d48153105f3f0 Mon Sep 17 00:00:00 2001 From: Xiangxuan Qu Date: Wed, 7 Jan 2026 15:40:35 +0900 Subject: [PATCH 04/21] refactor: remove unnecessary type: ignore from rag_pipeline_fields.py (#30666) Co-authored-by: fghpdf --- api/fields/rag_pipeline_fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/fields/rag_pipeline_fields.py b/api/fields/rag_pipeline_fields.py index f9e858c68b..97c02e7085 100644 --- a/api/fields/rag_pipeline_fields.py +++ b/api/fields/rag_pipeline_fields.py @@ -1,4 +1,4 @@ -from flask_restx import fields # type: ignore +from flask_restx import fields from fields.workflow_fields import workflow_partial_fields from libs.helper import AppIconUrlField, TimestampField From 187bfafe8b001cf8afad95e257374f2b2e7e5983 Mon Sep 17 00:00:00 2001 From: wangxiaolei Date: Wed, 7 Jan 2026 14:54:11 +0800 Subject: [PATCH 05/21] fix: fix assign value stand as default (#30651) Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .../advanced_chat/generate_task_pipeline.py | 19 + ...test_generate_task_pipeline_answer_node.py | 390 ++++++++++++++++++ 2 files changed, 409 insertions(+) create mode 100644 api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline_answer_node.py diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index da1e9f19b6..4dd95be52d 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -358,6 +358,25 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport): if node_finish_resp: yield node_finish_resp + # For ANSWER nodes, check if we need to send a message_replace event + # Only send if the final output differs from the accumulated task_state.answer + # This happens when variables were updated by variable_assigner during workflow execution + if event.node_type == NodeType.ANSWER and event.outputs: + final_answer = event.outputs.get("answer") + if final_answer is not None and final_answer != self._task_state.answer: + logger.info( + "ANSWER node final output '%s' differs from accumulated answer '%s', sending message_replace event", + final_answer, + self._task_state.answer, + ) + # Update the task state answer + self._task_state.answer = str(final_answer) + # Send message_replace event to update the UI + yield self._message_cycle_manager.message_replace_to_stream_response( + answer=str(final_answer), + reason="variable_update", + ) + def _handle_node_failed_events( self, event: Union[QueueNodeFailedEvent, QueueNodeExceptionEvent], diff --git a/api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline_answer_node.py b/api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline_answer_node.py new file mode 100644 index 0000000000..205b157542 --- /dev/null +++ b/api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline_answer_node.py @@ -0,0 +1,390 @@ +""" +Tests for AdvancedChatAppGenerateTaskPipeline._handle_node_succeeded_event method, +specifically testing the ANSWER node message_replace logic. +""" + +from datetime import datetime +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity +from core.app.entities.queue_entities import QueueNodeSucceededEvent +from core.workflow.enums import NodeType +from models import EndUser +from models.model import AppMode + + +class TestAnswerNodeMessageReplace: + """Test cases for ANSWER node message_replace event logic.""" + + @pytest.fixture + def mock_application_generate_entity(self): + """Create a mock application generate entity.""" + entity = Mock(spec=AdvancedChatAppGenerateEntity) + entity.task_id = "test-task-id" + entity.app_id = "test-app-id" + entity.workflow_run_id = "test-workflow-run-id" + # minimal app_config used by pipeline internals + entity.app_config = SimpleNamespace( + tenant_id="test-tenant-id", + app_id="test-app-id", + app_mode=AppMode.ADVANCED_CHAT, + app_model_config_dict={}, + additional_features=None, + sensitive_word_avoidance=None, + ) + entity.query = "test query" + entity.files = [] + entity.extras = {} + entity.trace_manager = None + entity.inputs = {} + entity.invoke_from = "debugger" + return entity + + @pytest.fixture + def mock_workflow(self): + """Create a mock workflow.""" + workflow = Mock() + workflow.id = "test-workflow-id" + workflow.features_dict = {} + return workflow + + @pytest.fixture + def mock_queue_manager(self): + """Create a mock queue manager.""" + manager = Mock() + manager.listen.return_value = [] + manager.graph_runtime_state = None + return manager + + @pytest.fixture + def mock_conversation(self): + """Create a mock conversation.""" + conversation = Mock() + conversation.id = "test-conversation-id" + conversation.mode = "advanced_chat" + return conversation + + @pytest.fixture + def mock_message(self): + """Create a mock message.""" + message = Mock() + message.id = "test-message-id" + message.query = "test query" + message.created_at = Mock() + message.created_at.timestamp.return_value = 1234567890 + return message + + @pytest.fixture + def mock_user(self): + """Create a mock end user.""" + user = MagicMock(spec=EndUser) + user.id = "test-user-id" + user.session_id = "test-session-id" + return user + + @pytest.fixture + def mock_draft_var_saver_factory(self): + """Create a mock draft variable saver factory.""" + return Mock() + + @pytest.fixture + def pipeline( + self, + mock_application_generate_entity, + mock_workflow, + mock_queue_manager, + mock_conversation, + mock_message, + mock_user, + mock_draft_var_saver_factory, + ): + """Create an AdvancedChatAppGenerateTaskPipeline instance with mocked dependencies.""" + from core.app.apps.advanced_chat.generate_task_pipeline import AdvancedChatAppGenerateTaskPipeline + + with patch("core.app.apps.advanced_chat.generate_task_pipeline.db"): + pipeline = AdvancedChatAppGenerateTaskPipeline( + application_generate_entity=mock_application_generate_entity, + workflow=mock_workflow, + queue_manager=mock_queue_manager, + conversation=mock_conversation, + message=mock_message, + user=mock_user, + stream=True, + dialogue_count=1, + draft_var_saver_factory=mock_draft_var_saver_factory, + ) + # Initialize workflow run id to avoid validation errors + pipeline._workflow_run_id = "test-workflow-run-id" + # Mock the message cycle manager methods we need to track + pipeline._message_cycle_manager.message_replace_to_stream_response = Mock() + return pipeline + + def test_answer_node_with_different_output_sends_message_replace(self, pipeline, mock_application_generate_entity): + """ + Test that when an ANSWER node's final output differs from accumulated answer, + a message_replace event is sent. + """ + # Arrange: Set initial accumulated answer + pipeline._task_state.answer = "initial answer" + + # Create ANSWER node succeeded event with different final output + event = QueueNodeSucceededEvent( + node_execution_id="test-node-execution-id", + node_id="test-answer-node", + node_type=NodeType.ANSWER, + start_at=datetime.now(), + outputs={"answer": "updated final answer"}, + ) + + # Mock the workflow response converter to avoid extra processing + pipeline._workflow_response_converter.workflow_node_finish_to_stream_response = Mock(return_value=None) + pipeline._save_output_for_event = Mock() + + # Act + responses = list(pipeline._handle_node_succeeded_event(event)) + + # Assert + assert pipeline._task_state.answer == "updated final answer" + # Verify message_replace was called + pipeline._message_cycle_manager.message_replace_to_stream_response.assert_called_once_with( + answer="updated final answer", reason="variable_update" + ) + + def test_answer_node_with_same_output_does_not_send_message_replace(self, pipeline): + """ + Test that when an ANSWER node's final output is the same as accumulated answer, + no message_replace event is sent. + """ + # Arrange: Set initial accumulated answer + pipeline._task_state.answer = "same answer" + + # Create ANSWER node succeeded event with same output + event = QueueNodeSucceededEvent( + node_execution_id="test-node-execution-id", + node_id="test-answer-node", + node_type=NodeType.ANSWER, + start_at=datetime.now(), + outputs={"answer": "same answer"}, + ) + + # Mock the workflow response converter + pipeline._workflow_response_converter.workflow_node_finish_to_stream_response = Mock(return_value=None) + pipeline._save_output_for_event = Mock() + + # Act + list(pipeline._handle_node_succeeded_event(event)) + + # Assert: answer should remain unchanged + assert pipeline._task_state.answer == "same answer" + # Verify message_replace was NOT called + pipeline._message_cycle_manager.message_replace_to_stream_response.assert_not_called() + + def test_answer_node_with_none_output_does_not_send_message_replace(self, pipeline): + """ + Test that when an ANSWER node's output is None or missing 'answer' key, + no message_replace event is sent. + """ + # Arrange: Set initial accumulated answer + pipeline._task_state.answer = "existing answer" + + # Create ANSWER node succeeded event with None output + event = QueueNodeSucceededEvent( + node_execution_id="test-node-execution-id", + node_id="test-answer-node", + node_type=NodeType.ANSWER, + start_at=datetime.now(), + outputs={"answer": None}, + ) + + # Mock the workflow response converter + pipeline._workflow_response_converter.workflow_node_finish_to_stream_response = Mock(return_value=None) + pipeline._save_output_for_event = Mock() + + # Act + list(pipeline._handle_node_succeeded_event(event)) + + # Assert: answer should remain unchanged + assert pipeline._task_state.answer == "existing answer" + # Verify message_replace was NOT called + pipeline._message_cycle_manager.message_replace_to_stream_response.assert_not_called() + + def test_answer_node_with_empty_outputs_does_not_send_message_replace(self, pipeline): + """ + Test that when an ANSWER node has empty outputs dict, + no message_replace event is sent. + """ + # Arrange: Set initial accumulated answer + pipeline._task_state.answer = "existing answer" + + # Create ANSWER node succeeded event with empty outputs + event = QueueNodeSucceededEvent( + node_execution_id="test-node-execution-id", + node_id="test-answer-node", + node_type=NodeType.ANSWER, + start_at=datetime.now(), + outputs={}, + ) + + # Mock the workflow response converter + pipeline._workflow_response_converter.workflow_node_finish_to_stream_response = Mock(return_value=None) + pipeline._save_output_for_event = Mock() + + # Act + list(pipeline._handle_node_succeeded_event(event)) + + # Assert: answer should remain unchanged + assert pipeline._task_state.answer == "existing answer" + # Verify message_replace was NOT called + pipeline._message_cycle_manager.message_replace_to_stream_response.assert_not_called() + + def test_answer_node_with_no_answer_key_in_outputs(self, pipeline): + """ + Test that when an ANSWER node's outputs don't contain 'answer' key, + no message_replace event is sent. + """ + # Arrange: Set initial accumulated answer + pipeline._task_state.answer = "existing answer" + + # Create ANSWER node succeeded event without 'answer' key in outputs + event = QueueNodeSucceededEvent( + node_execution_id="test-node-execution-id", + node_id="test-answer-node", + node_type=NodeType.ANSWER, + start_at=datetime.now(), + outputs={"other_key": "some value"}, + ) + + # Mock the workflow response converter + pipeline._workflow_response_converter.workflow_node_finish_to_stream_response = Mock(return_value=None) + pipeline._save_output_for_event = Mock() + + # Act + list(pipeline._handle_node_succeeded_event(event)) + + # Assert: answer should remain unchanged + assert pipeline._task_state.answer == "existing answer" + # Verify message_replace was NOT called + pipeline._message_cycle_manager.message_replace_to_stream_response.assert_not_called() + + def test_non_answer_node_does_not_send_message_replace(self, pipeline): + """ + Test that non-ANSWER nodes (e.g., LLM, END) don't trigger message_replace events. + """ + # Arrange: Set initial accumulated answer + pipeline._task_state.answer = "existing answer" + + # Test with LLM node + llm_event = QueueNodeSucceededEvent( + node_execution_id="test-llm-execution-id", + node_id="test-llm-node", + node_type=NodeType.LLM, + start_at=datetime.now(), + outputs={"answer": "different answer"}, + ) + + # Mock the workflow response converter + pipeline._workflow_response_converter.workflow_node_finish_to_stream_response = Mock(return_value=None) + pipeline._save_output_for_event = Mock() + + # Act + list(pipeline._handle_node_succeeded_event(llm_event)) + + # Assert: answer should remain unchanged + assert pipeline._task_state.answer == "existing answer" + # Verify message_replace was NOT called + pipeline._message_cycle_manager.message_replace_to_stream_response.assert_not_called() + + def test_end_node_does_not_send_message_replace(self, pipeline): + """ + Test that END nodes don't trigger message_replace events even with 'answer' output. + """ + # Arrange: Set initial accumulated answer + pipeline._task_state.answer = "existing answer" + + # Create END node succeeded event with answer output + event = QueueNodeSucceededEvent( + node_execution_id="test-end-execution-id", + node_id="test-end-node", + node_type=NodeType.END, + start_at=datetime.now(), + outputs={"answer": "different answer"}, + ) + + # Mock the workflow response converter + pipeline._workflow_response_converter.workflow_node_finish_to_stream_response = Mock(return_value=None) + pipeline._save_output_for_event = Mock() + + # Act + list(pipeline._handle_node_succeeded_event(event)) + + # Assert: answer should remain unchanged + assert pipeline._task_state.answer == "existing answer" + # Verify message_replace was NOT called + pipeline._message_cycle_manager.message_replace_to_stream_response.assert_not_called() + + def test_answer_node_with_numeric_output_converts_to_string(self, pipeline): + """ + Test that when an ANSWER node's final output is numeric, + it gets converted to string properly. + """ + # Arrange: Set initial accumulated answer + pipeline._task_state.answer = "text answer" + + # Create ANSWER node succeeded event with numeric output + event = QueueNodeSucceededEvent( + node_execution_id="test-node-execution-id", + node_id="test-answer-node", + node_type=NodeType.ANSWER, + start_at=datetime.now(), + outputs={"answer": 12345}, + ) + + # Mock the workflow response converter + pipeline._workflow_response_converter.workflow_node_finish_to_stream_response = Mock(return_value=None) + pipeline._save_output_for_event = Mock() + + # Act + list(pipeline._handle_node_succeeded_event(event)) + + # Assert: answer should be converted to string + assert pipeline._task_state.answer == "12345" + # Verify message_replace was called with string + pipeline._message_cycle_manager.message_replace_to_stream_response.assert_called_once_with( + answer="12345", reason="variable_update" + ) + + def test_answer_node_files_are_recorded(self, pipeline): + """ + Test that ANSWER nodes properly record files from outputs. + """ + # Arrange + pipeline._task_state.answer = "existing answer" + + # Create ANSWER node succeeded event with files + event = QueueNodeSucceededEvent( + node_execution_id="test-node-execution-id", + node_id="test-answer-node", + node_type=NodeType.ANSWER, + start_at=datetime.now(), + outputs={ + "answer": "same answer", + "files": [ + {"type": "image", "transfer_method": "remote_url", "remote_url": "http://example.com/img.png"} + ], + }, + ) + + # Mock the workflow response converter + pipeline._workflow_response_converter.fetch_files_from_node_outputs = Mock(return_value=event.outputs["files"]) + pipeline._workflow_response_converter.workflow_node_finish_to_stream_response = Mock(return_value=None) + pipeline._save_output_for_event = Mock() + + # Act + list(pipeline._handle_node_succeeded_event(event)) + + # Assert: files should be recorded + assert len(pipeline._recorded_files) == 1 + assert pipeline._recorded_files[0] == event.outputs["files"][0] From d8a0291382c30abf561f9b36c9b576a56453f7ff Mon Sep 17 00:00:00 2001 From: Xiangxuan Qu Date: Wed, 7 Jan 2026 22:15:43 +0900 Subject: [PATCH 06/21] refactor(web): remove unused type alias VoiceLanguageKey (#30694) Co-authored-by: fghpdf --- .../new-feature-panel/text-to-speech/param-config-content.tsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/web/app/components/base/features/new-feature-panel/text-to-speech/param-config-content.tsx b/web/app/components/base/features/new-feature-panel/text-to-speech/param-config-content.tsx index 41715b3c6b..cab41c66c1 100644 --- a/web/app/components/base/features/new-feature-panel/text-to-speech/param-config-content.tsx +++ b/web/app/components/base/features/new-feature-panel/text-to-speech/param-config-content.tsx @@ -1,7 +1,6 @@ 'use client' import type { OnFeaturesChange } from '@/app/components/base/features/types' import type { Item } from '@/app/components/base/select' -import type { I18nKeysWithPrefix } from '@/types/i18n' import { Listbox, ListboxButton, ListboxOption, ListboxOptions, Transition } from '@headlessui/react' import { CheckIcon, ChevronDownIcon } from '@heroicons/react/20/solid' import { RiCloseLine } from '@remixicon/react' @@ -20,8 +19,6 @@ import { useAppVoices } from '@/service/use-apps' import { TtsAutoPlay } from '@/types/app' import { cn } from '@/utils/classnames' -type VoiceLanguageKey = I18nKeysWithPrefix<'common', 'voice.language.'> - type VoiceParamConfigProps = { onClose: () => void onChange?: OnFeaturesChange From a422908efd39a0764f64402b1c97d8e2f3bccd01 Mon Sep 17 00:00:00 2001 From: yyh <92089059+lyzno1@users.noreply.github.com> Date: Wed, 7 Jan 2026 21:17:50 +0800 Subject: [PATCH 07/21] feat(i18n): Migrate translation workflow to Claude Code GitHub Actions (#30692) Co-authored-by: Claude Opus 4.5 Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../translate-i18n-base-on-english.yml | 94 ----- .github/workflows/translate-i18n-claude.yml | 395 ++++++++++++++++++ web/i18n-config/README.md | 30 +- web/package.json | 2 - web/pnpm-lock.yaml | 174 +------- web/scripts/auto-gen-i18n.js | 336 --------------- 6 files changed, 426 insertions(+), 605 deletions(-) delete mode 100644 .github/workflows/translate-i18n-base-on-english.yml create mode 100644 .github/workflows/translate-i18n-claude.yml delete mode 100644 web/scripts/auto-gen-i18n.js diff --git a/.github/workflows/translate-i18n-base-on-english.yml b/.github/workflows/translate-i18n-base-on-english.yml deleted file mode 100644 index 16d36361fd..0000000000 --- a/.github/workflows/translate-i18n-base-on-english.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Translate i18n Files Based on English - -on: - push: - branches: [main] - paths: - - 'web/i18n/en-US/*.json' - workflow_dispatch: - -permissions: - contents: write - pull-requests: write - -jobs: - check-and-update: - if: github.repository == 'langgenius/dify' - runs-on: ubuntu-latest - defaults: - run: - working-directory: web - steps: - # Keep use old checkout action version for https://github.com/peter-evans/create-pull-request/issues/4272 - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Check for file changes in i18n/en-US - id: check_files - run: | - # Skip check for manual trigger, translate all files - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - echo "FILES_CHANGED=true" >> $GITHUB_ENV - echo "FILE_ARGS=" >> $GITHUB_ENV - echo "Manual trigger: translating all files" - else - git fetch origin "${{ github.event.before }}" || true - git fetch origin "${{ github.sha }}" || true - changed_files=$(git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" -- 'i18n/en-US/*.json') - echo "Changed files: $changed_files" - if [ -n "$changed_files" ]; then - echo "FILES_CHANGED=true" >> $GITHUB_ENV - file_args="" - for file in $changed_files; do - filename=$(basename "$file" .json) - file_args="$file_args --file $filename" - done - echo "FILE_ARGS=$file_args" >> $GITHUB_ENV - echo "File arguments: $file_args" - else - echo "FILES_CHANGED=false" >> $GITHUB_ENV - fi - fi - - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - package_json_file: web/package.json - run_install: false - - - name: Set up Node.js - if: env.FILES_CHANGED == 'true' - uses: actions/setup-node@v6 - with: - node-version: 'lts/*' - cache: pnpm - cache-dependency-path: ./web/pnpm-lock.yaml - - - name: Install dependencies - if: env.FILES_CHANGED == 'true' - working-directory: ./web - run: pnpm install --frozen-lockfile - - - name: Generate i18n translations - if: env.FILES_CHANGED == 'true' - working-directory: ./web - run: pnpm run i18n:gen ${{ env.FILE_ARGS }} - - - name: Create Pull Request - if: env.FILES_CHANGED == 'true' - uses: peter-evans/create-pull-request@v6 - with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: 'chore(i18n): update translations based on en-US changes' - title: 'chore(i18n): translate i18n files based on en-US changes' - body: | - This PR was automatically created to update i18n translation files based on changes in en-US locale. - - **Triggered by:** ${{ github.sha }} - - **Changes included:** - - Updated translation files for all locales - branch: chore/automated-i18n-updates-${{ github.sha }} - delete-branch: true diff --git a/.github/workflows/translate-i18n-claude.yml b/.github/workflows/translate-i18n-claude.yml new file mode 100644 index 0000000000..8dccf8ef93 --- /dev/null +++ b/.github/workflows/translate-i18n-claude.yml @@ -0,0 +1,395 @@ +name: Translate i18n Files with Claude Code + +on: + push: + branches: [main] + paths: + - 'web/i18n/en-US/*.json' + workflow_dispatch: + inputs: + files: + description: 'Specific files to translate (space-separated, e.g., "app common"). Leave empty for all files.' + required: false + type: string + languages: + description: 'Specific languages to translate (space-separated, e.g., "zh-Hans ja-JP"). Leave empty for all supported languages.' + required: false + type: string + mode: + description: 'Sync mode: incremental (only changes) or full (re-check all keys)' + required: false + default: 'incremental' + type: choice + options: + - incremental + - full + +permissions: + contents: write + pull-requests: write + +jobs: + translate: + if: github.repository == 'langgenius/dify' + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Git + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + package_json_file: web/package.json + run_install: false + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: 'lts/*' + cache: pnpm + cache-dependency-path: ./web/pnpm-lock.yaml + + - name: Detect changed files and generate diff + id: detect_changes + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + # Manual trigger + if [ -n "${{ github.event.inputs.files }}" ]; then + echo "CHANGED_FILES=${{ github.event.inputs.files }}" >> $GITHUB_OUTPUT + else + # Get all JSON files in en-US directory + files=$(ls web/i18n/en-US/*.json 2>/dev/null | xargs -n1 basename | sed 's/.json$//' | tr '\n' ' ') + echo "CHANGED_FILES=$files" >> $GITHUB_OUTPUT + fi + echo "TARGET_LANGS=${{ github.event.inputs.languages }}" >> $GITHUB_OUTPUT + echo "SYNC_MODE=${{ github.event.inputs.mode || 'incremental' }}" >> $GITHUB_OUTPUT + + # For manual trigger with incremental mode, get diff from last commit + # For full mode, we'll do a complete check anyway + if [ "${{ github.event.inputs.mode }}" == "full" ]; then + echo "Full mode: will check all keys" > /tmp/i18n-diff.txt + echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT + else + git diff HEAD~1..HEAD -- 'web/i18n/en-US/*.json' > /tmp/i18n-diff.txt 2>/dev/null || echo "" > /tmp/i18n-diff.txt + if [ -s /tmp/i18n-diff.txt ]; then + echo "DIFF_AVAILABLE=true" >> $GITHUB_OUTPUT + else + echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT + fi + fi + else + # Push trigger - detect changed files from the push + BEFORE_SHA="${{ github.event.before }}" + # Handle edge case: first push or force push may have null/zero SHA + if [ -z "$BEFORE_SHA" ] || [ "$BEFORE_SHA" = "0000000000000000000000000000000000000000" ]; then + # Fallback to comparing with parent commit + BEFORE_SHA="HEAD~1" + fi + changed=$(git diff --name-only "$BEFORE_SHA" ${{ github.sha }} -- 'web/i18n/en-US/*.json' 2>/dev/null | xargs -n1 basename 2>/dev/null | sed 's/.json$//' | tr '\n' ' ' || echo "") + echo "CHANGED_FILES=$changed" >> $GITHUB_OUTPUT + echo "TARGET_LANGS=" >> $GITHUB_OUTPUT + echo "SYNC_MODE=incremental" >> $GITHUB_OUTPUT + + # Generate detailed diff for the push + git diff "$BEFORE_SHA"..${{ github.sha }} -- 'web/i18n/en-US/*.json' > /tmp/i18n-diff.txt 2>/dev/null || echo "" > /tmp/i18n-diff.txt + if [ -s /tmp/i18n-diff.txt ]; then + echo "DIFF_AVAILABLE=true" >> $GITHUB_OUTPUT + else + echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT + fi + fi + + # Truncate diff if too large (keep first 50KB) + if [ -f /tmp/i18n-diff.txt ]; then + head -c 50000 /tmp/i18n-diff.txt > /tmp/i18n-diff-truncated.txt + mv /tmp/i18n-diff-truncated.txt /tmp/i18n-diff.txt + fi + + echo "Detected files: $(cat $GITHUB_OUTPUT | grep CHANGED_FILES || echo 'none')" + + - name: Run Claude Code for Translation Sync + if: steps.detect_changes.outputs.CHANGED_FILES != '' + uses: anthropics/claude-code-action@v1 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + github_token: ${{ secrets.GITHUB_TOKEN }} + prompt: | + You are a professional i18n synchronization engineer for the Dify project. + Your task is to keep all language translations in sync with the English source (en-US). + + ## CRITICAL TOOL RESTRICTIONS + - Use **Read** tool to read files (NOT cat or bash) + - Use **Edit** tool to modify JSON files (NOT node, jq, or bash scripts) + - Use **Bash** ONLY for: git commands, gh commands, pnpm commands + - Run bash commands ONE BY ONE, never combine with && or || + + ## EFFICIENCY RULES + - **ONE Edit per language file** - batch all key additions into a single Edit + - Insert new keys at the beginning of JSON (after `{`), lint:fix will sort them + - Translate ALL keys for a language mentally first, then do ONE Edit + + ## Context + - Changed/target files: ${{ steps.detect_changes.outputs.CHANGED_FILES }} + - Target languages (empty means all supported): ${{ steps.detect_changes.outputs.TARGET_LANGS }} + - Sync mode: ${{ steps.detect_changes.outputs.SYNC_MODE }} + - Translation files are located in: web/i18n/{locale}/{filename}.json + - Language configuration is in: web/i18n-config/languages.ts + - Git diff is available: ${{ steps.detect_changes.outputs.DIFF_AVAILABLE }} + + ## CRITICAL DESIGN: Verify First, Then Sync + + You MUST follow this three-phase approach: + + ═══════════════════════════════════════════════════════════════ + ║ PHASE 1: VERIFY - Analyze and Generate Change Report ║ + ═══════════════════════════════════════════════════════════════ + + ### Step 1.1: Analyze Git Diff (for incremental mode) + Use the Read tool to read `/tmp/i18n-diff.txt` to see the git diff. + + Parse the diff to categorize changes: + - Lines with `+` (not `+++`): Added or modified values + - Lines with `-` (not `---`): Removed or old values + - Identify specific keys for each category: + * ADD: Keys that appear only in `+` lines (new keys) + * UPDATE: Keys that appear in both `-` and `+` lines (value changed) + * DELETE: Keys that appear only in `-` lines (removed keys) + + ### Step 1.2: Read Language Configuration + Use the Read tool to read `web/i18n-config/languages.ts`. + Extract all languages with `supported: true`. + + ### Step 1.3: Run i18n:check for Each Language + ```bash + pnpm --dir web install --frozen-lockfile + pnpm --dir web run i18n:check + ``` + + This will report: + - Missing keys (need to ADD) + - Extra keys (need to DELETE) + + ### Step 1.4: Generate Change Report + + Create a structured report identifying: + ``` + ╔══════════════════════════════════════════════════════════════╗ + ║ I18N SYNC CHANGE REPORT ║ + ╠══════════════════════════════════════════════════════════════╣ + ║ Files to process: [list] ║ + ║ Languages to sync: [list] ║ + ╠══════════════════════════════════════════════════════════════╣ + ║ ADD (New Keys): ║ + ║ - [filename].[key]: "English value" ║ + ║ ... ║ + ╠══════════════════════════════════════════════════════════════╣ + ║ UPDATE (Modified Keys - MUST re-translate): ║ + ║ - [filename].[key]: "Old value" → "New value" ║ + ║ ... ║ + ╠══════════════════════════════════════════════════════════════╣ + ║ DELETE (Extra Keys): ║ + ║ - [language]/[filename].[key] ║ + ║ ... ║ + ╚══════════════════════════════════════════════════════════════╝ + ``` + + **IMPORTANT**: For UPDATE detection, compare git diff to find keys where + the English value changed. These MUST be re-translated even if target + language already has a translation (it's now stale!). + + ═══════════════════════════════════════════════════════════════ + ║ PHASE 2: SYNC - Execute Changes Based on Report ║ + ═══════════════════════════════════════════════════════════════ + + ### Step 2.1: Process ADD Operations (BATCH per language file) + + **CRITICAL WORKFLOW for efficiency:** + 1. First, translate ALL new keys for ALL languages mentally + 2. Then, for EACH language file, do ONE Edit operation: + - Read the file once + - Insert ALL new keys at the beginning (right after the opening `{`) + - Don't worry about alphabetical order - lint:fix will sort them later + + Example Edit (adding 3 keys to zh-Hans/app.json): + ``` + old_string: '{\n "accessControl"' + new_string: '{\n "newKey1": "translation1",\n "newKey2": "translation2",\n "newKey3": "translation3",\n "accessControl"' + ``` + + **IMPORTANT**: + - ONE Edit per language file (not one Edit per key!) + - Always use the Edit tool. NEVER use bash scripts, node, or jq. + + ### Step 2.2: Process UPDATE Operations + + **IMPORTANT: Special handling for zh-Hans and ja-JP** + If zh-Hans or ja-JP files were ALSO modified in the same push: + - Run: `git diff HEAD~1 --name-only | grep -E "(zh-Hans|ja-JP)"` + - If found, it means someone manually translated them. Apply these rules: + + 1. **Missing keys**: Still ADD them (completeness required) + 2. **Existing translations**: Compare with the NEW English value: + - If translation is **completely wrong** or **unrelated** → Update it + - If translation is **roughly correct** (captures the meaning) → Keep it, respect manual work + - When in doubt, **keep the manual translation** + + Example: + - English changed: "Save" → "Save Changes" + - Manual translation: "保存更改" → Keep it (correct meaning) + - Manual translation: "删除" → Update it (completely wrong) + + For other languages: + Use Edit tool to replace the old value with the new translation. + You can batch multiple updates in one Edit if they are adjacent. + + ### Step 2.3: Process DELETE Operations + For extra keys reported by i18n:check: + - Run: `pnpm --dir web run i18n:check --auto-remove` + - Or manually remove from target language JSON files + + ## Translation Guidelines + + - PRESERVE all placeholders exactly as-is: + - `{{variable}}` - Mustache interpolation + - `${variable}` - Template literal + - `content` - HTML tags + - `_one`, `_other` - Pluralization suffixes (these are KEY suffixes, not values) + - Use appropriate language register (formal/informal) based on existing translations + - Match existing translation style in each language + - Technical terms: check existing conventions per language + - For CJK languages: no spaces between characters unless necessary + - For RTL languages (ar-TN, fa-IR): ensure proper text handling + + ## Output Format Requirements + - Alphabetical key ordering (if original file uses it) + - 2-space indentation + - Trailing newline at end of file + - Valid JSON (use proper escaping for special characters) + + ═══════════════════════════════════════════════════════════════ + ║ PHASE 3: RE-VERIFY - Confirm All Issues Resolved ║ + ═══════════════════════════════════════════════════════════════ + + ### Step 3.1: Run Lint Fix (IMPORTANT!) + ```bash + pnpm --dir web lint:fix --quiet -- 'i18n/**/*.json' + ``` + This ensures: + - JSON keys are sorted alphabetically (jsonc/sort-keys rule) + - Valid i18n keys (dify-i18n/valid-i18n-keys rule) + - No extra keys (dify-i18n/no-extra-keys rule) + + ### Step 3.2: Run Final i18n Check + ```bash + pnpm --dir web run i18n:check + ``` + + ### Step 3.3: Fix Any Remaining Issues + If check reports issues: + - Go back to PHASE 2 for unresolved items + - Repeat until check passes + + ### Step 3.4: Generate Final Summary + ``` + ╔══════════════════════════════════════════════════════════════╗ + ║ SYNC COMPLETED SUMMARY ║ + ╠══════════════════════════════════════════════════════════════╣ + ║ Language │ Added │ Updated │ Deleted │ Status ║ + ╠══════════════════════════════════════════════════════════════╣ + ║ zh-Hans │ 5 │ 2 │ 1 │ ✓ Complete ║ + ║ ja-JP │ 5 │ 2 │ 1 │ ✓ Complete ║ + ║ ... │ ... │ ... │ ... │ ... ║ + ╠══════════════════════════════════════════════════════════════╣ + ║ i18n:check │ PASSED - All keys in sync ║ + ╚══════════════════════════════════════════════════════════════╝ + ``` + + ## Mode-Specific Behavior + + **SYNC_MODE = "incremental"** (default): + - Focus on keys identified from git diff + - Also check i18n:check output for any missing/extra keys + - Efficient for small changes + + **SYNC_MODE = "full"**: + - Compare ALL keys between en-US and each language + - Run i18n:check to identify all discrepancies + - Use for first-time sync or fixing historical issues + + ## Important Notes + + 1. Always run i18n:check BEFORE and AFTER making changes + 2. The check script is the source of truth for missing/extra keys + 3. For UPDATE scenario: git diff is the source of truth for changed values + 4. Create a single commit with all translation changes + 5. If any translation fails, continue with others and report failures + + ═══════════════════════════════════════════════════════════════ + ║ PHASE 4: COMMIT AND CREATE PR ║ + ═══════════════════════════════════════════════════════════════ + + After all translations are complete and verified: + + ### Step 4.1: Check for changes + ```bash + git status --porcelain + ``` + + If there are changes: + + ### Step 4.2: Create a new branch and commit + Run these git commands ONE BY ONE (not combined with &&): + + 1. Create branch: + ```bash + git checkout -b "chore/i18n-sync-$(date +%Y%m%d-%H%M%S)" + ``` + + 2. Stage changes: + ```bash + git add web/i18n/ + ``` + + 3. Commit (use heredoc for multi-line message): + ```bash + git commit -m "chore(i18n): sync translations with en-US - Mode: ${{ steps.detect_changes.outputs.SYNC_MODE }}" + ``` + + 4. Push: + ```bash + git push origin HEAD + ``` + + ### Step 4.3: Create Pull Request + ```bash + gh pr create \ + --title "chore(i18n): sync translations with en-US" \ + --body "## Summary + + This PR was automatically generated to sync i18n translation files. + + ### Changes + - Mode: ${{ steps.detect_changes.outputs.SYNC_MODE }} + - Files processed: ${{ steps.detect_changes.outputs.CHANGED_FILES }} + + ### Verification + - [x] \`i18n:check\` passed + - [x] \`lint:fix\` applied + + 🤖 Generated with Claude Code GitHub Action" \ + --base main + ``` + + claude_args: | + --max-turns 150 + --allowedTools "Read,Write,Edit,Bash(git *),Bash(git:*),Bash(gh *),Bash(gh:*),Bash(pnpm *),Bash(pnpm:*),Glob,Grep" diff --git a/web/i18n-config/README.md b/web/i18n-config/README.md index 96c7157114..c90904459c 100644 --- a/web/i18n-config/README.md +++ b/web/i18n-config/README.md @@ -158,10 +158,32 @@ We have a list of languages that we support in the `languages.ts` file. But some ## Utility scripts -- Auto-fill translations: `pnpm run i18n:gen --file app common --lang zh-Hans ja-JP [--dry-run]` - - Use space-separated values; repeat `--file` / `--lang` as needed. Defaults to all en-US files and all supported locales except en-US. - - Protects placeholders (`{{var}}`, `${var}`, ``) before translation and restores them after. - Check missing/extra keys: `pnpm run i18n:check --file app billing --lang zh-Hans [--auto-remove]` - Use space-separated values; repeat `--file` / `--lang` as needed. Returns non-zero on missing/extra keys; `--auto-remove` deletes extra keys automatically. -Workflows: `.github/workflows/translate-i18n-base-on-english.yml` auto-runs the translation generator on `web/i18n/en-US/*.json` changes to main. `i18n:check` is a manual script (not run in CI). +## Automatic Translation + +Translation is handled automatically by Claude Code GitHub Actions. When changes are pushed to `web/i18n/en-US/*.json` on the main branch: + +1. Claude Code analyzes the git diff to detect changes +1. Identifies three types of changes: + - **ADD**: New keys that need translation + - **UPDATE**: Modified keys that need re-translation (even if target language has existing translation) + - **DELETE**: Removed keys that need to be deleted from other languages +1. Runs `i18n:check` to verify the initial sync status. +1. Translates missing/updated keys while preserving placeholders (`{{var}}`, `${var}`, ``) and removes deleted keys. +1. Runs `lint:fix` to sort JSON keys and `i18n:check` again to ensure everything is synchronized. +1. Creates a PR with the translations. + +### Manual Trigger + +To manually trigger translation: + +1. Go to Actions > "Translate i18n Files with Claude Code" +1. Click "Run workflow" +1. Optionally configure: + - **files**: Specific files to translate (space-separated, e.g., "app common") + - **languages**: Specific languages to translate (space-separated, e.g., "zh-Hans ja-JP") + - **mode**: `incremental` (default, only changes) or `full` (check all keys) + +Workflow: `.github/workflows/translate-i18n-claude.yml` diff --git a/web/package.json b/web/package.json index e94f2fd441..dcebe742a7 100644 --- a/web/package.json +++ b/web/package.json @@ -40,7 +40,6 @@ "gen-icons": "node ./scripts/gen-icons.mjs && eslint --fix app/components/base/icons/src/", "uglify-embed": "node ./bin/uglify-embed", "i18n:check": "tsx ./scripts/check-i18n.js", - "i18n:gen": "tsx ./scripts/auto-gen-i18n.js", "test": "vitest run", "test:coverage": "vitest run --coverage", "test:watch": "vitest --watch", @@ -196,7 +195,6 @@ "@vitest/coverage-v8": "4.0.16", "autoprefixer": "^10.4.21", "babel-loader": "^10.0.0", - "bing-translate-api": "^4.1.0", "code-inspector-plugin": "1.2.9", "cross-env": "^10.1.0", "eslint": "^9.39.2", diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index a2e95dadd7..5ad6d0481b 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -481,9 +481,6 @@ importers: babel-loader: specifier: ^10.0.0 version: 10.0.0(@babel/core@7.28.5)(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3)) - bing-translate-api: - specifier: ^4.1.0 - version: 4.2.0 code-inspector-plugin: specifier: 1.2.9 version: 1.2.9 @@ -3160,10 +3157,6 @@ packages: resolution: {integrity: sha512-TeheYy0ILzBEI/CO55CP6zJCSdSWeRtGnHy8U8dWSUH4I68iqTsy7HkMktR4xakThc9jotkPQUXT4ITdbV7cHA==} engines: {node: '>=18'} - '@sindresorhus/is@4.6.0': - resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} - engines: {node: '>=10'} - '@solid-primitives/event-listener@2.4.3': resolution: {integrity: sha512-h4VqkYFv6Gf+L7SQj+Y6puigL/5DIi7x5q07VZET7AWcS+9/G3WfIE9WheniHWJs51OEkRB43w6lDys5YeFceg==} peerDependencies: @@ -3324,10 +3317,6 @@ packages: '@swc/helpers@0.5.17': resolution: {integrity: sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==} - '@szmarczak/http-timer@4.0.6': - resolution: {integrity: sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==} - engines: {node: '>=10'} - '@tailwindcss/typography@0.5.19': resolution: {integrity: sha512-w31dd8HOx3k9vPtcQh5QHP9GwKcgbMp87j58qi6xgiBnFFtKEAgCWnDw4qUT8aHwkCp8bKvb/KGKWWHedP0AAg==} peerDependencies: @@ -3511,9 +3500,6 @@ packages: '@types/babel__traverse@7.28.0': resolution: {integrity: sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==} - '@types/cacheable-request@6.0.3': - resolution: {integrity: sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==} - '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -3652,9 +3638,6 @@ packages: '@types/html-minifier-terser@6.1.0': resolution: {integrity: sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg==} - '@types/http-cache-semantics@4.0.4': - resolution: {integrity: sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==} - '@types/js-cookie@3.0.6': resolution: {integrity: sha512-wkw9yd1kEXOPnvEeEV1Go1MmxtBJL0RR79aOTAApecWFVu7w0NNXNqhcWgvw2YgZDYadliXkl14pa3WXw5jlCQ==} @@ -3667,9 +3650,6 @@ packages: '@types/katex@0.16.7': resolution: {integrity: sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==} - '@types/keyv@3.1.4': - resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==} - '@types/mdast@4.0.4': resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} @@ -3732,9 +3712,6 @@ packages: '@types/resolve@1.20.6': resolution: {integrity: sha512-A4STmOXPhMUtHH+S6ymgE2GiBSMqf4oTvcQZMcHzokuTLVYzXTB8ttjcgxOVaAp2lGwEdzZ0J+cRbbeevQj1UQ==} - '@types/responselike@1.0.3': - resolution: {integrity: sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw==} - '@types/semver@7.7.1': resolution: {integrity: sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==} @@ -4338,9 +4315,6 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} - bing-translate-api@4.2.0: - resolution: {integrity: sha512-7a9yo1NbGcHPS8zXTdz8tCOymHZp2pvCuYOChCaXKjOX8EIwdV3SLd4D7RGIqZt1UhffypYBUcAV2gDcTgK0rA==} - bippy@0.3.34: resolution: {integrity: sha512-vmptmU/20UdIWHHhq7qCSHhHzK7Ro3YJ1utU0fBG7ujUc58LEfTtilKxcF0IOgSjT5XLcm7CBzDjbv4lcKApGQ==} peerDependencies: @@ -4427,14 +4401,6 @@ packages: resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} engines: {node: '>=8'} - cacheable-lookup@5.0.4: - resolution: {integrity: sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==} - engines: {node: '>=10.6.0'} - - cacheable-request@7.0.4: - resolution: {integrity: sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg==} - engines: {node: '>=8'} - callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} @@ -4588,9 +4554,6 @@ packages: client-only@0.0.1: resolution: {integrity: sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==} - clone-response@1.0.3: - resolution: {integrity: sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==} - clsx@1.2.1: resolution: {integrity: sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg==} engines: {node: '>=6'} @@ -4996,10 +4959,6 @@ packages: resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} engines: {node: '>=0.10.0'} - defer-to-connect@2.0.1: - resolution: {integrity: sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==} - engines: {node: '>=10'} - define-lazy-prop@2.0.0: resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==} engines: {node: '>=8'} @@ -5712,10 +5671,6 @@ packages: get-own-enumerable-property-symbols@3.0.2: resolution: {integrity: sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==} - get-stream@5.2.0: - resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} - engines: {node: '>=8'} - get-stream@8.0.1: resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} engines: {node: '>=16'} @@ -5772,10 +5727,6 @@ packages: peerDependencies: csstype: ^3.0.10 - got@11.8.6: - resolution: {integrity: sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g==} - engines: {node: '>=10.19.0'} - graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} @@ -5920,17 +5871,10 @@ packages: htmlparser2@6.1.0: resolution: {integrity: sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==} - http-cache-semantics@4.2.0: - resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} - http-proxy-agent@7.0.2: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} - http2-wrapper@1.0.3: - resolution: {integrity: sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==} - engines: {node: '>=10.19.0'} - https-browserify@1.0.0: resolution: {integrity: sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==} @@ -6453,10 +6397,6 @@ packages: lower-case@2.0.2: resolution: {integrity: sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==} - lowercase-keys@2.0.0: - resolution: {integrity: sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==} - engines: {node: '>=8'} - lowlight@1.20.0: resolution: {integrity: sha512-8Ktj+prEb1RoCPkEOrPMYUN/nCggB7qAWe3a7OpMjWQkh3l2RD5wKRQ+o8Q8YuI9RG/xs95waaI/E6ym/7NsTw==} @@ -6729,10 +6669,6 @@ packages: resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} engines: {node: '>=18'} - mimic-response@1.0.1: - resolution: {integrity: sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==} - engines: {node: '>=4'} - mimic-response@3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} @@ -6880,10 +6816,6 @@ packages: resolution: {integrity: sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==} engines: {node: '>=0.10.0'} - normalize-url@6.1.0: - resolution: {integrity: sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==} - engines: {node: '>=10'} - normalize-wheel@1.0.1: resolution: {integrity: sha512-1OnlAPZ3zgrk8B91HyRj+eVv+kS5u+Z0SCsak6Xil/kmgEia50ga7zfkumayonZrImffAxPU/5WcyGhzetHNPA==} @@ -6964,10 +6896,6 @@ packages: oxc-resolver@11.15.0: resolution: {integrity: sha512-Hk2J8QMYwmIO9XTCUiOH00+Xk2/+aBxRUnhrSlANDyCnLYc32R1WSIq1sU2yEdlqd53FfMpPEpnBYIKQMzliJw==} - p-cancelable@2.1.1: - resolution: {integrity: sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==} - engines: {node: '>=8'} - p-limit@2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} @@ -7359,10 +7287,6 @@ packages: queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} - quick-lru@5.1.1: - resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} - engines: {node: '>=10'} - randombytes@2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} @@ -7705,9 +7629,6 @@ packages: resize-observer-polyfill@1.5.1: resolution: {integrity: sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==} - resolve-alpn@1.2.1: - resolution: {integrity: sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==} - resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} @@ -7724,9 +7645,6 @@ packages: engines: {node: '>= 0.4'} hasBin: true - responselike@2.0.1: - resolution: {integrity: sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw==} - restore-cursor@5.1.0: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} @@ -11717,8 +11635,6 @@ snapshots: '@sindresorhus/base62@1.0.0': {} - '@sindresorhus/is@4.6.0': {} - '@solid-primitives/event-listener@2.4.3(solid-js@1.9.10)': dependencies: '@solid-primitives/utils': 6.3.2(solid-js@1.9.10) @@ -11971,10 +11887,6 @@ snapshots: dependencies: tslib: 2.8.1 - '@szmarczak/http-timer@4.0.6': - dependencies: - defer-to-connect: 2.0.1 - '@tailwindcss/typography@0.5.19(tailwindcss@3.4.18(tsx@4.21.0)(yaml@2.8.2))': dependencies: postcss-selector-parser: 6.0.10 @@ -12199,13 +12111,6 @@ snapshots: dependencies: '@babel/types': 7.28.5 - '@types/cacheable-request@6.0.3': - dependencies: - '@types/http-cache-semantics': 4.0.4 - '@types/keyv': 3.1.4 - '@types/node': 18.15.0 - '@types/responselike': 1.0.3 - '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 @@ -12373,8 +12278,6 @@ snapshots: '@types/html-minifier-terser@6.1.0': {} - '@types/http-cache-semantics@4.0.4': {} - '@types/js-cookie@3.0.6': {} '@types/js-yaml@4.0.9': {} @@ -12383,10 +12286,6 @@ snapshots: '@types/katex@0.16.7': {} - '@types/keyv@3.1.4': - dependencies: - '@types/node': 18.15.0 - '@types/mdast@4.0.4': dependencies: '@types/unist': 3.0.3 @@ -12450,10 +12349,6 @@ snapshots: '@types/resolve@1.20.6': {} - '@types/responselike@1.0.3': - dependencies: - '@types/node': 18.15.0 - '@types/semver@7.7.1': {} '@types/sortablejs@1.15.9': {} @@ -13160,10 +13055,6 @@ snapshots: binary-extensions@2.3.0: {} - bing-translate-api@4.2.0: - dependencies: - got: 11.8.6 - bippy@0.3.34(@types/react@19.2.7)(react@19.2.3): dependencies: '@types/react-reconciler': 0.28.9(@types/react@19.2.7) @@ -13273,18 +13164,6 @@ snapshots: cac@6.7.14: {} - cacheable-lookup@5.0.4: {} - - cacheable-request@7.0.4: - dependencies: - clone-response: 1.0.3 - get-stream: 5.2.0 - http-cache-semantics: 4.2.0 - keyv: 4.5.4 - lowercase-keys: 2.0.0 - normalize-url: 6.1.0 - responselike: 2.0.1 - callsites@3.1.0: {} camel-case@4.1.2: @@ -13425,10 +13304,6 @@ snapshots: client-only@0.0.1: {} - clone-response@1.0.3: - dependencies: - mimic-response: 1.0.1 - clsx@1.2.1: {} clsx@2.1.1: {} @@ -13859,6 +13734,7 @@ snapshots: decompress-response@6.0.0: dependencies: mimic-response: 3.1.0 + optional: true dedent@0.7.0: {} @@ -13871,8 +13747,6 @@ snapshots: deepmerge@4.3.1: {} - defer-to-connect@2.0.1: {} - define-lazy-prop@2.0.0: {} del@4.1.1: @@ -14013,6 +13887,7 @@ snapshots: end-of-stream@1.4.5: dependencies: once: 1.4.0 + optional: true endent@2.1.0: dependencies: @@ -14799,10 +14674,6 @@ snapshots: get-own-enumerable-property-symbols@3.0.2: {} - get-stream@5.2.0: - dependencies: - pump: 3.0.3 - get-stream@8.0.1: {} get-tsconfig@4.13.0: @@ -14862,20 +14733,6 @@ snapshots: dependencies: csstype: 3.2.3 - got@11.8.6: - dependencies: - '@sindresorhus/is': 4.6.0 - '@szmarczak/http-timer': 4.0.6 - '@types/cacheable-request': 6.0.3 - '@types/responselike': 1.0.3 - cacheable-lookup: 5.0.4 - cacheable-request: 7.0.4 - decompress-response: 6.0.0 - http2-wrapper: 1.0.3 - lowercase-keys: 2.0.0 - p-cancelable: 2.1.1 - responselike: 2.0.1 - graceful-fs@4.2.11: {} graphemer@1.4.0: {} @@ -15120,8 +14977,6 @@ snapshots: domutils: 2.8.0 entities: 2.2.0 - http-cache-semantics@4.2.0: {} - http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.4 @@ -15129,11 +14984,6 @@ snapshots: transitivePeerDependencies: - supports-color - http2-wrapper@1.0.3: - dependencies: - quick-lru: 5.1.1 - resolve-alpn: 1.2.1 - https-browserify@1.0.0: {} https-proxy-agent@7.0.6: @@ -15604,8 +15454,6 @@ snapshots: dependencies: tslib: 2.8.1 - lowercase-keys@2.0.0: {} - lowlight@1.20.0: dependencies: fault: 1.0.4 @@ -16186,9 +16034,8 @@ snapshots: mimic-function@5.0.1: {} - mimic-response@1.0.1: {} - - mimic-response@3.1.0: {} + mimic-response@3.1.0: + optional: true min-indent@1.0.1: {} @@ -16360,8 +16207,6 @@ snapshots: normalize-range@0.1.2: {} - normalize-url@6.1.0: {} - normalize-wheel@1.0.1: {} npm-run-path@5.3.0: @@ -16445,8 +16290,6 @@ snapshots: '@oxc-resolver/binding-win32-ia32-msvc': 11.15.0 '@oxc-resolver/binding-win32-x64-msvc': 11.15.0 - p-cancelable@2.1.1: {} - p-limit@2.3.0: dependencies: p-try: 2.2.0 @@ -16809,6 +16652,7 @@ snapshots: dependencies: end-of-stream: 1.4.5 once: 1.4.0 + optional: true punycode@1.4.1: {} @@ -16828,8 +16672,6 @@ snapshots: queue-microtask@1.2.3: {} - quick-lru@5.1.1: {} - randombytes@2.1.0: dependencies: safe-buffer: 5.2.1 @@ -17296,8 +17138,6 @@ snapshots: resize-observer-polyfill@1.5.1: {} - resolve-alpn@1.2.1: {} - resolve-from@4.0.0: {} resolve-pkg-maps@1.0.0: {} @@ -17316,10 +17156,6 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 - responselike@2.0.1: - dependencies: - lowercase-keys: 2.0.0 - restore-cursor@5.1.0: dependencies: onetime: 7.0.0 diff --git a/web/scripts/auto-gen-i18n.js b/web/scripts/auto-gen-i18n.js deleted file mode 100644 index bd73a18ab8..0000000000 --- a/web/scripts/auto-gen-i18n.js +++ /dev/null @@ -1,336 +0,0 @@ -import fs from 'node:fs' -import path from 'node:path' -import { fileURLToPath } from 'node:url' -import { translate } from 'bing-translate-api' -import data from '../i18n-config/languages' - -const __filename = fileURLToPath(import.meta.url) -const __dirname = path.dirname(__filename) - -const targetLanguage = 'en-US' -const i18nFolder = '../i18n' // Path to i18n folder relative to this script -// https://github.com/plainheart/bing-translate-api/blob/master/src/met/lang.json -const languageKeyMap = data.languages.reduce((map, language) => { - if (language.supported) { - if (language.value === 'zh-Hans' || language.value === 'zh-Hant') - map[language.value] = language.value - else - map[language.value] = language.value.split('-')[0] - } - - return map -}, {}) - -const supportedLanguages = Object.keys(languageKeyMap) - -function parseArgs(argv) { - const args = { - files: [], - languages: [], - isDryRun: false, - help: false, - errors: [], - } - - const collectValues = (startIndex) => { - const values = [] - let cursor = startIndex + 1 - while (cursor < argv.length && !argv[cursor].startsWith('--')) { - const value = argv[cursor].trim() - if (value) - values.push(value) - cursor++ - } - return { values, nextIndex: cursor - 1 } - } - - const validateList = (values, flag) => { - if (!values.length) { - args.errors.push(`${flag} requires at least one value. Example: ${flag} app billing`) - return false - } - - const invalid = values.find(value => value.includes(',')) - if (invalid) { - args.errors.push(`${flag} expects space-separated values. Example: ${flag} app billing`) - return false - } - - return true - } - - for (let index = 2; index < argv.length; index++) { - const arg = argv[index] - - if (arg === '--dry-run') { - args.isDryRun = true - continue - } - - if (arg === '--help' || arg === '-h') { - args.help = true - break - } - - if (arg.startsWith('--file=')) { - args.errors.push('--file expects space-separated values. Example: --file app billing') - continue - } - - if (arg === '--file') { - const { values, nextIndex } = collectValues(index) - if (validateList(values, '--file')) - args.files.push(...values) - index = nextIndex - continue - } - - if (arg.startsWith('--lang=')) { - args.errors.push('--lang expects space-separated values. Example: --lang zh-Hans ja-JP') - continue - } - - if (arg === '--lang') { - const { values, nextIndex } = collectValues(index) - if (validateList(values, '--lang')) - args.languages.push(...values) - index = nextIndex - continue - } - } - - return args -} - -function printHelp() { - console.log(`Usage: pnpm run i18n:gen [options] - -Options: - --file Process only specific files; provide space-separated names and repeat --file if needed - --lang Process only specific locales; provide space-separated locales and repeat --lang if needed (default: all supported except en-US) - --dry-run Preview changes without writing files - -h, --help Show help - -Examples: - pnpm run i18n:gen --file app common --lang zh-Hans ja-JP - pnpm run i18n:gen --dry-run -`) -} - -function protectPlaceholders(text) { - const placeholders = [] - let safeText = text - const patterns = [ - /\{\{[^{}]+\}\}/g, // mustache - /\$\{[^{}]+\}/g, // template expressions - /<[^>]+>/g, // html-like tags - ] - - patterns.forEach((pattern) => { - safeText = safeText.replace(pattern, (match) => { - const token = `__PH_${placeholders.length}__` - placeholders.push({ token, value: match }) - return token - }) - }) - - return { - safeText, - restore(translated) { - return placeholders.reduce((result, { token, value }) => result.replace(new RegExp(token, 'g'), value), translated) - }, - } -} - -async function translateText(source, toLanguage) { - if (typeof source !== 'string') - return { value: source, skipped: false } - - const trimmed = source.trim() - if (!trimmed) - return { value: source, skipped: false } - - const { safeText, restore } = protectPlaceholders(source) - - try { - const { translation } = await translate(safeText, null, languageKeyMap[toLanguage]) - return { value: restore(translation), skipped: false } - } - catch (error) { - console.error(`❌ Error translating to ${toLanguage}:`, error.message) - return { value: source, skipped: true, error: error.message } - } -} - -async function translateMissingKeys(sourceObj, targetObject, toLanguage) { - const skippedKeys = [] - const translatedKeys = [] - - for (const key of Object.keys(sourceObj)) { - const sourceValue = sourceObj[key] - const targetValue = targetObject[key] - - // Skip if target already has this key - if (targetValue !== undefined) - continue - - const translationResult = await translateText(sourceValue, toLanguage) - targetObject[key] = translationResult.value ?? '' - if (translationResult.skipped) - skippedKeys.push(`${key}: ${sourceValue}`) - else - translatedKeys.push(key) - } - - return { skipped: skippedKeys, translated: translatedKeys } -} -async function autoGenTrans(fileName, toGenLanguage, isDryRun = false) { - const fullKeyFilePath = path.resolve(__dirname, i18nFolder, targetLanguage, `${fileName}.json`) - const toGenLanguageFilePath = path.resolve(__dirname, i18nFolder, toGenLanguage, `${fileName}.json`) - - try { - const content = fs.readFileSync(fullKeyFilePath, 'utf8') - const fullKeyContent = JSON.parse(content) - - if (!fullKeyContent || typeof fullKeyContent !== 'object') - throw new Error(`Failed to extract translation object from ${fullKeyFilePath}`) - - // if toGenLanguageFilePath does not exist, create it with empty object - let toGenOutPut = {} - if (fs.existsSync(toGenLanguageFilePath)) { - const existingContent = fs.readFileSync(toGenLanguageFilePath, 'utf8') - toGenOutPut = JSON.parse(existingContent) - } - - console.log(`\n🌍 Processing ${fileName} for ${toGenLanguage}...`) - const result = await translateMissingKeys(fullKeyContent, toGenOutPut, toGenLanguage) - - // Generate summary report - console.log(`\n📊 Translation Summary for ${fileName} -> ${toGenLanguage}:`) - console.log(` ✅ Translated: ${result.translated.length} keys`) - console.log(` ⏭️ Skipped: ${result.skipped.length} keys`) - - if (result.skipped.length > 0) { - console.log(`\n⚠️ Skipped keys in ${fileName} (${toGenLanguage}):`) - result.skipped.slice(0, 5).forEach(item => console.log(` - ${item}`)) - if (result.skipped.length > 5) - console.log(` ... and ${result.skipped.length - 5} more`) - } - - const res = `${JSON.stringify(toGenOutPut, null, 2)}\n` - - if (!isDryRun) { - fs.writeFileSync(toGenLanguageFilePath, res) - console.log(`💾 Saved translations to ${toGenLanguageFilePath}`) - } - else { - console.log(`🔍 [DRY RUN] Would save translations to ${toGenLanguageFilePath}`) - } - - return result - } - catch (error) { - console.error(`Error processing file ${fullKeyFilePath}:`, error.message) - throw error - } -} - -// Add command line argument support -const args = parseArgs(process.argv) -const isDryRun = args.isDryRun -const targetFiles = args.files -const targetLangs = args.languages - -// Rate limiting helper -function delay(ms) { - return new Promise(resolve => setTimeout(resolve, ms)) -} - -async function main() { - if (args.help) { - printHelp() - return - } - - if (args.errors.length) { - args.errors.forEach(message => console.error(`❌ ${message}`)) - printHelp() - process.exit(1) - return - } - - console.log('🚀 Starting i18n:gen script...') - console.log(`📋 Mode: ${isDryRun ? 'DRY RUN (no files will be modified)' : 'LIVE MODE'}`) - - const filesInEn = fs - .readdirSync(path.resolve(__dirname, i18nFolder, targetLanguage)) - .filter(file => /\.json$/.test(file)) // Only process .json files - .map(file => file.replace(/\.json$/, '')) - - // Filter by target files if specified - const filesToProcess = targetFiles.length > 0 ? filesInEn.filter(f => targetFiles.includes(f)) : filesInEn - const languagesToProcess = Array.from(new Set((targetLangs.length > 0 ? targetLangs : supportedLanguages) - .filter(lang => lang !== targetLanguage))) - - const unknownLangs = languagesToProcess.filter(lang => !languageKeyMap[lang]) - if (unknownLangs.length) { - console.error(`❌ Unsupported languages: ${unknownLangs.join(', ')}`) - process.exit(1) - } - - if (!filesToProcess.length) { - console.log('ℹ️ No files to process based on provided arguments') - return - } - - if (!languagesToProcess.length) { - console.log('ℹ️ No languages to process (did you only specify en-US?)') - return - } - - console.log(`📁 Files to process: ${filesToProcess.join(', ')}`) - console.log(`🌍 Languages to process: ${languagesToProcess.join(', ')}`) - - let totalTranslated = 0 - let totalSkipped = 0 - let totalErrors = 0 - - // Process files sequentially to avoid API rate limits - for (const file of filesToProcess) { - console.log(`\n📄 Processing file: ${file}`) - - // Process languages with rate limiting - for (const language of languagesToProcess) { - try { - const result = await autoGenTrans(file, language, isDryRun) - totalTranslated += result.translated.length - totalSkipped += result.skipped.length - - // Rate limiting: wait 500ms between language processing - await delay(500) - } - catch (e) { - console.error(`❌ Error translating ${file} to ${language}:`, e.message) - totalErrors++ - } - } - } - - // Final summary - console.log('\n🎉 Auto-translation completed!') - console.log('📊 Final Summary:') - console.log(` ✅ Total keys translated: ${totalTranslated}`) - console.log(` ⏭️ Total keys skipped: ${totalSkipped}`) - console.log(` ❌ Total errors: ${totalErrors}`) - - if (isDryRun) - console.log('\n💡 This was a dry run. To actually translate, run without --dry-run flag.') - - if (totalErrors > 0) - process.exitCode = 1 -} - -main().catch((error) => { - console.error('❌ Unexpected error:', error.message) - process.exit(1) -}) From 885f226f77de86dad7cc67f6018f38cb3bdfd092 Mon Sep 17 00:00:00 2001 From: Asuka Minato Date: Wed, 7 Jan 2026 22:18:02 +0900 Subject: [PATCH 08/21] =?UTF-8?q?refactor:=20split=20changes=20for=20api/c?= =?UTF-8?q?ontrollers/console/workspace/trigger=E2=80=A6=20(#30627)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../console/workspace/trigger_providers.py | 133 +++++++----------- .../trigger/trigger_provider_service.py | 2 +- 2 files changed, 52 insertions(+), 83 deletions(-) diff --git a/api/controllers/console/workspace/trigger_providers.py b/api/controllers/console/workspace/trigger_providers.py index c13bfd986e..6b642af613 100644 --- a/api/controllers/console/workspace/trigger_providers.py +++ b/api/controllers/console/workspace/trigger_providers.py @@ -1,14 +1,14 @@ import logging -from collections.abc import Mapping from typing import Any from flask import make_response, redirect, request -from flask_restx import Resource, reqparse -from pydantic import BaseModel, Field, model_validator +from flask_restx import Resource +from pydantic import BaseModel, model_validator from sqlalchemy.orm import Session from werkzeug.exceptions import BadRequest, Forbidden from configs import dify_config +from controllers.common.schema import register_schema_models from controllers.web.error import NotFoundError from core.model_runtime.utils.encoders import jsonable_encoder from core.plugin.entities.plugin_daemon import CredentialType @@ -35,35 +35,38 @@ from ..wraps import ( logger = logging.getLogger(__name__) -class TriggerSubscriptionUpdateRequest(BaseModel): - """Request payload for updating a trigger subscription""" +class TriggerSubscriptionBuilderCreatePayload(BaseModel): + credential_type: str = CredentialType.UNAUTHORIZED - name: str | None = Field(default=None, description="The name for the subscription") - credentials: Mapping[str, Any] | None = Field(default=None, description="The credentials for the subscription") - parameters: Mapping[str, Any] | None = Field(default=None, description="The parameters for the subscription") - properties: Mapping[str, Any] | None = Field(default=None, description="The properties for the subscription") + +class TriggerSubscriptionBuilderVerifyPayload(BaseModel): + credentials: dict[str, Any] + + +class TriggerSubscriptionBuilderUpdatePayload(BaseModel): + name: str | None = None + parameters: dict[str, Any] | None = None + properties: dict[str, Any] | None = None + credentials: dict[str, Any] | None = None @model_validator(mode="after") def check_at_least_one_field(self): - if all(v is None for v in (self.name, self.credentials, self.parameters, self.properties)): + if all(v is None for v in self.model_dump().values()): raise ValueError("At least one of name, credentials, parameters, or properties must be provided") return self -class TriggerSubscriptionVerifyRequest(BaseModel): - """Request payload for verifying subscription credentials.""" - - credentials: Mapping[str, Any] = Field(description="The credentials to verify") +class TriggerOAuthClientPayload(BaseModel): + client_params: dict[str, Any] | None = None + enabled: bool | None = None -console_ns.schema_model( - TriggerSubscriptionUpdateRequest.__name__, - TriggerSubscriptionUpdateRequest.model_json_schema(ref_template="#/definitions/{model}"), -) - -console_ns.schema_model( - TriggerSubscriptionVerifyRequest.__name__, - TriggerSubscriptionVerifyRequest.model_json_schema(ref_template="#/definitions/{model}"), +register_schema_models( + console_ns, + TriggerSubscriptionBuilderCreatePayload, + TriggerSubscriptionBuilderVerifyPayload, + TriggerSubscriptionBuilderUpdatePayload, + TriggerOAuthClientPayload, ) @@ -132,16 +135,11 @@ class TriggerSubscriptionListApi(Resource): raise -parser = reqparse.RequestParser().add_argument( - "credential_type", type=str, required=False, nullable=True, location="json" -) - - @console_ns.route( "/workspaces/current/trigger-provider//subscriptions/builder/create", ) class TriggerSubscriptionBuilderCreateApi(Resource): - @console_ns.expect(parser) + @console_ns.expect(console_ns.models[TriggerSubscriptionBuilderCreatePayload.__name__]) @setup_required @login_required @edit_permission_required @@ -151,10 +149,10 @@ class TriggerSubscriptionBuilderCreateApi(Resource): user = current_user assert user.current_tenant_id is not None - args = parser.parse_args() + payload = TriggerSubscriptionBuilderCreatePayload.model_validate(console_ns.payload or {}) try: - credential_type = CredentialType.of(args.get("credential_type") or CredentialType.UNAUTHORIZED.value) + credential_type = CredentialType.of(payload.credential_type) subscription_builder = TriggerSubscriptionBuilderService.create_trigger_subscription_builder( tenant_id=user.current_tenant_id, user_id=user.id, @@ -182,18 +180,11 @@ class TriggerSubscriptionBuilderGetApi(Resource): ) -parser_api = ( - reqparse.RequestParser() - # The credentials of the subscription builder - .add_argument("credentials", type=dict, required=False, nullable=True, location="json") -) - - @console_ns.route( "/workspaces/current/trigger-provider//subscriptions/builder/verify-and-update/", ) -class TriggerSubscriptionBuilderVerifyAndUpdateApi(Resource): - @console_ns.expect(parser_api) +class TriggerSubscriptionBuilderVerifyApi(Resource): + @console_ns.expect(console_ns.models[TriggerSubscriptionBuilderVerifyPayload.__name__]) @setup_required @login_required @edit_permission_required @@ -203,7 +194,7 @@ class TriggerSubscriptionBuilderVerifyAndUpdateApi(Resource): user = current_user assert user.current_tenant_id is not None - args = parser_api.parse_args() + payload = TriggerSubscriptionBuilderVerifyPayload.model_validate(console_ns.payload or {}) try: # Use atomic update_and_verify to prevent race conditions @@ -213,7 +204,7 @@ class TriggerSubscriptionBuilderVerifyAndUpdateApi(Resource): provider_id=TriggerProviderID(provider), subscription_builder_id=subscription_builder_id, subscription_builder_updater=SubscriptionBuilderUpdater( - credentials=args.get("credentials", None), + credentials=payload.credentials, ), ) except Exception as e: @@ -221,24 +212,11 @@ class TriggerSubscriptionBuilderVerifyAndUpdateApi(Resource): raise ValueError(str(e)) from e -parser_update_api = ( - reqparse.RequestParser() - # The name of the subscription builder - .add_argument("name", type=str, required=False, nullable=True, location="json") - # The parameters of the subscription builder - .add_argument("parameters", type=dict, required=False, nullable=True, location="json") - # The properties of the subscription builder - .add_argument("properties", type=dict, required=False, nullable=True, location="json") - # The credentials of the subscription builder - .add_argument("credentials", type=dict, required=False, nullable=True, location="json") -) - - @console_ns.route( "/workspaces/current/trigger-provider//subscriptions/builder/update/", ) class TriggerSubscriptionBuilderUpdateApi(Resource): - @console_ns.expect(parser_update_api) + @console_ns.expect(console_ns.models[TriggerSubscriptionBuilderUpdatePayload.__name__]) @setup_required @login_required @edit_permission_required @@ -249,7 +227,7 @@ class TriggerSubscriptionBuilderUpdateApi(Resource): assert isinstance(user, Account) assert user.current_tenant_id is not None - args = parser_update_api.parse_args() + payload = TriggerSubscriptionBuilderUpdatePayload.model_validate(console_ns.payload or {}) try: return jsonable_encoder( TriggerSubscriptionBuilderService.update_trigger_subscription_builder( @@ -257,10 +235,10 @@ class TriggerSubscriptionBuilderUpdateApi(Resource): provider_id=TriggerProviderID(provider), subscription_builder_id=subscription_builder_id, subscription_builder_updater=SubscriptionBuilderUpdater( - name=args.get("name", None), - parameters=args.get("parameters", None), - properties=args.get("properties", None), - credentials=args.get("credentials", None), + name=payload.name, + parameters=payload.parameters, + properties=payload.properties, + credentials=payload.credentials, ), ) ) @@ -295,7 +273,7 @@ class TriggerSubscriptionBuilderLogsApi(Resource): "/workspaces/current/trigger-provider//subscriptions/builder/build/", ) class TriggerSubscriptionBuilderBuildApi(Resource): - @console_ns.expect(parser_update_api) + @console_ns.expect(console_ns.models[TriggerSubscriptionBuilderUpdatePayload.__name__]) @setup_required @login_required @edit_permission_required @@ -304,7 +282,7 @@ class TriggerSubscriptionBuilderBuildApi(Resource): """Build a subscription instance for a trigger provider""" user = current_user assert user.current_tenant_id is not None - args = parser_update_api.parse_args() + payload = TriggerSubscriptionBuilderUpdatePayload.model_validate(console_ns.payload or {}) try: # Use atomic update_and_build to prevent race conditions TriggerSubscriptionBuilderService.update_and_build_builder( @@ -313,9 +291,9 @@ class TriggerSubscriptionBuilderBuildApi(Resource): provider_id=TriggerProviderID(provider), subscription_builder_id=subscription_builder_id, subscription_builder_updater=SubscriptionBuilderUpdater( - name=args.get("name", None), - parameters=args.get("parameters", None), - properties=args.get("properties", None), + name=payload.name, + parameters=payload.parameters, + properties=payload.properties, ), ) return 200 @@ -328,7 +306,7 @@ class TriggerSubscriptionBuilderBuildApi(Resource): "/workspaces/current/trigger-provider//subscriptions/update", ) class TriggerSubscriptionUpdateApi(Resource): - @console_ns.expect(console_ns.models[TriggerSubscriptionUpdateRequest.__name__]) + @console_ns.expect(console_ns.models[TriggerSubscriptionBuilderUpdatePayload.__name__]) @setup_required @login_required @edit_permission_required @@ -338,7 +316,7 @@ class TriggerSubscriptionUpdateApi(Resource): user = current_user assert user.current_tenant_id is not None - request = TriggerSubscriptionUpdateRequest.model_validate(console_ns.payload) + request = TriggerSubscriptionBuilderUpdatePayload.model_validate(console_ns.payload or {}) subscription = TriggerProviderService.get_subscription_by_id( tenant_id=user.current_tenant_id, @@ -568,13 +546,6 @@ class TriggerOAuthCallbackApi(Resource): return redirect(f"{dify_config.CONSOLE_WEB_URL}/oauth-callback") -parser_oauth_client = ( - reqparse.RequestParser() - .add_argument("client_params", type=dict, required=False, nullable=True, location="json") - .add_argument("enabled", type=bool, required=False, nullable=True, location="json") -) - - @console_ns.route("/workspaces/current/trigger-provider//oauth/client") class TriggerOAuthClientManageApi(Resource): @setup_required @@ -622,7 +593,7 @@ class TriggerOAuthClientManageApi(Resource): logger.exception("Error getting OAuth client", exc_info=e) raise - @console_ns.expect(parser_oauth_client) + @console_ns.expect(console_ns.models[TriggerOAuthClientPayload.__name__]) @setup_required @login_required @is_admin_or_owner_required @@ -632,15 +603,15 @@ class TriggerOAuthClientManageApi(Resource): user = current_user assert user.current_tenant_id is not None - args = parser_oauth_client.parse_args() + payload = TriggerOAuthClientPayload.model_validate(console_ns.payload or {}) try: provider_id = TriggerProviderID(provider) return TriggerProviderService.save_custom_oauth_client_params( tenant_id=user.current_tenant_id, provider_id=provider_id, - client_params=args.get("client_params"), - enabled=args.get("enabled"), + client_params=payload.client_params, + enabled=payload.enabled, ) except ValueError as e: @@ -676,7 +647,7 @@ class TriggerOAuthClientManageApi(Resource): "/workspaces/current/trigger-provider//subscriptions/verify/", ) class TriggerSubscriptionVerifyApi(Resource): - @console_ns.expect(console_ns.models[TriggerSubscriptionVerifyRequest.__name__]) + @console_ns.expect(console_ns.models[TriggerSubscriptionBuilderVerifyPayload.__name__]) @setup_required @login_required @edit_permission_required @@ -686,9 +657,7 @@ class TriggerSubscriptionVerifyApi(Resource): user = current_user assert user.current_tenant_id is not None - verify_request: TriggerSubscriptionVerifyRequest = TriggerSubscriptionVerifyRequest.model_validate( - console_ns.payload - ) + verify_request = TriggerSubscriptionBuilderVerifyPayload.model_validate(console_ns.payload or {}) try: result = TriggerProviderService.verify_subscription_credentials( diff --git a/api/services/trigger/trigger_provider_service.py b/api/services/trigger/trigger_provider_service.py index 4131d75145..688993c798 100644 --- a/api/services/trigger/trigger_provider_service.py +++ b/api/services/trigger/trigger_provider_service.py @@ -799,7 +799,7 @@ class TriggerProviderService: user_id: str, provider_id: TriggerProviderID, subscription_id: str, - credentials: Mapping[str, Any], + credentials: dict[str, Any], ) -> dict[str, Any]: """ Verify credentials for an existing subscription without updating it. From 7ccf858ce61606ea4518a64ace44e4bbee4591b5 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Wed, 7 Jan 2026 21:47:23 +0800 Subject: [PATCH 09/21] fix(workflow): pass correct user_from/invoke_from into graph init (#30637) --- api/core/app/apps/advanced_chat/app_runner.py | 16 +++++----- api/core/app/apps/pipeline/pipeline_runner.py | 31 ++++++++++++------- api/core/app/apps/workflow/app_runner.py | 16 +++++----- api/core/app/apps/workflow_app_runner.py | 14 +++++++-- 4 files changed, 49 insertions(+), 28 deletions(-) diff --git a/api/core/app/apps/advanced_chat/app_runner.py b/api/core/app/apps/advanced_chat/app_runner.py index a2ae8dec5b..d636548f2b 100644 --- a/api/core/app/apps/advanced_chat/app_runner.py +++ b/api/core/app/apps/advanced_chat/app_runner.py @@ -39,7 +39,6 @@ from extensions.ext_database import db from extensions.ext_redis import redis_client from extensions.otel import WorkflowAppRunnerHandler, trace_span from models import Workflow -from models.enums import UserFrom from models.model import App, Conversation, Message, MessageAnnotation from models.workflow import ConversationVariable from services.conversation_variable_updater import ConversationVariableUpdater @@ -106,6 +105,11 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner): if not app_record: raise ValueError("App not found") + invoke_from = self.application_generate_entity.invoke_from + if self.application_generate_entity.single_iteration_run or self.application_generate_entity.single_loop_run: + invoke_from = InvokeFrom.DEBUGGER + user_from = self._resolve_user_from(invoke_from) + if self.application_generate_entity.single_iteration_run or self.application_generate_entity.single_loop_run: # Handle single iteration or single loop run graph, variable_pool, graph_runtime_state = self._prepare_single_node_execution( @@ -158,6 +162,8 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner): workflow_id=self._workflow.id, tenant_id=self._workflow.tenant_id, user_id=self.application_generate_entity.user_id, + user_from=user_from, + invoke_from=invoke_from, ) db.session.close() @@ -175,12 +181,8 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner): graph=graph, graph_config=self._workflow.graph_dict, user_id=self.application_generate_entity.user_id, - user_from=( - UserFrom.ACCOUNT - if self.application_generate_entity.invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER} - else UserFrom.END_USER - ), - invoke_from=self.application_generate_entity.invoke_from, + user_from=user_from, + invoke_from=invoke_from, call_depth=self.application_generate_entity.call_depth, variable_pool=variable_pool, graph_runtime_state=graph_runtime_state, diff --git a/api/core/app/apps/pipeline/pipeline_runner.py b/api/core/app/apps/pipeline/pipeline_runner.py index 4be9e01fbf..0157521ae9 100644 --- a/api/core/app/apps/pipeline/pipeline_runner.py +++ b/api/core/app/apps/pipeline/pipeline_runner.py @@ -73,9 +73,15 @@ class PipelineRunner(WorkflowBasedAppRunner): """ app_config = self.application_generate_entity.app_config app_config = cast(PipelineConfig, app_config) + invoke_from = self.application_generate_entity.invoke_from + + if self.application_generate_entity.single_iteration_run or self.application_generate_entity.single_loop_run: + invoke_from = InvokeFrom.DEBUGGER + + user_from = self._resolve_user_from(invoke_from) user_id = None - if self.application_generate_entity.invoke_from in {InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API}: + if invoke_from in {InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API}: end_user = db.session.query(EndUser).where(EndUser.id == self.application_generate_entity.user_id).first() if end_user: user_id = end_user.session_id @@ -117,7 +123,7 @@ class PipelineRunner(WorkflowBasedAppRunner): dataset_id=self.application_generate_entity.dataset_id, datasource_type=self.application_generate_entity.datasource_type, datasource_info=self.application_generate_entity.datasource_info, - invoke_from=self.application_generate_entity.invoke_from.value, + invoke_from=invoke_from.value, ) rag_pipeline_variables = [] @@ -149,6 +155,8 @@ class PipelineRunner(WorkflowBasedAppRunner): graph_runtime_state=graph_runtime_state, start_node_id=self.application_generate_entity.start_node_id, workflow=workflow, + user_from=user_from, + invoke_from=invoke_from, ) # RUN WORKFLOW @@ -159,12 +167,8 @@ class PipelineRunner(WorkflowBasedAppRunner): graph=graph, graph_config=workflow.graph_dict, user_id=self.application_generate_entity.user_id, - user_from=( - UserFrom.ACCOUNT - if self.application_generate_entity.invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER} - else UserFrom.END_USER - ), - invoke_from=self.application_generate_entity.invoke_from, + user_from=user_from, + invoke_from=invoke_from, call_depth=self.application_generate_entity.call_depth, graph_runtime_state=graph_runtime_state, variable_pool=variable_pool, @@ -210,7 +214,12 @@ class PipelineRunner(WorkflowBasedAppRunner): return workflow def _init_rag_pipeline_graph( - self, workflow: Workflow, graph_runtime_state: GraphRuntimeState, start_node_id: str | None = None + self, + workflow: Workflow, + graph_runtime_state: GraphRuntimeState, + start_node_id: str | None = None, + user_from: UserFrom = UserFrom.ACCOUNT, + invoke_from: InvokeFrom = InvokeFrom.SERVICE_API, ) -> Graph: """ Init pipeline graph @@ -253,8 +262,8 @@ class PipelineRunner(WorkflowBasedAppRunner): workflow_id=workflow.id, graph_config=graph_config, user_id=self.application_generate_entity.user_id, - user_from=UserFrom.ACCOUNT, - invoke_from=InvokeFrom.SERVICE_API, + user_from=user_from, + invoke_from=invoke_from, call_depth=0, ) diff --git a/api/core/app/apps/workflow/app_runner.py b/api/core/app/apps/workflow/app_runner.py index 894e6f397a..8dbdc1d58c 100644 --- a/api/core/app/apps/workflow/app_runner.py +++ b/api/core/app/apps/workflow/app_runner.py @@ -20,7 +20,6 @@ from core.workflow.workflow_entry import WorkflowEntry from extensions.ext_redis import redis_client from extensions.otel import WorkflowAppRunnerHandler, trace_span from libs.datetime_utils import naive_utc_now -from models.enums import UserFrom from models.workflow import Workflow logger = logging.getLogger(__name__) @@ -74,7 +73,12 @@ class WorkflowAppRunner(WorkflowBasedAppRunner): workflow_execution_id=self.application_generate_entity.workflow_execution_id, ) + invoke_from = self.application_generate_entity.invoke_from # if only single iteration or single loop run is requested + if self.application_generate_entity.single_iteration_run or self.application_generate_entity.single_loop_run: + invoke_from = InvokeFrom.DEBUGGER + user_from = self._resolve_user_from(invoke_from) + if self.application_generate_entity.single_iteration_run or self.application_generate_entity.single_loop_run: graph, variable_pool, graph_runtime_state = self._prepare_single_node_execution( workflow=self._workflow, @@ -102,6 +106,8 @@ class WorkflowAppRunner(WorkflowBasedAppRunner): workflow_id=self._workflow.id, tenant_id=self._workflow.tenant_id, user_id=self.application_generate_entity.user_id, + user_from=user_from, + invoke_from=invoke_from, root_node_id=self._root_node_id, ) @@ -120,12 +126,8 @@ class WorkflowAppRunner(WorkflowBasedAppRunner): graph=graph, graph_config=self._workflow.graph_dict, user_id=self.application_generate_entity.user_id, - user_from=( - UserFrom.ACCOUNT - if self.application_generate_entity.invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER} - else UserFrom.END_USER - ), - invoke_from=self.application_generate_entity.invoke_from, + user_from=user_from, + invoke_from=invoke_from, call_depth=self.application_generate_entity.call_depth, variable_pool=variable_pool, graph_runtime_state=graph_runtime_state, diff --git a/api/core/app/apps/workflow_app_runner.py b/api/core/app/apps/workflow_app_runner.py index 0e125b3538..7adf3504ac 100644 --- a/api/core/app/apps/workflow_app_runner.py +++ b/api/core/app/apps/workflow_app_runner.py @@ -77,10 +77,18 @@ class WorkflowBasedAppRunner: self._app_id = app_id self._graph_engine_layers = graph_engine_layers + @staticmethod + def _resolve_user_from(invoke_from: InvokeFrom) -> UserFrom: + if invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER}: + return UserFrom.ACCOUNT + return UserFrom.END_USER + def _init_graph( self, graph_config: Mapping[str, Any], graph_runtime_state: GraphRuntimeState, + user_from: UserFrom, + invoke_from: InvokeFrom, workflow_id: str = "", tenant_id: str = "", user_id: str = "", @@ -105,8 +113,8 @@ class WorkflowBasedAppRunner: workflow_id=workflow_id, graph_config=graph_config, user_id=user_id, - user_from=UserFrom.ACCOUNT, - invoke_from=InvokeFrom.SERVICE_API, + user_from=user_from, + invoke_from=invoke_from, call_depth=0, ) @@ -250,7 +258,7 @@ class WorkflowBasedAppRunner: graph_config=graph_config, user_id="", user_from=UserFrom.ACCOUNT, - invoke_from=InvokeFrom.SERVICE_API, + invoke_from=InvokeFrom.DEBUGGER, call_depth=0, ) From 25ff4ae5dad773837c9c36e499609efe98137b29 Mon Sep 17 00:00:00 2001 From: yyh <92089059+lyzno1@users.noreply.github.com> Date: Thu, 8 Jan 2026 09:53:32 +0800 Subject: [PATCH 10/21] fix(i18n): resolve Claude Code sandbox path issues in workflow (#30710) --- .github/workflows/translate-i18n-claude.yml | 71 +++++++++++++-------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/.github/workflows/translate-i18n-claude.yml b/.github/workflows/translate-i18n-claude.yml index 8dccf8ef93..0e05913576 100644 --- a/.github/workflows/translate-i18n-claude.yml +++ b/.github/workflows/translate-i18n-claude.yml @@ -132,6 +132,14 @@ jobs: - Use **Edit** tool to modify JSON files (NOT node, jq, or bash scripts) - Use **Bash** ONLY for: git commands, gh commands, pnpm commands - Run bash commands ONE BY ONE, never combine with && or || + - NEVER use `$()` command substitution - it's not supported. Split into separate commands instead. + + ## WORKING DIRECTORY & ABSOLUTE PATHS + Claude Code sandbox working directory may vary. Always use absolute paths: + - For pnpm: `pnpm --dir ${{ github.workspace }}/web ` + - For git: `git -C ${{ github.workspace }} ` + - For gh: `gh --repo ${{ github.repository }} ` + - For file paths: `${{ github.workspace }}/web/i18n/` ## EFFICIENCY RULES - **ONE Edit per language file** - batch all key additions into a single Edit @@ -142,8 +150,8 @@ jobs: - Changed/target files: ${{ steps.detect_changes.outputs.CHANGED_FILES }} - Target languages (empty means all supported): ${{ steps.detect_changes.outputs.TARGET_LANGS }} - Sync mode: ${{ steps.detect_changes.outputs.SYNC_MODE }} - - Translation files are located in: web/i18n/{locale}/{filename}.json - - Language configuration is in: web/i18n-config/languages.ts + - Translation files are located in: ${{ github.workspace }}/web/i18n/{locale}/{filename}.json + - Language configuration is in: ${{ github.workspace }}/web/i18n-config/languages.ts - Git diff is available: ${{ steps.detect_changes.outputs.DIFF_AVAILABLE }} ## CRITICAL DESIGN: Verify First, Then Sync @@ -166,13 +174,15 @@ jobs: * DELETE: Keys that appear only in `-` lines (removed keys) ### Step 1.2: Read Language Configuration - Use the Read tool to read `web/i18n-config/languages.ts`. + Use the Read tool to read `${{ github.workspace }}/web/i18n-config/languages.ts`. Extract all languages with `supported: true`. ### Step 1.3: Run i18n:check for Each Language ```bash - pnpm --dir web install --frozen-lockfile - pnpm --dir web run i18n:check + pnpm --dir ${{ github.workspace }}/web install --frozen-lockfile + ``` + ```bash + pnpm --dir ${{ github.workspace }}/web run i18n:check ``` This will report: @@ -234,7 +244,7 @@ jobs: **IMPORTANT: Special handling for zh-Hans and ja-JP** If zh-Hans or ja-JP files were ALSO modified in the same push: - - Run: `git diff HEAD~1 --name-only | grep -E "(zh-Hans|ja-JP)"` + - Run: `git -C ${{ github.workspace }} diff HEAD~1 --name-only` and check for zh-Hans or ja-JP files - If found, it means someone manually translated them. Apply these rules: 1. **Missing keys**: Still ADD them (completeness required) @@ -254,7 +264,7 @@ jobs: ### Step 2.3: Process DELETE Operations For extra keys reported by i18n:check: - - Run: `pnpm --dir web run i18n:check --auto-remove` + - Run: `pnpm --dir ${{ github.workspace }}/web run i18n:check --auto-remove` - Or manually remove from target language JSON files ## Translation Guidelines @@ -282,7 +292,7 @@ jobs: ### Step 3.1: Run Lint Fix (IMPORTANT!) ```bash - pnpm --dir web lint:fix --quiet -- 'i18n/**/*.json' + pnpm --dir ${{ github.workspace }}/web lint:fix --quiet -- 'i18n/**/*.json' ``` This ensures: - JSON keys are sorted alphabetically (jsonc/sort-keys rule) @@ -291,7 +301,7 @@ jobs: ### Step 3.2: Run Final i18n Check ```bash - pnpm --dir web run i18n:check + pnpm --dir ${{ github.workspace }}/web run i18n:check ``` ### Step 3.3: Fix Any Remaining Issues @@ -342,39 +352,45 @@ jobs: ### Step 4.1: Check for changes ```bash - git status --porcelain + git -C ${{ github.workspace }} status --porcelain ``` If there are changes: ### Step 4.2: Create a new branch and commit - Run these git commands ONE BY ONE (not combined with &&): + Run these git commands ONE BY ONE (not combined with &&). + **IMPORTANT**: Do NOT use `$()` command substitution. Use two separate commands: - 1. Create branch: + 1. First, get the timestamp: ```bash - git checkout -b "chore/i18n-sync-$(date +%Y%m%d-%H%M%S)" + date +%Y%m%d-%H%M%S + ``` + (Note the output, e.g., "20260115-143052") + + 2. Then create branch using the timestamp value: + ```bash + git -C ${{ github.workspace }} checkout -b chore/i18n-sync-20260115-143052 + ``` + (Replace "20260115-143052" with the actual timestamp from step 1) + + 3. Stage changes: + ```bash + git -C ${{ github.workspace }} add web/i18n/ ``` - 2. Stage changes: + 4. Commit: ```bash - git add web/i18n/ + git -C ${{ github.workspace }} commit -m "chore(i18n): sync translations with en-US - Mode: ${{ steps.detect_changes.outputs.SYNC_MODE }}" ``` - 3. Commit (use heredoc for multi-line message): + 5. Push: ```bash - git commit -m "chore(i18n): sync translations with en-US - Mode: ${{ steps.detect_changes.outputs.SYNC_MODE }}" - ``` - - 4. Push: - ```bash - git push origin HEAD + git -C ${{ github.workspace }} push origin HEAD ``` ### Step 4.3: Create Pull Request ```bash - gh pr create \ - --title "chore(i18n): sync translations with en-US" \ - --body "## Summary + gh pr create --repo ${{ github.repository }} --title "chore(i18n): sync translations with en-US" --body "## Summary This PR was automatically generated to sync i18n translation files. @@ -386,10 +402,9 @@ jobs: - [x] \`i18n:check\` passed - [x] \`lint:fix\` applied - 🤖 Generated with Claude Code GitHub Action" \ - --base main + 🤖 Generated with Claude Code GitHub Action" --base main ``` claude_args: | --max-turns 150 - --allowedTools "Read,Write,Edit,Bash(git *),Bash(git:*),Bash(gh *),Bash(gh:*),Bash(pnpm *),Bash(pnpm:*),Glob,Grep" + --allowedTools "Read,Write,Edit,Bash(git *),Bash(git:*),Bash(gh *),Bash(gh:*),Bash(pnpm *),Bash(pnpm:*),Bash(date *),Bash(date:*),Glob,Grep" From 27a803a6f01990d5a38fd6808fe7f51aa1703979 Mon Sep 17 00:00:00 2001 From: Rhon Joe Date: Thu, 8 Jan 2026 09:54:27 +0800 Subject: [PATCH 11/21] fix(web): resolve key-value input box height inconsistency on focus/blur (#30715) (#30716) --- .../components/key-value/key-value-edit/input-item.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/input-item.tsx b/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/input-item.tsx index 7e5fe7da7a..7f1e2df2a0 100644 --- a/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/input-item.tsx +++ b/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/input-item.tsx @@ -59,12 +59,12 @@ const InputItem: FC = ({ }, [onRemove]) return ( -
+
{(!readOnly) ? ( = ({ ) : (
{!hasValue &&
{placeholder}
} {hasValue && ( Date: Wed, 7 Jan 2026 18:03:39 -0800 Subject: [PATCH 12/21] feat: add decryption decorators for password and code fields in webapp (#30704) --- api/controllers/web/login.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/controllers/web/login.py b/api/controllers/web/login.py index 538d0c44be..5847f4ae3a 100644 --- a/api/controllers/web/login.py +++ b/api/controllers/web/login.py @@ -10,7 +10,12 @@ from controllers.console.auth.error import ( InvalidEmailError, ) from controllers.console.error import AccountBannedError -from controllers.console.wraps import only_edition_enterprise, setup_required +from controllers.console.wraps import ( + decrypt_code_field, + decrypt_password_field, + only_edition_enterprise, + setup_required, +) from controllers.web import web_ns from controllers.web.wraps import decode_jwt_token from libs.helper import email @@ -42,6 +47,7 @@ class LoginApi(Resource): 404: "Account not found", } ) + @decrypt_password_field def post(self): """Authenticate user and login.""" parser = ( @@ -181,6 +187,7 @@ class EmailCodeLoginApi(Resource): 404: "Account not found", } ) + @decrypt_code_field def post(self): parser = ( reqparse.RequestParser() From c5b99ebd1711b791d66e08aa9b021f123ab75e8e Mon Sep 17 00:00:00 2001 From: NFish Date: Thu, 8 Jan 2026 10:04:42 +0800 Subject: [PATCH 13/21] fix: web app login code encrypt (#30705) --- web/app/(shareLayout)/webapp-signin/check-code/page.tsx | 4 ++-- .../webapp-signin/components/mail-and-password-auth.tsx | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/web/app/(shareLayout)/webapp-signin/check-code/page.tsx b/web/app/(shareLayout)/webapp-signin/check-code/page.tsx index bda5484197..72e3b7f2ea 100644 --- a/web/app/(shareLayout)/webapp-signin/check-code/page.tsx +++ b/web/app/(shareLayout)/webapp-signin/check-code/page.tsx @@ -8,12 +8,12 @@ import Button from '@/app/components/base/button' import Input from '@/app/components/base/input' import Toast from '@/app/components/base/toast' import Countdown from '@/app/components/signin/countdown' - import { useLocale } from '@/context/i18n' import { useWebAppStore } from '@/context/web-app-context' import { sendWebAppEMailLoginCode, webAppEmailLoginWithCode } from '@/service/common' import { fetchAccessToken } from '@/service/share' import { setWebAppAccessToken, setWebAppPassport } from '@/service/webapp-auth' +import { encryptVerificationCode } from '@/utils/encryption' export default function CheckCode() { const { t } = useTranslation() @@ -64,7 +64,7 @@ export default function CheckCode() { return } setIsLoading(true) - const ret = await webAppEmailLoginWithCode({ email, code, token }) + const ret = await webAppEmailLoginWithCode({ email, code: encryptVerificationCode(code), token }) if (ret.result === 'success') { setWebAppAccessToken(ret.data.access_token) const { access_token } = await fetchAccessToken({ diff --git a/web/app/(shareLayout)/webapp-signin/components/mail-and-password-auth.tsx b/web/app/(shareLayout)/webapp-signin/components/mail-and-password-auth.tsx index 23ac83e76c..11aebe4a5b 100644 --- a/web/app/(shareLayout)/webapp-signin/components/mail-and-password-auth.tsx +++ b/web/app/(shareLayout)/webapp-signin/components/mail-and-password-auth.tsx @@ -13,6 +13,7 @@ import { useWebAppStore } from '@/context/web-app-context' import { webAppLogin } from '@/service/common' import { fetchAccessToken } from '@/service/share' import { setWebAppAccessToken, setWebAppPassport } from '@/service/webapp-auth' +import { encryptPassword } from '@/utils/encryption' type MailAndPasswordAuthProps = { isEmailSetup: boolean @@ -71,7 +72,7 @@ export default function MailAndPasswordAuth({ isEmailSetup }: MailAndPasswordAut setIsLoading(true) const loginData: Record = { email, - password, + password: encryptPassword(password), language: locale, remember_me: true, } From fe0802262c533c285a20d1352bdafc52156e575c Mon Sep 17 00:00:00 2001 From: zyssyz123 <916125788@qq.com> Date: Thu, 8 Jan 2026 13:17:30 +0800 Subject: [PATCH 14/21] feat: credit pool (#30720) Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../feature/hosted_service/__init__.py | 266 +++++++++++++++++- .../console/workspace/workspace.py | 3 + api/core/hosting_configuration.py | 137 ++++++++- api/core/provider_manager.py | 68 +++-- api/core/workflow/nodes/llm/llm_utils.py | 52 ++-- .../update_provider_when_message_created.py | 46 ++- ...12_25_1039-7df29de0f6be_add_credit_pool.py | 46 +++ api/models/__init__.py | 2 + api/models/model.py | 30 +- api/services/account_service.py | 5 + api/services/credit_pool_service.py | 85 ++++++ api/services/feature_service.py | 4 + api/services/workspace_service.py | 17 +- .../services/test_account_service.py | 9 +- 14 files changed, 694 insertions(+), 76 deletions(-) create mode 100644 api/migrations/versions/2025_12_25_1039-7df29de0f6be_add_credit_pool.py create mode 100644 api/services/credit_pool_service.py diff --git a/api/configs/feature/hosted_service/__init__.py b/api/configs/feature/hosted_service/__init__.py index 4ad30014c7..42ede718c4 100644 --- a/api/configs/feature/hosted_service/__init__.py +++ b/api/configs/feature/hosted_service/__init__.py @@ -8,6 +8,11 @@ class HostedCreditConfig(BaseSettings): default="", ) + HOSTED_POOL_CREDITS: int = Field( + description="Pool credits for hosted service", + default=200, + ) + def get_model_credits(self, model_name: str) -> int: """ Get credit value for a specific model name. @@ -60,19 +65,46 @@ class HostedOpenAiConfig(BaseSettings): HOSTED_OPENAI_TRIAL_MODELS: str = Field( description="Comma-separated list of available models for trial access", - default="gpt-3.5-turbo," - "gpt-3.5-turbo-1106," - "gpt-3.5-turbo-instruct," + default="gpt-4," + "gpt-4-turbo-preview," + "gpt-4-turbo-2024-04-09," + "gpt-4-1106-preview," + "gpt-4-0125-preview," + "gpt-4-turbo," + "gpt-4.1," + "gpt-4.1-2025-04-14," + "gpt-4.1-mini," + "gpt-4.1-mini-2025-04-14," + "gpt-4.1-nano," + "gpt-4.1-nano-2025-04-14," + "gpt-3.5-turbo," "gpt-3.5-turbo-16k," "gpt-3.5-turbo-16k-0613," + "gpt-3.5-turbo-1106," "gpt-3.5-turbo-0613," "gpt-3.5-turbo-0125," - "text-davinci-003", - ) - - HOSTED_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field( - description="Quota limit for hosted OpenAI service usage", - default=200, + "gpt-3.5-turbo-instruct," + "text-davinci-003," + "chatgpt-4o-latest," + "gpt-4o," + "gpt-4o-2024-05-13," + "gpt-4o-2024-08-06," + "gpt-4o-2024-11-20," + "gpt-4o-audio-preview," + "gpt-4o-audio-preview-2025-06-03," + "gpt-4o-mini," + "gpt-4o-mini-2024-07-18," + "o3-mini," + "o3-mini-2025-01-31," + "gpt-5-mini-2025-08-07," + "gpt-5-mini," + "o4-mini," + "o4-mini-2025-04-16," + "gpt-5-chat-latest," + "gpt-5," + "gpt-5-2025-08-07," + "gpt-5-nano," + "gpt-5-nano-2025-08-07", ) HOSTED_OPENAI_PAID_ENABLED: bool = Field( @@ -87,6 +119,13 @@ class HostedOpenAiConfig(BaseSettings): "gpt-4-turbo-2024-04-09," "gpt-4-1106-preview," "gpt-4-0125-preview," + "gpt-4-turbo," + "gpt-4.1," + "gpt-4.1-2025-04-14," + "gpt-4.1-mini," + "gpt-4.1-mini-2025-04-14," + "gpt-4.1-nano," + "gpt-4.1-nano-2025-04-14," "gpt-3.5-turbo," "gpt-3.5-turbo-16k," "gpt-3.5-turbo-16k-0613," @@ -94,7 +133,150 @@ class HostedOpenAiConfig(BaseSettings): "gpt-3.5-turbo-0613," "gpt-3.5-turbo-0125," "gpt-3.5-turbo-instruct," - "text-davinci-003", + "text-davinci-003," + "chatgpt-4o-latest," + "gpt-4o," + "gpt-4o-2024-05-13," + "gpt-4o-2024-08-06," + "gpt-4o-2024-11-20," + "gpt-4o-audio-preview," + "gpt-4o-audio-preview-2025-06-03," + "gpt-4o-mini," + "gpt-4o-mini-2024-07-18," + "o3-mini," + "o3-mini-2025-01-31," + "gpt-5-mini-2025-08-07," + "gpt-5-mini," + "o4-mini," + "o4-mini-2025-04-16," + "gpt-5-chat-latest," + "gpt-5," + "gpt-5-2025-08-07," + "gpt-5-nano," + "gpt-5-nano-2025-08-07", + ) + + +class HostedGeminiConfig(BaseSettings): + """ + Configuration for fetching Gemini service + """ + + HOSTED_GEMINI_API_KEY: str | None = Field( + description="API key for hosted Gemini service", + default=None, + ) + + HOSTED_GEMINI_API_BASE: str | None = Field( + description="Base URL for hosted Gemini API", + default=None, + ) + + HOSTED_GEMINI_API_ORGANIZATION: str | None = Field( + description="Organization ID for hosted Gemini service", + default=None, + ) + + HOSTED_GEMINI_TRIAL_ENABLED: bool = Field( + description="Enable trial access to hosted Gemini service", + default=False, + ) + + HOSTED_GEMINI_TRIAL_MODELS: str = Field( + description="Comma-separated list of available models for trial access", + default="gemini-2.5-flash,gemini-2.0-flash,gemini-2.0-flash-lite,", + ) + + HOSTED_GEMINI_PAID_ENABLED: bool = Field( + description="Enable paid access to hosted gemini service", + default=False, + ) + + HOSTED_GEMINI_PAID_MODELS: str = Field( + description="Comma-separated list of available models for paid access", + default="gemini-2.5-flash,gemini-2.0-flash,gemini-2.0-flash-lite,", + ) + + +class HostedXAIConfig(BaseSettings): + """ + Configuration for fetching XAI service + """ + + HOSTED_XAI_API_KEY: str | None = Field( + description="API key for hosted XAI service", + default=None, + ) + + HOSTED_XAI_API_BASE: str | None = Field( + description="Base URL for hosted XAI API", + default=None, + ) + + HOSTED_XAI_API_ORGANIZATION: str | None = Field( + description="Organization ID for hosted XAI service", + default=None, + ) + + HOSTED_XAI_TRIAL_ENABLED: bool = Field( + description="Enable trial access to hosted XAI service", + default=False, + ) + + HOSTED_XAI_TRIAL_MODELS: str = Field( + description="Comma-separated list of available models for trial access", + default="grok-3,grok-3-mini,grok-3-mini-fast", + ) + + HOSTED_XAI_PAID_ENABLED: bool = Field( + description="Enable paid access to hosted XAI service", + default=False, + ) + + HOSTED_XAI_PAID_MODELS: str = Field( + description="Comma-separated list of available models for paid access", + default="grok-3,grok-3-mini,grok-3-mini-fast", + ) + + +class HostedDeepseekConfig(BaseSettings): + """ + Configuration for fetching Deepseek service + """ + + HOSTED_DEEPSEEK_API_KEY: str | None = Field( + description="API key for hosted Deepseek service", + default=None, + ) + + HOSTED_DEEPSEEK_API_BASE: str | None = Field( + description="Base URL for hosted Deepseek API", + default=None, + ) + + HOSTED_DEEPSEEK_API_ORGANIZATION: str | None = Field( + description="Organization ID for hosted Deepseek service", + default=None, + ) + + HOSTED_DEEPSEEK_TRIAL_ENABLED: bool = Field( + description="Enable trial access to hosted Deepseek service", + default=False, + ) + + HOSTED_DEEPSEEK_TRIAL_MODELS: str = Field( + description="Comma-separated list of available models for trial access", + default="deepseek-chat,deepseek-reasoner", + ) + + HOSTED_DEEPSEEK_PAID_ENABLED: bool = Field( + description="Enable paid access to hosted Deepseek service", + default=False, + ) + + HOSTED_DEEPSEEK_PAID_MODELS: str = Field( + description="Comma-separated list of available models for paid access", + default="deepseek-chat,deepseek-reasoner", ) @@ -144,16 +326,66 @@ class HostedAnthropicConfig(BaseSettings): default=False, ) - HOSTED_ANTHROPIC_QUOTA_LIMIT: NonNegativeInt = Field( - description="Quota limit for hosted Anthropic service usage", - default=600000, - ) - HOSTED_ANTHROPIC_PAID_ENABLED: bool = Field( description="Enable paid access to hosted Anthropic service", default=False, ) + HOSTED_ANTHROPIC_TRIAL_MODELS: str = Field( + description="Comma-separated list of available models for paid access", + default="claude-opus-4-20250514," + "claude-sonnet-4-20250514," + "claude-3-5-haiku-20241022," + "claude-3-opus-20240229," + "claude-3-7-sonnet-20250219," + "claude-3-haiku-20240307", + ) + HOSTED_ANTHROPIC_PAID_MODELS: str = Field( + description="Comma-separated list of available models for paid access", + default="claude-opus-4-20250514," + "claude-sonnet-4-20250514," + "claude-3-5-haiku-20241022," + "claude-3-opus-20240229," + "claude-3-7-sonnet-20250219," + "claude-3-haiku-20240307", + ) + + +class HostedTongyiConfig(BaseSettings): + """ + Configuration for hosted Tongyi service + """ + + HOSTED_TONGYI_API_KEY: str | None = Field( + description="API key for hosted Tongyi service", + default=None, + ) + + HOSTED_TONGYI_USE_INTERNATIONAL_ENDPOINT: bool = Field( + description="Use international endpoint for hosted Tongyi service", + default=False, + ) + + HOSTED_TONGYI_TRIAL_ENABLED: bool = Field( + description="Enable trial access to hosted Tongyi service", + default=False, + ) + + HOSTED_TONGYI_PAID_ENABLED: bool = Field( + description="Enable paid access to hosted Anthropic service", + default=False, + ) + + HOSTED_TONGYI_TRIAL_MODELS: str = Field( + description="Comma-separated list of available models for trial access", + default="", + ) + + HOSTED_TONGYI_PAID_MODELS: str = Field( + description="Comma-separated list of available models for paid access", + default="", + ) + class HostedMinmaxConfig(BaseSettings): """ @@ -246,9 +478,13 @@ class HostedServiceConfig( HostedOpenAiConfig, HostedSparkConfig, HostedZhipuAIConfig, + HostedTongyiConfig, # moderation HostedModerationConfig, # credit config HostedCreditConfig, + HostedGeminiConfig, + HostedXAIConfig, + HostedDeepseekConfig, ): pass diff --git a/api/controllers/console/workspace/workspace.py b/api/controllers/console/workspace/workspace.py index 909a5ce201..52e6f7d737 100644 --- a/api/controllers/console/workspace/workspace.py +++ b/api/controllers/console/workspace/workspace.py @@ -80,6 +80,9 @@ tenant_fields = { "in_trial": fields.Boolean, "trial_end_reason": fields.String, "custom_config": fields.Raw(attribute="custom_config"), + "trial_credits": fields.Integer, + "trial_credits_used": fields.Integer, + "next_credit_reset_date": fields.Integer, } tenants_fields = { diff --git a/api/core/hosting_configuration.py b/api/core/hosting_configuration.py index af860a1070..370e64e385 100644 --- a/api/core/hosting_configuration.py +++ b/api/core/hosting_configuration.py @@ -56,6 +56,10 @@ class HostingConfiguration: self.provider_map[f"{DEFAULT_PLUGIN_ID}/minimax/minimax"] = self.init_minimax() self.provider_map[f"{DEFAULT_PLUGIN_ID}/spark/spark"] = self.init_spark() self.provider_map[f"{DEFAULT_PLUGIN_ID}/zhipuai/zhipuai"] = self.init_zhipuai() + self.provider_map[f"{DEFAULT_PLUGIN_ID}/gemini/google"] = self.init_gemini() + self.provider_map[f"{DEFAULT_PLUGIN_ID}/x/x"] = self.init_xai() + self.provider_map[f"{DEFAULT_PLUGIN_ID}/deepseek/deepseek"] = self.init_deepseek() + self.provider_map[f"{DEFAULT_PLUGIN_ID}/tongyi/tongyi"] = self.init_tongyi() self.moderation_config = self.init_moderation_config() @@ -128,7 +132,7 @@ class HostingConfiguration: quotas: list[HostingQuota] = [] if dify_config.HOSTED_OPENAI_TRIAL_ENABLED: - hosted_quota_limit = dify_config.HOSTED_OPENAI_QUOTA_LIMIT + hosted_quota_limit = 0 trial_models = self.parse_restrict_models_from_env("HOSTED_OPENAI_TRIAL_MODELS") trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit, restrict_models=trial_models) quotas.append(trial_quota) @@ -156,18 +160,49 @@ class HostingConfiguration: quota_unit=quota_unit, ) - @staticmethod - def init_anthropic() -> HostingProvider: - quota_unit = QuotaUnit.TOKENS + def init_gemini(self) -> HostingProvider: + quota_unit = QuotaUnit.CREDITS + quotas: list[HostingQuota] = [] + + if dify_config.HOSTED_GEMINI_TRIAL_ENABLED: + hosted_quota_limit = 0 + trial_models = self.parse_restrict_models_from_env("HOSTED_GEMINI_TRIAL_MODELS") + trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit, restrict_models=trial_models) + quotas.append(trial_quota) + + if dify_config.HOSTED_GEMINI_PAID_ENABLED: + paid_models = self.parse_restrict_models_from_env("HOSTED_GEMINI_PAID_MODELS") + paid_quota = PaidHostingQuota(restrict_models=paid_models) + quotas.append(paid_quota) + + if len(quotas) > 0: + credentials = { + "google_api_key": dify_config.HOSTED_GEMINI_API_KEY, + } + + if dify_config.HOSTED_GEMINI_API_BASE: + credentials["google_base_url"] = dify_config.HOSTED_GEMINI_API_BASE + + return HostingProvider(enabled=True, credentials=credentials, quota_unit=quota_unit, quotas=quotas) + + return HostingProvider( + enabled=False, + quota_unit=quota_unit, + ) + + def init_anthropic(self) -> HostingProvider: + quota_unit = QuotaUnit.CREDITS quotas: list[HostingQuota] = [] if dify_config.HOSTED_ANTHROPIC_TRIAL_ENABLED: - hosted_quota_limit = dify_config.HOSTED_ANTHROPIC_QUOTA_LIMIT - trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit) + hosted_quota_limit = 0 + trail_models = self.parse_restrict_models_from_env("HOSTED_ANTHROPIC_TRIAL_MODELS") + trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit, restrict_models=trail_models) quotas.append(trial_quota) if dify_config.HOSTED_ANTHROPIC_PAID_ENABLED: - paid_quota = PaidHostingQuota() + paid_models = self.parse_restrict_models_from_env("HOSTED_ANTHROPIC_PAID_MODELS") + paid_quota = PaidHostingQuota(restrict_models=paid_models) quotas.append(paid_quota) if len(quotas) > 0: @@ -185,6 +220,94 @@ class HostingConfiguration: quota_unit=quota_unit, ) + def init_tongyi(self) -> HostingProvider: + quota_unit = QuotaUnit.CREDITS + quotas: list[HostingQuota] = [] + + if dify_config.HOSTED_TONGYI_TRIAL_ENABLED: + hosted_quota_limit = 0 + trail_models = self.parse_restrict_models_from_env("HOSTED_TONGYI_TRIAL_MODELS") + trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit, restrict_models=trail_models) + quotas.append(trial_quota) + + if dify_config.HOSTED_TONGYI_PAID_ENABLED: + paid_models = self.parse_restrict_models_from_env("HOSTED_TONGYI_PAID_MODELS") + paid_quota = PaidHostingQuota(restrict_models=paid_models) + quotas.append(paid_quota) + + if len(quotas) > 0: + credentials = { + "dashscope_api_key": dify_config.HOSTED_TONGYI_API_KEY, + "use_international_endpoint": dify_config.HOSTED_TONGYI_USE_INTERNATIONAL_ENDPOINT, + } + + return HostingProvider(enabled=True, credentials=credentials, quota_unit=quota_unit, quotas=quotas) + + return HostingProvider( + enabled=False, + quota_unit=quota_unit, + ) + + def init_xai(self) -> HostingProvider: + quota_unit = QuotaUnit.CREDITS + quotas: list[HostingQuota] = [] + + if dify_config.HOSTED_XAI_TRIAL_ENABLED: + hosted_quota_limit = 0 + trail_models = self.parse_restrict_models_from_env("HOSTED_XAI_TRIAL_MODELS") + trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit, restrict_models=trail_models) + quotas.append(trial_quota) + + if dify_config.HOSTED_XAI_PAID_ENABLED: + paid_models = self.parse_restrict_models_from_env("HOSTED_XAI_PAID_MODELS") + paid_quota = PaidHostingQuota(restrict_models=paid_models) + quotas.append(paid_quota) + + if len(quotas) > 0: + credentials = { + "api_key": dify_config.HOSTED_XAI_API_KEY, + } + + if dify_config.HOSTED_XAI_API_BASE: + credentials["endpoint_url"] = dify_config.HOSTED_XAI_API_BASE + + return HostingProvider(enabled=True, credentials=credentials, quota_unit=quota_unit, quotas=quotas) + + return HostingProvider( + enabled=False, + quota_unit=quota_unit, + ) + + def init_deepseek(self) -> HostingProvider: + quota_unit = QuotaUnit.CREDITS + quotas: list[HostingQuota] = [] + + if dify_config.HOSTED_DEEPSEEK_TRIAL_ENABLED: + hosted_quota_limit = 0 + trail_models = self.parse_restrict_models_from_env("HOSTED_DEEPSEEK_TRIAL_MODELS") + trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit, restrict_models=trail_models) + quotas.append(trial_quota) + + if dify_config.HOSTED_DEEPSEEK_PAID_ENABLED: + paid_models = self.parse_restrict_models_from_env("HOSTED_DEEPSEEK_PAID_MODELS") + paid_quota = PaidHostingQuota(restrict_models=paid_models) + quotas.append(paid_quota) + + if len(quotas) > 0: + credentials = { + "api_key": dify_config.HOSTED_DEEPSEEK_API_KEY, + } + + if dify_config.HOSTED_DEEPSEEK_API_BASE: + credentials["endpoint_url"] = dify_config.HOSTED_DEEPSEEK_API_BASE + + return HostingProvider(enabled=True, credentials=credentials, quota_unit=quota_unit, quotas=quotas) + + return HostingProvider( + enabled=False, + quota_unit=quota_unit, + ) + @staticmethod def init_minimax() -> HostingProvider: quota_unit = QuotaUnit.TOKENS diff --git a/api/core/provider_manager.py b/api/core/provider_manager.py index 10d86d1762..fdbfca4330 100644 --- a/api/core/provider_manager.py +++ b/api/core/provider_manager.py @@ -618,18 +618,18 @@ class ProviderManager: ) for quota in configuration.quotas: - if quota.quota_type == ProviderQuotaType.TRIAL: + if quota.quota_type in (ProviderQuotaType.TRIAL, ProviderQuotaType.PAID): # Init trial provider records if not exists - if ProviderQuotaType.TRIAL not in provider_quota_to_provider_record_dict: + if quota.quota_type not in provider_quota_to_provider_record_dict: try: # FIXME ignore the type error, only TrialHostingQuota has limit need to change the logic new_provider_record = Provider( tenant_id=tenant_id, # TODO: Use provider name with prefix after the data migration. provider_name=ModelProviderID(provider_name).provider_name, - provider_type=ProviderType.SYSTEM, - quota_type=ProviderQuotaType.TRIAL, - quota_limit=quota.quota_limit, # type: ignore + provider_type=ProviderType.SYSTEM.value, + quota_type=quota.quota_type, + quota_limit=0, # type: ignore quota_used=0, is_valid=True, ) @@ -641,8 +641,8 @@ class ProviderManager: stmt = select(Provider).where( Provider.tenant_id == tenant_id, Provider.provider_name == ModelProviderID(provider_name).provider_name, - Provider.provider_type == ProviderType.SYSTEM, - Provider.quota_type == ProviderQuotaType.TRIAL, + Provider.provider_type == ProviderType.SYSTEM.value, + Provider.quota_type == quota.quota_type, ) existed_provider_record = db.session.scalar(stmt) if not existed_provider_record: @@ -912,6 +912,22 @@ class ProviderManager: provider_record ) quota_configurations = [] + + if dify_config.EDITION == "CLOUD": + from services.credit_pool_service import CreditPoolService + + trail_pool = CreditPoolService.get_pool( + tenant_id=tenant_id, + pool_type=ProviderQuotaType.TRIAL.value, + ) + paid_pool = CreditPoolService.get_pool( + tenant_id=tenant_id, + pool_type=ProviderQuotaType.PAID.value, + ) + else: + trail_pool = None + paid_pool = None + for provider_quota in provider_hosting_configuration.quotas: if provider_quota.quota_type not in quota_type_to_provider_records_dict: if provider_quota.quota_type == ProviderQuotaType.FREE: @@ -932,16 +948,36 @@ class ProviderManager: raise ValueError("quota_used is None") if provider_record.quota_limit is None: raise ValueError("quota_limit is None") + if provider_quota.quota_type == ProviderQuotaType.TRIAL and trail_pool is not None: + quota_configuration = QuotaConfiguration( + quota_type=provider_quota.quota_type, + quota_unit=provider_hosting_configuration.quota_unit or QuotaUnit.TOKENS, + quota_used=trail_pool.quota_used, + quota_limit=trail_pool.quota_limit, + is_valid=trail_pool.quota_limit > trail_pool.quota_used or trail_pool.quota_limit == -1, + restrict_models=provider_quota.restrict_models, + ) - quota_configuration = QuotaConfiguration( - quota_type=provider_quota.quota_type, - quota_unit=provider_hosting_configuration.quota_unit or QuotaUnit.TOKENS, - quota_used=provider_record.quota_used, - quota_limit=provider_record.quota_limit, - is_valid=provider_record.quota_limit > provider_record.quota_used - or provider_record.quota_limit == -1, - restrict_models=provider_quota.restrict_models, - ) + elif provider_quota.quota_type == ProviderQuotaType.PAID and paid_pool is not None: + quota_configuration = QuotaConfiguration( + quota_type=provider_quota.quota_type, + quota_unit=provider_hosting_configuration.quota_unit or QuotaUnit.TOKENS, + quota_used=paid_pool.quota_used, + quota_limit=paid_pool.quota_limit, + is_valid=paid_pool.quota_limit > paid_pool.quota_used or paid_pool.quota_limit == -1, + restrict_models=provider_quota.restrict_models, + ) + + else: + quota_configuration = QuotaConfiguration( + quota_type=provider_quota.quota_type, + quota_unit=provider_hosting_configuration.quota_unit or QuotaUnit.TOKENS, + quota_used=provider_record.quota_used, + quota_limit=provider_record.quota_limit, + is_valid=provider_record.quota_limit > provider_record.quota_used + or provider_record.quota_limit == -1, + restrict_models=provider_quota.restrict_models, + ) quota_configurations.append(quota_configuration) diff --git a/api/core/workflow/nodes/llm/llm_utils.py b/api/core/workflow/nodes/llm/llm_utils.py index 0c545469bc..01e25cbf5c 100644 --- a/api/core/workflow/nodes/llm/llm_utils.py +++ b/api/core/workflow/nodes/llm/llm_utils.py @@ -6,7 +6,7 @@ from sqlalchemy.orm import Session from configs import dify_config from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity -from core.entities.provider_entities import QuotaUnit +from core.entities.provider_entities import ProviderQuotaType, QuotaUnit from core.file.models import File from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance, ModelManager @@ -136,21 +136,37 @@ def deduct_llm_quota(tenant_id: str, model_instance: ModelInstance, usage: LLMUs used_quota = 1 if used_quota is not None and system_configuration.current_quota_type is not None: - with Session(db.engine) as session: - stmt = ( - update(Provider) - .where( - Provider.tenant_id == tenant_id, - # TODO: Use provider name with prefix after the data migration. - Provider.provider_name == ModelProviderID(model_instance.provider).provider_name, - Provider.provider_type == ProviderType.SYSTEM, - Provider.quota_type == system_configuration.current_quota_type.value, - Provider.quota_limit > Provider.quota_used, - ) - .values( - quota_used=Provider.quota_used + used_quota, - last_used=naive_utc_now(), - ) + if system_configuration.current_quota_type == ProviderQuotaType.TRIAL: + from services.credit_pool_service import CreditPoolService + + CreditPoolService.check_and_deduct_credits( + tenant_id=tenant_id, + credits_required=used_quota, ) - session.execute(stmt) - session.commit() + elif system_configuration.current_quota_type == ProviderQuotaType.PAID: + from services.credit_pool_service import CreditPoolService + + CreditPoolService.check_and_deduct_credits( + tenant_id=tenant_id, + credits_required=used_quota, + pool_type="paid", + ) + else: + with Session(db.engine) as session: + stmt = ( + update(Provider) + .where( + Provider.tenant_id == tenant_id, + # TODO: Use provider name with prefix after the data migration. + Provider.provider_name == ModelProviderID(model_instance.provider).provider_name, + Provider.provider_type == ProviderType.SYSTEM.value, + Provider.quota_type == system_configuration.current_quota_type.value, + Provider.quota_limit > Provider.quota_used, + ) + .values( + quota_used=Provider.quota_used + used_quota, + last_used=naive_utc_now(), + ) + ) + session.execute(stmt) + session.commit() diff --git a/api/events/event_handlers/update_provider_when_message_created.py b/api/events/event_handlers/update_provider_when_message_created.py index 84266ab0fa..1ddcc8f792 100644 --- a/api/events/event_handlers/update_provider_when_message_created.py +++ b/api/events/event_handlers/update_provider_when_message_created.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import Session from configs import dify_config from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, ChatAppGenerateEntity -from core.entities.provider_entities import QuotaUnit, SystemConfiguration +from core.entities.provider_entities import ProviderQuotaType, QuotaUnit, SystemConfiguration from events.message_event import message_was_created from extensions.ext_database import db from extensions.ext_redis import redis_client, redis_fallback @@ -134,22 +134,38 @@ def handle(sender: Message, **kwargs): system_configuration=system_configuration, model_name=model_config.model, ) - if used_quota is not None: - quota_update = _ProviderUpdateOperation( - filters=_ProviderUpdateFilters( + if provider_configuration.system_configuration.current_quota_type == ProviderQuotaType.TRIAL: + from services.credit_pool_service import CreditPoolService + + CreditPoolService.check_and_deduct_credits( tenant_id=tenant_id, - provider_name=ModelProviderID(model_config.provider).provider_name, - provider_type=ProviderType.SYSTEM, - quota_type=provider_configuration.system_configuration.current_quota_type.value, - ), - values=_ProviderUpdateValues(quota_used=Provider.quota_used + used_quota, last_used=current_time), - additional_filters=_ProviderUpdateAdditionalFilters( - quota_limit_check=True # Provider.quota_limit > Provider.quota_used - ), - description="quota_deduction_update", - ) - updates_to_perform.append(quota_update) + credits_required=used_quota, + pool_type="trial", + ) + elif provider_configuration.system_configuration.current_quota_type == ProviderQuotaType.PAID: + from services.credit_pool_service import CreditPoolService + + CreditPoolService.check_and_deduct_credits( + tenant_id=tenant_id, + credits_required=used_quota, + pool_type="paid", + ) + else: + quota_update = _ProviderUpdateOperation( + filters=_ProviderUpdateFilters( + tenant_id=tenant_id, + provider_name=ModelProviderID(model_config.provider).provider_name, + provider_type=ProviderType.SYSTEM.value, + quota_type=provider_configuration.system_configuration.current_quota_type.value, + ), + values=_ProviderUpdateValues(quota_used=Provider.quota_used + used_quota, last_used=current_time), + additional_filters=_ProviderUpdateAdditionalFilters( + quota_limit_check=True # Provider.quota_limit > Provider.quota_used + ), + description="quota_deduction_update", + ) + updates_to_perform.append(quota_update) # Execute all updates start_time = time_module.perf_counter() diff --git a/api/migrations/versions/2025_12_25_1039-7df29de0f6be_add_credit_pool.py b/api/migrations/versions/2025_12_25_1039-7df29de0f6be_add_credit_pool.py new file mode 100644 index 0000000000..e89fcee7e5 --- /dev/null +++ b/api/migrations/versions/2025_12_25_1039-7df29de0f6be_add_credit_pool.py @@ -0,0 +1,46 @@ +"""add credit pool + +Revision ID: 7df29de0f6be +Revises: 03ea244985ce +Create Date: 2025-12-25 10:39:15.139304 + +""" +from alembic import op +import models as models +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '7df29de0f6be' +down_revision = '03ea244985ce' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('tenant_credit_pools', + sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('tenant_id', models.types.StringUUID(), nullable=False), + sa.Column('pool_type', sa.String(length=40), server_default='trial', nullable=False), + sa.Column('quota_limit', sa.BigInteger(), nullable=False), + sa.Column('quota_used', sa.BigInteger(), nullable=False), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False), + sa.PrimaryKeyConstraint('id', name='tenant_credit_pool_pkey') + ) + with op.batch_alter_table('tenant_credit_pools', schema=None) as batch_op: + batch_op.create_index('tenant_credit_pool_pool_type_idx', ['pool_type'], unique=False) + batch_op.create_index('tenant_credit_pool_tenant_id_idx', ['tenant_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + + with op.batch_alter_table('tenant_credit_pools', schema=None) as batch_op: + batch_op.drop_index('tenant_credit_pool_tenant_id_idx') + batch_op.drop_index('tenant_credit_pool_pool_type_idx') + + op.drop_table('tenant_credit_pools') + # ### end Alembic commands ### diff --git a/api/models/__init__.py b/api/models/__init__.py index 906bc3198e..e23de832dc 100644 --- a/api/models/__init__.py +++ b/api/models/__init__.py @@ -60,6 +60,7 @@ from .model import ( Site, Tag, TagBinding, + TenantCreditPool, TraceAppConfig, UploadFile, ) @@ -177,6 +178,7 @@ __all__ = [ "Tenant", "TenantAccountJoin", "TenantAccountRole", + "TenantCreditPool", "TenantDefaultModel", "TenantPreferredModelProvider", "TenantStatus", diff --git a/api/models/model.py b/api/models/model.py index 46df047237..c791ae15b0 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -12,8 +12,8 @@ from uuid import uuid4 import sqlalchemy as sa from flask import request -from flask_login import UserMixin -from sqlalchemy import Float, Index, PrimaryKeyConstraint, String, exists, func, select, text +from flask_login import UserMixin # type: ignore[import-untyped] +from sqlalchemy import BigInteger, Float, Index, PrimaryKeyConstraint, String, exists, func, select, text from sqlalchemy.orm import Mapped, Session, mapped_column from configs import dify_config @@ -2073,3 +2073,29 @@ class TraceAppConfig(TypeBase): "created_at": str(self.created_at) if self.created_at else None, "updated_at": str(self.updated_at) if self.updated_at else None, } + + +class TenantCreditPool(Base): + __tablename__ = "tenant_credit_pools" + __table_args__ = ( + sa.PrimaryKeyConstraint("id", name="tenant_credit_pool_pkey"), + sa.Index("tenant_credit_pool_tenant_id_idx", "tenant_id"), + sa.Index("tenant_credit_pool_pool_type_idx", "pool_type"), + ) + + id = mapped_column(StringUUID, primary_key=True, server_default=text("uuid_generate_v4()")) + tenant_id = mapped_column(StringUUID, nullable=False) + pool_type = mapped_column(String(40), nullable=False, default="trial", server_default="trial") + quota_limit = mapped_column(BigInteger, nullable=False, default=0) + quota_used = mapped_column(BigInteger, nullable=False, default=0) + created_at = mapped_column(sa.DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP")) + updated_at = mapped_column( + sa.DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp() + ) + + @property + def remaining_credits(self) -> int: + return max(0, self.quota_limit - self.quota_used) + + def has_sufficient_credits(self, required_credits: int) -> bool: + return self.remaining_credits >= required_credits diff --git a/api/services/account_service.py b/api/services/account_service.py index 5a549dc318..d38c9d5a66 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -999,6 +999,11 @@ class TenantService: tenant.encrypt_public_key = generate_key_pair(tenant.id) db.session.commit() + + from services.credit_pool_service import CreditPoolService + + CreditPoolService.create_default_pool(tenant.id) + return tenant @staticmethod diff --git a/api/services/credit_pool_service.py b/api/services/credit_pool_service.py new file mode 100644 index 0000000000..1954602571 --- /dev/null +++ b/api/services/credit_pool_service.py @@ -0,0 +1,85 @@ +import logging + +from sqlalchemy import update +from sqlalchemy.orm import Session + +from configs import dify_config +from core.errors.error import QuotaExceededError +from extensions.ext_database import db +from models import TenantCreditPool + +logger = logging.getLogger(__name__) + + +class CreditPoolService: + @classmethod + def create_default_pool(cls, tenant_id: str) -> TenantCreditPool: + """create default credit pool for new tenant""" + credit_pool = TenantCreditPool( + tenant_id=tenant_id, quota_limit=dify_config.HOSTED_POOL_CREDITS, quota_used=0, pool_type="trial" + ) + db.session.add(credit_pool) + db.session.commit() + return credit_pool + + @classmethod + def get_pool(cls, tenant_id: str, pool_type: str = "trial") -> TenantCreditPool | None: + """get tenant credit pool""" + return ( + db.session.query(TenantCreditPool) + .filter_by( + tenant_id=tenant_id, + pool_type=pool_type, + ) + .first() + ) + + @classmethod + def check_credits_available( + cls, + tenant_id: str, + credits_required: int, + pool_type: str = "trial", + ) -> bool: + """check if credits are available without deducting""" + pool = cls.get_pool(tenant_id, pool_type) + if not pool: + return False + return pool.remaining_credits >= credits_required + + @classmethod + def check_and_deduct_credits( + cls, + tenant_id: str, + credits_required: int, + pool_type: str = "trial", + ) -> int: + """check and deduct credits, returns actual credits deducted""" + + pool = cls.get_pool(tenant_id, pool_type) + if not pool: + raise QuotaExceededError("Credit pool not found") + + if pool.remaining_credits <= 0: + raise QuotaExceededError("No credits remaining") + + # deduct all remaining credits if less than required + actual_credits = min(credits_required, pool.remaining_credits) + + try: + with Session(db.engine) as session: + stmt = ( + update(TenantCreditPool) + .where( + TenantCreditPool.tenant_id == tenant_id, + TenantCreditPool.pool_type == pool_type, + ) + .values(quota_used=TenantCreditPool.quota_used + actual_credits) + ) + session.execute(stmt) + session.commit() + except Exception: + logger.exception("Failed to deduct credits for tenant %s", tenant_id) + raise QuotaExceededError("Failed to deduct credits") + + return actual_credits diff --git a/api/services/feature_service.py b/api/services/feature_service.py index 8035adc734..9b853b8337 100644 --- a/api/services/feature_service.py +++ b/api/services/feature_service.py @@ -140,6 +140,7 @@ class FeatureModel(BaseModel): # pydantic configs model_config = ConfigDict(protected_namespaces=()) knowledge_pipeline: KnowledgePipeline = KnowledgePipeline() + next_credit_reset_date: int = 0 class KnowledgeRateLimitModel(BaseModel): @@ -301,6 +302,9 @@ class FeatureService: if "knowledge_pipeline_publish_enabled" in billing_info: features.knowledge_pipeline.publish_enabled = billing_info["knowledge_pipeline_publish_enabled"] + if "next_credit_reset_date" in billing_info: + features.next_credit_reset_date = billing_info["next_credit_reset_date"] + @classmethod def _fulfill_params_from_enterprise(cls, features: SystemFeatureModel): enterprise_info = EnterpriseService.get_info() diff --git a/api/services/workspace_service.py b/api/services/workspace_service.py index 292ac6e008..3ee41c2e8d 100644 --- a/api/services/workspace_service.py +++ b/api/services/workspace_service.py @@ -31,7 +31,8 @@ class WorkspaceService: assert tenant_account_join is not None, "TenantAccountJoin not found" tenant_info["role"] = tenant_account_join.role - can_replace_logo = FeatureService.get_features(tenant.id).can_replace_logo + feature = FeatureService.get_features(tenant.id) + can_replace_logo = feature.can_replace_logo if can_replace_logo and TenantService.has_roles(tenant, [TenantAccountRole.OWNER, TenantAccountRole.ADMIN]): base_url = dify_config.FILES_URL @@ -46,5 +47,19 @@ class WorkspaceService: "remove_webapp_brand": remove_webapp_brand, "replace_webapp_logo": replace_webapp_logo, } + if dify_config.EDITION == "CLOUD": + tenant_info["next_credit_reset_date"] = feature.next_credit_reset_date + + from services.credit_pool_service import CreditPoolService + + paid_pool = CreditPoolService.get_pool(tenant_id=tenant.id, pool_type="paid") + if paid_pool: + tenant_info["trial_credits"] = paid_pool.quota_limit + tenant_info["trial_credits_used"] = paid_pool.quota_used + else: + trial_pool = CreditPoolService.get_pool(tenant_id=tenant.id, pool_type="trial") + if trial_pool: + tenant_info["trial_credits"] = trial_pool.quota_limit + tenant_info["trial_credits_used"] = trial_pool.quota_used return tenant_info diff --git a/api/tests/unit_tests/services/test_account_service.py b/api/tests/unit_tests/services/test_account_service.py index 627a04bcd0..e35ba74c56 100644 --- a/api/tests/unit_tests/services/test_account_service.py +++ b/api/tests/unit_tests/services/test_account_service.py @@ -619,8 +619,13 @@ class TestTenantService: mock_tenant_instance.name = "Test User's Workspace" mock_tenant_class.return_value = mock_tenant_instance - # Execute test - TenantService.create_owner_tenant_if_not_exist(mock_account) + # Mock the db import in CreditPoolService to avoid database connection + with patch("services.credit_pool_service.db") as mock_credit_pool_db: + mock_credit_pool_db.session.add = MagicMock() + mock_credit_pool_db.session.commit = MagicMock() + + # Execute test + TenantService.create_owner_tenant_if_not_exist(mock_account) # Verify tenant was created with correct parameters mock_db_dependencies["db"].session.add.assert_called() From cd1af04dee4c579fa94ce56cb04381bead1ae0f3 Mon Sep 17 00:00:00 2001 From: Coding On Star <447357187@qq.com> Date: Thu, 8 Jan 2026 14:11:44 +0800 Subject: [PATCH 15/21] feat: model total credits (#30727) Co-authored-by: CodingOnStar Co-authored-by: Stephen Zhou <38493346+hyoban@users.noreply.github.com> --- .../base/icons/assets/public/llm/Tongyi.svg | 17 + .../public/llm/anthropic-short-light.svg | 4 + .../base/icons/assets/public/llm/deepseek.svg | 4 + .../base/icons/assets/public/llm/gemini.svg | 105 +++ .../base/icons/assets/public/llm/grok.svg | 11 + .../icons/assets/public/llm/openai-small.svg | 17 + .../src/public/llm/AnthropicShortLight.json | 36 + .../src/public/llm/AnthropicShortLight.tsx | 20 + .../base/icons/src/public/llm/Deepseek.json | 36 + .../base/icons/src/public/llm/Deepseek.tsx | 20 + .../base/icons/src/public/llm/Gemini.json | 807 ++++++++++++++++++ .../base/icons/src/public/llm/Gemini.tsx | 20 + .../base/icons/src/public/llm/Grok.json | 72 ++ .../base/icons/src/public/llm/Grok.tsx | 20 + .../base/icons/src/public/llm/OpenaiBlue.json | 37 + .../base/icons/src/public/llm/OpenaiBlue.tsx | 20 + .../icons/src/public/llm/OpenaiSmall.json | 128 +++ .../base/icons/src/public/llm/OpenaiSmall.tsx | 20 + .../base/icons/src/public/llm/OpenaiTeal.json | 37 + .../base/icons/src/public/llm/OpenaiTeal.tsx | 20 + .../icons/src/public/llm/OpenaiViolet.json | 37 + .../icons/src/public/llm/OpenaiViolet.tsx | 20 + .../base/icons/src/public/llm/Tongyi.json | 128 +++ .../base/icons/src/public/llm/Tongyi.tsx | 20 + .../base/icons/src/public/llm/index.ts | 9 + .../src/public/tracing/DatabricksIcon.tsx | 2 +- .../src/public/tracing/DatabricksIconBig.tsx | 2 +- .../icons/src/public/tracing/MlflowIcon.tsx | 2 +- .../src/public/tracing/MlflowIconBig.tsx | 2 +- .../icons/src/public/tracing/TencentIcon.json | 6 +- .../src/public/tracing/TencentIconBig.json | 10 +- .../apps-full-in-dialog/index.spec.tsx | 4 + .../model-provider-page/index.tsx | 11 +- .../provider-added-card/credential-panel.tsx | 8 +- .../provider-added-card/index.tsx | 15 +- .../provider-added-card/quota-panel.tsx | 185 +++- .../model-provider-page/utils.ts | 20 +- web/app/components/plugins/provider-card.tsx | 2 +- web/context/app-context.tsx | 8 +- web/i18n/en-US/billing.json | 2 +- web/i18n/en-US/common.json | 6 +- web/i18n/ja-JP/billing.json | 2 +- web/i18n/ja-JP/common.json | 6 +- web/i18n/zh-Hans/billing.json | 2 +- web/i18n/zh-Hans/common.json | 6 +- web/models/common.ts | 3 + 46 files changed, 1892 insertions(+), 77 deletions(-) create mode 100644 web/app/components/base/icons/assets/public/llm/Tongyi.svg create mode 100644 web/app/components/base/icons/assets/public/llm/anthropic-short-light.svg create mode 100644 web/app/components/base/icons/assets/public/llm/deepseek.svg create mode 100644 web/app/components/base/icons/assets/public/llm/gemini.svg create mode 100644 web/app/components/base/icons/assets/public/llm/grok.svg create mode 100644 web/app/components/base/icons/assets/public/llm/openai-small.svg create mode 100644 web/app/components/base/icons/src/public/llm/AnthropicShortLight.json create mode 100644 web/app/components/base/icons/src/public/llm/AnthropicShortLight.tsx create mode 100644 web/app/components/base/icons/src/public/llm/Deepseek.json create mode 100644 web/app/components/base/icons/src/public/llm/Deepseek.tsx create mode 100644 web/app/components/base/icons/src/public/llm/Gemini.json create mode 100644 web/app/components/base/icons/src/public/llm/Gemini.tsx create mode 100644 web/app/components/base/icons/src/public/llm/Grok.json create mode 100644 web/app/components/base/icons/src/public/llm/Grok.tsx create mode 100644 web/app/components/base/icons/src/public/llm/OpenaiBlue.json create mode 100644 web/app/components/base/icons/src/public/llm/OpenaiBlue.tsx create mode 100644 web/app/components/base/icons/src/public/llm/OpenaiSmall.json create mode 100644 web/app/components/base/icons/src/public/llm/OpenaiSmall.tsx create mode 100644 web/app/components/base/icons/src/public/llm/OpenaiTeal.json create mode 100644 web/app/components/base/icons/src/public/llm/OpenaiTeal.tsx create mode 100644 web/app/components/base/icons/src/public/llm/OpenaiViolet.json create mode 100644 web/app/components/base/icons/src/public/llm/OpenaiViolet.tsx create mode 100644 web/app/components/base/icons/src/public/llm/Tongyi.json create mode 100644 web/app/components/base/icons/src/public/llm/Tongyi.tsx diff --git a/web/app/components/base/icons/assets/public/llm/Tongyi.svg b/web/app/components/base/icons/assets/public/llm/Tongyi.svg new file mode 100644 index 0000000000..cca23b3aae --- /dev/null +++ b/web/app/components/base/icons/assets/public/llm/Tongyi.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/assets/public/llm/anthropic-short-light.svg b/web/app/components/base/icons/assets/public/llm/anthropic-short-light.svg new file mode 100644 index 0000000000..c8e2370803 --- /dev/null +++ b/web/app/components/base/icons/assets/public/llm/anthropic-short-light.svg @@ -0,0 +1,4 @@ + + + + diff --git a/web/app/components/base/icons/assets/public/llm/deepseek.svg b/web/app/components/base/icons/assets/public/llm/deepseek.svg new file mode 100644 index 0000000000..046f89e1ce --- /dev/null +++ b/web/app/components/base/icons/assets/public/llm/deepseek.svg @@ -0,0 +1,4 @@ + + + + diff --git a/web/app/components/base/icons/assets/public/llm/gemini.svg b/web/app/components/base/icons/assets/public/llm/gemini.svg new file mode 100644 index 0000000000..698f6ea629 --- /dev/null +++ b/web/app/components/base/icons/assets/public/llm/gemini.svg @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/assets/public/llm/grok.svg b/web/app/components/base/icons/assets/public/llm/grok.svg new file mode 100644 index 0000000000..6c0cbe227d --- /dev/null +++ b/web/app/components/base/icons/assets/public/llm/grok.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/web/app/components/base/icons/assets/public/llm/openai-small.svg b/web/app/components/base/icons/assets/public/llm/openai-small.svg new file mode 100644 index 0000000000..4af58790e4 --- /dev/null +++ b/web/app/components/base/icons/assets/public/llm/openai-small.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/src/public/llm/AnthropicShortLight.json b/web/app/components/base/icons/src/public/llm/AnthropicShortLight.json new file mode 100644 index 0000000000..2a8ff2f28a --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/AnthropicShortLight.json @@ -0,0 +1,36 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "40", + "height": "40", + "viewBox": "0 0 40 40", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "width": "40", + "height": "40", + "fill": "white" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M25.7926 10.1311H21.5089L29.3208 29.869H33.6045L25.7926 10.1311ZM13.4164 10.1311L5.60449 29.869H9.97273L11.5703 25.724H19.743L21.3405 29.869H25.7087L17.8969 10.1311H13.4164ZM12.9834 22.0583L15.6566 15.1217L18.3299 22.0583H12.9834Z", + "fill": "black" + }, + "children": [] + } + ] + }, + "name": "AnthropicShortLight" +} diff --git a/web/app/components/base/icons/src/public/llm/AnthropicShortLight.tsx b/web/app/components/base/icons/src/public/llm/AnthropicShortLight.tsx new file mode 100644 index 0000000000..2bd21f48da --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/AnthropicShortLight.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import type { IconData } from '@/app/components/base/icons/IconBase' +import * as React from 'react' +import IconBase from '@/app/components/base/icons/IconBase' +import data from './AnthropicShortLight.json' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject> + }, +) => + +Icon.displayName = 'AnthropicShortLight' + +export default Icon diff --git a/web/app/components/base/icons/src/public/llm/Deepseek.json b/web/app/components/base/icons/src/public/llm/Deepseek.json new file mode 100644 index 0000000000..1483974a02 --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/Deepseek.json @@ -0,0 +1,36 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "40", + "height": "40", + "viewBox": "0 0 40 40", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "width": "40", + "height": "40", + "fill": "white" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M36.6676 11.2917C36.3316 11.1277 36.1871 11.4402 35.9906 11.599C35.9242 11.6511 35.8668 11.7188 35.8108 11.7787C35.3199 12.3048 34.747 12.6485 33.9996 12.6068C32.9046 12.5469 31.971 12.8907 31.1455 13.7293C30.9696 12.6954 30.3863 12.0782 29.4996 11.6824C29.0348 11.4766 28.5647 11.2709 28.2406 10.823C28.0127 10.5053 27.9515 10.1511 27.8368 9.80214C27.7652 9.59121 27.6923 9.37506 27.4502 9.33861C27.1871 9.29694 27.0843 9.51829 26.9814 9.70318C26.5674 10.4584 26.4084 11.2917 26.4228 12.1355C26.4592 14.0313 27.26 15.5417 28.8486 16.6173C29.0296 16.7397 29.0764 16.8646 29.0191 17.0443C28.9111 17.4141 28.7822 17.7735 28.6676 18.1433C28.596 18.3803 28.4879 18.4323 28.2354 18.3282C27.363 17.9637 26.609 17.4246 25.9436 16.7709C24.8135 15.6771 23.7914 14.4689 22.5166 13.5235C22.2171 13.3021 21.919 13.0964 21.609 12.9011C20.3082 11.6355 21.7796 10.5964 22.1194 10.474C22.4762 10.3464 22.2431 9.9037 21.092 9.90891C19.9423 9.91413 18.889 10.2995 17.5478 10.8126C17.3512 10.8907 17.1455 10.948 16.9332 10.9922C15.7158 10.7631 14.4515 10.711 13.1298 10.8594C10.6428 11.1381 8.65587 12.3152 7.19493 14.3255C5.44102 16.7397 5.02826 19.4845 5.53347 22.349C6.06473 25.3646 7.60249 27.8646 9.96707 29.8178C12.4176 31.8413 15.2406 32.8334 18.4606 32.6433C20.4163 32.5313 22.5947 32.2683 25.0504 30.1875C25.6702 30.4949 26.3199 30.6173 27.3994 30.711C28.2302 30.7891 29.0296 30.6694 29.6494 30.5417C30.6194 30.3361 30.5518 29.4375 30.2015 29.2709C27.3578 27.9454 27.9814 28.4845 27.4136 28.0495C28.859 26.3361 31.0374 24.5574 31.889 18.797C31.9554 18.3386 31.898 18.0522 31.889 17.6798C31.8838 17.4558 31.9346 17.3673 32.1923 17.3413C32.9046 17.2605 33.596 17.0651 34.2314 16.7137C36.0739 15.7058 36.816 14.0522 36.9918 12.0678C37.0179 11.7657 36.9866 11.4506 36.6676 11.2917ZM20.613 29.1485C17.8564 26.9793 16.5204 26.2657 15.9684 26.297C15.4527 26.3255 15.5452 26.9167 15.6584 27.3022C15.777 27.6823 15.9319 27.9454 16.1494 28.2787C16.2991 28.5001 16.402 28.8307 15.9996 29.0755C15.1116 29.6277 13.5687 28.8907 13.4958 28.8542C11.7001 27.797 10.1988 26.3985 9.14025 24.487C8.11941 22.6459 7.52566 20.6719 7.42801 18.5651C7.40197 18.0547 7.5517 17.875 8.05691 17.7839C8.72227 17.6615 9.40978 17.6355 10.0751 17.7318C12.8876 18.1433 15.2822 19.4037 17.2887 21.3959C18.4346 22.5339 19.3018 23.8907 20.195 25.2162C21.1442 26.6251 22.1663 27.9662 23.4671 29.0651C23.9254 29.4506 24.2926 29.7449 24.6428 29.961C23.5856 30.0782 21.8199 30.1042 20.613 29.1485ZM21.9332 20.6407C21.9332 20.4141 22.1142 20.2345 22.342 20.2345C22.3928 20.2345 22.4398 20.2449 22.4814 20.2605C22.5374 20.2813 22.5895 20.3126 22.6299 20.3594C22.7027 20.4298 22.7444 20.5339 22.7444 20.6407C22.7444 20.8673 22.5635 21.047 22.3368 21.047C22.109 21.047 21.9332 20.8673 21.9332 20.6407ZM26.036 22.7501C25.7731 22.8569 25.51 22.9506 25.2575 22.961C24.8655 22.9793 24.4371 22.8203 24.204 22.6251C23.8434 22.323 23.5856 22.1537 23.4762 21.6225C23.4306 21.3959 23.4567 21.047 23.497 20.8465C23.5908 20.4141 23.4866 20.1381 23.1832 19.8855C22.9346 19.6798 22.6207 19.6251 22.2744 19.6251C22.1455 19.6251 22.027 19.5678 21.9384 19.5209C21.7939 19.4479 21.6754 19.2683 21.7887 19.047C21.8251 18.9766 22.001 18.8022 22.0426 18.7709C22.5114 18.5027 23.053 18.5913 23.5543 18.7918C24.0191 18.9818 24.3694 19.3307 24.8746 19.823C25.3915 20.4194 25.484 20.5861 25.7783 21.0313C26.01 21.3829 26.2223 21.7422 26.3668 22.1537C26.454 22.4089 26.3408 22.6198 26.036 22.7501Z", + "fill": "#4D6BFE" + }, + "children": [] + } + ] + }, + "name": "Deepseek" +} diff --git a/web/app/components/base/icons/src/public/llm/Deepseek.tsx b/web/app/components/base/icons/src/public/llm/Deepseek.tsx new file mode 100644 index 0000000000..b19beb8b8f --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/Deepseek.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import type { IconData } from '@/app/components/base/icons/IconBase' +import * as React from 'react' +import IconBase from '@/app/components/base/icons/IconBase' +import data from './Deepseek.json' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject> + }, +) => + +Icon.displayName = 'Deepseek' + +export default Icon diff --git a/web/app/components/base/icons/src/public/llm/Gemini.json b/web/app/components/base/icons/src/public/llm/Gemini.json new file mode 100644 index 0000000000..3121b1ea19 --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/Gemini.json @@ -0,0 +1,807 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "40", + "height": "40", + "viewBox": "0 0 40 40", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "width": "40", + "height": "40", + "fill": "white" + }, + "children": [] + }, + { + "type": "element", + "name": "mask", + "attributes": { + "id": "mask0_3892_95663", + "style": "mask-type:alpha", + "maskUnits": "userSpaceOnUse", + "x": "6", + "y": "6", + "width": "28", + "height": "29" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M20 6C20.2936 6 20.5488 6.2005 20.6205 6.48556C20.8393 7.3566 21.1277 8.20866 21.4828 9.03356C22.4116 11.191 23.6854 13.0791 25.3032 14.6968C26.9218 16.3146 28.8095 17.5888 30.9664 18.5172C31.7941 18.8735 32.6436 19.16 33.5149 19.3795C33.6533 19.4143 33.7762 19.4942 33.8641 19.6067C33.9519 19.7192 33.9998 19.8578 34 20.0005C34 20.2941 33.7995 20.5492 33.5149 20.621C32.6437 20.8399 31.7915 21.1282 30.9664 21.4833C28.8095 22.4121 26.9209 23.6859 25.3032 25.3036C23.6854 26.9223 22.4116 28.8099 21.4828 30.9669C21.1278 31.7919 20.8394 32.6439 20.6205 33.5149C20.586 33.6534 20.5062 33.7764 20.3937 33.8644C20.2813 33.9524 20.1427 34.0003 20 34.0005C19.8572 34.0003 19.7186 33.9525 19.6062 33.8645C19.4937 33.7765 19.414 33.6535 19.3795 33.5149C19.1605 32.6439 18.872 31.7918 18.5167 30.9669C17.5884 28.8099 16.3151 26.9214 14.6964 25.3036C13.0782 23.6859 11.1906 22.4121 9.03309 21.4833C8.20814 21.1283 7.35608 20.8399 6.48509 20.621C6.34667 20.5864 6.22377 20.5065 6.13589 20.3941C6.04801 20.2817 6.00018 20.1432 6 20.0005C6.00024 19.8578 6.04808 19.7192 6.13594 19.6067C6.2238 19.4942 6.34667 19.4143 6.48509 19.3795C7.35612 19.1607 8.20819 18.8723 9.03309 18.5172C11.1906 17.5888 13.0786 16.3146 14.6964 14.6968C16.3141 13.0791 17.5884 11.191 18.5167 9.03356C18.8719 8.20862 19.1604 7.35656 19.3795 6.48556C19.4508 6.2005 19.7064 6 20 6Z", + "fill": "black" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M20 6C20.2936 6 20.5488 6.2005 20.6205 6.48556C20.8393 7.3566 21.1277 8.20866 21.4828 9.03356C22.4116 11.191 23.6854 13.0791 25.3032 14.6968C26.9218 16.3146 28.8095 17.5888 30.9664 18.5172C31.7941 18.8735 32.6436 19.16 33.5149 19.3795C33.6533 19.4143 33.7762 19.4942 33.8641 19.6067C33.9519 19.7192 33.9998 19.8578 34 20.0005C34 20.2941 33.7995 20.5492 33.5149 20.621C32.6437 20.8399 31.7915 21.1282 30.9664 21.4833C28.8095 22.4121 26.9209 23.6859 25.3032 25.3036C23.6854 26.9223 22.4116 28.8099 21.4828 30.9669C21.1278 31.7919 20.8394 32.6439 20.6205 33.5149C20.586 33.6534 20.5062 33.7764 20.3937 33.8644C20.2813 33.9524 20.1427 34.0003 20 34.0005C19.8572 34.0003 19.7186 33.9525 19.6062 33.8645C19.4937 33.7765 19.414 33.6535 19.3795 33.5149C19.1605 32.6439 18.872 31.7918 18.5167 30.9669C17.5884 28.8099 16.3151 26.9214 14.6964 25.3036C13.0782 23.6859 11.1906 22.4121 9.03309 21.4833C8.20814 21.1283 7.35608 20.8399 6.48509 20.621C6.34667 20.5864 6.22377 20.5065 6.13589 20.3941C6.04801 20.2817 6.00018 20.1432 6 20.0005C6.00024 19.8578 6.04808 19.7192 6.13594 19.6067C6.2238 19.4942 6.34667 19.4143 6.48509 19.3795C7.35612 19.1607 8.20819 18.8723 9.03309 18.5172C11.1906 17.5888 13.0786 16.3146 14.6964 14.6968C16.3141 13.0791 17.5884 11.191 18.5167 9.03356C18.8719 8.20862 19.1604 7.35656 19.3795 6.48556C19.4508 6.2005 19.7064 6 20 6Z", + "fill": "url(#paint0_linear_3892_95663)" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "mask": "url(#mask0_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "filter": "url(#filter0_f_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M3.47232 27.8921C6.70753 29.0411 10.426 26.8868 11.7778 23.0804C13.1296 19.274 11.6028 15.2569 8.36763 14.108C5.13242 12.959 1.41391 15.1133 0.06211 18.9197C-1.28969 22.7261 0.23711 26.7432 3.47232 27.8921Z", + "fill": "#FFE432" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "filter": "url(#filter1_f_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M17.8359 15.341C22.2806 15.341 25.8838 11.6588 25.8838 7.11644C25.8838 2.57412 22.2806 -1.10815 17.8359 -1.10815C13.3912 -1.10815 9.78809 2.57412 9.78809 7.11644C9.78809 11.6588 13.3912 15.341 17.8359 15.341Z", + "fill": "#FC413D" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "filter": "url(#filter2_f_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M14.7081 41.6431C19.3478 41.4163 22.8707 36.3599 22.5768 30.3493C22.283 24.3387 18.2836 19.65 13.644 19.8769C9.00433 20.1037 5.48139 25.1601 5.77525 31.1707C6.06911 37.1813 10.0685 41.87 14.7081 41.6431Z", + "fill": "#00B95C" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "filter": "url(#filter3_f_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M14.7081 41.6431C19.3478 41.4163 22.8707 36.3599 22.5768 30.3493C22.283 24.3387 18.2836 19.65 13.644 19.8769C9.00433 20.1037 5.48139 25.1601 5.77525 31.1707C6.06911 37.1813 10.0685 41.87 14.7081 41.6431Z", + "fill": "#00B95C" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "filter": "url(#filter4_f_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M19.355 38.0071C23.2447 35.6405 24.2857 30.2506 21.6803 25.9684C19.0748 21.6862 13.8095 20.1334 9.91983 22.5C6.03016 24.8666 4.98909 30.2565 7.59454 34.5387C10.2 38.8209 15.4653 40.3738 19.355 38.0071Z", + "fill": "#00B95C" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "filter": "url(#filter5_f_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M35.0759 24.5504C39.4477 24.5504 42.9917 21.1377 42.9917 16.9278C42.9917 12.7179 39.4477 9.30518 35.0759 9.30518C30.7042 9.30518 27.1602 12.7179 27.1602 16.9278C27.1602 21.1377 30.7042 24.5504 35.0759 24.5504Z", + "fill": "#3186FF" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "filter": "url(#filter6_f_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M0.362818 23.6667C4.3882 26.7279 10.2688 25.7676 13.4976 21.5219C16.7264 17.2762 16.0806 11.3528 12.0552 8.29156C8.02982 5.23037 2.14917 6.19062 -1.07959 10.4364C-4.30835 14.6821 -3.66256 20.6055 0.362818 23.6667Z", + "fill": "#FBBC04" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "filter": "url(#filter7_f_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M20.9877 28.1903C25.7924 31.4936 32.1612 30.5732 35.2128 26.1346C38.2644 21.696 36.8432 15.4199 32.0385 12.1166C27.2338 8.81334 20.865 9.73372 17.8134 14.1723C14.7618 18.611 16.183 24.887 20.9877 28.1903Z", + "fill": "#3186FF" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "filter": "url(#filter8_f_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M29.7231 4.99175C30.9455 6.65415 29.3748 9.88535 26.2149 12.2096C23.0549 14.5338 19.5026 15.0707 18.2801 13.4088C17.0576 11.7468 18.6284 8.51514 21.7883 6.19092C24.9482 3.86717 28.5006 3.32982 29.7231 4.99175Z", + "fill": "#749BFF" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "filter": "url(#filter9_f_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M19.6891 12.9486C24.5759 8.41581 26.2531 2.27858 23.4354 -0.759249C20.6176 -3.79708 14.3718 -2.58516 9.485 1.94765C4.59823 6.48046 2.92099 12.6177 5.73879 15.6555C8.55658 18.6933 14.8024 17.4814 19.6891 12.9486Z", + "fill": "#FC413D" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "filter": "url(#filter10_f_3892_95663)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M9.6712 29.23C12.5757 31.3088 15.9102 31.6247 17.1191 29.9356C18.328 28.2465 16.9535 25.1921 14.049 23.1133C11.1446 21.0345 7.81003 20.7186 6.60113 22.4077C5.39223 24.0968 6.76675 27.1512 9.6712 29.23Z", + "fill": "#FFEE48" + }, + "children": [] + } + ] + } + ] + }, + { + "type": "element", + "name": "defs", + "attributes": {}, + "children": [ + { + "type": "element", + "name": "filter", + "attributes": { + "id": "filter0_f_3892_95663", + "x": "-3.44095", + "y": "10.7885", + "width": "18.7217", + "height": "20.4229", + "filterUnits": "userSpaceOnUse", + "color-interpolation-filters": "sRGB" + }, + "children": [ + { + "type": "element", + "name": "feFlood", + "attributes": { + "flood-opacity": "0", + "result": "BackgroundImageFix" + }, + "children": [] + }, + { + "type": "element", + "name": "feBlend", + "attributes": { + "mode": "normal", + "in": "SourceGraphic", + "in2": "BackgroundImageFix", + "result": "shape" + }, + "children": [] + }, + { + "type": "element", + "name": "feGaussianBlur", + "attributes": { + "stdDeviation": "1.50514", + "result": "effect1_foregroundBlur_3892_95663" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "filter", + "attributes": { + "id": "filter1_f_3892_95663", + "x": "-4.76352", + "y": "-15.6598", + "width": "45.1989", + "height": "45.5524", + "filterUnits": "userSpaceOnUse", + "color-interpolation-filters": "sRGB" + }, + "children": [ + { + "type": "element", + "name": "feFlood", + "attributes": { + "flood-opacity": "0", + "result": "BackgroundImageFix" + }, + "children": [] + }, + { + "type": "element", + "name": "feBlend", + "attributes": { + "mode": "normal", + "in": "SourceGraphic", + "in2": "BackgroundImageFix", + "result": "shape" + }, + "children": [] + }, + { + "type": "element", + "name": "feGaussianBlur", + "attributes": { + "stdDeviation": "7.2758", + "result": "effect1_foregroundBlur_3892_95663" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "filter", + "attributes": { + "id": "filter2_f_3892_95663", + "x": "-6.61209", + "y": "7.49899", + "width": "41.5757", + "height": "46.522", + "filterUnits": "userSpaceOnUse", + "color-interpolation-filters": "sRGB" + }, + "children": [ + { + "type": "element", + "name": "feFlood", + "attributes": { + "flood-opacity": "0", + "result": "BackgroundImageFix" + }, + "children": [] + }, + { + "type": "element", + "name": "feBlend", + "attributes": { + "mode": "normal", + "in": "SourceGraphic", + "in2": "BackgroundImageFix", + "result": "shape" + }, + "children": [] + }, + { + "type": "element", + "name": "feGaussianBlur", + "attributes": { + "stdDeviation": "6.18495", + "result": "effect1_foregroundBlur_3892_95663" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "filter", + "attributes": { + "id": "filter3_f_3892_95663", + "x": "-6.61209", + "y": "7.49899", + "width": "41.5757", + "height": "46.522", + "filterUnits": "userSpaceOnUse", + "color-interpolation-filters": "sRGB" + }, + "children": [ + { + "type": "element", + "name": "feFlood", + "attributes": { + "flood-opacity": "0", + "result": "BackgroundImageFix" + }, + "children": [] + }, + { + "type": "element", + "name": "feBlend", + "attributes": { + "mode": "normal", + "in": "SourceGraphic", + "in2": "BackgroundImageFix", + "result": "shape" + }, + "children": [] + }, + { + "type": "element", + "name": "feGaussianBlur", + "attributes": { + "stdDeviation": "6.18495", + "result": "effect1_foregroundBlur_3892_95663" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "filter", + "attributes": { + "id": "filter4_f_3892_95663", + "x": "-6.21073", + "y": "9.02316", + "width": "41.6959", + "height": "42.4608", + "filterUnits": "userSpaceOnUse", + "color-interpolation-filters": "sRGB" + }, + "children": [ + { + "type": "element", + "name": "feFlood", + "attributes": { + "flood-opacity": "0", + "result": "BackgroundImageFix" + }, + "children": [] + }, + { + "type": "element", + "name": "feBlend", + "attributes": { + "mode": "normal", + "in": "SourceGraphic", + "in2": "BackgroundImageFix", + "result": "shape" + }, + "children": [] + }, + { + "type": "element", + "name": "feGaussianBlur", + "attributes": { + "stdDeviation": "6.18495", + "result": "effect1_foregroundBlur_3892_95663" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "filter", + "attributes": { + "id": "filter5_f_3892_95663", + "x": "15.405", + "y": "-2.44994", + "width": "39.3423", + "height": "38.7556", + "filterUnits": "userSpaceOnUse", + "color-interpolation-filters": "sRGB" + }, + "children": [ + { + "type": "element", + "name": "feFlood", + "attributes": { + "flood-opacity": "0", + "result": "BackgroundImageFix" + }, + "children": [] + }, + { + "type": "element", + "name": "feBlend", + "attributes": { + "mode": "normal", + "in": "SourceGraphic", + "in2": "BackgroundImageFix", + "result": "shape" + }, + "children": [] + }, + { + "type": "element", + "name": "feGaussianBlur", + "attributes": { + "stdDeviation": "5.87756", + "result": "effect1_foregroundBlur_3892_95663" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "filter", + "attributes": { + "id": "filter6_f_3892_95663", + "x": "-13.7886", + "y": "-4.15284", + "width": "39.9951", + "height": "40.2639", + "filterUnits": "userSpaceOnUse", + "color-interpolation-filters": "sRGB" + }, + "children": [ + { + "type": "element", + "name": "feFlood", + "attributes": { + "flood-opacity": "0", + "result": "BackgroundImageFix" + }, + "children": [] + }, + { + "type": "element", + "name": "feBlend", + "attributes": { + "mode": "normal", + "in": "SourceGraphic", + "in2": "BackgroundImageFix", + "result": "shape" + }, + "children": [] + }, + { + "type": "element", + "name": "feGaussianBlur", + "attributes": { + "stdDeviation": "5.32691", + "result": "effect1_foregroundBlur_3892_95663" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "filter", + "attributes": { + "id": "filter7_f_3892_95663", + "x": "6.6925", + "y": "0.620963", + "width": "39.6414", + "height": "39.065", + "filterUnits": "userSpaceOnUse", + "color-interpolation-filters": "sRGB" + }, + "children": [ + { + "type": "element", + "name": "feFlood", + "attributes": { + "flood-opacity": "0", + "result": "BackgroundImageFix" + }, + "children": [] + }, + { + "type": "element", + "name": "feBlend", + "attributes": { + "mode": "normal", + "in": "SourceGraphic", + "in2": "BackgroundImageFix", + "result": "shape" + }, + "children": [] + }, + { + "type": "element", + "name": "feGaussianBlur", + "attributes": { + "stdDeviation": "4.75678", + "result": "effect1_foregroundBlur_3892_95663" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "filter", + "attributes": { + "id": "filter8_f_3892_95663", + "x": "9.35225", + "y": "-4.48661", + "width": "29.2984", + "height": "27.3739", + "filterUnits": "userSpaceOnUse", + "color-interpolation-filters": "sRGB" + }, + "children": [ + { + "type": "element", + "name": "feFlood", + "attributes": { + "flood-opacity": "0", + "result": "BackgroundImageFix" + }, + "children": [] + }, + { + "type": "element", + "name": "feBlend", + "attributes": { + "mode": "normal", + "in": "SourceGraphic", + "in2": "BackgroundImageFix", + "result": "shape" + }, + "children": [] + }, + { + "type": "element", + "name": "feGaussianBlur", + "attributes": { + "stdDeviation": "4.25649", + "result": "effect1_foregroundBlur_3892_95663" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "filter", + "attributes": { + "id": "filter9_f_3892_95663", + "x": "-2.81919", + "y": "-9.62339", + "width": "34.8122", + "height": "34.143", + "filterUnits": "userSpaceOnUse", + "color-interpolation-filters": "sRGB" + }, + "children": [ + { + "type": "element", + "name": "feFlood", + "attributes": { + "flood-opacity": "0", + "result": "BackgroundImageFix" + }, + "children": [] + }, + { + "type": "element", + "name": "feBlend", + "attributes": { + "mode": "normal", + "in": "SourceGraphic", + "in2": "BackgroundImageFix", + "result": "shape" + }, + "children": [] + }, + { + "type": "element", + "name": "feGaussianBlur", + "attributes": { + "stdDeviation": "3.59514", + "result": "effect1_foregroundBlur_3892_95663" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "filter", + "attributes": { + "id": "filter10_f_3892_95663", + "x": "-2.73761", + "y": "12.4221", + "width": "29.1949", + "height": "27.4994", + "filterUnits": "userSpaceOnUse", + "color-interpolation-filters": "sRGB" + }, + "children": [ + { + "type": "element", + "name": "feFlood", + "attributes": { + "flood-opacity": "0", + "result": "BackgroundImageFix" + }, + "children": [] + }, + { + "type": "element", + "name": "feBlend", + "attributes": { + "mode": "normal", + "in": "SourceGraphic", + "in2": "BackgroundImageFix", + "result": "shape" + }, + "children": [] + }, + { + "type": "element", + "name": "feGaussianBlur", + "attributes": { + "stdDeviation": "4.44986", + "result": "effect1_foregroundBlur_3892_95663" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "linearGradient", + "attributes": { + "id": "paint0_linear_3892_95663", + "x1": "13.9595", + "y1": "24.7349", + "x2": "28.5025", + "y2": "12.4738", + "gradientUnits": "userSpaceOnUse" + }, + "children": [ + { + "type": "element", + "name": "stop", + "attributes": { + "stop-color": "#4893FC" + }, + "children": [] + }, + { + "type": "element", + "name": "stop", + "attributes": { + "offset": "0.27", + "stop-color": "#4893FC" + }, + "children": [] + }, + { + "type": "element", + "name": "stop", + "attributes": { + "offset": "0.777", + "stop-color": "#969DFF" + }, + "children": [] + }, + { + "type": "element", + "name": "stop", + "attributes": { + "offset": "1", + "stop-color": "#BD99FE" + }, + "children": [] + } + ] + } + ] + } + ] + }, + "name": "Gemini" +} diff --git a/web/app/components/base/icons/src/public/llm/Gemini.tsx b/web/app/components/base/icons/src/public/llm/Gemini.tsx new file mode 100644 index 0000000000..f5430036bb --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/Gemini.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import type { IconData } from '@/app/components/base/icons/IconBase' +import * as React from 'react' +import IconBase from '@/app/components/base/icons/IconBase' +import data from './Gemini.json' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject> + }, +) => + +Icon.displayName = 'Gemini' + +export default Icon diff --git a/web/app/components/base/icons/src/public/llm/Grok.json b/web/app/components/base/icons/src/public/llm/Grok.json new file mode 100644 index 0000000000..590f845eeb --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/Grok.json @@ -0,0 +1,72 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "40", + "height": "40", + "viewBox": "0 0 40 40", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "width": "40", + "height": "40", + "fill": "white" + }, + "children": [] + }, + { + "type": "element", + "name": "g", + "attributes": { + "clip-path": "url(#clip0_3892_95659)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M15.745 24.54L26.715 16.35C27.254 15.95 28.022 16.106 28.279 16.73C29.628 20.018 29.025 23.971 26.341 26.685C23.658 29.399 19.924 29.995 16.511 28.639L12.783 30.384C18.13 34.081 24.623 33.166 28.681 29.06C31.9 25.805 32.897 21.368 31.965 17.367L31.973 17.376C30.622 11.498 32.305 9.149 35.755 4.345L36 4L31.46 8.59V8.576L15.743 24.544M13.48 26.531C9.643 22.824 10.305 17.085 13.58 13.776C16 11.327 19.968 10.328 23.432 11.797L27.152 10.06C26.482 9.57 25.622 9.043 24.637 8.673C20.182 6.819 14.848 7.742 11.227 11.401C7.744 14.924 6.648 20.341 8.53 24.962C9.935 28.416 7.631 30.86 5.31 33.326C4.49 34.2 3.666 35.074 3 36L13.478 26.534", + "fill": "black" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "defs", + "attributes": {}, + "children": [ + { + "type": "element", + "name": "clipPath", + "attributes": { + "id": "clip0_3892_95659" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "width": "33", + "height": "32", + "fill": "white", + "transform": "translate(3 4)" + }, + "children": [] + } + ] + } + ] + } + ] + }, + "name": "Grok" +} diff --git a/web/app/components/base/icons/src/public/llm/Grok.tsx b/web/app/components/base/icons/src/public/llm/Grok.tsx new file mode 100644 index 0000000000..8b378de490 --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/Grok.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import type { IconData } from '@/app/components/base/icons/IconBase' +import * as React from 'react' +import IconBase from '@/app/components/base/icons/IconBase' +import data from './Grok.json' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject> + }, +) => + +Icon.displayName = 'Grok' + +export default Icon diff --git a/web/app/components/base/icons/src/public/llm/OpenaiBlue.json b/web/app/components/base/icons/src/public/llm/OpenaiBlue.json new file mode 100644 index 0000000000..c5d4f974a2 --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/OpenaiBlue.json @@ -0,0 +1,37 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "24", + "height": "24", + "viewBox": "0 0 24 24", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "width": "24", + "height": "24", + "rx": "6", + "fill": "#03A4EE" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M19.7758 11.5959C19.9546 11.9948 20.0681 12.4213 20.1145 12.8563C20.1592 13.2913 20.1369 13.7315 20.044 14.1596C19.9529 14.5878 19.7947 14.9987 19.5746 15.377C19.4302 15.6298 19.2599 15.867 19.0639 16.0854C18.8696 16.3021 18.653 16.4981 18.4174 16.67C18.1801 16.842 17.9274 16.9864 17.6591 17.105C17.3926 17.222 17.1141 17.3114 16.8286 17.3698C16.6945 17.7859 16.4951 18.1797 16.2371 18.5339C15.9809 18.8881 15.6697 19.1993 15.3155 19.4555C14.9613 19.7134 14.5693 19.9129 14.1532 20.047C13.7371 20.1829 13.302 20.2499 12.8636 20.2499C12.573 20.2516 12.2807 20.2207 11.9953 20.1622C11.7116 20.102 11.433 20.0109 11.1665 19.8923C10.9 19.7736 10.6472 19.6258 10.4116 19.4538C10.1778 19.2819 9.96115 19.0841 9.76857 18.8658C9.33871 18.9586 8.89853 18.981 8.46351 18.9363C8.02849 18.8898 7.60207 18.7763 7.20143 18.5975C6.80252 18.4204 6.43284 18.1797 6.10786 17.8857C5.78289 17.5916 5.50606 17.2478 5.28769 16.8695C5.14153 16.6167 5.02117 16.3502 4.93004 16.0734C4.83891 15.7965 4.77873 15.5111 4.74778 15.2205C4.71683 14.9317 4.71855 14.6393 4.7495 14.3488C4.78045 14.0599 4.84407 13.7745 4.9352 13.4976C4.64289 13.1727 4.40217 12.803 4.22335 12.4041C4.04624 12.0034 3.93104 11.5787 3.88634 11.1437C3.83991 10.7087 3.86398 10.2685 3.95511 9.84036C4.04624 9.41222 4.20443 9.00127 4.42452 8.62299C4.56896 8.37023 4.73918 8.13123 4.93348 7.91458C5.12778 7.69793 5.34615 7.50191 5.58171 7.32997C5.81728 7.15802 6.07176 7.01187 6.33827 6.89495C6.6065 6.7763 6.88506 6.68861 7.17048 6.63015C7.3046 6.21232 7.50406 5.82029 7.76026 5.46608C8.01817 5.11188 8.32939 4.80066 8.6836 4.54274C9.03781 4.28654 9.42984 4.08708 9.84595 3.95125C10.2621 3.81713 10.6971 3.74835 11.1355 3.75007C11.4261 3.74835 11.7184 3.77758 12.0039 3.83776C12.2893 3.89794 12.5678 3.98736 12.8344 4.106C13.1009 4.22636 13.3536 4.37251 13.5892 4.54446C13.8248 4.71812 14.0414 4.91414 14.234 5.13251C14.6621 5.04138 15.1023 5.01903 15.5373 5.06373C15.9723 5.10844 16.3971 5.22364 16.7977 5.40074C17.1966 5.57957 17.5663 5.81857 17.8913 6.1126C18.2162 6.4049 18.4931 6.74707 18.7114 7.12707C18.8576 7.37811 18.9779 7.64463 19.0691 7.92318C19.1602 8.20001 19.2221 8.48544 19.2513 8.77602C19.2823 9.06661 19.2823 9.35892 19.2496 9.64951C19.2187 9.94009 19.155 10.2255 19.0639 10.5024C19.3579 10.8273 19.5969 11.1953 19.7758 11.5959ZM14.0466 18.9363C14.4214 18.7815 14.7619 18.5528 15.049 18.2657C15.3362 17.9785 15.5648 17.6381 15.7196 17.2615C15.8743 16.8867 15.9552 16.4843 15.9552 16.0785V12.2442C15.954 12.2407 15.9529 12.2367 15.9517 12.2321C15.9506 12.2287 15.9488 12.2252 15.9466 12.2218C15.9443 12.2184 15.9414 12.2155 15.938 12.2132C15.9345 12.2098 15.9311 12.2075 15.9276 12.2063L14.54 11.4051V16.0373C14.54 16.0837 14.5332 16.1318 14.5211 16.1765C14.5091 16.223 14.4919 16.2659 14.4678 16.3072C14.4438 16.3485 14.4162 16.3863 14.3819 16.419C14.3484 16.4523 14.3109 16.4812 14.2701 16.505L10.9842 18.4015C10.9567 18.4187 10.9103 18.4428 10.8862 18.4565C11.0221 18.5717 11.1699 18.6732 11.3247 18.7626C11.4811 18.852 11.6428 18.9277 11.8113 18.9896C11.9798 19.0497 12.1535 19.0962 12.3288 19.1271C12.5059 19.1581 12.6848 19.1735 12.8636 19.1735C13.2694 19.1735 13.6717 19.0927 14.0466 18.9363ZM6.22135 16.333C6.42596 16.6855 6.69592 16.9916 7.01745 17.2392C7.34071 17.4868 7.70695 17.6673 8.09899 17.7722C8.49102 17.8771 8.90025 17.9046 9.3026 17.8513C9.70495 17.798 10.0918 17.6673 10.4443 17.4644L13.7663 15.5472L13.7749 15.5386C13.7772 15.5363 13.7789 15.5329 13.78 15.5283C13.7823 15.5249 13.7841 15.5214 13.7852 15.518V13.9017L9.77545 16.2212C9.73418 16.2453 9.6912 16.2625 9.64649 16.2763C9.60007 16.2883 9.55364 16.2935 9.5055 16.2935C9.45907 16.2935 9.41265 16.2883 9.36622 16.2763C9.32152 16.2625 9.27681 16.2453 9.23554 16.2212L5.94967 14.323C5.92044 14.3058 5.87746 14.28 5.85339 14.2645C5.82244 14.4416 5.80696 14.6204 5.80696 14.7993C5.80696 14.9781 5.82415 15.1569 5.85511 15.334C5.88605 15.5094 5.9342 15.6831 5.99438 15.8516C6.05628 16.0201 6.13194 16.1817 6.22135 16.3364V16.333ZM5.35818 9.1629C5.15529 9.51539 5.02461 9.90398 4.97131 10.3063C4.918 10.7087 4.94552 11.1162 5.0504 11.51C5.15529 11.902 5.33583 12.2682 5.58343 12.5915C5.83103 12.913 6.13881 13.183 6.48958 13.3859L9.80984 15.3048C9.81328 15.3059 9.81729 15.3071 9.82188 15.3082H9.83391C9.8385 15.3082 9.84251 15.3071 9.84595 15.3048C9.84939 15.3036 9.85283 15.3019 9.85627 15.2996L11.249 14.4949L7.23926 12.1805C7.19971 12.1565 7.16189 12.1272 7.1275 12.0946C7.09418 12.0611 7.06529 12.0236 7.04153 11.9828C7.01917 11.9415 7.00026 11.8985 6.98822 11.8521C6.97619 11.8074 6.96931 11.761 6.97103 11.7128V7.80797C6.80252 7.86987 6.63917 7.94553 6.48442 8.03494C6.32967 8.12607 6.18352 8.22924 6.04596 8.34444C5.91013 8.45965 5.78289 8.58688 5.66769 8.72444C5.55248 8.86028 5.45103 9.00815 5.36162 9.1629H5.35818ZM16.7633 11.8177C16.8046 11.8418 16.8424 11.8693 16.8768 11.9037C16.9094 11.9364 16.9387 11.9742 16.9628 12.0155C16.9851 12.0567 17.004 12.1014 17.0161 12.1461C17.0264 12.1926 17.0332 12.239 17.0315 12.2871V16.192C17.5835 15.9891 18.0649 15.6332 18.4208 15.1655C18.7785 14.6978 18.9934 14.139 19.0433 13.5544C19.0931 12.9698 18.9762 12.3817 18.7046 11.8607C18.4329 11.3397 18.0185 10.9064 17.5095 10.6141L14.1893 8.69521C14.1858 8.69406 14.1818 8.69292 14.1772 8.69177H14.1652C14.1618 8.69292 14.1578 8.69406 14.1532 8.69521C14.1497 8.69636 14.1463 8.69808 14.1429 8.70037L12.757 9.50163L16.7667 11.8177H16.7633ZM18.1475 9.7372H18.1457V9.73892L18.1475 9.7372ZM18.1457 9.73548C18.2455 9.15774 18.1784 8.56281 17.9514 8.02119C17.7262 7.47956 17.3496 7.01359 16.8682 6.67658C16.3867 6.34128 15.8193 6.1487 15.233 6.12291C14.6449 6.09884 14.0638 6.24155 13.5548 6.53386L10.2345 8.45105C10.2311 8.45334 10.2282 8.45621 10.2259 8.45965L10.2191 8.46996C10.2179 8.4734 10.2168 8.47741 10.2156 8.482C10.2145 8.48544 10.2139 8.48945 10.2139 8.49403V10.0966L14.2237 7.78046C14.2649 7.75639 14.3096 7.7392 14.3543 7.72544C14.4008 7.7134 14.4472 7.70825 14.4936 7.70825C14.5418 7.70825 14.5882 7.7134 14.6346 7.72544C14.6793 7.7392 14.7223 7.75639 14.7636 7.78046L18.0494 9.67874C18.0787 9.69593 18.1217 9.72 18.1457 9.73548ZM9.45735 7.96101C9.45735 7.91458 9.46423 7.86816 9.47627 7.82173C9.4883 7.77702 9.5055 7.73232 9.52957 7.69105C9.55364 7.6515 9.58115 7.61368 9.61554 7.57929C9.64821 7.54662 9.68604 7.51739 9.72731 7.49503L13.0132 5.59848C13.0441 5.57957 13.0871 5.55549 13.1112 5.54346C12.6607 5.1669 12.1105 4.92618 11.5276 4.85224C10.9447 4.77658 10.3532 4.86943 9.82188 5.11875C9.28885 5.36807 8.83835 5.76527 8.52369 6.26047C8.20903 6.75739 8.04224 7.33169 8.04224 7.91974V11.7541C8.04339 11.7587 8.04454 11.7627 8.04568 11.7661C8.04683 11.7696 8.04855 11.773 8.05084 11.7765C8.05313 11.7799 8.056 11.7833 8.05944 11.7868C8.06173 11.7891 8.06517 11.7914 8.06976 11.7937L9.45735 12.5949V7.96101ZM10.2105 13.0282L11.997 14.0599L13.7835 13.0282V10.9666L11.9987 9.93493L10.2122 10.9666L10.2105 13.0282Z", + "fill": "white" + }, + "children": [] + } + ] + }, + "name": "OpenaiBlue" +} diff --git a/web/app/components/base/icons/src/public/llm/OpenaiBlue.tsx b/web/app/components/base/icons/src/public/llm/OpenaiBlue.tsx new file mode 100644 index 0000000000..9934a77591 --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/OpenaiBlue.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import type { IconData } from '@/app/components/base/icons/IconBase' +import * as React from 'react' +import IconBase from '@/app/components/base/icons/IconBase' +import data from './OpenaiBlue.json' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject> + }, +) => + +Icon.displayName = 'OpenaiBlue' + +export default Icon diff --git a/web/app/components/base/icons/src/public/llm/OpenaiSmall.json b/web/app/components/base/icons/src/public/llm/OpenaiSmall.json new file mode 100644 index 0000000000..aa72f614bc --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/OpenaiSmall.json @@ -0,0 +1,128 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "26", + "height": "26", + "viewBox": "0 0 26 26", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg", + "xmlns:xlink": "http://www.w3.org/1999/xlink" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "clip-path": "url(#clip0_3892_83671)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M1 13C1 9.27247 1 7.4087 1.60896 5.93853C2.42092 3.97831 3.97831 2.42092 5.93853 1.60896C7.4087 1 9.27247 1 13 1C16.7275 1 18.5913 1 20.0615 1.60896C22.0217 2.42092 23.5791 3.97831 24.391 5.93853C25 7.4087 25 9.27247 25 13C25 16.7275 25 18.5913 24.391 20.0615C23.5791 22.0217 22.0217 23.5791 20.0615 24.391C18.5913 25 16.7275 25 13 25C9.27247 25 7.4087 25 5.93853 24.391C3.97831 23.5791 2.42092 22.0217 1.60896 20.0615C1 18.5913 1 16.7275 1 13Z", + "fill": "white" + }, + "children": [] + }, + { + "type": "element", + "name": "rect", + "attributes": { + "width": "24", + "height": "24", + "transform": "translate(1 1)", + "fill": "url(#pattern0_3892_83671)" + }, + "children": [] + }, + { + "type": "element", + "name": "rect", + "attributes": { + "width": "24", + "height": "24", + "transform": "translate(1 1)", + "fill": "white", + "fill-opacity": "0.01" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M13 0.75C14.8603 0.75 16.2684 0.750313 17.3945 0.827148C18.5228 0.904144 19.3867 1.05876 20.1572 1.37793C22.1787 2.21525 23.7847 3.82133 24.6221 5.84277C24.9412 6.61333 25.0959 7.47723 25.1729 8.60547C25.2497 9.73161 25.25 11.1397 25.25 13C25.25 14.8603 25.2497 16.2684 25.1729 17.3945C25.0959 18.5228 24.9412 19.3867 24.6221 20.1572C23.7847 22.1787 22.1787 23.7847 20.1572 24.6221C19.3867 24.9412 18.5228 25.0959 17.3945 25.1729C16.2684 25.2497 14.8603 25.25 13 25.25C11.1397 25.25 9.73161 25.2497 8.60547 25.1729C7.47723 25.0959 6.61333 24.9412 5.84277 24.6221C3.82133 23.7847 2.21525 22.1787 1.37793 20.1572C1.05876 19.3867 0.904144 18.5228 0.827148 17.3945C0.750313 16.2684 0.75 14.8603 0.75 13C0.75 11.1397 0.750313 9.73161 0.827148 8.60547C0.904144 7.47723 1.05876 6.61333 1.37793 5.84277C2.21525 3.82133 3.82133 2.21525 5.84277 1.37793C6.61333 1.05876 7.47723 0.904144 8.60547 0.827148C9.73161 0.750313 11.1397 0.75 13 0.75Z", + "stroke": "#101828", + "stroke-opacity": "0.08", + "stroke-width": "0.5" + }, + "children": [] + }, + { + "type": "element", + "name": "defs", + "attributes": {}, + "children": [ + { + "type": "element", + "name": "pattern", + "attributes": { + "id": "pattern0_3892_83671", + "patternContentUnits": "objectBoundingBox", + "width": "1", + "height": "1" + }, + "children": [ + { + "type": "element", + "name": "use", + "attributes": { + "xlink:href": "#image0_3892_83671", + "transform": "scale(0.00625)" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "clipPath", + "attributes": { + "id": "clip0_3892_83671" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M1 13C1 9.27247 1 7.4087 1.60896 5.93853C2.42092 3.97831 3.97831 2.42092 5.93853 1.60896C7.4087 1 9.27247 1 13 1C16.7275 1 18.5913 1 20.0615 1.60896C22.0217 2.42092 23.5791 3.97831 24.391 5.93853C25 7.4087 25 9.27247 25 13C25 16.7275 25 18.5913 24.391 20.0615C23.5791 22.0217 22.0217 23.5791 20.0615 24.391C18.5913 25 16.7275 25 13 25C9.27247 25 7.4087 25 5.93853 24.391C3.97831 23.5791 2.42092 22.0217 1.60896 20.0615C1 18.5913 1 16.7275 1 13Z", + "fill": "white" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "image", + "attributes": { + "id": "image0_3892_83671", + "width": "160", + "height": "160", + "preserveAspectRatio": "none", + "xlink:href": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAKAAAACgCAYAAACLz2ctAAAACXBIWXMAACxLAAAsSwGlPZapAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAA0ySURBVHgB7Z3vsdM6E8Z93nm/QwdABUAFgQqACgIVABUAFcCpAKgAOoBUwKECOBUAFfj6yYzumNx4pV2vtKtkfzO+3CHESezHWu0frS7GiSEIjPjfEASGhAADU0KAgSkhwMCUEGBgSggwMCUEGJgSAgxMCQEGpoQAA1NCgIEpIcDAlBBgYEoIMDAlBBiYEgIMTAkBBqaEAANTQoCBKSHAwJQQYGBKCDAwJQQYmPL/4cz5/fv38PXr12G32w0/f/4crq6u9n+P/0/cvHlzuH379v7Pe/fuDZvNZv8n/i5Yx8U5LkyH6D5+/Dh8/vx5Lz4pEOGDBw+G58+fhxiljGfEly9fxkkweODUD5wX5w94nIUAf/z4UU14x4SIzwvKOHkBvn79epzmbk3ENz9evHgx/vr1awxoTnYOCCfi2bNnq+Z4a8G8cDLLMT8kOEkBwpN9+PDh3tngMvd4EzgPBC05H3j37t3eUTkknffw3PjsdMDROWnGE+PDhw8sk4s526tXr/YORM5k4vVPnz6NT58+HSeRskwypgJ4/yRG9vsnEe5NOj771DgpAUJ8JTcUAoXovn37Nq7h/fv3zZyb+XeHgE/F4z4ZAUJMJTdwMoXqzgFGJu6IqHHgM/HQ9cxJCBBhj5wA8PraES8HRtXWIsTRc+jnJHLBcDjmqbNDttvtMImv+oT+4uLiL+elFfD079y5M7x582bojrFzMLkfiNEBo1JtMB+zMMHHDjgsPY2GXYdhMOrhyV9iEt8wCXSo+fnSWCNG41TQcOvWrb9eQ0jm+vp6H07CwQ3/dBV/HDsG3uCwMBI8fvx4rAWcmNzIOyzM1eA5c50gzF3fvn3LGmXhLdee82rQrQBh9gbC4ahlhiAgbmoPD4PW9+EUVPQgwm4FSI1+EIk2kkqamhUy+I0lI2LNh1GDLgWIC9rK9MJcUmJfGnlgMmuD61Dy3eCYeC2M6FKA1PxLc8SRVNLUCHTnKIk/IpXnkS4FiCd6yeRpIAmrWAeDS0ToMX3XnQAp87t27icpXIVQvdzYnAjx4HqjOwFCZEsXWDoCpbAKx9ymggZvcyuYWup7e8sddyNA3GiIb8n8Sp9uSVgFE3+vk3p8L2r6gNc84V6AMG/wbHMi4U6yvYRVMGpri5mKkXqbC7oVIFcgKPQshZvFgPi1Y4v4vvOHStuJoa6dlrOmgTsBSlewlVYLU05Mi3le7sGCedcQYm4U9DKFcCVAbjm9xKyUjn7aIxInoK1VaEoJnWMxauJGgDnvTUuAORHUCKtIl4auTaNREYOaxRoczAWIkQEXY434tASILIYmWnWCUrOMa7t0TjwQHjAVIC7QUlhl6aLVFKDWvKhGJwYIWWI2qe/hoUjBtCQfxZypGxUFGgChwHJK8A81WVtOj8JRlMXfv39ffUE8il+nacq+ABdNlUqhliGUXPvamAkQNyp3ISEIiA7igwg9MzkNe+GhAru0ghm/aZqnsbprQYhPnjzZP7zUOpjE3bt3F1/78+fPYM5oADU5TsextQ3U+zRMsARJQPtYuVZpadXhAQeHglqumntvC5oLEBc65xFut9uj8zFPAsT3k3juubgirg9nXjwMdNiGinuepQBzTznEt4QXAR5mMUoOblyxtOJ5fhzzlikB4t9b01SAOdObKyiwFiA+QzI6SeOKGCkli91THxoQI+CMXJVGboSwEiC+FzdWmdJ4Gkjmh8ksexdgMy8YXiLltcEb9LaOdR5W4YQ+0nvRKUEDXBdcnynfzfKWJ9Huu0ZQ5zVnbEQuAV9CyxFQK4tRo4GQZH645prVpIkAcxUopZPzFgKs1U9au2WGNGwzPzysGW5igi8vLxdfg5nwYnphbpFpKM1ipC6mJSDrgHOXBpBzJLM8CVEUpHfTfXVsAOU5ckMTQ8URkHOksvw1DoImXLPsZYFSdQFS5pdbmetBgEtl+dIVddpmkBO2OYswzOQ9Ll4AbnWHpQBL43laAeQ1cEZlaxFWFyCVruJ6YRYClJblS7pZaYuh1JO3rI6uLkDKLHFpLcC1bTY8zA9LC36tPOLqAtRcx9tKgHhoNG+IRIjaZjk3N4TwLRYqVRfgUtJesjSwtgBrt9mQzA/ned215Kp3LBoYVRfg0o+VLIrxWA8ogSNAbbOc89RbZ0fMKqItusn3SsrrIpC9Noidyye37rRvJkDpvmvnTGrKviabggcfGZQlkAVqucFjdQEuPW0a6ahzBZVFqHLBru8SkLqj0nctR8EYAR0BUcDUljA3y9xSMZAbBVvdn+oCXEp4r9n+9Bi73W7onVRgwKmNnK+S41xPnJ8aBakCEk3MTDDQnGtgOWSXW1UdASMbqlyw0U6pswazDCFywPmXaDUPrC5A6inTHrUgQqlJ8gh+D/a4KzXLXAcC92ZJ4K3McHUBbjabxdfw1HIoqV/jLtz2zrzur8Qsf//+feBAibvFKFhdgHjClkZBPGGcHwkBlhZfQtzd7iB5BIgPIoQYKbPMHbVaWqhjNPGCNV1+3IBSkwSSWZaGLLyB345gshaUhWpiQcYGUG3CcEiS7pK8KtJ/mtU5UgaiAKEE7aWWS9exRUPzJiMgZYYB5mtcJJ4inJOWUf5esEyLNgtE51x+qTC4nmLwXyyzVc0EmEv/cAOpc5KnCCF6W9zeA2cxAgJqFEzhkzXAS06eYgixHMu0aFMBYgREl88lYIZfvnw5rAXmGE0tqc86Rm1vGb8Pn+GNJQE2GRnHxuS2khoG3ZVaksZC2uXwKO8vWbJp5QVrb3/GwaRDKtW1c5iFTDTXKFgsl+Q2sbQS4NK5WuyoZNYlH8sWczekxhoNbr89SXd6ye6bVgKkdlRqsUbEdJuGUjFYLtyeC7FkXcaarloWAsTDtXSu0u3P1mC+UY1lKwnNndElzco9CNB6HxEXW3Vx2t566beXRmXOHnDeBEidp1XzIhcClOyjpr2ZoMQs43tzvzs14rcWIPV7W60RNhdgiUdcMhJpIdkmgfvAeBCgVtPQtZhu1QXWRuHXrhA7BBkUpPS0sik4B4LiODxlZ6gyOCQMWn1XcwFSCe/SSPzaFWLHwPkgRCp9SIHvjvdKO5jWBNeLqkbfbrdDK8wFeH19vfgaCi+lK8Q0KjkgInS656akUqkYKnW8geuS65zftLJoNIbawTzNQ/CnZFusNY19pCGaksD5YDgHzKUlERFoibkAKU/s2LZT3LwuN2wjabULT5hz46wEmHuILbbu6mIEPEQSt8ttk5DSZ5xz4pB0T7UQYC7EhIeolec7x7UAc00icfMl+dbDC91i88E5LQVYOqK3Nr0JcwFSPf9KcpHSvC7MET5b0tl+bYFEKwHi95U8WFp72kkwF6BWF32MlhrbV1EmSutG1RYgJy8taRSqibkAqYsp7aKqLcS0KY0G1BJVDQFyphKYF1v0hZ7jOheMv5cgMcvHjtKwCgfKE9UQYOmhXfArxYUAqdDKGgFIwjZJ+NqT8pK4YisBYvsJL7gQIFUUqTFH4ZhlSViFghNX1CokaPlgrcWFAHOtO7QEQZVCaZd3ScrySz+fKqNv9fu0cCFAQI0SmrVph/PDGvO80vDH4chbChU5OCa81lsvcHAjwNxT7WFz5RySsnyJQHIZG4gfUxfPwku4ESCg5mle9rc9hqQsf818bEnkuEY4pwfvthRXAqSyItqmWAPJPE+6++acpc/z5mCU4EqAIFcOb7m16BzNFXUcqPmfRycjhzsBlnh42qviOEjKtTQX2C/FNT1PUSjcCRCUFJ+23umb22Zjbm610FiM5A2XAsTNLlmZppmjpZCUa9X4bpzi3V5wKUCAC1oyx6qxUD3RKqxSArV81aKSWQu3AgS46KUjj+aNl+SQa6a5ci3teoj3LeFagIAjwjQiYq7GDVzjJksKVHFo548P0aig8coF/jM4Jy0l5C61xHJKtO3FgeWGN27c+Ot1LAnFOa+urvYHF6z3rd0OGAvIqeWdPwp3UHLL2Am1WmZIjhp9C4+RS7lZltJr0Y0AE9wGk5qHdlhliZKQT4tNZFrQnQCBJBisddSOt2HumhvpIb5ewy6HdCnARI31H6UmWLt7KGddcg+VQaV0LcAE5mPwBiV9BtccGIXXxiAhJk5BQ48FBxQnIcA5GJkwf+J2XYWA0TgdI5GkkTnej8/OhWPwOh4YiI4zjfBYTq9BF2GYNSC8gh6E6UggRIPwBY5j3a+m9Jt405wU/pmDz0bIR9IPEedDf8GSDbt74+QFuAZp/FGTFrFGS8z7A3omdUvlbvmlQWpw6a2zqjpjUETL0I/XFWw1CAEy0dgPhBJez4UFEmIOKATzwsvLy/0OmJI8cgJzvEePHu3b4lru22tFCFCBVNCw2+3+9boPPd40j0uFEZvNZi++cxTdnBBgYEp4wYEpIcDAlBBgYEoIMDAlBBiYEgIMTAkBBqaEAANTQoCBKSHAwJQQYGBKCDAwJQQYmBICDEwJAQamhAADU0KAgSkhwMCUEGBgSggwMCUEGJgSAgxMCQEGpvwDojzI2oXtJzYAAAAASUVORK5CYII=" + }, + "children": [] + } + ] + } + ] + }, + "name": "OpenaiSmall" +} diff --git a/web/app/components/base/icons/src/public/llm/OpenaiSmall.tsx b/web/app/components/base/icons/src/public/llm/OpenaiSmall.tsx new file mode 100644 index 0000000000..6307091e0b --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/OpenaiSmall.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import type { IconData } from '@/app/components/base/icons/IconBase' +import * as React from 'react' +import IconBase from '@/app/components/base/icons/IconBase' +import data from './OpenaiSmall.json' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject> + }, +) => + +Icon.displayName = 'OpenaiSmall' + +export default Icon diff --git a/web/app/components/base/icons/src/public/llm/OpenaiTeal.json b/web/app/components/base/icons/src/public/llm/OpenaiTeal.json new file mode 100644 index 0000000000..ffd0981512 --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/OpenaiTeal.json @@ -0,0 +1,37 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "24", + "height": "24", + "viewBox": "0 0 24 24", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "width": "24", + "height": "24", + "rx": "6", + "fill": "#009688" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M19.7758 11.5959C19.9546 11.9948 20.0681 12.4213 20.1145 12.8563C20.1592 13.2913 20.1369 13.7315 20.044 14.1596C19.9529 14.5878 19.7947 14.9987 19.5746 15.377C19.4302 15.6298 19.2599 15.867 19.0639 16.0854C18.8696 16.3021 18.653 16.4981 18.4174 16.67C18.1801 16.842 17.9274 16.9864 17.6591 17.105C17.3926 17.222 17.1141 17.3114 16.8286 17.3698C16.6945 17.7859 16.4951 18.1797 16.2371 18.5339C15.9809 18.8881 15.6697 19.1993 15.3155 19.4555C14.9613 19.7134 14.5693 19.9129 14.1532 20.047C13.7371 20.1829 13.302 20.2499 12.8636 20.2499C12.573 20.2516 12.2807 20.2207 11.9953 20.1622C11.7116 20.102 11.433 20.0109 11.1665 19.8923C10.9 19.7736 10.6472 19.6258 10.4116 19.4538C10.1778 19.2819 9.96115 19.0841 9.76857 18.8658C9.33871 18.9586 8.89853 18.981 8.46351 18.9363C8.02849 18.8898 7.60207 18.7763 7.20143 18.5975C6.80252 18.4204 6.43284 18.1797 6.10786 17.8857C5.78289 17.5916 5.50606 17.2478 5.28769 16.8695C5.14153 16.6167 5.02117 16.3502 4.93004 16.0734C4.83891 15.7965 4.77873 15.5111 4.74778 15.2205C4.71683 14.9317 4.71855 14.6393 4.7495 14.3488C4.78045 14.0599 4.84407 13.7745 4.9352 13.4976C4.64289 13.1727 4.40217 12.803 4.22335 12.4041C4.04624 12.0034 3.93104 11.5787 3.88634 11.1437C3.83991 10.7087 3.86398 10.2685 3.95511 9.84036C4.04624 9.41222 4.20443 9.00127 4.42452 8.62299C4.56896 8.37023 4.73918 8.13123 4.93348 7.91458C5.12778 7.69793 5.34615 7.50191 5.58171 7.32997C5.81728 7.15802 6.07176 7.01187 6.33827 6.89495C6.6065 6.7763 6.88506 6.68861 7.17048 6.63015C7.3046 6.21232 7.50406 5.82029 7.76026 5.46608C8.01817 5.11188 8.32939 4.80066 8.6836 4.54274C9.03781 4.28654 9.42984 4.08708 9.84595 3.95125C10.2621 3.81713 10.6971 3.74835 11.1355 3.75007C11.4261 3.74835 11.7184 3.77758 12.0039 3.83776C12.2893 3.89794 12.5678 3.98736 12.8344 4.106C13.1009 4.22636 13.3536 4.37251 13.5892 4.54446C13.8248 4.71812 14.0414 4.91414 14.234 5.13251C14.6621 5.04138 15.1023 5.01903 15.5373 5.06373C15.9723 5.10844 16.3971 5.22364 16.7977 5.40074C17.1966 5.57957 17.5663 5.81857 17.8913 6.1126C18.2162 6.4049 18.4931 6.74707 18.7114 7.12707C18.8576 7.37811 18.9779 7.64463 19.0691 7.92318C19.1602 8.20001 19.2221 8.48544 19.2513 8.77602C19.2823 9.06661 19.2823 9.35892 19.2496 9.64951C19.2187 9.94009 19.155 10.2255 19.0639 10.5024C19.3579 10.8273 19.5969 11.1953 19.7758 11.5959ZM14.0466 18.9363C14.4214 18.7815 14.7619 18.5528 15.049 18.2657C15.3362 17.9785 15.5648 17.6381 15.7196 17.2615C15.8743 16.8867 15.9552 16.4843 15.9552 16.0785V12.2442C15.954 12.2407 15.9529 12.2367 15.9517 12.2321C15.9506 12.2287 15.9488 12.2252 15.9466 12.2218C15.9443 12.2184 15.9414 12.2155 15.938 12.2132C15.9345 12.2098 15.9311 12.2075 15.9276 12.2063L14.54 11.4051V16.0373C14.54 16.0837 14.5332 16.1318 14.5211 16.1765C14.5091 16.223 14.4919 16.2659 14.4678 16.3072C14.4438 16.3485 14.4162 16.3863 14.3819 16.419C14.3484 16.4523 14.3109 16.4812 14.2701 16.505L10.9842 18.4015C10.9567 18.4187 10.9103 18.4428 10.8862 18.4565C11.0221 18.5717 11.1699 18.6732 11.3247 18.7626C11.4811 18.852 11.6428 18.9277 11.8113 18.9896C11.9798 19.0497 12.1535 19.0962 12.3288 19.1271C12.5059 19.1581 12.6848 19.1735 12.8636 19.1735C13.2694 19.1735 13.6717 19.0927 14.0466 18.9363ZM6.22135 16.333C6.42596 16.6855 6.69592 16.9916 7.01745 17.2392C7.34071 17.4868 7.70695 17.6673 8.09899 17.7722C8.49102 17.8771 8.90025 17.9046 9.3026 17.8513C9.70495 17.798 10.0918 17.6673 10.4443 17.4644L13.7663 15.5472L13.7749 15.5386C13.7772 15.5363 13.7789 15.5329 13.78 15.5283C13.7823 15.5249 13.7841 15.5214 13.7852 15.518V13.9017L9.77545 16.2212C9.73418 16.2453 9.6912 16.2625 9.64649 16.2763C9.60007 16.2883 9.55364 16.2935 9.5055 16.2935C9.45907 16.2935 9.41265 16.2883 9.36622 16.2763C9.32152 16.2625 9.27681 16.2453 9.23554 16.2212L5.94967 14.323C5.92044 14.3058 5.87746 14.28 5.85339 14.2645C5.82244 14.4416 5.80696 14.6204 5.80696 14.7993C5.80696 14.9781 5.82415 15.1569 5.85511 15.334C5.88605 15.5094 5.9342 15.6831 5.99438 15.8516C6.05628 16.0201 6.13194 16.1817 6.22135 16.3364V16.333ZM5.35818 9.1629C5.15529 9.51539 5.02461 9.90398 4.97131 10.3063C4.918 10.7087 4.94552 11.1162 5.0504 11.51C5.15529 11.902 5.33583 12.2682 5.58343 12.5915C5.83103 12.913 6.13881 13.183 6.48958 13.3859L9.80984 15.3048C9.81328 15.3059 9.81729 15.3071 9.82188 15.3082H9.83391C9.8385 15.3082 9.84251 15.3071 9.84595 15.3048C9.84939 15.3036 9.85283 15.3019 9.85627 15.2996L11.249 14.4949L7.23926 12.1805C7.19971 12.1565 7.16189 12.1272 7.1275 12.0946C7.09418 12.0611 7.06529 12.0236 7.04153 11.9828C7.01917 11.9415 7.00026 11.8985 6.98822 11.8521C6.97619 11.8074 6.96931 11.761 6.97103 11.7128V7.80797C6.80252 7.86987 6.63917 7.94553 6.48442 8.03494C6.32967 8.12607 6.18352 8.22924 6.04596 8.34444C5.91013 8.45965 5.78289 8.58688 5.66769 8.72444C5.55248 8.86028 5.45103 9.00815 5.36162 9.1629H5.35818ZM16.7633 11.8177C16.8046 11.8418 16.8424 11.8693 16.8768 11.9037C16.9094 11.9364 16.9387 11.9742 16.9628 12.0155C16.9851 12.0567 17.004 12.1014 17.0161 12.1461C17.0264 12.1926 17.0332 12.239 17.0315 12.2871V16.192C17.5835 15.9891 18.0649 15.6332 18.4208 15.1655C18.7785 14.6978 18.9934 14.139 19.0433 13.5544C19.0931 12.9698 18.9762 12.3817 18.7046 11.8607C18.4329 11.3397 18.0185 10.9064 17.5095 10.6141L14.1893 8.69521C14.1858 8.69406 14.1818 8.69292 14.1772 8.69177H14.1652C14.1618 8.69292 14.1578 8.69406 14.1532 8.69521C14.1497 8.69636 14.1463 8.69808 14.1429 8.70037L12.757 9.50163L16.7667 11.8177H16.7633ZM18.1475 9.7372H18.1457V9.73892L18.1475 9.7372ZM18.1457 9.73548C18.2455 9.15774 18.1784 8.56281 17.9514 8.02119C17.7262 7.47956 17.3496 7.01359 16.8682 6.67658C16.3867 6.34128 15.8193 6.1487 15.233 6.12291C14.6449 6.09884 14.0638 6.24155 13.5548 6.53386L10.2345 8.45105C10.2311 8.45334 10.2282 8.45621 10.2259 8.45965L10.2191 8.46996C10.2179 8.4734 10.2168 8.47741 10.2156 8.482C10.2145 8.48544 10.2139 8.48945 10.2139 8.49403V10.0966L14.2237 7.78046C14.2649 7.75639 14.3096 7.7392 14.3543 7.72544C14.4008 7.7134 14.4472 7.70825 14.4936 7.70825C14.5418 7.70825 14.5882 7.7134 14.6346 7.72544C14.6793 7.7392 14.7223 7.75639 14.7636 7.78046L18.0494 9.67874C18.0787 9.69593 18.1217 9.72 18.1457 9.73548ZM9.45735 7.96101C9.45735 7.91458 9.46423 7.86816 9.47627 7.82173C9.4883 7.77702 9.5055 7.73232 9.52957 7.69105C9.55364 7.6515 9.58115 7.61368 9.61554 7.57929C9.64821 7.54662 9.68604 7.51739 9.72731 7.49503L13.0132 5.59848C13.0441 5.57957 13.0871 5.55549 13.1112 5.54346C12.6607 5.1669 12.1105 4.92618 11.5276 4.85224C10.9447 4.77658 10.3532 4.86943 9.82188 5.11875C9.28885 5.36807 8.83835 5.76527 8.52369 6.26047C8.20903 6.75739 8.04224 7.33169 8.04224 7.91974V11.7541C8.04339 11.7587 8.04454 11.7627 8.04568 11.7661C8.04683 11.7696 8.04855 11.773 8.05084 11.7765C8.05313 11.7799 8.056 11.7833 8.05944 11.7868C8.06173 11.7891 8.06517 11.7914 8.06976 11.7937L9.45735 12.5949V7.96101ZM10.2105 13.0282L11.997 14.0599L13.7835 13.0282V10.9666L11.9987 9.93493L10.2122 10.9666L10.2105 13.0282Z", + "fill": "white" + }, + "children": [] + } + ] + }, + "name": "OpenaiTeal" +} diff --git a/web/app/components/base/icons/src/public/llm/OpenaiTeal.tsx b/web/app/components/base/icons/src/public/llm/OpenaiTeal.tsx new file mode 100644 index 0000000000..ef803ea52f --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/OpenaiTeal.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import type { IconData } from '@/app/components/base/icons/IconBase' +import * as React from 'react' +import IconBase from '@/app/components/base/icons/IconBase' +import data from './OpenaiTeal.json' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject> + }, +) => + +Icon.displayName = 'OpenaiTeal' + +export default Icon diff --git a/web/app/components/base/icons/src/public/llm/OpenaiViolet.json b/web/app/components/base/icons/src/public/llm/OpenaiViolet.json new file mode 100644 index 0000000000..e80a85507e --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/OpenaiViolet.json @@ -0,0 +1,37 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "24", + "height": "24", + "viewBox": "0 0 24 24", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "width": "24", + "height": "24", + "rx": "6", + "fill": "#AB68FF" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M19.7758 11.5959C19.9546 11.9948 20.0681 12.4213 20.1145 12.8563C20.1592 13.2913 20.1369 13.7315 20.044 14.1596C19.9529 14.5878 19.7947 14.9987 19.5746 15.377C19.4302 15.6298 19.2599 15.867 19.0639 16.0854C18.8696 16.3021 18.653 16.4981 18.4174 16.67C18.1801 16.842 17.9274 16.9864 17.6591 17.105C17.3926 17.222 17.1141 17.3114 16.8286 17.3698C16.6945 17.7859 16.4951 18.1797 16.2371 18.5339C15.9809 18.8881 15.6697 19.1993 15.3155 19.4555C14.9613 19.7134 14.5693 19.9129 14.1532 20.047C13.7371 20.1829 13.302 20.2499 12.8636 20.2499C12.573 20.2516 12.2807 20.2207 11.9953 20.1622C11.7116 20.102 11.433 20.0109 11.1665 19.8923C10.9 19.7736 10.6472 19.6258 10.4116 19.4538C10.1778 19.2819 9.96115 19.0841 9.76857 18.8658C9.33871 18.9586 8.89853 18.981 8.46351 18.9363C8.02849 18.8898 7.60207 18.7763 7.20143 18.5975C6.80252 18.4204 6.43284 18.1797 6.10786 17.8857C5.78289 17.5916 5.50606 17.2478 5.28769 16.8695C5.14153 16.6167 5.02117 16.3502 4.93004 16.0734C4.83891 15.7965 4.77873 15.5111 4.74778 15.2205C4.71683 14.9317 4.71855 14.6393 4.7495 14.3488C4.78045 14.0599 4.84407 13.7745 4.9352 13.4976C4.64289 13.1727 4.40217 12.803 4.22335 12.4041C4.04624 12.0034 3.93104 11.5787 3.88634 11.1437C3.83991 10.7087 3.86398 10.2685 3.95511 9.84036C4.04624 9.41222 4.20443 9.00127 4.42452 8.62299C4.56896 8.37023 4.73918 8.13123 4.93348 7.91458C5.12778 7.69793 5.34615 7.50191 5.58171 7.32997C5.81728 7.15802 6.07176 7.01187 6.33827 6.89495C6.6065 6.7763 6.88506 6.68861 7.17048 6.63015C7.3046 6.21232 7.50406 5.82029 7.76026 5.46608C8.01817 5.11188 8.32939 4.80066 8.6836 4.54274C9.03781 4.28654 9.42984 4.08708 9.84595 3.95125C10.2621 3.81713 10.6971 3.74835 11.1355 3.75007C11.4261 3.74835 11.7184 3.77758 12.0039 3.83776C12.2893 3.89794 12.5678 3.98736 12.8344 4.106C13.1009 4.22636 13.3536 4.37251 13.5892 4.54446C13.8248 4.71812 14.0414 4.91414 14.234 5.13251C14.6621 5.04138 15.1023 5.01903 15.5373 5.06373C15.9723 5.10844 16.3971 5.22364 16.7977 5.40074C17.1966 5.57957 17.5663 5.81857 17.8913 6.1126C18.2162 6.4049 18.4931 6.74707 18.7114 7.12707C18.8576 7.37811 18.9779 7.64463 19.0691 7.92318C19.1602 8.20001 19.2221 8.48544 19.2513 8.77602C19.2823 9.06661 19.2823 9.35892 19.2496 9.64951C19.2187 9.94009 19.155 10.2255 19.0639 10.5024C19.3579 10.8273 19.5969 11.1953 19.7758 11.5959ZM14.0466 18.9363C14.4214 18.7815 14.7619 18.5528 15.049 18.2657C15.3362 17.9785 15.5648 17.6381 15.7196 17.2615C15.8743 16.8867 15.9552 16.4843 15.9552 16.0785V12.2442C15.954 12.2407 15.9529 12.2367 15.9517 12.2321C15.9506 12.2287 15.9488 12.2252 15.9466 12.2218C15.9443 12.2184 15.9414 12.2155 15.938 12.2132C15.9345 12.2098 15.9311 12.2075 15.9276 12.2063L14.54 11.4051V16.0373C14.54 16.0837 14.5332 16.1318 14.5211 16.1765C14.5091 16.223 14.4919 16.2659 14.4678 16.3072C14.4438 16.3485 14.4162 16.3863 14.3819 16.419C14.3484 16.4523 14.3109 16.4812 14.2701 16.505L10.9842 18.4015C10.9567 18.4187 10.9103 18.4428 10.8862 18.4565C11.0221 18.5717 11.1699 18.6732 11.3247 18.7626C11.4811 18.852 11.6428 18.9277 11.8113 18.9896C11.9798 19.0497 12.1535 19.0962 12.3288 19.1271C12.5059 19.1581 12.6848 19.1735 12.8636 19.1735C13.2694 19.1735 13.6717 19.0927 14.0466 18.9363ZM6.22135 16.333C6.42596 16.6855 6.69592 16.9916 7.01745 17.2392C7.34071 17.4868 7.70695 17.6673 8.09899 17.7722C8.49102 17.8771 8.90025 17.9046 9.3026 17.8513C9.70495 17.798 10.0918 17.6673 10.4443 17.4644L13.7663 15.5472L13.7749 15.5386C13.7772 15.5363 13.7789 15.5329 13.78 15.5283C13.7823 15.5249 13.7841 15.5214 13.7852 15.518V13.9017L9.77545 16.2212C9.73418 16.2453 9.6912 16.2625 9.64649 16.2763C9.60007 16.2883 9.55364 16.2935 9.5055 16.2935C9.45907 16.2935 9.41265 16.2883 9.36622 16.2763C9.32152 16.2625 9.27681 16.2453 9.23554 16.2212L5.94967 14.323C5.92044 14.3058 5.87746 14.28 5.85339 14.2645C5.82244 14.4416 5.80696 14.6204 5.80696 14.7993C5.80696 14.9781 5.82415 15.1569 5.85511 15.334C5.88605 15.5094 5.9342 15.6831 5.99438 15.8516C6.05628 16.0201 6.13194 16.1817 6.22135 16.3364V16.333ZM5.35818 9.1629C5.15529 9.51539 5.02461 9.90398 4.97131 10.3063C4.918 10.7087 4.94552 11.1162 5.0504 11.51C5.15529 11.902 5.33583 12.2682 5.58343 12.5915C5.83103 12.913 6.13881 13.183 6.48958 13.3859L9.80984 15.3048C9.81328 15.3059 9.81729 15.3071 9.82188 15.3082H9.83391C9.8385 15.3082 9.84251 15.3071 9.84595 15.3048C9.84939 15.3036 9.85283 15.3019 9.85627 15.2996L11.249 14.4949L7.23926 12.1805C7.19971 12.1565 7.16189 12.1272 7.1275 12.0946C7.09418 12.0611 7.06529 12.0236 7.04153 11.9828C7.01917 11.9415 7.00026 11.8985 6.98822 11.8521C6.97619 11.8074 6.96931 11.761 6.97103 11.7128V7.80797C6.80252 7.86987 6.63917 7.94553 6.48442 8.03494C6.32967 8.12607 6.18352 8.22924 6.04596 8.34444C5.91013 8.45965 5.78289 8.58688 5.66769 8.72444C5.55248 8.86028 5.45103 9.00815 5.36162 9.1629H5.35818ZM16.7633 11.8177C16.8046 11.8418 16.8424 11.8693 16.8768 11.9037C16.9094 11.9364 16.9387 11.9742 16.9628 12.0155C16.9851 12.0567 17.004 12.1014 17.0161 12.1461C17.0264 12.1926 17.0332 12.239 17.0315 12.2871V16.192C17.5835 15.9891 18.0649 15.6332 18.4208 15.1655C18.7785 14.6978 18.9934 14.139 19.0433 13.5544C19.0931 12.9698 18.9762 12.3817 18.7046 11.8607C18.4329 11.3397 18.0185 10.9064 17.5095 10.6141L14.1893 8.69521C14.1858 8.69406 14.1818 8.69292 14.1772 8.69177H14.1652C14.1618 8.69292 14.1578 8.69406 14.1532 8.69521C14.1497 8.69636 14.1463 8.69808 14.1429 8.70037L12.757 9.50163L16.7667 11.8177H16.7633ZM18.1475 9.7372H18.1457V9.73892L18.1475 9.7372ZM18.1457 9.73548C18.2455 9.15774 18.1784 8.56281 17.9514 8.02119C17.7262 7.47956 17.3496 7.01359 16.8682 6.67658C16.3867 6.34128 15.8193 6.1487 15.233 6.12291C14.6449 6.09884 14.0638 6.24155 13.5548 6.53386L10.2345 8.45105C10.2311 8.45334 10.2282 8.45621 10.2259 8.45965L10.2191 8.46996C10.2179 8.4734 10.2168 8.47741 10.2156 8.482C10.2145 8.48544 10.2139 8.48945 10.2139 8.49403V10.0966L14.2237 7.78046C14.2649 7.75639 14.3096 7.7392 14.3543 7.72544C14.4008 7.7134 14.4472 7.70825 14.4936 7.70825C14.5418 7.70825 14.5882 7.7134 14.6346 7.72544C14.6793 7.7392 14.7223 7.75639 14.7636 7.78046L18.0494 9.67874C18.0787 9.69593 18.1217 9.72 18.1457 9.73548ZM9.45735 7.96101C9.45735 7.91458 9.46423 7.86816 9.47627 7.82173C9.4883 7.77702 9.5055 7.73232 9.52957 7.69105C9.55364 7.6515 9.58115 7.61368 9.61554 7.57929C9.64821 7.54662 9.68604 7.51739 9.72731 7.49503L13.0132 5.59848C13.0441 5.57957 13.0871 5.55549 13.1112 5.54346C12.6607 5.1669 12.1105 4.92618 11.5276 4.85224C10.9447 4.77658 10.3532 4.86943 9.82188 5.11875C9.28885 5.36807 8.83835 5.76527 8.52369 6.26047C8.20903 6.75739 8.04224 7.33169 8.04224 7.91974V11.7541C8.04339 11.7587 8.04454 11.7627 8.04568 11.7661C8.04683 11.7696 8.04855 11.773 8.05084 11.7765C8.05313 11.7799 8.056 11.7833 8.05944 11.7868C8.06173 11.7891 8.06517 11.7914 8.06976 11.7937L9.45735 12.5949V7.96101ZM10.2105 13.0282L11.997 14.0599L13.7835 13.0282V10.9666L11.9987 9.93493L10.2122 10.9666L10.2105 13.0282Z", + "fill": "white" + }, + "children": [] + } + ] + }, + "name": "OpenaiViolet" +} diff --git a/web/app/components/base/icons/src/public/llm/OpenaiViolet.tsx b/web/app/components/base/icons/src/public/llm/OpenaiViolet.tsx new file mode 100644 index 0000000000..9aa08c0f3b --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/OpenaiViolet.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import type { IconData } from '@/app/components/base/icons/IconBase' +import * as React from 'react' +import IconBase from '@/app/components/base/icons/IconBase' +import data from './OpenaiViolet.json' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject> + }, +) => + +Icon.displayName = 'OpenaiViolet' + +export default Icon diff --git a/web/app/components/base/icons/src/public/llm/Tongyi.json b/web/app/components/base/icons/src/public/llm/Tongyi.json new file mode 100644 index 0000000000..9150ca226b --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/Tongyi.json @@ -0,0 +1,128 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "25", + "height": "25", + "viewBox": "0 0 25 25", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg", + "xmlns:xlink": "http://www.w3.org/1999/xlink" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "clip-path": "url(#clip0_6305_73327)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M0.5 12.5C0.5 8.77247 0.5 6.9087 1.10896 5.43853C1.92092 3.47831 3.47831 1.92092 5.43853 1.10896C6.9087 0.5 8.77247 0.5 12.5 0.5C16.2275 0.5 18.0913 0.5 19.5615 1.10896C21.5217 1.92092 23.0791 3.47831 23.891 5.43853C24.5 6.9087 24.5 8.77247 24.5 12.5C24.5 16.2275 24.5 18.0913 23.891 19.5615C23.0791 21.5217 21.5217 23.0791 19.5615 23.891C18.0913 24.5 16.2275 24.5 12.5 24.5C8.77247 24.5 6.9087 24.5 5.43853 23.891C3.47831 23.0791 1.92092 21.5217 1.10896 19.5615C0.5 18.0913 0.5 16.2275 0.5 12.5Z", + "fill": "white" + }, + "children": [] + }, + { + "type": "element", + "name": "rect", + "attributes": { + "width": "24", + "height": "24", + "transform": "translate(0.5 0.5)", + "fill": "url(#pattern0_6305_73327)" + }, + "children": [] + }, + { + "type": "element", + "name": "rect", + "attributes": { + "width": "24", + "height": "24", + "transform": "translate(0.5 0.5)", + "fill": "white", + "fill-opacity": "0.01" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M12.5 0.25C14.3603 0.25 15.7684 0.250313 16.8945 0.327148C18.0228 0.404144 18.8867 0.558755 19.6572 0.87793C21.6787 1.71525 23.2847 3.32133 24.1221 5.34277C24.4412 6.11333 24.5959 6.97723 24.6729 8.10547C24.7497 9.23161 24.75 10.6397 24.75 12.5C24.75 14.3603 24.7497 15.7684 24.6729 16.8945C24.5959 18.0228 24.4412 18.8867 24.1221 19.6572C23.2847 21.6787 21.6787 23.2847 19.6572 24.1221C18.8867 24.4412 18.0228 24.5959 16.8945 24.6729C15.7684 24.7497 14.3603 24.75 12.5 24.75C10.6397 24.75 9.23161 24.7497 8.10547 24.6729C6.97723 24.5959 6.11333 24.4412 5.34277 24.1221C3.32133 23.2847 1.71525 21.6787 0.87793 19.6572C0.558755 18.8867 0.404144 18.0228 0.327148 16.8945C0.250313 15.7684 0.25 14.3603 0.25 12.5C0.25 10.6397 0.250313 9.23161 0.327148 8.10547C0.404144 6.97723 0.558755 6.11333 0.87793 5.34277C1.71525 3.32133 3.32133 1.71525 5.34277 0.87793C6.11333 0.558755 6.97723 0.404144 8.10547 0.327148C9.23161 0.250313 10.6397 0.25 12.5 0.25Z", + "stroke": "#101828", + "stroke-opacity": "0.08", + "stroke-width": "0.5" + }, + "children": [] + }, + { + "type": "element", + "name": "defs", + "attributes": {}, + "children": [ + { + "type": "element", + "name": "pattern", + "attributes": { + "id": "pattern0_6305_73327", + "patternContentUnits": "objectBoundingBox", + "width": "1", + "height": "1" + }, + "children": [ + { + "type": "element", + "name": "use", + "attributes": { + "xlink:href": "#image0_6305_73327", + "transform": "scale(0.00625)" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "clipPath", + "attributes": { + "id": "clip0_6305_73327" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M0.5 12.5C0.5 8.77247 0.5 6.9087 1.10896 5.43853C1.92092 3.47831 3.47831 1.92092 5.43853 1.10896C6.9087 0.5 8.77247 0.5 12.5 0.5C16.2275 0.5 18.0913 0.5 19.5615 1.10896C21.5217 1.92092 23.0791 3.47831 23.891 5.43853C24.5 6.9087 24.5 8.77247 24.5 12.5C24.5 16.2275 24.5 18.0913 23.891 19.5615C23.0791 21.5217 21.5217 23.0791 19.5615 23.891C18.0913 24.5 16.2275 24.5 12.5 24.5C8.77247 24.5 6.9087 24.5 5.43853 23.891C3.47831 23.0791 1.92092 21.5217 1.10896 19.5615C0.5 18.0913 0.5 16.2275 0.5 12.5Z", + "fill": "white" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "image", + "attributes": { + "id": "image0_6305_73327", + "width": "160", + "height": "160", + "preserveAspectRatio": "none", + "xlink:href": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAKAAAACgCAYAAACLz2ctAAAACXBIWXMAACxLAAAsSwGlPZapAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAm6SURBVHgB7Z09jFVFGIZn/Wf9I5FSKIVSKYndErolVJJQQUKliRWFNhY2UGxFItUmUJFARZaOQGekA0poF8slUcFFRVnPu9cDw/HuPWdmvpnvu3PeJ9m4MQH23H3P/DzzzczCVoMjRInXHCGKMIBEFQaQqMIAElUYQKIKA0hUYQCJKgwgUYUBJKowgEQVBpCowgASVRhAogoDSFRhAIkqDCBRhQEkqjCARBUGkKjCABJVGECiCgNIVGEAiSpvuJGyubnlLq0+cQ/u/739fSyLiwvu8JF33PKxXY6EM9oW8Mrl393dO8+Swgfw59euPXU//finI+GMMoCPNp43gflr+3u0YBJcubzZhPG5I2GMMoA/nH/84vvjJxbd/gPpIxG0hDdvsBUMZXQBRFf5cP2f7e8Pff729tfysUUnwa0bf7iNDbaCIYwugNeb8VpLO3FAC4ggpoJW8GoztiTDGVUAEb62hULg9ux5+fjoiiXGg5jYYGZNhjGaAGLicbPpIsFHTfC62gThW2p0igTXr206MozRBHDt2uYL5XK0CZ/f+rXA5037/6GgBaSWGcYoAuhrF7R+O4330Ap+cUJmQkItM4xRBNDXLkd7Viw+O/gWtUxBqg/gNO3SB7VMOaoP4DTt0gdaQIkJCbVMP1UXI8zSLn2gq77dtJ6pa8XQMheaIcCuxfLvOiZVe/e97ixTbQD7tEsfrZbxW9BYEEINMPw4880HImPaXFTbBQ/RLn1IaRlNLq4+cZapMoBDtUsfUpUymuCzWBNoxXNRZQBDtMss/DHkPIPZuFUnWV0AY7TLNCataB0eD0OR60ZbweoCGKNdpoExZE0OD1L84bq9IomqAuh3mUsJEwh/DFkTWB60RjUB7GqXwwki2R9D1gSKJKyVilUTQAntAvwxZI1Y0zJVBFBKuwCrg3UprGmZKgLY3WQUSy3apQ9LWmbuA9jVLiinisEfQ9aOJS1jYpEQA+NHG3HjLkntklp4ME9Ay+CF3btPNwLqAYQakGh5qF3CwWePYgVNVLtgdJ9S3d6Ci2+9atUufVjQMqotoN99Yuz26cE3XQzrzRgQQV46Eq5f0O0eFtoNNy/cu/PM3b0zafG1JyNqAeyWqz+4/8ydPI29ueGN8qHm6+dmmQmnXYV2Kah4kdiUPk/4L772GFClC54240zdxIN9HBZNvzWkliulUPnXd1roT9nEg6pffFkvwNREcrlSiuIBnDXjTN3Ec+r0e+5p83fcGonPC0VquVKS4j9BX0VGytkqeKvRrWCpiZvCX0VyuVKSogGEdmlnX7NIOVul7VZqX9MNRWq5UpqiARwaipSzVTCrxYoIJjTcFD5BarkyB8UCGDrBSDlbBa0gJiSXOCHZRmq5MgdFAhiz0E8tI4M17dKlyE8Tu7+CWiYNi9qlS/YApiz0U8ukYVG7dMn+E6W2QNQycVjVLl2yLwRKjMGgZfZHlg2h20ELiEnN/gNxxQ7ziD/mtqRdumQPIE5nSt3k02qZmLe41TII4Bhr/sC9xr1aUi8+C1sNLiMIzsXV9DPyEKSzKx9GVcuAlXO/zc2MGF3muZXdLhX/ma2ekpV9DIhWy8KRt1KnnpZAqsv0n9nqyf1FpkUWjrxttYx1JFcq/Ge2enJ/kQBauYkIWsb6kWvSKxX+M1u0AcXEEDyU9k1ErZaxSo6VCv+ZJ2LaVitYLICSv/zUahmLrWDOlQr/ma2d3F9UjVu4iajVMtbIuVLhP/NkU7qdCUnRAEr+8iWqZaxQYqXCf2b4UCtKqvjiILXM/ym1UmFRy6isTlPLvKRkgahFLaMSQGqZl5Qej1rTMmr1OdQyOgWi1rSMWgDHrmVStUtKmZslLaNaoThmLZN6jDBmshLPrK1lVAM4Vi0jdYxwyhjOipZRr9Eeo5aROkY4dQxnQcuY2CQwJi2Teoxw94BxqWfW0jImAjgmLZN6jHCX1DGctpYxs01qDFpmOWHiMWt3YcoYTlvLmAlg7VomdeKB8vpZSD1zaS1jaqOoFS2TY202Vbv0hUJKRZXWMqYCaEXLSM3MW0rd3jSPWsbcVvkatQwG+rGE3N40j1rG3lkNzo6WkahSSXmhYu51mzctYzKAVrQMxoKpExJp7dKHpJYpcXWZ2X2KuDNE4g1stUxMK9TOzGNPW82lXfrAn3u6+djtitzEjwAiyCWurTUbwKuCt3tLnC0Teo9cbu3SB168VGIvDgrBZBc8RDuEoKFlcmuXEhw/8a7LjbkAouvJccB4SS1Tw6XZ+PlLFMuaC2Cut7+klimlXXKBF6hUjaSpAOa+Tr6ElimtXXIgtSI1BFMBXMssP0tomdLaRZrSZ0mbCeBkopD/AMmc1TJa2kWSo4W3J5gJYMk7PXJUy2hrFwmgXUqfJW0igKW1A1rA2JPzd9Iy1C5xqAcwl3bpI6VypDvRoHaJRz2AWm+/pJahdolHNYDa2iFVy6A7pnZJQ3UteH1dpugRV0Hs3Rf3KLjCIEY7oOVGK0rtkoZqAGOvXHj171hwX379ftE3uB23Uruko9oFS+zF1TjgBy0XamPmXbvgs9O+wku9HAtT/++/+9XFgO6j9BvctlynTr+rrl1Snh9DFgxdtFEPID6Epf9q7kLR6D7Q+qVol0nFsszEA89v9RLCoZgQ0TGb0j9uglv6w29PpUrRLlL7bjWePwcmAojwhW5K/6qZeJQGLZcV7aLx/DkwdTTH0DGVhrVvx20WtIvWqkUOTD3FyQFdm8aBkhLaBRt8JLSL1XtOYjEVwCFaZl61y4Xzj50ES4qrFjkw9ySzKjI0tYuFaheN58+NuQC2WmYa1C51hQ+YbMunaRkN7YDqaWqXvJgM4DQto6EdsH+E2iUvZk9GQCt42xs7fXvmF6fB3oQja6ld+jH9VCcTuj4pYjcxUbsMY2GrwRkGv8gSpzR1weQBtYIAXfCZwLNl0GJLjP0QvhonHy3mA6jJhfNPmhZwEkJUvwydBEC7XFyN33/cgtn3uZXdrmbqHFgI4W9EH3q2DLVLGAzgDPyN6EM3MVG7hMEA9hByhQG1SzgMYA/+RvS+s2WoXcJhAAfgy+idtAy1SxwM4ED6rjBgtUscDOBA/PMBu2fLsNolHgYwAF/LtGfLULukwQAGME3LULukwZWQQBA8LLPhv+19GhKcbVY8xjT2a2ELGEhXy0gwJu3ShQGMAIFZFrpg+5NmcpPjeth5gV0wUYUtIFGFASSqMIBEFQaQqMIAElUYQKIKA0hUYQCJKgwgUYUBJKowgEQVBpCowgASVRhAogoDSFRhAIkqDCBRhQEkqjCARBUGkKjCABJVGECiCgNIVPkXGPWKHZj1nMYAAAAASUVORK5CYII=" + }, + "children": [] + } + ] + } + ] + }, + "name": "Tongyi" +} diff --git a/web/app/components/base/icons/src/public/llm/Tongyi.tsx b/web/app/components/base/icons/src/public/llm/Tongyi.tsx new file mode 100644 index 0000000000..9934dee856 --- /dev/null +++ b/web/app/components/base/icons/src/public/llm/Tongyi.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import type { IconData } from '@/app/components/base/icons/IconBase' +import * as React from 'react' +import IconBase from '@/app/components/base/icons/IconBase' +import data from './Tongyi.json' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject> + }, +) => + +Icon.displayName = 'Tongyi' + +export default Icon diff --git a/web/app/components/base/icons/src/public/llm/index.ts b/web/app/components/base/icons/src/public/llm/index.ts index 3a4306391e..0c5cef4a36 100644 --- a/web/app/components/base/icons/src/public/llm/index.ts +++ b/web/app/components/base/icons/src/public/llm/index.ts @@ -1,6 +1,7 @@ export { default as Anthropic } from './Anthropic' export { default as AnthropicDark } from './AnthropicDark' export { default as AnthropicLight } from './AnthropicLight' +export { default as AnthropicShortLight } from './AnthropicShortLight' export { default as AnthropicText } from './AnthropicText' export { default as Azureai } from './Azureai' export { default as AzureaiText } from './AzureaiText' @@ -12,8 +13,11 @@ export { default as Chatglm } from './Chatglm' export { default as ChatglmText } from './ChatglmText' export { default as Cohere } from './Cohere' export { default as CohereText } from './CohereText' +export { default as Deepseek } from './Deepseek' +export { default as Gemini } from './Gemini' export { default as Gpt3 } from './Gpt3' export { default as Gpt4 } from './Gpt4' +export { default as Grok } from './Grok' export { default as Huggingface } from './Huggingface' export { default as HuggingfaceText } from './HuggingfaceText' export { default as HuggingfaceTextHub } from './HuggingfaceTextHub' @@ -26,14 +30,19 @@ export { default as Localai } from './Localai' export { default as LocalaiText } from './LocalaiText' export { default as Microsoft } from './Microsoft' export { default as OpenaiBlack } from './OpenaiBlack' +export { default as OpenaiBlue } from './OpenaiBlue' export { default as OpenaiGreen } from './OpenaiGreen' +export { default as OpenaiSmall } from './OpenaiSmall' +export { default as OpenaiTeal } from './OpenaiTeal' export { default as OpenaiText } from './OpenaiText' export { default as OpenaiTransparent } from './OpenaiTransparent' +export { default as OpenaiViolet } from './OpenaiViolet' export { default as OpenaiYellow } from './OpenaiYellow' export { default as Openllm } from './Openllm' export { default as OpenllmText } from './OpenllmText' export { default as Replicate } from './Replicate' export { default as ReplicateText } from './ReplicateText' +export { default as Tongyi } from './Tongyi' export { default as XorbitsInference } from './XorbitsInference' export { default as XorbitsInferenceText } from './XorbitsInferenceText' export { default as Zhipuai } from './Zhipuai' diff --git a/web/app/components/base/icons/src/public/tracing/DatabricksIcon.tsx b/web/app/components/base/icons/src/public/tracing/DatabricksIcon.tsx index 87abe453ec..a1e45d8bdf 100644 --- a/web/app/components/base/icons/src/public/tracing/DatabricksIcon.tsx +++ b/web/app/components/base/icons/src/public/tracing/DatabricksIcon.tsx @@ -11,7 +11,7 @@ const Icon = ( ref, ...props }: React.SVGProps & { - ref?: React.RefObject> + ref?: React.RefObject> }, ) => diff --git a/web/app/components/base/icons/src/public/tracing/DatabricksIconBig.tsx b/web/app/components/base/icons/src/public/tracing/DatabricksIconBig.tsx index bebaa1b40e..ef21c05a23 100644 --- a/web/app/components/base/icons/src/public/tracing/DatabricksIconBig.tsx +++ b/web/app/components/base/icons/src/public/tracing/DatabricksIconBig.tsx @@ -11,7 +11,7 @@ const Icon = ( ref, ...props }: React.SVGProps & { - ref?: React.RefObject> + ref?: React.RefObject> }, ) => diff --git a/web/app/components/base/icons/src/public/tracing/MlflowIcon.tsx b/web/app/components/base/icons/src/public/tracing/MlflowIcon.tsx index 3c86ed61f4..09a31882c9 100644 --- a/web/app/components/base/icons/src/public/tracing/MlflowIcon.tsx +++ b/web/app/components/base/icons/src/public/tracing/MlflowIcon.tsx @@ -11,7 +11,7 @@ const Icon = ( ref, ...props }: React.SVGProps & { - ref?: React.RefObject> + ref?: React.RefObject> }, ) => diff --git a/web/app/components/base/icons/src/public/tracing/MlflowIconBig.tsx b/web/app/components/base/icons/src/public/tracing/MlflowIconBig.tsx index fbb288d46a..03fef44991 100644 --- a/web/app/components/base/icons/src/public/tracing/MlflowIconBig.tsx +++ b/web/app/components/base/icons/src/public/tracing/MlflowIconBig.tsx @@ -11,7 +11,7 @@ const Icon = ( ref, ...props }: React.SVGProps & { - ref?: React.RefObject> + ref?: React.RefObject> }, ) => diff --git a/web/app/components/base/icons/src/public/tracing/TencentIcon.json b/web/app/components/base/icons/src/public/tracing/TencentIcon.json index 642fa75a92..9fd54c0ce9 100644 --- a/web/app/components/base/icons/src/public/tracing/TencentIcon.json +++ b/web/app/components/base/icons/src/public/tracing/TencentIcon.json @@ -1,14 +1,16 @@ { "icon": { "type": "element", + "isRootNode": true, "name": "svg", "attributes": { "width": "80px", "height": "18px", "viewBox": "0 0 80 18", - "version": "1.1" + "version": "1.1", + "xmlns": "http://www.w3.org/2000/svg", + "xmlns:xlink": "http://www.w3.org/1999/xlink" }, - "isRootNode": true, "children": [ { "type": "element", diff --git a/web/app/components/base/icons/src/public/tracing/TencentIconBig.json b/web/app/components/base/icons/src/public/tracing/TencentIconBig.json index d0582e7f8d..9abd81455f 100644 --- a/web/app/components/base/icons/src/public/tracing/TencentIconBig.json +++ b/web/app/components/base/icons/src/public/tracing/TencentIconBig.json @@ -1,14 +1,16 @@ { "icon": { "type": "element", + "isRootNode": true, "name": "svg", "attributes": { - "width": "80px", - "height": "18px", + "width": "120px", + "height": "27px", "viewBox": "0 0 80 18", - "version": "1.1" + "version": "1.1", + "xmlns": "http://www.w3.org/2000/svg", + "xmlns:xlink": "http://www.w3.org/1999/xlink" }, - "isRootNode": true, "children": [ { "type": "element", diff --git a/web/app/components/billing/apps-full-in-dialog/index.spec.tsx b/web/app/components/billing/apps-full-in-dialog/index.spec.tsx index a11b582b0f..d006a3222d 100644 --- a/web/app/components/billing/apps-full-in-dialog/index.spec.tsx +++ b/web/app/components/billing/apps-full-in-dialog/index.spec.tsx @@ -75,6 +75,9 @@ const buildAppContext = (overrides: Partial = {}): AppContextVa created_at: 0, role: 'normal', providers: [], + trial_credits: 200, + trial_credits_used: 0, + next_credit_reset_date: 0, } const langGeniusVersionInfo: LangGeniusVersionResponse = { current_env: '', @@ -96,6 +99,7 @@ const buildAppContext = (overrides: Partial = {}): AppContextVa mutateCurrentWorkspace: vi.fn(), langGeniusVersionInfo, isLoadingCurrentWorkspace: false, + isValidatingCurrentWorkspace: false, } const useSelector: AppContextValue['useSelector'] = selector => selector({ ...base, useSelector }) return { diff --git a/web/app/components/header/account-setting/model-provider-page/index.tsx b/web/app/components/header/account-setting/model-provider-page/index.tsx index d3daaee859..7606bbc04f 100644 --- a/web/app/components/header/account-setting/model-provider-page/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/index.tsx @@ -6,8 +6,10 @@ import { RiBrainLine, } from '@remixicon/react' import { useDebounce } from 'ahooks' -import { useMemo } from 'react' +import { useEffect, useMemo } from 'react' import { useTranslation } from 'react-i18next' +import { IS_CLOUD_EDITION } from '@/config' +import { useAppContext } from '@/context/app-context' import { useGlobalPublicStore } from '@/context/global-public-context' import { useProviderContext } from '@/context/provider-context' import { cn } from '@/utils/classnames' @@ -20,6 +22,7 @@ import { } from './hooks' import InstallFromMarketplace from './install-from-marketplace' import ProviderAddedCard from './provider-added-card' +import QuotaPanel from './provider-added-card/quota-panel' import SystemModelSelector from './system-model-selector' type Props = { @@ -31,6 +34,7 @@ const FixedModelProvider = ['langgenius/openai/openai', 'langgenius/anthropic/an const ModelProviderPage = ({ searchText }: Props) => { const debouncedSearchText = useDebounce(searchText, { wait: 500 }) const { t } = useTranslation() + const { mutateCurrentWorkspace, isValidatingCurrentWorkspace } = useAppContext() const { data: textGenerationDefaultModel, isLoading: isTextGenerationDefaultModelLoading } = useDefaultModel(ModelTypeEnum.textGeneration) const { data: embeddingsDefaultModel, isLoading: isEmbeddingsDefaultModelLoading } = useDefaultModel(ModelTypeEnum.textEmbedding) const { data: rerankDefaultModel, isLoading: isRerankDefaultModelLoading } = useDefaultModel(ModelTypeEnum.rerank) @@ -88,6 +92,10 @@ const ModelProviderPage = ({ searchText }: Props) => { return [filteredConfiguredProviders, filteredNotConfiguredProviders] }, [configuredProviders, debouncedSearchText, notConfiguredProviders]) + useEffect(() => { + mutateCurrentWorkspace() + }, [mutateCurrentWorkspace]) + return (
@@ -115,6 +123,7 @@ const ModelProviderPage = ({ searchText }: Props) => { />
+ {IS_CLOUD_EDITION && } {!filteredConfiguredProviders?.length && (
diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/credential-panel.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/credential-panel.tsx index 59d7b2c0c8..c46f9d56bd 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/credential-panel.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/credential-panel.tsx @@ -7,6 +7,7 @@ import { useToastContext } from '@/app/components/base/toast' import { ConfigProvider } from '@/app/components/header/account-setting/model-provider-page/model-auth' import { useCredentialStatus } from '@/app/components/header/account-setting/model-provider-page/model-auth/hooks' import Indicator from '@/app/components/header/indicator' +import { IS_CLOUD_EDITION } from '@/config' import { useEventEmitterContextContext } from '@/context/event-emitter' import { changeModelProviderPriority } from '@/service/common' import { cn } from '@/utils/classnames' @@ -26,6 +27,7 @@ import PriorityUseTip from './priority-use-tip' type CredentialPanelProps = { provider: ModelProvider } + const CredentialPanel = ({ provider, }: CredentialPanelProps) => { @@ -47,6 +49,8 @@ const CredentialPanel = ({ notAllowedToUse, } = useCredentialStatus(provider) + const showPrioritySelector = systemConfig.enabled && isCustomConfigured && IS_CLOUD_EDITION + const handleChangePriority = async (key: PreferredProviderTypeEnum) => { const res = await changeModelProviderPriority({ url: `/workspaces/current/model-providers/${provider.provider}/preferred-provider-type`, @@ -114,7 +118,7 @@ const CredentialPanel = ({ provider={provider} /> { - systemConfig.enabled && isCustomConfigured && ( + showPrioritySelector && ( = ({ const systemConfig = provider.system_configuration const hasModelList = fetched && !!modelList.length const { isCurrentWorkspaceManager } = useAppContext() - const showQuota = systemConfig.enabled && [...MODEL_PROVIDER_QUOTA_GET_PAID].includes(provider.provider) && !IS_CE_EDITION + const showModelProvider = systemConfig.enabled && MODEL_PROVIDER_QUOTA_GET_PAID.includes(provider.provider as ModelProviderQuotaGetPaid) && !IS_CE_EDITION const showCredential = configurationMethods.includes(ConfigurationMethodEnum.predefinedModel) && isCurrentWorkspaceManager const getModelList = async (providerName: string) => { @@ -104,13 +104,6 @@ const ProviderAddedCard: FC = ({ }
- { - showQuota && ( - - ) - } { showCredential && ( = ({ { collapsed && (
- {(showQuota || !notConfigured) && ( + {(showModelProvider || !notConfigured) && ( <>
{ @@ -150,7 +143,7 @@ const ProviderAddedCard: FC = ({
)} - {!showQuota && notConfigured && ( + {!showModelProvider && notConfigured && (
{t('modelProvider.configureTip', { ns: 'common' })} diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/quota-panel.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/quota-panel.tsx index cd49148403..e296bc4555 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/quota-panel.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/quota-panel.tsx @@ -1,66 +1,163 @@ import type { FC } from 'react' import type { ModelProvider } from '../declarations' +import type { Plugin } from '@/app/components/plugins/types' +import { useBoolean } from 'ahooks' +import * as React from 'react' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' +import { AnthropicShortLight, Deepseek, Gemini, Grok, OpenaiSmall, Tongyi } from '@/app/components/base/icons/src/public/llm' +import Loading from '@/app/components/base/loading' import Tooltip from '@/app/components/base/tooltip' +import InstallFromMarketplace from '@/app/components/plugins/install-plugin/install-from-marketplace' +import { useAppContext } from '@/context/app-context' +import useTimestamp from '@/hooks/use-timestamp' +import { cn } from '@/utils/classnames' import { formatNumber } from '@/utils/format' -import { - CustomConfigurationStatusEnum, - PreferredProviderTypeEnum, - QuotaUnitEnum, -} from '../declarations' -import { - MODEL_PROVIDER_QUOTA_GET_PAID, -} from '../utils' -import PriorityUseTip from './priority-use-tip' +import { PreferredProviderTypeEnum } from '../declarations' +import { useMarketplaceAllPlugins } from '../hooks' +import { modelNameMap, ModelProviderQuotaGetPaid } from '../utils' + +const allProviders = [ + { key: ModelProviderQuotaGetPaid.OPENAI, Icon: OpenaiSmall }, + { key: ModelProviderQuotaGetPaid.ANTHROPIC, Icon: AnthropicShortLight }, + { key: ModelProviderQuotaGetPaid.GEMINI, Icon: Gemini }, + { key: ModelProviderQuotaGetPaid.X, Icon: Grok }, + { key: ModelProviderQuotaGetPaid.DEEPSEEK, Icon: Deepseek }, + { key: ModelProviderQuotaGetPaid.TONGYI, Icon: Tongyi }, +] as const + +// Map provider key to plugin ID +// provider key format: langgenius/provider/model, plugin ID format: langgenius/provider +const providerKeyToPluginId: Record = { + [ModelProviderQuotaGetPaid.OPENAI]: 'langgenius/openai', + [ModelProviderQuotaGetPaid.ANTHROPIC]: 'langgenius/anthropic', + [ModelProviderQuotaGetPaid.GEMINI]: 'langgenius/gemini', + [ModelProviderQuotaGetPaid.X]: 'langgenius/x', + [ModelProviderQuotaGetPaid.DEEPSEEK]: 'langgenius/deepseek', + [ModelProviderQuotaGetPaid.TONGYI]: 'langgenius/tongyi', +} type QuotaPanelProps = { - provider: ModelProvider + providers: ModelProvider[] + isLoading?: boolean } const QuotaPanel: FC = ({ - provider, + providers, + isLoading = false, }) => { const { t } = useTranslation() + const { currentWorkspace } = useAppContext() + const credits = Math.max((currentWorkspace.trial_credits - currentWorkspace.trial_credits_used) || 0, 0) + const providerMap = useMemo(() => new Map( + providers.map(p => [p.provider, p.preferred_provider_type]), + ), [providers]) + const { formatTime } = useTimestamp() + const { + plugins: allPlugins, + } = useMarketplaceAllPlugins(providers, '') + const [selectedPlugin, setSelectedPlugin] = useState(null) + const [isShowInstallModal, { + setTrue: showInstallFromMarketplace, + setFalse: hideInstallFromMarketplace, + }] = useBoolean(false) + const selectedPluginIdRef = useRef(null) - const customConfig = provider.custom_configuration - const priorityUseType = provider.preferred_provider_type - const systemConfig = provider.system_configuration - const currentQuota = systemConfig.enabled && systemConfig.quota_configurations.find(item => item.quota_type === systemConfig.current_quota_type) - const openaiOrAnthropic = MODEL_PROVIDER_QUOTA_GET_PAID.includes(provider.provider) + const handleIconClick = useCallback((key: string) => { + const providerType = providerMap.get(key) + if (!providerType && allPlugins) { + const pluginId = providerKeyToPluginId[key] + const plugin = allPlugins.find(p => p.plugin_id === pluginId) + if (plugin) { + setSelectedPlugin(plugin) + selectedPluginIdRef.current = pluginId + showInstallFromMarketplace() + } + } + }, [allPlugins, providerMap, showInstallFromMarketplace]) + + useEffect(() => { + if (isShowInstallModal && selectedPluginIdRef.current) { + const isInstalled = providers.some(p => p.provider.startsWith(selectedPluginIdRef.current!)) + if (isInstalled) { + hideInstallFromMarketplace() + selectedPluginIdRef.current = null + } + } + }, [providers, isShowInstallModal, hideInstallFromMarketplace]) + + if (isLoading) { + return ( +
+ +
+ ) + } return ( -
+
{t('modelProvider.quota', { ns: 'common' })} - +
- { - currentQuota && ( -
- {formatNumber(Math.max((currentQuota?.quota_limit || 0) - (currentQuota?.quota_used || 0), 0))} - { - currentQuota?.quota_unit === QuotaUnitEnum.tokens && 'Tokens' +
+
+ {formatNumber(credits)} + {t('modelProvider.credits', { ns: 'common' })} + {currentWorkspace.next_credit_reset_date + ? ( + <> + · + + {t('modelProvider.resetDate', { + ns: 'common', + date: formatTime(currentWorkspace.next_credit_reset_date, t('dateFormat', { ns: 'appLog' })), + interpolation: { escapeValue: false }, + })} + + + ) + : null} +
+
+ {allProviders.map(({ key, Icon }) => { + const providerType = providerMap.get(key) + const usingQuota = providerType === PreferredProviderTypeEnum.system + const getTooltipKey = () => { + if (usingQuota) + return 'modelProvider.card.modelSupported' + if (providerType === PreferredProviderTypeEnum.custom) + return 'modelProvider.card.modelAPI' + return 'modelProvider.card.modelNotSupported' } - { - currentQuota?.quota_unit === QuotaUnitEnum.times && t('modelProvider.callTimes', { ns: 'common' }) - } - { - currentQuota?.quota_unit === QuotaUnitEnum.credits && t('modelProvider.credits', { ns: 'common' }) - } -
- ) - } - { - priorityUseType === PreferredProviderTypeEnum.system && customConfig.status === CustomConfigurationStatusEnum.active && ( - - ) - } + return ( + +
handleIconClick(key)} + > + + {!usingQuota && ( +
+ )} +
+ + ) + })} +
+
+ {isShowInstallModal && selectedPlugin && ( + + )}
) } -export default QuotaPanel +export default React.memo(QuotaPanel) diff --git a/web/app/components/header/account-setting/model-provider-page/utils.ts b/web/app/components/header/account-setting/model-provider-page/utils.ts index b60d6a0c7b..7cfa7fc654 100644 --- a/web/app/components/header/account-setting/model-provider-page/utils.ts +++ b/web/app/components/header/account-setting/model-provider-page/utils.ts @@ -17,7 +17,25 @@ import { ModelTypeEnum, } from './declarations' -export const MODEL_PROVIDER_QUOTA_GET_PAID = ['langgenius/anthropic/anthropic', 'langgenius/openai/openai', 'langgenius/azure_openai/azure_openai'] +export enum ModelProviderQuotaGetPaid { + ANTHROPIC = 'langgenius/anthropic/anthropic', + OPENAI = 'langgenius/openai/openai', + // AZURE_OPENAI = 'langgenius/azure_openai/azure_openai', + GEMINI = 'langgenius/gemini/google', + X = 'langgenius/x/x', + DEEPSEEK = 'langgenius/deepseek/deepseek', + TONGYI = 'langgenius/tongyi/tongyi', +} +export const MODEL_PROVIDER_QUOTA_GET_PAID = [ModelProviderQuotaGetPaid.ANTHROPIC, ModelProviderQuotaGetPaid.OPENAI, ModelProviderQuotaGetPaid.GEMINI, ModelProviderQuotaGetPaid.X, ModelProviderQuotaGetPaid.DEEPSEEK, ModelProviderQuotaGetPaid.TONGYI] + +export const modelNameMap = { + [ModelProviderQuotaGetPaid.OPENAI]: 'OpenAI', + [ModelProviderQuotaGetPaid.ANTHROPIC]: 'Anthropic', + [ModelProviderQuotaGetPaid.GEMINI]: 'Gemini', + [ModelProviderQuotaGetPaid.X]: 'xAI', + [ModelProviderQuotaGetPaid.DEEPSEEK]: 'DeepSeek', + [ModelProviderQuotaGetPaid.TONGYI]: 'Tongyi', +} export const isNullOrUndefined = (value: any) => { return value === undefined || value === null diff --git a/web/app/components/plugins/provider-card.tsx b/web/app/components/plugins/provider-card.tsx index a3bba8d774..d76e222c4a 100644 --- a/web/app/components/plugins/provider-card.tsx +++ b/web/app/components/plugins/provider-card.tsx @@ -92,7 +92,7 @@ const ProviderCardComponent: FC = ({ manifest={payload} uniqueIdentifier={payload.latest_package_identifier} onClose={hideInstallFromMarketplace} - onSuccess={() => hideInstallFromMarketplace()} + onSuccess={hideInstallFromMarketplace} /> ) } diff --git a/web/context/app-context.tsx b/web/context/app-context.tsx index 335f96fcce..12000044d6 100644 --- a/web/context/app-context.tsx +++ b/web/context/app-context.tsx @@ -29,6 +29,7 @@ export type AppContextValue = { langGeniusVersionInfo: LangGeniusVersionResponse useSelector: typeof useSelector isLoadingCurrentWorkspace: boolean + isValidatingCurrentWorkspace: boolean } const userProfilePlaceholder = { @@ -58,6 +59,9 @@ const initialWorkspaceInfo: ICurrentWorkspace = { created_at: 0, role: 'normal', providers: [], + trial_credits: 200, + trial_credits_used: 0, + next_credit_reset_date: 0, } const AppContext = createContext({ @@ -72,6 +76,7 @@ const AppContext = createContext({ langGeniusVersionInfo: initialLangGeniusVersionInfo, useSelector, isLoadingCurrentWorkspace: false, + isValidatingCurrentWorkspace: false, }) export function useSelector(selector: (value: AppContextValue) => T): T { @@ -86,7 +91,7 @@ export const AppContextProvider: FC = ({ children }) => const queryClient = useQueryClient() const systemFeatures = useGlobalPublicStore(s => s.systemFeatures) const { data: userProfileResp } = useUserProfile() - const { data: currentWorkspaceResp, isPending: isLoadingCurrentWorkspace } = useCurrentWorkspace() + const { data: currentWorkspaceResp, isPending: isLoadingCurrentWorkspace, isFetching: isValidatingCurrentWorkspace } = useCurrentWorkspace() const langGeniusVersionQuery = useLangGeniusVersion( userProfileResp?.meta.currentVersion, !systemFeatures.branding.enabled, @@ -195,6 +200,7 @@ export const AppContextProvider: FC = ({ children }) => isCurrentWorkspaceDatasetOperator, mutateCurrentWorkspace, isLoadingCurrentWorkspace, + isValidatingCurrentWorkspace, }} >
diff --git a/web/i18n/en-US/billing.json b/web/i18n/en-US/billing.json index 1f10a49966..3242aa8e78 100644 --- a/web/i18n/en-US/billing.json +++ b/web/i18n/en-US/billing.json @@ -96,7 +96,7 @@ "plansCommon.memberAfter": "Member", "plansCommon.messageRequest.title": "{{count,number}} message credits", "plansCommon.messageRequest.titlePerMonth": "{{count,number}} message credits/month", - "plansCommon.messageRequest.tooltip": "Message credits are provided to help you easily try out different OpenAI models in Dify. Credits are consumed based on the model type. Once they’re used up, you can switch to your own OpenAI API key.", + "plansCommon.messageRequest.tooltip": "Message credits are provided to help you easily try out different models from OpenAI, Anthropic, Gemini, xAI, DeepSeek and Tongyi in Dify. Credits are consumed based on the model type. Once they're used up, you can switch to your own API key.", "plansCommon.modelProviders": "Support OpenAI/Anthropic/Llama2/Azure OpenAI/Hugging Face/Replicate", "plansCommon.month": "month", "plansCommon.mostPopular": "Popular", diff --git a/web/i18n/en-US/common.json b/web/i18n/en-US/common.json index f971ff1668..64ac47d804 100644 --- a/web/i18n/en-US/common.json +++ b/web/i18n/en-US/common.json @@ -339,13 +339,16 @@ "modelProvider.callTimes": "Call times", "modelProvider.card.buyQuota": "Buy Quota", "modelProvider.card.callTimes": "Call times", + "modelProvider.card.modelAPI": "{{modelName}} models are using the API Key.", + "modelProvider.card.modelNotSupported": "{{modelName}} models are not installed.", + "modelProvider.card.modelSupported": "{{modelName}} models are using this quota.", "modelProvider.card.onTrial": "On Trial", "modelProvider.card.paid": "Paid", "modelProvider.card.priorityUse": "Priority use", "modelProvider.card.quota": "QUOTA", "modelProvider.card.quotaExhausted": "Quota exhausted", "modelProvider.card.removeKey": "Remove API Key", - "modelProvider.card.tip": "Priority will be given to the paid quota. The Trial quota will be used after the paid quota is exhausted.", + "modelProvider.card.tip": "Message Credits supports models from OpenAI, Anthropic, Gemini, xAI, DeepSeek and Tongyi. Priority will be given to the paid quota. The free quota will be used after the paid quota is exhausted.", "modelProvider.card.tokens": "Tokens", "modelProvider.collapse": "Collapse", "modelProvider.config": "Config", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Remaining available free tokens", "modelProvider.rerankModel.key": "Rerank Model", "modelProvider.rerankModel.tip": "Rerank model will reorder the candidate document list based on the semantic match with user query, improving the results of semantic ranking", + "modelProvider.resetDate": "Reset on {{date}}", "modelProvider.searchModel": "Search model", "modelProvider.selectModel": "Select your model", "modelProvider.selector.emptySetting": "Please go to settings to configure", diff --git a/web/i18n/ja-JP/billing.json b/web/i18n/ja-JP/billing.json index 344e934948..b23ae6c959 100644 --- a/web/i18n/ja-JP/billing.json +++ b/web/i18n/ja-JP/billing.json @@ -96,7 +96,7 @@ "plansCommon.memberAfter": "メンバー", "plansCommon.messageRequest.title": "{{count,number}}メッセージクレジット", "plansCommon.messageRequest.titlePerMonth": "{{count,number}}メッセージクレジット/月", - "plansCommon.messageRequest.tooltip": "メッセージクレジットは、Dify でさまざまな OpenAI モデルを簡単にお試しいただくためのものです。モデルタイプに応じてクレジットが消費され、使い切った後はご自身の OpenAI API キーに切り替えていただけます。", + "plansCommon.messageRequest.tooltip": "メッセージクレジットは、DifyでOpenAI、Anthropic、Gemini、xAI、DeepSeek、Tongyiなどのさまざまなモデルを簡単に試すために提供されています。クレジットはモデルの種類に基づいて消費されます。使い切ったら、独自のAPIキーに切り替えることができます。", "plansCommon.modelProviders": "OpenAI/Anthropic/Llama2/Azure OpenAI/Hugging Face/Replicateをサポート", "plansCommon.month": "月", "plansCommon.mostPopular": "人気", diff --git a/web/i18n/ja-JP/common.json b/web/i18n/ja-JP/common.json index e7481830d8..11f543e7e5 100644 --- a/web/i18n/ja-JP/common.json +++ b/web/i18n/ja-JP/common.json @@ -339,13 +339,16 @@ "modelProvider.callTimes": "呼び出し回数", "modelProvider.card.buyQuota": "クォータを購入", "modelProvider.card.callTimes": "通話回数", + "modelProvider.card.modelAPI": "{{modelName}} は現在 APIキーを使用しています。", + "modelProvider.card.modelNotSupported": "{{modelName}} 未インストール。", + "modelProvider.card.modelSupported": "このクォータは現在{{modelName}}に使用されています。", "modelProvider.card.onTrial": "トライアル中", "modelProvider.card.paid": "有料", "modelProvider.card.priorityUse": "優先利用", "modelProvider.card.quota": "クォータ", "modelProvider.card.quotaExhausted": "クォータが使い果たされました", "modelProvider.card.removeKey": "API キーを削除", - "modelProvider.card.tip": "有料クォータは優先して使用されます。有料クォータを使用し終えた後、トライアルクォータが利用されます。", + "modelProvider.card.tip": "メッセージ枠はOpenAI、Anthropic、Gemini、xAI、DeepSeek、Tongyiのモデルを使用することをサポートしています。無料枠は有料枠が使い果たされた後に消費されます。", "modelProvider.card.tokens": "トークン", "modelProvider.collapse": "折り畳み", "modelProvider.config": "設定", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "残りの無料トークン", "modelProvider.rerankModel.key": "Rerank モデル", "modelProvider.rerankModel.tip": "Rerank モデルは、ユーザークエリとの意味的一致に基づいて候補文書リストを再配置し、意味的ランキングの結果を向上させます。", + "modelProvider.resetDate": "{{date}} にリセット", "modelProvider.searchModel": "検索モデル", "modelProvider.selectModel": "モデルを選択", "modelProvider.selector.emptySetting": "設定に移動して構成してください", diff --git a/web/i18n/zh-Hans/billing.json b/web/i18n/zh-Hans/billing.json index e42edf0dc6..9111c1a6d1 100644 --- a/web/i18n/zh-Hans/billing.json +++ b/web/i18n/zh-Hans/billing.json @@ -96,7 +96,7 @@ "plansCommon.memberAfter": "个成员", "plansCommon.messageRequest.title": "{{count,number}} 条消息额度", "plansCommon.messageRequest.titlePerMonth": "{{count,number}} 条消息额度/月", - "plansCommon.messageRequest.tooltip": "消息额度旨在帮助您便捷地试用 Dify 中的各类 OpenAI 模型。不同模型会消耗不同额度。额度用尽后，您可以切换为使用自己的 OpenAI API 密钥。", + "plansCommon.messageRequest.tooltip": "消息额度旨在帮助您便捷地试用 Dify 中来自 OpenAI、Anthropic、Gemini、xAI、深度求索、通义 的不同模型。不同模型会消耗不同额度。额度用尽后，您可以切换为使用自己的 API 密钥。", "plansCommon.modelProviders": "支持 OpenAI/Anthropic/Llama2/Azure OpenAI/Hugging Face/Replicate", "plansCommon.month": "月", "plansCommon.mostPopular": "最受欢迎", diff --git a/web/i18n/zh-Hans/common.json b/web/i18n/zh-Hans/common.json index ca4ecce821..be7d4690af 100644 --- a/web/i18n/zh-Hans/common.json +++ b/web/i18n/zh-Hans/common.json @@ -339,13 +339,16 @@ "modelProvider.callTimes": "调用次数", "modelProvider.card.buyQuota": "购买额度", "modelProvider.card.callTimes": "调用次数", + "modelProvider.card.modelAPI": "{{modelName}} 模型正在使用 API Key。", + "modelProvider.card.modelNotSupported": "{{modelName}} 模型未安装。", + "modelProvider.card.modelSupported": "{{modelName}} 模型正在使用此额度。", "modelProvider.card.onTrial": "试用中", "modelProvider.card.paid": "已购买", "modelProvider.card.priorityUse": "优先使用", "modelProvider.card.quota": "额度", "modelProvider.card.quotaExhausted": "配额已用完", "modelProvider.card.removeKey": "删除 API 密钥", - "modelProvider.card.tip": "已付费额度将优先考虑。试用额度将在付费额度用完后使用。", + "modelProvider.card.tip": "消息额度支持使用 OpenAI、Anthropic、Gemini、xAI、深度求索、通义 的模型；免费额度会在付费额度用尽后才会消耗。", "modelProvider.card.tokens": "Tokens", "modelProvider.collapse": "收起", "modelProvider.config": "配置", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "剩余免费额度", "modelProvider.rerankModel.key": "Rerank 模型", "modelProvider.rerankModel.tip": "重排序模型将根据候选文档列表与用户问题语义匹配度进行重新排序，从而改进语义排序的结果", + "modelProvider.resetDate": "于 {{date}} 重置", "modelProvider.searchModel": "搜索模型", "modelProvider.selectModel": "选择您的模型", "modelProvider.selector.emptySetting": "请前往设置进行配置", diff --git a/web/models/common.ts b/web/models/common.ts index 0e034ffa33..62a543672b 100644 --- a/web/models/common.ts +++ b/web/models/common.ts @@ -142,6 +142,9 @@ export type IWorkspace = { export type ICurrentWorkspace = Omit & { role: 'owner' | 'admin' | 'editor' | 'dataset_operator' | 'normal' providers: Provider[] + trial_credits: number + trial_credits_used: number + next_credit_reset_date: number trial_end_reason?: string custom_config?: { remove_webapp_brand?: boolean From b2cbeeae929fc1a593a04994d032dfb2b4a7fbd5 Mon Sep 17 00:00:00 2001 From: xuwei95 <1013104194@qq.com> Date: Thu, 8 Jan 2026 17:23:27 +0800 Subject: [PATCH 16/21] fix(web): restrict postMessage targetOrigin from wildcard to specific origins (#30690) Co-authored-by: XW --- .../base/chat/embedded-chatbot/header/index.tsx | 4 +++- web/hooks/use-oauth.ts | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/web/app/components/base/chat/embedded-chatbot/header/index.tsx b/web/app/components/base/chat/embedded-chatbot/header/index.tsx index 869f88efb6..95ba6d212d 100644 --- a/web/app/components/base/chat/embedded-chatbot/header/index.tsx +++ b/web/app/components/base/chat/embedded-chatbot/header/index.tsx @@ -66,7 +66,9 @@ const Header: FC = ({ const listener = (event: MessageEvent) => handleMessageReceived(event) window.addEventListener('message', listener) - window.parent.postMessage({ type: 'dify-chatbot-iframe-ready' }, '*') + // Security: Use document.referrer to get parent origin + const targetOrigin = document.referrer ? new URL(document.referrer).origin : '*' + window.parent.postMessage({ type: 'dify-chatbot-iframe-ready' }, targetOrigin) return () => window.removeEventListener('message', listener) }, [isIframe, handleMessageReceived]) diff --git a/web/hooks/use-oauth.ts b/web/hooks/use-oauth.ts index 34ed8bafb0..8fb2707804 100644 --- a/web/hooks/use-oauth.ts +++ b/web/hooks/use-oauth.ts @@ -10,12 +10,15 @@ export const useOAuthCallback = () => { const errorDescription = urlParams.get('error_description') if (window.opener) { + // Use window.opener.origin instead of '*' for security + const targetOrigin = window.opener?.origin || '*' + if (subscriptionId) { window.opener.postMessage({ type: 'oauth_callback', success: true, subscriptionId, - }, '*') + }, targetOrigin) } else if (error) { window.opener.postMessage({ @@ -23,12 +26,12 @@ export const useOAuthCallback = () => { success: false, error, errorDescription, - }, '*') + }, targetOrigin) } else { window.opener.postMessage({ type: 'oauth_callback', - }, '*') + }, targetOrigin) } window.close() } From 91d44719f455f2988f3ebdd18768c283805706da Mon Sep 17 00:00:00 2001 From: MkDev11 Date: Thu, 8 Jan 2026 02:05:32 -0800 Subject: [PATCH 17/21] fix(web): resolve chat message loading race conditions and infinite loops (#30695) Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> --- web/app/components/app/log/list.spec.tsx | 228 ++++++++++++++++++++ web/app/components/app/log/list.tsx | 252 +++++++++-------------- 2 files changed, 322 insertions(+), 158 deletions(-) create mode 100644 web/app/components/app/log/list.spec.tsx diff --git a/web/app/components/app/log/list.spec.tsx b/web/app/components/app/log/list.spec.tsx new file mode 100644 index 0000000000..81901c6cad --- /dev/null +++ b/web/app/components/app/log/list.spec.tsx @@ -0,0 +1,228 @@ +/** + * Tests for race condition prevention logic in chat message loading. + * These tests verify the core algorithms used in fetchData and loadMoreMessages + * to prevent race conditions, infinite loops, and stale state issues. + * See GitHub issue #30259 for context. + */ + +// Test the race condition prevention logic in isolation +describe('Chat Message Loading Race Condition Prevention', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + describe('Request Deduplication', () => { + it('should deduplicate messages with same IDs when merging responses', async () => { + // Simulate the deduplication logic used in setAllChatItems + const existingItems = [ + { id: 'msg-1', isAnswer: false }, + { id: 'msg-2', isAnswer: true }, + ] + const newItems = [ + { id: 'msg-2', isAnswer: true }, // duplicate + { id: 'msg-3', isAnswer: false }, // new + ] + + const existingIds = new Set(existingItems.map(item => item.id)) + const uniqueNewItems = newItems.filter(item => !existingIds.has(item.id)) + const mergedItems = [...uniqueNewItems, ...existingItems] + + expect(uniqueNewItems).toHaveLength(1) + expect(uniqueNewItems[0].id).toBe('msg-3') + expect(mergedItems).toHaveLength(3) + }) + }) + + describe('Retry Counter Logic', () => { + const MAX_RETRY_COUNT = 3 + + it('should increment retry counter when no unique items found', () => { + const state = { retryCount: 0 } + const prevItemsLength = 5 + + // Simulate the retry logic from loadMoreMessages + const uniqueNewItemsLength = 0 + + if (uniqueNewItemsLength === 0) { + if (state.retryCount < MAX_RETRY_COUNT && prevItemsLength > 1) { + state.retryCount++ + } + else { + state.retryCount = 0 + } + } + + expect(state.retryCount).toBe(1) + }) + + it('should reset retry counter after MAX_RETRY_COUNT attempts', () => { + const state = { retryCount: MAX_RETRY_COUNT } + const prevItemsLength = 5 + const uniqueNewItemsLength = 0 + + if (uniqueNewItemsLength === 0) { + if (state.retryCount < MAX_RETRY_COUNT && prevItemsLength > 1) { + state.retryCount++ + } + else { + state.retryCount = 0 + } + } + + expect(state.retryCount).toBe(0) + }) + + it('should reset retry counter when unique items are found', () => { + const state = { retryCount: 2 } + + // Simulate finding unique items (length > 0) + const processRetry = (uniqueCount: number) => { + if (uniqueCount === 0) { + state.retryCount++ + } + else { + state.retryCount = 0 + } + } + + processRetry(3) // Found 3 unique items + + expect(state.retryCount).toBe(0) + }) + }) + + describe('Throttling Logic', () => { + const SCROLL_DEBOUNCE_MS = 200 + + it('should throttle requests within debounce window', () => { + const state = { lastLoadTime: 0 } + const results: boolean[] = [] + + const tryRequest = (now: number): boolean => { + if (now - state.lastLoadTime >= SCROLL_DEBOUNCE_MS) { + state.lastLoadTime = now + return true + } + return false + } + + // First request - should pass + results.push(tryRequest(1000)) + // Second request within debounce - should be blocked + results.push(tryRequest(1100)) + // Third request after debounce - should pass + results.push(tryRequest(1300)) + + expect(results).toEqual([true, false, true]) + }) + }) + + describe('AbortController Cancellation', () => { + it('should abort previous request when new request starts', () => { + const state: { controller: AbortController | null } = { controller: null } + const abortedSignals: boolean[] = [] + + // First request + const controller1 = new AbortController() + state.controller = controller1 + + // Second request - should abort first + if (state.controller) { + state.controller.abort() + abortedSignals.push(state.controller.signal.aborted) + } + const controller2 = new AbortController() + state.controller = controller2 + + expect(abortedSignals).toEqual([true]) + expect(controller1.signal.aborted).toBe(true) + expect(controller2.signal.aborted).toBe(false) + }) + }) + + describe('Stale Response Detection', () => { + it('should ignore responses from outdated requests', () => { + const state = { requestId: 0 } + const processedResponses: number[] = [] + + // Simulate concurrent requests - each gets its own captured ID + const request1Id = ++state.requestId + const request2Id = ++state.requestId + + // Request 2 completes first (current requestId is 2) + if (request2Id === state.requestId) { + processedResponses.push(request2Id) + } + + // Request 1 completes later (stale - requestId is still 2) + if (request1Id === state.requestId) { + processedResponses.push(request1Id) + } + + expect(processedResponses).toEqual([2]) + expect(processedResponses).not.toContain(1) + }) + }) + + describe('Pagination Anchor Management', () => { + it('should track oldest answer ID for pagination', () => { + let oldestAnswerIdRef: string | undefined + + const chatItems = [ + { id: 'question-1', isAnswer: false }, + { id: 'answer-1', isAnswer: true }, + { id: 'question-2', isAnswer: false }, + { id: 'answer-2', isAnswer: true }, + ] + + // Update pagination anchor with oldest answer ID + const answerItems = chatItems.filter(item => item.isAnswer) + const oldestAnswer = answerItems[answerItems.length - 1] + if (oldestAnswer?.id) { + oldestAnswerIdRef = oldestAnswer.id + } + + expect(oldestAnswerIdRef).toBe('answer-2') + }) + + it('should use pagination anchor in subsequent requests', () => { + const oldestAnswerIdRef = 'answer-123' + const params: { conversation_id: string, limit: number, first_id?: string } = { + conversation_id: 'conv-1', + limit: 10, + } + + if (oldestAnswerIdRef) { + params.first_id = oldestAnswerIdRef + } + + expect(params.first_id).toBe('answer-123') + }) + }) +}) + +describe('Functional State Update Pattern', () => { + it('should use functional update to avoid stale closures', () => { + // Simulate the functional update pattern used in setAllChatItems + let state = [{ id: '1' }, { id: '2' }] + + const newItems = [{ id: '3' }, { id: '2' }] // id '2' is duplicate + + // Functional update pattern + const updater = (prevItems: { id: string }[]) => { + const existingIds = new Set(prevItems.map(item => item.id)) + const uniqueNewItems = newItems.filter(item => !existingIds.has(item.id)) + return [...uniqueNewItems, ...prevItems] + } + + state = updater(state) + + expect(state).toHaveLength(3) + expect(state.map(i => i.id)).toEqual(['3', '1', '2']) + }) +}) diff --git a/web/app/components/app/log/list.tsx b/web/app/components/app/log/list.tsx index a17177bf7e..410953ccf7 100644 --- a/web/app/components/app/log/list.tsx +++ b/web/app/components/app/log/list.tsx @@ -209,7 +209,6 @@ type IDetailPanel = { function DetailPanel({ detail, onFeedback }: IDetailPanel) { const MIN_ITEMS_FOR_SCROLL_LOADING = 8 - const SCROLL_THRESHOLD_PX = 50 const SCROLL_DEBOUNCE_MS = 200 const { userProfile: { timezone } } = useAppContext() const { formatTime } = useTimestamp() @@ -228,69 +227,103 @@ function DetailPanel({ detail, onFeedback }: IDetailPanel) { const [hasMore, setHasMore] = useState(true) const [varValues, setVarValues] = useState>({}) const isLoadingRef = useRef(false) + const abortControllerRef = useRef(null) + const requestIdRef = useRef(0) + const lastLoadTimeRef = useRef(0) + const retryCountRef = useRef(0) + const oldestAnswerIdRef = useRef(undefined) + const MAX_RETRY_COUNT = 3 const [allChatItems, setAllChatItems] = useState([]) const [chatItemTree, setChatItemTree] = useState([]) const [threadChatItems, setThreadChatItems] = useState([]) const fetchData = useCallback(async () => { - if (isLoadingRef.current) + if (isLoadingRef.current || !hasMore) return + // Cancel any in-flight request + if (abortControllerRef.current) { + abortControllerRef.current.abort() + } + + const controller = new AbortController() + abortControllerRef.current = controller + const currentRequestId = ++requestIdRef.current + try { isLoadingRef.current = true - if (!hasMore) - return - const params: ChatMessagesRequest = { conversation_id: detail.id, limit: 10, } - // Use the oldest answer item ID for pagination - const answerItems = allChatItems.filter(item => item.isAnswer) - const oldestAnswerItem = answerItems[answerItems.length - 1] - if (oldestAnswerItem?.id) - params.first_id = oldestAnswerItem.id + // Use ref for pagination anchor to avoid stale closure issues + if (oldestAnswerIdRef.current) + params.first_id = oldestAnswerIdRef.current + const messageRes = await fetchChatMessages({ url: `/apps/${appDetail?.id}/chat-messages`, params, }) + + // Ignore stale responses + if (currentRequestId !== requestIdRef.current || controller.signal.aborted) + return if (messageRes.data.length > 0) { const varValues = messageRes.data.at(-1)!.inputs setVarValues(varValues) } setHasMore(messageRes.has_more) - const newAllChatItems = [ - ...getFormattedChatList(messageRes.data, detail.id, timezone!, t('dateTimeFormat', { ns: 'appLog' }) as string), - ...allChatItems, - ] - setAllChatItems(newAllChatItems) + const newItems = getFormattedChatList(messageRes.data, detail.id, timezone!, t('dateTimeFormat', { ns: 'appLog' }) as string) - let tree = buildChatItemTree(newAllChatItems) - if (messageRes.has_more === false && detail?.model_config?.configs?.introduction) { - tree = [{ - id: 'introduction', - isAnswer: true, - isOpeningStatement: true, - content: detail?.model_config?.configs?.introduction ?? 'hello', - feedbackDisabled: true, - children: tree, - }] - } - setChatItemTree(tree) - - const lastMessageId = newAllChatItems.length > 0 ? newAllChatItems[newAllChatItems.length - 1].id : undefined - setThreadChatItems(getThreadMessages(tree, lastMessageId)) + // Use functional update to avoid stale state issues + setAllChatItems((prevItems: IChatItem[]) => { + const existingIds = new Set(prevItems.map(item => item.id)) + const uniqueNewItems = newItems.filter(item => !existingIds.has(item.id)) + return [...uniqueNewItems, ...prevItems] + }) } - catch (err) { + catch (err: unknown) { + if (err instanceof Error && err.name === 'AbortError') + return console.error('fetchData execution failed:', err) } finally { isLoadingRef.current = false + if (abortControllerRef.current === controller) + abortControllerRef.current = null } - }, [allChatItems, detail.id, hasMore, timezone, t, appDetail, detail?.model_config?.configs?.introduction]) + }, [detail.id, hasMore, timezone, t, appDetail, detail?.model_config?.configs?.introduction]) + + // Derive chatItemTree, threadChatItems, and oldestAnswerIdRef from allChatItems + useEffect(() => { + if (allChatItems.length === 0) + return + + let tree = buildChatItemTree(allChatItems) + if (!hasMore && detail?.model_config?.configs?.introduction) { + tree = [{ + id: 'introduction', + isAnswer: true, + isOpeningStatement: true, + content: detail?.model_config?.configs?.introduction ?? 'hello', + feedbackDisabled: true, + children: tree, + }] + } + setChatItemTree(tree) + + const lastMessageId = allChatItems.length > 0 ? allChatItems[allChatItems.length - 1].id : undefined + setThreadChatItems(getThreadMessages(tree, lastMessageId)) + + // Update pagination anchor ref with the oldest answer ID + const answerItems = allChatItems.filter(item => item.isAnswer) + const oldestAnswer = answerItems[answerItems.length - 1] + if (oldestAnswer?.id) + oldestAnswerIdRef.current = oldestAnswer.id + }, [allChatItems, hasMore, detail?.model_config?.configs?.introduction]) const switchSibling = useCallback((siblingMessageId: string) => { const newThreadChatItems = getThreadMessages(chatItemTree, siblingMessageId) @@ -397,6 +430,12 @@ function DetailPanel({ detail, onFeedback }: IDetailPanel) { if (isLoading || !hasMore || !appDetail?.id || !detail.id) return + // Throttle using ref to persist across re-renders + const now = Date.now() + if (now - lastLoadTimeRef.current < SCROLL_DEBOUNCE_MS) + return + lastLoadTimeRef.current = now + setIsLoading(true) try { @@ -405,15 +444,9 @@ function DetailPanel({ detail, onFeedback }: IDetailPanel) { limit: 10, } - // Use the earliest response item as the first_id - const answerItems = allChatItems.filter(item => item.isAnswer) - const oldestAnswerItem = answerItems[answerItems.length - 1] - if (oldestAnswerItem?.id) { - params.first_id = oldestAnswerItem.id - } - else if (allChatItems.length > 0 && allChatItems[0]?.id) { - const firstId = allChatItems[0].id.replace('question-', '').replace('answer-', '') - params.first_id = firstId + // Use ref for pagination anchor to avoid stale closure issues + if (oldestAnswerIdRef.current) { + params.first_id = oldestAnswerIdRef.current } const messageRes = await fetchChatMessages({ @@ -423,6 +456,7 @@ function DetailPanel({ detail, onFeedback }: IDetailPanel) { if (!messageRes.data || messageRes.data.length === 0) { setHasMore(false) + retryCountRef.current = 0 return } @@ -440,91 +474,36 @@ function DetailPanel({ detail, onFeedback }: IDetailPanel) { t('dateTimeFormat', { ns: 'appLog' }) as string, ) - // Check for duplicate messages - const existingIds = new Set(allChatItems.map(item => item.id)) - const uniqueNewItems = newItems.filter(item => !existingIds.has(item.id)) + // Use functional update to get latest state and avoid stale closures + setAllChatItems((prevItems: IChatItem[]) => { + const existingIds = new Set(prevItems.map(item => item.id)) + const uniqueNewItems = newItems.filter(item => !existingIds.has(item.id)) - if (uniqueNewItems.length === 0) { - if (allChatItems.length > 1) { - const nextId = allChatItems[1].id.replace('question-', '').replace('answer-', '') - - const retryParams = { - ...params, - first_id: nextId, + // If no unique items and we haven't exceeded retry limit, signal retry needed + if (uniqueNewItems.length === 0) { + if (retryCountRef.current < MAX_RETRY_COUNT && prevItems.length > 1) { + retryCountRef.current++ + return prevItems } - - const retryRes = await fetchChatMessages({ - url: `/apps/${appDetail.id}/chat-messages`, - params: retryParams, - }) - - if (retryRes.data && retryRes.data.length > 0) { - const retryItems = getFormattedChatList( - retryRes.data, - detail.id, - timezone!, - t('dateTimeFormat', { ns: 'appLog' }) as string, - ) - - const retryUniqueItems = retryItems.filter(item => !existingIds.has(item.id)) - if (retryUniqueItems.length > 0) { - const newAllChatItems = [ - ...retryUniqueItems, - ...allChatItems, - ] - - setAllChatItems(newAllChatItems) - - let tree = buildChatItemTree(newAllChatItems) - if (retryRes.has_more === false && detail?.model_config?.configs?.introduction) { - tree = [{ - id: 'introduction', - isAnswer: true, - isOpeningStatement: true, - content: detail?.model_config?.configs?.introduction ?? 'hello', - feedbackDisabled: true, - children: tree, - }] - } - setChatItemTree(tree) - setHasMore(retryRes.has_more) - setThreadChatItems(getThreadMessages(tree, newAllChatItems.at(-1)?.id)) - return - } + else { + retryCountRef.current = 0 + return prevItems } } - } - const newAllChatItems = [ - ...uniqueNewItems, - ...allChatItems, - ] - - setAllChatItems(newAllChatItems) - - let tree = buildChatItemTree(newAllChatItems) - if (messageRes.has_more === false && detail?.model_config?.configs?.introduction) { - tree = [{ - id: 'introduction', - isAnswer: true, - isOpeningStatement: true, - content: detail?.model_config?.configs?.introduction ?? 'hello', - feedbackDisabled: true, - children: tree, - }] - } - setChatItemTree(tree) - - setThreadChatItems(getThreadMessages(tree, newAllChatItems.at(-1)?.id)) + retryCountRef.current = 0 + return [...uniqueNewItems, ...prevItems] + }) } catch (error) { console.error(error) setHasMore(false) + retryCountRef.current = 0 } finally { setIsLoading(false) } - }, [allChatItems, detail.id, hasMore, isLoading, timezone, t, appDetail]) + }, [detail.id, hasMore, isLoading, timezone, t, appDetail, detail?.model_config?.configs?.introduction]) useEffect(() => { const scrollableDiv = document.getElementById('scrollableDiv') @@ -556,24 +535,11 @@ function DetailPanel({ detail, onFeedback }: IDetailPanel) { if (!scrollContainer) return - let lastLoadTime = 0 - const throttleDelay = 200 - const handleScroll = () => { const currentScrollTop = scrollContainer!.scrollTop - const scrollHeight = scrollContainer!.scrollHeight - const clientHeight = scrollContainer!.clientHeight + const isNearTop = currentScrollTop < 30 - const distanceFromTop = currentScrollTop - const distanceFromBottom = scrollHeight - currentScrollTop - clientHeight - - const now = Date.now() - - const isNearTop = distanceFromTop < 30 - // eslint-disable-next-line sonarjs/no-unused-vars - const _distanceFromBottom = distanceFromBottom < 30 - if (isNearTop && hasMore && !isLoading && (now - lastLoadTime > throttleDelay)) { - lastLoadTime = now + if (isNearTop && hasMore && !isLoading) { loadMoreMessages() } } @@ -619,36 +585,6 @@ function DetailPanel({ detail, onFeedback }: IDetailPanel) { return () => cancelAnimationFrame(raf) }, []) - // Add scroll listener to ensure loading is triggered - useEffect(() => { - if (threadChatItems.length >= MIN_ITEMS_FOR_SCROLL_LOADING && hasMore) { - const scrollableDiv = document.getElementById('scrollableDiv') - - if (scrollableDiv) { - let loadingTimeout: NodeJS.Timeout | null = null - - const handleScroll = () => { - const { scrollTop } = scrollableDiv - - // Trigger loading when scrolling near the top - if (scrollTop < SCROLL_THRESHOLD_PX && !isLoadingRef.current) { - if (loadingTimeout) - clearTimeout(loadingTimeout) - - loadingTimeout = setTimeout(fetchData, SCROLL_DEBOUNCE_MS) // 200ms debounce - } - } - - scrollableDiv.addEventListener('scroll', handleScroll) - return () => { - scrollableDiv.removeEventListener('scroll', handleScroll) - if (loadingTimeout) - clearTimeout(loadingTimeout) - } - } - } - }, [threadChatItems.length, hasMore, fetchData]) - return (
{/* Panel Header */} From 7774a1312e37b4536d031e6179fea7e919a60314 Mon Sep 17 00:00:00 2001 From: yyh <92089059+lyzno1@users.noreply.github.com> Date: Thu, 8 Jan 2026 21:28:49 +0800 Subject: [PATCH 18/21] fix(ci): use repository_dispatch for i18n sync workflow (#30744) --- .github/workflows/translate-i18n-claude.yml | 47 +++++++++------ .github/workflows/trigger-i18n-sync.yml | 66 +++++++++++++++++++++ 2 files changed, 95 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/trigger-i18n-sync.yml diff --git a/.github/workflows/translate-i18n-claude.yml b/.github/workflows/translate-i18n-claude.yml index 0e05913576..003e7ffc6e 100644 --- a/.github/workflows/translate-i18n-claude.yml +++ b/.github/workflows/translate-i18n-claude.yml @@ -1,10 +1,12 @@ name: Translate i18n Files with Claude Code +# Note: claude-code-action doesn't support push events directly. +# Push events are handled by trigger-i18n-sync.yml which sends repository_dispatch. +# See: https://github.com/langgenius/dify/issues/30743 + on: - push: - branches: [main] - paths: - - 'web/i18n/en-US/*.json' + repository_dispatch: + types: [i18n-sync] workflow_dispatch: inputs: files: @@ -87,26 +89,35 @@ jobs: echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT fi fi - else - # Push trigger - detect changed files from the push - BEFORE_SHA="${{ github.event.before }}" - # Handle edge case: first push or force push may have null/zero SHA - if [ -z "$BEFORE_SHA" ] || [ "$BEFORE_SHA" = "0000000000000000000000000000000000000000" ]; then - # Fallback to comparing with parent commit - BEFORE_SHA="HEAD~1" + elif [ "${{ github.event_name }}" == "repository_dispatch" ]; then + # Triggered by push via trigger-i18n-sync.yml workflow + # Validate required payload fields + if [ -z "${{ github.event.client_payload.changed_files }}" ]; then + echo "Error: repository_dispatch payload missing required 'changed_files' field" >&2 + exit 1 fi - changed=$(git diff --name-only "$BEFORE_SHA" ${{ github.sha }} -- 'web/i18n/en-US/*.json' 2>/dev/null | xargs -n1 basename 2>/dev/null | sed 's/.json$//' | tr '\n' ' ' || echo "") - echo "CHANGED_FILES=$changed" >> $GITHUB_OUTPUT + echo "CHANGED_FILES=${{ github.event.client_payload.changed_files }}" >> $GITHUB_OUTPUT echo "TARGET_LANGS=" >> $GITHUB_OUTPUT - echo "SYNC_MODE=incremental" >> $GITHUB_OUTPUT + echo "SYNC_MODE=${{ github.event.client_payload.sync_mode || 'incremental' }}" >> $GITHUB_OUTPUT - # Generate detailed diff for the push - git diff "$BEFORE_SHA"..${{ github.sha }} -- 'web/i18n/en-US/*.json' > /tmp/i18n-diff.txt 2>/dev/null || echo "" > /tmp/i18n-diff.txt - if [ -s /tmp/i18n-diff.txt ]; then - echo "DIFF_AVAILABLE=true" >> $GITHUB_OUTPUT + # Decode the base64-encoded diff from the trigger workflow + if [ -n "${{ github.event.client_payload.diff_base64 }}" ]; then + if ! echo "${{ github.event.client_payload.diff_base64 }}" | base64 -d > /tmp/i18n-diff.txt 2>&1; then + echo "Warning: Failed to decode base64 diff payload" >&2 + echo "" > /tmp/i18n-diff.txt + echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT + elif [ -s /tmp/i18n-diff.txt ]; then + echo "DIFF_AVAILABLE=true" >> $GITHUB_OUTPUT + else + echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT + fi else + echo "" > /tmp/i18n-diff.txt echo "DIFF_AVAILABLE=false" >> $GITHUB_OUTPUT fi + else + echo "Unsupported event type: ${{ github.event_name }}" + exit 1 fi # Truncate diff if too large (keep first 50KB) diff --git a/.github/workflows/trigger-i18n-sync.yml b/.github/workflows/trigger-i18n-sync.yml new file mode 100644 index 0000000000..de093c9235 --- /dev/null +++ b/.github/workflows/trigger-i18n-sync.yml @@ -0,0 +1,66 @@ +name: Trigger i18n Sync on Push + +# This workflow bridges the push event to repository_dispatch +# because claude-code-action doesn't support push events directly. +# See: https://github.com/langgenius/dify/issues/30743 + +on: + push: + branches: [main] + paths: + - 'web/i18n/en-US/*.json' + +permissions: + contents: write + +jobs: + trigger: + if: github.repository == 'langgenius/dify' + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect changed files and generate diff + id: detect + run: | + BEFORE_SHA="${{ github.event.before }}" + # Handle edge case: force push may have null/zero SHA + if [ -z "$BEFORE_SHA" ] || [ "$BEFORE_SHA" = "0000000000000000000000000000000000000000" ]; then + BEFORE_SHA="HEAD~1" + fi + + # Detect changed i18n files + changed=$(git diff --name-only "$BEFORE_SHA" "${{ github.sha }}" -- 'web/i18n/en-US/*.json' 2>/dev/null | xargs -n1 basename 2>/dev/null | sed 's/.json$//' | tr '\n' ' ' || echo "") + echo "changed_files=$changed" >> $GITHUB_OUTPUT + + # Generate diff for context + git diff "$BEFORE_SHA" "${{ github.sha }}" -- 'web/i18n/en-US/*.json' > /tmp/i18n-diff.txt 2>/dev/null || echo "" > /tmp/i18n-diff.txt + + # Truncate if too large (keep first 50KB to match receiving workflow) + head -c 50000 /tmp/i18n-diff.txt > /tmp/i18n-diff-truncated.txt + mv /tmp/i18n-diff-truncated.txt /tmp/i18n-diff.txt + + # Base64 encode the diff for safe JSON transport (portable, single-line) + diff_base64=$(base64 < /tmp/i18n-diff.txt | tr -d '\n') + echo "diff_base64=$diff_base64" >> $GITHUB_OUTPUT + + if [ -n "$changed" ]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + echo "Detected changed files: $changed" + else + echo "has_changes=false" >> $GITHUB_OUTPUT + echo "No i18n changes detected" + fi + + - name: Trigger i18n sync workflow + if: steps.detect.outputs.has_changes == 'true' + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + event-type: i18n-sync + client-payload: '{"changed_files": "${{ steps.detect.outputs.changed_files }}", "diff_base64": "${{ steps.detect.outputs.diff_base64 }}", "sync_mode": "incremental", "trigger_sha": "${{ github.sha }}"}' From 5ad238579938f6d904edaced2e36c3d39962fa8d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 8 Jan 2026 22:53:04 +0800 Subject: [PATCH 19/21] chore(i18n): sync translations with en-US (#30750) Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> --- web/i18n/ar-TN/common.json | 4 ++++ web/i18n/de-DE/common.json | 4 ++++ web/i18n/es-ES/common.json | 4 ++++ web/i18n/fa-IR/common.json | 4 ++++ web/i18n/fr-FR/common.json | 4 ++++ web/i18n/hi-IN/common.json | 4 ++++ web/i18n/id-ID/common.json | 4 ++++ web/i18n/it-IT/common.json | 4 ++++ web/i18n/ko-KR/common.json | 4 ++++ web/i18n/pl-PL/common.json | 4 ++++ web/i18n/pt-BR/common.json | 4 ++++ web/i18n/ro-RO/common.json | 4 ++++ web/i18n/ru-RU/common.json | 4 ++++ web/i18n/sl-SI/common.json | 4 ++++ web/i18n/th-TH/common.json | 4 ++++ web/i18n/tr-TR/common.json | 4 ++++ web/i18n/uk-UA/common.json | 4 ++++ web/i18n/vi-VN/common.json | 4 ++++ web/i18n/zh-Hant/common.json | 4 ++++ 19 files changed, 76 insertions(+) diff --git a/web/i18n/ar-TN/common.json b/web/i18n/ar-TN/common.json index d015f1ae0b..998466c649 100644 --- a/web/i18n/ar-TN/common.json +++ b/web/i18n/ar-TN/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "أوقات الاتصال", "modelProvider.card.buyQuota": "شراء حصة", "modelProvider.card.callTimes": "أوقات الاتصال", + "modelProvider.card.modelAPI": "نماذج {{modelName}} تستخدم مفتاح API.", + "modelProvider.card.modelNotSupported": "نماذج {{modelName}} غير مثبتة.", + "modelProvider.card.modelSupported": "نماذج {{modelName}} تستخدم هذا الحصة.", "modelProvider.card.onTrial": "في التجربة", "modelProvider.card.paid": "مدفوع", "modelProvider.card.priorityUse": "أولوية الاستخدام", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "الرموز المجانية المتاحة المتبقية", "modelProvider.rerankModel.key": "نموذج إعادة الترتيب", "modelProvider.rerankModel.tip": "سيعيد نموذج إعادة الترتيب ترتيب قائمة المستندات المرشحة بناءً على المطابقة الدلالية مع استعلام المستخدم، مما يحسن نتائج الترتيب الدلالي", + "modelProvider.resetDate": "إعادة التعيين في {{date}}", "modelProvider.searchModel": "نموذج البحث", "modelProvider.selectModel": "اختر نموذجك", "modelProvider.selector.emptySetting": "يرجى الانتقال إلى الإعدادات للتكوين", diff --git a/web/i18n/de-DE/common.json b/web/i18n/de-DE/common.json index f54f6a939f..bd2d083fb0 100644 --- a/web/i18n/de-DE/common.json +++ b/web/i18n/de-DE/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Anrufzeiten", "modelProvider.card.buyQuota": "Kontingent kaufen", "modelProvider.card.callTimes": "Anrufzeiten", + "modelProvider.card.modelAPI": "{{modelName}}-Modelle verwenden den API-Schlüssel.", + "modelProvider.card.modelNotSupported": "{{modelName}}-Modelle sind nicht installiert.", + "modelProvider.card.modelSupported": "{{modelName}}-Modelle verwenden dieses Kontingent.", "modelProvider.card.onTrial": "In Probe", "modelProvider.card.paid": "Bezahlt", "modelProvider.card.priorityUse": "Priorisierte Nutzung", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Verbleibende verfügbare kostenlose Token", "modelProvider.rerankModel.key": "Rerank-Modell", "modelProvider.rerankModel.tip": "Rerank-Modell wird die Kandidatendokumentenliste basierend auf der semantischen Übereinstimmung mit der Benutzeranfrage neu ordnen und die Ergebnisse der semantischen Rangordnung verbessern", + "modelProvider.resetDate": "Zurücksetzen am {{date}}", "modelProvider.searchModel": "Suchmodell", "modelProvider.selectModel": "Wählen Sie Ihr Modell", "modelProvider.selector.emptySetting": "Bitte gehen Sie zu den Einstellungen, um zu konfigurieren", diff --git a/web/i18n/es-ES/common.json b/web/i18n/es-ES/common.json index ec08f11ed7..8175f97946 100644 --- a/web/i18n/es-ES/common.json +++ b/web/i18n/es-ES/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Tiempos de llamada", "modelProvider.card.buyQuota": "Comprar Cuota", "modelProvider.card.callTimes": "Tiempos de llamada", + "modelProvider.card.modelAPI": "Los modelos {{modelName}} están usando la clave API.", + "modelProvider.card.modelNotSupported": "Los modelos {{modelName}} no están instalados.", + "modelProvider.card.modelSupported": "Los modelos {{modelName}} están usando esta cuota.", "modelProvider.card.onTrial": "En prueba", "modelProvider.card.paid": "Pagado", "modelProvider.card.priorityUse": "Uso prioritario", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Tokens gratuitos restantes disponibles", "modelProvider.rerankModel.key": "Modelo de Reordenar", "modelProvider.rerankModel.tip": "El modelo de reordenar reordenará la lista de documentos candidatos basada en la coincidencia semántica con la consulta del usuario, mejorando los resultados de clasificación semántica", + "modelProvider.resetDate": "Restablecer el {{date}}", "modelProvider.searchModel": "Modelo de búsqueda", "modelProvider.selectModel": "Selecciona tu modelo", "modelProvider.selector.emptySetting": "Por favor ve a configuraciones para configurar", diff --git a/web/i18n/fa-IR/common.json b/web/i18n/fa-IR/common.json index 78f9b9e388..90ca2fbce3 100644 --- a/web/i18n/fa-IR/common.json +++ b/web/i18n/fa-IR/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "تعداد فراخوانی", "modelProvider.card.buyQuota": "خرید سهمیه", "modelProvider.card.callTimes": "تعداد فراخوانی", + "modelProvider.card.modelAPI": "مدل‌های {{modelName}} از کلید API استفاده می‌کنند.", + "modelProvider.card.modelNotSupported": "مدل‌های {{modelName}} نصب نشده‌اند.", + "modelProvider.card.modelSupported": "مدل‌های {{modelName}} از این سهمیه استفاده می‌کنند.", "modelProvider.card.onTrial": "در حال آزمایش", "modelProvider.card.paid": "پرداخت شده", "modelProvider.card.priorityUse": "استفاده با اولویت", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "توکن‌های رایگان باقی‌مانده در دسترس", "modelProvider.rerankModel.key": "مدل رتبه‌بندی مجدد", "modelProvider.rerankModel.tip": "مدل رتبه‌بندی مجدد، لیست اسناد کاندید را بر اساس تطابق معنایی با پرسش کاربر مرتب می‌کند و نتایج رتبه‌بندی معنایی را بهبود می‌بخشد", + "modelProvider.resetDate": "بازنشانی در {{date}}", "modelProvider.searchModel": "جستجوی مدل", "modelProvider.selectModel": "مدل خود را انتخاب کنید", "modelProvider.selector.emptySetting": "لطفاً به تنظیمات بروید تا پیکربندی کنید", diff --git a/web/i18n/fr-FR/common.json b/web/i18n/fr-FR/common.json index 7cc1af2d80..d2b4c70d7c 100644 --- a/web/i18n/fr-FR/common.json +++ b/web/i18n/fr-FR/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Temps d'appel", "modelProvider.card.buyQuota": "Acheter Quota", "modelProvider.card.callTimes": "Temps d'appel", + "modelProvider.card.modelAPI": "Les modèles {{modelName}} utilisent la clé API.", + "modelProvider.card.modelNotSupported": "Les modèles {{modelName}} ne sont pas installés.", + "modelProvider.card.modelSupported": "Les modèles {{modelName}} utilisent ce quota.", "modelProvider.card.onTrial": "En Essai", "modelProvider.card.paid": "Payé", "modelProvider.card.priorityUse": "Utilisation prioritaire", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Tokens gratuits restants disponibles", "modelProvider.rerankModel.key": "Modèle de Réorganisation", "modelProvider.rerankModel.tip": "Le modèle de réorganisation réorganisera la liste des documents candidats en fonction de la correspondance sémantique avec la requête de l'utilisateur, améliorant ainsi les résultats du classement sémantique.", + "modelProvider.resetDate": "Réinitialiser le {{date}}", "modelProvider.searchModel": "Modèle de recherche", "modelProvider.selectModel": "Sélectionnez votre modèle", "modelProvider.selector.emptySetting": "Veuillez aller dans les paramètres pour configurer", diff --git a/web/i18n/hi-IN/common.json b/web/i18n/hi-IN/common.json index 4670d5a545..c7b2402f81 100644 --- a/web/i18n/hi-IN/common.json +++ b/web/i18n/hi-IN/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "कॉल समय", "modelProvider.card.buyQuota": "कोटा खरीदें", "modelProvider.card.callTimes": "कॉल समय", + "modelProvider.card.modelAPI": "{{modelName}} मॉडल API कुंजी का उपयोग कर रहे हैं।", + "modelProvider.card.modelNotSupported": "{{modelName}} मॉडल इंस्टॉल नहीं हैं।", + "modelProvider.card.modelSupported": "{{modelName}} मॉडल इस कोटा का उपयोग कर रहे हैं।", "modelProvider.card.onTrial": "परीक्षण पर", "modelProvider.card.paid": "भुगतान किया हुआ", "modelProvider.card.priorityUse": "प्राथमिकता उपयोग", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "बचे हुए उपलब्ध मुफ्त टोकन", "modelProvider.rerankModel.key": "रीरैंक मॉडल", "modelProvider.rerankModel.tip": "रीरैंक मॉडल उपयोगकर्ता प्रश्न के साथ सांविधिक मेल के आधार पर उम्मीदवार दस्तावेज़ सूची को पुनः क्रमित करेगा, सांविधिक रैंकिंग के परिणामों में सुधार करेगा।", + "modelProvider.resetDate": "{{date}} को रीसेट करें", "modelProvider.searchModel": "खोज मॉडल", "modelProvider.selectModel": "अपने मॉडल का चयन करें", "modelProvider.selector.emptySetting": "कॉन्फ़िगर करने के लिए कृपया सेटिंग्स पर जाएं", diff --git a/web/i18n/id-ID/common.json b/web/i18n/id-ID/common.json index ede4d3ae44..541ee74b10 100644 --- a/web/i18n/id-ID/common.json +++ b/web/i18n/id-ID/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Waktu panggilan", "modelProvider.card.buyQuota": "Beli Kuota", "modelProvider.card.callTimes": "Waktu panggilan", + "modelProvider.card.modelAPI": "Model {{modelName}} menggunakan Kunci API.", + "modelProvider.card.modelNotSupported": "Model {{modelName}} tidak terpasang.", + "modelProvider.card.modelSupported": "Model {{modelName}} menggunakan kuota ini.", "modelProvider.card.onTrial": "Sedang Diadili", "modelProvider.card.paid": "Dibayar", "modelProvider.card.priorityUse": "Penggunaan prioritas", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Token gratis yang masih tersedia", "modelProvider.rerankModel.key": "Peringkat ulang Model", "modelProvider.rerankModel.tip": "Model rerank akan menyusun ulang daftar dokumen kandidat berdasarkan kecocokan semantik dengan kueri pengguna, meningkatkan hasil peringkat semantik", + "modelProvider.resetDate": "Setel ulang pada {{date}}", "modelProvider.searchModel": "Model pencarian", "modelProvider.selectModel": "Pilih model Anda", "modelProvider.selector.emptySetting": "Silakan buka pengaturan untuk mengonfigurasi", diff --git a/web/i18n/it-IT/common.json b/web/i18n/it-IT/common.json index 737ef923b1..49e14591a7 100644 --- a/web/i18n/it-IT/common.json +++ b/web/i18n/it-IT/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Numero di chiamate", "modelProvider.card.buyQuota": "Acquista Quota", "modelProvider.card.callTimes": "Numero di chiamate", + "modelProvider.card.modelAPI": "I modelli {{modelName}} stanno utilizzando la chiave API.", + "modelProvider.card.modelNotSupported": "I modelli {{modelName}} non sono installati.", + "modelProvider.card.modelSupported": "I modelli {{modelName}} stanno utilizzando questa quota.", "modelProvider.card.onTrial": "In Prova", "modelProvider.card.paid": "Pagato", "modelProvider.card.priorityUse": "Uso prioritario", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Token gratuiti rimanenti disponibili", "modelProvider.rerankModel.key": "Modello di Rerank", "modelProvider.rerankModel.tip": "Il modello di rerank riordinerà la lista dei documenti candidati basandosi sulla corrispondenza semantica con la query dell'utente, migliorando i risultati del ranking semantico", + "modelProvider.resetDate": "Ripristina il {{date}}", "modelProvider.searchModel": "Modello di ricerca", "modelProvider.selectModel": "Seleziona il tuo modello", "modelProvider.selector.emptySetting": "Per favore vai alle impostazioni per configurare", diff --git a/web/i18n/ko-KR/common.json b/web/i18n/ko-KR/common.json index 5640cb353d..a8ae974530 100644 --- a/web/i18n/ko-KR/common.json +++ b/web/i18n/ko-KR/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "호출 횟수", "modelProvider.card.buyQuota": "Buy Quota", "modelProvider.card.callTimes": "호출 횟수", + "modelProvider.card.modelAPI": "{{modelName}} 모델이 API 키를 사용하고 있습니다.", + "modelProvider.card.modelNotSupported": "{{modelName}} 모델이 설치되지 않았습니다.", + "modelProvider.card.modelSupported": "{{modelName}} 모델이 이 할당량을 사용하고 있습니다.", "modelProvider.card.onTrial": "트라이얼 중", "modelProvider.card.paid": "유료", "modelProvider.card.priorityUse": "우선 사용", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "남은 무료 토큰 사용 가능", "modelProvider.rerankModel.key": "재랭크 모델", "modelProvider.rerankModel.tip": "재랭크 모델은 사용자 쿼리와의 의미적 일치를 기반으로 후보 문서 목록을 재배열하여 의미적 순위를 향상시킵니다.", + "modelProvider.resetDate": "{{date}}에 재설정", "modelProvider.searchModel": "검색 모델", "modelProvider.selectModel": "모델 선택", "modelProvider.selector.emptySetting": "설정으로 이동하여 구성하세요", diff --git a/web/i18n/pl-PL/common.json b/web/i18n/pl-PL/common.json index ae654e04ac..963ecf865d 100644 --- a/web/i18n/pl-PL/common.json +++ b/web/i18n/pl-PL/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Czasy wywołań", "modelProvider.card.buyQuota": "Kup limit", "modelProvider.card.callTimes": "Czasy wywołań", + "modelProvider.card.modelAPI": "Modele {{modelName}} używają klucza API.", + "modelProvider.card.modelNotSupported": "Modele {{modelName}} nie są zainstalowane.", + "modelProvider.card.modelSupported": "Modele {{modelName}} używają tego limitu.", "modelProvider.card.onTrial": "Na próbę", "modelProvider.card.paid": "Płatny", "modelProvider.card.priorityUse": "Używanie z priorytetem", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Pozostałe dostępne darmowe tokeny", "modelProvider.rerankModel.key": "Model ponownego rankingu", "modelProvider.rerankModel.tip": "Model ponownego rankingu zmieni kolejność listy dokumentów kandydatów na podstawie semantycznego dopasowania z zapytaniem użytkownika, poprawiając wyniki rankingu semantycznego", + "modelProvider.resetDate": "Reset {{date}}", "modelProvider.searchModel": "Model wyszukiwania", "modelProvider.selectModel": "Wybierz swój model", "modelProvider.selector.emptySetting": "Przejdź do ustawień, aby skonfigurować", diff --git a/web/i18n/pt-BR/common.json b/web/i18n/pt-BR/common.json index 2e7f49de7e..7efc250349 100644 --- a/web/i18n/pt-BR/common.json +++ b/web/i18n/pt-BR/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Chamadas", "modelProvider.card.buyQuota": "Comprar Quota", "modelProvider.card.callTimes": "Chamadas", + "modelProvider.card.modelAPI": "Os modelos {{modelName}} estão usando a Chave API.", + "modelProvider.card.modelNotSupported": "Os modelos {{modelName}} não estão instalados.", + "modelProvider.card.modelSupported": "Os modelos {{modelName}} estão usando esta cota.", "modelProvider.card.onTrial": "Em Teste", "modelProvider.card.paid": "Pago", "modelProvider.card.priorityUse": "Uso prioritário", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Tokens gratuitos disponíveis restantes", "modelProvider.rerankModel.key": "Modelo de Reordenação", "modelProvider.rerankModel.tip": "O modelo de reordenaenação reorganizará a lista de documentos candidatos com base na correspondência semântica com a consulta do usuário, melhorando os resultados da classificação semântica", + "modelProvider.resetDate": "Redefinir em {{date}}", "modelProvider.searchModel": "Modelo de pesquisa", "modelProvider.selectModel": "Selecione seu modelo", "modelProvider.selector.emptySetting": "Por favor, vá para configurações para configurar", diff --git a/web/i18n/ro-RO/common.json b/web/i18n/ro-RO/common.json index c21e755b3c..bafe3542dc 100644 --- a/web/i18n/ro-RO/common.json +++ b/web/i18n/ro-RO/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Apeluri", "modelProvider.card.buyQuota": "Cumpără cotă", "modelProvider.card.callTimes": "Apeluri", + "modelProvider.card.modelAPI": "Modelele {{modelName}} folosesc cheia API.", + "modelProvider.card.modelNotSupported": "Modelele {{modelName}} nu sunt instalate.", + "modelProvider.card.modelSupported": "Modelele {{modelName}} folosesc această cotă.", "modelProvider.card.onTrial": "În probă", "modelProvider.card.paid": "Plătit", "modelProvider.card.priorityUse": "Utilizare prioritară", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Jetoane gratuite disponibile rămase", "modelProvider.rerankModel.key": "Model de reordonare", "modelProvider.rerankModel.tip": "Modelul de reordonare va reordona lista de documente candidate pe baza potrivirii semantice cu interogarea utilizatorului, îmbunătățind rezultatele clasificării semantice", + "modelProvider.resetDate": "Resetare la {{date}}", "modelProvider.searchModel": "Model de căutare", "modelProvider.selectModel": "Selectați modelul dvs.", "modelProvider.selector.emptySetting": "Vă rugăm să mergeți la setări pentru a configura", diff --git a/web/i18n/ru-RU/common.json b/web/i18n/ru-RU/common.json index e763a7ec2a..0210db777f 100644 --- a/web/i18n/ru-RU/common.json +++ b/web/i18n/ru-RU/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Количество вызовов", "modelProvider.card.buyQuota": "Купить квоту", "modelProvider.card.callTimes": "Количество вызовов", + "modelProvider.card.modelAPI": "Модели {{modelName}} используют API-ключ.", + "modelProvider.card.modelNotSupported": "Модели {{modelName}} не установлены.", + "modelProvider.card.modelSupported": "Модели {{modelName}} используют эту квоту.", "modelProvider.card.onTrial": "Пробная версия", "modelProvider.card.paid": "Платный", "modelProvider.card.priorityUse": "Приоритетное использование", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Оставшиеся доступные бесплатные токены", "modelProvider.rerankModel.key": "Модель повторного ранжирования", "modelProvider.rerankModel.tip": "Модель повторного ранжирования изменит порядок списка документов-кандидатов на основе семантического соответствия запросу пользователя, улучшая результаты семантического ранжирования", + "modelProvider.resetDate": "Сброс {{date}}", "modelProvider.searchModel": "Поиск модели", "modelProvider.selectModel": "Выберите свою модель", "modelProvider.selector.emptySetting": "Пожалуйста, перейдите в настройки для настройки", diff --git a/web/i18n/sl-SI/common.json b/web/i18n/sl-SI/common.json index d092fe10c8..c33686ac03 100644 --- a/web/i18n/sl-SI/common.json +++ b/web/i18n/sl-SI/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Število klicev", "modelProvider.card.buyQuota": "Kupi kvoto", "modelProvider.card.callTimes": "Časi klicev", + "modelProvider.card.modelAPI": "Modeli {{modelName}} uporabljajo API ključ.", + "modelProvider.card.modelNotSupported": "Modeli {{modelName}} niso nameščeni.", + "modelProvider.card.modelSupported": "Modeli {{modelName}} uporabljajo to kvoto.", "modelProvider.card.onTrial": "Na preizkusu", "modelProvider.card.paid": "Plačano", "modelProvider.card.priorityUse": "Prednostna uporaba", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Preostali razpoložljivi brezplačni žetoni", "modelProvider.rerankModel.key": "Model za prerazvrstitev", "modelProvider.rerankModel.tip": "Model za prerazvrstitev bo prerazporedil seznam kandidatskih dokumentov na podlagi semantične ujemanja z uporabniško poizvedbo, s čimer se izboljšajo rezultati semantičnega razvrščanja.", + "modelProvider.resetDate": "Ponastavi {{date}}", "modelProvider.searchModel": "Model iskanja", "modelProvider.selectModel": "Izberite svoj model", "modelProvider.selector.emptySetting": "Prosimo, pojdite v nastavitve za konfiguracijo", diff --git a/web/i18n/th-TH/common.json b/web/i18n/th-TH/common.json index 9a38f7f683..2a6b575618 100644 --- a/web/i18n/th-TH/common.json +++ b/web/i18n/th-TH/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "เวลาโทร", "modelProvider.card.buyQuota": "ซื้อโควต้า", "modelProvider.card.callTimes": "เวลาโทร", + "modelProvider.card.modelAPI": "โมเดล {{modelName}} กำลังใช้คีย์ API", + "modelProvider.card.modelNotSupported": "โมเดล {{modelName}} ไม่ได้ติดตั้ง", + "modelProvider.card.modelSupported": "โมเดล {{modelName}} กำลังใช้โควต้านี้", "modelProvider.card.onTrial": "ทดลองใช้", "modelProvider.card.paid": "จ่าย", "modelProvider.card.priorityUse": "ลําดับความสําคัญในการใช้งาน", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "โทเค็นฟรีที่เหลืออยู่", "modelProvider.rerankModel.key": "จัดอันดับโมเดลใหม่", "modelProvider.rerankModel.tip": "โมเดล Rerank จะจัดลําดับรายการเอกสารผู้สมัครใหม่ตามการจับคู่ความหมายกับการสืบค้นของผู้ใช้ ซึ่งช่วยปรับปรุงผลลัพธ์ของการจัดอันดับความหมาย", + "modelProvider.resetDate": "รีเซ็ตเมื่อ {{date}}", "modelProvider.searchModel": "ค้นหารุ่น", "modelProvider.selectModel": "เลือกรุ่นของคุณ", "modelProvider.selector.emptySetting": "โปรดไปที่การตั้งค่าเพื่อกําหนดค่า", diff --git a/web/i18n/tr-TR/common.json b/web/i18n/tr-TR/common.json index 0ee51e161c..c45b453180 100644 --- a/web/i18n/tr-TR/common.json +++ b/web/i18n/tr-TR/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Çağrı Süreleri", "modelProvider.card.buyQuota": "Kota Satın Al", "modelProvider.card.callTimes": "Çağrı Süreleri", + "modelProvider.card.modelAPI": "{{modelName}} modelleri API Anahtarını kullanıyor.", + "modelProvider.card.modelNotSupported": "{{modelName}} modelleri kurulu değil.", + "modelProvider.card.modelSupported": "{{modelName}} modelleri bu kotayı kullanıyor.", "modelProvider.card.onTrial": "Deneme Sürümünde", "modelProvider.card.paid": "Ücretli", "modelProvider.card.priorityUse": "Öncelikli Kullan", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Kalan kullanılabilir ücretsiz tokenler", "modelProvider.rerankModel.key": "Yeniden Sıralama Modeli", "modelProvider.rerankModel.tip": "Yeniden sıralama modeli, kullanıcı sorgusuyla anlam eşleştirmesine dayalı olarak aday belge listesini yeniden sıralayacak ve anlam sıralama sonuçlarını iyileştirecektir.", + "modelProvider.resetDate": "{{date}} tarihinde sıfırla", "modelProvider.searchModel": "Model ara", "modelProvider.selectModel": "Modelinizi seçin", "modelProvider.selector.emptySetting": "Lütfen ayarlara gidip yapılandırın", diff --git a/web/i18n/uk-UA/common.json b/web/i18n/uk-UA/common.json index ddec8637e1..e9e810da45 100644 --- a/web/i18n/uk-UA/common.json +++ b/web/i18n/uk-UA/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Кількість викликів", "modelProvider.card.buyQuota": "Придбати квоту", "modelProvider.card.callTimes": "Кількість викликів", + "modelProvider.card.modelAPI": "Моделі {{modelName}} використовують API-ключ.", + "modelProvider.card.modelNotSupported": "Моделі {{modelName}} не встановлено.", + "modelProvider.card.modelSupported": "Моделі {{modelName}} використовують цю квоту.", "modelProvider.card.onTrial": "У пробному періоді", "modelProvider.card.paid": "Оплачено", "modelProvider.card.priorityUse": "Пріоритетне використання", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Залишилося доступних безкоштовних токенів", "modelProvider.rerankModel.key": "Модель повторного ранжування", "modelProvider.rerankModel.tip": "Модель повторного ранжування змінить порядок списку документів-кандидатів на основі семантичної відповідності запиту користувача, покращуючи результати семантичного ранжування.", + "modelProvider.resetDate": "Скидання {{date}}", "modelProvider.searchModel": "Пошукова модель", "modelProvider.selectModel": "Виберіть свою модель", "modelProvider.selector.emptySetting": "Перейдіть до налаштувань, щоб налаштувати", diff --git a/web/i18n/vi-VN/common.json b/web/i18n/vi-VN/common.json index f8fa9c07d5..1fec0e10e2 100644 --- a/web/i18n/vi-VN/common.json +++ b/web/i18n/vi-VN/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "Số lần gọi", "modelProvider.card.buyQuota": "Mua Quota", "modelProvider.card.callTimes": "Số lần gọi", + "modelProvider.card.modelAPI": "Các mô hình {{modelName}} đang sử dụng Khóa API.", + "modelProvider.card.modelNotSupported": "Các mô hình {{modelName}} chưa được cài đặt.", + "modelProvider.card.modelSupported": "Các mô hình {{modelName}} đang sử dụng hạn mức này.", "modelProvider.card.onTrial": "Thử nghiệm", "modelProvider.card.paid": "Đã thanh toán", "modelProvider.card.priorityUse": "Ưu tiên sử dụng", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "Số lượng mã thông báo miễn phí còn lại", "modelProvider.rerankModel.key": "Mô hình Sắp xếp lại", "modelProvider.rerankModel.tip": "Mô hình sắp xếp lại sẽ sắp xếp lại danh sách tài liệu ứng cử viên dựa trên sự phù hợp ngữ nghĩa với truy vấn của người dùng, cải thiện kết quả của việc xếp hạng ngữ nghĩa", + "modelProvider.resetDate": "Đặt lại vào {{date}}", "modelProvider.searchModel": "Mô hình tìm kiếm", "modelProvider.selectModel": "Chọn mô hình của bạn", "modelProvider.selector.emptySetting": "Vui lòng vào cài đặt để cấu hình", diff --git a/web/i18n/zh-Hant/common.json b/web/i18n/zh-Hant/common.json index 8fe3e5bd07..52be863c6d 100644 --- a/web/i18n/zh-Hant/common.json +++ b/web/i18n/zh-Hant/common.json @@ -339,6 +339,9 @@ "modelProvider.callTimes": "呼叫次數", "modelProvider.card.buyQuota": "購買額度", "modelProvider.card.callTimes": "呼叫次數", + "modelProvider.card.modelAPI": "{{modelName}} 模型正在使用 API Key。", + "modelProvider.card.modelNotSupported": "{{modelName}} 模型未安裝。", + "modelProvider.card.modelSupported": "{{modelName}} 模型正在使用此配額。", "modelProvider.card.onTrial": "試用中", "modelProvider.card.paid": "已購買", "modelProvider.card.priorityUse": "優先使用", @@ -394,6 +397,7 @@ "modelProvider.quotaTip": "剩餘免費額度", "modelProvider.rerankModel.key": "Rerank 模型", "modelProvider.rerankModel.tip": "重排序模型將根據候選文件列表與使用者問題語義匹配度進行重新排序，從而改進語義排序的結果", + "modelProvider.resetDate": "於 {{date}} 重置", "modelProvider.searchModel": "搜尋模型", "modelProvider.selectModel": "選擇您的模型", "modelProvider.selector.emptySetting": "請前往設定進行配置", From 9848823dcd1679c375171e474ab1f0dac85dd318 Mon Sep 17 00:00:00 2001 From: Coding On Star <447357187@qq.com> Date: Fri, 9 Jan 2026 10:21:18 +0800 Subject: [PATCH 20/21] feat: implement step two of dataset creation with comprehensive UI components and hooks (#30681) Co-authored-by: CodingOnStar --- .../components/general-chunking-options.tsx | 199 ++ .../create/step-two/components/index.ts | 5 + .../components/indexing-mode-section.tsx | 253 ++ .../step-two/{ => components}/inputs.tsx | 0 .../step-two/{ => components}/option-card.tsx | 0 .../components/parent-child-options.tsx | 191 ++ .../step-two/components/preview-panel.tsx | 171 ++ .../step-two/components/step-two-footer.tsx | 58 + .../create/step-two/{ => hooks}/escape.ts | 0 .../datasets/create/step-two/hooks/index.ts | 14 + .../create/step-two/{ => hooks}/unescape.ts | 0 .../step-two/hooks/use-document-creation.ts | 279 +++ .../step-two/hooks/use-indexing-config.ts | 143 ++ .../step-two/hooks/use-indexing-estimate.ts | 123 + .../step-two/hooks/use-preview-state.ts | 127 + .../step-two/hooks/use-segmentation-state.ts | 222 ++ .../datasets/create/step-two/index.spec.tsx | 2197 +++++++++++++++++ .../datasets/create/step-two/index.tsx | 1366 ++-------- .../datasets/create/step-two/types.ts | 28 + 19 files changed, 4209 insertions(+), 1167 deletions(-) create mode 100644 web/app/components/datasets/create/step-two/components/general-chunking-options.tsx create mode 100644 web/app/components/datasets/create/step-two/components/index.ts create mode 100644 web/app/components/datasets/create/step-two/components/indexing-mode-section.tsx rename web/app/components/datasets/create/step-two/{ => components}/inputs.tsx (100%) rename web/app/components/datasets/create/step-two/{ => components}/option-card.tsx (100%) create mode 100644 web/app/components/datasets/create/step-two/components/parent-child-options.tsx create mode 100644 web/app/components/datasets/create/step-two/components/preview-panel.tsx create mode 100644 web/app/components/datasets/create/step-two/components/step-two-footer.tsx rename web/app/components/datasets/create/step-two/{ => hooks}/escape.ts (100%) create mode 100644 web/app/components/datasets/create/step-two/hooks/index.ts rename web/app/components/datasets/create/step-two/{ => hooks}/unescape.ts (100%) create mode 100644 web/app/components/datasets/create/step-two/hooks/use-document-creation.ts create mode 100644 web/app/components/datasets/create/step-two/hooks/use-indexing-config.ts create mode 100644 web/app/components/datasets/create/step-two/hooks/use-indexing-estimate.ts create mode 100644 web/app/components/datasets/create/step-two/hooks/use-preview-state.ts create mode 100644 web/app/components/datasets/create/step-two/hooks/use-segmentation-state.ts create mode 100644 web/app/components/datasets/create/step-two/index.spec.tsx create mode 100644 web/app/components/datasets/create/step-two/types.ts diff --git a/web/app/components/datasets/create/step-two/components/general-chunking-options.tsx b/web/app/components/datasets/create/step-two/components/general-chunking-options.tsx new file mode 100644 index 0000000000..5140c902f5 --- /dev/null +++ b/web/app/components/datasets/create/step-two/components/general-chunking-options.tsx @@ -0,0 +1,199 @@ +'use client' + +import type { FC } from 'react' +import type { PreProcessingRule } from '@/models/datasets' +import { + RiAlertFill, + RiSearchEyeLine, +} from '@remixicon/react' +import Image from 'next/image' +import { useTranslation } from 'react-i18next' +import Button from '@/app/components/base/button' +import Checkbox from '@/app/components/base/checkbox' +import Divider from '@/app/components/base/divider' +import Tooltip from '@/app/components/base/tooltip' +import { IS_CE_EDITION } from '@/config' +import { ChunkingMode } from '@/models/datasets' +import SettingCog from '../../assets/setting-gear-mod.svg' +import s from '../index.module.css' +import LanguageSelect from '../language-select' +import { DelimiterInput, MaxLengthInput, OverlapInput } from './inputs' +import { OptionCard } from './option-card' + +type TextLabelProps = { + children: React.ReactNode +} + +const TextLabel: FC = ({ children }) => { + return +} + +type GeneralChunkingOptionsProps = { + // State + segmentIdentifier: string + maxChunkLength: number + overlap: number + rules: PreProcessingRule[] + currentDocForm: ChunkingMode + docLanguage: string + // Flags + isActive: boolean + isInUpload: boolean + isNotUploadInEmptyDataset: boolean + hasCurrentDatasetDocForm: boolean + // Actions + onSegmentIdentifierChange: (value: string) => void + onMaxChunkLengthChange: (value: number) => void + onOverlapChange: (value: number) => void + onRuleToggle: (id: string) => void + onDocFormChange: (form: ChunkingMode) => void + onDocLanguageChange: (lang: string) => void + onPreview: () => void + onReset: () => void + // Locale + locale: string +} + +export const GeneralChunkingOptions: FC = ({ + segmentIdentifier, + maxChunkLength, + overlap, + rules, + currentDocForm, + docLanguage, + isActive, + isInUpload, + isNotUploadInEmptyDataset, + hasCurrentDatasetDocForm, + onSegmentIdentifierChange, + onMaxChunkLengthChange, + onOverlapChange, + onRuleToggle, + onDocFormChange, + onDocLanguageChange, + onPreview, + onReset, + locale, +}) => { + const { t } = useTranslation() + + const getRuleName = (key: string): string => { + const ruleNameMap: Record = { + remove_extra_spaces: t('stepTwo.removeExtraSpaces', { ns: 'datasetCreation' }), + remove_urls_emails: t('stepTwo.removeUrlEmails', { ns: 'datasetCreation' }), + remove_stopwords: t('stepTwo.removeStopwords', { ns: 'datasetCreation' }), + } + return ruleNameMap[key] ?? key + } + + return ( + } + activeHeaderClassName="bg-dataset-option-card-blue-gradient" + description={t('stepTwo.generalTip', { ns: 'datasetCreation' })} + isActive={isActive} + onSwitched={() => onDocFormChange(ChunkingMode.text)} + actions={( + <> + + + + )} + noHighlight={isInUpload && isNotUploadInEmptyDataset} + > +
+
+ onSegmentIdentifierChange(e.target.value)} + /> + + +
+
+
+
+ {t('stepTwo.rules', { ns: 'datasetCreation' })} +
+ +
+
+ {rules.map(rule => ( +
onRuleToggle(rule.id)} + > + + +
+ ))} + {IS_CE_EDITION && ( + <> + +
+
{ + if (hasCurrentDatasetDocForm) + return + if (currentDocForm === ChunkingMode.qa) + onDocFormChange(ChunkingMode.text) + else + onDocFormChange(ChunkingMode.qa) + }} + > + + +
+ + +
+ {currentDocForm === ChunkingMode.qa && ( +
+ + + {t('stepTwo.QATip', { ns: 'datasetCreation' })} + +
+ )} + + )} +
+
+
+ + ) +} diff --git a/web/app/components/datasets/create/step-two/components/index.ts b/web/app/components/datasets/create/step-two/components/index.ts new file mode 100644 index 0000000000..d5382e0c4b --- /dev/null +++ b/web/app/components/datasets/create/step-two/components/index.ts @@ -0,0 +1,5 @@ +export { GeneralChunkingOptions } from './general-chunking-options' +export { IndexingModeSection } from './indexing-mode-section' +export { ParentChildOptions } from './parent-child-options' +export { PreviewPanel } from './preview-panel' +export { StepTwoFooter } from './step-two-footer' diff --git a/web/app/components/datasets/create/step-two/components/indexing-mode-section.tsx b/web/app/components/datasets/create/step-two/components/indexing-mode-section.tsx new file mode 100644 index 0000000000..ee49f42903 --- /dev/null +++ b/web/app/components/datasets/create/step-two/components/indexing-mode-section.tsx @@ -0,0 +1,253 @@ +'use client' + +import type { FC } from 'react' +import type { DefaultModel, Model } from '@/app/components/header/account-setting/model-provider-page/declarations' +import type { RetrievalConfig } from '@/types/app' +import Image from 'next/image' +import Link from 'next/link' +import { useTranslation } from 'react-i18next' +import Badge from '@/app/components/base/badge' +import Button from '@/app/components/base/button' +import CustomDialog from '@/app/components/base/dialog' +import Divider from '@/app/components/base/divider' +import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback' +import Tooltip from '@/app/components/base/tooltip' +import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config' +import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config' +import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector' +import { useDocLink } from '@/context/i18n' +import { ChunkingMode } from '@/models/datasets' +import { cn } from '@/utils/classnames' +import { indexMethodIcon } from '../../icons' +import { IndexingType } from '../hooks' +import s from '../index.module.css' +import { OptionCard } from './option-card' + +type IndexingModeSectionProps = { + // State + indexType: IndexingType + hasSetIndexType: boolean + docForm: ChunkingMode + embeddingModel: DefaultModel + embeddingModelList?: Model[] + retrievalConfig: RetrievalConfig + showMultiModalTip: boolean + // Flags + isModelAndRetrievalConfigDisabled: boolean + datasetId?: string + // Modal state + isQAConfirmDialogOpen: boolean + // Actions + onIndexTypeChange: (type: IndexingType) => void + onEmbeddingModelChange: (model: DefaultModel) => void + onRetrievalConfigChange: (config: RetrievalConfig) => void + onQAConfirmDialogClose: () => void + onQAConfirmDialogConfirm: () => void +} + +export const IndexingModeSection: FC = ({ + indexType, + hasSetIndexType, + docForm, + embeddingModel, + embeddingModelList, + retrievalConfig, + showMultiModalTip, + isModelAndRetrievalConfigDisabled, + datasetId, + isQAConfirmDialogOpen, + onIndexTypeChange, + onEmbeddingModelChange, + onRetrievalConfigChange, + onQAConfirmDialogClose, + onQAConfirmDialogConfirm, +}) => { + const { t } = useTranslation() + const docLink = useDocLink() + + const getIndexingTechnique = () => indexType + + return ( + <> + {/* Index Mode */} +
+ {t('stepTwo.indexMode', { ns: 'datasetCreation' })} +
+
+ {/* Qualified option */} + {(!hasSetIndexType || (hasSetIndexType && indexType === IndexingType.QUALIFIED)) && ( + + {t('stepTwo.qualified', { ns: 'datasetCreation' })} + + {t('stepTwo.recommend', { ns: 'datasetCreation' })} + + + {!hasSetIndexType && } + +
+ )} + description={t('stepTwo.qualifiedTip', { ns: 'datasetCreation' })} + icon={} + isActive={!hasSetIndexType && indexType === IndexingType.QUALIFIED} + disabled={hasSetIndexType} + onSwitched={() => onIndexTypeChange(IndexingType.QUALIFIED)} + /> + )} + + {/* Economical option */} + {(!hasSetIndexType || (hasSetIndexType && indexType === IndexingType.ECONOMICAL)) && ( + <> + +
+

+ {t('stepTwo.qaSwitchHighQualityTipTitle', { ns: 'datasetCreation' })} +

+

+ {t('stepTwo.qaSwitchHighQualityTipContent', { ns: 'datasetCreation' })} +

+
+
+ + +
+ + + {docForm === ChunkingMode.qa + ? t('stepTwo.notAvailableForQA', { ns: 'datasetCreation' }) + : t('stepTwo.notAvailableForParentChild', { ns: 'datasetCreation' })} +
+ )} + noDecoration + position="top" + asChild={false} + triggerClassName="flex-1 self-stretch" + > + } + isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL} + disabled={hasSetIndexType || docForm !== ChunkingMode.text} + onSwitched={() => onIndexTypeChange(IndexingType.ECONOMICAL)} + /> + + + )} +
+ + {/* High quality tip */} + {!hasSetIndexType && indexType === IndexingType.QUALIFIED && ( +
+
+
+ +
+ + {t('stepTwo.highQualityTip', { ns: 'datasetCreation' })} + +
+ )} + + {/* Economical index setting tip */} + {hasSetIndexType && indexType === IndexingType.ECONOMICAL && ( +
+ {t('stepTwo.indexSettingTip', { ns: 'datasetCreation' })} + + {t('stepTwo.datasetSettingLink', { ns: 'datasetCreation' })} + +
+ )} + + {/* Embedding model */} + {indexType === IndexingType.QUALIFIED && ( +
+
+ {t('form.embeddingModel', { ns: 'datasetSettings' })} +
+ + {isModelAndRetrievalConfigDisabled && ( +
+ {t('stepTwo.indexSettingTip', { ns: 'datasetCreation' })} + + {t('stepTwo.datasetSettingLink', { ns: 'datasetCreation' })} + +
+ )} +
+ )} + + + + {/* Retrieval Method Config */} +
+ {!isModelAndRetrievalConfigDisabled + ? ( +
+
+ {t('form.retrievalSetting.title', { ns: 'datasetSettings' })} +
+
+ + {t('form.retrievalSetting.learnMore', { ns: 'datasetSettings' })} + + {t('form.retrievalSetting.longDescription', { ns: 'datasetSettings' })} +
+
+ ) + : ( +
+
{t('form.retrievalSetting.title', { ns: 'datasetSettings' })}
+
+ )} + +
+ {getIndexingTechnique() === IndexingType.QUALIFIED + ? ( + + ) + : ( + + )} +
+
+ + ) +} diff --git a/web/app/components/datasets/create/step-two/inputs.tsx b/web/app/components/datasets/create/step-two/components/inputs.tsx similarity index 100% rename from web/app/components/datasets/create/step-two/inputs.tsx rename to web/app/components/datasets/create/step-two/components/inputs.tsx diff --git a/web/app/components/datasets/create/step-two/option-card.tsx b/web/app/components/datasets/create/step-two/components/option-card.tsx similarity index 100% rename from web/app/components/datasets/create/step-two/option-card.tsx rename to web/app/components/datasets/create/step-two/components/option-card.tsx diff --git a/web/app/components/datasets/create/step-two/components/parent-child-options.tsx b/web/app/components/datasets/create/step-two/components/parent-child-options.tsx new file mode 100644 index 0000000000..e46aa5817b --- /dev/null +++ b/web/app/components/datasets/create/step-two/components/parent-child-options.tsx @@ -0,0 +1,191 @@ +'use client' + +import type { FC } from 'react' +import type { ParentChildConfig } from '../hooks' +import type { ParentMode, PreProcessingRule } from '@/models/datasets' +import { RiSearchEyeLine } from '@remixicon/react' +import Image from 'next/image' +import { useTranslation } from 'react-i18next' +import Button from '@/app/components/base/button' +import Checkbox from '@/app/components/base/checkbox' +import Divider from '@/app/components/base/divider' +import { ParentChildChunk } from '@/app/components/base/icons/src/vender/knowledge' +import RadioCard from '@/app/components/base/radio-card' +import { ChunkingMode } from '@/models/datasets' +import FileList from '../../assets/file-list-3-fill.svg' +import Note from '../../assets/note-mod.svg' +import BlueEffect from '../../assets/option-card-effect-blue.svg' +import s from '../index.module.css' +import { DelimiterInput, MaxLengthInput } from './inputs' +import { OptionCard } from './option-card' + +type TextLabelProps = { + children: React.ReactNode +} + +const TextLabel: FC = ({ children }) => { + return +} + +type ParentChildOptionsProps = { + // State + parentChildConfig: ParentChildConfig + rules: PreProcessingRule[] + currentDocForm: ChunkingMode + // Flags + isActive: boolean + isInUpload: boolean + isNotUploadInEmptyDataset: boolean + // Actions + onDocFormChange: (form: ChunkingMode) => void + onChunkForContextChange: (mode: ParentMode) => void + onParentDelimiterChange: (value: string) => void + onParentMaxLengthChange: (value: number) => void + onChildDelimiterChange: (value: string) => void + onChildMaxLengthChange: (value: number) => void + onRuleToggle: (id: string) => void + onPreview: () => void + onReset: () => void +} + +export const ParentChildOptions: FC = ({ + parentChildConfig, + rules, + currentDocForm: _currentDocForm, + isActive, + isInUpload, + isNotUploadInEmptyDataset, + onDocFormChange, + onChunkForContextChange, + onParentDelimiterChange, + onParentMaxLengthChange, + onChildDelimiterChange, + onChildMaxLengthChange, + onRuleToggle, + onPreview, + onReset, +}) => { + const { t } = useTranslation() + + const getRuleName = (key: string): string => { + const ruleNameMap: Record = { + remove_extra_spaces: t('stepTwo.removeExtraSpaces', { ns: 'datasetCreation' }), + remove_urls_emails: t('stepTwo.removeUrlEmails', { ns: 'datasetCreation' }), + remove_stopwords: t('stepTwo.removeStopwords', { ns: 'datasetCreation' }), + } + return ruleNameMap[key] ?? key + } + + return ( + } + effectImg={BlueEffect.src} + className="text-util-colors-blue-light-blue-light-500" + activeHeaderClassName="bg-dataset-option-card-blue-gradient" + description={t('stepTwo.parentChildTip', { ns: 'datasetCreation' })} + isActive={isActive} + onSwitched={() => onDocFormChange(ChunkingMode.parentChild)} + actions={( + <> + + + + )} + noHighlight={isInUpload && isNotUploadInEmptyDataset} + > +
+ {/* Parent chunk for context */} +
+
+
+ {t('stepTwo.parentChunkForContext', { ns: 'datasetCreation' })} +
+ +
+ } + title={t('stepTwo.paragraph', { ns: 'datasetCreation' })} + description={t('stepTwo.paragraphTip', { ns: 'datasetCreation' })} + isChosen={parentChildConfig.chunkForContext === 'paragraph'} + onChosen={() => onChunkForContextChange('paragraph')} + chosenConfig={( +
+ onParentDelimiterChange(e.target.value)} + /> + +
+ )} + /> + } + title={t('stepTwo.fullDoc', { ns: 'datasetCreation' })} + description={t('stepTwo.fullDocTip', { ns: 'datasetCreation' })} + onChosen={() => onChunkForContextChange('full-doc')} + isChosen={parentChildConfig.chunkForContext === 'full-doc'} + /> +
+ + {/* Child chunk for retrieval */} +
+
+
+ {t('stepTwo.childChunkForRetrieval', { ns: 'datasetCreation' })} +
+ +
+
+ onChildDelimiterChange(e.target.value)} + /> + +
+
+ + {/* Rules */} +
+
+
+ {t('stepTwo.rules', { ns: 'datasetCreation' })} +
+ +
+
+ {rules.map(rule => ( +
onRuleToggle(rule.id)} + > + + +
+ ))} +
+
+
+ + ) +} diff --git a/web/app/components/datasets/create/step-two/components/preview-panel.tsx b/web/app/components/datasets/create/step-two/components/preview-panel.tsx new file mode 100644 index 0000000000..4f25cee5bd --- /dev/null +++ b/web/app/components/datasets/create/step-two/components/preview-panel.tsx @@ -0,0 +1,171 @@ +'use client' + +import type { FC } from 'react' +import type { ParentChildConfig } from '../hooks' +import type { DataSourceType, FileIndexingEstimateResponse } from '@/models/datasets' +import { RiSearchEyeLine } from '@remixicon/react' +import { noop } from 'es-toolkit/function' +import { useTranslation } from 'react-i18next' +import Badge from '@/app/components/base/badge' +import FloatRightContainer from '@/app/components/base/float-right-container' +import { SkeletonContainer, SkeletonPoint, SkeletonRectangle, SkeletonRow } from '@/app/components/base/skeleton' +import { FULL_DOC_PREVIEW_LENGTH } from '@/config' +import { ChunkingMode } from '@/models/datasets' +import { cn } from '@/utils/classnames' +import { ChunkContainer, QAPreview } from '../../../chunk' +import PreviewDocumentPicker from '../../../common/document-picker/preview-document-picker' +import { PreviewSlice } from '../../../formatted-text/flavours/preview-slice' +import { FormattedText } from '../../../formatted-text/formatted' +import PreviewContainer from '../../../preview/container' +import { PreviewHeader } from '../../../preview/header' + +type PreviewPanelProps = { + // State + isMobile: boolean + dataSourceType: DataSourceType + currentDocForm: ChunkingMode + estimate?: FileIndexingEstimateResponse + parentChildConfig: ParentChildConfig + isSetting?: boolean + // Picker + pickerFiles: Array<{ id: string, name: string, extension: string }> + pickerValue: { id: string, name: string, extension: string } + // Mutation state + isIdle: boolean + isPending: boolean + // Actions + onPickerChange: (selected: { id: string, name: string }) => void +} + +export const PreviewPanel: FC = ({ + isMobile, + dataSourceType: _dataSourceType, + currentDocForm, + estimate, + parentChildConfig, + isSetting, + pickerFiles, + pickerValue, + isIdle, + isPending, + onPickerChange, +}) => { + const { t } = useTranslation() + + return ( + + +
+ >} + onChange={onPickerChange} + value={isSetting ? pickerFiles[0] : pickerValue} + /> + {currentDocForm !== ChunkingMode.qa && ( + + )} +
+ + )} + className={cn('relative flex h-full w-1/2 shrink-0 p-4 pr-0', isMobile && 'w-full max-w-[524px]')} + mainClassName="space-y-6" + > + {/* QA Preview */} + {currentDocForm === ChunkingMode.qa && estimate?.qa_preview && ( + estimate.qa_preview.map((item, index) => ( + + + + )) + )} + + {/* Text Preview */} + {currentDocForm === ChunkingMode.text && estimate?.preview && ( + estimate.preview.map((item, index) => ( + + {item.content} + + )) + )} + + {/* Parent-Child Preview */} + {currentDocForm === ChunkingMode.parentChild && estimate?.preview && ( + estimate.preview.map((item, index) => { + const indexForLabel = index + 1 + const childChunks = parentChildConfig.chunkForContext === 'full-doc' + ? item.child_chunks.slice(0, FULL_DOC_PREVIEW_LENGTH) + : item.child_chunks + return ( + + + {childChunks.map((child, childIndex) => { + const childIndexForLabel = childIndex + 1 + return ( + + ) + })} + + + ) + }) + )} + + {/* Idle State */} + {isIdle && ( +
+
+ +

+ {t('stepTwo.previewChunkTip', { ns: 'datasetCreation' })} +

+
+
+ )} + + {/* Loading State */} + {isPending && ( +
+ {Array.from({ length: 10 }, (_, i) => ( + + + + + + + + + + + ))} +
+ )} + + + ) +} diff --git a/web/app/components/datasets/create/step-two/components/step-two-footer.tsx b/web/app/components/datasets/create/step-two/components/step-two-footer.tsx new file mode 100644 index 0000000000..a22be64a75 --- /dev/null +++ b/web/app/components/datasets/create/step-two/components/step-two-footer.tsx @@ -0,0 +1,58 @@ +'use client' + +import type { FC } from 'react' +import { RiArrowLeftLine } from '@remixicon/react' +import { useTranslation } from 'react-i18next' +import Button from '@/app/components/base/button' + +type StepTwoFooterProps = { + isSetting?: boolean + isCreating: boolean + onPrevious: () => void + onCreate: () => void + onCancel?: () => void +} + +export const StepTwoFooter: FC = ({ + isSetting, + isCreating, + onPrevious, + onCreate, + onCancel, +}) => { + const { t } = useTranslation() + + if (!isSetting) { + return ( +
+ + +
+ ) + } + + return ( +
+ + +
+ ) +} diff --git a/web/app/components/datasets/create/step-two/escape.ts b/web/app/components/datasets/create/step-two/hooks/escape.ts similarity index 100% rename from web/app/components/datasets/create/step-two/escape.ts rename to web/app/components/datasets/create/step-two/hooks/escape.ts diff --git a/web/app/components/datasets/create/step-two/hooks/index.ts b/web/app/components/datasets/create/step-two/hooks/index.ts new file mode 100644 index 0000000000..f16daaaea5 --- /dev/null +++ b/web/app/components/datasets/create/step-two/hooks/index.ts @@ -0,0 +1,14 @@ +export { useDocumentCreation } from './use-document-creation' +export type { DocumentCreation, ValidationParams } from './use-document-creation' + +export { IndexingType, useIndexingConfig } from './use-indexing-config' +export type { IndexingConfig } from './use-indexing-config' + +export { useIndexingEstimate } from './use-indexing-estimate' +export type { IndexingEstimate } from './use-indexing-estimate' + +export { usePreviewState } from './use-preview-state' +export type { PreviewState } from './use-preview-state' + +export { DEFAULT_MAXIMUM_CHUNK_LENGTH, DEFAULT_OVERLAP, DEFAULT_SEGMENT_IDENTIFIER, defaultParentChildConfig, MAXIMUM_CHUNK_TOKEN_LENGTH, useSegmentationState } from './use-segmentation-state' +export type { ParentChildConfig, SegmentationState } from './use-segmentation-state' diff --git a/web/app/components/datasets/create/step-two/unescape.ts b/web/app/components/datasets/create/step-two/hooks/unescape.ts similarity index 100% rename from web/app/components/datasets/create/step-two/unescape.ts rename to web/app/components/datasets/create/step-two/hooks/unescape.ts diff --git a/web/app/components/datasets/create/step-two/hooks/use-document-creation.ts b/web/app/components/datasets/create/step-two/hooks/use-document-creation.ts new file mode 100644 index 0000000000..fd132b38ef --- /dev/null +++ b/web/app/components/datasets/create/step-two/hooks/use-document-creation.ts @@ -0,0 +1,279 @@ +import type { DefaultModel, Model } from '@/app/components/header/account-setting/model-provider-page/declarations' +import type { NotionPage } from '@/models/common' +import type { + ChunkingMode, + CrawlOptions, + CrawlResultItem, + CreateDocumentReq, + createDocumentResponse, + CustomFile, + FullDocumentDetail, + ProcessRule, +} from '@/models/datasets' +import type { RetrievalConfig, RETRIEVE_METHOD } from '@/types/app' +import { useCallback } from 'react' +import { useTranslation } from 'react-i18next' +import { trackEvent } from '@/app/components/base/amplitude' +import Toast from '@/app/components/base/toast' +import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model' +import { DataSourceProvider } from '@/models/common' +import { + DataSourceType, +} from '@/models/datasets' +import { getNotionInfo, getWebsiteInfo, useCreateDocument, useCreateFirstDocument } from '@/service/knowledge/use-create-dataset' +import { useInvalidDatasetList } from '@/service/knowledge/use-dataset' +import { IndexingType } from './use-indexing-config' +import { MAXIMUM_CHUNK_TOKEN_LENGTH } from './use-segmentation-state' + +export type UseDocumentCreationOptions = { + datasetId?: string + isSetting?: boolean + documentDetail?: FullDocumentDetail + dataSourceType: DataSourceType + files: CustomFile[] + notionPages: NotionPage[] + notionCredentialId: string + websitePages: CrawlResultItem[] + crawlOptions?: CrawlOptions + websiteCrawlProvider?: DataSourceProvider + websiteCrawlJobId?: string + // Callbacks + onStepChange?: (delta: number) => void + updateIndexingTypeCache?: (type: string) => void + updateResultCache?: (res: createDocumentResponse) => void + updateRetrievalMethodCache?: (method: RETRIEVE_METHOD | '') => void + onSave?: () => void + mutateDatasetRes?: () => void +} + +export type ValidationParams = { + segmentationType: string + maxChunkLength: number + limitMaxChunkLength: number + overlap: number + indexType: IndexingType + embeddingModel: DefaultModel + rerankModelList: Model[] + retrievalConfig: RetrievalConfig +} + +export const useDocumentCreation = (options: UseDocumentCreationOptions) => { + const { t } = useTranslation() + const { + datasetId, + isSetting, + documentDetail, + dataSourceType, + files, + notionPages, + notionCredentialId, + websitePages, + crawlOptions, + websiteCrawlProvider = DataSourceProvider.jinaReader, + websiteCrawlJobId = '', + onStepChange, + updateIndexingTypeCache, + updateResultCache, + updateRetrievalMethodCache, + onSave, + mutateDatasetRes, + } = options + + const createFirstDocumentMutation = useCreateFirstDocument() + const createDocumentMutation = useCreateDocument(datasetId!) + const invalidDatasetList = useInvalidDatasetList() + + const isCreating = createFirstDocumentMutation.isPending || createDocumentMutation.isPending + + // Validate creation params + const validateParams = useCallback((params: ValidationParams): boolean => { + const { + segmentationType, + maxChunkLength, + limitMaxChunkLength, + overlap, + indexType, + embeddingModel, + rerankModelList, + retrievalConfig, + } = params + + if (segmentationType === 'general' && overlap > maxChunkLength) { + Toast.notify({ type: 'error', message: t('stepTwo.overlapCheck', { ns: 'datasetCreation' }) }) + return false + } + + if (segmentationType === 'general' && maxChunkLength > limitMaxChunkLength) { + Toast.notify({ + type: 'error', + message: t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: limitMaxChunkLength }), + }) + return false + } + + if (!isSetting) { + if (indexType === IndexingType.QUALIFIED && (!embeddingModel.model || !embeddingModel.provider)) { + Toast.notify({ + type: 'error', + message: t('datasetConfig.embeddingModelRequired', { ns: 'appDebug' }), + }) + return false + } + + if (!isReRankModelSelected({ + rerankModelList, + retrievalConfig, + indexMethod: indexType, + })) { + Toast.notify({ type: 'error', message: t('datasetConfig.rerankModelRequired', { ns: 'appDebug' }) }) + return false + } + } + + return true + }, [t, isSetting]) + + // Build creation params + const buildCreationParams = useCallback(( + currentDocForm: ChunkingMode, + docLanguage: string, + processRule: ProcessRule, + retrievalConfig: RetrievalConfig, + embeddingModel: DefaultModel, + indexingTechnique: string, + ): CreateDocumentReq | null => { + if (isSetting) { + return { + original_document_id: documentDetail?.id, + doc_form: currentDocForm, + doc_language: docLanguage, + process_rule: processRule, + retrieval_model: retrievalConfig, + embedding_model: embeddingModel.model, + embedding_model_provider: embeddingModel.provider, + indexing_technique: indexingTechnique, + } as CreateDocumentReq + } + + const params: CreateDocumentReq = { + data_source: { + type: dataSourceType, + info_list: { + data_source_type: dataSourceType, + }, + }, + indexing_technique: indexingTechnique, + process_rule: processRule, + doc_form: currentDocForm, + doc_language: docLanguage, + retrieval_model: retrievalConfig, + embedding_model: embeddingModel.model, + embedding_model_provider: embeddingModel.provider, + } as CreateDocumentReq + + // Add data source specific info + if (dataSourceType === DataSourceType.FILE) { + params.data_source!.info_list.file_info_list = { + file_ids: files.map(file => file.id || '').filter(Boolean), + } + } + if (dataSourceType === DataSourceType.NOTION) + params.data_source!.info_list.notion_info_list = getNotionInfo(notionPages, notionCredentialId) + + if (dataSourceType === DataSourceType.WEB) { + params.data_source!.info_list.website_info_list = getWebsiteInfo({ + websiteCrawlProvider, + websiteCrawlJobId, + websitePages, + crawlOptions, + }) + } + + return params + }, [ + isSetting, + documentDetail, + dataSourceType, + files, + notionPages, + notionCredentialId, + websitePages, + websiteCrawlProvider, + websiteCrawlJobId, + crawlOptions, + ]) + + // Execute creation + const executeCreation = useCallback(async ( + params: CreateDocumentReq, + indexType: IndexingType, + retrievalConfig: RetrievalConfig, + ) => { + if (!datasetId) { + await createFirstDocumentMutation.mutateAsync(params, { + onSuccess(data) { + updateIndexingTypeCache?.(indexType) + updateResultCache?.(data) + updateRetrievalMethodCache?.(retrievalConfig.search_method as RETRIEVE_METHOD) + }, + }) + } + else { + await createDocumentMutation.mutateAsync(params, { + onSuccess(data) { + updateIndexingTypeCache?.(indexType) + updateResultCache?.(data) + updateRetrievalMethodCache?.(retrievalConfig.search_method as RETRIEVE_METHOD) + }, + }) + } + + mutateDatasetRes?.() + invalidDatasetList() + + trackEvent('create_datasets', { + data_source_type: dataSourceType, + indexing_technique: indexType, + }) + + onStepChange?.(+1) + + if (isSetting) + onSave?.() + }, [ + datasetId, + createFirstDocumentMutation, + createDocumentMutation, + updateIndexingTypeCache, + updateResultCache, + updateRetrievalMethodCache, + mutateDatasetRes, + invalidDatasetList, + dataSourceType, + onStepChange, + isSetting, + onSave, + ]) + + // Validate preview params + const validatePreviewParams = useCallback((maxChunkLength: number): boolean => { + if (maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH) { + Toast.notify({ + type: 'error', + message: t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: MAXIMUM_CHUNK_TOKEN_LENGTH }), + }) + return false + } + return true + }, [t]) + + return { + isCreating, + validateParams, + buildCreationParams, + executeCreation, + validatePreviewParams, + } +} + +export type DocumentCreation = ReturnType diff --git a/web/app/components/datasets/create/step-two/hooks/use-indexing-config.ts b/web/app/components/datasets/create/step-two/hooks/use-indexing-config.ts new file mode 100644 index 0000000000..97fc9c260f --- /dev/null +++ b/web/app/components/datasets/create/step-two/hooks/use-indexing-config.ts @@ -0,0 +1,143 @@ +import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations' +import type { RetrievalConfig } from '@/types/app' +import { useEffect, useMemo, useState } from 'react' +import { checkShowMultiModalTip } from '@/app/components/datasets/settings/utils' +import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { useDefaultModel, useModelList, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' +import { RETRIEVE_METHOD } from '@/types/app' + +export enum IndexingType { + QUALIFIED = 'high_quality', + ECONOMICAL = 'economy', +} + +const DEFAULT_RETRIEVAL_CONFIG: RetrievalConfig = { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { + reranking_provider_name: '', + reranking_model_name: '', + }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, +} + +export type UseIndexingConfigOptions = { + initialIndexType?: IndexingType + initialEmbeddingModel?: DefaultModel + initialRetrievalConfig?: RetrievalConfig + isAPIKeySet: boolean + hasSetIndexType: boolean +} + +export const useIndexingConfig = (options: UseIndexingConfigOptions) => { + const { + initialIndexType, + initialEmbeddingModel, + initialRetrievalConfig, + isAPIKeySet, + hasSetIndexType, + } = options + + // Rerank model + const { + modelList: rerankModelList, + defaultModel: rerankDefaultModel, + currentModel: isRerankDefaultModelValid, + } = useModelListAndDefaultModelAndCurrentProviderAndModel(ModelTypeEnum.rerank) + + // Embedding model list + const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding) + const { data: defaultEmbeddingModel } = useDefaultModel(ModelTypeEnum.textEmbedding) + + // Index type state + const [indexType, setIndexType] = useState(() => { + if (initialIndexType) + return initialIndexType + return isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL + }) + + // Embedding model state + const [embeddingModel, setEmbeddingModel] = useState( + initialEmbeddingModel ?? { + provider: defaultEmbeddingModel?.provider.provider || '', + model: defaultEmbeddingModel?.model || '', + }, + ) + + // Retrieval config state + const [retrievalConfig, setRetrievalConfig] = useState( + initialRetrievalConfig ?? DEFAULT_RETRIEVAL_CONFIG, + ) + + // Sync retrieval config with rerank model when available + useEffect(() => { + if (initialRetrievalConfig) + return + + setRetrievalConfig({ + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: !!isRerankDefaultModelValid, + reranking_model: { + reranking_provider_name: isRerankDefaultModelValid ? rerankDefaultModel?.provider.provider ?? '' : '', + reranking_model_name: isRerankDefaultModelValid ? rerankDefaultModel?.model ?? '' : '', + }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }) + }, [rerankDefaultModel, isRerankDefaultModelValid, initialRetrievalConfig]) + + // Sync index type with props + useEffect(() => { + if (initialIndexType) + setIndexType(initialIndexType) + else + setIndexType(isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL) + }, [isAPIKeySet, initialIndexType]) + + // Show multimodal tip + const showMultiModalTip = useMemo(() => { + return checkShowMultiModalTip({ + embeddingModel, + rerankingEnable: retrievalConfig.reranking_enable, + rerankModel: { + rerankingProviderName: retrievalConfig.reranking_model.reranking_provider_name, + rerankingModelName: retrievalConfig.reranking_model.reranking_model_name, + }, + indexMethod: indexType, + embeddingModelList, + rerankModelList, + }) + }, [embeddingModel, retrievalConfig, indexType, embeddingModelList, rerankModelList]) + + // Get effective indexing technique + const getIndexingTechnique = () => initialIndexType || indexType + + return { + // Index type + indexType, + setIndexType, + hasSetIndexType, + getIndexingTechnique, + + // Embedding model + embeddingModel, + setEmbeddingModel, + embeddingModelList, + defaultEmbeddingModel, + + // Retrieval config + retrievalConfig, + setRetrievalConfig, + rerankModelList, + rerankDefaultModel, + isRerankDefaultModelValid, + + // Computed + showMultiModalTip, + } +} + +export type IndexingConfig = ReturnType diff --git a/web/app/components/datasets/create/step-two/hooks/use-indexing-estimate.ts b/web/app/components/datasets/create/step-two/hooks/use-indexing-estimate.ts new file mode 100644 index 0000000000..cc5a2bcf33 --- /dev/null +++ b/web/app/components/datasets/create/step-two/hooks/use-indexing-estimate.ts @@ -0,0 +1,123 @@ +import type { IndexingType } from './use-indexing-config' +import type { NotionPage } from '@/models/common' +import type { ChunkingMode, CrawlOptions, CrawlResultItem, CustomFile, ProcessRule } from '@/models/datasets' +import { useCallback } from 'react' +import { DataSourceProvider } from '@/models/common' +import { DataSourceType } from '@/models/datasets' +import { + useFetchFileIndexingEstimateForFile, + useFetchFileIndexingEstimateForNotion, + useFetchFileIndexingEstimateForWeb, +} from '@/service/knowledge/use-create-dataset' + +export type UseIndexingEstimateOptions = { + dataSourceType: DataSourceType + datasetId?: string + // Document settings + currentDocForm: ChunkingMode + docLanguage: string + // File data source + files: CustomFile[] + previewFileName?: string + // Notion data source + previewNotionPage: NotionPage + notionCredentialId: string + // Website data source + previewWebsitePage: CrawlResultItem + crawlOptions?: CrawlOptions + websiteCrawlProvider?: DataSourceProvider + websiteCrawlJobId?: string + // Processing + indexingTechnique: IndexingType + processRule: ProcessRule +} + +export const useIndexingEstimate = (options: UseIndexingEstimateOptions) => { + const { + dataSourceType, + datasetId, + currentDocForm, + docLanguage, + files, + previewFileName, + previewNotionPage, + notionCredentialId, + previewWebsitePage, + crawlOptions, + websiteCrawlProvider, + websiteCrawlJobId, + indexingTechnique, + processRule, + } = options + + // File indexing estimate + const fileQuery = useFetchFileIndexingEstimateForFile({ + docForm: currentDocForm, + docLanguage, + dataSourceType: DataSourceType.FILE, + files: previewFileName + ? [files.find(file => file.name === previewFileName)!] + : files, + indexingTechnique, + processRule, + dataset_id: datasetId!, + }) + + // Notion indexing estimate + const notionQuery = useFetchFileIndexingEstimateForNotion({ + docForm: currentDocForm, + docLanguage, + dataSourceType: DataSourceType.NOTION, + notionPages: [previewNotionPage], + indexingTechnique, + processRule, + dataset_id: datasetId || '', + credential_id: notionCredentialId, + }) + + // Website indexing estimate + const websiteQuery = useFetchFileIndexingEstimateForWeb({ + docForm: currentDocForm, + docLanguage, + dataSourceType: DataSourceType.WEB, + websitePages: [previewWebsitePage], + crawlOptions, + websiteCrawlProvider: websiteCrawlProvider ?? DataSourceProvider.jinaReader, + websiteCrawlJobId: websiteCrawlJobId ?? '', + indexingTechnique, + processRule, + dataset_id: datasetId || '', + }) + + // Get current mutation based on data source type + const getCurrentMutation = useCallback(() => { + if (dataSourceType === DataSourceType.FILE) + return fileQuery + if (dataSourceType === DataSourceType.NOTION) + return notionQuery + return websiteQuery + }, [dataSourceType, fileQuery, notionQuery, websiteQuery]) + + const currentMutation = getCurrentMutation() + + // Trigger estimate fetch + const fetchEstimate = useCallback(() => { + if (dataSourceType === DataSourceType.FILE) + fileQuery.mutate() + else if (dataSourceType === DataSourceType.NOTION) + notionQuery.mutate() + else + websiteQuery.mutate() + }, [dataSourceType, fileQuery, notionQuery, websiteQuery]) + + return { + currentMutation, + estimate: currentMutation.data, + isIdle: currentMutation.isIdle, + isPending: currentMutation.isPending, + fetchEstimate, + reset: currentMutation.reset, + } +} + +export type IndexingEstimate = ReturnType diff --git a/web/app/components/datasets/create/step-two/hooks/use-preview-state.ts b/web/app/components/datasets/create/step-two/hooks/use-preview-state.ts new file mode 100644 index 0000000000..94171c5947 --- /dev/null +++ b/web/app/components/datasets/create/step-two/hooks/use-preview-state.ts @@ -0,0 +1,127 @@ +import type { NotionPage } from '@/models/common' +import type { CrawlResultItem, CustomFile, DocumentItem, FullDocumentDetail } from '@/models/datasets' +import { useCallback, useState } from 'react' +import { DataSourceType } from '@/models/datasets' + +export type UsePreviewStateOptions = { + dataSourceType: DataSourceType + files: CustomFile[] + notionPages: NotionPage[] + websitePages: CrawlResultItem[] + documentDetail?: FullDocumentDetail + datasetId?: string +} + +export const usePreviewState = (options: UsePreviewStateOptions) => { + const { + dataSourceType, + files, + notionPages, + websitePages, + documentDetail, + datasetId, + } = options + + // File preview state + const [previewFile, setPreviewFile] = useState( + (datasetId && documentDetail) + ? documentDetail.file + : files[0], + ) + + // Notion page preview state + const [previewNotionPage, setPreviewNotionPage] = useState( + (datasetId && documentDetail) + ? documentDetail.notion_page + : notionPages[0], + ) + + // Website page preview state + const [previewWebsitePage, setPreviewWebsitePage] = useState( + (datasetId && documentDetail) + ? documentDetail.website_page + : websitePages[0], + ) + + // Get preview items for document picker based on data source type + const getPreviewPickerItems = useCallback(() => { + if (dataSourceType === DataSourceType.FILE) { + return files as Array> + } + if (dataSourceType === DataSourceType.NOTION) { + return notionPages.map(page => ({ + id: page.page_id, + name: page.page_name, + extension: 'md', + })) + } + if (dataSourceType === DataSourceType.WEB) { + return websitePages.map(page => ({ + id: page.source_url, + name: page.title, + extension: 'md', + })) + } + return [] + }, [dataSourceType, files, notionPages, websitePages]) + + // Get current preview value for picker + const getPreviewPickerValue = useCallback(() => { + if (dataSourceType === DataSourceType.FILE) { + return previewFile as Required + } + if (dataSourceType === DataSourceType.NOTION) { + return { + id: previewNotionPage?.page_id || '', + name: previewNotionPage?.page_name || '', + extension: 'md', + } + } + if (dataSourceType === DataSourceType.WEB) { + return { + id: previewWebsitePage?.source_url || '', + name: previewWebsitePage?.title || '', + extension: 'md', + } + } + return { id: '', name: '', extension: '' } + }, [dataSourceType, previewFile, previewNotionPage, previewWebsitePage]) + + // Handle preview change + const handlePreviewChange = useCallback((selected: { id: string, name: string }) => { + if (dataSourceType === DataSourceType.FILE) { + setPreviewFile(selected as DocumentItem) + } + else if (dataSourceType === DataSourceType.NOTION) { + const selectedPage = notionPages.find(page => page.page_id === selected.id) + if (selectedPage) + setPreviewNotionPage(selectedPage) + } + else if (dataSourceType === DataSourceType.WEB) { + const selectedPage = websitePages.find(page => page.source_url === selected.id) + if (selectedPage) + setPreviewWebsitePage(selectedPage) + } + }, [dataSourceType, notionPages, websitePages]) + + return { + // File preview + previewFile, + setPreviewFile, + + // Notion preview + previewNotionPage, + setPreviewNotionPage, + + // Website preview + previewWebsitePage, + setPreviewWebsitePage, + + // Picker helpers + getPreviewPickerItems, + getPreviewPickerValue, + handlePreviewChange, + } +} + +export type PreviewState = ReturnType diff --git a/web/app/components/datasets/create/step-two/hooks/use-segmentation-state.ts b/web/app/components/datasets/create/step-two/hooks/use-segmentation-state.ts new file mode 100644 index 0000000000..69cc089b4f --- /dev/null +++ b/web/app/components/datasets/create/step-two/hooks/use-segmentation-state.ts @@ -0,0 +1,222 @@ +import type { ParentMode, PreProcessingRule, ProcessRule, Rules } from '@/models/datasets' +import { useCallback, useState } from 'react' +import { ChunkingMode, ProcessMode } from '@/models/datasets' +import escape from './escape' +import unescape from './unescape' + +// Constants +export const DEFAULT_SEGMENT_IDENTIFIER = '\\n\\n' +export const DEFAULT_MAXIMUM_CHUNK_LENGTH = 1024 +export const DEFAULT_OVERLAP = 50 +export const MAXIMUM_CHUNK_TOKEN_LENGTH = Number.parseInt( + globalThis.document?.body?.getAttribute('data-public-indexing-max-segmentation-tokens-length') || '4000', + 10, +) + +export type ParentChildConfig = { + chunkForContext: ParentMode + parent: { + delimiter: string + maxLength: number + } + child: { + delimiter: string + maxLength: number + } +} + +export const defaultParentChildConfig: ParentChildConfig = { + chunkForContext: 'paragraph', + parent: { + delimiter: '\\n\\n', + maxLength: 1024, + }, + child: { + delimiter: '\\n', + maxLength: 512, + }, +} + +export type UseSegmentationStateOptions = { + initialSegmentationType?: ProcessMode +} + +export const useSegmentationState = (options: UseSegmentationStateOptions = {}) => { + const { initialSegmentationType } = options + + // Segmentation type (general or parent-child) + const [segmentationType, setSegmentationType] = useState( + initialSegmentationType ?? ProcessMode.general, + ) + + // General chunking settings + const [segmentIdentifier, doSetSegmentIdentifier] = useState(DEFAULT_SEGMENT_IDENTIFIER) + const [maxChunkLength, setMaxChunkLength] = useState(DEFAULT_MAXIMUM_CHUNK_LENGTH) + const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(MAXIMUM_CHUNK_TOKEN_LENGTH) + const [overlap, setOverlap] = useState(DEFAULT_OVERLAP) + + // Pre-processing rules + const [rules, setRules] = useState([]) + const [defaultConfig, setDefaultConfig] = useState() + + // Parent-child config + const [parentChildConfig, setParentChildConfig] = useState(defaultParentChildConfig) + + // Escaped segment identifier setter + const setSegmentIdentifier = useCallback((value: string, canEmpty?: boolean) => { + if (value) { + doSetSegmentIdentifier(escape(value)) + } + else { + doSetSegmentIdentifier(canEmpty ? '' : DEFAULT_SEGMENT_IDENTIFIER) + } + }, []) + + // Rule toggle handler + const toggleRule = useCallback((id: string) => { + setRules(prev => prev.map(rule => + rule.id === id ? { ...rule, enabled: !rule.enabled } : rule, + )) + }, []) + + // Reset to defaults + const resetToDefaults = useCallback(() => { + if (defaultConfig) { + setSegmentIdentifier(defaultConfig.segmentation.separator) + setMaxChunkLength(defaultConfig.segmentation.max_tokens) + setOverlap(defaultConfig.segmentation.chunk_overlap!) + setRules(defaultConfig.pre_processing_rules) + } + setParentChildConfig(defaultParentChildConfig) + }, [defaultConfig, setSegmentIdentifier]) + + // Apply config from document detail + const applyConfigFromRules = useCallback((rulesConfig: Rules, isHierarchical: boolean) => { + const separator = rulesConfig.segmentation.separator + const max = rulesConfig.segmentation.max_tokens + const chunkOverlap = rulesConfig.segmentation.chunk_overlap + + setSegmentIdentifier(separator) + setMaxChunkLength(max) + setOverlap(chunkOverlap!) + setRules(rulesConfig.pre_processing_rules) + setDefaultConfig(rulesConfig) + + if (isHierarchical) { + setParentChildConfig({ + chunkForContext: rulesConfig.parent_mode || 'paragraph', + parent: { + delimiter: escape(rulesConfig.segmentation.separator), + maxLength: rulesConfig.segmentation.max_tokens, + }, + child: { + delimiter: escape(rulesConfig.subchunk_segmentation!.separator), + maxLength: rulesConfig.subchunk_segmentation!.max_tokens, + }, + }) + } + }, [setSegmentIdentifier]) + + // Get process rule for API + const getProcessRule = useCallback((docForm: ChunkingMode): ProcessRule => { + if (docForm === ChunkingMode.parentChild) { + return { + rules: { + pre_processing_rules: rules, + segmentation: { + separator: unescape(parentChildConfig.parent.delimiter), + max_tokens: parentChildConfig.parent.maxLength, + }, + parent_mode: parentChildConfig.chunkForContext, + subchunk_segmentation: { + separator: unescape(parentChildConfig.child.delimiter), + max_tokens: parentChildConfig.child.maxLength, + }, + }, + mode: 'hierarchical', + } as ProcessRule + } + + return { + rules: { + pre_processing_rules: rules, + segmentation: { + separator: unescape(segmentIdentifier), + max_tokens: maxChunkLength, + chunk_overlap: overlap, + }, + }, + mode: segmentationType, + } as ProcessRule + }, [rules, parentChildConfig, segmentIdentifier, maxChunkLength, overlap, segmentationType]) + + // Update parent config field + const updateParentConfig = useCallback((field: 'delimiter' | 'maxLength', value: string | number) => { + setParentChildConfig((prev) => { + let newValue: string | number + if (field === 'delimiter') + newValue = value ? escape(value as string) : '' + else + newValue = value + return { + ...prev, + parent: { ...prev.parent, [field]: newValue }, + } + }) + }, []) + + // Update child config field + const updateChildConfig = useCallback((field: 'delimiter' | 'maxLength', value: string | number) => { + setParentChildConfig((prev) => { + let newValue: string | number + if (field === 'delimiter') + newValue = value ? escape(value as string) : '' + else + newValue = value + return { + ...prev, + child: { ...prev.child, [field]: newValue }, + } + }) + }, []) + + // Set chunk for context mode + const setChunkForContext = useCallback((mode: ParentMode) => { + setParentChildConfig(prev => ({ ...prev, chunkForContext: mode })) + }, []) + + return { + // General chunking state + segmentationType, + setSegmentationType, + segmentIdentifier, + setSegmentIdentifier, + maxChunkLength, + setMaxChunkLength, + limitMaxChunkLength, + setLimitMaxChunkLength, + overlap, + setOverlap, + + // Rules + rules, + setRules, + defaultConfig, + setDefaultConfig, + toggleRule, + + // Parent-child config + parentChildConfig, + setParentChildConfig, + updateParentConfig, + updateChildConfig, + setChunkForContext, + + // Actions + resetToDefaults, + applyConfigFromRules, + getProcessRule, + } +} + +export type SegmentationState = ReturnType diff --git a/web/app/components/datasets/create/step-two/index.spec.tsx b/web/app/components/datasets/create/step-two/index.spec.tsx new file mode 100644 index 0000000000..7145920f60 --- /dev/null +++ b/web/app/components/datasets/create/step-two/index.spec.tsx @@ -0,0 +1,2197 @@ +import type { Model } from '@/app/components/header/account-setting/model-provider-page/declarations' +import type { DataSourceProvider, NotionPage } from '@/models/common' +import type { + CrawlOptions, + CrawlResultItem, + CustomFile, + FileIndexingEstimateResponse, + FullDocumentDetail, + PreProcessingRule, + Rules, +} from '@/models/datasets' +import type { RetrievalConfig } from '@/types/app' +import { act, fireEvent, render, renderHook, screen } from '@testing-library/react' +import { ConfigurationMethodEnum, ModelStatusEnum, ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { ChunkingMode, DataSourceType, ProcessMode } from '@/models/datasets' +import { RETRIEVE_METHOD } from '@/types/app' +import { PreviewPanel } from './components/preview-panel' +import { StepTwoFooter } from './components/step-two-footer' +import { + DEFAULT_MAXIMUM_CHUNK_LENGTH, + DEFAULT_OVERLAP, + DEFAULT_SEGMENT_IDENTIFIER, + defaultParentChildConfig, + IndexingType, + useDocumentCreation, + useIndexingConfig, + useIndexingEstimate, + usePreviewState, + useSegmentationState, +} from './hooks' +import escape from './hooks/escape' +import unescape from './hooks/unescape' + +// ============================================ +// Mock external dependencies +// ============================================ + +// Mock dataset detail context +const mockDataset = { + id: 'test-dataset-id', + doc_form: ChunkingMode.text, + data_source_type: DataSourceType.FILE, + embedding_model: 'text-embedding-ada-002', + embedding_model_provider: 'openai', + retrieval_model_dict: { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + } as RetrievalConfig, +} + +let mockCurrentDataset: typeof mockDataset | null = null +const mockMutateDatasetRes = vi.fn() + +vi.mock('@/context/dataset-detail', () => ({ + useDatasetDetailContextWithSelector: (selector: (state: { dataset: typeof mockDataset | null, mutateDatasetRes: () => void }) => unknown) => + selector({ dataset: mockCurrentDataset, mutateDatasetRes: mockMutateDatasetRes }), +})) + +// Note: @/context/i18n is globally mocked in vitest.setup.ts, no need to mock here +// Note: @/hooks/use-breakpoints uses real import + +// Mock model hooks +const mockEmbeddingModelList = [ + { provider: 'openai', model: 'text-embedding-ada-002' }, + { provider: 'cohere', model: 'embed-english-v3.0' }, +] +const mockDefaultEmbeddingModel = { provider: { provider: 'openai' }, model: 'text-embedding-ada-002' } +// Model[] type structure for rerank model list (simplified mock) +const mockRerankModelList: Model[] = [{ + provider: 'cohere', + icon_small: { en_US: 'cohere-icon', zh_Hans: 'cohere-icon' }, + label: { en_US: 'Cohere', zh_Hans: 'Cohere' }, + models: [{ + model: 'rerank-english-v3.0', + label: { en_US: 'Rerank English v3.0', zh_Hans: 'Rerank English v3.0' }, + model_type: ModelTypeEnum.rerank, + features: [], + fetch_from: ConfigurationMethodEnum.predefinedModel, + status: ModelStatusEnum.active, + model_properties: {}, + load_balancing_enabled: false, + }], + status: ModelStatusEnum.active, +}] +const mockRerankDefaultModel = { provider: { provider: 'cohere' }, model: 'rerank-english-v3.0' } +let mockIsRerankDefaultModelValid = true + +vi.mock('@/app/components/header/account-setting/model-provider-page/hooks', () => ({ + useModelListAndDefaultModelAndCurrentProviderAndModel: () => ({ + modelList: mockRerankModelList, + defaultModel: mockRerankDefaultModel, + currentModel: mockIsRerankDefaultModelValid, + }), + useModelList: () => ({ data: mockEmbeddingModelList }), + useDefaultModel: () => ({ data: mockDefaultEmbeddingModel }), +})) + +// Mock service hooks +const mockFetchDefaultProcessRuleMutate = vi.fn() +vi.mock('@/service/knowledge/use-create-dataset', () => ({ + useFetchDefaultProcessRule: ({ onSuccess }: { onSuccess: (data: { rules: Rules, limits: { indexing_max_segmentation_tokens_length: number } }) => void }) => ({ + mutate: (url: string) => { + mockFetchDefaultProcessRuleMutate(url) + onSuccess({ + rules: { + segmentation: { separator: '\\n', max_tokens: 500, chunk_overlap: 50 }, + pre_processing_rules: [ + { id: 'remove_extra_spaces', enabled: true }, + { id: 'remove_urls_emails', enabled: false }, + ], + parent_mode: 'paragraph', + subchunk_segmentation: { separator: '\\n', max_tokens: 256 }, + }, + limits: { indexing_max_segmentation_tokens_length: 4000 }, + }) + }, + isPending: false, + }), + useFetchFileIndexingEstimateForFile: () => ({ + mutate: vi.fn(), + data: undefined, + isIdle: true, + isPending: false, + reset: vi.fn(), + }), + useFetchFileIndexingEstimateForNotion: () => ({ + mutate: vi.fn(), + data: undefined, + isIdle: true, + isPending: false, + reset: vi.fn(), + }), + useFetchFileIndexingEstimateForWeb: () => ({ + mutate: vi.fn(), + data: undefined, + isIdle: true, + isPending: false, + reset: vi.fn(), + }), + useCreateFirstDocument: () => ({ + mutateAsync: vi.fn().mockImplementation(async (params: unknown, options?: { onSuccess?: (data: unknown) => void }) => { + const data = { dataset: { id: 'new-dataset-id' } } + options?.onSuccess?.(data) + return data + }), + isPending: false, + }), + useCreateDocument: () => ({ + mutateAsync: vi.fn().mockImplementation(async (params: unknown, options?: { onSuccess?: (data: unknown) => void }) => { + const data = { document: { id: 'new-doc-id' } } + options?.onSuccess?.(data) + return data + }), + isPending: false, + }), + getNotionInfo: vi.fn().mockReturnValue([{ workspace_id: 'ws-1', pages: [{ page_id: 'page-1' }] }]), + getWebsiteInfo: vi.fn().mockReturnValue({ provider: 'jinaReader', job_id: 'job-123', urls: ['https://test.com'] }), +})) + +vi.mock('@/service/knowledge/use-dataset', () => ({ + useInvalidDatasetList: () => vi.fn(), +})) + +// Mock amplitude tracking (external service) +vi.mock('@/app/components/base/amplitude', () => ({ + trackEvent: vi.fn(), +})) + +// Note: @/app/components/base/toast - uses real import (base component) +// Note: @/app/components/datasets/common/check-rerank-model - uses real import +// Note: @/app/components/base/float-right-container - uses real import (base component) + +// Mock checkShowMultiModalTip - requires complex model list structure +vi.mock('@/app/components/datasets/settings/utils', () => ({ + checkShowMultiModalTip: () => false, +})) + +// ============================================ +// Test data factories +// ============================================ + +const createMockFile = (overrides?: Partial): CustomFile => ({ + id: 'file-1', + name: 'test-file.pdf', + extension: 'pdf', + size: 1024, + type: 'application/pdf', + lastModified: Date.now(), + ...overrides, +} as CustomFile) + +const createMockNotionPage = (overrides?: Partial): NotionPage => ({ + page_id: 'notion-page-1', + page_name: 'Test Notion Page', + page_icon: null, + type: 'page', + ...overrides, +} as NotionPage) + +const createMockWebsitePage = (overrides?: Partial): CrawlResultItem => ({ + source_url: 'https://example.com/page1', + title: 'Test Website Page', + description: 'Test description', + markdown: '# Test Content', + ...overrides, +} as CrawlResultItem) + +const createMockDocumentDetail = (overrides?: Partial): FullDocumentDetail => ({ + id: 'doc-1', + doc_form: ChunkingMode.text, + doc_language: 'English', + file: { id: 'file-1', name: 'test.pdf', extension: 'pdf' }, + notion_page: createMockNotionPage(), + website_page: createMockWebsitePage(), + dataset_process_rule: { + mode: ProcessMode.general, + rules: { + segmentation: { separator: '\\n\\n', max_tokens: 1024, chunk_overlap: 50 }, + pre_processing_rules: [{ id: 'remove_extra_spaces', enabled: true }], + }, + }, + ...overrides, +} as FullDocumentDetail) + +const createMockRules = (overrides?: Partial): Rules => ({ + segmentation: { separator: '\\n\\n', max_tokens: 1024, chunk_overlap: 50 }, + pre_processing_rules: [ + { id: 'remove_extra_spaces', enabled: true }, + { id: 'remove_urls_emails', enabled: false }, + ], + parent_mode: 'paragraph', + subchunk_segmentation: { separator: '\\n', max_tokens: 512 }, + ...overrides, +}) + +const createMockEstimate = (overrides?: Partial): FileIndexingEstimateResponse => ({ + total_segments: 10, + total_nodes: 10, + tokens: 5000, + total_price: 0.01, + currency: 'USD', + qa_preview: [{ question: 'Q1', answer: 'A1' }], + preview: [{ content: 'Chunk 1 content', child_chunks: ['Child 1', 'Child 2'] }], + ...overrides, +}) + +// ============================================ +// Utility Functions Tests (escape/unescape) +// ============================================ + +describe('escape utility', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + // Tests for escape function + describe('escape function', () => { + it('should return empty string for null/undefined input', () => { + expect(escape(null as unknown as string)).toBe('') + expect(escape(undefined as unknown as string)).toBe('') + expect(escape('')).toBe('') + }) + + it('should escape newline characters', () => { + expect(escape('\n')).toBe('\\n') + expect(escape('\r')).toBe('\\r') + expect(escape('\n\r')).toBe('\\n\\r') + }) + + it('should escape tab characters', () => { + expect(escape('\t')).toBe('\\t') + }) + + it('should escape other special characters', () => { + expect(escape('\0')).toBe('\\0') + expect(escape('\b')).toBe('\\b') + expect(escape('\f')).toBe('\\f') + expect(escape('\v')).toBe('\\v') + }) + + it('should escape single quotes', () => { + expect(escape('\'')).toBe('\\\'') + }) + + it('should handle mixed content', () => { + expect(escape('Hello\nWorld\t!')).toBe('Hello\\nWorld\\t!') + }) + + it('should not escape regular characters', () => { + expect(escape('Hello World')).toBe('Hello World') + expect(escape('abc123')).toBe('abc123') + }) + + it('should return empty string for non-string input', () => { + expect(escape(123 as unknown as string)).toBe('') + expect(escape({} as unknown as string)).toBe('') + }) + }) +}) + +describe('unescape utility', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + // Tests for unescape function + describe('unescape function', () => { + it('should unescape newline characters', () => { + expect(unescape('\\n')).toBe('\n') + expect(unescape('\\r')).toBe('\r') + }) + + it('should unescape tab characters', () => { + expect(unescape('\\t')).toBe('\t') + }) + + it('should unescape other special characters', () => { + expect(unescape('\\0')).toBe('\0') + expect(unescape('\\b')).toBe('\b') + expect(unescape('\\f')).toBe('\f') + expect(unescape('\\v')).toBe('\v') + }) + + it('should unescape single and double quotes', () => { + expect(unescape('\\\'')).toBe('\'') + expect(unescape('\\"')).toBe('"') + }) + + it('should unescape backslash', () => { + expect(unescape('\\\\')).toBe('\\') + }) + + it('should unescape hex sequences', () => { + expect(unescape('\\x41')).toBe('A') // 0x41 = 65 = 'A' + expect(unescape('\\x5A')).toBe('Z') // 0x5A = 90 = 'Z' + }) + + it('should unescape short hex (2-digit) sequences', () => { + // Short hex format: \xNN (2 hexadecimal digits) + expect(unescape('\\xA5')).toBe('¥') // Yen sign + expect(unescape('\\x7F')).toBe('\x7F') // Delete character + expect(unescape('\\x00')).toBe('\x00') // Null character via hex + }) + + it('should unescape octal sequences', () => { + expect(unescape('\\101')).toBe('A') // Octal 101 = 65 = 'A' + expect(unescape('\\132')).toBe('Z') // Octal 132 = 90 = 'Z' + expect(unescape('\\7')).toBe('\x07') // Single digit octal + }) + + it('should unescape unicode sequences', () => { + expect(unescape('\\u0041')).toBe('A') + expect(unescape('\\u{41}')).toBe('A') + }) + + it('should unescape Python-style unicode', () => { + expect(unescape('\\U00000041')).toBe('A') + }) + + it('should handle mixed content', () => { + expect(unescape('Hello\\nWorld\\t!')).toBe('Hello\nWorld\t!') + }) + + it('should not modify regular text', () => { + expect(unescape('Hello World')).toBe('Hello World') + }) + }) +}) + +// ============================================ +// useSegmentationState Hook Tests +// ============================================ + +describe('useSegmentationState', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + // Tests for initial state + describe('Initial State', () => { + it('should initialize with default values', () => { + const { result } = renderHook(() => useSegmentationState()) + + expect(result.current.segmentationType).toBe(ProcessMode.general) + expect(result.current.segmentIdentifier).toBe(DEFAULT_SEGMENT_IDENTIFIER) + expect(result.current.maxChunkLength).toBe(DEFAULT_MAXIMUM_CHUNK_LENGTH) + expect(result.current.overlap).toBe(DEFAULT_OVERLAP) + expect(result.current.rules).toEqual([]) + expect(result.current.parentChildConfig).toEqual(defaultParentChildConfig) + }) + + it('should initialize with custom segmentation type', () => { + const { result } = renderHook(() => + useSegmentationState({ initialSegmentationType: ProcessMode.parentChild }), + ) + + expect(result.current.segmentationType).toBe(ProcessMode.parentChild) + }) + }) + + // Tests for state setters + describe('State Management', () => { + it('should update segmentation type', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setSegmentationType(ProcessMode.parentChild) + }) + + expect(result.current.segmentationType).toBe(ProcessMode.parentChild) + }) + + it('should update max chunk length', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setMaxChunkLength(2048) + }) + + expect(result.current.maxChunkLength).toBe(2048) + }) + + it('should update overlap', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setOverlap(100) + }) + + expect(result.current.overlap).toBe(100) + }) + + it('should update rules', () => { + const { result } = renderHook(() => useSegmentationState()) + const newRules: PreProcessingRule[] = [{ id: 'test', enabled: true }] + + act(() => { + result.current.setRules(newRules) + }) + + expect(result.current.rules).toEqual(newRules) + }) + }) + + // Tests for setSegmentIdentifier with escape + describe('setSegmentIdentifier', () => { + it('should escape special characters', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setSegmentIdentifier('\n\n') + }) + + expect(result.current.segmentIdentifier).toBe('\\n\\n') + }) + + it('should use default when empty and canEmpty is false', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setSegmentIdentifier('') + }) + + expect(result.current.segmentIdentifier).toBe(DEFAULT_SEGMENT_IDENTIFIER) + }) + + it('should allow empty when canEmpty is true', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setSegmentIdentifier('', true) + }) + + expect(result.current.segmentIdentifier).toBe('') + }) + }) + + // Tests for toggleRule + describe('toggleRule', () => { + it('should toggle rule enabled state', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setRules([ + { id: 'rule1', enabled: true }, + { id: 'rule2', enabled: false }, + ]) + }) + + act(() => { + result.current.toggleRule('rule1') + }) + + expect(result.current.rules.find(r => r.id === 'rule1')?.enabled).toBe(false) + expect(result.current.rules.find(r => r.id === 'rule2')?.enabled).toBe(false) + }) + + it('should not affect other rules', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setRules([ + { id: 'rule1', enabled: true }, + { id: 'rule2', enabled: false }, + ]) + }) + + act(() => { + result.current.toggleRule('rule2') + }) + + expect(result.current.rules.find(r => r.id === 'rule1')?.enabled).toBe(true) + expect(result.current.rules.find(r => r.id === 'rule2')?.enabled).toBe(true) + }) + }) + + // Tests for parent-child config + describe('Parent-Child Configuration', () => { + it('should update parent config delimiter with truthy value', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.updateParentConfig('delimiter', '\n\n\n') + }) + + expect(result.current.parentChildConfig.parent.delimiter).toBe('\\n\\n\\n') + }) + + it('should update parent config delimiter with empty value', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.updateParentConfig('delimiter', '') + }) + + expect(result.current.parentChildConfig.parent.delimiter).toBe('') + }) + + it('should update parent config maxLength', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.updateParentConfig('maxLength', 2048) + }) + + expect(result.current.parentChildConfig.parent.maxLength).toBe(2048) + }) + + it('should update child config delimiter with truthy value', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.updateChildConfig('delimiter', '\n') + }) + + expect(result.current.parentChildConfig.child.delimiter).toBe('\\n') + }) + + it('should update child config delimiter with empty value', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.updateChildConfig('delimiter', '') + }) + + expect(result.current.parentChildConfig.child.delimiter).toBe('') + }) + + it('should update child config maxLength', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.updateChildConfig('maxLength', 256) + }) + + expect(result.current.parentChildConfig.child.maxLength).toBe(256) + }) + + it('should set chunk for context mode', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setChunkForContext('full-doc') + }) + + expect(result.current.parentChildConfig.chunkForContext).toBe('full-doc') + }) + }) + + // Tests for resetToDefaults + describe('resetToDefaults', () => { + it('should reset to default config when available', () => { + const { result } = renderHook(() => useSegmentationState()) + + // Set non-default values and default config + act(() => { + result.current.setMaxChunkLength(2048) + result.current.setOverlap(100) + result.current.setDefaultConfig(createMockRules()) + }) + + // Reset - should use default config values + act(() => { + result.current.resetToDefaults() + }) + + expect(result.current.maxChunkLength).toBe(1024) + expect(result.current.overlap).toBe(50) + expect(result.current.parentChildConfig).toEqual(defaultParentChildConfig) + }) + + it('should only reset parentChildConfig when no default config', () => { + const { result } = renderHook(() => useSegmentationState()) + + // Set non-default values without setting defaultConfig + act(() => { + result.current.setMaxChunkLength(2048) + result.current.setOverlap(100) + result.current.setChunkForContext('full-doc') + }) + + // Reset - should only reset parentChildConfig since no default config + act(() => { + result.current.resetToDefaults() + }) + + // Values stay the same since no defaultConfig + expect(result.current.maxChunkLength).toBe(2048) + expect(result.current.overlap).toBe(100) + // But parentChildConfig is always reset + expect(result.current.parentChildConfig).toEqual(defaultParentChildConfig) + }) + }) + + // Tests for applyConfigFromRules + describe('applyConfigFromRules', () => { + it('should apply general config from rules', () => { + const { result } = renderHook(() => useSegmentationState()) + const rules = createMockRules({ + segmentation: { separator: '---', max_tokens: 512, chunk_overlap: 25 }, + }) + + act(() => { + result.current.applyConfigFromRules(rules, false) + }) + + expect(result.current.maxChunkLength).toBe(512) + expect(result.current.overlap).toBe(25) + }) + + it('should apply hierarchical config from rules', () => { + const { result } = renderHook(() => useSegmentationState()) + const rules = createMockRules({ + parent_mode: 'paragraph', + subchunk_segmentation: { separator: '\n', max_tokens: 256 }, + }) + + act(() => { + result.current.applyConfigFromRules(rules, true) + }) + + expect(result.current.parentChildConfig.chunkForContext).toBe('paragraph') + expect(result.current.parentChildConfig.child.maxLength).toBe(256) + }) + + it('should apply full hierarchical parent-child config from rules', () => { + const { result } = renderHook(() => useSegmentationState()) + const rules = createMockRules({ + segmentation: { separator: '\n\n', max_tokens: 1024, chunk_overlap: 50 }, + parent_mode: 'full-doc', + subchunk_segmentation: { separator: '\n', max_tokens: 128 }, + }) + + act(() => { + result.current.applyConfigFromRules(rules, true) + }) + + // Should set parent config from segmentation + expect(result.current.parentChildConfig.parent.delimiter).toBe('\\n\\n') + expect(result.current.parentChildConfig.parent.maxLength).toBe(1024) + // Should set child config from subchunk_segmentation + expect(result.current.parentChildConfig.child.delimiter).toBe('\\n') + expect(result.current.parentChildConfig.child.maxLength).toBe(128) + // Should set chunkForContext + expect(result.current.parentChildConfig.chunkForContext).toBe('full-doc') + }) + }) + + // Tests for getProcessRule + describe('getProcessRule', () => { + it('should return general process rule', () => { + const { result } = renderHook(() => useSegmentationState()) + + const processRule = result.current.getProcessRule(ChunkingMode.text) + + expect(processRule.mode).toBe(ProcessMode.general) + expect(processRule.rules.segmentation.max_tokens).toBe(DEFAULT_MAXIMUM_CHUNK_LENGTH) + }) + + it('should return hierarchical process rule for parent-child', () => { + const { result } = renderHook(() => useSegmentationState()) + + const processRule = result.current.getProcessRule(ChunkingMode.parentChild) + + expect(processRule.mode).toBe('hierarchical') + expect(processRule.rules.parent_mode).toBe('paragraph') + expect(processRule.rules.subchunk_segmentation).toBeDefined() + }) + }) +}) + +// ============================================ +// useIndexingConfig Hook Tests +// ============================================ + +describe('useIndexingConfig', () => { + beforeEach(() => { + vi.clearAllMocks() + mockIsRerankDefaultModelValid = true + }) + + // Tests for initial state + // Note: Hook has useEffect that syncs state, so we test the state after effects settle + describe('Initial State', () => { + it('should initialize with QUALIFIED when API key is set', async () => { + const { result } = renderHook(() => + useIndexingConfig({ isAPIKeySet: true, hasSetIndexType: false }), + ) + + // After effects settle, indexType should be QUALIFIED + await vi.waitFor(() => { + expect(result.current.indexType).toBe(IndexingType.QUALIFIED) + }) + }) + + it('should initialize with ECONOMICAL when API key is not set', async () => { + const { result } = renderHook(() => + useIndexingConfig({ isAPIKeySet: false, hasSetIndexType: false }), + ) + + await vi.waitFor(() => { + expect(result.current.indexType).toBe(IndexingType.ECONOMICAL) + }) + }) + + it('should use initial index type when provided', async () => { + const { result } = renderHook(() => + useIndexingConfig({ + isAPIKeySet: false, + hasSetIndexType: true, + initialIndexType: IndexingType.QUALIFIED, + }), + ) + + await vi.waitFor(() => { + expect(result.current.indexType).toBe(IndexingType.QUALIFIED) + }) + }) + }) + + // Tests for state setters + describe('State Management', () => { + it('should update index type', async () => { + const { result } = renderHook(() => + useIndexingConfig({ isAPIKeySet: true, hasSetIndexType: false }), + ) + + // Wait for initial effects to settle + await vi.waitFor(() => { + expect(result.current.indexType).toBeDefined() + }) + + act(() => { + result.current.setIndexType(IndexingType.ECONOMICAL) + }) + + expect(result.current.indexType).toBe(IndexingType.ECONOMICAL) + }) + + it('should update embedding model', async () => { + const { result } = renderHook(() => + useIndexingConfig({ isAPIKeySet: true, hasSetIndexType: false }), + ) + + await vi.waitFor(() => { + expect(result.current.embeddingModel).toBeDefined() + }) + + act(() => { + result.current.setEmbeddingModel({ provider: 'cohere', model: 'embed-v3' }) + }) + + expect(result.current.embeddingModel).toEqual({ provider: 'cohere', model: 'embed-v3' }) + }) + + it('should update retrieval config', async () => { + const { result } = renderHook(() => + useIndexingConfig({ isAPIKeySet: true, hasSetIndexType: false }), + ) + + await vi.waitFor(() => { + expect(result.current.retrievalConfig).toBeDefined() + }) + + const newConfig: RetrievalConfig = { + search_method: RETRIEVE_METHOD.hybrid, + reranking_enable: true, + reranking_model: { reranking_provider_name: 'cohere', reranking_model_name: 'rerank-v3' }, + top_k: 5, + score_threshold_enabled: true, + score_threshold: 0.7, + } + + act(() => { + result.current.setRetrievalConfig(newConfig) + }) + + expect(result.current.retrievalConfig).toEqual(newConfig) + }) + }) + + // Tests for getIndexingTechnique + describe('getIndexingTechnique', () => { + it('should return initial type when set', async () => { + const { result } = renderHook(() => + useIndexingConfig({ + isAPIKeySet: true, + hasSetIndexType: true, + initialIndexType: IndexingType.ECONOMICAL, + }), + ) + + await vi.waitFor(() => { + expect(result.current.getIndexingTechnique()).toBe(IndexingType.ECONOMICAL) + }) + }) + + it('should return current type when no initial type', async () => { + const { result } = renderHook(() => + useIndexingConfig({ isAPIKeySet: true, hasSetIndexType: false }), + ) + + await vi.waitFor(() => { + expect(result.current.indexType).toBeDefined() + }) + + act(() => { + result.current.setIndexType(IndexingType.ECONOMICAL) + }) + + expect(result.current.getIndexingTechnique()).toBe(IndexingType.ECONOMICAL) + }) + }) + + // Tests for initialRetrievalConfig handling + describe('initialRetrievalConfig', () => { + it('should skip retrieval config sync when initialRetrievalConfig is provided', async () => { + const customRetrievalConfig: RetrievalConfig = { + search_method: RETRIEVE_METHOD.hybrid, + reranking_enable: true, + reranking_model: { reranking_provider_name: 'custom', reranking_model_name: 'custom-model' }, + top_k: 10, + score_threshold_enabled: true, + score_threshold: 0.8, + } + + const { result } = renderHook(() => + useIndexingConfig({ + isAPIKeySet: true, + hasSetIndexType: false, + initialRetrievalConfig: customRetrievalConfig, + }), + ) + + await vi.waitFor(() => { + expect(result.current.retrievalConfig).toBeDefined() + }) + + // Should use the provided initial config, not the default synced one + expect(result.current.retrievalConfig.search_method).toBe(RETRIEVE_METHOD.hybrid) + expect(result.current.retrievalConfig.top_k).toBe(10) + }) + }) +}) + +// ============================================ +// usePreviewState Hook Tests +// ============================================ + +describe('usePreviewState', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + const defaultOptions = { + dataSourceType: DataSourceType.FILE, + files: [createMockFile()], + notionPages: [createMockNotionPage()], + websitePages: [createMockWebsitePage()], + } + + // Tests for initial state + describe('Initial State', () => { + it('should initialize with first file for FILE data source', () => { + const { result } = renderHook(() => usePreviewState(defaultOptions)) + + expect(result.current.previewFile).toEqual(defaultOptions.files[0]) + }) + + it('should initialize with first notion page for NOTION data source', () => { + const { result } = renderHook(() => + usePreviewState({ ...defaultOptions, dataSourceType: DataSourceType.NOTION }), + ) + + expect(result.current.previewNotionPage).toEqual(defaultOptions.notionPages[0]) + }) + + it('should initialize with document detail when provided', () => { + const documentDetail = createMockDocumentDetail() + const { result } = renderHook(() => + usePreviewState({ + ...defaultOptions, + documentDetail, + datasetId: 'test-id', + }), + ) + + expect(result.current.previewFile).toEqual(documentDetail.file) + }) + }) + + // Tests for getPreviewPickerItems + describe('getPreviewPickerItems', () => { + it('should return files for FILE data source', () => { + const { result } = renderHook(() => usePreviewState(defaultOptions)) + + const items = result.current.getPreviewPickerItems() + expect(items).toEqual(defaultOptions.files) + }) + + it('should return mapped notion pages for NOTION data source', () => { + const { result } = renderHook(() => + usePreviewState({ ...defaultOptions, dataSourceType: DataSourceType.NOTION }), + ) + + const items = result.current.getPreviewPickerItems() + expect(items[0]).toEqual({ + id: 'notion-page-1', + name: 'Test Notion Page', + extension: 'md', + }) + }) + + it('should return mapped website pages for WEB data source', () => { + const { result } = renderHook(() => + usePreviewState({ ...defaultOptions, dataSourceType: DataSourceType.WEB }), + ) + + const items = result.current.getPreviewPickerItems() + expect(items[0]).toEqual({ + id: 'https://example.com/page1', + name: 'Test Website Page', + extension: 'md', + }) + }) + + it('should return empty array for unknown data source', () => { + const { result } = renderHook(() => + usePreviewState({ ...defaultOptions, dataSourceType: 'unknown' as DataSourceType }), + ) + + const items = result.current.getPreviewPickerItems() + expect(items).toEqual([]) + }) + }) + + // Tests for getPreviewPickerValue + describe('getPreviewPickerValue', () => { + it('should return file value for FILE data source', () => { + const { result } = renderHook(() => usePreviewState(defaultOptions)) + + const value = result.current.getPreviewPickerValue() + expect(value).toEqual(defaultOptions.files[0]) + }) + + it('should return mapped notion page value for NOTION data source', () => { + const notionPage = createMockNotionPage({ page_id: 'page-123', page_name: 'My Page' }) + const { result } = renderHook(() => + usePreviewState({ + ...defaultOptions, + dataSourceType: DataSourceType.NOTION, + notionPages: [notionPage], + }), + ) + + const value = result.current.getPreviewPickerValue() + expect(value).toEqual({ + id: 'page-123', + name: 'My Page', + extension: 'md', + }) + }) + + it('should return mapped website page value for WEB data source', () => { + const websitePage = createMockWebsitePage({ source_url: 'https://test.com', title: 'Test Title' }) + const { result } = renderHook(() => + usePreviewState({ + ...defaultOptions, + dataSourceType: DataSourceType.WEB, + websitePages: [websitePage], + }), + ) + + const value = result.current.getPreviewPickerValue() + expect(value).toEqual({ + id: 'https://test.com', + name: 'Test Title', + extension: 'md', + }) + }) + + it('should return empty value for unknown data source', () => { + const { result } = renderHook(() => + usePreviewState({ ...defaultOptions, dataSourceType: 'unknown' as DataSourceType }), + ) + + const value = result.current.getPreviewPickerValue() + expect(value).toEqual({ id: '', name: '', extension: '' }) + }) + + it('should handle undefined notion page gracefully', () => { + const { result } = renderHook(() => + usePreviewState({ + ...defaultOptions, + dataSourceType: DataSourceType.NOTION, + notionPages: [], + }), + ) + + const value = result.current.getPreviewPickerValue() + expect(value).toEqual({ + id: '', + name: '', + extension: 'md', + }) + }) + + it('should handle undefined website page gracefully', () => { + const { result } = renderHook(() => + usePreviewState({ + ...defaultOptions, + dataSourceType: DataSourceType.WEB, + websitePages: [], + }), + ) + + const value = result.current.getPreviewPickerValue() + expect(value).toEqual({ + id: '', + name: '', + extension: 'md', + }) + }) + }) + + // Tests for handlePreviewChange + describe('handlePreviewChange', () => { + it('should update preview file for FILE data source', () => { + const files = [createMockFile(), createMockFile({ id: 'file-2', name: 'second.pdf' })] + const { result } = renderHook(() => + usePreviewState({ ...defaultOptions, files }), + ) + + act(() => { + result.current.handlePreviewChange({ id: 'file-2', name: 'second.pdf' }) + }) + + expect(result.current.previewFile).toEqual({ id: 'file-2', name: 'second.pdf' }) + }) + + it('should update preview notion page for NOTION data source', () => { + const notionPages = [ + createMockNotionPage(), + createMockNotionPage({ page_id: 'notion-page-2', page_name: 'Second Page' }), + ] + const { result } = renderHook(() => + usePreviewState({ ...defaultOptions, dataSourceType: DataSourceType.NOTION, notionPages }), + ) + + act(() => { + result.current.handlePreviewChange({ id: 'notion-page-2', name: 'Second Page' }) + }) + + expect(result.current.previewNotionPage?.page_id).toBe('notion-page-2') + }) + + it('should update preview website page for WEB data source', () => { + const websitePages = [ + createMockWebsitePage(), + createMockWebsitePage({ source_url: 'https://example.com/page2', title: 'Second Page' }), + ] + const { result } = renderHook(() => + usePreviewState({ ...defaultOptions, dataSourceType: DataSourceType.WEB, websitePages }), + ) + + act(() => { + result.current.handlePreviewChange({ id: 'https://example.com/page2', name: 'Second Page' }) + }) + + expect(result.current.previewWebsitePage?.source_url).toBe('https://example.com/page2') + }) + }) +}) + +// ============================================ +// useDocumentCreation Hook Tests +// ============================================ + +describe('useDocumentCreation', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + const defaultOptions = { + dataSourceType: DataSourceType.FILE, + files: [createMockFile()], + notionPages: [] as NotionPage[], + notionCredentialId: '', + websitePages: [] as CrawlResultItem[], + } + + // Tests for validateParams + describe('validateParams', () => { + it('should return false when overlap exceeds max chunk length', () => { + const { result } = renderHook(() => useDocumentCreation(defaultOptions)) + + const isValid = result.current.validateParams({ + segmentationType: 'general', + maxChunkLength: 100, + limitMaxChunkLength: 4000, + overlap: 200, + indexType: IndexingType.QUALIFIED, + embeddingModel: { provider: 'openai', model: 'text-embedding-ada-002' }, + rerankModelList: [], + retrievalConfig: { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + }) + + expect(isValid).toBe(false) + }) + + it('should return false when max chunk length exceeds limit', () => { + const { result } = renderHook(() => useDocumentCreation(defaultOptions)) + + const isValid = result.current.validateParams({ + segmentationType: 'general', + maxChunkLength: 5000, + limitMaxChunkLength: 4000, + overlap: 50, + indexType: IndexingType.QUALIFIED, + embeddingModel: { provider: 'openai', model: 'text-embedding-ada-002' }, + rerankModelList: [], + retrievalConfig: { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + }) + + expect(isValid).toBe(false) + }) + + it('should return true for valid params', () => { + const { result } = renderHook(() => useDocumentCreation(defaultOptions)) + + const isValid = result.current.validateParams({ + segmentationType: 'general', + maxChunkLength: 1000, + limitMaxChunkLength: 4000, + overlap: 50, + indexType: IndexingType.QUALIFIED, + embeddingModel: { provider: 'openai', model: 'text-embedding-ada-002' }, + rerankModelList: [], + retrievalConfig: { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + }) + + expect(isValid).toBe(true) + }) + }) + + // Tests for buildCreationParams + describe('buildCreationParams', () => { + it('should build params for file upload', () => { + const { result } = renderHook(() => useDocumentCreation(defaultOptions)) + + const params = result.current.buildCreationParams( + ChunkingMode.text, + 'English', + { mode: ProcessMode.general, rules: createMockRules() }, + { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + { provider: 'openai', model: 'text-embedding-ada-002' }, + IndexingType.QUALIFIED, + ) + + expect(params).toBeDefined() + expect(params?.doc_form).toBe(ChunkingMode.text) + expect(params?.doc_language).toBe('English') + expect(params?.data_source?.type).toBe(DataSourceType.FILE) + }) + + it('should build params for setting mode', () => { + const documentDetail = createMockDocumentDetail() + const { result } = renderHook(() => + useDocumentCreation({ + ...defaultOptions, + isSetting: true, + documentDetail, + }), + ) + + const params = result.current.buildCreationParams( + ChunkingMode.text, + 'English', + { mode: ProcessMode.general, rules: createMockRules() }, + { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + { provider: 'openai', model: 'text-embedding-ada-002' }, + IndexingType.QUALIFIED, + ) + + expect(params?.original_document_id).toBe(documentDetail.id) + }) + + it('should build params for notion_import data source', () => { + const { result } = renderHook(() => + useDocumentCreation({ + ...defaultOptions, + dataSourceType: DataSourceType.NOTION, + notionPages: [createMockNotionPage()], + notionCredentialId: 'notion-cred-123', + }), + ) + + const params = result.current.buildCreationParams( + ChunkingMode.text, + 'English', + { mode: ProcessMode.general, rules: createMockRules() }, + { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + { provider: 'openai', model: 'text-embedding-ada-002' }, + IndexingType.QUALIFIED, + ) + + expect(params).toBeDefined() + expect(params?.data_source?.type).toBe(DataSourceType.NOTION) + expect(params?.data_source?.info_list.notion_info_list).toBeDefined() + }) + + it('should build params for website_crawl data source', () => { + const { result } = renderHook(() => + useDocumentCreation({ + ...defaultOptions, + dataSourceType: DataSourceType.WEB, + websitePages: [createMockWebsitePage()], + websiteCrawlProvider: 'jinaReader' as DataSourceProvider, + websiteCrawlJobId: 'job-123', + crawlOptions: { max_depth: 2 } as CrawlOptions, + }), + ) + + const params = result.current.buildCreationParams( + ChunkingMode.text, + 'English', + { mode: ProcessMode.general, rules: createMockRules() }, + { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + { provider: 'openai', model: 'text-embedding-ada-002' }, + IndexingType.QUALIFIED, + ) + + expect(params).toBeDefined() + expect(params?.data_source?.type).toBe(DataSourceType.WEB) + expect(params?.data_source?.info_list.website_info_list).toBeDefined() + }) + }) + + // Tests for validateParams edge cases + describe('validateParams - additional cases', () => { + it('should return false when embedding model is missing for QUALIFIED index type', () => { + const { result } = renderHook(() => useDocumentCreation(defaultOptions)) + + const isValid = result.current.validateParams({ + segmentationType: 'general', + maxChunkLength: 500, + limitMaxChunkLength: 4000, + overlap: 50, + indexType: IndexingType.QUALIFIED, + embeddingModel: { provider: '', model: '' }, + rerankModelList: mockRerankModelList, + retrievalConfig: { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + }) + + expect(isValid).toBe(false) + }) + + it('should return false when rerank model is required but not selected', () => { + const { result } = renderHook(() => useDocumentCreation(defaultOptions)) + + // isReRankModelSelected returns false when: + // - indexMethod === 'high_quality' (IndexingType.QUALIFIED) + // - reranking_enable === true + // - rerankModelSelected === false (model not found in list) + const isValid = result.current.validateParams({ + segmentationType: 'general', + maxChunkLength: 500, + limitMaxChunkLength: 4000, + overlap: 50, + indexType: IndexingType.QUALIFIED, + embeddingModel: { provider: 'openai', model: 'text-embedding-ada-002' }, + rerankModelList: [], // Empty list means model won't be found + retrievalConfig: { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: true, // Reranking enabled + reranking_model: { + reranking_provider_name: 'nonexistent', + reranking_model_name: 'nonexistent-model', + }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + }) + + expect(isValid).toBe(false) + }) + }) + + // Tests for executeCreation + describe('executeCreation', () => { + it('should call createFirstDocumentMutation when datasetId is not provided', async () => { + const mockOnStepChange = vi.fn() + const mockUpdateIndexingTypeCache = vi.fn() + const mockUpdateResultCache = vi.fn() + const mockUpdateRetrievalMethodCache = vi.fn() + const mockOnSave = vi.fn() + + const { result } = renderHook(() => + useDocumentCreation({ + ...defaultOptions, + datasetId: undefined, + onStepChange: mockOnStepChange, + updateIndexingTypeCache: mockUpdateIndexingTypeCache, + updateResultCache: mockUpdateResultCache, + updateRetrievalMethodCache: mockUpdateRetrievalMethodCache, + onSave: mockOnSave, + }), + ) + + const params = result.current.buildCreationParams( + ChunkingMode.text, + 'English', + { mode: ProcessMode.general, rules: createMockRules() }, + { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + { provider: 'openai', model: 'text-embedding-ada-002' }, + IndexingType.QUALIFIED, + ) + + await act(async () => { + await result.current.executeCreation(params!, IndexingType.QUALIFIED, { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }) + }) + + expect(mockOnStepChange).toHaveBeenCalledWith(1) + }) + + it('should call createDocumentMutation when datasetId is provided', async () => { + const mockOnStepChange = vi.fn() + const { result } = renderHook(() => + useDocumentCreation({ + ...defaultOptions, + datasetId: 'existing-dataset-id', + onStepChange: mockOnStepChange, + }), + ) + + const params = result.current.buildCreationParams( + ChunkingMode.text, + 'English', + { mode: ProcessMode.general, rules: createMockRules() }, + { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + { provider: 'openai', model: 'text-embedding-ada-002' }, + IndexingType.QUALIFIED, + ) + + await act(async () => { + await result.current.executeCreation(params!, IndexingType.QUALIFIED, { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }) + }) + + expect(mockOnStepChange).toHaveBeenCalledWith(1) + }) + + it('should call onSave when in setting mode', async () => { + const mockOnSave = vi.fn() + const documentDetail = createMockDocumentDetail() + const { result } = renderHook(() => + useDocumentCreation({ + ...defaultOptions, + datasetId: 'existing-dataset-id', + isSetting: true, + documentDetail, + onSave: mockOnSave, + }), + ) + + const params = result.current.buildCreationParams( + ChunkingMode.text, + 'English', + { mode: ProcessMode.general, rules: createMockRules() }, + { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + { provider: 'openai', model: 'text-embedding-ada-002' }, + IndexingType.QUALIFIED, + ) + + await act(async () => { + await result.current.executeCreation(params!, IndexingType.QUALIFIED, { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }) + }) + + expect(mockOnSave).toHaveBeenCalled() + }) + }) + + // Tests for validatePreviewParams + describe('validatePreviewParams', () => { + it('should return true for valid max chunk length', () => { + const { result } = renderHook(() => useDocumentCreation(defaultOptions)) + + const isValid = result.current.validatePreviewParams(1000) + expect(isValid).toBe(true) + }) + + it('should return false when max chunk length exceeds maximum', () => { + const { result } = renderHook(() => useDocumentCreation(defaultOptions)) + + const isValid = result.current.validatePreviewParams(10000) + expect(isValid).toBe(false) + }) + }) +}) + +// ============================================ +// useIndexingEstimate Hook Tests +// ============================================ + +describe('useIndexingEstimate', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + const defaultOptions = { + dataSourceType: DataSourceType.FILE, + currentDocForm: ChunkingMode.text, + docLanguage: 'English', + files: [createMockFile()], + previewNotionPage: createMockNotionPage(), + notionCredentialId: '', + previewWebsitePage: createMockWebsitePage(), + indexingTechnique: IndexingType.QUALIFIED, + processRule: { mode: ProcessMode.general, rules: createMockRules() }, + } + + // Tests for initial state + describe('Initial State', () => { + it('should initialize with idle state', () => { + const { result } = renderHook(() => useIndexingEstimate(defaultOptions)) + + expect(result.current.isIdle).toBe(true) + expect(result.current.isPending).toBe(false) + expect(result.current.estimate).toBeUndefined() + }) + }) + + // Tests for fetchEstimate + describe('fetchEstimate', () => { + it('should have fetchEstimate function', () => { + const { result } = renderHook(() => useIndexingEstimate(defaultOptions)) + + expect(typeof result.current.fetchEstimate).toBe('function') + }) + + it('should have reset function', () => { + const { result } = renderHook(() => useIndexingEstimate(defaultOptions)) + + expect(typeof result.current.reset).toBe('function') + }) + + it('should call fetchEstimate for FILE data source', () => { + const { result } = renderHook(() => + useIndexingEstimate({ + ...defaultOptions, + dataSourceType: DataSourceType.FILE, + previewFileName: 'test-file.pdf', + }), + ) + + act(() => { + result.current.fetchEstimate() + }) + + // fetchEstimate should be callable without error + expect(result.current.fetchEstimate).toBeDefined() + }) + + it('should call fetchEstimate for NOTION data source', () => { + const { result } = renderHook(() => + useIndexingEstimate({ + ...defaultOptions, + dataSourceType: DataSourceType.NOTION, + previewNotionPage: createMockNotionPage(), + notionCredentialId: 'cred-123', + }), + ) + + act(() => { + result.current.fetchEstimate() + }) + + expect(result.current.fetchEstimate).toBeDefined() + }) + + it('should call fetchEstimate for WEB data source', () => { + const { result } = renderHook(() => + useIndexingEstimate({ + ...defaultOptions, + dataSourceType: DataSourceType.WEB, + previewWebsitePage: createMockWebsitePage(), + websiteCrawlProvider: 'jinaReader' as DataSourceProvider, + websiteCrawlJobId: 'job-123', + crawlOptions: { max_depth: 2 } as CrawlOptions, + }), + ) + + act(() => { + result.current.fetchEstimate() + }) + + expect(result.current.fetchEstimate).toBeDefined() + }) + }) + + // Tests for getCurrentMutation based on data source type + describe('Data Source Selection', () => { + it('should use file query for FILE data source', () => { + const { result } = renderHook(() => + useIndexingEstimate({ + ...defaultOptions, + dataSourceType: DataSourceType.FILE, + }), + ) + + expect(result.current.currentMutation).toBeDefined() + expect(result.current.isIdle).toBe(true) + }) + + it('should use notion query for NOTION data source', () => { + const { result } = renderHook(() => + useIndexingEstimate({ + ...defaultOptions, + dataSourceType: DataSourceType.NOTION, + }), + ) + + expect(result.current.currentMutation).toBeDefined() + expect(result.current.isIdle).toBe(true) + }) + + it('should use website query for WEB data source', () => { + const { result } = renderHook(() => + useIndexingEstimate({ + ...defaultOptions, + dataSourceType: DataSourceType.WEB, + websiteCrawlProvider: 'jinaReader' as DataSourceProvider, + websiteCrawlJobId: 'job-123', + }), + ) + + expect(result.current.currentMutation).toBeDefined() + expect(result.current.isIdle).toBe(true) + }) + }) +}) + +// ============================================ +// StepTwoFooter Component Tests +// ============================================ + +describe('StepTwoFooter', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + const defaultProps = { + isSetting: false, + isCreating: false, + onPrevious: vi.fn(), + onCreate: vi.fn(), + onCancel: vi.fn(), + } + + // Tests for rendering + describe('Rendering', () => { + it('should render without crashing', () => { + render() + + // Should render Previous and Next buttons with correct text + expect(screen.getByText(/previousStep/i)).toBeInTheDocument() + expect(screen.getByText(/nextStep/i)).toBeInTheDocument() + }) + + it('should render Previous and Next buttons when not in setting mode', () => { + render() + + expect(screen.getByText(/previousStep/i)).toBeInTheDocument() + expect(screen.getByText(/nextStep/i)).toBeInTheDocument() + }) + + it('should render Save and Cancel buttons when in setting mode', () => { + render() + + expect(screen.getByText(/save/i)).toBeInTheDocument() + expect(screen.getByText(/cancel/i)).toBeInTheDocument() + }) + }) + + // Tests for user interactions + describe('User Interactions', () => { + it('should call onPrevious when Previous button is clicked', () => { + const onPrevious = vi.fn() + render() + + fireEvent.click(screen.getByText(/previousStep/i)) + + expect(onPrevious).toHaveBeenCalledTimes(1) + }) + + it('should call onCreate when Next/Save button is clicked', () => { + const onCreate = vi.fn() + render() + + fireEvent.click(screen.getByText(/nextStep/i)) + + expect(onCreate).toHaveBeenCalledTimes(1) + }) + + it('should call onCancel when Cancel button is clicked in setting mode', () => { + const onCancel = vi.fn() + render() + + fireEvent.click(screen.getByText(/cancel/i)) + + expect(onCancel).toHaveBeenCalledTimes(1) + }) + }) + + // Tests for loading state + describe('Loading State', () => { + it('should show loading state on Next button when creating', () => { + render() + + const nextButton = screen.getByText(/nextStep/i).closest('button') + // Button has disabled:btn-disabled class which handles the loading state + expect(nextButton).toHaveClass('disabled:btn-disabled') + }) + + it('should show loading state on Save button when creating in setting mode', () => { + render() + + const saveButton = screen.getByText(/save/i).closest('button') + // Button has disabled:btn-disabled class which handles the loading state + expect(saveButton).toHaveClass('disabled:btn-disabled') + }) + }) +}) + +// ============================================ +// PreviewPanel Component Tests +// ============================================ + +describe('PreviewPanel', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + const defaultProps = { + isMobile: false, + dataSourceType: DataSourceType.FILE, + currentDocForm: ChunkingMode.text, + estimate: undefined as FileIndexingEstimateResponse | undefined, + parentChildConfig: defaultParentChildConfig, + isSetting: false, + pickerFiles: [{ id: 'file-1', name: 'test.pdf', extension: 'pdf' }], + pickerValue: { id: 'file-1', name: 'test.pdf', extension: 'pdf' }, + isIdle: true, + isPending: false, + onPickerChange: vi.fn(), + } + + // Tests for rendering + describe('Rendering', () => { + it('should render without crashing', () => { + render() + + // Check for the preview header title text + expect(screen.getByText('datasetCreation.stepTwo.preview')).toBeInTheDocument() + }) + + it('should render idle state when isIdle is true', () => { + render() + + expect(screen.getByText(/previewChunkTip/i)).toBeInTheDocument() + }) + + it('should render loading skeleton when isPending is true', () => { + render() + + // Should show skeleton containers + expect(screen.queryByText(/previewChunkTip/i)).not.toBeInTheDocument() + }) + }) + + // Tests for different doc forms + describe('Preview Content', () => { + it('should render text preview when docForm is text', () => { + const estimate = createMockEstimate() + render( + , + ) + + expect(screen.getByText('Chunk 1 content')).toBeInTheDocument() + }) + + it('should render QA preview when docForm is qa', () => { + const estimate = createMockEstimate() + render( + , + ) + + expect(screen.getByText('Q1')).toBeInTheDocument() + expect(screen.getByText('A1')).toBeInTheDocument() + }) + + it('should show chunk count badge for non-QA doc form', () => { + const estimate = createMockEstimate({ total_segments: 25 }) + render( + , + ) + + expect(screen.getByText(/25/)).toBeInTheDocument() + }) + + it('should render parent-child preview when docForm is parentChild', () => { + const estimate = createMockEstimate({ + preview: [ + { content: 'Parent chunk content', child_chunks: ['Child 1', 'Child 2', 'Child 3'] }, + ], + }) + render( + , + ) + + // Should render parent chunk label + expect(screen.getByText('Chunk-1')).toBeInTheDocument() + // Should render child chunks + expect(screen.getByText('Child 1')).toBeInTheDocument() + expect(screen.getByText('Child 2')).toBeInTheDocument() + expect(screen.getByText('Child 3')).toBeInTheDocument() + }) + + it('should limit child chunks when chunkForContext is full-doc', () => { + // FULL_DOC_PREVIEW_LENGTH is 50, so we need more than 50 chunks to test the limit + const manyChildChunks = Array.from({ length: 60 }, (_, i) => `ChildChunk${i + 1}`) + const estimate = createMockEstimate({ + preview: [{ content: 'Parent content', child_chunks: manyChildChunks }], + }) + render( + , + ) + + // Should render parent chunk + expect(screen.getByText('Chunk-1')).toBeInTheDocument() + // full-doc mode limits to FULL_DOC_PREVIEW_LENGTH (50) + expect(screen.getByText('ChildChunk1')).toBeInTheDocument() + expect(screen.getByText('ChildChunk50')).toBeInTheDocument() + // Should not render beyond the limit + expect(screen.queryByText('ChildChunk51')).not.toBeInTheDocument() + }) + + it('should render multiple parent chunks in parent-child mode', () => { + const estimate = createMockEstimate({ + preview: [ + { content: 'Parent 1', child_chunks: ['P1-C1'] }, + { content: 'Parent 2', child_chunks: ['P2-C1'] }, + ], + }) + render( + , + ) + + expect(screen.getByText('Chunk-1')).toBeInTheDocument() + expect(screen.getByText('Chunk-2')).toBeInTheDocument() + expect(screen.getByText('P1-C1')).toBeInTheDocument() + expect(screen.getByText('P2-C1')).toBeInTheDocument() + }) + }) + + // Tests for picker + describe('Document Picker', () => { + it('should call onPickerChange when document is selected', () => { + const onPickerChange = vi.fn() + render() + + // The picker interaction would be tested through the actual component + expect(onPickerChange).not.toHaveBeenCalled() + }) + }) +}) + +// ============================================ +// Edge Cases Tests +// ============================================ + +describe('Edge Cases', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + describe('Empty/Null Values', () => { + it('should handle empty files array in usePreviewState', () => { + const { result } = renderHook(() => + usePreviewState({ + dataSourceType: DataSourceType.FILE, + files: [], + notionPages: [], + websitePages: [], + }), + ) + + expect(result.current.previewFile).toBeUndefined() + }) + + it('should handle empty notion pages array', () => { + const { result } = renderHook(() => + usePreviewState({ + dataSourceType: DataSourceType.NOTION, + files: [], + notionPages: [], + websitePages: [], + }), + ) + + expect(result.current.previewNotionPage).toBeUndefined() + }) + + it('should handle empty website pages array', () => { + const { result } = renderHook(() => + usePreviewState({ + dataSourceType: DataSourceType.WEB, + files: [], + notionPages: [], + websitePages: [], + }), + ) + + expect(result.current.previewWebsitePage).toBeUndefined() + }) + }) + + describe('Boundary Conditions', () => { + it('should handle very large chunk length', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setMaxChunkLength(999999) + }) + + expect(result.current.maxChunkLength).toBe(999999) + }) + + it('should handle zero overlap', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setOverlap(0) + }) + + expect(result.current.overlap).toBe(0) + }) + + it('should handle special characters in segment identifier', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setSegmentIdentifier('<<>>') + }) + + expect(result.current.segmentIdentifier).toBe('<<>>') + }) + }) + + describe('Callback Stability', () => { + it('should maintain stable setSegmentIdentifier reference', () => { + const { result, rerender } = renderHook(() => useSegmentationState()) + const initialSetter = result.current.setSegmentIdentifier + + rerender() + + expect(result.current.setSegmentIdentifier).toBe(initialSetter) + }) + + it('should maintain stable toggleRule reference', () => { + const { result, rerender } = renderHook(() => useSegmentationState()) + const initialToggle = result.current.toggleRule + + rerender() + + expect(result.current.toggleRule).toBe(initialToggle) + }) + + it('should maintain stable getProcessRule reference', () => { + const { result, rerender } = renderHook(() => useSegmentationState()) + + // Update some state to trigger re-render + act(() => { + result.current.setMaxChunkLength(2048) + }) + + rerender() + + // getProcessRule depends on state, so it may change but should remain a function + expect(typeof result.current.getProcessRule).toBe('function') + }) + }) +}) + +// ============================================ +// Integration Scenarios +// ============================================ + +describe('Integration Scenarios', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCurrentDataset = null + }) + + describe('Document Creation Flow', () => { + it('should build and validate params for file upload workflow', () => { + const files = [createMockFile()] + + const { result: segResult } = renderHook(() => useSegmentationState()) + const { result: creationResult } = renderHook(() => + useDocumentCreation({ + dataSourceType: DataSourceType.FILE, + files, + notionPages: [], + notionCredentialId: '', + websitePages: [], + }), + ) + + // Build params + const params = creationResult.current.buildCreationParams( + ChunkingMode.text, + 'English', + segResult.current.getProcessRule(ChunkingMode.text), + { + search_method: RETRIEVE_METHOD.semantic, + reranking_enable: false, + reranking_model: { reranking_provider_name: '', reranking_model_name: '' }, + top_k: 3, + score_threshold_enabled: false, + score_threshold: 0.5, + }, + { provider: 'openai', model: 'text-embedding-ada-002' }, + IndexingType.QUALIFIED, + ) + + expect(params).toBeDefined() + expect(params?.data_source?.info_list.file_info_list?.file_ids).toContain('file-1') + }) + + it('should handle parent-child document form', () => { + const { result } = renderHook(() => useSegmentationState()) + + act(() => { + result.current.setSegmentationType(ProcessMode.parentChild) + result.current.setChunkForContext('full-doc') + result.current.updateParentConfig('maxLength', 2048) + result.current.updateChildConfig('maxLength', 512) + }) + + const processRule = result.current.getProcessRule(ChunkingMode.parentChild) + + expect(processRule.mode).toBe('hierarchical') + expect(processRule.rules.parent_mode).toBe('full-doc') + expect(processRule.rules.segmentation.max_tokens).toBe(2048) + expect(processRule.rules.subchunk_segmentation?.max_tokens).toBe(512) + }) + }) + + describe('Preview Flow', () => { + it('should handle preview file change flow', () => { + const files = [ + createMockFile({ id: 'file-1', name: 'first.pdf' }), + createMockFile({ id: 'file-2', name: 'second.pdf' }), + ] + + const { result } = renderHook(() => + usePreviewState({ + dataSourceType: DataSourceType.FILE, + files, + notionPages: [], + websitePages: [], + }), + ) + + // Initial state + expect(result.current.getPreviewPickerValue().name).toBe('first.pdf') + + // Change preview + act(() => { + result.current.handlePreviewChange({ id: 'file-2', name: 'second.pdf' }) + }) + + expect(result.current.previewFile).toEqual({ id: 'file-2', name: 'second.pdf' }) + }) + }) + + describe('Escape/Unescape Round Trip', () => { + it('should preserve original string through escape/unescape', () => { + const original = '\n\n' + const escaped = escape(original) + const unescaped = unescape(escaped) + + expect(unescaped).toBe(original) + }) + + it('should handle complex strings without backslashes', () => { + // This string contains control characters but no literal backslashes. + const original = 'Hello\nWorld\t!\r\n' + const escaped = escape(original) + const unescaped = unescape(escaped) + expect(unescaped).toBe(original) + }) + + it('should document behavior for strings with existing backslashes', () => { + // When the original string already contains backslash sequences, + // escape/unescape are not perfectly symmetric because escape() + // does not escape backslashes. + const original = 'Hello\\nWorld' + const escaped = escape(original) + const unescaped = unescape(escaped) + // The unescaped value interprets "\n" as a newline, so it differs from the original. + expect(unescaped).toBe('Hello\nWorld') + expect(unescaped).not.toBe(original) + }) + }) +}) diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 51b5c15178..b4d2c5f6e9 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -1,137 +1,30 @@ 'use client' -import type { FC, PropsWithChildren } from 'react' -import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations' -import type { NotionPage } from '@/models/common' -import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, createDocumentResponse, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules } from '@/models/datasets' -import type { RetrievalConfig } from '@/types/app' -import { - RiAlertFill, - RiArrowLeftLine, - RiSearchEyeLine, -} from '@remixicon/react' -import { noop } from 'es-toolkit/function' -import Image from 'next/image' -import Link from 'next/link' -import { useCallback, useEffect, useMemo, useState } from 'react' -import { useTranslation } from 'react-i18next' -import { trackEvent } from '@/app/components/base/amplitude' -import Badge from '@/app/components/base/badge' -import Button from '@/app/components/base/button' -import Checkbox from '@/app/components/base/checkbox' -import CustomDialog from '@/app/components/base/dialog' -import Divider from '@/app/components/base/divider' -import FloatRightContainer from '@/app/components/base/float-right-container' -import { ParentChildChunk } from '@/app/components/base/icons/src/vender/knowledge' -import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback' -import RadioCard from '@/app/components/base/radio-card' -import { SkeletonContainer, SkeletonPoint, SkeletonRectangle, SkeletonRow } from '@/app/components/base/skeleton' -import Toast from '@/app/components/base/toast' -import Tooltip from '@/app/components/base/tooltip' -import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model' -import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config' -import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config' -import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' -import { useDefaultModel, useModelList, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' -import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector' -import { FULL_DOC_PREVIEW_LENGTH, IS_CE_EDITION } from '@/config' +import type { FC } from 'react' +import type { StepTwoProps } from './types' +import { useCallback, useEffect, useState } from 'react' +import { useTranslation } from 'react-i18next' +import Divider from '@/app/components/base/divider' +import Toast from '@/app/components/base/toast' import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' -import { useDocLink, useLocale } from '@/context/i18n' +import { useLocale } from '@/context/i18n' import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' import { LanguagesSupported } from '@/i18n-config/language' import { DataSourceProvider } from '@/models/common' -import { ChunkingMode, DataSourceType, ProcessMode } from '@/models/datasets' -import { getNotionInfo, getWebsiteInfo, useCreateDocument, useCreateFirstDocument, useFetchDefaultProcessRule, useFetchFileIndexingEstimateForFile, useFetchFileIndexingEstimateForNotion, useFetchFileIndexingEstimateForWeb } from '@/service/knowledge/use-create-dataset' -import { useInvalidDatasetList } from '@/service/knowledge/use-dataset' -import { RETRIEVE_METHOD } from '@/types/app' +import { ChunkingMode, ProcessMode } from '@/models/datasets' +import { useFetchDefaultProcessRule } from '@/service/knowledge/use-create-dataset' import { cn } from '@/utils/classnames' -import { ChunkContainer, QAPreview } from '../../chunk' -import PreviewDocumentPicker from '../../common/document-picker/preview-document-picker' -import { PreviewSlice } from '../../formatted-text/flavours/preview-slice' -import { FormattedText } from '../../formatted-text/formatted' -import PreviewContainer from '../../preview/container' -import { PreviewHeader } from '../../preview/header' -import { checkShowMultiModalTip } from '../../settings/utils' -import FileList from '../assets/file-list-3-fill.svg' -import Note from '../assets/note-mod.svg' -import BlueEffect from '../assets/option-card-effect-blue.svg' -import SettingCog from '../assets/setting-gear-mod.svg' -import { indexMethodIcon } from '../icons' -import escape from './escape' -import s from './index.module.css' -import { DelimiterInput, MaxLengthInput, OverlapInput } from './inputs' -import LanguageSelect from './language-select' -import { OptionCard } from './option-card' -import unescape from './unescape' +import { GeneralChunkingOptions, IndexingModeSection, ParentChildOptions, PreviewPanel, StepTwoFooter } from './components' +import { IndexingType, MAXIMUM_CHUNK_TOKEN_LENGTH, useDocumentCreation, useIndexingConfig, useIndexingEstimate, usePreviewState, useSegmentationState } from './hooks' -const TextLabel: FC = (props) => { - return -} +export { IndexingType } -type StepTwoProps = { - isSetting?: boolean - documentDetail?: FullDocumentDetail - isAPIKeySet: boolean - onSetting: () => void - datasetId?: string - indexingType?: IndexingType - retrievalMethod?: string - dataSourceType: DataSourceType - files: CustomFile[] - notionPages?: NotionPage[] - notionCredentialId: string - websitePages?: CrawlResultItem[] - crawlOptions?: CrawlOptions - websiteCrawlProvider?: DataSourceProvider - websiteCrawlJobId?: string - onStepChange?: (delta: number) => void - updateIndexingTypeCache?: (type: string) => void - updateRetrievalMethodCache?: (method: RETRIEVE_METHOD | '') => void - updateResultCache?: (res: createDocumentResponse) => void - onSave?: () => void - onCancel?: () => void -} - -export enum IndexingType { - QUALIFIED = 'high_quality', - ECONOMICAL = 'economy', -} - -const DEFAULT_SEGMENT_IDENTIFIER = '\\n\\n' -const DEFAULT_MAXIMUM_CHUNK_LENGTH = 1024 -const DEFAULT_OVERLAP = 50 -const MAXIMUM_CHUNK_TOKEN_LENGTH = Number.parseInt(globalThis.document?.body?.getAttribute('data-public-indexing-max-segmentation-tokens-length') || '4000', 10) - -type ParentChildConfig = { - chunkForContext: ParentMode - parent: { - delimiter: string - maxLength: number - } - child: { - delimiter: string - maxLength: number - } -} - -const defaultParentChildConfig: ParentChildConfig = { - chunkForContext: 'paragraph', - parent: { - delimiter: '\\n\\n', - maxLength: 1024, - }, - child: { - delimiter: '\\n', - maxLength: 512, - }, -} - -const StepTwo = ({ +const StepTwo: FC = ({ isSetting, documentDetail, isAPIKeySet, datasetId, - indexingType, + indexingType: propsIndexingType, dataSourceType: inCreatePageDataSourceType, files, notionPages = [], @@ -146,1099 +39,238 @@ const StepTwo = ({ onSave, onCancel, updateRetrievalMethodCache, -}: StepTwoProps) => { +}) => { const { t } = useTranslation() - const docLink = useDocLink() const locale = useLocale() - const media = useBreakpoints() - const isMobile = media === MediaType.mobile - - const currentDataset = useDatasetDetailContextWithSelector(state => state.dataset) - const mutateDatasetRes = useDatasetDetailContextWithSelector(state => state.mutateDatasetRes) + const isMobile = useBreakpoints() === MediaType.mobile + const currentDataset = useDatasetDetailContextWithSelector(s => s.dataset) + const mutateDatasetRes = useDatasetDetailContextWithSelector(s => s.mutateDatasetRes) + // Computed flags const isInUpload = Boolean(currentDataset) const isUploadInEmptyDataset = isInUpload && !currentDataset?.doc_form const isNotUploadInEmptyDataset = !isUploadInEmptyDataset const isInInit = !isInUpload && !isSetting - const isInCreatePage = !datasetId || (datasetId && !currentDataset?.data_source_type) - const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : currentDataset?.data_source_type - const [segmentationType, setSegmentationType] = useState( - currentDataset?.doc_form === ChunkingMode.parentChild ? ProcessMode.parentChild : ProcessMode.general, - ) - const [segmentIdentifier, doSetSegmentIdentifier] = useState(DEFAULT_SEGMENT_IDENTIFIER) - const setSegmentIdentifier = useCallback((value: string, canEmpty?: boolean) => { - doSetSegmentIdentifier(value ? escape(value) : (canEmpty ? '' : DEFAULT_SEGMENT_IDENTIFIER)) - }, []) - const [maxChunkLength, setMaxChunkLength] = useState(DEFAULT_MAXIMUM_CHUNK_LENGTH) // default chunk length - const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(MAXIMUM_CHUNK_TOKEN_LENGTH) - const [overlap, setOverlap] = useState(DEFAULT_OVERLAP) - const [rules, setRules] = useState([]) - const [defaultConfig, setDefaultConfig] = useState() - const hasSetIndexType = !!indexingType - const [indexType, setIndexType] = useState(() => { - if (hasSetIndexType) - return indexingType - return isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL - }) + const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : (currentDataset?.data_source_type ?? inCreatePageDataSourceType) + const hasSetIndexType = !!propsIndexingType + const isModelAndRetrievalConfigDisabled = !!datasetId && !!currentDataset?.data_source_type - const [previewFile, setPreviewFile] = useState( - (datasetId && documentDetail) - ? documentDetail.file - : files[0], - ) - const [previewNotionPage, setPreviewNotionPage] = useState( - (datasetId && documentDetail) - ? documentDetail.notion_page - : notionPages[0], - ) - - const [previewWebsitePage, setPreviewWebsitePage] = useState( - (datasetId && documentDetail) - ? documentDetail.website_page - : websitePages[0], - ) - - // QA Related + // Document form state + const [docForm, setDocForm] = useState((datasetId && documentDetail) ? documentDetail.doc_form as ChunkingMode : ChunkingMode.text) + const [docLanguage, setDocLanguage] = useState(() => (datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese Simplified')) const [isQAConfirmDialogOpen, setIsQAConfirmDialogOpen] = useState(false) - const [docForm, setDocForm] = useState( - (datasetId && documentDetail) ? documentDetail.doc_form as ChunkingMode : ChunkingMode.text, - ) - const handleChangeDocform = (value: ChunkingMode) => { - if (value === ChunkingMode.qa && indexType === IndexingType.ECONOMICAL) { - setIsQAConfirmDialogOpen(true) - return - } - if (value === ChunkingMode.parentChild && indexType === IndexingType.ECONOMICAL) - setIndexType(IndexingType.QUALIFIED) - - setDocForm(value) - - if (value === ChunkingMode.parentChild) - setSegmentationType(ProcessMode.parentChild) - else - setSegmentationType(ProcessMode.general) - - // eslint-disable-next-line ts/no-use-before-define - currentEstimateMutation.reset() - } - - const [docLanguage, setDocLanguage] = useState( - (datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese Simplified'), - ) - - const [parentChildConfig, setParentChildConfig] = useState(defaultParentChildConfig) - - const getIndexing_technique = () => indexingType || indexType const currentDocForm = currentDataset?.doc_form || docForm - const getProcessRule = (): ProcessRule => { - if (currentDocForm === ChunkingMode.parentChild) { - return { - rules: { - pre_processing_rules: rules, - segmentation: { - separator: unescape( - parentChildConfig.parent.delimiter, - ), - max_tokens: parentChildConfig.parent.maxLength, - }, - parent_mode: parentChildConfig.chunkForContext, - subchunk_segmentation: { - separator: unescape(parentChildConfig.child.delimiter), - max_tokens: parentChildConfig.child.maxLength, - }, - }, - mode: 'hierarchical', - } as ProcessRule - } - return { - rules: { - pre_processing_rules: rules, - segmentation: { - separator: unescape(segmentIdentifier), - max_tokens: maxChunkLength, - chunk_overlap: overlap, - }, - }, // api will check this. It will be removed after api refactored. - mode: segmentationType, - } as ProcessRule - } - - const fileIndexingEstimateQuery = useFetchFileIndexingEstimateForFile({ - docForm: currentDocForm, - docLanguage, - dataSourceType: DataSourceType.FILE, - files: previewFile - ? [files.find(file => file.name === previewFile.name)!] - : files, - indexingTechnique: getIndexing_technique() as any, - processRule: getProcessRule(), - dataset_id: datasetId!, + // Custom hooks + const segmentation = useSegmentationState({ + initialSegmentationType: currentDataset?.doc_form === ChunkingMode.parentChild ? ProcessMode.parentChild : ProcessMode.general, }) - const notionIndexingEstimateQuery = useFetchFileIndexingEstimateForNotion({ - docForm: currentDocForm, - docLanguage, - dataSourceType: DataSourceType.NOTION, - notionPages: [previewNotionPage], - indexingTechnique: getIndexing_technique() as any, - processRule: getProcessRule(), - dataset_id: datasetId || '', - credential_id: notionCredentialId, + const indexing = useIndexingConfig({ + initialIndexType: propsIndexingType, + initialEmbeddingModel: currentDataset?.embedding_model ? { provider: currentDataset.embedding_model_provider, model: currentDataset.embedding_model } : undefined, + initialRetrievalConfig: currentDataset?.retrieval_model_dict, + isAPIKeySet, + hasSetIndexType, }) - - const websiteIndexingEstimateQuery = useFetchFileIndexingEstimateForWeb({ - docForm: currentDocForm, - docLanguage, - dataSourceType: DataSourceType.WEB, - websitePages: [previewWebsitePage], + const preview = usePreviewState({ dataSourceType, files, notionPages, websitePages, documentDetail, datasetId }) + const creation = useDocumentCreation({ + datasetId, + isSetting, + documentDetail, + dataSourceType, + files, + notionPages, + notionCredentialId, + websitePages, crawlOptions, websiteCrawlProvider, websiteCrawlJobId, - indexingTechnique: getIndexing_technique() as any, - processRule: getProcessRule(), - dataset_id: datasetId || '', + onStepChange, + updateIndexingTypeCache, + updateResultCache, + updateRetrievalMethodCache, + onSave, + mutateDatasetRes, + }) + const estimateHook = useIndexingEstimate({ + dataSourceType, + datasetId, + currentDocForm, + docLanguage, + files, + previewFileName: preview.previewFile?.name, + previewNotionPage: preview.previewNotionPage, + notionCredentialId, + previewWebsitePage: preview.previewWebsitePage, + crawlOptions, + websiteCrawlProvider, + websiteCrawlJobId, + indexingTechnique: indexing.getIndexingTechnique() as IndexingType, + processRule: segmentation.getProcessRule(currentDocForm), }) - const currentEstimateMutation = dataSourceType === DataSourceType.FILE - ? fileIndexingEstimateQuery - : dataSourceType === DataSourceType.NOTION - ? notionIndexingEstimateQuery - : websiteIndexingEstimateQuery + // Fetch default process rule + const fetchDefaultProcessRuleMutation = useFetchDefaultProcessRule({ + onSuccess(data) { + segmentation.setSegmentIdentifier(data.rules.segmentation.separator) + segmentation.setMaxChunkLength(data.rules.segmentation.max_tokens) + segmentation.setOverlap(data.rules.segmentation.chunk_overlap!) + segmentation.setRules(data.rules.pre_processing_rules) + segmentation.setDefaultConfig(data.rules) + segmentation.setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length) + }, + }) - const fetchEstimate = useCallback(() => { - if (dataSourceType === DataSourceType.FILE) - fileIndexingEstimateQuery.mutate() - - if (dataSourceType === DataSourceType.NOTION) - notionIndexingEstimateQuery.mutate() - - if (dataSourceType === DataSourceType.WEB) - websiteIndexingEstimateQuery.mutate() - }, [dataSourceType, fileIndexingEstimateQuery, notionIndexingEstimateQuery, websiteIndexingEstimateQuery]) - - const estimate - = dataSourceType === DataSourceType.FILE - ? fileIndexingEstimateQuery.data - : dataSourceType === DataSourceType.NOTION - ? notionIndexingEstimateQuery.data - : websiteIndexingEstimateQuery.data - - const getRuleName = (key: string) => { - if (key === 'remove_extra_spaces') - return t('stepTwo.removeExtraSpaces', { ns: 'datasetCreation' }) - - if (key === 'remove_urls_emails') - return t('stepTwo.removeUrlEmails', { ns: 'datasetCreation' }) - - if (key === 'remove_stopwords') - return t('stepTwo.removeStopwords', { ns: 'datasetCreation' }) - } - const ruleChangeHandle = (id: string) => { - const newRules = rules.map((rule) => { - if (rule.id === id) { - return { - id: rule.id, - enabled: !rule.enabled, - } - } - return rule - }) - setRules(newRules) - } - const resetRules = () => { - if (defaultConfig) { - setSegmentIdentifier(defaultConfig.segmentation.separator) - setMaxChunkLength(defaultConfig.segmentation.max_tokens) - setOverlap(defaultConfig.segmentation.chunk_overlap!) - setRules(defaultConfig.pre_processing_rules) + // Event handlers + const handleDocFormChange = useCallback((value: ChunkingMode) => { + if (value === ChunkingMode.qa && indexing.indexType === IndexingType.ECONOMICAL) { + setIsQAConfirmDialogOpen(true) + return } - setParentChildConfig(defaultParentChildConfig) - } + if (value === ChunkingMode.parentChild && indexing.indexType === IndexingType.ECONOMICAL) + indexing.setIndexType(IndexingType.QUALIFIED) + setDocForm(value) + segmentation.setSegmentationType(value === ChunkingMode.parentChild ? ProcessMode.parentChild : ProcessMode.general) + estimateHook.reset() + }, [indexing, segmentation, estimateHook]) - const updatePreview = () => { - if (segmentationType === ProcessMode.general && maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH) { + const updatePreview = useCallback(() => { + if (segmentation.segmentationType === ProcessMode.general && segmentation.maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH) { Toast.notify({ type: 'error', message: t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: MAXIMUM_CHUNK_TOKEN_LENGTH }) }) return } - fetchEstimate() - } + estimateHook.fetchEstimate() + }, [segmentation, t, estimateHook]) - const { - modelList: rerankModelList, - defaultModel: rerankDefaultModel, - currentModel: isRerankDefaultModelValid, - } = useModelListAndDefaultModelAndCurrentProviderAndModel(ModelTypeEnum.rerank) - const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding) - const { data: defaultEmbeddingModel } = useDefaultModel(ModelTypeEnum.textEmbedding) - const [embeddingModel, setEmbeddingModel] = useState( - currentDataset?.embedding_model - ? { - provider: currentDataset.embedding_model_provider, - model: currentDataset.embedding_model, - } - : { - provider: defaultEmbeddingModel?.provider.provider || '', - model: defaultEmbeddingModel?.model || '', - }, - ) - const [retrievalConfig, setRetrievalConfig] = useState(currentDataset?.retrieval_model_dict || { - search_method: RETRIEVE_METHOD.semantic, - reranking_enable: false, - reranking_model: { - reranking_provider_name: '', - reranking_model_name: '', - }, - top_k: 3, - score_threshold_enabled: false, - score_threshold: 0.5, - } as RetrievalConfig) - - useEffect(() => { - if (currentDataset?.retrieval_model_dict) - return - setRetrievalConfig({ - search_method: RETRIEVE_METHOD.semantic, - reranking_enable: !!isRerankDefaultModelValid, - reranking_model: { - reranking_provider_name: isRerankDefaultModelValid ? rerankDefaultModel?.provider.provider ?? '' : '', - reranking_model_name: isRerankDefaultModelValid ? rerankDefaultModel?.model ?? '' : '', - }, - top_k: 3, - score_threshold_enabled: false, - score_threshold: 0.5, + const handleCreate = useCallback(async () => { + const isValid = creation.validateParams({ + segmentationType: segmentation.segmentationType, + maxChunkLength: segmentation.maxChunkLength, + limitMaxChunkLength: segmentation.limitMaxChunkLength, + overlap: segmentation.overlap, + indexType: indexing.indexType, + embeddingModel: indexing.embeddingModel, + rerankModelList: indexing.rerankModelList, + retrievalConfig: indexing.retrievalConfig, }) - }, [rerankDefaultModel, isRerankDefaultModelValid]) - - const getCreationParams = () => { - let params - if (segmentationType === ProcessMode.general && overlap > maxChunkLength) { - Toast.notify({ type: 'error', message: t('stepTwo.overlapCheck', { ns: 'datasetCreation' }) }) + if (!isValid) return - } - if (segmentationType === ProcessMode.general && maxChunkLength > limitMaxChunkLength) { - Toast.notify({ type: 'error', message: t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: limitMaxChunkLength }) }) - return - } - if (isSetting) { - params = { - original_document_id: documentDetail?.id, - doc_form: currentDocForm, - doc_language: docLanguage, - process_rule: getProcessRule(), - retrieval_model: retrievalConfig, // Readonly. If want to changed, just go to settings page. - embedding_model: embeddingModel.model, // Readonly - embedding_model_provider: embeddingModel.provider, // Readonly - indexing_technique: getIndexing_technique(), - } as CreateDocumentReq - } - else { // create - const indexMethod = getIndexing_technique() - if (indexMethod === IndexingType.QUALIFIED && (!embeddingModel.model || !embeddingModel.provider)) { - Toast.notify({ - type: 'error', - message: t('datasetConfig.embeddingModelRequired', { ns: 'appDebug' }), - }) - return - } - if ( - !isReRankModelSelected({ - rerankModelList, - retrievalConfig, - indexMethod: indexMethod as string, - }) - ) { - Toast.notify({ type: 'error', message: t('datasetConfig.rerankModelRequired', { ns: 'appDebug' }) }) - return - } - params = { - data_source: { - type: dataSourceType, - info_list: { - data_source_type: dataSourceType, - }, - }, - indexing_technique: getIndexing_technique(), - process_rule: getProcessRule(), - doc_form: currentDocForm, - doc_language: docLanguage, - retrieval_model: retrievalConfig, - embedding_model: embeddingModel.model, - embedding_model_provider: embeddingModel.provider, - } as CreateDocumentReq - if (dataSourceType === DataSourceType.FILE) { - params.data_source.info_list.file_info_list = { - file_ids: files.map(file => file.id || '').filter(Boolean), - } - } - if (dataSourceType === DataSourceType.NOTION) - params.data_source.info_list.notion_info_list = getNotionInfo(notionPages, notionCredentialId) - - if (dataSourceType === DataSourceType.WEB) { - params.data_source.info_list.website_info_list = getWebsiteInfo({ - websiteCrawlProvider, - websiteCrawlJobId, - websitePages, - }) - } - } - return params - } - - const fetchDefaultProcessRuleMutation = useFetchDefaultProcessRule({ - onSuccess(data) { - const separator = data.rules.segmentation.separator - setSegmentIdentifier(separator) - setMaxChunkLength(data.rules.segmentation.max_tokens) - setOverlap(data.rules.segmentation.chunk_overlap!) - setRules(data.rules.pre_processing_rules) - setDefaultConfig(data.rules) - setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length) - }, - }) - - const getRulesFromDetail = () => { - if (documentDetail) { - const rules = documentDetail.dataset_process_rule.rules - const separator = rules.segmentation.separator - const max = rules.segmentation.max_tokens - const overlap = rules.segmentation.chunk_overlap - const isHierarchicalDocument = documentDetail.doc_form === ChunkingMode.parentChild - || (rules.parent_mode && rules.subchunk_segmentation) - setSegmentIdentifier(separator) - setMaxChunkLength(max) - setOverlap(overlap!) - setRules(rules.pre_processing_rules) - setDefaultConfig(rules) - - if (isHierarchicalDocument) { - setParentChildConfig({ - chunkForContext: rules.parent_mode || 'paragraph', - parent: { - delimiter: escape(rules.segmentation.separator), - maxLength: rules.segmentation.max_tokens, - }, - child: { - delimiter: escape(rules.subchunk_segmentation.separator), - maxLength: rules.subchunk_segmentation.max_tokens, - }, - }) - } - } - } - - const getDefaultMode = () => { - if (documentDetail) - setSegmentationType(documentDetail.dataset_process_rule.mode) - } - - const createFirstDocumentMutation = useCreateFirstDocument() - const createDocumentMutation = useCreateDocument(datasetId!) - - const isCreating = createFirstDocumentMutation.isPending || createDocumentMutation.isPending - const invalidDatasetList = useInvalidDatasetList() - - const createHandle = async () => { - const params = getCreationParams() + const params = creation.buildCreationParams(currentDocForm, docLanguage, segmentation.getProcessRule(currentDocForm), indexing.retrievalConfig, indexing.embeddingModel, indexing.getIndexingTechnique()) if (!params) - return false + return + await creation.executeCreation(params, indexing.indexType, indexing.retrievalConfig) + }, [creation, segmentation, indexing, currentDocForm, docLanguage]) - if (!datasetId) { - await createFirstDocumentMutation.mutateAsync( - params, - { - onSuccess(data) { - updateIndexingTypeCache?.(indexType as string) - updateResultCache?.(data) - updateRetrievalMethodCache?.(retrievalConfig.search_method as RETRIEVE_METHOD) - }, - }, - ) - } - else { - await createDocumentMutation.mutateAsync(params, { - onSuccess(data) { - updateIndexingTypeCache?.(indexType as string) - updateResultCache?.(data) - updateRetrievalMethodCache?.(retrievalConfig.search_method as RETRIEVE_METHOD) - }, - }) - } - if (mutateDatasetRes) - mutateDatasetRes() - invalidDatasetList() - trackEvent('create_datasets', { - data_source_type: dataSourceType, - indexing_technique: getIndexing_technique(), - }) - onStepChange?.(+1) - if (isSetting) - onSave?.() - } + const handlePickerChange = useCallback((selected: { id: string, name: string }) => { + estimateHook.reset() + preview.handlePreviewChange(selected) + estimateHook.fetchEstimate() + }, [estimateHook, preview]) + const handleQAConfirm = useCallback(() => { + setIsQAConfirmDialogOpen(false) + indexing.setIndexType(IndexingType.QUALIFIED) + setDocForm(ChunkingMode.qa) + }, [indexing]) + + // Initialize rules useEffect(() => { - // fetch rules if (!isSetting) { fetchDefaultProcessRuleMutation.mutate('/datasets/process-rule') } - else { - getRulesFromDetail() - getDefaultMode() + else if (documentDetail) { + const rules = documentDetail.dataset_process_rule.rules + const isHierarchical = documentDetail.doc_form === ChunkingMode.parentChild || Boolean(rules.parent_mode && rules.subchunk_segmentation) + segmentation.applyConfigFromRules(rules, isHierarchical) + segmentation.setSegmentationType(documentDetail.dataset_process_rule.mode) } + // eslint-disable-next-line react-hooks/exhaustive-deps }, []) - useEffect(() => { - // get indexing type by props - if (indexingType) - setIndexType(indexingType as IndexingType) - else - setIndexType(isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL) - }, [isAPIKeySet, indexingType, datasetId]) - - const isModelAndRetrievalConfigDisabled = !!datasetId && !!currentDataset?.data_source_type - - const showMultiModalTip = useMemo(() => { - return checkShowMultiModalTip({ - embeddingModel, - rerankingEnable: retrievalConfig.reranking_enable, - rerankModel: { - rerankingProviderName: retrievalConfig.reranking_model.reranking_provider_name, - rerankingModelName: retrievalConfig.reranking_model.reranking_model_name, - }, - indexMethod: indexType, - embeddingModelList, - rerankModelList, - }) - }, [embeddingModel, retrievalConfig.reranking_enable, retrievalConfig.reranking_model, indexType, embeddingModelList, rerankModelList]) + // Show options conditions + const showGeneralOption = (isInUpload && [ChunkingMode.text, ChunkingMode.qa].includes(currentDataset!.doc_form)) || isUploadInEmptyDataset || isInInit + const showParentChildOption = (isInUpload && currentDataset!.doc_form === ChunkingMode.parentChild) || isUploadInEmptyDataset || isInInit return (
{t('stepTwo.segmentation', { ns: 'datasetCreation' })}
- {((isInUpload && [ChunkingMode.text, ChunkingMode.qa].includes(currentDataset!.doc_form)) - || isUploadInEmptyDataset - || isInInit) - && ( - } - activeHeaderClassName="bg-dataset-option-card-blue-gradient" - description={t('stepTwo.generalTip', { ns: 'datasetCreation' })} - isActive={ - [ChunkingMode.text, ChunkingMode.qa].includes(currentDocForm) - } - onSwitched={() => - handleChangeDocform(ChunkingMode.text)} - actions={( - <> - - - - )} - noHighlight={isInUpload && isNotUploadInEmptyDataset} - > -
-
- setSegmentIdentifier(e.target.value, true)} - /> - - -
-
-
-
- {t('stepTwo.rules', { ns: 'datasetCreation' })} -
- -
-
- {rules.map(rule => ( -
{ - ruleChangeHandle(rule.id) - }} - > - - -
- ))} - {IS_CE_EDITION && ( - <> - -
-
{ - if (currentDataset?.doc_form) - return - if (docForm === ChunkingMode.qa) - handleChangeDocform(ChunkingMode.text) - else - handleChangeDocform(ChunkingMode.qa) - }} - > - - -
- - -
- {currentDocForm === ChunkingMode.qa && ( -
- - - {t('stepTwo.QATip', { ns: 'datasetCreation' })} - -
- )} - - )} -
-
-
- + {showGeneralOption && ( + segmentation.setSegmentIdentifier(value, true)} + onMaxChunkLengthChange={segmentation.setMaxChunkLength} + onOverlapChange={segmentation.setOverlap} + onRuleToggle={segmentation.toggleRule} + onDocFormChange={handleDocFormChange} + onDocLanguageChange={setDocLanguage} + onPreview={updatePreview} + onReset={segmentation.resetToDefaults} + locale={locale} + /> )} - { - ( - (isInUpload && currentDataset!.doc_form === ChunkingMode.parentChild) - || isUploadInEmptyDataset - || isInInit - ) - && ( - } - effectImg={BlueEffect.src} - className="text-util-colors-blue-light-blue-light-500" - activeHeaderClassName="bg-dataset-option-card-blue-gradient" - description={t('stepTwo.parentChildTip', { ns: 'datasetCreation' })} - isActive={currentDocForm === ChunkingMode.parentChild} - onSwitched={() => handleChangeDocform(ChunkingMode.parentChild)} - actions={( - <> - - - - )} - noHighlight={isInUpload && isNotUploadInEmptyDataset} - > -
-
-
-
- {t('stepTwo.parentChunkForContext', { ns: 'datasetCreation' })} -
- -
- } - title={t('stepTwo.paragraph', { ns: 'datasetCreation' })} - description={t('stepTwo.paragraphTip', { ns: 'datasetCreation' })} - isChosen={parentChildConfig.chunkForContext === 'paragraph'} - onChosen={() => setParentChildConfig( - { - ...parentChildConfig, - chunkForContext: 'paragraph', - }, - )} - chosenConfig={( -
- setParentChildConfig({ - ...parentChildConfig, - parent: { - ...parentChildConfig.parent, - delimiter: e.target.value ? escape(e.target.value) : '', - }, - })} - /> - setParentChildConfig({ - ...parentChildConfig, - parent: { - ...parentChildConfig.parent, - maxLength: value, - }, - })} - /> -
- )} - /> - } - title={t('stepTwo.fullDoc', { ns: 'datasetCreation' })} - description={t('stepTwo.fullDocTip', { ns: 'datasetCreation' })} - onChosen={() => setParentChildConfig( - { - ...parentChildConfig, - chunkForContext: 'full-doc', - }, - )} - isChosen={parentChildConfig.chunkForContext === 'full-doc'} - /> -
- -
-
-
- {t('stepTwo.childChunkForRetrieval', { ns: 'datasetCreation' })} -
- -
-
- setParentChildConfig({ - ...parentChildConfig, - child: { - ...parentChildConfig.child, - delimiter: e.target.value ? escape(e.target.value) : '', - }, - })} - /> - setParentChildConfig({ - ...parentChildConfig, - child: { - ...parentChildConfig.child, - maxLength: value, - }, - })} - /> -
-
-
-
-
- {t('stepTwo.rules', { ns: 'datasetCreation' })} -
- -
-
- {rules.map(rule => ( -
{ - ruleChangeHandle(rule.id) - }} - > - - -
- ))} -
-
-
- - ) - } - -
{t('stepTwo.indexMode', { ns: 'datasetCreation' })}
-
- {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && ( - - {t('stepTwo.qualified', { ns: 'datasetCreation' })} - - {t('stepTwo.recommend', { ns: 'datasetCreation' })} - - - {!hasSetIndexType && } - -
- )} - description={t('stepTwo.qualifiedTip', { ns: 'datasetCreation' })} - icon={} - isActive={!hasSetIndexType && indexType === IndexingType.QUALIFIED} - disabled={hasSetIndexType} - onSwitched={() => { - setIndexType(IndexingType.QUALIFIED) - }} - /> - )} - - {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && ( - <> - setIsQAConfirmDialogOpen(false)} className="w-[432px]"> -
-

- {t('stepTwo.qaSwitchHighQualityTipTitle', { ns: 'datasetCreation' })} -

-

- {t('stepTwo.qaSwitchHighQualityTipContent', { ns: 'datasetCreation' })} -

-
-
- - -
- - - { - docForm === ChunkingMode.qa - ? t('stepTwo.notAvailableForQA', { ns: 'datasetCreation' }) - : t('stepTwo.notAvailableForParentChild', { ns: 'datasetCreation' }) - } -
- )} - noDecoration - position="top" - asChild={false} - triggerClassName="flex-1 self-stretch" - > - } - isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL} - disabled={hasSetIndexType || docForm !== ChunkingMode.text} - onSwitched={() => { - setIndexType(IndexingType.ECONOMICAL) - }} - /> - - - )} -
- {!hasSetIndexType && indexType === IndexingType.QUALIFIED && ( -
-
-
- -
- {t('stepTwo.highQualityTip', { ns: 'datasetCreation' })} -
- )} - {hasSetIndexType && indexType === IndexingType.ECONOMICAL && ( -
- {t('stepTwo.indexSettingTip', { ns: 'datasetCreation' })} - {t('stepTwo.datasetSettingLink', { ns: 'datasetCreation' })} -
- )} - {/* Embedding model */} - {indexType === IndexingType.QUALIFIED && ( -
-
{t('form.embeddingModel', { ns: 'datasetSettings' })}
- { - setEmbeddingModel(model) - }} - /> - {isModelAndRetrievalConfigDisabled && ( -
- {t('stepTwo.indexSettingTip', { ns: 'datasetCreation' })} - {t('stepTwo.datasetSettingLink', { ns: 'datasetCreation' })} -
- )} -
+ {showParentChildOption && ( + segmentation.updateParentConfig('delimiter', v)} + onParentMaxLengthChange={v => segmentation.updateParentConfig('maxLength', v)} + onChildDelimiterChange={v => segmentation.updateChildConfig('delimiter', v)} + onChildMaxLengthChange={v => segmentation.updateChildConfig('maxLength', v)} + onRuleToggle={segmentation.toggleRule} + onPreview={updatePreview} + onReset={segmentation.resetToDefaults} + /> )} - {/* Retrieval Method Config */} -
- {!isModelAndRetrievalConfigDisabled - ? ( -
-
{t('form.retrievalSetting.title', { ns: 'datasetSettings' })}
-
- - {t('form.retrievalSetting.learnMore', { ns: 'datasetSettings' })} - - {t('form.retrievalSetting.longDescription', { ns: 'datasetSettings' })} -
-
- ) - : ( -
-
{t('form.retrievalSetting.title', { ns: 'datasetSettings' })}
-
- )} - -
- { - getIndexing_technique() === IndexingType.QUALIFIED - ? ( - - ) - : ( - - ) - } -
-
- - {!isSetting - ? ( -
- - -
- ) - : ( -
- - -
- )} + setIsQAConfirmDialogOpen(false)} + onQAConfirmDialogConfirm={handleQAConfirm} + /> + onStepChange?.(-1)} onCreate={handleCreate} onCancel={onCancel} />
- - -
- {dataSourceType === DataSourceType.FILE - && ( - >} - onChange={(selected) => { - currentEstimateMutation.reset() - setPreviewFile(selected) - currentEstimateMutation.mutate() - }} - // when it is from setting, it just has one file - value={isSetting ? (files[0]! as Required) : previewFile} - /> - )} - {dataSourceType === DataSourceType.NOTION - && ( - ({ - id: page.page_id, - name: page.page_name, - extension: 'md', - })) - } - onChange={(selected) => { - currentEstimateMutation.reset() - const selectedPage = notionPages.find(page => page.page_id === selected.id) - setPreviewNotionPage(selectedPage!) - currentEstimateMutation.mutate() - }} - value={{ - id: previewNotionPage?.page_id || '', - name: previewNotionPage?.page_name || '', - extension: 'md', - }} - /> - )} - {dataSourceType === DataSourceType.WEB - && ( - ({ - id: page.source_url, - name: page.title, - extension: 'md', - })) - } - onChange={(selected) => { - currentEstimateMutation.reset() - const selectedPage = websitePages.find(page => page.source_url === selected.id) - setPreviewWebsitePage(selectedPage!) - currentEstimateMutation.mutate() - }} - value={ - { - id: previewWebsitePage?.source_url || '', - name: previewWebsitePage?.title || '', - extension: 'md', - } - } - /> - )} - { - currentDocForm !== ChunkingMode.qa - && ( - - ) - } -
- - )} - className={cn('relative flex h-full w-1/2 shrink-0 p-4 pr-0', isMobile && 'w-full max-w-[524px]')} - mainClassName="space-y-6" - > - {currentDocForm === ChunkingMode.qa && estimate?.qa_preview && ( - estimate?.qa_preview.map((item, index) => ( - - - - )) - )} - {currentDocForm === ChunkingMode.text && estimate?.preview && ( - estimate?.preview.map((item, index) => ( - - {item.content} - - )) - )} - {currentDocForm === ChunkingMode.parentChild && currentEstimateMutation.data?.preview && ( - estimate?.preview?.map((item, index) => { - const indexForLabel = index + 1 - const childChunks = parentChildConfig.chunkForContext === 'full-doc' - ? item.child_chunks.slice(0, FULL_DOC_PREVIEW_LENGTH) - : item.child_chunks - return ( - - - {childChunks.map((child, index) => { - const indexForLabel = index + 1 - return ( - - ) - })} - - - ) - }) - )} - {currentEstimateMutation.isIdle && ( -
-
- -

- {t('stepTwo.previewChunkTip', { ns: 'datasetCreation' })} -

-
-
- )} - {currentEstimateMutation.isPending && ( -
- {Array.from({ length: 10 }, (_, i) => ( - - - - - - - - - - - ))} -
- )} - - + } + pickerValue={preview.getPreviewPickerValue()} + isIdle={estimateHook.isIdle} + isPending={estimateHook.isPending} + onPickerChange={handlePickerChange} + />
) } diff --git a/web/app/components/datasets/create/step-two/types.ts b/web/app/components/datasets/create/step-two/types.ts new file mode 100644 index 0000000000..7f5291fb13 --- /dev/null +++ b/web/app/components/datasets/create/step-two/types.ts @@ -0,0 +1,28 @@ +import type { IndexingType } from './hooks' +import type { DataSourceProvider, NotionPage } from '@/models/common' +import type { CrawlOptions, CrawlResultItem, createDocumentResponse, CustomFile, DataSourceType, FullDocumentDetail } from '@/models/datasets' +import type { RETRIEVE_METHOD } from '@/types/app' + +export type StepTwoProps = { + isSetting?: boolean + documentDetail?: FullDocumentDetail + isAPIKeySet: boolean + onSetting: () => void + datasetId?: string + indexingType?: IndexingType + retrievalMethod?: string + dataSourceType: DataSourceType + files: CustomFile[] + notionPages?: NotionPage[] + notionCredentialId: string + websitePages?: CrawlResultItem[] + crawlOptions?: CrawlOptions + websiteCrawlProvider?: DataSourceProvider + websiteCrawlJobId?: string + onStepChange?: (delta: number) => void + updateIndexingTypeCache?: (type: string) => void + updateRetrievalMethodCache?: (method: RETRIEVE_METHOD | '') => void + updateResultCache?: (res: createDocumentResponse) => void + onSave?: () => void + onCancel?: () => void +} From 98df99b0ca2f13db64eba18e37489db5c534eadd Mon Sep 17 00:00:00 2001 From: Coding On Star <447357187@qq.com> Date: Fri, 9 Jan 2026 10:21:27 +0800 Subject: [PATCH 21/21] feat(embedding-process): implement embedding process components and polling logic (#30622) Co-authored-by: CodingOnStar --- .../create/embedding-process/index.spec.tsx | 1562 +++++++++++++++++ .../create/embedding-process/index.tsx | 420 +---- .../indexing-progress-item.tsx | 120 ++ .../create/embedding-process/rule-detail.tsx | 133 ++ .../embedding-process/upgrade-banner.tsx | 22 + .../use-indexing-status-polling.ts | 90 + .../create/embedding-process/utils.ts | 64 + 7 files changed, 2086 insertions(+), 325 deletions(-) create mode 100644 web/app/components/datasets/create/embedding-process/index.spec.tsx create mode 100644 web/app/components/datasets/create/embedding-process/indexing-progress-item.tsx create mode 100644 web/app/components/datasets/create/embedding-process/rule-detail.tsx create mode 100644 web/app/components/datasets/create/embedding-process/upgrade-banner.tsx create mode 100644 web/app/components/datasets/create/embedding-process/use-indexing-status-polling.ts create mode 100644 web/app/components/datasets/create/embedding-process/utils.ts diff --git a/web/app/components/datasets/create/embedding-process/index.spec.tsx b/web/app/components/datasets/create/embedding-process/index.spec.tsx new file mode 100644 index 0000000000..8d2bae03cd --- /dev/null +++ b/web/app/components/datasets/create/embedding-process/index.spec.tsx @@ -0,0 +1,1562 @@ +import type { FullDocumentDetail, IndexingStatusResponse, ProcessRuleResponse } from '@/models/datasets' +import { act, render, renderHook, screen } from '@testing-library/react' +import { DataSourceType, ProcessMode } from '@/models/datasets' +import { RETRIEVE_METHOD } from '@/types/app' +import IndexingProgressItem from './indexing-progress-item' +import RuleDetail from './rule-detail' +import UpgradeBanner from './upgrade-banner' +import { useIndexingStatusPolling } from './use-indexing-status-polling' +import { + createDocumentLookup, + getFileType, + getSourcePercent, + isLegacyDataSourceInfo, + isSourceEmbedding, +} from './utils' + +// ============================================================================= +// Mock External Dependencies +// ============================================================================= + +// Mock next/navigation +const mockPush = vi.fn() +const mockRouter = { push: mockPush } +vi.mock('next/navigation', () => ({ + useRouter: () => mockRouter, +})) + +// Mock next/image +vi.mock('next/image', () => ({ + default: ({ src, alt, className }: { src: string, alt: string, className?: string }) => ( + // eslint-disable-next-line next/no-img-element + {alt} + ), +})) + +// Mock API service +const mockFetchIndexingStatusBatch = vi.fn() +vi.mock('@/service/datasets', () => ({ + fetchIndexingStatusBatch: (params: { datasetId: string, batchId: string }) => + mockFetchIndexingStatusBatch(params), +})) + +// Mock service hooks +const mockProcessRuleData: ProcessRuleResponse | undefined = undefined +vi.mock('@/service/knowledge/use-dataset', () => ({ + useProcessRule: vi.fn(() => ({ data: mockProcessRuleData })), +})) + +const mockInvalidDocumentList = vi.fn() +vi.mock('@/service/knowledge/use-document', () => ({ + useInvalidDocumentList: () => mockInvalidDocumentList, +})) + +// Mock useDatasetApiAccessUrl hook +vi.mock('@/hooks/use-api-access-url', () => ({ + useDatasetApiAccessUrl: () => 'https://api.example.com/docs', +})) + +// Mock provider context +let mockEnableBilling = false +let mockPlanType = 'sandbox' +vi.mock('@/context/provider-context', () => ({ + useProviderContext: () => ({ + enableBilling: mockEnableBilling, + plan: { type: mockPlanType }, + }), +})) + +// Mock icons +vi.mock('../icons', () => ({ + indexMethodIcon: { + economical: '/icons/economical.svg', + high_quality: '/icons/high-quality.svg', + }, + retrievalIcon: { + fullText: '/icons/full-text.svg', + hybrid: '/icons/hybrid.svg', + vector: '/icons/vector.svg', + }, +})) + +// Mock IndexingType enum from step-two +vi.mock('../step-two', () => ({ + IndexingType: { + QUALIFIED: 'high_quality', + ECONOMICAL: 'economy', + }, +})) + +// ============================================================================= +// Factory Functions for Test Data +// ============================================================================= + +/** + * Create a mock IndexingStatusResponse + */ +const createMockIndexingStatus = ( + overrides: Partial = {}, +): IndexingStatusResponse => ({ + id: 'doc-1', + indexing_status: 'completed', + processing_started_at: Date.now(), + parsing_completed_at: Date.now(), + cleaning_completed_at: Date.now(), + splitting_completed_at: Date.now(), + completed_at: Date.now(), + paused_at: null, + error: null, + stopped_at: null, + completed_segments: 10, + total_segments: 10, + ...overrides, +}) + +/** + * Create a mock FullDocumentDetail + */ +const createMockDocument = ( + overrides: Partial = {}, +): FullDocumentDetail => ({ + id: 'doc-1', + name: 'test-document.txt', + data_source_type: DataSourceType.FILE, + data_source_info: { + upload_file: { + id: 'file-1', + name: 'test-document.txt', + extension: 'txt', + mime_type: 'text/plain', + size: 1024, + created_by: 'user-1', + created_at: Date.now(), + }, + }, + batch: 'batch-1', + created_api_request_id: 'req-1', + processing_started_at: Date.now(), + parsing_completed_at: Date.now(), + cleaning_completed_at: Date.now(), + splitting_completed_at: Date.now(), + tokens: 100, + indexing_latency: 5000, + completed_at: Date.now(), + paused_by: '', + paused_at: 0, + stopped_at: 0, + indexing_status: 'completed', + disabled_at: 0, + ...overrides, +} as FullDocumentDetail) + +/** + * Create a mock ProcessRuleResponse + */ +const createMockProcessRule = ( + overrides: Partial = {}, +): ProcessRuleResponse => ({ + mode: ProcessMode.general, + rules: { + segmentation: { + separator: '\n', + max_tokens: 500, + chunk_overlap: 50, + }, + pre_processing_rules: [ + { id: 'remove_extra_spaces', enabled: true }, + { id: 'remove_urls_emails', enabled: false }, + ], + }, + ...overrides, +} as ProcessRuleResponse) + +// ============================================================================= +// Utils Tests +// ============================================================================= + +describe('utils', () => { + // Test utility functions for document handling + + describe('isLegacyDataSourceInfo', () => { + it('should return true for legacy data source with upload_file object', () => { + // Arrange + const info = { + upload_file: { id: 'file-1', name: 'test.txt' }, + } + + // Act & Assert + expect(isLegacyDataSourceInfo(info as Parameters[0])).toBe(true) + }) + + it('should return false for null', () => { + expect(isLegacyDataSourceInfo(null as unknown as Parameters[0])).toBe(false) + }) + + it('should return false for undefined', () => { + expect(isLegacyDataSourceInfo(undefined as unknown as Parameters[0])).toBe(false) + }) + + it('should return false when upload_file is not an object', () => { + // Arrange + const info = { upload_file: 'string-value' } + + // Act & Assert + expect(isLegacyDataSourceInfo(info as unknown as Parameters[0])).toBe(false) + }) + }) + + describe('isSourceEmbedding', () => { + it.each([ + ['indexing', true], + ['splitting', true], + ['parsing', true], + ['cleaning', true], + ['waiting', true], + ['completed', false], + ['error', false], + ['paused', false], + ])('should return %s for status "%s"', (status, expected) => { + // Arrange + const detail = createMockIndexingStatus({ indexing_status: status as IndexingStatusResponse['indexing_status'] }) + + // Act & Assert + expect(isSourceEmbedding(detail)).toBe(expected) + }) + }) + + describe('getSourcePercent', () => { + it('should return 0 when total_segments is 0', () => { + // Arrange + const detail = createMockIndexingStatus({ + completed_segments: 0, + total_segments: 0, + }) + + // Act & Assert + expect(getSourcePercent(detail)).toBe(0) + }) + + it('should calculate correct percentage', () => { + // Arrange + const detail = createMockIndexingStatus({ + completed_segments: 5, + total_segments: 10, + }) + + // Act & Assert + expect(getSourcePercent(detail)).toBe(50) + }) + + it('should cap percentage at 100', () => { + // Arrange + const detail = createMockIndexingStatus({ + completed_segments: 15, + total_segments: 10, + }) + + // Act & Assert + expect(getSourcePercent(detail)).toBe(100) + }) + + it('should handle undefined values', () => { + // Arrange + const detail = { indexing_status: 'indexing' } as IndexingStatusResponse + + // Act & Assert + expect(getSourcePercent(detail)).toBe(0) + }) + + it('should round to nearest integer', () => { + // Arrange + const detail = createMockIndexingStatus({ + completed_segments: 1, + total_segments: 3, + }) + + // Act & Assert + expect(getSourcePercent(detail)).toBe(33) + }) + }) + + describe('getFileType', () => { + it('should extract extension from filename', () => { + expect(getFileType('document.pdf')).toBe('pdf') + expect(getFileType('file.name.txt')).toBe('txt') + expect(getFileType('archive.tar.gz')).toBe('gz') + }) + + it('should return "txt" for undefined', () => { + expect(getFileType(undefined)).toBe('txt') + }) + + it('should return filename without extension', () => { + expect(getFileType('filename')).toBe('filename') + }) + }) + + describe('createDocumentLookup', () => { + it('should create lookup functions for documents', () => { + // Arrange + const documents = [ + createMockDocument({ id: 'doc-1', name: 'file1.txt' }), + createMockDocument({ id: 'doc-2', name: 'file2.pdf', data_source_type: DataSourceType.NOTION }), + ] + + // Act + const lookup = createDocumentLookup(documents) + + // Assert + expect(lookup.getName('doc-1')).toBe('file1.txt') + expect(lookup.getName('doc-2')).toBe('file2.pdf') + expect(lookup.getName('non-existent')).toBeUndefined() + }) + + it('should return source type correctly', () => { + // Arrange + const documents = [ + createMockDocument({ id: 'doc-1', data_source_type: DataSourceType.FILE }), + createMockDocument({ id: 'doc-2', data_source_type: DataSourceType.NOTION }), + ] + const lookup = createDocumentLookup(documents) + + // Assert + expect(lookup.getSourceType('doc-1')).toBe(DataSourceType.FILE) + expect(lookup.getSourceType('doc-2')).toBe(DataSourceType.NOTION) + }) + + it('should return notion icon for legacy data source', () => { + // Arrange + const documents = [ + createMockDocument({ + id: 'doc-1', + data_source_info: { + upload_file: { id: 'f1' }, + notion_page_icon: '📄', + } as FullDocumentDetail['data_source_info'], + }), + ] + const lookup = createDocumentLookup(documents) + + // Assert + expect(lookup.getNotionIcon('doc-1')).toBe('📄') + }) + + it('should return undefined for non-legacy notion icon', () => { + // Arrange + const documents = [ + createMockDocument({ + id: 'doc-1', + data_source_info: { some_other_field: 'value' } as unknown as FullDocumentDetail['data_source_info'], + }), + ] + const lookup = createDocumentLookup(documents) + + // Assert + expect(lookup.getNotionIcon('doc-1')).toBeUndefined() + }) + + it('should memoize lookups with Map for performance', () => { + // Arrange + const documents = Array.from({ length: 1000 }, (_, i) => + createMockDocument({ id: `doc-${i}`, name: `file${i}.txt` })) + + // Act + const lookup = createDocumentLookup(documents) + const startTime = performance.now() + for (let i = 0; i < 1000; i++) + lookup.getName(`doc-${i}`) + + const duration = performance.now() - startTime + + // Assert - should be very fast due to Map lookup + expect(duration).toBeLessThan(50) + }) + }) +}) + +// ============================================================================= +// useIndexingStatusPolling Hook Tests +// ============================================================================= + +describe('useIndexingStatusPolling', () => { + // Test the polling hook for indexing status + + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('should fetch status on mount', async () => { + // Arrange + const mockStatus = [createMockIndexingStatus({ indexing_status: 'completed' })] + mockFetchIndexingStatusBatch.mockResolvedValue({ data: mockStatus }) + + // Act + const { result } = renderHook(() => + useIndexingStatusPolling({ datasetId: 'ds-1', batchId: 'batch-1' }), + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(mockFetchIndexingStatusBatch).toHaveBeenCalledWith({ + datasetId: 'ds-1', + batchId: 'batch-1', + }) + expect(result.current.statusList).toEqual(mockStatus) + }) + + it('should stop polling when all statuses are completed', async () => { + // Arrange + const mockStatus = [createMockIndexingStatus({ indexing_status: 'completed' })] + mockFetchIndexingStatusBatch.mockResolvedValue({ data: mockStatus }) + + // Act + renderHook(() => + useIndexingStatusPolling({ datasetId: 'ds-1', batchId: 'batch-1' }), + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert - should only be called once since status is completed + expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(1) + }) + + it('should continue polling when status is indexing', async () => { + // Arrange + const indexingStatus = [createMockIndexingStatus({ indexing_status: 'indexing' })] + const completedStatus = [createMockIndexingStatus({ indexing_status: 'completed' })] + + mockFetchIndexingStatusBatch + .mockResolvedValueOnce({ data: indexingStatus }) + .mockResolvedValueOnce({ data: completedStatus }) + + // Act + renderHook(() => + useIndexingStatusPolling({ datasetId: 'ds-1', batchId: 'batch-1' }), + ) + + // First poll + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Advance timer for next poll (2500ms) + await act(async () => { + await vi.advanceTimersByTimeAsync(2500) + }) + + // Assert + expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(2) + }) + + it('should stop polling when status is error', async () => { + // Arrange + const mockStatus = [createMockIndexingStatus({ indexing_status: 'error', error: 'Some error' })] + mockFetchIndexingStatusBatch.mockResolvedValue({ data: mockStatus }) + + // Act + const { result } = renderHook(() => + useIndexingStatusPolling({ datasetId: 'ds-1', batchId: 'batch-1' }), + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(result.current.isEmbeddingCompleted).toBe(true) + expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(1) + }) + + it('should stop polling when status is paused', async () => { + // Arrange + const mockStatus = [createMockIndexingStatus({ indexing_status: 'paused' })] + mockFetchIndexingStatusBatch.mockResolvedValue({ data: mockStatus }) + + // Act + const { result } = renderHook(() => + useIndexingStatusPolling({ datasetId: 'ds-1', batchId: 'batch-1' }), + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(result.current.isEmbeddingCompleted).toBe(true) + }) + + it('should continue polling on API error', async () => { + // Arrange + mockFetchIndexingStatusBatch + .mockRejectedValueOnce(new Error('Network error')) + .mockResolvedValueOnce({ data: [createMockIndexingStatus({ indexing_status: 'completed' })] }) + + // Act + renderHook(() => + useIndexingStatusPolling({ datasetId: 'ds-1', batchId: 'batch-1' }), + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + await act(async () => { + await vi.advanceTimersByTimeAsync(2500) + }) + + // Assert - should retry after error + expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(2) + }) + + it('should return correct isEmbedding state', async () => { + // Arrange + const mockStatus = [createMockIndexingStatus({ indexing_status: 'indexing' })] + mockFetchIndexingStatusBatch.mockResolvedValue({ data: mockStatus }) + + // Act + const { result } = renderHook(() => + useIndexingStatusPolling({ datasetId: 'ds-1', batchId: 'batch-1' }), + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(result.current.isEmbedding).toBe(true) + expect(result.current.isEmbeddingCompleted).toBe(false) + }) + + it('should cleanup timeout on unmount', async () => { + // Arrange + const mockStatus = [createMockIndexingStatus({ indexing_status: 'indexing' })] + mockFetchIndexingStatusBatch.mockResolvedValue({ data: mockStatus }) + + // Act + const { unmount } = renderHook(() => + useIndexingStatusPolling({ datasetId: 'ds-1', batchId: 'batch-1' }), + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + const callCountBeforeUnmount = mockFetchIndexingStatusBatch.mock.calls.length + + unmount() + + // Advance timers - should not trigger more calls after unmount + await act(async () => { + await vi.advanceTimersByTimeAsync(5000) + }) + + // Assert - no additional calls after unmount + expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(callCountBeforeUnmount) + }) + + it('should handle multiple documents with mixed statuses', async () => { + // Arrange + const mockStatus = [ + createMockIndexingStatus({ id: 'doc-1', indexing_status: 'completed' }), + createMockIndexingStatus({ id: 'doc-2', indexing_status: 'indexing' }), + ] + mockFetchIndexingStatusBatch.mockResolvedValue({ data: mockStatus }) + + // Act + const { result } = renderHook(() => + useIndexingStatusPolling({ datasetId: 'ds-1', batchId: 'batch-1' }), + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(result.current.isEmbedding).toBe(true) + expect(result.current.isEmbeddingCompleted).toBe(false) + expect(result.current.statusList).toHaveLength(2) + }) + + it('should return empty statusList initially', () => { + // Arrange & Act + const { result } = renderHook(() => + useIndexingStatusPolling({ datasetId: 'ds-1', batchId: 'batch-1' }), + ) + + // Assert + expect(result.current.statusList).toEqual([]) + expect(result.current.isEmbedding).toBe(false) + expect(result.current.isEmbeddingCompleted).toBe(false) + }) +}) + +// ============================================================================= +// UpgradeBanner Component Tests +// ============================================================================= + +describe('UpgradeBanner', () => { + // Test the upgrade banner component + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('should render upgrade message', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText(/billing\.plansCommon\.documentProcessingPriorityUpgrade/i)).toBeInTheDocument() + }) + + it('should render ZapFast icon', () => { + // Arrange & Act + const { container } = render() + + // Assert + expect(container.querySelector('svg')).toBeInTheDocument() + }) + + it('should render UpgradeBtn component', () => { + // Arrange & Act + render() + + // Assert - UpgradeBtn should be rendered + const upgradeContainer = screen.getByText(/billing\.plansCommon\.documentProcessingPriorityUpgrade/i).parentElement + expect(upgradeContainer).toBeInTheDocument() + }) +}) + +// ============================================================================= +// IndexingProgressItem Component Tests +// ============================================================================= + +describe('IndexingProgressItem', () => { + // Test the progress item component for individual documents + + beforeEach(() => { + vi.clearAllMocks() + }) + + describe('Rendering', () => { + it('should render document name', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render() + + // Assert + expect(screen.getByText('test-document.txt')).toBeInTheDocument() + }) + + it('should render progress percentage when embedding', () => { + // Arrange + const detail = createMockIndexingStatus({ + indexing_status: 'indexing', + completed_segments: 5, + total_segments: 10, + }) + + // Act + render() + + // Assert + expect(screen.getByText('50%')).toBeInTheDocument() + }) + + it('should not render progress percentage when completed', () => { + // Arrange + const detail = createMockIndexingStatus({ indexing_status: 'completed' }) + + // Act + render() + + // Assert + expect(screen.queryByText('%')).not.toBeInTheDocument() + }) + }) + + describe('Status Icons', () => { + it('should render success icon for completed status', () => { + // Arrange + const detail = createMockIndexingStatus({ indexing_status: 'completed' }) + + // Act + const { container } = render() + + // Assert + expect(container.querySelector('.text-text-success')).toBeInTheDocument() + }) + + it('should render error icon for error status', () => { + // Arrange + const detail = createMockIndexingStatus({ + indexing_status: 'error', + error: 'Processing failed', + }) + + // Act + const { container } = render() + + // Assert + expect(container.querySelector('.text-text-destructive')).toBeInTheDocument() + }) + + it('should not render status icon for indexing status', () => { + // Arrange + const detail = createMockIndexingStatus({ indexing_status: 'indexing' }) + + // Act + const { container } = render() + + // Assert + expect(container.querySelector('.text-text-success')).not.toBeInTheDocument() + expect(container.querySelector('.text-text-destructive')).not.toBeInTheDocument() + }) + }) + + describe('Source Type Icons', () => { + it('should render file icon for FILE source type', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render( + , + ) + + // Assert - DocumentFileIcon should be rendered + expect(screen.getByText('document.pdf')).toBeInTheDocument() + }) + + // DocumentFileIcon branch coverage: different file extensions + describe('DocumentFileIcon file extensions', () => { + it.each([ + ['document.pdf', 'pdf'], + ['data.json', 'json'], + ['page.html', 'html'], + ['readme.txt', 'txt'], + ['notes.markdown', 'markdown'], + ['readme.md', 'md'], + ['spreadsheet.xlsx', 'xlsx'], + ['legacy.xls', 'xls'], + ['data.csv', 'csv'], + ['letter.doc', 'doc'], + ['report.docx', 'docx'], + ])('should render file icon for %s (%s extension)', (filename) => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render( + , + ) + + // Assert + expect(screen.getByText(filename)).toBeInTheDocument() + }) + + it('should handle unknown file extension with default icon', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render( + , + ) + + // Assert - should still render with default document icon + expect(screen.getByText('archive.zip')).toBeInTheDocument() + }) + + it('should handle uppercase extension', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render( + , + ) + + // Assert + expect(screen.getByText('REPORT.PDF')).toBeInTheDocument() + }) + + it('should handle mixed case extension', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render( + , + ) + + // Assert + expect(screen.getByText('Document.Docx')).toBeInTheDocument() + }) + + it('should handle filename with multiple dots', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render( + , + ) + + // Assert - should extract "pdf" as extension + expect(screen.getByText('my.file.name.pdf')).toBeInTheDocument() + }) + + it('should handle filename without extension', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render( + , + ) + + // Assert - should use filename itself as fallback + expect(screen.getByText('noextension')).toBeInTheDocument() + }) + }) + + it('should render notion icon for NOTION source type', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render( + , + ) + + // Assert + expect(screen.getByText('Notion Page')).toBeInTheDocument() + }) + }) + + describe('Progress Bar', () => { + it('should render progress bar when embedding', () => { + // Arrange + const detail = createMockIndexingStatus({ + indexing_status: 'indexing', + completed_segments: 30, + total_segments: 100, + }) + + // Act + const { container } = render() + + // Assert + const progressBar = container.querySelector('[style*="width: 30%"]') + expect(progressBar).toBeInTheDocument() + }) + + it('should not render progress bar when completed', () => { + // Arrange + const detail = createMockIndexingStatus({ indexing_status: 'completed' }) + + // Act + const { container } = render() + + // Assert + const progressBar = container.querySelector('.bg-components-progress-bar-progress') + expect(progressBar).not.toBeInTheDocument() + }) + + it('should apply error styling for error status', () => { + // Arrange + const detail = createMockIndexingStatus({ indexing_status: 'error' }) + + // Act + const { container } = render() + + // Assert + expect(container.querySelector('.bg-state-destructive-hover-alt')).toBeInTheDocument() + }) + }) + + describe('Billing', () => { + it('should render PriorityLabel when enableBilling is true', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render() + + // Assert - PriorityLabel component should be in the DOM + const container = screen.getByText('test.txt').parentElement + expect(container).toBeInTheDocument() + }) + + it('should not render PriorityLabel when enableBilling is false', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render() + + // Assert + expect(screen.getByText('test.txt')).toBeInTheDocument() + }) + }) + + describe('Edge Cases', () => { + it('should handle undefined name', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render() + + // Assert - should not crash + expect(document.body).toBeInTheDocument() + }) + + it('should handle undefined sourceType', () => { + // Arrange + const detail = createMockIndexingStatus() + + // Act + render() + + // Assert - should render without source icon + expect(screen.getByText('test.txt')).toBeInTheDocument() + }) + }) +}) + +// ============================================================================= +// RuleDetail Component Tests +// ============================================================================= + +describe('RuleDetail', () => { + // Test the rule detail component for process configuration display + + beforeEach(() => { + vi.clearAllMocks() + }) + + describe('Rendering', () => { + it('should render without crashing', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText(/datasetDocuments\.embedding\.mode/i)).toBeInTheDocument() + }) + + it('should render all field labels', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText(/datasetDocuments\.embedding\.mode/i)).toBeInTheDocument() + expect(screen.getByText(/datasetDocuments\.embedding\.segmentLength/i)).toBeInTheDocument() + expect(screen.getByText(/datasetDocuments\.embedding\.textCleaning/i)).toBeInTheDocument() + expect(screen.getByText(/datasetCreation\.stepTwo\.indexMode/i)).toBeInTheDocument() + expect(screen.getByText(/datasetSettings\.form\.retrievalSetting\.title/i)).toBeInTheDocument() + }) + }) + + describe('Mode Display', () => { + it('should show "-" when sourceData is undefined', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getAllByText('-')).toHaveLength(3) // mode, segmentLength, textCleaning + }) + + it('should show "custom" for general process mode', () => { + // Arrange + const sourceData = createMockProcessRule({ mode: ProcessMode.general }) + + // Act + render() + + // Assert + expect(screen.getByText(/datasetDocuments\.embedding\.custom/i)).toBeInTheDocument() + }) + + it('should show hierarchical mode with paragraph parent', () => { + // Arrange + const sourceData = createMockProcessRule({ + mode: ProcessMode.parentChild, + rules: { + parent_mode: 'paragraph', + segmentation: { max_tokens: 500 }, + }, + } as Partial) + + // Act + render() + + // Assert + expect(screen.getByText(/datasetDocuments\.embedding\.hierarchical/i)).toBeInTheDocument() + }) + }) + + describe('Segment Length Display', () => { + it('should show max_tokens for general mode', () => { + // Arrange + const sourceData = createMockProcessRule({ + mode: ProcessMode.general, + rules: { + segmentation: { max_tokens: 500 }, + }, + } as Partial) + + // Act + render() + + // Assert + expect(screen.getByText('500')).toBeInTheDocument() + }) + + it('should show parent and child tokens for hierarchical mode', () => { + // Arrange + const sourceData = createMockProcessRule({ + mode: ProcessMode.parentChild, + rules: { + segmentation: { max_tokens: 1000 }, + subchunk_segmentation: { max_tokens: 200 }, + }, + } as Partial) + + // Act + render() + + // Assert + expect(screen.getByText(/1000/)).toBeInTheDocument() + expect(screen.getByText(/200/)).toBeInTheDocument() + }) + }) + + describe('Text Cleaning Rules', () => { + it('should show enabled rule names', () => { + // Arrange + const sourceData = createMockProcessRule({ + mode: ProcessMode.general, + rules: { + pre_processing_rules: [ + { id: 'remove_extra_spaces', enabled: true }, + { id: 'remove_urls_emails', enabled: true }, + { id: 'remove_stopwords', enabled: false }, + ], + }, + } as Partial) + + // Act + render() + + // Assert + expect(screen.getByText(/removeExtraSpaces/i)).toBeInTheDocument() + expect(screen.getByText(/removeUrlEmails/i)).toBeInTheDocument() + }) + + it('should show "-" when no rules are enabled', () => { + // Arrange + const sourceData = createMockProcessRule({ + mode: ProcessMode.general, + rules: { + pre_processing_rules: [ + { id: 'remove_extra_spaces', enabled: false }, + ], + }, + } as Partial) + + // Act + render() + + // Assert - textCleaning should show "-" + const dashElements = screen.getAllByText('-') + expect(dashElements.length).toBeGreaterThan(0) + }) + }) + + describe('Indexing Type', () => { + it('should show qualified for high_quality indexing', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText(/datasetCreation\.stepTwo\.qualified/i)).toBeInTheDocument() + }) + + it('should show economical for economy indexing', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText(/datasetCreation\.stepTwo\.economical/i)).toBeInTheDocument() + }) + + it('should render correct icon for indexing type', () => { + // Arrange & Act + render() + + // Assert + const images = screen.getAllByTestId('next-image') + expect(images.length).toBeGreaterThan(0) + }) + }) + + describe('Retrieval Method', () => { + it('should show semantic search by default', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText(/dataset\.retrieval\.semantic_search\.title/i)).toBeInTheDocument() + }) + + it('should show keyword search for economical indexing', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText(/dataset\.retrieval\.keyword_search\.title/i)).toBeInTheDocument() + }) + + it.each([ + [RETRIEVE_METHOD.fullText, 'full_text_search'], + [RETRIEVE_METHOD.hybrid, 'hybrid_search'], + [RETRIEVE_METHOD.semantic, 'semantic_search'], + ])('should show correct label for %s retrieval method', (method, expectedKey) => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText(new RegExp(`dataset\\.retrieval\\.${expectedKey}\\.title`, 'i'))).toBeInTheDocument() + }) + }) +}) + +// ============================================================================= +// EmbeddingProcess Integration Tests +// ============================================================================= + +describe('EmbeddingProcess', () => { + // Integration tests for the main EmbeddingProcess component + + // Import the main component after mocks are set up + let EmbeddingProcess: typeof import('./index').default + + beforeEach(async () => { + vi.clearAllMocks() + vi.useFakeTimers() + mockEnableBilling = false + mockPlanType = 'sandbox' + + // Dynamically import to get fresh component with mocks + const embeddingModule = await import('./index') + EmbeddingProcess = embeddingModule.default + }) + + afterEach(() => { + vi.useRealTimers() + }) + + describe('Rendering', () => { + it('should render without crashing', async () => { + // Arrange + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Act + render() + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(document.body).toBeInTheDocument() + }) + + it('should render status header', async () => { + // Arrange + const mockStatus = [createMockIndexingStatus({ indexing_status: 'indexing' })] + mockFetchIndexingStatusBatch.mockResolvedValue({ data: mockStatus }) + + // Act + render() + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(screen.getByText(/datasetDocuments\.embedding\.processing/i)).toBeInTheDocument() + }) + + it('should show completed status when all documents are done', async () => { + // Arrange + const mockStatus = [createMockIndexingStatus({ indexing_status: 'completed' })] + mockFetchIndexingStatusBatch.mockResolvedValue({ data: mockStatus }) + + // Act + render() + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(screen.getByText(/datasetDocuments\.embedding\.completed/i)).toBeInTheDocument() + }) + }) + + describe('Progress Items', () => { + it('should render progress items for each document', async () => { + // Arrange + const documents = [ + createMockDocument({ id: 'doc-1', name: 'file1.txt' }), + createMockDocument({ id: 'doc-2', name: 'file2.pdf' }), + ] + const mockStatus = [ + createMockIndexingStatus({ id: 'doc-1' }), + createMockIndexingStatus({ id: 'doc-2' }), + ] + mockFetchIndexingStatusBatch.mockResolvedValue({ data: mockStatus }) + + // Act + render( + , + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(screen.getByText('file1.txt')).toBeInTheDocument() + expect(screen.getByText('file2.pdf')).toBeInTheDocument() + }) + }) + + describe('Upgrade Banner', () => { + it('should show upgrade banner when billing is enabled and not team plan', async () => { + // Arrange + mockEnableBilling = true + mockPlanType = 'sandbox' + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Re-import to get updated mock values + const embeddingModule = await import('./index') + EmbeddingProcess = embeddingModule.default + + // Act + render() + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(screen.getByText(/billing\.plansCommon\.documentProcessingPriorityUpgrade/i)).toBeInTheDocument() + }) + + it('should not show upgrade banner when billing is disabled', async () => { + // Arrange + mockEnableBilling = false + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Act + render() + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(screen.queryByText(/billing\.plansCommon\.documentProcessingPriorityUpgrade/i)).not.toBeInTheDocument() + }) + + it('should not show upgrade banner for team plan', async () => { + // Arrange + mockEnableBilling = true + mockPlanType = 'team' + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Re-import to get updated mock values + const embeddingModule = await import('./index') + EmbeddingProcess = embeddingModule.default + + // Act + render() + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(screen.queryByText(/billing\.plansCommon\.documentProcessingPriorityUpgrade/i)).not.toBeInTheDocument() + }) + }) + + describe('Action Buttons', () => { + it('should render API access button with correct link', async () => { + // Arrange + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Act + render() + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + const apiButton = screen.getByText('Access the API') + expect(apiButton).toBeInTheDocument() + expect(apiButton.closest('a')).toHaveAttribute('href', 'https://api.example.com/docs') + }) + + it('should render navigation button', async () => { + // Arrange + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Act + render() + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(screen.getByText(/datasetCreation\.stepThree\.navTo/i)).toBeInTheDocument() + }) + + it('should navigate to documents list when nav button clicked', async () => { + // Arrange + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Act + render() + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + const navButton = screen.getByText(/datasetCreation\.stepThree\.navTo/i) + + await act(async () => { + navButton.click() + }) + + // Assert + expect(mockInvalidDocumentList).toHaveBeenCalled() + expect(mockPush).toHaveBeenCalledWith('/datasets/ds-1/documents') + }) + }) + + describe('Rule Detail', () => { + it('should render RuleDetail component', async () => { + // Arrange + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Act + render( + , + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(screen.getByText(/datasetDocuments\.embedding\.mode/i)).toBeInTheDocument() + }) + + it('should pass indexingType to RuleDetail', async () => { + // Arrange + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Act + render( + , + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert + expect(screen.getByText(/datasetCreation\.stepTwo\.economical/i)).toBeInTheDocument() + }) + }) + + describe('Document Lookup Memoization', () => { + it('should memoize document lookup based on documents array', async () => { + // Arrange + const documents = [createMockDocument({ id: 'doc-1', name: 'test.txt' })] + mockFetchIndexingStatusBatch.mockResolvedValue({ + data: [createMockIndexingStatus({ id: 'doc-1' })], + }) + + // Act + const { rerender } = render( + , + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Rerender with same documents reference + rerender( + , + ) + + // Assert - component should render without issues + expect(screen.getByText('test.txt')).toBeInTheDocument() + }) + }) + + describe('Edge Cases', () => { + it('should handle empty documents array', async () => { + // Arrange + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Act + render() + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert - should render without crashing + expect(document.body).toBeInTheDocument() + }) + + it('should handle undefined documents', async () => { + // Arrange + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Act + render() + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert - should render without crashing + expect(document.body).toBeInTheDocument() + }) + + it('should handle status with missing document', async () => { + // Arrange + const documents = [createMockDocument({ id: 'doc-1', name: 'test.txt' })] + mockFetchIndexingStatusBatch.mockResolvedValue({ + data: [ + createMockIndexingStatus({ id: 'doc-1' }), + createMockIndexingStatus({ id: 'doc-unknown' }), // No matching document + ], + }) + + // Act + render( + , + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert - should render known document and handle unknown gracefully + expect(screen.getByText('test.txt')).toBeInTheDocument() + }) + + it('should handle undefined retrievalMethod', async () => { + // Arrange + mockFetchIndexingStatusBatch.mockResolvedValue({ data: [] }) + + // Act + render( + , + ) + + await act(async () => { + await vi.runOnlyPendingTimersAsync() + }) + + // Assert - should use default semantic search + expect(screen.getByText(/dataset\.retrieval\.semantic_search\.title/i)).toBeInTheDocument() + }) + }) +}) diff --git a/web/app/components/datasets/create/embedding-process/index.tsx b/web/app/components/datasets/create/embedding-process/index.tsx index aa1f6cee50..e9cea84f00 100644 --- a/web/app/components/datasets/create/embedding-process/index.tsx +++ b/web/app/components/datasets/create/embedding-process/index.tsx @@ -1,47 +1,29 @@ import type { FC } from 'react' -import type { - DataSourceInfo, - FullDocumentDetail, - IndexingStatusResponse, - LegacyDataSourceInfo, - ProcessRuleResponse, -} from '@/models/datasets' +import type { FullDocumentDetail } from '@/models/datasets' +import type { RETRIEVE_METHOD } from '@/types/app' import { RiArrowRightLine, - RiCheckboxCircleFill, - RiErrorWarningFill, RiLoader2Fill, RiTerminalBoxLine, } from '@remixicon/react' -import Image from 'next/image' import Link from 'next/link' import { useRouter } from 'next/navigation' -import * as React from 'react' -import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { useMemo } from 'react' import { useTranslation } from 'react-i18next' import Button from '@/app/components/base/button' import Divider from '@/app/components/base/divider' -import { ZapFast } from '@/app/components/base/icons/src/vender/solid/general' -import NotionIcon from '@/app/components/base/notion-icon' -import Tooltip from '@/app/components/base/tooltip' -import PriorityLabel from '@/app/components/billing/priority-label' import { Plan } from '@/app/components/billing/type' -import UpgradeBtn from '@/app/components/billing/upgrade-btn' -import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata' import { useProviderContext } from '@/context/provider-context' import { useDatasetApiAccessUrl } from '@/hooks/use-api-access-url' -import { DataSourceType, ProcessMode } from '@/models/datasets' -import { fetchIndexingStatusBatch as doFetchIndexingStatus } from '@/service/datasets' import { useProcessRule } from '@/service/knowledge/use-dataset' import { useInvalidDocumentList } from '@/service/knowledge/use-document' -import { RETRIEVE_METHOD } from '@/types/app' -import { sleep } from '@/utils' -import { cn } from '@/utils/classnames' -import DocumentFileIcon from '../../common/document-file-icon' -import { indexMethodIcon, retrievalIcon } from '../icons' -import { IndexingType } from '../step-two' +import IndexingProgressItem from './indexing-progress-item' +import RuleDetail from './rule-detail' +import UpgradeBanner from './upgrade-banner' +import { useIndexingStatusPolling } from './use-indexing-status-polling' +import { createDocumentLookup } from './utils' -type Props = { +type EmbeddingProcessProps = { datasetId: string batchId: string documents?: FullDocumentDetail[] @@ -49,333 +31,121 @@ type Props = { retrievalMethod?: RETRIEVE_METHOD } -const RuleDetail: FC<{ - sourceData?: ProcessRuleResponse - indexingType?: string - retrievalMethod?: RETRIEVE_METHOD -}> = ({ sourceData, indexingType, retrievalMethod }) => { +// Status header component +const StatusHeader: FC<{ isEmbedding: boolean, isCompleted: boolean }> = ({ + isEmbedding, + isCompleted, +}) => { const { t } = useTranslation() - const segmentationRuleMap = { - mode: t('embedding.mode', { ns: 'datasetDocuments' }), - segmentLength: t('embedding.segmentLength', { ns: 'datasetDocuments' }), - textCleaning: t('embedding.textCleaning', { ns: 'datasetDocuments' }), - } - - const getRuleName = (key: string) => { - if (key === 'remove_extra_spaces') - return t('stepTwo.removeExtraSpaces', { ns: 'datasetCreation' }) - - if (key === 'remove_urls_emails') - return t('stepTwo.removeUrlEmails', { ns: 'datasetCreation' }) - - if (key === 'remove_stopwords') - return t('stepTwo.removeStopwords', { ns: 'datasetCreation' }) - } - - const isNumber = (value: unknown) => { - return typeof value === 'number' - } - - const getValue = useCallback((field: string) => { - let value: string | number | undefined = '-' - const maxTokens = isNumber(sourceData?.rules?.segmentation?.max_tokens) - ? sourceData.rules.segmentation.max_tokens - : value - const childMaxTokens = isNumber(sourceData?.rules?.subchunk_segmentation?.max_tokens) - ? sourceData.rules.subchunk_segmentation.max_tokens - : value - switch (field) { - case 'mode': - value = !sourceData?.mode - ? value - : sourceData.mode === ProcessMode.general - ? (t('embedding.custom', { ns: 'datasetDocuments' }) as string) - : `${t('embedding.hierarchical', { ns: 'datasetDocuments' })} · ${sourceData?.rules?.parent_mode === 'paragraph' - ? t('parentMode.paragraph', { ns: 'dataset' }) - : t('parentMode.fullDoc', { ns: 'dataset' })}` - break - case 'segmentLength': - value = !sourceData?.mode - ? value - : sourceData.mode === ProcessMode.general - ? maxTokens - : `${t('embedding.parentMaxTokens', { ns: 'datasetDocuments' })} ${maxTokens}; ${t('embedding.childMaxTokens', { ns: 'datasetDocuments' })} ${childMaxTokens}` - break - default: - value = !sourceData?.mode - ? value - : sourceData?.rules?.pre_processing_rules?.filter(rule => - rule.enabled).map(rule => getRuleName(rule.id)).join(',') - break - } - return value - }, [sourceData]) - return ( -
- {Object.keys(segmentationRuleMap).map((field) => { - return ( - - ) - })} - - )} - /> - - )} - /> +
+ {isEmbedding && ( + <> + + {t('embedding.processing', { ns: 'datasetDocuments' })} + + )} + {isCompleted && t('embedding.completed', { ns: 'datasetDocuments' })}
) } -const EmbeddingProcess: FC = ({ datasetId, batchId, documents = [], indexingType, retrievalMethod }) => { +// Action buttons component +const ActionButtons: FC<{ + apiReferenceUrl: string + onNavToDocuments: () => void +}> = ({ apiReferenceUrl, onNavToDocuments }) => { const { t } = useTranslation() + + return ( +
+ + + + +
+ ) +} + +const EmbeddingProcess: FC = ({ + datasetId, + batchId, + documents = [], + indexingType, + retrievalMethod, +}) => { const { enableBilling, plan } = useProviderContext() - - const getFirstDocument = documents[0] - - const [indexingStatusBatchDetail, setIndexingStatusDetail] = useState([]) - const fetchIndexingStatus = async () => { - const status = await doFetchIndexingStatus({ datasetId, batchId }) - setIndexingStatusDetail(status.data) - return status.data - } - - const [isStopQuery, setIsStopQuery] = useState(false) - const isStopQueryRef = useRef(isStopQuery) - useEffect(() => { - isStopQueryRef.current = isStopQuery - }, [isStopQuery]) - const stopQueryStatus = () => { - setIsStopQuery(true) - } - - const startQueryStatus = async () => { - if (isStopQueryRef.current) - return - - try { - const indexingStatusBatchDetail = await fetchIndexingStatus() - const isCompleted = indexingStatusBatchDetail.every(indexingStatusDetail => ['completed', 'error', 'paused'].includes(indexingStatusDetail.indexing_status)) - if (isCompleted) { - stopQueryStatus() - return - } - await sleep(2500) - await startQueryStatus() - } - catch { - await sleep(2500) - await startQueryStatus() - } - } - - useEffect(() => { - setIsStopQuery(false) - startQueryStatus() - return () => { - stopQueryStatus() - } - }, []) - - // get rule - const { data: ruleDetail } = useProcessRule(getFirstDocument?.id) - const router = useRouter() const invalidDocumentList = useInvalidDocumentList() - const navToDocumentList = () => { + const apiReferenceUrl = useDatasetApiAccessUrl() + + // Polling hook for indexing status + const { statusList, isEmbedding, isEmbeddingCompleted } = useIndexingStatusPolling({ + datasetId, + batchId, + }) + + // Get process rule for the first document + const firstDocumentId = documents[0]?.id + const { data: ruleDetail } = useProcessRule(firstDocumentId) + + // Document lookup utilities - memoized for performance + const documentLookup = useMemo( + () => createDocumentLookup(documents), + [documents], + ) + + const handleNavToDocuments = () => { invalidDocumentList() router.push(`/datasets/${datasetId}/documents`) } - const apiReferenceUrl = useDatasetApiAccessUrl() - const isEmbedding = useMemo(() => { - return indexingStatusBatchDetail.some(indexingStatusDetail => ['indexing', 'splitting', 'parsing', 'cleaning'].includes(indexingStatusDetail?.indexing_status || '')) - }, [indexingStatusBatchDetail]) - const isEmbeddingCompleted = useMemo(() => { - return indexingStatusBatchDetail.every(indexingStatusDetail => ['completed', 'error', 'paused'].includes(indexingStatusDetail?.indexing_status || '')) - }, [indexingStatusBatchDetail]) - - const getSourceName = (id: string) => { - const doc = documents.find(document => document.id === id) - return doc?.name - } - const getFileType = (name?: string) => name?.split('.').pop() || 'txt' - const getSourcePercent = (detail: IndexingStatusResponse) => { - const completedCount = detail.completed_segments || 0 - const totalCount = detail.total_segments || 0 - if (totalCount === 0) - return 0 - const percent = Math.round(completedCount * 100 / totalCount) - return percent > 100 ? 100 : percent - } - const getSourceType = (id: string) => { - const doc = documents.find(document => document.id === id) - return doc?.data_source_type as DataSourceType - } - - const isLegacyDataSourceInfo = (info: DataSourceInfo): info is LegacyDataSourceInfo => { - return info != null && typeof (info as LegacyDataSourceInfo).upload_file === 'object' - } - - const getIcon = (id: string) => { - const doc = documents.find(document => document.id === id) - const info = doc?.data_source_info - if (info && isLegacyDataSourceInfo(info)) - return info.notion_page_icon - return undefined - } - const isSourceEmbedding = (detail: IndexingStatusResponse) => - ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'].includes(detail.indexing_status || '') + const showUpgradeBanner = enableBilling && plan.type !== Plan.team return ( <>
-
- {isEmbedding && ( - <> - - {t('embedding.processing', { ns: 'datasetDocuments' })} - - )} - {isEmbeddingCompleted && t('embedding.completed', { ns: 'datasetDocuments' })} -
- { - enableBilling && plan.type !== Plan.team && ( -
-
- -
-
- {t('plansCommon.documentProcessingPriorityUpgrade', { ns: 'billing' })} -
- -
- ) - } + + + {showUpgradeBanner && } +
- {indexingStatusBatchDetail.map(indexingStatusDetail => ( -
- {isSourceEmbedding(indexingStatusDetail) && ( -
- )} -
- {getSourceType(indexingStatusDetail.id) === DataSourceType.FILE && ( - - )} - {getSourceType(indexingStatusDetail.id) === DataSourceType.NOTION && ( - - )} -
-
- {getSourceName(indexingStatusDetail.id)} -
- { - enableBilling && ( - - ) - } -
- {isSourceEmbedding(indexingStatusDetail) && ( -
{`${getSourcePercent(indexingStatusDetail)}%`}
- )} - {indexingStatusDetail.indexing_status === 'error' && ( - - - - - - )} - {indexingStatusDetail.indexing_status === 'completed' && ( - - )} -
-
+ {statusList.map(detail => ( + ))}
+ +
-
- - - - -
+ + ) } diff --git a/web/app/components/datasets/create/embedding-process/indexing-progress-item.tsx b/web/app/components/datasets/create/embedding-process/indexing-progress-item.tsx new file mode 100644 index 0000000000..b7c085cff9 --- /dev/null +++ b/web/app/components/datasets/create/embedding-process/indexing-progress-item.tsx @@ -0,0 +1,120 @@ +import type { FC } from 'react' +import type { IndexingStatusResponse } from '@/models/datasets' +import { + RiCheckboxCircleFill, + RiErrorWarningFill, +} from '@remixicon/react' +import NotionIcon from '@/app/components/base/notion-icon' +import Tooltip from '@/app/components/base/tooltip' +import PriorityLabel from '@/app/components/billing/priority-label' +import { DataSourceType } from '@/models/datasets' +import { cn } from '@/utils/classnames' +import DocumentFileIcon from '../../common/document-file-icon' +import { getFileType, getSourcePercent, isSourceEmbedding } from './utils' + +type IndexingProgressItemProps = { + detail: IndexingStatusResponse + name?: string + sourceType?: DataSourceType + notionIcon?: string + enableBilling?: boolean +} + +// Status icon component for completed/error states +const StatusIcon: FC<{ status: string, error?: string }> = ({ status, error }) => { + if (status === 'completed') + return + + if (status === 'error') { + return ( + + + + + + ) + } + + return null +} + +// Source type icon component +const SourceTypeIcon: FC<{ + sourceType?: DataSourceType + name?: string + notionIcon?: string +}> = ({ sourceType, name, notionIcon }) => { + if (sourceType === DataSourceType.FILE) { + return ( + + ) + } + + if (sourceType === DataSourceType.NOTION) { + return ( + + ) + } + + return null +} + +const IndexingProgressItem: FC = ({ + detail, + name, + sourceType, + notionIcon, + enableBilling, +}) => { + const isEmbedding = isSourceEmbedding(detail) + const percent = getSourcePercent(detail) + const isError = detail.indexing_status === 'error' + + return ( +
+ {isEmbedding && ( +
+ )} +
+ +
+
+ {name} +
+ {enableBilling && } +
+ {isEmbedding && ( +
{`${percent}%`}
+ )} + +
+
+ ) +} + +export default IndexingProgressItem diff --git a/web/app/components/datasets/create/embedding-process/rule-detail.tsx b/web/app/components/datasets/create/embedding-process/rule-detail.tsx new file mode 100644 index 0000000000..dff35100cb --- /dev/null +++ b/web/app/components/datasets/create/embedding-process/rule-detail.tsx @@ -0,0 +1,133 @@ +import type { FC } from 'react' +import type { ProcessRuleResponse } from '@/models/datasets' +import Image from 'next/image' +import { useCallback } from 'react' +import { useTranslation } from 'react-i18next' +import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata' +import { ProcessMode } from '@/models/datasets' +import { RETRIEVE_METHOD } from '@/types/app' +import { indexMethodIcon, retrievalIcon } from '../icons' +import { IndexingType } from '../step-two' + +type RuleDetailProps = { + sourceData?: ProcessRuleResponse + indexingType?: string + retrievalMethod?: RETRIEVE_METHOD +} + +// Lookup table for pre-processing rule names +const PRE_PROCESSING_RULE_KEYS = { + remove_extra_spaces: 'stepTwo.removeExtraSpaces', + remove_urls_emails: 'stepTwo.removeUrlEmails', + remove_stopwords: 'stepTwo.removeStopwords', +} as const + +// Lookup table for retrieval method icons +const RETRIEVAL_ICON_MAP: Partial> = { + [RETRIEVE_METHOD.fullText]: retrievalIcon.fullText, + [RETRIEVE_METHOD.hybrid]: retrievalIcon.hybrid, + [RETRIEVE_METHOD.semantic]: retrievalIcon.vector, + [RETRIEVE_METHOD.invertedIndex]: retrievalIcon.fullText, + [RETRIEVE_METHOD.keywordSearch]: retrievalIcon.fullText, +} + +const isNumber = (value: unknown): value is number => typeof value === 'number' + +const RuleDetail: FC = ({ sourceData, indexingType, retrievalMethod }) => { + const { t } = useTranslation() + + const segmentationRuleLabels = { + mode: t('embedding.mode', { ns: 'datasetDocuments' }), + segmentLength: t('embedding.segmentLength', { ns: 'datasetDocuments' }), + textCleaning: t('embedding.textCleaning', { ns: 'datasetDocuments' }), + } + + const getRuleName = useCallback((key: string): string | undefined => { + const translationKey = PRE_PROCESSING_RULE_KEYS[key as keyof typeof PRE_PROCESSING_RULE_KEYS] + return translationKey ? t(translationKey, { ns: 'datasetCreation' }) : undefined + }, [t]) + + const getModeValue = useCallback((): string => { + if (!sourceData?.mode) + return '-' + + if (sourceData.mode === ProcessMode.general) + return t('embedding.custom', { ns: 'datasetDocuments' }) + + const parentModeLabel = sourceData.rules?.parent_mode === 'paragraph' + ? t('parentMode.paragraph', { ns: 'dataset' }) + : t('parentMode.fullDoc', { ns: 'dataset' }) + + return `${t('embedding.hierarchical', { ns: 'datasetDocuments' })} · ${parentModeLabel}` + }, [sourceData, t]) + + const getSegmentLengthValue = useCallback((): string | number => { + if (!sourceData?.mode) + return '-' + + const maxTokens = isNumber(sourceData.rules?.segmentation?.max_tokens) + ? sourceData.rules.segmentation.max_tokens + : '-' + + if (sourceData.mode === ProcessMode.general) + return maxTokens + + const childMaxTokens = isNumber(sourceData.rules?.subchunk_segmentation?.max_tokens) + ? sourceData.rules.subchunk_segmentation.max_tokens + : '-' + + return `${t('embedding.parentMaxTokens', { ns: 'datasetDocuments' })} ${maxTokens}; ${t('embedding.childMaxTokens', { ns: 'datasetDocuments' })} ${childMaxTokens}` + }, [sourceData, t]) + + const getTextCleaningValue = useCallback((): string => { + if (!sourceData?.mode) + return '-' + + const enabledRules = sourceData.rules?.pre_processing_rules?.filter(rule => rule.enabled) || [] + const ruleNames = enabledRules + .map((rule) => { + const name = getRuleName(rule.id) + return typeof name === 'string' ? name : '' + }) + .filter(name => name) + return ruleNames.length > 0 ? ruleNames.join(',') : '-' + }, [sourceData, getRuleName]) + + const fieldValueGetters: Record string | number> = { + mode: getModeValue, + segmentLength: getSegmentLengthValue, + textCleaning: getTextCleaningValue, + } + + const isEconomical = indexingType === IndexingType.ECONOMICAL + const indexMethodIconSrc = isEconomical ? indexMethodIcon.economical : indexMethodIcon.high_quality + const indexModeLabel = t(`stepTwo.${isEconomical ? 'economical' : 'qualified'}`, { ns: 'datasetCreation' }) + + const effectiveRetrievalMethod = isEconomical ? 'keyword_search' : (retrievalMethod ?? 'semantic_search') + const retrievalLabel = t(`retrieval.${effectiveRetrievalMethod}.title`, { ns: 'dataset' }) + const retrievalIconSrc = RETRIEVAL_ICON_MAP[retrievalMethod as keyof typeof RETRIEVAL_ICON_MAP] ?? retrievalIcon.vector + + return ( +
+ {Object.keys(segmentationRuleLabels).map(field => ( + + ))} + } + /> + } + /> +
+ ) +} + +export default RuleDetail diff --git a/web/app/components/datasets/create/embedding-process/upgrade-banner.tsx b/web/app/components/datasets/create/embedding-process/upgrade-banner.tsx new file mode 100644 index 0000000000..49e5fe99a1 --- /dev/null +++ b/web/app/components/datasets/create/embedding-process/upgrade-banner.tsx @@ -0,0 +1,22 @@ +import type { FC } from 'react' +import { useTranslation } from 'react-i18next' +import { ZapFast } from '@/app/components/base/icons/src/vender/solid/general' +import UpgradeBtn from '@/app/components/billing/upgrade-btn' + +const UpgradeBanner: FC = () => { + const { t } = useTranslation() + + return ( +
+
+ +
+
+ {t('plansCommon.documentProcessingPriorityUpgrade', { ns: 'billing' })} +
+ +
+ ) +} + +export default UpgradeBanner diff --git a/web/app/components/datasets/create/embedding-process/use-indexing-status-polling.ts b/web/app/components/datasets/create/embedding-process/use-indexing-status-polling.ts new file mode 100644 index 0000000000..f8e69e47af --- /dev/null +++ b/web/app/components/datasets/create/embedding-process/use-indexing-status-polling.ts @@ -0,0 +1,90 @@ +import type { IndexingStatusResponse } from '@/models/datasets' +import { useEffect, useRef, useState } from 'react' +import { fetchIndexingStatusBatch } from '@/service/datasets' + +const POLLING_INTERVAL = 2500 +const COMPLETED_STATUSES = ['completed', 'error', 'paused'] as const +const EMBEDDING_STATUSES = ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'] as const + +type IndexingStatusPollingParams = { + datasetId: string + batchId: string +} + +type IndexingStatusPollingResult = { + statusList: IndexingStatusResponse[] + isEmbedding: boolean + isEmbeddingCompleted: boolean +} + +const isStatusCompleted = (status: string): boolean => + COMPLETED_STATUSES.includes(status as typeof COMPLETED_STATUSES[number]) + +const isAllCompleted = (statusList: IndexingStatusResponse[]): boolean => + statusList.every(item => isStatusCompleted(item.indexing_status)) + +/** + * Custom hook for polling indexing status with automatic stop on completion. + * Handles the polling lifecycle and provides derived states for UI rendering. + */ +export const useIndexingStatusPolling = ({ + datasetId, + batchId, +}: IndexingStatusPollingParams): IndexingStatusPollingResult => { + const [statusList, setStatusList] = useState([]) + const isStopPollingRef = useRef(false) + + useEffect(() => { + // Reset polling state on mount + isStopPollingRef.current = false + let timeoutId: ReturnType | null = null + + const fetchStatus = async (): Promise => { + const response = await fetchIndexingStatusBatch({ datasetId, batchId }) + setStatusList(response.data) + return response.data + } + + const poll = async (): Promise => { + if (isStopPollingRef.current) + return + + try { + const data = await fetchStatus() + if (isAllCompleted(data)) { + isStopPollingRef.current = true + return + } + } + catch { + // Continue polling on error + } + + if (!isStopPollingRef.current) { + timeoutId = setTimeout(() => { + poll() + }, POLLING_INTERVAL) + } + } + + poll() + + return () => { + isStopPollingRef.current = true + if (timeoutId) + clearTimeout(timeoutId) + } + }, [datasetId, batchId]) + + const isEmbedding = statusList.some(item => + EMBEDDING_STATUSES.includes(item?.indexing_status as typeof EMBEDDING_STATUSES[number]), + ) + + const isEmbeddingCompleted = statusList.length > 0 && isAllCompleted(statusList) + + return { + statusList, + isEmbedding, + isEmbeddingCompleted, + } +} diff --git a/web/app/components/datasets/create/embedding-process/utils.ts b/web/app/components/datasets/create/embedding-process/utils.ts new file mode 100644 index 0000000000..6fbefb0230 --- /dev/null +++ b/web/app/components/datasets/create/embedding-process/utils.ts @@ -0,0 +1,64 @@ +import type { + DataSourceInfo, + DataSourceType, + FullDocumentDetail, + IndexingStatusResponse, + LegacyDataSourceInfo, +} from '@/models/datasets' + +const EMBEDDING_STATUSES = ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'] as const + +/** + * Type guard for legacy data source info with upload_file property + */ +export const isLegacyDataSourceInfo = (info: DataSourceInfo): info is LegacyDataSourceInfo => { + return info != null && typeof (info as LegacyDataSourceInfo).upload_file === 'object' +} + +/** + * Check if a status indicates the source is being embedded + */ +export const isSourceEmbedding = (detail: IndexingStatusResponse): boolean => + EMBEDDING_STATUSES.includes(detail.indexing_status as typeof EMBEDDING_STATUSES[number]) + +/** + * Calculate the progress percentage for a document + */ +export const getSourcePercent = (detail: IndexingStatusResponse): number => { + const completedCount = detail.completed_segments || 0 + const totalCount = detail.total_segments || 0 + + if (totalCount === 0) + return 0 + + const percent = Math.round(completedCount * 100 / totalCount) + return Math.min(percent, 100) +} + +/** + * Get file extension from filename, defaults to 'txt' + */ +export const getFileType = (name?: string): string => + name?.split('.').pop() || 'txt' + +/** + * Document lookup utilities - provides document info by ID from a list + */ +export const createDocumentLookup = (documents: FullDocumentDetail[]) => { + const documentMap = new Map(documents.map(doc => [doc.id, doc])) + + return { + getDocument: (id: string) => documentMap.get(id), + + getName: (id: string) => documentMap.get(id)?.name, + + getSourceType: (id: string) => documentMap.get(id)?.data_source_type as DataSourceType | undefined, + + getNotionIcon: (id: string) => { + const info = documentMap.get(id)?.data_source_info + if (info && isLegacyDataSourceInfo(info)) + return info.notion_page_icon + return undefined + }, + } +}