From 470e72c820323edd20e3924d160f7cb26add1793 Mon Sep 17 00:00:00 2001 From: Nite Knite Date: Fri, 7 Feb 2025 14:02:57 +0800 Subject: [PATCH 1/8] chore: bump katex version and tweak UI copy (#13280) --- .../model-provider-page/index.tsx | 6 +++--- web/i18n/en-US/common.ts | 2 +- web/i18n/zh-Hans/common.ts | 2 +- web/package.json | 4 ++-- web/pnpm-lock.yaml | 16 ++++++++-------- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/web/app/components/header/account-setting/model-provider-page/index.tsx b/web/app/components/header/account-setting/model-provider-page/index.tsx index f77974abaf..949352a265 100644 --- a/web/app/components/header/account-setting/model-provider-page/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/index.tsx @@ -71,11 +71,11 @@ const ModelProviderPage = ({ searchText }: Props) => { const [filteredConfiguredProviders, filteredNotConfiguredProviders] = useMemo(() => { const filteredConfiguredProviders = configuredProviders.filter( provider => provider.provider.toLowerCase().includes(debouncedSearchText.toLowerCase()) - || Object.values(provider.label).some(text => text.toLowerCase().includes(debouncedSearchText.toLowerCase())), + || Object.values(provider.label).some(text => text.toLowerCase().includes(debouncedSearchText.toLowerCase())), ) const filteredNotConfiguredProviders = notConfiguredProviders.filter( provider => provider.provider.toLowerCase().includes(debouncedSearchText.toLowerCase()) - || Object.values(provider.label).some(text => text.toLowerCase().includes(debouncedSearchText.toLowerCase())), + || Object.values(provider.label).some(text => text.toLowerCase().includes(debouncedSearchText.toLowerCase())), ) return [filteredConfiguredProviders, filteredNotConfiguredProviders] @@ -143,7 +143,7 @@ const ModelProviderPage = ({ searchText }: Props) => { )} {!!filteredNotConfiguredProviders?.length && ( <> -
{t('common.modelProvider.configureRequired')}
+
{t('common.modelProvider.toBeConfigured')}
{filteredNotConfiguredProviders?.map(provider => ( =18'} - katex@0.16.11: - resolution: {integrity: sha512-RQrI8rlHY92OLf3rho/Ts8i/XvjgguEjOkO1BEXcU3N8BqPpSzBNwV/G0Ukr+P/l3ivvJUE/Fa/CwbS6HesGNQ==} + katex@0.16.21: + resolution: {integrity: sha512-XvqR7FgOHtWupfMiigNzmh+MgUVmDGU2kXZm899ZkPfcuoPuFxyHmXsgATDpFZDAXCI8tvinaVcDo8PIIJSo4A==} hasBin: true keyv@4.5.4: @@ -15130,7 +15130,7 @@ snapshots: jwt-decode@4.0.0: {} - katex@0.16.11: + katex@0.16.21: dependencies: commander: 8.3.0 @@ -15558,7 +15558,7 @@ snapshots: dagre-d3-es: 7.0.11 dayjs: 1.11.13 dompurify: 3.2.3 - katex: 0.16.11 + katex: 0.16.21 khroma: 2.1.0 lodash-es: 4.17.21 marked: 13.0.3 @@ -15652,7 +15652,7 @@ snapshots: dependencies: '@types/katex': 0.16.7 devlop: 1.1.0 - katex: 0.16.11 + katex: 0.16.21 micromark-factory-space: 2.0.0 micromark-util-character: 2.1.0 micromark-util-symbol: 2.0.0 @@ -16991,7 +16991,7 @@ snapshots: '@types/katex': 0.16.7 hast-util-from-html-isomorphic: 2.0.0 hast-util-to-text: 4.0.2 - katex: 0.16.11 + katex: 0.16.21 unist-util-visit-parents: 6.0.1 vfile: 6.0.3 From a8a8a5513cc0e0c1275073521b03a4b37b9617e0 Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Fri, 7 Feb 2025 14:04:49 +0800 Subject: [PATCH 2/8] fix: app check dependency (#13320) --- .../install-plugin/install-bundle/steps/install-multi.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/plugins/install-plugin/install-bundle/steps/install-multi.tsx b/web/app/components/plugins/install-plugin/install-bundle/steps/install-multi.tsx index 48b1ecd325..803be4abd4 100644 --- a/web/app/components/plugins/install-plugin/install-bundle/steps/install-multi.tsx +++ b/web/app/components/plugins/install-plugin/install-bundle/steps/install-multi.tsx @@ -26,7 +26,7 @@ const InstallByDSLList: FC = ({ isFromMarketPlace, }) => { // DSL has id, to get plugin info to show more info - const { isLoading: isFetchingMarketplaceDataById, data: infoGetById, error: infoByIdError } = useFetchPluginsInMarketPlaceByIds(allPlugins.filter(d => d.type === 'marketplace').map(d => (d as GitHubItemAndMarketPlaceDependency).value.plugin_unique_identifier!)) + const { isLoading: isFetchingMarketplaceDataById, data: infoGetById, error: infoByIdError } = useFetchPluginsInMarketPlaceByIds(allPlugins.filter(d => d.type === 'marketplace').map(d => (d as GitHubItemAndMarketPlaceDependency).value.marketplace_plugin_unique_identifier!)) // has meta(org,name,version), to get id const { isLoading: isFetchingDataByMeta, data: infoByMeta, error: infoByMetaError } = useFetchPluginsInMarketPlaceByInfo(allPlugins.filter(d => d.type === 'marketplace').map(d => (d as GitHubItemAndMarketPlaceDependency).value!)) From 7e1d9894fbd865d008ad777397ad74d0aa229862 Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Fri, 7 Feb 2025 16:02:12 +0800 Subject: [PATCH 3/8] fix: plugins task permission (#13330) --- web/service/use-plugins.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web/service/use-plugins.ts b/web/service/use-plugins.ts index 2eb99055ad..a1cf1a62e6 100644 --- a/web/service/use-plugins.ts +++ b/web/service/use-plugins.ts @@ -29,6 +29,7 @@ import { useQueryClient, } from '@tanstack/react-query' import { useInvalidateAllBuiltInTools } from './use-tools' +import usePermission from '@/app/components/plugins/plugin-page/use-permission' const NAME_SPACE = 'plugins' @@ -356,12 +357,16 @@ export const useFetchPluginsInMarketPlaceByInfo = (infos: Record[]) const usePluginTaskListKey = [NAME_SPACE, 'pluginTaskList'] export const usePluginTaskList = () => { + const { + canManagement, + } = usePermission() const { data, isFetched, refetch, ...rest } = useQuery({ + enabled: canManagement, queryKey: usePluginTaskListKey, queryFn: () => get<{ tasks: PluginTask[] }>('/workspaces/current/plugin/tasks?page=1&page_size=100'), refetchInterval: (lastQuery) => { From d4a09805a31dc3013ecc9ed05fc1143023060c7f Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Fri, 7 Feb 2025 16:08:25 +0800 Subject: [PATCH 4/8] improve preview document tokenizer (#13328) --- .../index_processor/processor/parent_child_index_processor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/core/rag/index_processor/processor/parent_child_index_processor.py b/api/core/rag/index_processor/processor/parent_child_index_processor.py index 3140122081..894b85339a 100644 --- a/api/core/rag/index_processor/processor/parent_child_index_processor.py +++ b/api/core/rag/index_processor/processor/parent_child_index_processor.py @@ -47,6 +47,8 @@ class ParentChildIndexProcessor(BaseIndexProcessor): embedding_model_instance=kwargs.get("embedding_model_instance"), ) for document in documents: + if kwargs.get("preview") and len(all_documents) >= 10: + return all_documents # document clean document_text = CleanProcessor.clean(document.page_content, process_rule) document.page_content = document_text From fec3bb44695927014cd6b5fd7ede8c2305808dff Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Fri, 7 Feb 2025 17:30:04 +0800 Subject: [PATCH 5/8] fix: models sort in model page (#13334) --- .../header/account-setting/model-provider-page/hooks.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/app/components/header/account-setting/model-provider-page/hooks.ts b/web/app/components/header/account-setting/model-provider-page/hooks.ts index 231df0e9db..48acaeb64a 100644 --- a/web/app/components/header/account-setting/model-provider-page/hooks.ts +++ b/web/app/components/header/account-setting/model-provider-page/hooks.ts @@ -275,6 +275,8 @@ export const useMarketplaceAllPlugins = (providers: ModelProvider[], searchText: category: PluginType.model, exclude, type: 'plugin', + sortBy: 'install_count', + sortOrder: 'DESC', }) } else { @@ -284,6 +286,8 @@ export const useMarketplaceAllPlugins = (providers: ModelProvider[], searchText: type: 'plugin', pageSize: 1000, exclude, + sortBy: 'install_count', + sortOrder: 'DESC', }) } }, [queryPlugins, queryPluginsWithDebounced, searchText, exclude]) From 3e9c3d0bb7274bd4b5bf8cf17fe5d8ba0fab7b8c Mon Sep 17 00:00:00 2001 From: Joel Date: Sat, 8 Feb 2025 11:31:55 +0800 Subject: [PATCH 6/8] fix: install installed plugin problem (#13384) --- .../hooks/use-check-installed.tsx | 1 + .../steps/install.tsx | 34 ++++++------------- web/app/components/plugins/types.ts | 1 + web/service/use-plugins.ts | 19 ++++++++--- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/web/app/components/plugins/install-plugin/hooks/use-check-installed.tsx b/web/app/components/plugins/install-plugin/hooks/use-check-installed.tsx index e72648fcec..e9d83c8c87 100644 --- a/web/app/components/plugins/install-plugin/hooks/use-check-installed.tsx +++ b/web/app/components/plugins/install-plugin/hooks/use-check-installed.tsx @@ -16,6 +16,7 @@ const useCheckInstalled = (props: Props) => { const res: Record = {} data?.plugins.forEach((plugin) => { res[plugin.plugin_id] = { + installedId: plugin.id, installedVersion: plugin.declaration.version, uniqueIdentifier: plugin.plugin_unique_identifier, } diff --git a/web/app/components/plugins/install-plugin/install-from-local-package/steps/install.tsx b/web/app/components/plugins/install-plugin/install-from-local-package/steps/install.tsx index 19baa86d73..1deb8d8282 100644 --- a/web/app/components/plugins/install-plugin/install-from-local-package/steps/install.tsx +++ b/web/app/components/plugins/install-plugin/install-from-local-package/steps/install.tsx @@ -8,8 +8,9 @@ import Button from '@/app/components/base/button' import { Trans, useTranslation } from 'react-i18next' import { RiLoader2Line } from '@remixicon/react' import checkTaskStatus from '../../base/check-task-status' -import { useInstallPackageFromLocal, usePluginTaskList, useUpdatePackageFromMarketPlace } from '@/service/use-plugins' +import { useInstallPackageFromLocal, usePluginTaskList } from '@/service/use-plugins' import useCheckInstalled from '@/app/components/plugins/install-plugin/hooks/use-check-installed' +import { uninstallPlugin } from '@/service/plugins' import Version from '../../base/version' const i18nPrefix = 'plugin.installModal' @@ -50,7 +51,6 @@ const Installed: FC = ({ const [isInstalling, setIsInstalling] = React.useState(false) const { mutateAsync: installPackageFromLocal } = useInstallPackageFromLocal() - const { mutateAsync: updatePackageFromMarketPlace } = useUpdatePackageFromMarketPlace() const { check, @@ -69,27 +69,15 @@ const Installed: FC = ({ onStartToInstall?.() try { - let taskId - let isInstalled - if (hasInstalled) { - const { - all_installed, - task_id, - } = await updatePackageFromMarketPlace({ - original_plugin_unique_identifier: installedInfoPayload.uniqueIdentifier, - new_plugin_unique_identifier: uniqueIdentifier, - }) - taskId = task_id - isInstalled = all_installed - } - else { - const { - all_installed, - task_id, - } = await installPackageFromLocal(uniqueIdentifier) - taskId = task_id - isInstalled = all_installed - } + if (hasInstalled) + await uninstallPlugin(installedInfoPayload.installedId) + + const { + all_installed, + task_id, + } = await installPackageFromLocal(uniqueIdentifier) + const taskId = task_id + const isInstalled = all_installed if (isInstalled) { onInstalled() diff --git a/web/app/components/plugins/types.ts b/web/app/components/plugins/types.ts index e9f7884257..9a94f7740c 100644 --- a/web/app/components/plugins/types.ts +++ b/web/app/components/plugins/types.ts @@ -373,6 +373,7 @@ export type VersionListResponse = { } export type VersionInfo = { + installedId: string, // use to uninstall installedVersion: string, uniqueIdentifier: string } diff --git a/web/service/use-plugins.ts b/web/service/use-plugins.ts index a1cf1a62e6..841e4bfc11 100644 --- a/web/service/use-plugins.ts +++ b/web/service/use-plugins.ts @@ -30,6 +30,7 @@ import { } from '@tanstack/react-query' import { useInvalidateAllBuiltInTools } from './use-tools' import usePermission from '@/app/components/plugins/plugin-page/use-permission' +import { uninstallPlugin } from '@/service/plugins' const NAME_SPACE = 'plugins' @@ -237,10 +238,20 @@ export const useInstallOrUpdate = ({ } } if (isInstalled) { - await updatePackageFromMarketPlace({ - original_plugin_unique_identifier: installedPayload?.uniqueIdentifier, - new_plugin_unique_identifier: uniqueIdentifier, - }) + if (item.type === 'package') { + await uninstallPlugin(installedPayload.installedId) + await post('/workspaces/current/plugin/install/pkg', { + body: { + plugin_unique_identifiers: [uniqueIdentifier], + }, + }) + } + else { + await updatePackageFromMarketPlace({ + original_plugin_unique_identifier: installedPayload?.uniqueIdentifier, + new_plugin_unique_identifier: uniqueIdentifier, + }) + } } return ({ success: true }) } From 8d5a8f01532b7776a0c7b743db22b362dbbcd782 Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Sat, 8 Feb 2025 14:53:43 +0800 Subject: [PATCH 7/8] fix: using latest plugin as the dep of DSL under 0.1.15 --- api/services/app_dsl_service.py | 36 +++++++++++++++----- api/services/plugin/dependencies_analysis.py | 16 +++++++++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index cb0f3dfbc0..960ce4f7df 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -1,7 +1,7 @@ import logging import uuid from enum import StrEnum -from typing import Optional +from typing import Mapping, Optional from urllib.parse import urlparse from uuid import uuid4 @@ -256,6 +256,16 @@ class AppDslService: check_dependencies_pending_data = None if dependencies: check_dependencies_pending_data = [PluginDependency.model_validate(d) for d in dependencies] + elif imported_version <= "0.1.5": + if "workflow" in data: + graph = data.get("workflow", {}).get("graph", {}) + dependencies_list = self._extract_dependencies_from_workflow_graph(graph) + else: + dependencies_list = self._extract_dependencies_from_model_config(data.get("model_config", {})) + + check_dependencies_pending_data = DependenciesAnalysisService.generate_latest_dependencies( + dependencies_list + ) # Create or update app app = self._create_or_update_app( @@ -558,7 +568,7 @@ class AppDslService: raise ValueError("Missing app configuration, please check.") export_data["model_config"] = app_model_config.to_dict() - dependencies = cls._extract_dependencies_from_model_config(app_model_config) + dependencies = cls._extract_dependencies_from_model_config(app_model_config.to_dict()) export_data["dependencies"] = [ jsonable_encoder(d.model_dump()) for d in DependenciesAnalysisService.generate_dependencies( @@ -574,6 +584,16 @@ class AppDslService: :return: dependencies list format like ["langgenius/google"] """ graph = workflow.graph_dict + dependencies = cls._extract_dependencies_from_workflow_graph(graph) + return dependencies + + @classmethod + def _extract_dependencies_from_workflow_graph(cls, graph: Mapping) -> list[str]: + """ + Extract dependencies from workflow graph + :param graph: Workflow graph + :return: dependencies list format like ["langgenius/google"] + """ dependencies = [] for node in graph.get("nodes", []): try: @@ -647,24 +667,24 @@ class AppDslService: return dependencies @classmethod - def _extract_dependencies_from_model_config(cls, model_config: AppModelConfig) -> list[str]: + def _extract_dependencies_from_model_config(cls, model_config: Mapping) -> list[str]: """ Extract dependencies from model config - :param model_config: AppModelConfig instance - :return: dependencies list format like ["langgenius/google:1.0.0@abcdef1234567890"] + :param model_config: model config dict + :return: dependencies list format like ["langgenius/google"] """ dependencies = [] try: # completion model - model_dict = model_config.model_dict + model_dict = model_config.get("model", {}) if model_dict: dependencies.append( DependenciesAnalysisService.analyze_model_provider_dependency(model_dict.get("provider", "")) ) # reranking model - dataset_configs = model_config.dataset_configs_dict + dataset_configs = model_config.get("dataset_configs", {}) if dataset_configs: for dataset_config in dataset_configs.get("datasets", {}).get("datasets", []): if dataset_config.get("reranking_model"): @@ -677,7 +697,7 @@ class AppDslService: ) # tools - agent_configs = model_config.agent_mode_dict + agent_configs = model_config.get("agent_mode", {}) if agent_configs: for agent_config in agent_configs.get("tools", []): dependencies.append( diff --git a/api/services/plugin/dependencies_analysis.py b/api/services/plugin/dependencies_analysis.py index d425483863..df0c321a8b 100644 --- a/api/services/plugin/dependencies_analysis.py +++ b/api/services/plugin/dependencies_analysis.py @@ -1,3 +1,4 @@ +from core.helper import marketplace from core.plugin.entities.plugin import GenericProviderID, PluginDependency, PluginInstallationSource from core.plugin.manager.plugin import PluginInstallationManager @@ -98,3 +99,18 @@ class DependenciesAnalysisService: raise ValueError(f"Unknown plugin source: {plugin.source}") return result + + @classmethod + def generate_latest_dependencies(cls, dependencies: list[str]) -> list[PluginDependency]: + """ + Generate the latest version of dependencies + """ + dependencies = list(set(dependencies)) + deps = marketplace.batch_fetch_plugin_manifests(dependencies) + return [ + PluginDependency( + type=PluginDependency.Type.Marketplace, + value=PluginDependency.Marketplace(marketplace_plugin_unique_identifier=dep.latest_package_identifier), + ) + for dep in deps + ] From e3a81f09a976293668de897b56f0bb3b50a20fca Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Sat, 8 Feb 2025 14:54:02 +0800 Subject: [PATCH 8/8] fix: ruff --- api/services/app_dsl_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index 960ce4f7df..4ddb56981e 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -1,7 +1,8 @@ import logging import uuid +from collections.abc import Mapping from enum import StrEnum -from typing import Mapping, Optional +from typing import Optional from urllib.parse import urlparse from uuid import uuid4