diff --git a/.devcontainer/post_create_command.sh b/.devcontainer/post_create_command.sh index ddf976c47a..b0322dd2b2 100755 --- a/.devcontainer/post_create_command.sh +++ b/.devcontainer/post_create_command.sh @@ -3,8 +3,8 @@ cd web && npm install pipx install poetry -echo 'alias start-api="cd /workspaces/dify/api && flask run --host 0.0.0.0 --port=5001 --debug"' >> ~/.bashrc -echo 'alias start-worker="cd /workspaces/dify/api && celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion"' >> ~/.bashrc +echo 'alias start-api="cd /workspaces/dify/api && poetry run python -m flask run --host 0.0.0.0 --port=5001 --debug"' >> ~/.bashrc +echo 'alias start-worker="cd /workspaces/dify/api && poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion"' >> ~/.bashrc echo 'alias start-web="cd /workspaces/dify/web && npm run dev"' >> ~/.bashrc echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify up -d"' >> ~/.bashrc diff --git a/.github/DISCUSSION_TEMPLATE/general.yml b/.github/DISCUSSION_TEMPLATE/general.yml new file mode 100644 index 0000000000..5af61ea64c --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/general.yml @@ -0,0 +1,24 @@ +title: "General Discussion" +body: + - type: checkboxes + attributes: + label: Self Checks + description: "To make sure we get to you in time, please check the following :)" + options: + - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones. + required: true + - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)). + required: true + - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）" + required: true + - label: "Please do not modify this template :) and fill in all the required fields." + required: true + - type: textarea + attributes: + label: Content + placeholder: Please describe the content you would like to discuss. + validations: + required: true + - type: markdown + attributes: + value: Please limit one request per issue. diff --git a/.github/DISCUSSION_TEMPLATE/help.yml b/.github/DISCUSSION_TEMPLATE/help.yml new file mode 100644 index 0000000000..abebaa9727 --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/help.yml @@ -0,0 +1,30 @@ +title: "Help" +body: + - type: checkboxes + attributes: + label: Self Checks + description: "To make sure we get to you in time, please check the following :)" + options: + - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones. + required: true + - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)). + required: true + - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）" + required: true + - label: "Please do not modify this template :) and fill in all the required fields." + required: true + - type: textarea + attributes: + label: 1. Is this request related to a challenge you're experiencing? Tell me about your story. + placeholder: Please describe the specific scenario or problem you're facing as clearly as possible. For instance "I was trying to use [feature] for [specific task], and [what happened]... It was frustrating because...." + validations: + required: true + - type: textarea + attributes: + label: 2. Additional context or comments + placeholder: (Any other information, comments, documentations, links, or screenshots that would provide more clarity. This is the place to add anything else not covered above.) + validations: + required: false + - type: markdown + attributes: + value: Please limit one request per issue. diff --git a/.github/DISCUSSION_TEMPLATE/suggestion.yml b/.github/DISCUSSION_TEMPLATE/suggestion.yml new file mode 100644 index 0000000000..0893a10b2d --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/suggestion.yml @@ -0,0 +1,37 @@ +title: Suggestions for New Features +body: + - type: checkboxes + attributes: + label: Self Checks + description: "To make sure we get to you in time, please check the following :)" + options: + - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones. + required: true + - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)). + required: true + - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）" + required: true + - label: "Please do not modify this template :) and fill in all the required fields." + required: true + - type: textarea + attributes: + label: 1. Is this request related to a challenge you're experiencing? Tell me about your story. + placeholder: Please describe the specific scenario or problem you're facing as clearly as possible. For instance "I was trying to use [feature] for [specific task], and [what happened]... It was frustrating because...." + validations: + required: true + - type: textarea + attributes: + label: 2. Additional context or comments + placeholder: (Any other information, comments, documentations, links, or screenshots that would provide more clarity. This is the place to add anything else not covered above.) + validations: + required: false + - type: checkboxes + attributes: + label: 3. Can you help us with this feature? + description: Let us know! This is not a commitment, but a starting point for collaboration. + options: + - label: I am interested in contributing to this feature. + required: false + - type: markdown + attributes: + value: Please limit one request per issue. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 8824c5dba6..cf4fce3700 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -22,7 +22,7 @@ body: - type: input attributes: label: Dify version - placeholder: 0.6.11 + placeholder: 0.6.15 description: See about section in Dify console validations: required: true diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index aa7e68dbac..4ba91c467f 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -89,6 +89,5 @@ jobs: pgvecto-rs pgvector chroma - myscale - name: Test Vector Stores run: poetry run -C api bash dev/pytest/pytest_vdb.sh diff --git a/.gitignore b/.gitignore index 2f44cf7934..97b7333dde 100644 --- a/.gitignore +++ b/.gitignore @@ -174,5 +174,6 @@ sdks/python-client/dify_client.egg-info .vscode/* !.vscode/launch.json pyrightconfig.json +api/.vscode .idea/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce7ad7db98..f810584f24 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,7 +81,7 @@ Dify requires the following dependencies to build, make sure they're installed o Dify is composed of a backend and a frontend. Navigate to the backend directory by `cd api/`, then follow the [Backend README](api/README.md) to install it. In a separate terminal, navigate to the frontend directory by `cd web/`, then follow the [Frontend README](web/README.md) to install. -Check the [installation FAQ](https://docs.dify.ai/getting-started/faq/install-faq) for a list of common issues and steps to troubleshoot. +Check the [installation FAQ](https://docs.dify.ai/learn-more/faq/self-host-faq) for a list of common issues and steps to troubleshoot. ### 5. Visit dify in your browser diff --git a/CONTRIBUTING_CN.md b/CONTRIBUTING_CN.md index 6ddfa9c84a..303c2513f5 100644 --- a/CONTRIBUTING_CN.md +++ b/CONTRIBUTING_CN.md @@ -2,17 +2,17 @@ 考虑到我们的现状，我们需要灵活快速地交付，但我们也希望确保像你这样的贡献者在贡献过程中获得尽可能顺畅的体验。我们为此编写了这份贡献指南，旨在让你熟悉代码库和我们与贡献者的合作方式，以便你能快速进入有趣的部分。 -这份指南，就像 Dify 本身一样，是一个不断改进的工作。如果有时它落后于实际项目，我们非常感谢你的理解，并欢迎任何反馈以供我们改进。 +这份指南，就像 Dify 本身一样，是一个不断改进的工作。如果有时它落后于实际项目，我们非常感谢你的理解，并欢迎提供任何反馈以供我们改进。 -在许可方面，请花一分钟阅读我们简短的[许可证和贡献者协议](./LICENSE)。社区还遵守[行为准则](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md)。 +在许可方面，请花一分钟阅读我们简短的 [许可证和贡献者协议](./LICENSE)。社区还遵守 [行为准则](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md)。 ## 在开始之前 -[查找](https://github.com/langgenius/dify/issues?q=is:issue+is:closed)现有问题，或[创建](https://github.com/langgenius/dify/issues/new/choose)一个新问题。我们将问题分为两类： +[查找](https://github.com/langgenius/dify/issues?q=is:issue+is:closed)现有问题，或 [创建](https://github.com/langgenius/dify/issues/new/choose) 一个新问题。我们将问题分为两类： ### 功能请求： -* 如果您要提出新的功能请求，请解释所提议的功能的目标，并尽可能提供详细的上下文。[@perzeusss](https://github.com/perzeuss)制作了一个很好的[功能请求助手](https://udify.app/chat/MK2kVSnw1gakVwMX)，可以帮助您起草需求。随时尝试一下。 +* 如果您要提出新的功能请求，请解释所提议的功能的目标，并尽可能提供详细的上下文。[@perzeusss](https://github.com/perzeuss) 制作了一个很好的 [功能请求助手](https://udify.app/chat/MK2kVSnw1gakVwMX)，可以帮助您起草需求。随时尝试一下。 * 如果您想从现有问题中选择一个，请在其下方留下评论表示您的意愿。 @@ -20,45 +20,44 @@ 根据所提议的功能所属的领域不同，您可能需要与不同的团队成员交流。以下是我们团队成员目前正在从事的各个领域的概述： - | Member | Scope | + | 团队成员 | 工作范围 | | ------------------------------------------------------------ | ---------------------------------------------------- | - | [@yeuoly](https://github.com/Yeuoly) | Architecting Agents | - | [@jyong](https://github.com/JohnJyong) | RAG pipeline design | - | [@GarfieldDai](https://github.com/GarfieldDai) | Building workflow orchestrations | - | [@iamjoel](https://github.com/iamjoel) & [@zxhlyh](https://github.com/zxhlyh) | Making our frontend a breeze to use | - | [@guchenhe](https://github.com/guchenhe) & [@crazywoola](https://github.com/crazywoola) | Developer experience, points of contact for anything | - | [@takatost](https://github.com/takatost) | Overall product direction and architecture | + | [@yeuoly](https://github.com/Yeuoly) | 架构 Agents | + | [@jyong](https://github.com/JohnJyong) | RAG 流水线设计 | + | [@GarfieldDai](https://github.com/GarfieldDai) | 构建 workflow 编排 | + | [@iamjoel](https://github.com/iamjoel) & [@zxhlyh](https://github.com/zxhlyh) | 让我们的前端更易用 | + | [@guchenhe](https://github.com/guchenhe) & [@crazywoola](https://github.com/crazywoola) | 开发人员体验, 综合事项联系人 | + | [@takatost](https://github.com/takatost) | 产品整体方向和架构 | - How we prioritize: + 事项优先级： - | Feature Type | Priority | + | 功能类型 | 优先级 | | ------------------------------------------------------------ | --------------- | - | High-Priority Features as being labeled by a team member | High Priority | - | Popular feature requests from our [community feedback board](https://github.com/langgenius/dify/discussions/categories/feedbacks) | Medium Priority | - | Non-core features and minor enhancements | Low Priority | - | Valuable but not immediate | Future-Feature | + | 被团队成员标记为高优先级的功能 | 高优先级 | + | 在 [community feedback board](https://github.com/langgenius/dify/discussions/categories/feedbacks) 内反馈的常见功能请求 | 中等优先级 | + | 非核心功能和小幅改进 | 低优先级 | + | 有价值当不紧急 | 未来功能 | -### 其他任何事情（例如bug报告、性能优化、拼写错误更正）： +### 其他任何事情（例如 bug 报告、性能优化、拼写错误更正）： * 立即开始编码。 - How we prioritize: + 事项优先级： - | Issue Type | Priority | + | Issue 类型 | 优先级 | | ------------------------------------------------------------ | --------------- | - | Bugs in core functions (cannot login, applications not working, security loopholes) | Critical | - | Non-critical bugs, performance boosts | Medium Priority | - | Minor fixes (typos, confusing but working UI) | Low Priority | - + | 核心功能的 Bugs（例如无法登录、应用无法工作、安全漏洞） | 紧急 | + | 非紧急 bugs, 性能提升 | 中等优先级 | + | 小幅修复(错别字, 能正常工作但存在误导的 UI) | 低优先级 | ## 安装 -以下是设置Dify进行开发的步骤： +以下是设置 Dify 进行开发的步骤： -### 1. Fork该仓库 +### 1. Fork 该仓库 ### 2. 克隆仓库 -从终端克隆fork的仓库： +从终端克隆代码仓库： ``` git clone git@github.com:/dify.git @@ -76,72 +75,72 @@ Dify 依赖以下工具和库： ### 4. 安装 -Dify由后端和前端组成。通过`cd api/`导航到后端目录，然后按照[后端README](api/README.md)进行安装。在另一个终端中，通过`cd web/`导航到前端目录，然后按照[前端README](web/README.md)进行安装。 +Dify 由后端和前端组成。通过 `cd api/` 导航到后端目录，然后按照 [后端 README](api/README.md) 进行安装。在另一个终端中，通过 `cd web/` 导航到前端目录，然后按照 [前端 README](web/README.md) 进行安装。 -查看[安装常见问题解答](https://docs.dify.ai/getting-started/faq/install-faq)以获取常见问题列表和故障排除步骤。 +查看 [安装常见问题解答](https://docs.dify.ai/v/zh-hans/learn-more/faq/install-faq) 以获取常见问题列表和故障排除步骤。 -### 5. 在浏览器中访问Dify +### 5. 在浏览器中访问 Dify -为了验证您的设置，打开浏览器并访问[http://localhost:3000](http://localhost:3000)（默认或您自定义的URL和端口）。现在您应该看到Dify正在运行。 +为了验证您的设置，打开浏览器并访问 [http://localhost:3000](http://localhost:3000)（默认或您自定义的 URL 和端口）。现在您应该看到 Dify 正在运行。 ## 开发 -如果您要添加模型提供程序，请参考[此指南](https://github.com/langgenius/dify/blob/main/api/core/model_runtime/README.md)。 +如果您要添加模型提供程序，请参考 [此指南](https://github.com/langgenius/dify/blob/main/api/core/model_runtime/README.md)。 -如果您要向Agent或Workflow添加工具提供程序，请参考[此指南](./api/core/tools/README.md)。 +如果您要向 Agent 或 Workflow 添加工具提供程序，请参考 [此指南](./api/core/tools/README.md)。 -为了帮助您快速了解您的贡献在哪个部分，以下是Dify后端和前端的简要注释大纲： +为了帮助您快速了解您的贡献在哪个部分，以下是 Dify 后端和前端的简要注释大纲： ### 后端 -Dify的后端使用Python编写，使用[Flask](https://flask.palletsprojects.com/en/3.0.x/)框架。它使用[SQLAlchemy](https://www.sqlalchemy.org/)作为ORM，使用[Celery](https://docs.celeryq.dev/en/stable/getting-started/introduction.html)作为任务队列。授权逻辑通过Flask-login进行处理。 +Dify 的后端使用 Python 编写，使用 [Flask](https://flask.palletsprojects.com/en/3.0.x/) 框架。它使用 [SQLAlchemy](https://www.sqlalchemy.org/) 作为 ORM，使用 [Celery](https://docs.celeryq.dev/en/stable/getting-started/introduction.html) 作为任务队列。授权逻辑通过 Flask-login 进行处理。 ``` [api/] -├── constants // Constant settings used throughout code base. -├── controllers // API route definitions and request handling logic. -├── core // Core application orchestration, model integrations, and tools. -├── docker // Docker & containerization related configurations. -├── events // Event handling and processing -├── extensions // Extensions with 3rd party frameworks/platforms. -├── fields // field definitions for serialization/marshalling. -├── libs // Reusable libraries and helpers. -├── migrations // Scripts for database migration. -├── models // Database models & schema definitions. -├── services // Specifies business logic. -├── storage // Private key storage. -├── tasks // Handling of async tasks and background jobs. +├── constants // 用于整个代码库的常量设置。 +├── controllers // API 路由定义和请求处理逻辑。 +├── core // 核心应用编排、模型集成和工具。 +├── docker // Docker 和容器化相关配置。 +├── events // 事件处理和处理。 +├── extensions // 与第三方框架/平台的扩展。 +├── fields // 用于序列化/封装的字段定义。 +├── libs // 可重用的库和助手。 +├── migrations // 数据库迁移脚本。 +├── models // 数据库模型和架构定义。 +├── services // 指定业务逻辑。 +├── storage // 私钥存储。 +├── tasks // 异步任务和后台作业的处理。 └── tests ``` ### 前端 -该网站使用基于Typescript的[Next.js](https://nextjs.org/)模板进行引导，并使用[Tailwind CSS](https://tailwindcss.com/)进行样式设计。[React-i18next](https://react.i18next.com/)用于国际化。 +该网站使用基于 Typescript 的 [Next.js](https://nextjs.org/) 模板进行引导，并使用 [Tailwind CSS](https://tailwindcss.com/) 进行样式设计。[React-i18next](https://react.i18next.com/) 用于国际化。 ``` [web/] -├── app // layouts, pages, and components -│ ├── (commonLayout) // common layout used throughout the app -│ ├── (shareLayout) // layouts specifically shared across token-specific sessions -│ ├── activate // activate page -│ ├── components // shared by pages and layouts -│ ├── install // install page -│ ├── signin // signin page -│ └── styles // globally shared styles -├── assets // Static assets -├── bin // scripts ran at build step -├── config // adjustable settings and options -├── context // shared contexts used by different portions of the app -├── dictionaries // Language-specific translate files -├── docker // container configurations -├── hooks // Reusable hooks -├── i18n // Internationalization configuration -├── models // describes data models & shapes of API responses -├── public // meta assets like favicon -├── service // specifies shapes of API actions +├── app // 布局、页面和组件 +│ ├── (commonLayout) // 整个应用通用的布局 +│ ├── (shareLayout) // 在特定会话中共享的布局 +│ ├── activate // 激活页面 +│ ├── components // 页面和布局共享的组件 +│ ├── install // 安装页面 +│ ├── signin // 登录页面 +│ └── styles // 全局共享的样式 +├── assets // 静态资源 +├── bin // 构建步骤运行的脚本 +├── config // 可调整的设置和选项 +├── context // 应用中不同部分使用的共享上下文 +├── dictionaries // 语言特定的翻译文件 +├── docker // 容器配置 +├── hooks // 可重用的钩子 +├── i18n // 国际化配置 +├── models // 描述数据模型和 API 响应的形状 +├── public // 如 favicon 等元资源 +├── service // 定义 API 操作的形状 ├── test -├── types // descriptions of function params and return values -└── utils // Shared utility functions +├── types // 函数参数和返回值的描述 +└── utils // 共享的实用函数 ``` ## 提交你的 PR diff --git a/CONTRIBUTING_JA.md b/CONTRIBUTING_JA.md index e8f5456a3c..6d5bfb205c 100644 --- a/CONTRIBUTING_JA.md +++ b/CONTRIBUTING_JA.md @@ -1,7 +1,7 @@ Dify にコントリビュートしたいとお考えなのですね。それは素晴らしいことです。 私たちは、LLM アプリケーションの構築と管理のための最も直感的なワークフローを設計するという壮大な野望を持っています。人数も資金も限られている新興企業として、コミュニティからの支援は本当に重要です。 -私たちは現状を鑑み、機敏かつ迅速に開発をする必要がありますが、同時にあなたのようなコントリビューターの方々に、可能な限りスムーズな貢献体験をしていただきたいと思っています。そのためにこのコントリビュートガイドを作成しました。 +私たちは現状を鑑み、機敏かつ迅速に開発をする必要がありますが、同時にあなた様のようなコントリビューターの方々に、可能な限りスムーズな貢献体験をしていただきたいと思っています。そのためにこのコントリビュートガイドを作成しました。 コードベースやコントリビュータの方々と私たちがどのように仕事をしているのかに慣れていただき、楽しいパートにすぐに飛び込めるようにすることが目的です。 このガイドは Dify そのものと同様に、継続的に改善されています。実際のプロジェクトに遅れをとることがあるかもしれませんが、ご理解のほどよろしくお願いいたします。 @@ -14,13 +14,13 @@ Dify にコントリビュートしたいとお考えなのですね。それは ### 機能リクエスト -* 新しい機能要望を出す場合は、提案する機能が何を実現するものなのかを説明し、可能な限り多くのコンテキストを含めてください。[@perzeusss](https://github.com/perzeuss)は、あなたの要望を書き出すのに役立つ [Feature Request Copilot](https://udify.app/chat/MK2kVSnw1gakVwMX) を作ってくれました。気軽に試してみてください。 +* 新しい機能要望を出す場合は、提案する機能が何を実現するものなのかを説明し、可能な限り多くのコンテキストを含めてください。[@perzeusss](https://github.com/perzeuss)は、あなた様の要望を書き出すのに役立つ [Feature Request Copilot](https://udify.app/chat/MK2kVSnw1gakVwMX) を作ってくれました。気軽に試してみてください。 * 既存の課題から 1 つ選びたい場合は、その下にコメントを書いてください。 - 関連する方向で作業しているチームメンバーが参加します。すべてが良好であれば、コーディングを開始する許可が与えられます。私たちが変更を提案した場合にあなたの作業が無駄になることがないよう、それまでこの機能の作業を控えていただくようお願いいたします。 + 関連する方向で作業しているチームメンバーが参加します。すべてが良好であれば、コーディングを開始する許可が与えられます。私たちが変更を提案した場合にあなた様の作業が無駄になることがないよう、それまでこの機能の作業を控えていただくようお願いいたします。 - 提案された機能がどの分野に属するかによって、あなたは異なるチーム・メンバーと話をするかもしれません。以下は、各チームメンバーが現在取り組んでいる分野の概要です。 + 提案された機能がどの分野に属するかによって、あなた様は異なるチーム・メンバーと話をするかもしれません。以下は、各チームメンバーが現在取り組んでいる分野の概要です。 | Member | Scope | | --------------------------------------------------------------------------------------- | ------------------------------------ | @@ -82,7 +82,7 @@ Dify はバックエンドとフロントエンドから構成されています まず`cd api/`でバックエンドのディレクトリに移動し、[Backend README](api/README.md)に従ってインストールします。 次に別のターミナルで、`cd web/`でフロントエンドのディレクトリに移動し、[Frontend README](web/README.md)に従ってインストールしてください。 -よくある問題とトラブルシューティングの手順については、[installation FAQ](https://docs.dify.ai/getting-started/faq/install-faq) を確認してください。 +よくある問題とトラブルシューティングの手順については、[installation FAQ](https://docs.dify.ai/v/japanese/learn-more/faq/install-faq) を確認してください。 ### 5. ブラウザで dify にアクセスする @@ -153,7 +153,7 @@ Dify のバックエンドは[Flask](https://flask.palletsprojects.com/en/3.0.x/ いよいよ、私たちのリポジトリにプルリクエスト (PR) を提出する時が来ました。主要な機能については、まず `deploy/dev` ブランチにマージしてテストしてから `main` ブランチにマージします。 マージ競合などの問題が発生した場合、またはプル リクエストを開く方法がわからない場合は、[GitHub's pull request tutorial](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests) をチェックしてみてください。 -これで完了です！あなたの PR がマージされると、[README](https://github.com/langgenius/dify/blob/main/README.md) にコントリビューターとして紹介されます。 +これで完了です！あなた様の PR がマージされると、[README](https://github.com/langgenius/dify/blob/main/README.md) にコントリビューターとして紹介されます。 ## ヘルプを得る diff --git a/README.md b/README.md index 40a6837c42..5d8221bcbd 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,6 @@ At the same time, please consider supporting Dify by sharing it on social media * [Github Discussion](https://github.com/langgenius/dify/discussions). Best for: sharing feedback and asking questions. * [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs you encounter using Dify.AI, and feature proposals. See our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). -* [Email](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). Best for: questions you have about using Dify.AI. * [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community. * [Twitter](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community. diff --git a/README_AR.md b/README_AR.md index 35be2ba9b6..c91602721e 100644 --- a/README_AR.md +++ b/README_AR.md @@ -199,7 +199,6 @@ docker compose up -d ## المجتمع والاتصال * [مناقشة Github](https://github.com/langgenius/dify/discussions). الأفضل لـ: مشاركة التعليقات وطرح الأسئلة. * [المشكلات على GitHub](https://github.com/langgenius/dify/issues). الأفضل لـ: الأخطاء التي تواجهها في استخدام Dify.AI، واقتراحات الميزات. انظر [دليل المساهمة](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). -* [البريد الإلكتروني](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). الأفضل لـ: الأسئلة التي تتعلق باستخدام Dify.AI. * [Discord](https://discord.gg/FngNHpbcY7). الأفضل لـ: مشاركة تطبيقاتك والترفيه مع المجتمع. * [تويتر](https://twitter.com/dify_ai). الأفضل لـ: مشاركة تطبيقاتك والترفيه مع المجتمع. diff --git a/README_ES.md b/README_ES.md index ed613be8d4..84c06a2503 100644 --- a/README_ES.md +++ b/README_ES.md @@ -224,7 +224,6 @@ Al mismo tiempo, considera apoyar a Dify compartiéndolo en redes sociales y en * [Discusión en GitHub](https://github.com/langgenius/dify/discussions). Lo mejor para: compartir comentarios y hacer preguntas. * [Reporte de problemas en GitHub](https://github.com/langgenius/dify/issues). Lo mejor para: errores que encuentres usando Dify.AI y propuestas de características. Consulta nuestra [Guía de contribución](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). -* [Correo electrónico](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). Lo mejor para: preguntas que tengas sobre el uso de Dify.AI. * [Discord](https://discord.gg/FngNHpbcY7). Lo mejor para: compartir tus aplicaciones y pasar el rato con la comunidad. * [Twitter](https://twitter.com/dify_ai). Lo mejor para: compartir tus aplicaciones y pasar el rato con la comunidad. diff --git a/README_FR.md b/README_FR.md index 6f09773bf2..768c9390d8 100644 --- a/README_FR.md +++ b/README_FR.md @@ -222,7 +222,6 @@ Dans le même temps, veuillez envisager de soutenir Dify en le partageant sur le * [Discussion GitHub](https://github.com/langgenius/dify/discussions). Meilleur pour: partager des commentaires et poser des questions. * [Problèmes GitHub](https://github.com/langgenius/dify/issues). Meilleur pour: les bogues que vous rencontrez en utilisant Dify.AI et les propositions de fonctionnalités. Consultez notre [Guide de contribution](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). -* [E-mail](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). Meilleur pour: les questions que vous avez sur l'utilisation de Dify.AI. * [Discord](https://discord.gg/FngNHpbcY7). Meilleur pour: partager vos applications et passer du temps avec la communauté. * [Twitter](https://twitter.com/dify_ai). Meilleur pour: partager vos applications et passer du temps avec la communauté. diff --git a/README_JA.md b/README_JA.md index 55f6e173fd..f4cccd5271 100644 --- a/README_JA.md +++ b/README_JA.md @@ -221,7 +221,6 @@ docker compose up -d * [Github Discussion](https://github.com/langgenius/dify/discussions). 主に: フィードバックの共有や質問。 * [GitHub Issues](https://github.com/langgenius/dify/issues). 主に: Dify.AIを使用する際に発生するエラーや問題については、[貢献ガイド](CONTRIBUTING_JA.md)を参照してください -* [Email](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). 主に: Dify.AIの使用に関する質問。 * [Discord](https://discord.gg/FngNHpbcY7). 主に: アプリケーションの共有やコミュニティとの交流。 * [Twitter](https://twitter.com/dify_ai). 主に: アプリケーションの共有やコミュニティとの交流。 @@ -239,7 +238,7 @@ docker compose up -d 無料の30分間のミーティングをスケジュール - 技術サポート + 技術サポート 技術的な問題やサポートに関する質問 diff --git a/README_KL.md b/README_KL.md index 7fdc0b5181..6a15f39bc6 100644 --- a/README_KL.md +++ b/README_KL.md @@ -224,7 +224,6 @@ At the same time, please consider supporting Dify by sharing it on social media ). Best for: sharing feedback and asking questions. * [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs you encounter using Dify.AI, and feature proposals. See our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). -* [Email](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). Best for: questions you have about using Dify.AI. * [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community. * [Twitter](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community. diff --git a/README_KR.md b/README_KR.md index fa1980a99f..bb15fac8ef 100644 --- a/README_KR.md +++ b/README_KR.md @@ -214,7 +214,6 @@ Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했 * [Github 토론](https://github.com/langgenius/dify/discussions). 피드백 공유 및 질문하기에 적합합니다. * [GitHub 이슈](https://github.com/langgenius/dify/issues). Dify.AI 사용 중 발견한 버그와 기능 제안에 적합합니다. [기여 가이드](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md)를 참조하세요. -* [이메일](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). Dify.AI 사용에 대한 질문하기에 적합합니다. * [디스코드](https://discord.gg/FngNHpbcY7). 애플리케이션 공유 및 커뮤니티와 소통하기에 적합합니다. * [트위터](https://twitter.com/dify_ai). 애플리케이션 공유 및 커뮤니티와 소통하기에 적합합니다. diff --git a/api/.env.example b/api/.env.example index eb0574e252..82bee7cc78 100644 --- a/api/.env.example +++ b/api/.env.example @@ -183,6 +183,7 @@ UPLOAD_IMAGE_FILE_SIZE_LIMIT=10 # Model Configuration MULTIMODAL_SEND_IMAGE_FORMAT=base64 +PROMPT_GENERATION_MAX_TOKENS=512 # Mail configuration, support: resend, smtp MAIL_TYPE= @@ -216,6 +217,7 @@ UNSTRUCTURED_API_KEY= SSRF_PROXY_HTTP_URL= SSRF_PROXY_HTTPS_URL= +SSRF_DEFAULT_MAX_RETRIES=3 BATCH_UPLOAD_LIMIT=10 KEYWORD_DATA_SOURCE_TYPE=database @@ -260,3 +262,6 @@ APP_MAX_ACTIVE_REQUESTS=0 # Plugin configuration PLUGIN_INNER_API_URL=http://127.0.0.1:5002 PLUGIN_INNER_API_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi + +# Celery beat configuration +CELERY_BEAT_SCHEDULER_TIME=1 \ No newline at end of file diff --git a/api/README.md b/api/README.md index 8d2afa5e80..70ca2e86a8 100644 --- a/api/README.md +++ b/api/README.md @@ -12,7 +12,8 @@ ```bash cd ../docker cp middleware.env.example middleware.env - docker compose -f docker-compose.middleware.yaml -p dify up -d + # change the profile to other vector database if you are not using weaviate + docker compose -f docker-compose.middleware.yaml --profile weaviate -p dify up -d cd ../api ``` diff --git a/api/app.py b/api/app.py index f5a6d40e1a..2c484ace85 100644 --- a/api/app.py +++ b/api/app.py @@ -1,7 +1,5 @@ import os -from configs import dify_config - if os.environ.get("DEBUG", "false").lower() != 'true': from gevent import monkey @@ -23,7 +21,9 @@ from flask import Flask, Response, request from flask_cors import CORS from werkzeug.exceptions import Unauthorized +import contexts from commands import register_commands +from configs import dify_config # DO NOT REMOVE BELOW from events import event_handlers @@ -181,7 +181,10 @@ def load_user_from_request(request_from_flask_login): decoded = PassportService().verify(auth_token) user_id = decoded.get('user_id') - return AccountService.load_logged_in_account(account_id=user_id, token=auth_token) + account = AccountService.load_logged_in_account(account_id=user_id, token=auth_token) + if account: + contexts.tenant_id.set(account.current_tenant_id) + return account @login_manager.unauthorized_handler diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 565823e119..d6a149faec 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -23,6 +23,7 @@ class SecurityConfig(BaseSettings): default=24, ) + class AppExecutionConfig(BaseSettings): """ App Execution configs @@ -418,7 +419,6 @@ class DataSetConfig(BaseSettings): default=False, ) - class WorkspaceConfig(BaseSettings): """ Workspace configs @@ -448,6 +448,13 @@ class ImageFormatConfig(BaseSettings): ) +class CeleryBeatConfig(BaseSettings): + CELERY_BEAT_SCHEDULER_TIME: int = Field( + description='the time of the celery scheduler, default to 1 day', + default=1, + ) + + class FeatureConfig( # place the configs in alphabet order AppExecutionConfig, @@ -476,5 +483,6 @@ class FeatureConfig( # hosted services config HostedServiceConfig, + CeleryBeatConfig, ): pass diff --git a/api/configs/feature/hosted_service/__init__.py b/api/configs/feature/hosted_service/__init__.py index 209d46bb76..88fe188587 100644 --- a/api/configs/feature/hosted_service/__init__.py +++ b/api/configs/feature/hosted_service/__init__.py @@ -79,7 +79,7 @@ class HostedAzureOpenAiConfig(BaseSettings): default=False, ) - HOSTED_OPENAI_API_KEY: Optional[str] = Field( + HOSTED_AZURE_OPENAI_API_KEY: Optional[str] = Field( description='', default=None, ) diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index a32b70bdc7..07688e9aeb 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -1,4 +1,5 @@ from typing import Any, Optional +from urllib.parse import quote_plus from pydantic import Field, NonNegativeInt, PositiveInt, computed_field from pydantic_settings import BaseSettings @@ -104,7 +105,7 @@ class DatabaseConfig: ).strip("&") db_extras = f"?{db_extras}" if db_extras else "" return (f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://" - f"{self.DB_USERNAME}:{self.DB_PASSWORD}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_DATABASE}" + f"{quote_plus(self.DB_USERNAME)}:{quote_plus(self.DB_PASSWORD)}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_DATABASE}" f"{db_extras}") SQLALCHEMY_POOL_SIZE: NonNegativeInt = Field( diff --git a/api/configs/middleware/vdb/myscale_config.py b/api/configs/middleware/vdb/myscale_config.py index e513cad0e8..895cd6f176 100644 --- a/api/configs/middleware/vdb/myscale_config.py +++ b/api/configs/middleware/vdb/myscale_config.py @@ -1,4 +1,3 @@ -from typing import Optional from pydantic import BaseModel, Field, PositiveInt @@ -8,32 +7,32 @@ class MyScaleConfig(BaseModel): MyScale configs """ - MYSCALE_HOST: Optional[str] = Field( + MYSCALE_HOST: str = Field( description='MyScale host', - default=None, + default='localhost', ) - MYSCALE_PORT: Optional[PositiveInt] = Field( + MYSCALE_PORT: PositiveInt = Field( description='MyScale port', default=8123, ) - MYSCALE_USER: Optional[str] = Field( + MYSCALE_USER: str = Field( description='MyScale user', - default=None, + default='default', ) - MYSCALE_PASSWORD: Optional[str] = Field( + MYSCALE_PASSWORD: str = Field( description='MyScale password', - default=None, + default='', ) - MYSCALE_DATABASE: Optional[str] = Field( + MYSCALE_DATABASE: str = Field( description='MyScale database name', - default=None, + default='default', ) - MYSCALE_FTS_PARAMS: Optional[str] = Field( + MYSCALE_FTS_PARAMS: str = Field( description='MyScale fts index parameters', - default=None, + default='', ) diff --git a/api/configs/packaging/__init__.py b/api/configs/packaging/__init__.py index 30888d0b71..13c55ca425 100644 --- a/api/configs/packaging/__init__.py +++ b/api/configs/packaging/__init__.py @@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings): CURRENT_VERSION: str = Field( description='Dify version', - default='0.6.13', + default='0.6.15', ) COMMIT_SHA: str = Field( diff --git a/api/constants/__init__.py b/api/constants/__init__.py index e69de29bb2..08a2786994 100644 --- a/api/constants/__init__.py +++ b/api/constants/__init__.py @@ -0,0 +1,2 @@ +# TODO: Update all string in code to use this constant +HIDDEN_VALUE = '[__HIDDEN__]' \ No newline at end of file diff --git a/api/contexts/__init__.py b/api/contexts/__init__.py new file mode 100644 index 0000000000..306fac3a93 --- /dev/null +++ b/api/contexts/__init__.py @@ -0,0 +1,3 @@ +from contextvars import ContextVar + +tenant_id: ContextVar[str] = ContextVar('tenant_id') \ No newline at end of file diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index 6952940649..2f304b970c 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -15,6 +15,7 @@ from fields.app_fields import ( app_pagination_fields, ) from libs.login import login_required +from services.app_dsl_service import AppDslService from services.app_service import AppService ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow', 'completion'] @@ -97,8 +98,42 @@ class AppImportApi(Resource): parser.add_argument('icon_background', type=str, location='json') args = parser.parse_args() - app_service = AppService() - app = app_service.import_app(current_user.current_tenant_id, args['data'], args, current_user) + app = AppDslService.import_and_create_new_app( + tenant_id=current_user.current_tenant_id, + data=args['data'], + args=args, + account=current_user + ) + + return app, 201 + + +class AppImportFromUrlApi(Resource): + @setup_required + @login_required + @account_initialization_required + @marshal_with(app_detail_fields_with_site) + @cloud_edition_billing_resource_check('apps') + def post(self): + """Import app from url""" + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: + raise Forbidden() + + parser = reqparse.RequestParser() + parser.add_argument('url', type=str, required=True, nullable=False, location='json') + parser.add_argument('name', type=str, location='json') + parser.add_argument('description', type=str, location='json') + parser.add_argument('icon', type=str, location='json') + parser.add_argument('icon_background', type=str, location='json') + args = parser.parse_args() + + app = AppDslService.import_and_create_new_app_from_url( + tenant_id=current_user.current_tenant_id, + url=args['url'], + args=args, + account=current_user + ) return app, 201 @@ -177,9 +212,13 @@ class AppCopyApi(Resource): parser.add_argument('icon_background', type=str, location='json') args = parser.parse_args() - app_service = AppService() - data = app_service.export_app(app_model) - app = app_service.import_app(current_user.current_tenant_id, data, args, current_user) + data = AppDslService.export_dsl(app_model=app_model, include_secret=True) + app = AppDslService.import_and_create_new_app( + tenant_id=current_user.current_tenant_id, + data=data, + args=args, + account=current_user + ) return app, 201 @@ -195,10 +234,13 @@ class AppExportApi(Resource): if not current_user.is_editor: raise Forbidden() - app_service = AppService() + # Add include_secret params + parser = reqparse.RequestParser() + parser.add_argument('include_secret', type=inputs.boolean, default=False, location='args') + args = parser.parse_args() return { - "data": app_service.export_app(app_model) + "data": AppDslService.export_dsl(app_model=app_model, include_secret=args['include_secret']) } @@ -322,6 +364,7 @@ class AppTraceApi(Resource): api.add_resource(AppListApi, '/apps') api.add_resource(AppImportApi, '/apps/import') +api.add_resource(AppImportFromUrlApi, '/apps/import/url') api.add_resource(AppApi, '/apps/') api.add_resource(AppCopyApi, '/apps//copy') api.add_resource(AppExportApi, '/apps//export') diff --git a/api/controllers/console/app/conversation.py b/api/controllers/console/app/conversation.py index 6475150405..96cd9a6ea1 100644 --- a/api/controllers/console/app/conversation.py +++ b/api/controllers/console/app/conversation.py @@ -22,7 +22,7 @@ from fields.conversation_fields import ( ) from libs.helper import datetime_string from libs.login import login_required -from models.model import AppMode, Conversation, Message, MessageAnnotation +from models.model import AppMode, Conversation, EndUser, Message, MessageAnnotation class CompletionConversationApi(Resource): @@ -156,19 +156,31 @@ class ChatConversationApi(Resource): parser.add_argument('limit', type=int_range(1, 100), required=False, default=20, location='args') args = parser.parse_args() + subquery = ( + db.session.query( + Conversation.id.label('conversation_id'), + EndUser.session_id.label('from_end_user_session_id') + ) + .outerjoin(EndUser, Conversation.from_end_user_id == EndUser.id) + .subquery() + ) + query = db.select(Conversation).where(Conversation.app_id == app_model.id) if args['keyword']: + keyword_filter = '%{}%'.format(args['keyword']) query = query.join( - Message, Message.conversation_id == Conversation.id + Message, Message.conversation_id == Conversation.id, + ).join( + subquery, subquery.c.conversation_id == Conversation.id ).filter( or_( - Message.query.ilike('%{}%'.format(args['keyword'])), - Message.answer.ilike('%{}%'.format(args['keyword'])), - Conversation.name.ilike('%{}%'.format(args['keyword'])), - Conversation.introduction.ilike('%{}%'.format(args['keyword'])), + Message.query.ilike(keyword_filter), + Message.answer.ilike(keyword_filter), + Conversation.name.ilike(keyword_filter), + Conversation.introduction.ilike(keyword_filter), + subquery.c.from_end_user_session_id.ilike(keyword_filter) ), - ) account = current_user diff --git a/api/controllers/console/app/generator.py b/api/controllers/console/app/generator.py index ee02fc1846..6803775e20 100644 --- a/api/controllers/console/app/generator.py +++ b/api/controllers/console/app/generator.py @@ -1,3 +1,5 @@ +import os + from flask_login import current_user from flask_restful import Resource, reqparse @@ -22,17 +24,21 @@ class RuleGenerateApi(Resource): @account_initialization_required def post(self): parser = reqparse.RequestParser() - parser.add_argument('audiences', type=str, required=True, nullable=False, location='json') - parser.add_argument('hoping_to_solve', type=str, required=True, nullable=False, location='json') + parser.add_argument('instruction', type=str, required=True, nullable=False, location='json') + parser.add_argument('model_config', type=dict, required=True, nullable=False, location='json') + parser.add_argument('no_variable', type=bool, required=True, default=False, location='json') args = parser.parse_args() account = current_user + PROMPT_GENERATION_MAX_TOKENS = int(os.getenv('PROMPT_GENERATION_MAX_TOKENS', '512')) try: rules = LLMGenerator.generate_rule_config( - account.current_tenant_id, - args['audiences'], - args['hoping_to_solve'] + tenant_id=account.current_tenant_id, + instruction=args['instruction'], + model_config=args['model_config'], + no_variable=args['no_variable'], + rule_config_max_tokens=PROMPT_GENERATION_MAX_TOKENS ) except ProviderTokenNotInitError as ex: raise ProviderNotInitializeError(ex.description) diff --git a/api/controllers/console/app/statistic.py b/api/controllers/console/app/statistic.py index d687b52dc8..b882ffef34 100644 --- a/api/controllers/console/app/statistic.py +++ b/api/controllers/console/app/statistic.py @@ -281,7 +281,7 @@ class UserSatisfactionRateStatistic(Resource): SELECT date(DATE_TRUNC('day', m.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, COUNT(m.id) as message_count, COUNT(mf.id) as feedback_count FROM messages m - LEFT JOIN message_feedbacks mf on mf.message_id=m.id + LEFT JOIN message_feedbacks mf on mf.message_id=m.id and mf.rating='like' WHERE m.app_id = :app_id ''' arg_dict = {'tz': account.timezone, 'app_id': app_model.id} diff --git a/api/controllers/console/app/workflow.py b/api/controllers/console/app/workflow.py index cadb75c547..686ef7b4be 100644 --- a/api/controllers/console/app/workflow.py +++ b/api/controllers/console/app/workflow.py @@ -13,6 +13,7 @@ from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import InvokeFrom +from core.app.segments import factory from core.errors.error import AppInvokeQuotaExceededError from fields.workflow_fields import workflow_fields from fields.workflow_run_fields import workflow_run_node_execution_fields @@ -20,6 +21,7 @@ from libs import helper from libs.helper import TimestampField, uuid_value from libs.login import current_user, login_required from models.model import App, AppMode +from services.app_dsl_service import AppDslService from services.app_generate_service import AppGenerateService from services.errors.app import WorkflowHashNotEqualError from services.workflow_service import WorkflowService @@ -40,7 +42,7 @@ class DraftWorkflowApi(Resource): # The role of the current user in the ta table must be admin, owner, or editor if not current_user.is_editor: raise Forbidden() - + # fetch draft workflow by app_model workflow_service = WorkflowService() workflow = workflow_service.get_draft_workflow(app_model=app_model) @@ -63,13 +65,15 @@ class DraftWorkflowApi(Resource): if not current_user.is_editor: raise Forbidden() - content_type = request.headers.get('Content-Type') + content_type = request.headers.get('Content-Type', '') if 'application/json' in content_type: parser = reqparse.RequestParser() parser.add_argument('graph', type=dict, required=True, nullable=False, location='json') parser.add_argument('features', type=dict, required=True, nullable=False, location='json') parser.add_argument('hash', type=str, required=False, location='json') + # TODO: set this to required=True after frontend is updated + parser.add_argument('environment_variables', type=list, required=False, location='json') args = parser.parse_args() elif 'text/plain' in content_type: try: @@ -83,7 +87,8 @@ class DraftWorkflowApi(Resource): args = { 'graph': data.get('graph'), 'features': data.get('features'), - 'hash': data.get('hash') + 'hash': data.get('hash'), + 'environment_variables': data.get('environment_variables') } except json.JSONDecodeError: return {'message': 'Invalid JSON data'}, 400 @@ -93,12 +98,15 @@ class DraftWorkflowApi(Resource): workflow_service = WorkflowService() try: + environment_variables_list = args.get('environment_variables') or [] + environment_variables = [factory.build_variable_from_mapping(obj) for obj in environment_variables_list] workflow = workflow_service.sync_draft_workflow( app_model=app_model, - graph=args.get('graph'), - features=args.get('features'), + graph=args['graph'], + features=args['features'], unique_hash=args.get('hash'), - account=current_user + account=current_user, + environment_variables=environment_variables, ) except WorkflowHashNotEqualError: raise DraftWorkflowNotSync() @@ -128,8 +136,7 @@ class DraftWorkflowImportApi(Resource): parser.add_argument('data', type=str, required=True, nullable=False, location='json') args = parser.parse_args() - workflow_service = WorkflowService() - workflow = workflow_service.import_draft_workflow( + workflow = AppDslService.import_and_overwrite_workflow( app_model=app_model, data=args['data'], account=current_user diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 70c506bb0e..c446f523b6 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -1,10 +1,11 @@ import flask_restful -from flask import current_app, request +from flask import request from flask_login import current_user from flask_restful import Resource, marshal, marshal_with, reqparse from werkzeug.exceptions import Forbidden, NotFound import services +from configs import dify_config from controllers.console import api from controllers.console.apikey import api_key_fields, api_key_list from controllers.console.app.error import ProviderNotInitializeError @@ -530,7 +531,7 @@ class DatasetApiBaseUrlApi(Resource): @account_initialization_required def get(self): return { - 'api_base_url': (current_app.config['SERVICE_API_URL'] if current_app.config['SERVICE_API_URL'] + 'api_base_url': (dify_config.SERVICE_API_URL if dify_config.SERVICE_API_URL else request.host_url.rstrip('/')) + '/v1' } @@ -540,20 +541,20 @@ class DatasetRetrievalSettingApi(Resource): @login_required @account_initialization_required def get(self): - vector_type = current_app.config['VECTOR_STORE'] + vector_type = dify_config.VECTOR_STORE match vector_type: - case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE: + case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT: return { 'retrieval_method': [ - RetrievalMethod.SEMANTIC_SEARCH + RetrievalMethod.SEMANTIC_SEARCH.value ] } - case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH | VectorType.ANALYTICDB | VectorType.MYSCALE: + case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH | VectorType.ANALYTICDB | VectorType.MYSCALE | VectorType.ORACLE: return { 'retrieval_method': [ - RetrievalMethod.SEMANTIC_SEARCH, - RetrievalMethod.FULL_TEXT_SEARCH, - RetrievalMethod.HYBRID_SEARCH, + RetrievalMethod.SEMANTIC_SEARCH.value, + RetrievalMethod.FULL_TEXT_SEARCH.value, + RetrievalMethod.HYBRID_SEARCH.value, ] } case _: @@ -566,18 +567,18 @@ class DatasetRetrievalSettingMockApi(Resource): @account_initialization_required def get(self, vector_type): match vector_type: - case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE: + case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT: return { 'retrieval_method': [ - RetrievalMethod.SEMANTIC_SEARCH + RetrievalMethod.SEMANTIC_SEARCH.value ] } - case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH| VectorType.ANALYTICDB | VectorType.MYSCALE: + case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH| VectorType.ANALYTICDB | VectorType.MYSCALE | VectorType.ORACLE: return { 'retrieval_method': [ - RetrievalMethod.SEMANTIC_SEARCH, - RetrievalMethod.FULL_TEXT_SEARCH, - RetrievalMethod.HYBRID_SEARCH, + RetrievalMethod.SEMANTIC_SEARCH.value, + RetrievalMethod.FULL_TEXT_SEARCH.value, + RetrievalMethod.HYBRID_SEARCH.value, ] } case _: diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index a189aac3f1..3dcade6152 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -75,7 +75,7 @@ class DatasetDocumentSegmentListApi(Resource): ) if last_id is not None: - last_segment = DocumentSegment.query.get(str(last_id)) + last_segment = db.session.get(DocumentSegment, str(last_id)) if last_segment: query = query.filter( DocumentSegment.position > last_segment.position) diff --git a/api/controllers/console/datasets/file.py b/api/controllers/console/datasets/file.py index c13bd45abb..3b2083bcc3 100644 --- a/api/controllers/console/datasets/file.py +++ b/api/controllers/console/datasets/file.py @@ -1,8 +1,9 @@ -from flask import current_app, request +from flask import request from flask_login import current_user from flask_restful import Resource, marshal_with import services +from configs import dify_config from controllers.console import api from controllers.console.datasets.error import ( FileTooLargeError, @@ -26,9 +27,9 @@ class FileApi(Resource): @account_initialization_required @marshal_with(upload_config_fields) def get(self): - file_size_limit = current_app.config.get("UPLOAD_FILE_SIZE_LIMIT") - batch_count_limit = current_app.config.get("UPLOAD_FILE_BATCH_LIMIT") - image_file_size_limit = current_app.config.get("UPLOAD_IMAGE_FILE_SIZE_LIMIT") + file_size_limit = dify_config.UPLOAD_FILE_SIZE_LIMIT + batch_count_limit = dify_config.UPLOAD_FILE_BATCH_LIMIT + image_file_size_limit = dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT return { 'file_size_limit': file_size_limit, 'batch_count_limit': batch_count_limit, @@ -76,7 +77,7 @@ class FileSupportTypeApi(Resource): @login_required @account_initialization_required def get(self): - etl_type = current_app.config['ETL_TYPE'] + etl_type = dify_config.ETL_TYPE allowed_extensions = UNSTRUCTURED_ALLOWED_EXTENSIONS if etl_type == 'Unstructured' else ALLOWED_EXTENSIONS return {'allowed_extensions': allowed_extensions} diff --git a/api/controllers/console/explore/audio.py b/api/controllers/console/explore/audio.py index 920b1d8383..27cc83042a 100644 --- a/api/controllers/console/explore/audio.py +++ b/api/controllers/console/explore/audio.py @@ -78,10 +78,12 @@ class ChatTextApi(InstalledAppResource): parser = reqparse.RequestParser() parser.add_argument('message_id', type=str, required=False, location='json') parser.add_argument('voice', type=str, location='json') + parser.add_argument('text', type=str, location='json') parser.add_argument('streaming', type=bool, location='json') args = parser.parse_args() - message_id = args.get('message_id') + message_id = args.get('message_id', None) + text = args.get('text', None) if (app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value] and app_model.workflow and app_model.workflow.features_dict): @@ -95,7 +97,8 @@ class ChatTextApi(InstalledAppResource): response = AudioService.transcript_tts( app_model=app_model, message_id=message_id, - voice=voice + voice=voice, + text=text ) return response except services.errors.app_model_config.AppModelConfigBrokenError: diff --git a/api/controllers/console/explore/parameter.py b/api/controllers/console/explore/parameter.py index 45255edb3a..0a168d6306 100644 --- a/api/controllers/console/explore/parameter.py +++ b/api/controllers/console/explore/parameter.py @@ -1,7 +1,7 @@ -from flask import current_app from flask_restful import fields, marshal_with +from configs import dify_config from controllers.console import api from controllers.console.app.error import AppUnavailableError from controllers.console.explore.wraps import InstalledAppResource @@ -78,7 +78,7 @@ class AppParameterApi(InstalledAppResource): "transfer_methods": ["remote_url", "local_file"] }}), 'system_parameters': { - 'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT') + 'image_file_size_limit': dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT } } diff --git a/api/controllers/console/init_validate.py b/api/controllers/console/init_validate.py index b319f706b4..6feb1003a9 100644 --- a/api/controllers/console/init_validate.py +++ b/api/controllers/console/init_validate.py @@ -1,8 +1,9 @@ import os -from flask import current_app, session +from flask import session from flask_restful import Resource, reqparse +from configs import dify_config from libs.helper import str_len from models.model import DifySetup from services.account_service import TenantService @@ -40,7 +41,7 @@ class InitValidateAPI(Resource): return {'result': 'success'}, 201 def get_init_validate_status(): - if current_app.config['EDITION'] == 'SELF_HOSTED': + if dify_config.EDITION == 'SELF_HOSTED': if os.environ.get('INIT_PASSWORD'): return session.get('is_init_validated') or DifySetup.query.first() diff --git a/api/controllers/console/setup.py b/api/controllers/console/setup.py index def50212a1..ef7cc6bc03 100644 --- a/api/controllers/console/setup.py +++ b/api/controllers/console/setup.py @@ -1,8 +1,9 @@ from functools import wraps -from flask import current_app, request +from flask import request from flask_restful import Resource, reqparse +from configs import dify_config from libs.helper import email, get_remote_ip, str_len from libs.password import valid_password from models.model import DifySetup @@ -17,7 +18,7 @@ from .wraps import only_edition_self_hosted class SetupApi(Resource): def get(self): - if current_app.config['EDITION'] == 'SELF_HOSTED': + if dify_config.EDITION == 'SELF_HOSTED': setup_status = get_setup_status() if setup_status: return { @@ -77,7 +78,7 @@ def setup_required(view): def get_setup_status(): - if current_app.config['EDITION'] == 'SELF_HOSTED': + if dify_config.EDITION == 'SELF_HOSTED': return DifySetup.query.first() else: return True diff --git a/api/controllers/console/version.py b/api/controllers/console/version.py index faf36c4f40..1fcf4bdc00 100644 --- a/api/controllers/console/version.py +++ b/api/controllers/console/version.py @@ -3,9 +3,10 @@ import json import logging import requests -from flask import current_app from flask_restful import Resource, reqparse +from configs import dify_config + from . import api @@ -15,16 +16,16 @@ class VersionApi(Resource): parser = reqparse.RequestParser() parser.add_argument('current_version', type=str, required=True, location='args') args = parser.parse_args() - check_update_url = current_app.config['CHECK_UPDATE_URL'] + check_update_url = dify_config.CHECK_UPDATE_URL result = { - 'version': current_app.config['CURRENT_VERSION'], + 'version': dify_config.CURRENT_VERSION, 'release_date': '', 'release_notes': '', 'can_auto_update': False, 'features': { - 'can_replace_logo': current_app.config['CAN_REPLACE_LOGO'], - 'model_load_balancing_enabled': current_app.config['MODEL_LB_ENABLED'] + 'can_replace_logo': dify_config.CAN_REPLACE_LOGO, + 'model_load_balancing_enabled': dify_config.MODEL_LB_ENABLED } } diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index 0b5c84c2a3..1056d5eb62 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -1,10 +1,11 @@ import datetime import pytz -from flask import current_app, request +from flask import request from flask_login import current_user from flask_restful import Resource, fields, marshal_with, reqparse +from configs import dify_config from constants.languages import supported_language from controllers.console import api from controllers.console.setup import setup_required @@ -36,7 +37,7 @@ class AccountInitApi(Resource): parser = reqparse.RequestParser() - if current_app.config['EDITION'] == 'CLOUD': + if dify_config.EDITION == 'CLOUD': parser.add_argument('invitation_code', type=str, location='json') parser.add_argument( @@ -45,7 +46,7 @@ class AccountInitApi(Resource): required=True, location='json') args = parser.parse_args() - if current_app.config['EDITION'] == 'CLOUD': + if dify_config.EDITION == 'CLOUD': if not args['invitation_code']: raise ValueError('invitation_code is required') diff --git a/api/controllers/console/workspace/members.py b/api/controllers/console/workspace/members.py index e8c88850a4..34e9da3841 100644 --- a/api/controllers/console/workspace/members.py +++ b/api/controllers/console/workspace/members.py @@ -1,8 +1,8 @@ -from flask import current_app from flask_login import current_user from flask_restful import Resource, abort, marshal_with, reqparse import services +from configs import dify_config from controllers.console import api from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check @@ -48,7 +48,7 @@ class MemberInviteEmailApi(Resource): inviter = current_user invitation_results = [] - console_web_url = current_app.config.get("CONSOLE_WEB_URL") + console_web_url = dify_config.CONSOLE_WEB_URL for invitee_email in invitee_emails: try: token = RegisterService.invite_new_member(inviter.current_tenant, invitee_email, interface_language, role=invitee_role, inviter=inviter) @@ -117,7 +117,7 @@ class MemberUpdateRoleApi(Resource): if not TenantAccountRole.is_valid_role(new_role): return {'code': 'invalid-role', 'message': 'Invalid role'}, 400 - member = Account.query.get(str(member_id)) + member = db.session.get(Account, str(member_id)) if not member: abort(404) diff --git a/api/controllers/console/workspace/tool_providers.py b/api/controllers/console/workspace/tool_providers.py index 6e3f78d4e2..bafeabb08a 100644 --- a/api/controllers/console/workspace/tool_providers.py +++ b/api/controllers/console/workspace/tool_providers.py @@ -1,10 +1,11 @@ import io -from flask import current_app, send_file +from flask import send_file from flask_login import current_user from flask_restful import Resource, reqparse from werkzeug.exceptions import Forbidden +from configs import dify_config from controllers.console import api from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required @@ -104,7 +105,7 @@ class ToolBuiltinProviderIconApi(Resource): @setup_required def get(self, provider): icon_bytes, mimetype = BuiltinToolManageService.get_builtin_tool_provider_icon(provider) - icon_cache_max_age = current_app.config.get('TOOL_ICON_CACHE_MAX_AGE') + icon_cache_max_age = dify_config.TOOL_ICON_CACHE_MAX_AGE return send_file(io.BytesIO(icon_bytes), mimetype=mimetype, max_age=icon_cache_max_age) class ToolApiProviderAddApi(Resource): diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index 7c8ad11078..3baf69acfd 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -1,9 +1,10 @@ import json from functools import wraps -from flask import abort, current_app, request +from flask import abort, request from flask_login import current_user +from configs import dify_config from controllers.console.workspace.error import AccountNotInitializedError from services.feature_service import FeatureService from services.operation_service import OperationService @@ -26,7 +27,7 @@ def account_initialization_required(view): def only_edition_cloud(view): @wraps(view) def decorated(*args, **kwargs): - if current_app.config['EDITION'] != 'CLOUD': + if dify_config.EDITION != 'CLOUD': abort(404) return view(*args, **kwargs) @@ -37,7 +38,7 @@ def only_edition_cloud(view): def only_edition_self_hosted(view): @wraps(view) def decorated(*args, **kwargs): - if current_app.config['EDITION'] != 'SELF_HOSTED': + if dify_config.EDITION != 'SELF_HOSTED': abort(404) return view(*args, **kwargs) diff --git a/api/controllers/service_api/app/audio.py b/api/controllers/service_api/app/audio.py index 607d71598f..3c009af343 100644 --- a/api/controllers/service_api/app/audio.py +++ b/api/controllers/service_api/app/audio.py @@ -76,10 +76,12 @@ class TextApi(Resource): parser = reqparse.RequestParser() parser.add_argument('message_id', type=str, required=False, location='json') parser.add_argument('voice', type=str, location='json') + parser.add_argument('text', type=str, location='json') parser.add_argument('streaming', type=bool, location='json') args = parser.parse_args() - message_id = args.get('message_id') + message_id = args.get('message_id', None) + text = args.get('text', None) if (app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value] and app_model.workflow and app_model.workflow.features_dict): @@ -87,15 +89,15 @@ class TextApi(Resource): voice = args.get('voice') if args.get('voice') else text_to_speech.get('voice') else: try: - voice = args.get('voice') if args.get('voice') else app_model.app_model_config.text_to_speech_dict.get( - 'voice') + voice = args.get('voice') if args.get('voice') else app_model.app_model_config.text_to_speech_dict.get('voice') except Exception: voice = None response = AudioService.transcript_tts( app_model=app_model, message_id=message_id, end_user=end_user.external_user_id, - voice=voice + voice=voice, + text=text ) return response diff --git a/api/controllers/service_api/app/workflow.py b/api/controllers/service_api/app/workflow.py index dd11949e84..10484c9027 100644 --- a/api/controllers/service_api/app/workflow.py +++ b/api/controllers/service_api/app/workflow.py @@ -1,6 +1,6 @@ import logging -from flask_restful import Resource, reqparse +from flask_restful import Resource, fields, marshal_with, reqparse from werkzeug.exceptions import InternalServerError from controllers.service_api import api @@ -21,14 +21,43 @@ from core.errors.error import ( QuotaExceededError, ) from core.model_runtime.errors.invoke import InvokeError +from extensions.ext_database import db from libs import helper from models.model import App, AppMode, EndUser +from models.workflow import WorkflowRun from services.app_generate_service import AppGenerateService logger = logging.getLogger(__name__) class WorkflowRunApi(Resource): + workflow_run_fields = { + 'id': fields.String, + 'workflow_id': fields.String, + 'status': fields.String, + 'inputs': fields.Raw, + 'outputs': fields.Raw, + 'error': fields.String, + 'total_steps': fields.Integer, + 'total_tokens': fields.Integer, + 'created_at': fields.DateTime, + 'finished_at': fields.DateTime, + 'elapsed_time': fields.Float, + } + + @validate_app_token + @marshal_with(workflow_run_fields) + def get(self, app_model: App, workflow_id: str): + """ + Get a workflow task running detail + """ + app_mode = AppMode.value_of(app_model.mode) + if app_mode != AppMode.WORKFLOW: + raise NotWorkflowAppError() + + workflow_run = db.session.query(WorkflowRun).filter(WorkflowRun.id == workflow_id).first() + return workflow_run + @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True)) def post(self, app_model: App, end_user: EndUser): """ @@ -88,5 +117,5 @@ class WorkflowTaskStopApi(Resource): } -api.add_resource(WorkflowRunApi, '/workflows/run') +api.add_resource(WorkflowRunApi, '/workflows/run/', '/workflows/run') api.add_resource(WorkflowTaskStopApi, '/workflows/tasks//stop') diff --git a/api/controllers/web/audio.py b/api/controllers/web/audio.py index 8be872f5f9..0e905f905a 100644 --- a/api/controllers/web/audio.py +++ b/api/controllers/web/audio.py @@ -74,10 +74,12 @@ class TextApi(WebApiResource): parser = reqparse.RequestParser() parser.add_argument('message_id', type=str, required=False, location='json') parser.add_argument('voice', type=str, location='json') + parser.add_argument('text', type=str, location='json') parser.add_argument('streaming', type=bool, location='json') args = parser.parse_args() - message_id = args.get('message_id') + message_id = args.get('message_id', None) + text = args.get('text', None) if (app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value] and app_model.workflow and app_model.workflow.features_dict): @@ -94,7 +96,8 @@ class TextApi(WebApiResource): app_model=app_model, message_id=message_id, end_user=end_user.external_user_id, - voice=voice + voice=voice, + text=text ) return response diff --git a/api/core/agent/fc_agent_runner.py b/api/core/agent/fc_agent_runner.py index bec76e7a24..7019b5e39f 100644 --- a/api/core/agent/fc_agent_runner.py +++ b/api/core/agent/fc_agent_runner.py @@ -342,10 +342,14 @@ class FunctionCallAgentRunner(BaseAgentRunner): """ tool_calls = [] for prompt_message in llm_result_chunk.delta.message.tool_calls: + args = {} + if prompt_message.function.arguments != '': + args = json.loads(prompt_message.function.arguments) + tool_calls.append(( prompt_message.id, prompt_message.function.name, - json.loads(prompt_message.function.arguments), + args, )) return tool_calls @@ -359,10 +363,14 @@ class FunctionCallAgentRunner(BaseAgentRunner): """ tool_calls = [] for prompt_message in llm_result.message.tool_calls: + args = {} + if prompt_message.function.arguments != '': + args = json.loads(prompt_message.function.arguments) + tool_calls.append(( prompt_message.id, prompt_message.function.name, - json.loads(prompt_message.function.arguments), + args, )) return tool_calls diff --git a/api/core/app/app_config/base_app_config_manager.py b/api/core/app/app_config/base_app_config_manager.py index 353fe85b74..3dea305e98 100644 --- a/api/core/app/app_config/base_app_config_manager.py +++ b/api/core/app/app_config/base_app_config_manager.py @@ -1,6 +1,7 @@ -from typing import Optional, Union +from collections.abc import Mapping +from typing import Any -from core.app.app_config.entities import AppAdditionalFeatures, EasyUIBasedAppModelConfigFrom +from core.app.app_config.entities import AppAdditionalFeatures from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.app.app_config.features.more_like_this.manager import MoreLikeThisConfigManager from core.app.app_config.features.opening_statement.manager import OpeningStatementConfigManager @@ -10,37 +11,19 @@ from core.app.app_config.features.suggested_questions_after_answer.manager impor SuggestedQuestionsAfterAnswerConfigManager, ) from core.app.app_config.features.text_to_speech.manager import TextToSpeechConfigManager -from models.model import AppMode, AppModelConfig +from models.model import AppMode class BaseAppConfigManager: - @classmethod - def convert_to_config_dict(cls, config_from: EasyUIBasedAppModelConfigFrom, - app_model_config: Union[AppModelConfig, dict], - config_dict: Optional[dict] = None) -> dict: - """ - Convert app model config to config dict - :param config_from: app model config from - :param app_model_config: app model config - :param config_dict: app model config dict - :return: - """ - if config_from != EasyUIBasedAppModelConfigFrom.ARGS: - app_model_config_dict = app_model_config.to_dict() - config_dict = app_model_config_dict.copy() - - return config_dict - - @classmethod - def convert_features(cls, config_dict: dict, app_mode: AppMode) -> AppAdditionalFeatures: + def convert_features(cls, config_dict: Mapping[str, Any], app_mode: AppMode) -> AppAdditionalFeatures: """ Convert app config to app model config :param config_dict: app config :param app_mode: app mode """ - config_dict = config_dict.copy() + config_dict = dict(config_dict.items()) additional_features = AppAdditionalFeatures() additional_features.show_retrieve_source = RetrievalResourceConfigManager.convert( diff --git a/api/core/app/app_config/easy_ui_based_app/dataset/manager.py b/api/core/app/app_config/easy_ui_based_app/dataset/manager.py index c10aa98dba..13da5514d1 100644 --- a/api/core/app/app_config/easy_ui_based_app/dataset/manager.py +++ b/api/core/app/app_config/easy_ui_based_app/dataset/manager.py @@ -62,7 +62,12 @@ class DatasetConfigManager: return None # dataset configs - dataset_configs = config.get('dataset_configs', {'retrieval_model': 'single'}) + if 'dataset_configs' in config and config.get('dataset_configs'): + dataset_configs = config.get('dataset_configs') + else: + dataset_configs = { + 'retrieval_model': 'multiple' + } query_variable = config.get('dataset_query_variable') if dataset_configs['retrieval_model'] == 'single': @@ -83,9 +88,10 @@ class DatasetConfigManager: retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.value_of( dataset_configs['retrieval_model'] ), - top_k=dataset_configs.get('top_k'), + top_k=dataset_configs.get('top_k', 4), score_threshold=dataset_configs.get('score_threshold'), - reranking_model=dataset_configs.get('reranking_model') + reranking_model=dataset_configs.get('reranking_model'), + weights=dataset_configs.get('weights') ) ) @@ -114,12 +120,6 @@ class DatasetConfigManager: if not isinstance(config["dataset_configs"], dict): raise ValueError("dataset_configs must be of object type") - if config["dataset_configs"]['retrieval_model'] == 'multiple': - if not config["dataset_configs"]['reranking_model']: - raise ValueError("reranking_model has not been set") - if not isinstance(config["dataset_configs"]['reranking_model'], dict): - raise ValueError("reranking_model must be of object type") - if not isinstance(config["dataset_configs"], dict): raise ValueError("dataset_configs must be of object type") diff --git a/api/core/app/app_config/entities.py b/api/core/app/app_config/entities.py index 9b7012c3fb..9133a35c08 100644 --- a/api/core/app/app_config/entities.py +++ b/api/core/app/app_config/entities.py @@ -159,7 +159,11 @@ class DatasetRetrieveConfigEntity(BaseModel): retrieve_strategy: RetrieveStrategy top_k: Optional[int] = None score_threshold: Optional[float] = None + rerank_mode: Optional[str] = 'reranking_model' reranking_model: Optional[dict] = None + weights: Optional[dict] = None + + class DatasetEntity(BaseModel): diff --git a/api/core/app/app_config/features/file_upload/manager.py b/api/core/app/app_config/features/file_upload/manager.py index 2049b573cd..86799fb1ab 100644 --- a/api/core/app/app_config/features/file_upload/manager.py +++ b/api/core/app/app_config/features/file_upload/manager.py @@ -1,11 +1,12 @@ -from typing import Optional +from collections.abc import Mapping +from typing import Any, Optional from core.app.app_config.entities import FileExtraConfig class FileUploadConfigManager: @classmethod - def convert(cls, config: dict, is_vision: bool = True) -> Optional[FileExtraConfig]: + def convert(cls, config: Mapping[str, Any], is_vision: bool = True) -> Optional[FileExtraConfig]: """ Convert model config to model config diff --git a/api/core/app/app_config/features/text_to_speech/manager.py b/api/core/app/app_config/features/text_to_speech/manager.py index b516fa46ab..f11e268e73 100644 --- a/api/core/app/app_config/features/text_to_speech/manager.py +++ b/api/core/app/app_config/features/text_to_speech/manager.py @@ -3,13 +3,13 @@ from core.app.app_config.entities import TextToSpeechEntity class TextToSpeechConfigManager: @classmethod - def convert(cls, config: dict) -> bool: + def convert(cls, config: dict): """ Convert model config to model config :param config: model config args """ - text_to_speech = False + text_to_speech = None text_to_speech_dict = config.get('text_to_speech') if text_to_speech_dict: if text_to_speech_dict.get('enabled'): diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index 84723cb5c7..0141dbec58 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -1,3 +1,4 @@ +import contextvars import logging import os import threading @@ -8,6 +9,7 @@ from typing import Union from flask import Flask, current_app from pydantic import ValidationError +import contexts from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager from core.app.apps.advanced_chat.app_runner import AdvancedChatAppRunner @@ -107,6 +109,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): extras=extras, trace_manager=trace_manager ) + contexts.tenant_id.set(application_generate_entity.app_config.tenant_id) return self._generate( app_model=app_model, @@ -173,6 +176,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): inputs=args['inputs'] ) ) + contexts.tenant_id.set(application_generate_entity.app_config.tenant_id) return self._generate( app_model=app_model, @@ -225,6 +229,8 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): 'queue_manager': queue_manager, 'conversation_id': conversation.id, 'message_id': message.id, + 'user': user, + 'context': contextvars.copy_context() }) worker_thread.start() @@ -249,7 +255,9 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): application_generate_entity: AdvancedChatAppGenerateEntity, queue_manager: AppQueueManager, conversation_id: str, - message_id: str) -> None: + message_id: str, + user: Account, + context: contextvars.Context) -> None: """ Generate worker in a new thread. :param flask_app: Flask app @@ -259,6 +267,8 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): :param message_id: message ID :return: """ + for var, val in context.items(): + var.set(val) with flask_app.app_context(): try: runner = AdvancedChatAppRunner() diff --git a/api/core/app/apps/advanced_chat/app_runner.py b/api/core/app/apps/advanced_chat/app_runner.py index 3856621700..18db0ab22d 100644 --- a/api/core/app/apps/advanced_chat/app_runner.py +++ b/api/core/app/apps/advanced_chat/app_runner.py @@ -1,7 +1,8 @@ import logging import os import time -from typing import Optional, cast +from collections.abc import Mapping +from typing import Any, Optional, cast from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfig from core.app.apps.advanced_chat.workflow_event_trigger_callback import WorkflowEventTriggerCallback @@ -14,6 +15,7 @@ from core.app.entities.app_invoke_entities import ( ) from core.app.entities.queue_entities import QueueAnnotationReplyEvent, QueueStopEvent, QueueTextChunkEvent from core.moderation.base import ModerationException +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.node_entities import SystemVariable from core.workflow.nodes.base_node import UserFrom from core.workflow.workflow_engine_manager import WorkflowEngineManager @@ -87,7 +89,7 @@ class AdvancedChatAppRunner(AppRunner): db.session.close() - workflow_callbacks = [WorkflowEventTriggerCallback( + workflow_callbacks: list[WorkflowCallback] = [WorkflowEventTriggerCallback( queue_manager=queue_manager, workflow=workflow )] @@ -161,7 +163,7 @@ class AdvancedChatAppRunner(AppRunner): self, queue_manager: AppQueueManager, app_record: App, app_generate_entity: AdvancedChatAppGenerateEntity, - inputs: dict, + inputs: Mapping[str, Any], query: str, message_id: str ) -> bool: diff --git a/api/core/app/apps/advanced_chat/generate_response_converter.py b/api/core/app/apps/advanced_chat/generate_response_converter.py index 08069332ba..ef579827b4 100644 --- a/api/core/app/apps/advanced_chat/generate_response_converter.py +++ b/api/core/app/apps/advanced_chat/generate_response_converter.py @@ -1,9 +1,11 @@ import json from collections.abc import Generator -from typing import cast +from typing import Any, cast from core.app.apps.base_app_generate_response_converter import AppGenerateResponseConverter from core.app.entities.task_entities import ( + AppBlockingResponse, + AppStreamResponse, ChatbotAppBlockingResponse, ChatbotAppStreamResponse, ErrorStreamResponse, @@ -18,12 +20,13 @@ class AdvancedChatAppGenerateResponseConverter(AppGenerateResponseConverter): _blocking_response_type = ChatbotAppBlockingResponse @classmethod - def convert_blocking_full_response(cls, blocking_response: ChatbotAppBlockingResponse) -> dict: + def convert_blocking_full_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]: """ Convert blocking full response. :param blocking_response: blocking response :return: """ + blocking_response = cast(ChatbotAppBlockingResponse, blocking_response) response = { 'event': 'message', 'task_id': blocking_response.task_id, @@ -39,7 +42,7 @@ class AdvancedChatAppGenerateResponseConverter(AppGenerateResponseConverter): return response @classmethod - def convert_blocking_simple_response(cls, blocking_response: ChatbotAppBlockingResponse) -> dict: + def convert_blocking_simple_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]: """ Convert blocking simple response. :param blocking_response: blocking response @@ -53,8 +56,7 @@ class AdvancedChatAppGenerateResponseConverter(AppGenerateResponseConverter): return response @classmethod - def convert_stream_full_response(cls, stream_response: Generator[ChatbotAppStreamResponse, None, None]) \ - -> Generator[str, None, None]: + def convert_stream_full_response(cls, stream_response: Generator[AppStreamResponse, None, None]) -> Generator[str, Any, None]: """ Convert stream full response. :param stream_response: stream response @@ -83,8 +85,7 @@ class AdvancedChatAppGenerateResponseConverter(AppGenerateResponseConverter): yield json.dumps(response_chunk) @classmethod - def convert_stream_simple_response(cls, stream_response: Generator[ChatbotAppStreamResponse, None, None]) \ - -> Generator[str, None, None]: + def convert_stream_simple_response(cls, stream_response: Generator[AppStreamResponse, None, None]) -> Generator[str, Any, None]: """ Convert stream simple response. :param stream_response: stream response diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index 4b089f033f..be72d89c1e 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -118,7 +118,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc self._stream_generate_routes = self._get_stream_generate_routes() self._conversation_name_generate_thread = None - def process(self) -> Union[ChatbotAppBlockingResponse, Generator[ChatbotAppStreamResponse, None, None]]: + def process(self): """ Process generate task pipeline. :return: @@ -141,8 +141,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc else: return self._to_blocking_response(generator) - def _to_blocking_response(self, generator: Generator[StreamResponse, None, None]) \ - -> ChatbotAppBlockingResponse: + def _to_blocking_response(self, generator: Generator[StreamResponse, None, None]) -> ChatbotAppBlockingResponse: """ Process blocking response. :return: @@ -172,8 +171,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc raise Exception('Queue listening stopped unexpectedly.') - def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) \ - -> Generator[ChatbotAppStreamResponse, None, None]: + def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) -> Generator[ChatbotAppStreamResponse, Any, None]: """ To stream response. :return: diff --git a/api/core/app/apps/advanced_chat/workflow_event_trigger_callback.py b/api/core/app/apps/advanced_chat/workflow_event_trigger_callback.py index 78fe077e6b..8d43155a08 100644 --- a/api/core/app/apps/advanced_chat/workflow_event_trigger_callback.py +++ b/api/core/app/apps/advanced_chat/workflow_event_trigger_callback.py @@ -14,13 +14,13 @@ from core.app.entities.queue_entities import ( QueueWorkflowStartedEvent, QueueWorkflowSucceededEvent, ) -from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.base_node_data_entities import BaseNodeData from core.workflow.entities.node_entities import NodeType from models.workflow import Workflow -class WorkflowEventTriggerCallback(BaseWorkflowCallback): +class WorkflowEventTriggerCallback(WorkflowCallback): def __init__(self, queue_manager: AppQueueManager, workflow: Workflow): self._queue_manager = queue_manager diff --git a/api/core/app/apps/base_app_generate_response_converter.py b/api/core/app/apps/base_app_generate_response_converter.py index bacd1a5477..1165314a7f 100644 --- a/api/core/app/apps/base_app_generate_response_converter.py +++ b/api/core/app/apps/base_app_generate_response_converter.py @@ -1,7 +1,7 @@ import logging from abc import ABC, abstractmethod from collections.abc import Generator -from typing import Union +from typing import Any, Union from core.app.entities.app_invoke_entities import InvokeFrom from core.app.entities.task_entities import AppBlockingResponse, AppStreamResponse @@ -15,44 +15,41 @@ class AppGenerateResponseConverter(ABC): @classmethod def convert(cls, response: Union[ AppBlockingResponse, - Generator[AppStreamResponse, None, None] - ], invoke_from: InvokeFrom) -> Union[ - dict, - Generator[str, None, None] - ]: + Generator[AppStreamResponse, Any, None] + ], invoke_from: InvokeFrom): if invoke_from in [InvokeFrom.DEBUGGER, InvokeFrom.SERVICE_API]: - if isinstance(response, cls._blocking_response_type): + if isinstance(response, AppBlockingResponse): return cls.convert_blocking_full_response(response) else: - def _generate(): + def _generate_full_response() -> Generator[str, Any, None]: for chunk in cls.convert_stream_full_response(response): if chunk == 'ping': yield f'event: {chunk}\n\n' else: yield f'data: {chunk}\n\n' - return _generate() + return _generate_full_response() else: - if isinstance(response, cls._blocking_response_type): + if isinstance(response, AppBlockingResponse): return cls.convert_blocking_simple_response(response) else: - def _generate(): + def _generate_simple_response() -> Generator[str, Any, None]: for chunk in cls.convert_stream_simple_response(response): if chunk == 'ping': yield f'event: {chunk}\n\n' else: yield f'data: {chunk}\n\n' - return _generate() + return _generate_simple_response() @classmethod @abstractmethod - def convert_blocking_full_response(cls, blocking_response: AppBlockingResponse) -> dict: + def convert_blocking_full_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]: raise NotImplementedError @classmethod @abstractmethod - def convert_blocking_simple_response(cls, blocking_response: AppBlockingResponse) -> dict: + def convert_blocking_simple_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]: raise NotImplementedError @classmethod @@ -68,7 +65,7 @@ class AppGenerateResponseConverter(ABC): raise NotImplementedError @classmethod - def _get_simple_metadata(cls, metadata: dict) -> dict: + def _get_simple_metadata(cls, metadata: dict[str, Any]): """ Get simple metadata. :param metadata: metadata diff --git a/api/core/app/apps/base_app_queue_manager.py b/api/core/app/apps/base_app_queue_manager.py index dd2343d0b1..f929a979f1 100644 --- a/api/core/app/apps/base_app_queue_manager.py +++ b/api/core/app/apps/base_app_queue_manager.py @@ -5,9 +5,9 @@ from collections.abc import Generator from enum import Enum from typing import Any -from flask import current_app from sqlalchemy.orm import DeclarativeMeta +from configs import dify_config from core.app.entities.app_invoke_entities import InvokeFrom from core.app.entities.queue_entities import ( AppQueueEvent, @@ -48,7 +48,7 @@ class AppQueueManager: :return: """ # wait for APP_MAX_EXECUTION_TIME seconds to stop listen - listen_timeout = current_app.config.get("APP_MAX_EXECUTION_TIME") + listen_timeout = dify_config.APP_MAX_EXECUTION_TIME start_time = time.time() last_ping_time = 0 while True: diff --git a/api/core/app/apps/workflow/app_generator.py b/api/core/app/apps/workflow/app_generator.py index 0f547ca164..b1986dbcee 100644 --- a/api/core/app/apps/workflow/app_generator.py +++ b/api/core/app/apps/workflow/app_generator.py @@ -1,3 +1,4 @@ +import contextvars import logging import os import threading @@ -8,6 +9,7 @@ from typing import Union from flask import Flask, current_app from pydantic import ValidationError +import contexts from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.app.apps.base_app_generator import BaseAppGenerator from core.app.apps.base_app_queue_manager import AppQueueManager, GenerateTaskStoppedException, PublishFrom @@ -38,7 +40,7 @@ class WorkflowAppGenerator(BaseAppGenerator): invoke_from: InvokeFrom, stream: bool = True, call_depth: int = 0, - ) -> Union[dict, Generator[dict, None, None]]: + ): """ Generate App response. @@ -86,6 +88,7 @@ class WorkflowAppGenerator(BaseAppGenerator): call_depth=call_depth, trace_manager=trace_manager ) + contexts.tenant_id.set(application_generate_entity.app_config.tenant_id) return self._generate( app_model=app_model, @@ -126,7 +129,8 @@ class WorkflowAppGenerator(BaseAppGenerator): worker_thread = threading.Thread(target=self._generate_worker, kwargs={ 'flask_app': current_app._get_current_object(), 'application_generate_entity': application_generate_entity, - 'queue_manager': queue_manager + 'queue_manager': queue_manager, + 'context': contextvars.copy_context() }) worker_thread.start() @@ -150,8 +154,7 @@ class WorkflowAppGenerator(BaseAppGenerator): node_id: str, user: Account, args: dict, - stream: bool = True) \ - -> Union[dict, Generator[dict, None, None]]: + stream: bool = True): """ Generate App response. @@ -193,6 +196,7 @@ class WorkflowAppGenerator(BaseAppGenerator): inputs=args['inputs'] ) ) + contexts.tenant_id.set(application_generate_entity.app_config.tenant_id) return self._generate( app_model=app_model, @@ -205,7 +209,8 @@ class WorkflowAppGenerator(BaseAppGenerator): def _generate_worker(self, flask_app: Flask, application_generate_entity: WorkflowAppGenerateEntity, - queue_manager: AppQueueManager) -> None: + queue_manager: AppQueueManager, + context: contextvars.Context) -> None: """ Generate worker in a new thread. :param flask_app: Flask app @@ -213,6 +218,8 @@ class WorkflowAppGenerator(BaseAppGenerator): :param queue_manager: queue manager :return: """ + for var, val in context.items(): + var.set(val) with flask_app.app_context(): try: # workflow app diff --git a/api/core/app/apps/workflow/app_runner.py b/api/core/app/apps/workflow/app_runner.py index 050319e552..24f4a83217 100644 --- a/api/core/app/apps/workflow/app_runner.py +++ b/api/core/app/apps/workflow/app_runner.py @@ -10,6 +10,7 @@ from core.app.entities.app_invoke_entities import ( InvokeFrom, WorkflowAppGenerateEntity, ) +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.node_entities import SystemVariable from core.workflow.nodes.base_node import UserFrom from core.workflow.workflow_engine_manager import WorkflowEngineManager @@ -57,7 +58,7 @@ class WorkflowAppRunner: db.session.close() - workflow_callbacks = [WorkflowEventTriggerCallback( + workflow_callbacks: list[WorkflowCallback] = [WorkflowEventTriggerCallback( queue_manager=queue_manager, workflow=workflow )] diff --git a/api/core/app/apps/workflow/workflow_event_trigger_callback.py b/api/core/app/apps/workflow/workflow_event_trigger_callback.py index e423a40bcb..4472a7e9b5 100644 --- a/api/core/app/apps/workflow/workflow_event_trigger_callback.py +++ b/api/core/app/apps/workflow/workflow_event_trigger_callback.py @@ -14,13 +14,13 @@ from core.app.entities.queue_entities import ( QueueWorkflowStartedEvent, QueueWorkflowSucceededEvent, ) -from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.base_node_data_entities import BaseNodeData from core.workflow.entities.node_entities import NodeType from models.workflow import Workflow -class WorkflowEventTriggerCallback(BaseWorkflowCallback): +class WorkflowEventTriggerCallback(WorkflowCallback): def __init__(self, queue_manager: AppQueueManager, workflow: Workflow): self._queue_manager = queue_manager diff --git a/api/core/app/apps/workflow_logging_callback.py b/api/core/app/apps/workflow_logging_callback.py index f617c671e9..2e6431d6d0 100644 --- a/api/core/app/apps/workflow_logging_callback.py +++ b/api/core/app/apps/workflow_logging_callback.py @@ -2,7 +2,7 @@ from typing import Optional from core.app.entities.queue_entities import AppQueueEvent from core.model_runtime.utils.encoders import jsonable_encoder -from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.base_node_data_entities import BaseNodeData from core.workflow.entities.node_entities import NodeType @@ -15,7 +15,7 @@ _TEXT_COLOR_MAPPING = { } -class WorkflowLoggingCallback(BaseWorkflowCallback): +class WorkflowLoggingCallback(WorkflowCallback): def __init__(self) -> None: self.current_node_id = None diff --git a/api/core/app/entities/app_invoke_entities.py b/api/core/app/entities/app_invoke_entities.py index 1d2ad4a373..9a861c29e2 100644 --- a/api/core/app/entities/app_invoke_entities.py +++ b/api/core/app/entities/app_invoke_entities.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from enum import Enum from typing import Any, Optional @@ -76,7 +77,7 @@ class AppGenerateEntity(BaseModel): # app config app_config: AppConfig - inputs: dict[str, Any] + inputs: Mapping[str, Any] files: list[FileVar] = [] user_id: str @@ -140,7 +141,7 @@ class AdvancedChatAppGenerateEntity(AppGenerateEntity): app_config: WorkflowUIBasedAppConfig conversation_id: Optional[str] = None - query: Optional[str] = None + query: str class SingleIterationRunEntity(BaseModel): """ diff --git a/api/core/app/segments/__init__.py b/api/core/app/segments/__init__.py new file mode 100644 index 0000000000..d5cd0a589c --- /dev/null +++ b/api/core/app/segments/__init__.py @@ -0,0 +1,53 @@ +from .segment_group import SegmentGroup +from .segments import ( + ArrayAnySegment, + FileSegment, + FloatSegment, + IntegerSegment, + NoneSegment, + ObjectSegment, + Segment, + StringSegment, +) +from .types import SegmentType +from .variables import ( + ArrayAnyVariable, + ArrayFileVariable, + ArrayNumberVariable, + ArrayObjectVariable, + ArrayStringVariable, + FileVariable, + FloatVariable, + IntegerVariable, + NoneVariable, + ObjectVariable, + SecretVariable, + StringVariable, + Variable, +) + +__all__ = [ + 'IntegerVariable', + 'FloatVariable', + 'ObjectVariable', + 'SecretVariable', + 'FileVariable', + 'StringVariable', + 'ArrayAnyVariable', + 'Variable', + 'SegmentType', + 'SegmentGroup', + 'Segment', + 'NoneSegment', + 'NoneVariable', + 'IntegerSegment', + 'FloatSegment', + 'ObjectSegment', + 'ArrayAnySegment', + 'FileSegment', + 'StringSegment', + 'ArrayStringVariable', + 'ArrayNumberVariable', + 'ArrayObjectVariable', + 'ArrayFileVariable', +] diff --git a/api/core/app/segments/factory.py b/api/core/app/segments/factory.py new file mode 100644 index 0000000000..f62e44bf07 --- /dev/null +++ b/api/core/app/segments/factory.py @@ -0,0 +1,86 @@ +from collections.abc import Mapping +from typing import Any + +from core.file.file_obj import FileVar + +from .segments import ( + ArrayAnySegment, + FileSegment, + FloatSegment, + IntegerSegment, + NoneSegment, + ObjectSegment, + Segment, + StringSegment, +) +from .types import SegmentType +from .variables import ( + ArrayFileVariable, + ArrayNumberVariable, + ArrayObjectVariable, + ArrayStringVariable, + FileVariable, + FloatVariable, + IntegerVariable, + ObjectVariable, + SecretVariable, + StringVariable, + Variable, +) + + +def build_variable_from_mapping(m: Mapping[str, Any], /) -> Variable: + if (value_type := m.get('value_type')) is None: + raise ValueError('missing value type') + if not m.get('name'): + raise ValueError('missing name') + if (value := m.get('value')) is None: + raise ValueError('missing value') + match value_type: + case SegmentType.STRING: + return StringVariable.model_validate(m) + case SegmentType.SECRET: + return SecretVariable.model_validate(m) + case SegmentType.NUMBER if isinstance(value, int): + return IntegerVariable.model_validate(m) + case SegmentType.NUMBER if isinstance(value, float): + return FloatVariable.model_validate(m) + case SegmentType.NUMBER if not isinstance(value, float | int): + raise ValueError(f'invalid number value {value}') + case SegmentType.FILE: + return FileVariable.model_validate(m) + case SegmentType.OBJECT if isinstance(value, dict): + return ObjectVariable.model_validate( + {**m, 'value': {k: build_variable_from_mapping(v) for k, v in value.items()}} + ) + case SegmentType.ARRAY_STRING if isinstance(value, list): + return ArrayStringVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]}) + case SegmentType.ARRAY_NUMBER if isinstance(value, list): + return ArrayNumberVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]}) + case SegmentType.ARRAY_OBJECT if isinstance(value, list): + return ArrayObjectVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]}) + case SegmentType.ARRAY_FILE if isinstance(value, list): + return ArrayFileVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]}) + raise ValueError(f'not supported value type {value_type}') + + +def build_segment(value: Any, /) -> Segment: + if value is None: + return NoneSegment() + if isinstance(value, str): + return StringSegment(value=value) + if isinstance(value, int): + return IntegerSegment(value=value) + if isinstance(value, float): + return FloatSegment(value=value) + if isinstance(value, dict): + # TODO: Limit the depth of the object + obj = {k: build_segment(v) for k, v in value.items()} + return ObjectSegment(value=obj) + if isinstance(value, list): + # TODO: Limit the depth of the array + elements = [build_segment(v) for v in value] + return ArrayAnySegment(value=elements) + if isinstance(value, FileVar): + return FileSegment(value=value) + raise ValueError(f'not supported value {value}') diff --git a/api/core/app/segments/parser.py b/api/core/app/segments/parser.py new file mode 100644 index 0000000000..de6c796652 --- /dev/null +++ b/api/core/app/segments/parser.py @@ -0,0 +1,18 @@ +import re + +from core.workflow.entities.variable_pool import VariablePool + +from . import SegmentGroup, factory + +VARIABLE_PATTERN = re.compile(r'\{\{#([a-zA-Z0-9_]{1,50}(?:\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10})#\}\}') + + +def convert_template(*, template: str, variable_pool: VariablePool): + parts = re.split(VARIABLE_PATTERN, template) + segments = [] + for part in filter(lambda x: x, parts): + if '.' in part and (value := variable_pool.get(part.split('.'))): + segments.append(value) + else: + segments.append(factory.build_segment(part)) + return SegmentGroup(value=segments) diff --git a/api/core/app/segments/segment_group.py b/api/core/app/segments/segment_group.py new file mode 100644 index 0000000000..b4ff09b6d3 --- /dev/null +++ b/api/core/app/segments/segment_group.py @@ -0,0 +1,22 @@ +from .segments import Segment +from .types import SegmentType + + +class SegmentGroup(Segment): + value_type: SegmentType = SegmentType.GROUP + value: list[Segment] + + @property + def text(self): + return ''.join([segment.text for segment in self.value]) + + @property + def log(self): + return ''.join([segment.log for segment in self.value]) + + @property + def markdown(self): + return ''.join([segment.markdown for segment in self.value]) + + def to_object(self): + return [segment.to_object() for segment in self.value] diff --git a/api/core/app/segments/segments.py b/api/core/app/segments/segments.py new file mode 100644 index 0000000000..4227f154e6 --- /dev/null +++ b/api/core/app/segments/segments.py @@ -0,0 +1,140 @@ +import json +from collections.abc import Mapping, Sequence +from typing import Any + +from pydantic import BaseModel, ConfigDict, field_validator + +from core.file.file_obj import FileVar + +from .types import SegmentType + + +class Segment(BaseModel): + model_config = ConfigDict(frozen=True) + + value_type: SegmentType + value: Any + + @field_validator('value_type') + def validate_value_type(cls, value): + """ + This validator checks if the provided value is equal to the default value of the 'value_type' field. + If the value is different, a ValueError is raised. + """ + if value != cls.model_fields['value_type'].default: + raise ValueError("Cannot modify 'value_type'") + return value + + @property + def text(self) -> str: + return str(self.value) + + @property + def log(self) -> str: + return str(self.value) + + @property + def markdown(self) -> str: + return str(self.value) + + def to_object(self) -> Any: + return self.value + + +class NoneSegment(Segment): + value_type: SegmentType = SegmentType.NONE + value: None = None + + @property + def text(self) -> str: + return 'null' + + @property + def log(self) -> str: + return 'null' + + @property + def markdown(self) -> str: + return 'null' + + +class StringSegment(Segment): + value_type: SegmentType = SegmentType.STRING + value: str + + +class FloatSegment(Segment): + value_type: SegmentType = SegmentType.NUMBER + value: float + + +class IntegerSegment(Segment): + value_type: SegmentType = SegmentType.NUMBER + value: int + + +class FileSegment(Segment): + value_type: SegmentType = SegmentType.FILE + # TODO: embed FileVar in this model. + value: FileVar + + @property + def markdown(self) -> str: + return self.value.to_markdown() + + +class ObjectSegment(Segment): + value_type: SegmentType = SegmentType.OBJECT + value: Mapping[str, Segment] + + @property + def text(self) -> str: + # TODO: Process variables. + return json.dumps(self.model_dump()['value'], ensure_ascii=False) + + @property + def log(self) -> str: + # TODO: Process variables. + return json.dumps(self.model_dump()['value'], ensure_ascii=False, indent=2) + + @property + def markdown(self) -> str: + # TODO: Use markdown code block + return json.dumps(self.model_dump()['value'], ensure_ascii=False, indent=2) + + def to_object(self): + return {k: v.to_object() for k, v in self.value.items()} + + +class ArraySegment(Segment): + @property + def markdown(self) -> str: + return '\n'.join(['- ' + item.markdown for item in self.value]) + + def to_object(self): + return [v.to_object() for v in self.value] + + +class ArrayAnySegment(ArraySegment): + value_type: SegmentType = SegmentType.ARRAY_ANY + value: Sequence[Segment] + + +class ArrayStringSegment(ArraySegment): + value_type: SegmentType = SegmentType.ARRAY_STRING + value: Sequence[StringSegment] + + +class ArrayNumberSegment(ArraySegment): + value_type: SegmentType = SegmentType.ARRAY_NUMBER + value: Sequence[FloatSegment | IntegerSegment] + + +class ArrayObjectSegment(ArraySegment): + value_type: SegmentType = SegmentType.ARRAY_OBJECT + value: Sequence[ObjectSegment] + + +class ArrayFileSegment(ArraySegment): + value_type: SegmentType = SegmentType.ARRAY_FILE + value: Sequence[FileSegment] diff --git a/api/core/app/segments/types.py b/api/core/app/segments/types.py new file mode 100644 index 0000000000..a371058ef5 --- /dev/null +++ b/api/core/app/segments/types.py @@ -0,0 +1,17 @@ +from enum import Enum + + +class SegmentType(str, Enum): + NONE = 'none' + NUMBER = 'number' + STRING = 'string' + SECRET = 'secret' + ARRAY_ANY = 'array[any]' + ARRAY_STRING = 'array[string]' + ARRAY_NUMBER = 'array[number]' + ARRAY_OBJECT = 'array[object]' + ARRAY_FILE = 'array[file]' + OBJECT = 'object' + FILE = 'file' + + GROUP = 'group' diff --git a/api/core/app/segments/variables.py b/api/core/app/segments/variables.py new file mode 100644 index 0000000000..ac26e16542 --- /dev/null +++ b/api/core/app/segments/variables.py @@ -0,0 +1,85 @@ +from pydantic import Field + +from core.helper import encrypter + +from .segments import ( + ArrayAnySegment, + ArrayFileSegment, + ArrayNumberSegment, + ArrayObjectSegment, + ArrayStringSegment, + FileSegment, + FloatSegment, + IntegerSegment, + NoneSegment, + ObjectSegment, + Segment, + StringSegment, +) +from .types import SegmentType + + +class Variable(Segment): + """ + A variable is a segment that has a name. + """ + + id: str = Field( + default='', + description="Unique identity for variable. It's only used by environment variables now.", + ) + name: str + description: str = Field(default='', description='Description of the variable.') + + +class StringVariable(StringSegment, Variable): + pass + + +class FloatVariable(FloatSegment, Variable): + pass + + +class IntegerVariable(IntegerSegment, Variable): + pass + + +class FileVariable(FileSegment, Variable): + pass + + +class ObjectVariable(ObjectSegment, Variable): + pass + + +class ArrayAnyVariable(ArrayAnySegment, Variable): + pass + + +class ArrayStringVariable(ArrayStringSegment, Variable): + pass + + +class ArrayNumberVariable(ArrayNumberSegment, Variable): + pass + + +class ArrayObjectVariable(ArrayObjectSegment, Variable): + pass + + +class ArrayFileVariable(ArrayFileSegment, Variable): + pass + + +class SecretVariable(StringVariable): + value_type: SegmentType = SegmentType.SECRET + + @property + def log(self) -> str: + return encrypter.obfuscated_token(self.value) + + +class NoneVariable(NoneSegment, Variable): + value_type: SegmentType = SegmentType.NONE + value: None = None diff --git a/api/core/callback_handler/agent_tool_callback_handler.py b/api/core/callback_handler/agent_tool_callback_handler.py index f973b7e1ce..03f8244bab 100644 --- a/api/core/callback_handler/agent_tool_callback_handler.py +++ b/api/core/callback_handler/agent_tool_callback_handler.py @@ -1,9 +1,11 @@ import os +from collections.abc import Mapping, Sequence from typing import Any, Optional, TextIO, Union from pydantic import BaseModel from core.ops.ops_trace_manager import TraceQueueManager, TraceTask, TraceTaskName +from core.tools.entities.tool_entities import ToolInvokeMessage _TEXT_COLOR_MAPPING = { "blue": "36;1", @@ -43,7 +45,7 @@ class DifyAgentCallbackHandler(BaseModel): def on_tool_start( self, tool_name: str, - tool_inputs: dict[str, Any], + tool_inputs: Mapping[str, Any], ) -> None: """Do nothing.""" print_text("\n[on_tool_start] ToolCall:" + tool_name + "\n" + str(tool_inputs) + "\n", color=self.color) @@ -51,8 +53,8 @@ class DifyAgentCallbackHandler(BaseModel): def on_tool_end( self, tool_name: str, - tool_inputs: dict[str, Any], - tool_outputs: str, + tool_inputs: Mapping[str, Any], + tool_outputs: Sequence[ToolInvokeMessage], message_id: Optional[str] = None, timer: Optional[Any] = None, trace_manager: Optional[TraceQueueManager] = None diff --git a/api/core/file/message_file_parser.py b/api/core/file/message_file_parser.py index 842b539ad1..7b2f8217f9 100644 --- a/api/core/file/message_file_parser.py +++ b/api/core/file/message_file_parser.py @@ -1,4 +1,5 @@ -from typing import Union +from collections.abc import Mapping, Sequence +from typing import Any, Union import requests @@ -16,7 +17,7 @@ class MessageFileParser: self.tenant_id = tenant_id self.app_id = app_id - def validate_and_transform_files_arg(self, files: list[dict], file_extra_config: FileExtraConfig, + def validate_and_transform_files_arg(self, files: Sequence[Mapping[str, Any]], file_extra_config: FileExtraConfig, user: Union[Account, EndUser]) -> list[FileVar]: """ validate and transform files arg diff --git a/api/core/file/upload_file_parser.py b/api/core/file/upload_file_parser.py index 9e454f08d4..737a11e426 100644 --- a/api/core/file/upload_file_parser.py +++ b/api/core/file/upload_file_parser.py @@ -6,8 +6,7 @@ import os import time from typing import Optional -from flask import current_app - +from configs import dify_config from extensions.ext_storage import storage IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'webp', 'gif', 'svg'] @@ -23,7 +22,7 @@ class UploadFileParser: if upload_file.extension not in IMAGE_EXTENSIONS: return None - if current_app.config['MULTIMODAL_SEND_IMAGE_FORMAT'] == 'url' or force_url: + if dify_config.MULTIMODAL_SEND_IMAGE_FORMAT == 'url' or force_url: return cls.get_signed_temp_image_url(upload_file.id) else: # get image file base64 @@ -44,13 +43,13 @@ class UploadFileParser: :param upload_file: UploadFile object :return: """ - base_url = current_app.config.get('FILES_URL') + base_url = dify_config.FILES_URL image_preview_url = f'{base_url}/files/{upload_file_id}/image-preview' timestamp = str(int(time.time())) nonce = os.urandom(16).hex() data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}" - secret_key = current_app.config['SECRET_KEY'].encode() + secret_key = dify_config.SECRET_KEY.encode() sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest() encoded_sign = base64.urlsafe_b64encode(sign).decode() @@ -68,7 +67,7 @@ class UploadFileParser: :return: """ data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}" - secret_key = current_app.config['SECRET_KEY'].encode() + secret_key = dify_config.SECRET_KEY.encode() recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest() recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode() @@ -77,4 +76,4 @@ class UploadFileParser: return False current_time = int(time.time()) - return current_time - int(timestamp) <= current_app.config.get('FILES_ACCESS_TIMEOUT') + return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT diff --git a/api/core/helper/code_executor/code_executor.py b/api/core/helper/code_executor/code_executor.py index f094f7d79b..5b69d3af4b 100644 --- a/api/core/helper/code_executor/code_executor.py +++ b/api/core/helper/code_executor/code_executor.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) CODE_EXECUTION_ENDPOINT = dify_config.CODE_EXECUTION_ENDPOINT CODE_EXECUTION_API_KEY = dify_config.CODE_EXECUTION_API_KEY -CODE_EXECUTION_TIMEOUT= (10, 60) +CODE_EXECUTION_TIMEOUT = (10, 60) class CodeExecutionException(Exception): pass @@ -64,7 +64,7 @@ class CodeExecutor: @classmethod def execute_code(cls, - language: Literal['python3', 'javascript', 'jinja2'], + language: CodeLanguage, preload: str, code: str, dependencies: Optional[list[CodeDependency]] = None) -> str: @@ -119,7 +119,7 @@ class CodeExecutor: return response.data.stdout @classmethod - def execute_workflow_code_template(cls, language: Literal['python3', 'javascript', 'jinja2'], code: str, inputs: dict, dependencies: Optional[list[CodeDependency]] = None) -> dict: + def execute_workflow_code_template(cls, language: CodeLanguage, code: str, inputs: dict, dependencies: Optional[list[CodeDependency]] = None) -> dict: """ Execute code :param language: code language diff --git a/api/core/helper/encrypter.py b/api/core/helper/encrypter.py index fcf293dc1c..bf87a842c0 100644 --- a/api/core/helper/encrypter.py +++ b/api/core/helper/encrypter.py @@ -6,11 +6,16 @@ from models.account import Tenant def obfuscated_token(token: str): - return token[:6] + '*' * (len(token) - 8) + token[-2:] + if not token: + return token + if len(token) <= 8: + return '*' * 20 + return token[:6] + '*' * 12 + token[-2:] def encrypt_token(tenant_id: str, token: str): - tenant = db.session.query(Tenant).filter(Tenant.id == tenant_id).first() + if not (tenant := db.session.query(Tenant).filter(Tenant.id == tenant_id).first()): + raise ValueError(f'Tenant with id {tenant_id} not found') encrypted_token = rsa.encrypt(token, tenant.encrypt_public_key) return base64.b64encode(encrypted_token).decode() diff --git a/api/core/helper/position_helper.py b/api/core/helper/position_helper.py index e4ceeb652e..dd1534c791 100644 --- a/api/core/helper/position_helper.py +++ b/api/core/helper/position_helper.py @@ -13,15 +13,10 @@ def get_position_map(folder_path: str, *, file_name: str = "_position.yaml") -> :param file_name: the YAML file name, default to '_position.yaml' :return: a dict with name as key and index as value """ - position_file_name = os.path.join(folder_path, file_name) - positions = load_yaml_file(position_file_name, ignore_error=True) - position_map = {} - index = 0 - for _, name in enumerate(positions): - if name and isinstance(name, str): - position_map[name.strip()] = index - index += 1 - return position_map + position_file_path = os.path.join(folder_path, file_name) + yaml_content = load_yaml_file(file_path=position_file_path, default_value=[]) + positions = [item.strip() for item in yaml_content if item and isinstance(item, str) and item.strip()] + return {name: index for index, name in enumerate(positions)} def sort_by_position_map( diff --git a/api/core/helper/ssrf_proxy.py b/api/core/helper/ssrf_proxy.py index 019b27f28a..14ca8e943c 100644 --- a/api/core/helper/ssrf_proxy.py +++ b/api/core/helper/ssrf_proxy.py @@ -1,48 +1,75 @@ """ Proxy requests to avoid SSRF """ +import logging import os +import time import httpx SSRF_PROXY_ALL_URL = os.getenv('SSRF_PROXY_ALL_URL', '') SSRF_PROXY_HTTP_URL = os.getenv('SSRF_PROXY_HTTP_URL', '') SSRF_PROXY_HTTPS_URL = os.getenv('SSRF_PROXY_HTTPS_URL', '') +SSRF_DEFAULT_MAX_RETRIES = int(os.getenv('SSRF_DEFAULT_MAX_RETRIES', '3')) proxies = { 'http://': SSRF_PROXY_HTTP_URL, 'https://': SSRF_PROXY_HTTPS_URL } if SSRF_PROXY_HTTP_URL and SSRF_PROXY_HTTPS_URL else None +BACKOFF_FACTOR = 0.5 +STATUS_FORCELIST = [429, 500, 502, 503, 504] -def make_request(method, url, **kwargs): - if SSRF_PROXY_ALL_URL: - return httpx.request(method=method, url=url, proxy=SSRF_PROXY_ALL_URL, **kwargs) - elif proxies: - return httpx.request(method=method, url=url, proxies=proxies, **kwargs) - else: - return httpx.request(method=method, url=url, **kwargs) +def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs): + if "allow_redirects" in kwargs: + allow_redirects = kwargs.pop("allow_redirects") + if "follow_redirects" not in kwargs: + kwargs["follow_redirects"] = allow_redirects + + retries = 0 + while retries <= max_retries: + try: + if SSRF_PROXY_ALL_URL: + response = httpx.request(method=method, url=url, proxy=SSRF_PROXY_ALL_URL, **kwargs) + elif proxies: + response = httpx.request(method=method, url=url, proxies=proxies, **kwargs) + else: + response = httpx.request(method=method, url=url, **kwargs) + + if response.status_code not in STATUS_FORCELIST: + return response + else: + logging.warning(f"Received status code {response.status_code} for URL {url} which is in the force list") + + except httpx.RequestError as e: + logging.warning(f"Request to URL {url} failed on attempt {retries + 1}: {e}") + + retries += 1 + if retries <= max_retries: + time.sleep(BACKOFF_FACTOR * (2 ** (retries - 1))) + + raise Exception(f"Reached maximum retries ({max_retries}) for URL {url}") -def get(url, **kwargs): - return make_request('GET', url, **kwargs) +def get(url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs): + return make_request('GET', url, max_retries=max_retries, **kwargs) -def post(url, **kwargs): - return make_request('POST', url, **kwargs) +def post(url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs): + return make_request('POST', url, max_retries=max_retries, **kwargs) -def put(url, **kwargs): - return make_request('PUT', url, **kwargs) +def put(url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs): + return make_request('PUT', url, max_retries=max_retries, **kwargs) -def patch(url, **kwargs): - return make_request('PATCH', url, **kwargs) +def patch(url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs): + return make_request('PATCH', url, max_retries=max_retries, **kwargs) -def delete(url, **kwargs): - return make_request('DELETE', url, **kwargs) +def delete(url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs): + return make_request('DELETE', url, max_retries=max_retries, **kwargs) -def head(url, **kwargs): - return make_request('HEAD', url, **kwargs) +def head(url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs): + return make_request('HEAD', url, max_retries=max_retries, **kwargs) diff --git a/api/core/llm_generator/llm_generator.py b/api/core/llm_generator/llm_generator.py index 70d3befbbd..0b5029460a 100644 --- a/api/core/llm_generator/llm_generator.py +++ b/api/core/llm_generator/llm_generator.py @@ -3,10 +3,13 @@ import logging import re from typing import Optional -from core.llm_generator.output_parser.errors import OutputParserException from core.llm_generator.output_parser.rule_config_generator import RuleConfigGeneratorOutputParser from core.llm_generator.output_parser.suggested_questions_after_answer import SuggestedQuestionsAfterAnswerOutputParser -from core.llm_generator.prompts import CONVERSATION_TITLE_PROMPT, GENERATOR_QA_PROMPT +from core.llm_generator.prompts import ( + CONVERSATION_TITLE_PROMPT, + GENERATOR_QA_PROMPT, + WORKFLOW_RULE_CONFIG_PROMPT_GENERATE_TEMPLATE, +) from core.model_manager import ModelManager from core.model_runtime.entities.message_entities import SystemPromptMessage, UserPromptMessage from core.model_runtime.entities.model_entities import ModelType @@ -115,55 +118,158 @@ class LLMGenerator: return questions @classmethod - def generate_rule_config(cls, tenant_id: str, audiences: str, hoping_to_solve: str) -> dict: + def generate_rule_config(cls, tenant_id: str, instruction: str, model_config: dict, no_variable: bool, rule_config_max_tokens: int = 512) -> dict: output_parser = RuleConfigGeneratorOutputParser() + error = "" + error_step = "" + rule_config = { + "prompt": "", + "variables": [], + "opening_statement": "", + "error": "" + } + model_parameters = { + "max_tokens": rule_config_max_tokens, + "temperature": 0.01 + } + + if no_variable: + prompt_template = PromptTemplateParser( + WORKFLOW_RULE_CONFIG_PROMPT_GENERATE_TEMPLATE + ) + + prompt_generate = prompt_template.format( + inputs={ + "TASK_DESCRIPTION": instruction, + }, + remove_template_variables=False + ) + + prompt_messages = [UserPromptMessage(content=prompt_generate)] + + model_manager = ModelManager() + + model_instance = model_manager.get_default_model_instance( + tenant_id=tenant_id, + model_type=ModelType.LLM, + ) + + try: + response = model_instance.invoke_llm( + prompt_messages=prompt_messages, + model_parameters=model_parameters, + stream=False + ) + + rule_config["prompt"] = response.message.content + + except InvokeError as e: + error = str(e) + error_step = "generate rule config" + except Exception as e: + logging.exception(e) + rule_config["error"] = str(e) + + rule_config["error"] = f"Failed to {error_step}. Error: {error}" if error else "" + + return rule_config + + # get rule config prompt, parameter and statement + prompt_generate, parameter_generate, statement_generate = output_parser.get_format_instructions() + prompt_template = PromptTemplateParser( - template=output_parser.get_format_instructions() + prompt_generate ) - prompt = prompt_template.format( + parameter_template = PromptTemplateParser( + parameter_generate + ) + + statement_template = PromptTemplateParser( + statement_generate + ) + + # format the prompt_generate_prompt + prompt_generate_prompt = prompt_template.format( inputs={ - "audiences": audiences, - "hoping_to_solve": hoping_to_solve, - "variable": "{{variable}}", - "lanA": "{{lanA}}", - "lanB": "{{lanB}}", - "topic": "{{topic}}" + "TASK_DESCRIPTION": instruction, }, remove_template_variables=False ) + prompt_messages = [UserPromptMessage(content=prompt_generate_prompt)] + # get model instance model_manager = ModelManager() - model_instance = model_manager.get_default_model_instance( + model_instance = model_manager.get_model_instance( tenant_id=tenant_id, model_type=ModelType.LLM, + provider=model_config.get("provider") if model_config else None, + model=model_config.get("name") if model_config else None, ) - prompt_messages = [UserPromptMessage(content=prompt)] - try: - response = model_instance.invoke_llm( - prompt_messages=prompt_messages, - model_parameters={ - "max_tokens": 512, - "temperature": 0 - }, - stream=False - ) + try: + # the first step to generate the task prompt + prompt_content = model_instance.invoke_llm( + prompt_messages=prompt_messages, + model_parameters=model_parameters, + stream=False + ) + except InvokeError as e: + error = str(e) + error_step = "generate prefix prompt" + rule_config["error"] = f"Failed to {error_step}. Error: {error}" if error else "" + + return rule_config + + rule_config["prompt"] = prompt_content.message.content + + parameter_generate_prompt = parameter_template.format( + inputs={ + "INPUT_TEXT": prompt_content.message.content, + }, + remove_template_variables=False + ) + parameter_messages = [UserPromptMessage(content=parameter_generate_prompt)] + + # the second step to generate the task_parameter and task_statement + statement_generate_prompt = statement_template.format( + inputs={ + "TASK_DESCRIPTION": instruction, + "INPUT_TEXT": prompt_content.message.content, + }, + remove_template_variables=False + ) + statement_messages = [UserPromptMessage(content=statement_generate_prompt)] + + try: + parameter_content = model_instance.invoke_llm( + prompt_messages=parameter_messages, + model_parameters=model_parameters, + stream=False + ) + rule_config["variables"] = re.findall(r'"\s*([^"]+)\s*"', parameter_content.message.content) + except InvokeError as e: + error = str(e) + error_step = "generate variables" + + try: + statement_content = model_instance.invoke_llm( + prompt_messages=statement_messages, + model_parameters=model_parameters, + stream=False + ) + rule_config["opening_statement"] = statement_content.message.content + except InvokeError as e: + error = str(e) + error_step = "generate conversation opener" - rule_config = output_parser.parse(response.message.content) - except InvokeError as e: - raise e - except OutputParserException: - raise ValueError('Please give a valid input for intended audience or hoping to solve problems.') except Exception as e: logging.exception(e) - rule_config = { - "prompt": "", - "variables": [], - "opening_statement": "" - } + rule_config["error"] = str(e) + + rule_config["error"] = f"Failed to {error_step}. Error: {error}" if error else "" return rule_config diff --git a/api/core/llm_generator/output_parser/rule_config_generator.py b/api/core/llm_generator/output_parser/rule_config_generator.py index f6d4bcf11a..8856f0c685 100644 --- a/api/core/llm_generator/output_parser/rule_config_generator.py +++ b/api/core/llm_generator/output_parser/rule_config_generator.py @@ -1,14 +1,18 @@ from typing import Any from core.llm_generator.output_parser.errors import OutputParserException -from core.llm_generator.prompts import RULE_CONFIG_GENERATE_TEMPLATE +from core.llm_generator.prompts import ( + RULE_CONFIG_PARAMETER_GENERATE_TEMPLATE, + RULE_CONFIG_PROMPT_GENERATE_TEMPLATE, + RULE_CONFIG_STATEMENT_GENERATE_TEMPLATE, +) from libs.json_in_md_parser import parse_and_check_json_markdown class RuleConfigGeneratorOutputParser: - def get_format_instructions(self) -> str: - return RULE_CONFIG_GENERATE_TEMPLATE + def get_format_instructions(self) -> tuple[str, str, str]: + return RULE_CONFIG_PROMPT_GENERATE_TEMPLATE, RULE_CONFIG_PARAMETER_GENERATE_TEMPLATE, RULE_CONFIG_STATEMENT_GENERATE_TEMPLATE def parse(self, text: str) -> Any: try: diff --git a/api/core/llm_generator/prompts.py b/api/core/llm_generator/prompts.py index 170a28432b..87361b385a 100644 --- a/api/core/llm_generator/prompts.py +++ b/api/core/llm_generator/prompts.py @@ -64,6 +64,7 @@ User Input: SUGGESTED_QUESTIONS_AFTER_ANSWER_INSTRUCTION_PROMPT = ( "Please help me predict the three most likely questions that human would ask, " "and keeping each question under 20 characters.\n" + "MAKE SURE your output is the SAME language as the Assistant's latest response(if the main response is written in Chinese, then the language of your output must be using Chinese.)!\n" "The output must be an array in JSON format following the specified schema:\n" "[\"question1\",\"question2\",\"question3\"]\n" ) @@ -80,65 +81,73 @@ GENERATOR_QA_PROMPT = ( '' ) -RULE_CONFIG_GENERATE_TEMPLATE = """Given MY INTENDED AUDIENCES and HOPING TO SOLVE using a language model, please select \ -the model prompt that best suits the input. -You will be provided with the prompt, variables, and an opening statement. -Only the content enclosed in double curly braces, such as {{variable}}, in the prompt can be considered as a variable; \ -otherwise, it cannot exist as a variable in the variables. -If you believe revising the original input will result in a better response from the language model, you may \ -suggest revisions. +WORKFLOW_RULE_CONFIG_PROMPT_GENERATE_TEMPLATE = """ +Here is a task description for which I would like you to create a high-quality prompt template for: + +{{TASK_DESCRIPTION}} + +Based on task description, please create a well-structured prompt template that another AI could use to consistently complete the task. The prompt template should include: +- Do not inlcude or section and variables in the prompt, assume user will add them at their own will. +- Clear instructions for the AI that will be using this prompt, demarcated with tags. The instructions should provide step-by-step directions on how to complete the task using the input variables. Also Specifies in the instructions that the output should not contain any xml tag. +- Relevant examples if needed to clarify the task further, demarcated with tags. Do not include variables in the prompt. Give three pairs of input and output examples. +- Include other relevant sections demarcated with appropriate XML tags like , . +- Use the same language as task description. +- Output in ``` xml ``` and start with +Please generate the full prompt template with at least 300 words and output only the prompt template. +""" -<> -Integrate the intended audience in the prompt e.g. the audience is an expert in the field. -Break down complex tasks into a sequence of simpler prompts in an interactive conversation. -Implement example-driven prompting (Use few-shot prompting). -When formatting your prompt start with Instruction followed by either Example if relevant. \ -Subsequently present your content. Use one or more line breaks to separate instructions examples questions context and input data. -Incorporate the following phrases: “Your task is” and “You MUST”. -Incorporate the following phrases: “You will be penalized”. -Use leading words like writing “think step by step”. -Add to your prompt the following phrase “Ensure that your answer is unbiased and does not rely on stereotypes”. -Assign a role to the large language models. -Use Delimiters. -To write an essay /text /paragraph /article or any type of text that should be detailed: “Write a detailed [essay/text/paragraph] for me on [topic] in detail by adding all the information necessary”. -Clearly state the requirements that the model must follow in order to produce content in the form of the keywords regulations hint or instructions +RULE_CONFIG_PROMPT_GENERATE_TEMPLATE = """ +Here is a task description for which I would like you to create a high-quality prompt template for: + +{{TASK_DESCRIPTION}} + +Based on task description, please create a well-structured prompt template that another AI could use to consistently complete the task. The prompt template should include: +- Descriptive variable names surrounded by {{ }} (two curly brackets) to indicate where the actual values will be substituted in. Choose variable names that clearly indicate the type of value expected. Variable names have to be composed of number, english alphabets and underline and nothing else. +- Clear instructions for the AI that will be using this prompt, demarcated with tags. The instructions should provide step-by-step directions on how to complete the task using the input variables. Also Specifies in the instructions that the output should not contain any xml tag. +- Relevant examples if needed to clarify the task further, demarcated with tags. Do not use curly brackets any other than in section. +- Any other relevant sections demarcated with appropriate XML tags like , , etc. +- Use the same language as task description. +- Output in ``` xml ``` and start with +Please generate the full prompt template and output only the prompt template. +""" -<< FORMATTING >> -Return a markdown code snippet with a JSON object formatted to look like, \ -no any other string out of markdown code snippet: -```json -{{{{ - "prompt": string \\ generated prompt - "variables": list of string \\ variables - "opening_statement": string \\ an opening statement to guide users on how to ask questions with generated prompt \ -and fill in variables, with a welcome sentence, and keep TLDR. -}}}} -``` +RULE_CONFIG_PARAMETER_GENERATE_TEMPLATE = """ +I need to extract the following information from the input text. The tag specifies the 'type', 'description' and 'required' of the information to be extracted. + +variables name bounded two double curly brackets. Variable name has to be composed of number, english alphabets and underline and nothing else. + -<< EXAMPLES >> -[EXAMPLE A] -```json -{ - "prompt": "I need your help to translate the following {{Input_language}}paper paragraph into {{Target_language}}, in a style similar to a popular science magazine in {{Target_language}}. #### Rules Ensure accurate conveyance of the original text's facts and context during translation. Maintain the original paragraph format and retain technical terms and company abbreviations ", - "variables": ["Input_language", "Target_language"], - "opening_statement": " Hi. I am your translation assistant. I can help you with any translation and ensure accurate conveyance of information. " -} -``` +Step 1: Carefully read the input and understand the structure of the expected output. +Step 2: Extract relevant parameters from the provided text based on the name and description of object. +Step 3: Structure the extracted parameters to JSON object as specified in . +Step 4: Ensure that the list of variable_names is properly formatted and valid. The output should not contain any XML tags. Output an empty list if there is no valid variable name in input text. -[EXAMPLE B] -```json -{ - "prompt": "Your task is to review the provided meeting notes and create a concise summary that captures the essential information, focusing on key takeaways and action items assigned to specific individuals or departments during the meeting. Use clear and professional language, and organize the summary in a logical manner using appropriate formatting such as headings, subheadings, and bullet points. Ensure that the summary is easy to understand and provides a comprehensive but succinct overview of the meeting's content, with a particular focus on clearly indicating who is responsible for each action item.", - "variables": ["meeting_notes"], - "opening_statement": "Hi! I'm your meeting notes summarizer AI. I can help you with any meeting notes and ensure accurate conveyance of information." -} -``` +### Structure +Here is the structure of the expected output, I should always follow the output structure. +["variable_name_1", "variable_name_2"] -<< MY INTENDED AUDIENCES >> -{{audiences}} +### Input Text +Inside XML tags, there is a text that I should extract parameters and convert to a JSON object. + +{{INPUT_TEXT}} + -<< HOPING TO SOLVE >> -{{hoping_to_solve}} +### Answer +I should always output a valid list. Output nothing other than the list of variable_name. Output an empty list if there is no variable name in input text. +""" -<< OUTPUT >> -""" \ No newline at end of file +RULE_CONFIG_STATEMENT_GENERATE_TEMPLATE = """ + +Step 1: Identify the purpose of the chatbot from the variable {{TASK_DESCRIPTION}} and infer chatbot's tone (e.g., friendly, professional, etc.) to add personality traits. +Step 2: Create a coherent and engaging opening statement. +Step 3: Ensure the output is welcoming and clearly explains what the chatbot is designed to do. Do not include any XML tags in the output. +Please use the same language as the user's input language. If user uses chinese then generate opening statement in chinese, if user uses english then generate opening statement in english. +Example Input: +Provide customer support for an e-commerce website +Example Output: +Welcome! I'm here to assist you with any questions or issues you might have with your shopping experience. Whether you're looking for product information, need help with your order, or have any other inquiries, feel free to ask. I'm friendly, helpful, and ready to support you in any way I can. + +Here is the task description: {{INPUT_TEXT}} + +You just need to generate the output +""" diff --git a/api/core/memory/token_buffer_memory.py b/api/core/memory/token_buffer_memory.py index 21f1965e93..b33d4dd7cb 100644 --- a/api/core/memory/token_buffer_memory.py +++ b/api/core/memory/token_buffer_memory.py @@ -103,7 +103,7 @@ class TokenBufferMemory: if curr_message_tokens > max_token_limit: pruned_memory = [] - while curr_message_tokens > max_token_limit and prompt_messages: + while curr_message_tokens > max_token_limit and len(prompt_messages)>1: pruned_memory.append(prompt_messages.pop(0)) curr_message_tokens = self.model_instance.get_llm_num_tokens( prompt_messages diff --git a/api/core/model_manager.py b/api/core/model_manager.py index d64db890f9..8e99ad3dec 100644 --- a/api/core/model_manager.py +++ b/api/core/model_manager.py @@ -410,7 +410,7 @@ class LBModelManager: self._model = model self._load_balancing_configs = load_balancing_configs - for load_balancing_config in self._load_balancing_configs: + for load_balancing_config in self._load_balancing_configs[:]: # Iterate over a shallow copy of the list if load_balancing_config.name == "__inherit__": if not managed_credentials: # remove __inherit__ if managed credentials is not provided diff --git a/api/core/model_runtime/docs/en_US/schema.md b/api/core/model_runtime/docs/en_US/schema.md index 2e55d05b0f..67f4e0879d 100644 --- a/api/core/model_runtime/docs/en_US/schema.md +++ b/api/core/model_runtime/docs/en_US/schema.md @@ -86,6 +86,9 @@ - `agent-thought` Agent reasoning, generally over 70B with thought chain capability. - `vision` Vision, i.e., image understanding. +- `tool-call` +- `multi-tool-call` +- `stream-tool-call` ### FetchFrom diff --git a/api/core/model_runtime/docs/zh_Hans/schema.md b/api/core/model_runtime/docs/zh_Hans/schema.md index f40a3f8698..681f49c435 100644 --- a/api/core/model_runtime/docs/zh_Hans/schema.md +++ b/api/core/model_runtime/docs/zh_Hans/schema.md @@ -87,6 +87,9 @@ - `agent-thought` Agent 推理，一般超过 70B 有思维链能力。 - `vision` 视觉，即：图像理解。 +- `tool-call` 工具调用 +- `multi-tool-call` 多工具调用 +- `stream-tool-call` 流式工具调用 ### FetchFrom diff --git a/api/core/model_runtime/model_providers/__base/ai_model.py b/api/core/model_runtime/model_providers/__base/ai_model.py index 04b539433c..0de216bf89 100644 --- a/api/core/model_runtime/model_providers/__base/ai_model.py +++ b/api/core/model_runtime/model_providers/__base/ai_model.py @@ -162,7 +162,7 @@ class AIModel(ABC): # traverse all model_schema_yaml_paths for model_schema_yaml_path in model_schema_yaml_paths: # read yaml data from yaml file - yaml_data = load_yaml_file(model_schema_yaml_path, ignore_error=True) + yaml_data = load_yaml_file(model_schema_yaml_path) new_parameter_rules = [] for parameter_rule in yaml_data.get('parameter_rules', []): diff --git a/api/core/model_runtime/model_providers/__base/model_provider.py b/api/core/model_runtime/model_providers/__base/model_provider.py index 51dd3b7e28..780460a3f7 100644 --- a/api/core/model_runtime/model_providers/__base/model_provider.py +++ b/api/core/model_runtime/model_providers/__base/model_provider.py @@ -44,7 +44,7 @@ class ModelProvider(ABC): # read provider schema from yaml file yaml_path = os.path.join(current_path, f'{provider_name}.yaml') - yaml_data = load_yaml_file(yaml_path, ignore_error=True) + yaml_data = load_yaml_file(yaml_path) try: # yaml_data to entity diff --git a/api/core/model_runtime/model_providers/_position.yaml b/api/core/model_runtime/model_providers/_position.yaml index cf4ac10828..c2fa0e5a6e 100644 --- a/api/core/model_runtime/model_providers/_position.yaml +++ b/api/core/model_runtime/model_providers/_position.yaml @@ -23,6 +23,7 @@ - tongyi - wenxin - moonshot +- tencent - jina - chatglm - yi diff --git a/api/core/model_runtime/model_providers/anthropic/llm/claude-3-5-sonnet-20240620.yaml b/api/core/model_runtime/model_providers/anthropic/llm/claude-3-5-sonnet-20240620.yaml index 72d4d8545b..e02c5517fe 100644 --- a/api/core/model_runtime/model_providers/anthropic/llm/claude-3-5-sonnet-20240620.yaml +++ b/api/core/model_runtime/model_providers/anthropic/llm/claude-3-5-sonnet-20240620.yaml @@ -27,9 +27,9 @@ parameter_rules: - name: max_tokens use_template: max_tokens required: true - default: 4096 + default: 8192 min: 1 - max: 4096 + max: 8192 - name: response_format use_template: response_format pricing: diff --git a/api/core/model_runtime/model_providers/anthropic/llm/llm.py b/api/core/model_runtime/model_providers/anthropic/llm/llm.py index fbc0b722b1..107efe4867 100644 --- a/api/core/model_runtime/model_providers/anthropic/llm/llm.py +++ b/api/core/model_runtime/model_providers/anthropic/llm/llm.py @@ -113,6 +113,11 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): if system: extra_model_kwargs['system'] = system + # Add the new header for claude-3-5-sonnet-20240620 model + extra_headers = {} + if model == "claude-3-5-sonnet-20240620": + extra_headers["anthropic-beta"] = "max-tokens-3-5-sonnet-2024-07-15" + if tools: extra_model_kwargs['tools'] = [ self._transform_tool_prompt(tool) for tool in tools @@ -121,6 +126,7 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): model=model, messages=prompt_message_dicts, stream=stream, + extra_headers=extra_headers, **model_parameters, **extra_model_kwargs ) @@ -130,6 +136,7 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): model=model, messages=prompt_message_dicts, stream=stream, + extra_headers=extra_headers, **model_parameters, **extra_model_kwargs ) @@ -138,7 +145,7 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): return self._handle_chat_generate_stream_response(model, credentials, response, prompt_messages) return self._handle_chat_generate_response(model, credentials, response, prompt_messages) - + def _code_block_mode_wrapper(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], model_parameters: dict, tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, stream: bool = True, user: Optional[str] = None, diff --git a/api/core/model_runtime/model_providers/azure_openai/azure_openai.yaml b/api/core/model_runtime/model_providers/azure_openai/azure_openai.yaml index 7e57c3ed2e..875e94167d 100644 --- a/api/core/model_runtime/model_providers/azure_openai/azure_openai.yaml +++ b/api/core/model_runtime/model_providers/azure_openai/azure_openai.yaml @@ -71,6 +71,9 @@ model_credential_schema: - label: en_US: '2024-02-01' value: '2024-02-01' + - label: + en_US: '2024-06-01' + value: '2024-06-01' placeholder: zh_Hans: 在此选择您的 API 版本 en_US: Select your API Version here diff --git a/api/core/model_runtime/model_providers/azure_openai/llm/llm.py b/api/core/model_runtime/model_providers/azure_openai/llm/llm.py index 25bc94cde6..1911caa952 100644 --- a/api/core/model_runtime/model_providers/azure_openai/llm/llm.py +++ b/api/core/model_runtime/model_providers/azure_openai/llm/llm.py @@ -375,6 +375,10 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): continue delta = chunk.choices[0] + # NOTE: For fix https://github.com/langgenius/dify/issues/5790 + if delta.delta is None: + continue + # extract tool calls from response self._update_tool_calls(tool_calls=tool_calls, tool_calls_response=delta.delta.tool_calls) @@ -501,7 +505,7 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): sub_messages.append(sub_message_dict) message_dict = {"role": "user", "content": sub_messages} elif isinstance(message, AssistantPromptMessage): - message = cast(AssistantPromptMessage, message) + # message = cast(AssistantPromptMessage, message) message_dict = {"role": "assistant", "content": message.content} if message.tool_calls: message_dict["tool_calls"] = [helper.dump_model(tool_call) for tool_call in message.tool_calls] diff --git a/api/core/model_runtime/model_providers/bedrock/bedrock.yaml b/api/core/model_runtime/model_providers/bedrock/bedrock.yaml index aa364fb63f..c540ee23b3 100644 --- a/api/core/model_runtime/model_providers/bedrock/bedrock.yaml +++ b/api/core/model_runtime/model_providers/bedrock/bedrock.yaml @@ -66,6 +66,10 @@ provider_credential_schema: label: en_US: Europe (Frankfurt) zh_Hans: 欧洲 (法兰克福) + - value: eu-west-2 + label: + en_US: Eu west London (London) + zh_Hans: 欧洲西部 (伦敦) - value: us-gov-west-1 label: en_US: AWS GovCloud (US-West) diff --git a/api/core/model_runtime/model_providers/bedrock/llm/_position.yaml b/api/core/model_runtime/model_providers/bedrock/llm/_position.yaml index 3a79a929ba..c523596b57 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/_position.yaml +++ b/api/core/model_runtime/model_providers/bedrock/llm/_position.yaml @@ -10,10 +10,13 @@ - cohere.command-text-v14 - cohere.command-r-plus-v1.0 - cohere.command-r-v1.0 +- meta.llama3-1-8b-instruct-v1:0 +- meta.llama3-1-70b-instruct-v1:0 - meta.llama3-8b-instruct-v1:0 - meta.llama3-70b-instruct-v1:0 - meta.llama2-13b-chat-v1 - meta.llama2-70b-chat-v1 +- mistral.mistral-large-2407-v1:0 - mistral.mistral-small-2402-v1:0 - mistral.mistral-large-2402-v1:0 - mistral.mixtral-8x7b-instruct-v0:1 diff --git a/api/core/model_runtime/model_providers/bedrock/llm/llm.py b/api/core/model_runtime/model_providers/bedrock/llm/llm.py index efb8c395fa..e9906c8294 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/llm.py +++ b/api/core/model_runtime/model_providers/bedrock/llm/llm.py @@ -48,6 +48,28 @@ logger = logging.getLogger(__name__) class BedrockLargeLanguageModel(LargeLanguageModel): + # please refer to the documentation: https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference.html + # TODO There is invoke issue: context limit on Cohere Model, will add them after fixed. + CONVERSE_API_ENABLED_MODEL_INFO=[ + {'prefix': 'anthropic.claude-v2', 'support_system_prompts': True, 'support_tool_use': False}, + {'prefix': 'anthropic.claude-v1', 'support_system_prompts': True, 'support_tool_use': False}, + {'prefix': 'anthropic.claude-3', 'support_system_prompts': True, 'support_tool_use': True}, + {'prefix': 'meta.llama', 'support_system_prompts': True, 'support_tool_use': False}, + {'prefix': 'mistral.mistral-7b-instruct', 'support_system_prompts': False, 'support_tool_use': False}, + {'prefix': 'mistral.mixtral-8x7b-instruct', 'support_system_prompts': False, 'support_tool_use': False}, + {'prefix': 'mistral.mistral-large', 'support_system_prompts': True, 'support_tool_use': True}, + {'prefix': 'mistral.mistral-small', 'support_system_prompts': True, 'support_tool_use': True}, + {'prefix': 'amazon.titan', 'support_system_prompts': False, 'support_tool_use': False} + ] + + @staticmethod + def _find_model_info(model_id): + for model in BedrockLargeLanguageModel.CONVERSE_API_ENABLED_MODEL_INFO: + if model_id.startswith(model['prefix']): + return model + logger.info(f"current model id: {model_id} did not support by Converse API") + return None + def _invoke(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], model_parameters: dict, tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, @@ -66,10 +88,12 @@ class BedrockLargeLanguageModel(LargeLanguageModel): :param user: unique user id :return: full response or stream response chunk generator result """ - # TODO: consolidate different invocation methods for models based on base model capabilities - # invoke anthropic models via boto3 client - if "anthropic" in model: - return self._generate_anthropic(model, credentials, prompt_messages, model_parameters, stop, stream, user, tools) + + model_info= BedrockLargeLanguageModel._find_model_info(model) + if model_info: + model_info['model'] = model + # invoke models via boto3 converse API + return self._generate_with_converse(model_info, credentials, prompt_messages, model_parameters, stop, stream, user, tools) # invoke Cohere models via boto3 client if "cohere.command-r" in model: return self._generate_cohere_chat(model, credentials, prompt_messages, model_parameters, stop, stream, user, tools) @@ -151,12 +175,12 @@ class BedrockLargeLanguageModel(LargeLanguageModel): return self._handle_generate_response(model, credentials, response, prompt_messages) - def _generate_anthropic(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], model_parameters: dict, + def _generate_with_converse(self, model_info: dict, credentials: dict, prompt_messages: list[PromptMessage], model_parameters: dict, stop: Optional[list[str]] = None, stream: bool = True, user: Optional[str] = None, tools: Optional[list[PromptMessageTool]] = None,) -> Union[LLMResult, Generator]: """ - Invoke Anthropic large language model + Invoke large language model with converse API - :param model: model name + :param model_info: model information :param credentials: model credentials :param prompt_messages: prompt messages :param model_parameters: model parameters @@ -173,25 +197,36 @@ class BedrockLargeLanguageModel(LargeLanguageModel): inference_config, additional_model_fields = self._convert_converse_api_model_parameters(model_parameters, stop) parameters = { - 'modelId': model, + 'modelId': model_info['model'], 'messages': prompt_message_dicts, 'inferenceConfig': inference_config, 'additionalModelRequestFields': additional_model_fields, } - if system and len(system) > 0: + if model_info['support_system_prompts'] and system and len(system) > 0: parameters['system'] = system - if tools: + if model_info['support_tool_use'] and tools: parameters['toolConfig'] = self._convert_converse_tool_config(tools=tools) + try: + if stream: + response = bedrock_client.converse_stream(**parameters) + return self._handle_converse_stream_response(model_info['model'], credentials, response, prompt_messages) + else: + response = bedrock_client.converse(**parameters) + return self._handle_converse_response(model_info['model'], credentials, response, prompt_messages) + except ClientError as ex: + error_code = ex.response['Error']['Code'] + full_error_msg = f"{error_code}: {ex.response['Error']['Message']}" + raise self._map_client_to_invoke_error(error_code, full_error_msg) + except (EndpointConnectionError, NoRegionError, ServiceNotInRegionError) as ex: + raise InvokeConnectionError(str(ex)) - if stream: - response = bedrock_client.converse_stream(**parameters) - return self._handle_converse_stream_response(model, credentials, response, prompt_messages) - else: - response = bedrock_client.converse(**parameters) - return self._handle_converse_response(model, credentials, response, prompt_messages) + except UnknownServiceError as ex: + raise InvokeServerUnavailableError(str(ex)) + except Exception as ex: + raise InvokeError(str(ex)) def _handle_converse_response(self, model: str, credentials: dict, response: dict, prompt_messages: list[PromptMessage]) -> LLMResult: """ @@ -203,10 +238,30 @@ class BedrockLargeLanguageModel(LargeLanguageModel): :param prompt_messages: prompt messages :return: full response chunk generator result """ + response_content = response['output']['message']['content'] # transform assistant message to prompt message - assistant_prompt_message = AssistantPromptMessage( - content=response['output']['message']['content'][0]['text'] - ) + if response['stopReason'] == 'tool_use': + tool_calls = [] + text, tool_use = self._extract_tool_use(response_content) + + tool_call = AssistantPromptMessage.ToolCall( + id=tool_use['toolUseId'], + type='function', + function=AssistantPromptMessage.ToolCall.ToolCallFunction( + name=tool_use['name'], + arguments=json.dumps(tool_use['input']) + ) + ) + tool_calls.append(tool_call) + + assistant_prompt_message = AssistantPromptMessage( + content=text, + tool_calls=tool_calls + ) + else: + assistant_prompt_message = AssistantPromptMessage( + content=response_content[0]['text'] + ) # calculate num tokens if response['usage']: @@ -229,6 +284,18 @@ class BedrockLargeLanguageModel(LargeLanguageModel): ) return result + def _extract_tool_use(self, content:dict)-> tuple[str, dict]: + tool_use = {} + text = '' + for item in content: + if 'toolUse' in item: + tool_use = item['toolUse'] + elif 'text' in item: + text = item['text'] + else: + raise ValueError(f"Got unknown item: {item}") + return text, tool_use + def _handle_converse_stream_response(self, model: str, credentials: dict, response: dict, prompt_messages: list[PromptMessage], ) -> Generator: """ @@ -340,14 +407,12 @@ class BedrockLargeLanguageModel(LargeLanguageModel): """ system = [] + prompt_message_dicts = [] for message in prompt_messages: if isinstance(message, SystemPromptMessage): message.content=message.content.strip() system.append({"text": message.content}) - - prompt_message_dicts = [] - for message in prompt_messages: - if not isinstance(message, SystemPromptMessage): + else: prompt_message_dicts.append(self._convert_prompt_message_to_dict(message)) return system, prompt_message_dicts @@ -448,7 +513,6 @@ class BedrockLargeLanguageModel(LargeLanguageModel): } else: raise ValueError(f"Got unknown type {message}") - return message_dict def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage] | str, @@ -505,7 +569,6 @@ class BedrockLargeLanguageModel(LargeLanguageModel): except ClientError as ex: error_code = ex.response['Error']['Code'] full_error_msg = f"{error_code}: {ex.response['Error']['Message']}" - raise CredentialsValidateFailedError(str(self._map_client_to_invoke_error(error_code, full_error_msg))) except Exception as ex: diff --git a/api/core/model_runtime/model_providers/bedrock/llm/meta.llama3-1-70b-instruct-v1.0.yaml b/api/core/model_runtime/model_providers/bedrock/llm/meta.llama3-1-70b-instruct-v1.0.yaml new file mode 100644 index 0000000000..10bfa7b1d5 --- /dev/null +++ b/api/core/model_runtime/model_providers/bedrock/llm/meta.llama3-1-70b-instruct-v1.0.yaml @@ -0,0 +1,25 @@ +model: meta.llama3-1-70b-instruct-v1:0 +label: + en_US: Llama 3.1 Instruct 70B +model_type: llm +model_properties: + mode: completion + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + default: 0.5 + - name: top_p + use_template: top_p + default: 0.9 + - name: max_gen_len + use_template: max_tokens + required: true + default: 512 + min: 1 + max: 2048 +pricing: + input: '0.00265' + output: '0.0035' + unit: '0.001' + currency: USD diff --git a/api/core/model_runtime/model_providers/bedrock/llm/meta.llama3-1-8b-instruct-v1.0.yaml b/api/core/model_runtime/model_providers/bedrock/llm/meta.llama3-1-8b-instruct-v1.0.yaml new file mode 100644 index 0000000000..81cd53243f --- /dev/null +++ b/api/core/model_runtime/model_providers/bedrock/llm/meta.llama3-1-8b-instruct-v1.0.yaml @@ -0,0 +1,25 @@ +model: meta.llama3-1-8b-instruct-v1:0 +label: + en_US: Llama 3.1 Instruct 8B +model_type: llm +model_properties: + mode: completion + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + default: 0.5 + - name: top_p + use_template: top_p + default: 0.9 + - name: max_gen_len + use_template: max_tokens + required: true + default: 512 + min: 1 + max: 2048 +pricing: + input: '0.0003' + output: '0.0006' + unit: '0.001' + currency: USD diff --git a/api/core/model_runtime/model_providers/bedrock/llm/mistral.mistral-large-2402-v1.0.yaml b/api/core/model_runtime/model_providers/bedrock/llm/mistral.mistral-large-2402-v1.0.yaml index 8b9a3fecd7..65eed5926b 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/mistral.mistral-large-2402-v1.0.yaml +++ b/api/core/model_runtime/model_providers/bedrock/llm/mistral.mistral-large-2402-v1.0.yaml @@ -2,6 +2,9 @@ model: mistral.mistral-large-2402-v1:0 label: en_US: Mistral Large model_type: llm +features: + - tool-call + - agent-thought model_properties: mode: completion context_size: 32000 diff --git a/api/core/model_runtime/model_providers/bedrock/llm/mistral.mistral-large-2407-v1.0.yaml b/api/core/model_runtime/model_providers/bedrock/llm/mistral.mistral-large-2407-v1.0.yaml new file mode 100644 index 0000000000..19d7843a57 --- /dev/null +++ b/api/core/model_runtime/model_providers/bedrock/llm/mistral.mistral-large-2407-v1.0.yaml @@ -0,0 +1,29 @@ +model: mistral.mistral-large-2407-v1:0 +label: + en_US: Mistral Large 2 (24.07) +model_type: llm +features: + - tool-call +model_properties: + mode: completion + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + required: false + default: 0.7 + - name: top_p + use_template: top_p + required: false + default: 1 + - name: max_tokens + use_template: max_tokens + required: true + default: 512 + min: 1 + max: 8192 +pricing: + input: '0.003' + output: '0.009' + unit: '0.001' + currency: USD diff --git a/api/core/model_runtime/model_providers/bedrock/llm/mistral.mistral-small-2402-v1.0.yaml b/api/core/model_runtime/model_providers/bedrock/llm/mistral.mistral-small-2402-v1.0.yaml index 582f4a6d9f..b97c2a9493 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/mistral.mistral-small-2402-v1.0.yaml +++ b/api/core/model_runtime/model_providers/bedrock/llm/mistral.mistral-small-2402-v1.0.yaml @@ -2,6 +2,8 @@ model: mistral.mistral-small-2402-v1:0 label: en_US: Mistral Small model_type: llm +features: + - tool-call model_properties: mode: completion context_size: 32000 diff --git a/api/core/model_runtime/model_providers/groq/llm/_position.yaml b/api/core/model_runtime/model_providers/groq/llm/_position.yaml new file mode 100644 index 0000000000..be115ca920 --- /dev/null +++ b/api/core/model_runtime/model_providers/groq/llm/_position.yaml @@ -0,0 +1,7 @@ +- llama-3.1-405b-reasoning +- llama-3.1-70b-versatile +- llama-3.1-8b-instant +- llama3-70b-8192 +- llama3-8b-8192 +- mixtral-8x7b-32768 +- llama2-70b-4096 diff --git a/api/core/model_runtime/model_providers/groq/llm/llama-3.1-405b-reasoning.yaml b/api/core/model_runtime/model_providers/groq/llm/llama-3.1-405b-reasoning.yaml new file mode 100644 index 0000000000..217785cea2 --- /dev/null +++ b/api/core/model_runtime/model_providers/groq/llm/llama-3.1-405b-reasoning.yaml @@ -0,0 +1,25 @@ +model: llama-3.1-405b-reasoning +label: + zh_Hans: Llama-3.1-405b-reasoning + en_US: Llama-3.1-405b-reasoning +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 131072 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 8192 +pricing: + input: '0.05' + output: '0.1' + unit: '0.000001' + currency: USD diff --git a/api/core/model_runtime/model_providers/groq/llm/llama-3.1-70b-versatile.yaml b/api/core/model_runtime/model_providers/groq/llm/llama-3.1-70b-versatile.yaml new file mode 100644 index 0000000000..ab5f6ab05e --- /dev/null +++ b/api/core/model_runtime/model_providers/groq/llm/llama-3.1-70b-versatile.yaml @@ -0,0 +1,25 @@ +model: llama-3.1-70b-versatile +label: + zh_Hans: Llama-3.1-70b-versatile + en_US: Llama-3.1-70b-versatile +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 131072 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 8192 +pricing: + input: '0.05' + output: '0.1' + unit: '0.000001' + currency: USD diff --git a/api/core/model_runtime/model_providers/groq/llm/llama-3.1-8b-instant.yaml b/api/core/model_runtime/model_providers/groq/llm/llama-3.1-8b-instant.yaml new file mode 100644 index 0000000000..a82e64532e --- /dev/null +++ b/api/core/model_runtime/model_providers/groq/llm/llama-3.1-8b-instant.yaml @@ -0,0 +1,25 @@ +model: llama-3.1-8b-instant +label: + zh_Hans: Llama-3.1-8b-instant + en_US: Llama-3.1-8b-instant +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 131072 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 8192 +pricing: + input: '0.05' + output: '0.1' + unit: '0.000001' + currency: USD diff --git a/api/core/model_runtime/model_providers/hunyuan/llm/llm.py b/api/core/model_runtime/model_providers/hunyuan/llm/llm.py index 2b6d8e0047..8859dd72bd 100644 --- a/api/core/model_runtime/model_providers/hunyuan/llm/llm.py +++ b/api/core/model_runtime/model_providers/hunyuan/llm/llm.py @@ -14,6 +14,7 @@ from core.model_runtime.entities.message_entities import ( PromptMessage, PromptMessageTool, SystemPromptMessage, + ToolPromptMessage, UserPromptMessage, ) from core.model_runtime.errors.invoke import InvokeError @@ -44,6 +45,17 @@ class HunyuanLargeLanguageModel(LargeLanguageModel): "Stream": stream, **custom_parameters, } + # add Tools and ToolChoice + if (tools and len(tools) > 0): + params['ToolChoice'] = "auto" + params['Tools'] = [{ + "Type": "function", + "Function": { + "Name": tool.name, + "Description": tool.description, + "Parameters": json.dumps(tool.parameters) + } + } for tool in tools] request.from_json_string(json.dumps(params)) response = client.ChatCompletions(request) @@ -89,9 +101,43 @@ class HunyuanLargeLanguageModel(LargeLanguageModel): def _convert_prompt_messages_to_dicts(self, prompt_messages: list[PromptMessage]) -> list[dict]: """Convert a list of PromptMessage objects to a list of dictionaries with 'Role' and 'Content' keys.""" - return [{"Role": message.role.value, "Content": message.content} for message in prompt_messages] + dict_list = [] + for message in prompt_messages: + if isinstance(message, AssistantPromptMessage): + tool_calls = message.tool_calls + if (tool_calls and len(tool_calls) > 0): + dict_tool_calls = [ + { + "Id": tool_call.id, + "Type": tool_call.type, + "Function": { + "Name": tool_call.function.name, + "Arguments": tool_call.function.arguments if (tool_call.function.arguments == "") else "{}" + } + } for tool_call in tool_calls] + + dict_list.append({ + "Role": message.role.value, + # fix set content = "" while tool_call request + # fix [hunyuan] None, [TencentCloudSDKException] code:InvalidParameter message:Messages Content and Contents not allowed empty at the same time. + "Content": " ", # message.content if (message.content is not None) else "", + "ToolCalls": dict_tool_calls + }) + else: + dict_list.append({ "Role": message.role.value, "Content": message.content }) + elif isinstance(message, ToolPromptMessage): + tool_execute_result = { "result": message.content } + content =json.dumps(tool_execute_result, ensure_ascii=False) + dict_list.append({ "Role": message.role.value, "Content": content, "ToolCallId": message.tool_call_id }) + else: + dict_list.append({ "Role": message.role.value, "Content": message.content }) + return dict_list def _handle_stream_chat_response(self, model, credentials, prompt_messages, resp): + + tool_call = None + tool_calls = [] + for index, event in enumerate(resp): logging.debug("_handle_stream_chat_response, event: %s", event) @@ -109,20 +155,54 @@ class HunyuanLargeLanguageModel(LargeLanguageModel): usage = data.get('Usage', {}) prompt_tokens = usage.get('PromptTokens', 0) completion_tokens = usage.get('CompletionTokens', 0) - usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens) + + response_tool_calls = delta.get('ToolCalls') + if (response_tool_calls is not None): + new_tool_calls = self._extract_response_tool_calls(response_tool_calls) + if (len(new_tool_calls) > 0): + new_tool_call = new_tool_calls[0] + if (tool_call is None): tool_call = new_tool_call + elif (tool_call.id != new_tool_call.id): + tool_calls.append(tool_call) + tool_call = new_tool_call + else: + tool_call.function.name += new_tool_call.function.name + tool_call.function.arguments += new_tool_call.function.arguments + if (tool_call is not None and len(tool_call.function.name) > 0 and len(tool_call.function.arguments) > 0): + tool_calls.append(tool_call) + tool_call = None assistant_prompt_message = AssistantPromptMessage( content=message_content, tool_calls=[] ) + # rewrite content = "" while tool_call to avoid show content on web page + if (len(tool_calls) > 0): assistant_prompt_message.content = "" + + # add tool_calls to assistant_prompt_message + if (finish_reason == 'tool_calls'): + assistant_prompt_message.tool_calls = tool_calls + tool_call = None + tool_calls = [] - delta_chunk = LLMResultChunkDelta( - index=index, - role=delta.get('Role', 'assistant'), - message=assistant_prompt_message, - usage=usage, - finish_reason=finish_reason, - ) + if (len(finish_reason) > 0): + usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens) + + delta_chunk = LLMResultChunkDelta( + index=index, + role=delta.get('Role', 'assistant'), + message=assistant_prompt_message, + usage=usage, + finish_reason=finish_reason, + ) + tool_call = None + tool_calls = [] + + else: + delta_chunk = LLMResultChunkDelta( + index=index, + message=assistant_prompt_message, + ) yield LLMResultChunk( model=model, @@ -177,12 +257,15 @@ class HunyuanLargeLanguageModel(LargeLanguageModel): """ human_prompt = "\n\nHuman:" ai_prompt = "\n\nAssistant:" + tool_prompt = "\n\nTool:" content = message.content if isinstance(message, UserPromptMessage): message_text = f"{human_prompt} {content}" elif isinstance(message, AssistantPromptMessage): message_text = f"{ai_prompt} {content}" + elif isinstance(message, ToolPromptMessage): + message_text = f"{tool_prompt} {content}" elif isinstance(message, SystemPromptMessage): message_text = content else: @@ -203,3 +286,30 @@ class HunyuanLargeLanguageModel(LargeLanguageModel): return { InvokeError: [TencentCloudSDKException], } + + def _extract_response_tool_calls(self, + response_tool_calls: list[dict]) \ + -> list[AssistantPromptMessage.ToolCall]: + """ + Extract tool calls from response + + :param response_tool_calls: response tool calls + :return: list of tool calls + """ + tool_calls = [] + if response_tool_calls: + for response_tool_call in response_tool_calls: + response_function = response_tool_call.get('Function', {}) + function = AssistantPromptMessage.ToolCall.ToolCallFunction( + name=response_function.get('Name', ''), + arguments=response_function.get('Arguments', '') + ) + + tool_call = AssistantPromptMessage.ToolCall( + id=response_tool_call.get('Id', 0), + type='function', + function=function + ) + tool_calls.append(tool_call) + + return tool_calls \ No newline at end of file diff --git a/api/core/model_runtime/model_providers/novita/llm/Nous-Hermes-2-Mixtral-8x7B-DPO.yaml b/api/core/model_runtime/model_providers/novita/llm/Nous-Hermes-2-Mixtral-8x7B-DPO.yaml index 8b19316473..7ff30458e2 100644 --- a/api/core/model_runtime/model_providers/novita/llm/Nous-Hermes-2-Mixtral-8x7B-DPO.yaml +++ b/api/core/model_runtime/model_providers/novita/llm/Nous-Hermes-2-Mixtral-8x7B-DPO.yaml @@ -34,3 +34,8 @@ parameter_rules: min: -2 max: 2 default: 0 +pricing: + input: '0.0027' + output: '0.0027' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/airoboros-l2-70b.yaml b/api/core/model_runtime/model_providers/novita/llm/airoboros-l2-70b.yaml new file mode 100644 index 0000000000..b599418461 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/airoboros-l2-70b.yaml @@ -0,0 +1,41 @@ +model: jondurbin/airoboros-l2-70b +label: + zh_Hans: jondurbin/airoboros-l2-70b + en_US: jondurbin/airoboros-l2-70b +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 4096 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 +pricing: + input: '0.005' + output: '0.005' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/dolphin-mixtral-8x22b.yaml b/api/core/model_runtime/model_providers/novita/llm/dolphin-mixtral-8x22b.yaml new file mode 100644 index 0000000000..72a181f5d3 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/dolphin-mixtral-8x22b.yaml @@ -0,0 +1,41 @@ +model: cognitivecomputations/dolphin-mixtral-8x22b +label: + zh_Hans: cognitivecomputations/dolphin-mixtral-8x22b + en_US: cognitivecomputations/dolphin-mixtral-8x22b +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 16000 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 +pricing: + input: '0.009' + output: '0.009' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/gemma-2-9b-it.yaml b/api/core/model_runtime/model_providers/novita/llm/gemma-2-9b-it.yaml new file mode 100644 index 0000000000..d1749bc882 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/gemma-2-9b-it.yaml @@ -0,0 +1,41 @@ +model: google/gemma-2-9b-it +label: + zh_Hans: google/gemma-2-9b-it + en_US: google/gemma-2-9b-it +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 +pricing: + input: '0.0008' + output: '0.0008' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/hermes-2-pro-llama-3-8b.yaml b/api/core/model_runtime/model_providers/novita/llm/hermes-2-pro-llama-3-8b.yaml new file mode 100644 index 0000000000..8b3228e56a --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/hermes-2-pro-llama-3-8b.yaml @@ -0,0 +1,41 @@ +model: nousresearch/hermes-2-pro-llama-3-8b +label: + zh_Hans: nousresearch/hermes-2-pro-llama-3-8b + en_US: nousresearch/hermes-2-pro-llama-3-8b +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 +pricing: + input: '0.0014' + output: '0.0014' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/l3-70b-euryale-v2.1.yaml b/api/core/model_runtime/model_providers/novita/llm/l3-70b-euryale-v2.1.yaml new file mode 100644 index 0000000000..5e27941c52 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/l3-70b-euryale-v2.1.yaml @@ -0,0 +1,41 @@ +model: sao10k/l3-70b-euryale-v2.1 +label: + zh_Hans: sao10k/l3-70b-euryale-v2.1 + en_US: sao10k/l3-70b-euryale-v2.1 +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 16000 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 +pricing: + input: '0.0148' + output: '0.0148' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/llama-3-70b-instruct.yaml b/api/core/model_runtime/model_providers/novita/llm/llama-3-70b-instruct.yaml index 5298296de3..39709e1063 100644 --- a/api/core/model_runtime/model_providers/novita/llm/llama-3-70b-instruct.yaml +++ b/api/core/model_runtime/model_providers/novita/llm/llama-3-70b-instruct.yaml @@ -34,3 +34,8 @@ parameter_rules: min: -2 max: 2 default: 0 +pricing: + input: '0.0051' + output: '0.0074' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/llama-3-8b-instruct.yaml b/api/core/model_runtime/model_providers/novita/llm/llama-3-8b-instruct.yaml index 45e62ee52a..9b5e5df4d0 100644 --- a/api/core/model_runtime/model_providers/novita/llm/llama-3-8b-instruct.yaml +++ b/api/core/model_runtime/model_providers/novita/llm/llama-3-8b-instruct.yaml @@ -34,3 +34,8 @@ parameter_rules: min: -2 max: 2 default: 0 +pricing: + input: '0.00063' + output: '0.00063' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/llama-3.1-405b-instruct.yaml b/api/core/model_runtime/model_providers/novita/llm/llama-3.1-405b-instruct.yaml new file mode 100644 index 0000000000..c5a45271ae --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/llama-3.1-405b-instruct.yaml @@ -0,0 +1,41 @@ +model: meta-llama/llama-3.1-405b-instruct +label: + zh_Hans: meta-llama/llama-3.1-405b-instruct + en_US: meta-llama/llama-3.1-405b-instruct +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 32768 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 +pricing: + input: '0.03' + output: '0.05' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/llama-3.1-70b-instruct.yaml b/api/core/model_runtime/model_providers/novita/llm/llama-3.1-70b-instruct.yaml new file mode 100644 index 0000000000..3a5c29c40f --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/llama-3.1-70b-instruct.yaml @@ -0,0 +1,41 @@ +model: meta-llama/llama-3.1-70b-instruct +label: + zh_Hans: meta-llama/llama-3.1-70b-instruct + en_US: meta-llama/llama-3.1-70b-instruct +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 +pricing: + input: '0.0055' + output: '0.0076' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/llama-3.1-8b-instruct.yaml b/api/core/model_runtime/model_providers/novita/llm/llama-3.1-8b-instruct.yaml new file mode 100644 index 0000000000..e6ef772a3f --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/llama-3.1-8b-instruct.yaml @@ -0,0 +1,41 @@ +model: meta-llama/llama-3.1-8b-instruct +label: + zh_Hans: meta-llama/llama-3.1-8b-instruct + en_US: meta-llama/llama-3.1-8b-instruct +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 +pricing: + input: '0.001' + output: '0.001' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/lzlv_70b.yaml b/api/core/model_runtime/model_providers/novita/llm/lzlv_70b.yaml index 0facc0c112..0cc68a8c45 100644 --- a/api/core/model_runtime/model_providers/novita/llm/lzlv_70b.yaml +++ b/api/core/model_runtime/model_providers/novita/llm/lzlv_70b.yaml @@ -34,3 +34,8 @@ parameter_rules: min: -2 max: 2 default: 0 +pricing: + input: '0.0058' + output: '0.0078' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/midnight-rose-70b.yaml b/api/core/model_runtime/model_providers/novita/llm/midnight-rose-70b.yaml new file mode 100644 index 0000000000..19876bee17 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/midnight-rose-70b.yaml @@ -0,0 +1,41 @@ +model: sophosympatheia/midnight-rose-70b +label: + zh_Hans: sophosympatheia/midnight-rose-70b + en_US: sophosympatheia/midnight-rose-70b +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 4096 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 +pricing: + input: '0.008' + output: '0.008' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/mistral-7b-instruct.yaml b/api/core/model_runtime/model_providers/novita/llm/mistral-7b-instruct.yaml new file mode 100644 index 0000000000..6fba47bcf0 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/mistral-7b-instruct.yaml @@ -0,0 +1,41 @@ +model: mistralai/mistral-7b-instruct +label: + zh_Hans: mistralai/mistral-7b-instruct + en_US: mistralai/mistral-7b-instruct +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 32768 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 +pricing: + input: '0.00059' + output: '0.00059' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/mythomax-l2-13b.yaml b/api/core/model_runtime/model_providers/novita/llm/mythomax-l2-13b.yaml index 28a8630ff2..7e4ac3ffe0 100644 --- a/api/core/model_runtime/model_providers/novita/llm/mythomax-l2-13b.yaml +++ b/api/core/model_runtime/model_providers/novita/llm/mythomax-l2-13b.yaml @@ -34,3 +34,8 @@ parameter_rules: min: -2 max: 2 default: 0 +pricing: + input: '0.00119' + output: '0.00119' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/nous-hermes-llama2-13b.yaml b/api/core/model_runtime/model_providers/novita/llm/nous-hermes-llama2-13b.yaml index ce714a118b..75671c414c 100644 --- a/api/core/model_runtime/model_providers/novita/llm/nous-hermes-llama2-13b.yaml +++ b/api/core/model_runtime/model_providers/novita/llm/nous-hermes-llama2-13b.yaml @@ -34,3 +34,8 @@ parameter_rules: min: -2 max: 2 default: 0 +pricing: + input: '0.0017' + output: '0.0017' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/openhermes-2.5-mistral-7b.yaml b/api/core/model_runtime/model_providers/novita/llm/openhermes-2.5-mistral-7b.yaml index 6cef39f847..8b0deba4f7 100644 --- a/api/core/model_runtime/model_providers/novita/llm/openhermes-2.5-mistral-7b.yaml +++ b/api/core/model_runtime/model_providers/novita/llm/openhermes-2.5-mistral-7b.yaml @@ -34,3 +34,8 @@ parameter_rules: min: -2 max: 2 default: 0 +pricing: + input: '0.0017' + output: '0.0017' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/llm/wizardlm-2-8x22b.yaml b/api/core/model_runtime/model_providers/novita/llm/wizardlm-2-8x22b.yaml index b3e3a03697..ef42568e8f 100644 --- a/api/core/model_runtime/model_providers/novita/llm/wizardlm-2-8x22b.yaml +++ b/api/core/model_runtime/model_providers/novita/llm/wizardlm-2-8x22b.yaml @@ -34,3 +34,8 @@ parameter_rules: min: -2 max: 2 default: 0 +pricing: + input: '0.0064' + output: '0.0064' + unit: '0.0001' + currency: USD diff --git a/api/core/model_runtime/model_providers/novita/novita.yaml b/api/core/model_runtime/model_providers/novita/novita.yaml index ef6a863569..f634197989 100644 --- a/api/core/model_runtime/model_providers/novita/novita.yaml +++ b/api/core/model_runtime/model_providers/novita/novita.yaml @@ -1,6 +1,9 @@ provider: novita label: en_US: novita.ai +description: + en_US: An LLM API that matches various application scenarios with high cost-effectiveness. + zh_Hans: 适配多种海外应用场景的高性价比 LLM API icon_small: en_US: icon_s_en.svg icon_large: @@ -11,7 +14,7 @@ help: en_US: Get your API key from novita.ai zh_Hans: 从 novita.ai 获取 API Key url: - en_US: https://novita.ai/dashboard/key?utm_source=dify + en_US: https://novita.ai/settings#key-management?utm_source=dify&utm_medium=ch&utm_campaign=api supported_model_types: - llm configurate_methods: diff --git a/api/core/model_runtime/model_providers/openai/llm/_position.yaml b/api/core/model_runtime/model_providers/openai/llm/_position.yaml index 566055e3f7..91b9215829 100644 --- a/api/core/model_runtime/model_providers/openai/llm/_position.yaml +++ b/api/core/model_runtime/model_providers/openai/llm/_position.yaml @@ -1,6 +1,8 @@ - gpt-4 - gpt-4o - gpt-4o-2024-05-13 +- gpt-4o-mini +- gpt-4o-mini-2024-07-18 - gpt-4-turbo - gpt-4-turbo-2024-04-09 - gpt-4-turbo-preview diff --git a/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml new file mode 100644 index 0000000000..6f23e0647d --- /dev/null +++ b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml @@ -0,0 +1,44 @@ +model: gpt-4o-mini-2024-07-18 +label: + zh_Hans: gpt-4o-mini-2024-07-18 + en_US: gpt-4o-mini-2024-07-18 +model_type: llm +features: + - multi-tool-call + - agent-thought + - stream-tool-call + - vision +model_properties: + mode: chat + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: presence_penalty + use_template: presence_penalty + - name: frequency_penalty + use_template: frequency_penalty + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 16384 + - name: response_format + label: + zh_Hans: 回复格式 + en_US: response_format + type: string + help: + zh_Hans: 指定模型必须输出的格式 + en_US: specifying the format that the model must output + required: false + options: + - text + - json_object +pricing: + input: '0.15' + output: '0.60' + unit: '0.000001' + currency: USD diff --git a/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml new file mode 100644 index 0000000000..b97fbf8aab --- /dev/null +++ b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml @@ -0,0 +1,44 @@ +model: gpt-4o-mini +label: + zh_Hans: gpt-4o-mini + en_US: gpt-4o-mini +model_type: llm +features: + - multi-tool-call + - agent-thought + - stream-tool-call + - vision +model_properties: + mode: chat + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: presence_penalty + use_template: presence_penalty + - name: frequency_penalty + use_template: frequency_penalty + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 16384 + - name: response_format + label: + zh_Hans: 回复格式 + en_US: response_format + type: string + help: + zh_Hans: 指定模型必须输出的格式 + en_US: specifying the format that the model must output + required: false + options: + - text + - json_object +pricing: + input: '0.15' + output: '0.60' + unit: '0.000001' + currency: USD diff --git a/api/core/model_runtime/model_providers/openai/tts/tts.py b/api/core/model_runtime/model_providers/openai/tts/tts.py index 608ed897e0..d3fcf731f1 100644 --- a/api/core/model_runtime/model_providers/openai/tts/tts.py +++ b/api/core/model_runtime/model_providers/openai/tts/tts.py @@ -114,7 +114,8 @@ class OpenAIText2SpeechModel(_CommonOpenAI, TTSModel): # doc: https://platform.openai.com/docs/guides/text-to-speech credentials_kwargs = self._to_credential_kwargs(credentials) client = OpenAI(**credentials_kwargs) - if not voice or voice not in self.get_tts_model_voices(model=model, credentials=credentials): + model_support_voice = [x.get("value") for x in self.get_tts_model_voices(model=model, credentials=credentials)] + if not voice or voice not in model_support_voice: voice = self._get_model_default_voice(model, credentials) word_limit = self._get_model_word_limit(model, credentials) if len(content_text) > word_limit: diff --git a/api/core/model_runtime/model_providers/openrouter/llm/_position.yaml b/api/core/model_runtime/model_providers/openrouter/llm/_position.yaml index 51131249e5..7e00dd3f4b 100644 --- a/api/core/model_runtime/model_providers/openrouter/llm/_position.yaml +++ b/api/core/model_runtime/model_providers/openrouter/llm/_position.yaml @@ -1,4 +1,5 @@ - openai/gpt-4o +- openai/gpt-4o-mini - openai/gpt-4 - openai/gpt-4-32k - openai/gpt-3.5-turbo @@ -11,6 +12,9 @@ - google/gemini-pro - cohere/command-r-plus - cohere/command-r +- meta-llama/llama-3.1-405b-instruct +- meta-llama/llama-3.1-70b-instruct +- meta-llama/llama-3.1-8b-instruct - meta-llama/llama-3-70b-instruct - meta-llama/llama-3-8b-instruct - mistralai/mixtral-8x22b-instruct diff --git a/api/core/model_runtime/model_providers/openrouter/llm/gpt-4o-mini.yaml b/api/core/model_runtime/model_providers/openrouter/llm/gpt-4o-mini.yaml new file mode 100644 index 0000000000..de0bad4136 --- /dev/null +++ b/api/core/model_runtime/model_providers/openrouter/llm/gpt-4o-mini.yaml @@ -0,0 +1,43 @@ +model: openai/gpt-4o-mini +label: + en_US: gpt-4o-mini +model_type: llm +features: + - multi-tool-call + - agent-thought + - stream-tool-call + - vision +model_properties: + mode: chat + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: presence_penalty + use_template: presence_penalty + - name: frequency_penalty + use_template: frequency_penalty + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 16384 + - name: response_format + label: + zh_Hans: 回复格式 + en_US: response_format + type: string + help: + zh_Hans: 指定模型必须输出的格式 + en_US: specifying the format that the model must output + required: false + options: + - text + - json_object +pricing: + input: "0.15" + output: "0.60" + unit: "0.000001" + currency: USD diff --git a/api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-405b-instruct.yaml b/api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-405b-instruct.yaml new file mode 100644 index 0000000000..7d68e708b7 --- /dev/null +++ b/api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-405b-instruct.yaml @@ -0,0 +1,23 @@ +model: meta-llama/llama-3.1-405b-instruct +label: + en_US: llama-3.1-405b-instruct +model_type: llm +model_properties: + mode: chat + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + required: true + default: 512 + min: 1 + max: 128000 +pricing: + input: "3" + output: "3" + unit: "0.000001" + currency: USD diff --git a/api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-70b-instruct.yaml b/api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-70b-instruct.yaml new file mode 100644 index 0000000000..78e3b45435 --- /dev/null +++ b/api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-70b-instruct.yaml @@ -0,0 +1,23 @@ +model: meta-llama/llama-3.1-70b-instruct +label: + en_US: llama-3.1-70b-instruct +model_type: llm +model_properties: + mode: chat + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + required: true + default: 512 + min: 1 + max: 128000 +pricing: + input: "0.9" + output: "0.9" + unit: "0.000001" + currency: USD diff --git a/api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-8b-instruct.yaml b/api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-8b-instruct.yaml new file mode 100644 index 0000000000..6e69b7deb7 --- /dev/null +++ b/api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-8b-instruct.yaml @@ -0,0 +1,23 @@ +model: meta-llama/llama-3.1-8b-instruct +label: + en_US: llama-3.1-8b-instruct +model_type: llm +model_properties: + mode: chat + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + required: true + default: 512 + min: 1 + max: 128000 +pricing: + input: "0.2" + output: "0.2" + unit: "0.000001" + currency: USD diff --git a/api/core/model_runtime/model_providers/sagemaker/__init__.py b/api/core/model_runtime/model_providers/sagemaker/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/model_runtime/model_providers/sagemaker/_assets/icon_l_en.png b/api/core/model_runtime/model_providers/sagemaker/_assets/icon_l_en.png new file mode 100644 index 0000000000..0abe07a78f Binary files /dev/null and b/api/core/model_runtime/model_providers/sagemaker/_assets/icon_l_en.png differ diff --git a/api/core/model_runtime/model_providers/sagemaker/_assets/icon_s_en.png b/api/core/model_runtime/model_providers/sagemaker/_assets/icon_s_en.png new file mode 100644 index 0000000000..6b88942a5c Binary files /dev/null and b/api/core/model_runtime/model_providers/sagemaker/_assets/icon_s_en.png differ diff --git a/api/core/model_runtime/model_providers/sagemaker/llm/__init__.py b/api/core/model_runtime/model_providers/sagemaker/llm/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/model_runtime/model_providers/sagemaker/llm/llm.py b/api/core/model_runtime/model_providers/sagemaker/llm/llm.py new file mode 100644 index 0000000000..f8e7757a96 --- /dev/null +++ b/api/core/model_runtime/model_providers/sagemaker/llm/llm.py @@ -0,0 +1,238 @@ +import json +import logging +from collections.abc import Generator +from typing import Any, Optional, Union + +import boto3 + +from core.model_runtime.entities.llm_entities import LLMMode, LLMResult +from core.model_runtime.entities.message_entities import ( + AssistantPromptMessage, + PromptMessage, + PromptMessageTool, +) +from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, I18nObject, ModelType +from core.model_runtime.errors.invoke import ( + InvokeAuthorizationError, + InvokeBadRequestError, + InvokeConnectionError, + InvokeError, + InvokeRateLimitError, + InvokeServerUnavailableError, +) +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel + +logger = logging.getLogger(__name__) + + +class SageMakerLargeLanguageModel(LargeLanguageModel): + """ + Model class for Cohere large language model. + """ + sagemaker_client: Any = None + + def _invoke(self, model: str, credentials: dict, + prompt_messages: list[PromptMessage], model_parameters: dict, + tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, + stream: bool = True, user: Optional[str] = None) \ + -> Union[LLMResult, Generator]: + """ + Invoke large language model + + :param model: model name + :param credentials: model credentials + :param prompt_messages: prompt messages + :param model_parameters: model parameters + :param tools: tools for tool calling + :param stop: stop words + :param stream: is stream response + :param user: unique user id + :return: full response or stream response chunk generator result + """ + # get model mode + model_mode = self.get_model_mode(model, credentials) + + if not self.sagemaker_client: + access_key = credentials.get('access_key') + secret_key = credentials.get('secret_key') + aws_region = credentials.get('aws_region') + if aws_region: + if access_key and secret_key: + self.sagemaker_client = boto3.client("sagemaker-runtime", + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime", region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime") + + + sagemaker_endpoint = credentials.get('sagemaker_endpoint') + response_model = self.sagemaker_client.invoke_endpoint( + EndpointName=sagemaker_endpoint, + Body=json.dumps( + { + "inputs": prompt_messages[0].content, + "parameters": { "stop" : stop}, + "history" : [] + } + ), + ContentType="application/json", + ) + + assistant_text = response_model['Body'].read().decode('utf8') + + # transform assistant message to prompt message + assistant_prompt_message = AssistantPromptMessage( + content=assistant_text + ) + + usage = self._calc_response_usage(model, credentials, 0, 0) + + response = LLMResult( + model=model, + prompt_messages=prompt_messages, + message=assistant_prompt_message, + usage=usage + ) + + return response + + def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], + tools: Optional[list[PromptMessageTool]] = None) -> int: + """ + Get number of tokens for given prompt messages + + :param model: model name + :param credentials: model credentials + :param prompt_messages: prompt messages + :param tools: tools for tool calling + :return: + """ + # get model mode + model_mode = self.get_model_mode(model) + + try: + return 0 + except Exception as e: + raise self._transform_invoke_error(e) + + def validate_credentials(self, model: str, credentials: dict) -> None: + """ + Validate model credentials + + :param model: model name + :param credentials: model credentials + :return: + """ + try: + # get model mode + model_mode = self.get_model_mode(model) + except Exception as ex: + raise CredentialsValidateFailedError(str(ex)) + + @property + def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + """ + Map model invoke error to unified error + The key is the error type thrown to the caller + The value is the error type thrown by the model, + which needs to be converted into a unified error type for the caller. + + :return: Invoke error mapping + """ + return { + InvokeConnectionError: [ + InvokeConnectionError + ], + InvokeServerUnavailableError: [ + InvokeServerUnavailableError + ], + InvokeRateLimitError: [ + InvokeRateLimitError + ], + InvokeAuthorizationError: [ + InvokeAuthorizationError + ], + InvokeBadRequestError: [ + InvokeBadRequestError, + KeyError, + ValueError + ] + } + + def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None: + """ + used to define customizable model schema + """ + rules = [ + ParameterRule( + name='temperature', + type=ParameterType.FLOAT, + use_template='temperature', + label=I18nObject( + zh_Hans='温度', + en_US='Temperature' + ), + ), + ParameterRule( + name='top_p', + type=ParameterType.FLOAT, + use_template='top_p', + label=I18nObject( + zh_Hans='Top P', + en_US='Top P' + ) + ), + ParameterRule( + name='max_tokens', + type=ParameterType.INT, + use_template='max_tokens', + min=1, + max=credentials.get('context_length', 2048), + default=512, + label=I18nObject( + zh_Hans='最大生成长度', + en_US='Max Tokens' + ) + ) + ] + + completion_type = LLMMode.value_of(credentials["mode"]) + + if completion_type == LLMMode.CHAT: + print(f"completion_type : {LLMMode.CHAT.value}") + + if completion_type == LLMMode.COMPLETION: + print(f"completion_type : {LLMMode.COMPLETION.value}") + + features = [] + + support_function_call = credentials.get('support_function_call', False) + if support_function_call: + features.append(ModelFeature.TOOL_CALL) + + support_vision = credentials.get('support_vision', False) + if support_vision: + features.append(ModelFeature.VISION) + + context_length = credentials.get('context_length', 2048) + + entity = AIModelEntity( + model=model, + label=I18nObject( + en_US=model + ), + fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, + model_type=ModelType.LLM, + features=features, + model_properties={ + ModelPropertyKey.MODE: completion_type, + ModelPropertyKey.CONTEXT_SIZE: context_length + }, + parameter_rules=rules + ) + + return entity diff --git a/api/core/model_runtime/model_providers/sagemaker/rerank/__init__.py b/api/core/model_runtime/model_providers/sagemaker/rerank/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/model_runtime/model_providers/sagemaker/rerank/rerank.py b/api/core/model_runtime/model_providers/sagemaker/rerank/rerank.py new file mode 100644 index 0000000000..0b06f54ef1 --- /dev/null +++ b/api/core/model_runtime/model_providers/sagemaker/rerank/rerank.py @@ -0,0 +1,190 @@ +import json +import logging +from typing import Any, Optional + +import boto3 + +from core.model_runtime.entities.common_entities import I18nObject +from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelType +from core.model_runtime.entities.rerank_entities import RerankDocument, RerankResult +from core.model_runtime.errors.invoke import ( + InvokeAuthorizationError, + InvokeBadRequestError, + InvokeConnectionError, + InvokeError, + InvokeRateLimitError, + InvokeServerUnavailableError, +) +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.rerank_model import RerankModel + +logger = logging.getLogger(__name__) + +class SageMakerRerankModel(RerankModel): + """ + Model class for Cohere rerank model. + """ + sagemaker_client: Any = None + + def _sagemaker_rerank(self, query_input: str, docs: list[str], rerank_endpoint:str): + inputs = [query_input]*len(docs) + response_model = self.sagemaker_client.invoke_endpoint( + EndpointName=rerank_endpoint, + Body=json.dumps( + { + "inputs": inputs, + "docs": docs + } + ), + ContentType="application/json", + ) + json_str = response_model['Body'].read().decode('utf8') + json_obj = json.loads(json_str) + scores = json_obj['scores'] + return scores if isinstance(scores, list) else [scores] + + + def _invoke(self, model: str, credentials: dict, + query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, + user: Optional[str] = None) \ + -> RerankResult: + """ + Invoke rerank model + + :param model: model name + :param credentials: model credentials + :param query: search query + :param docs: docs for reranking + :param score_threshold: score threshold + :param top_n: top n + :param user: unique user id + :return: rerank result + """ + line = 0 + try: + if len(docs) == 0: + return RerankResult( + model=model, + docs=docs + ) + + line = 1 + if not self.sagemaker_client: + access_key = credentials.get('aws_access_key_id') + secret_key = credentials.get('aws_secret_access_key') + aws_region = credentials.get('aws_region') + if aws_region: + if access_key and secret_key: + self.sagemaker_client = boto3.client("sagemaker-runtime", + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime", region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime") + + line = 2 + + sagemaker_endpoint = credentials.get('sagemaker_endpoint') + candidate_docs = [] + + scores = self._sagemaker_rerank(query, docs, sagemaker_endpoint) + for idx in range(len(scores)): + candidate_docs.append({"content" : docs[idx], "score": scores[idx]}) + + sorted(candidate_docs, key=lambda x: x['score'], reverse=True) + + line = 3 + rerank_documents = [] + for idx, result in enumerate(candidate_docs): + rerank_document = RerankDocument( + index=idx, + text=result.get('content'), + score=result.get('score', -100.0) + ) + + if score_threshold is not None: + if rerank_document.score >= score_threshold: + rerank_documents.append(rerank_document) + else: + rerank_documents.append(rerank_document) + + return RerankResult( + model=model, + docs=rerank_documents + ) + + except Exception as e: + logger.exception(f'Exception {e}, line : {line}') + + def validate_credentials(self, model: str, credentials: dict) -> None: + """ + Validate model credentials + + :param model: model name + :param credentials: model credentials + :return: + """ + try: + self._invoke( + model=model, + credentials=credentials, + query="What is the capital of the United States?", + docs=[ + "Carson City is the capital city of the American state of Nevada. At the 2010 United States " + "Census, Carson City had a population of 55,274.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that " + "are a political division controlled by the United States. Its capital is Saipan.", + ], + score_threshold=0.8 + ) + except Exception as ex: + raise CredentialsValidateFailedError(str(ex)) + + @property + def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + """ + Map model invoke error to unified error + The key is the error type thrown to the caller + The value is the error type thrown by the model, + which needs to be converted into a unified error type for the caller. + + :return: Invoke error mapping + """ + return { + InvokeConnectionError: [ + InvokeConnectionError + ], + InvokeServerUnavailableError: [ + InvokeServerUnavailableError + ], + InvokeRateLimitError: [ + InvokeRateLimitError + ], + InvokeAuthorizationError: [ + InvokeAuthorizationError + ], + InvokeBadRequestError: [ + InvokeBadRequestError, + KeyError, + ValueError + ] + } + + def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None: + """ + used to define customizable model schema + """ + entity = AIModelEntity( + model=model, + label=I18nObject( + en_US=model + ), + fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, + model_type=ModelType.RERANK, + model_properties={ }, + parameter_rules=[] + ) + + return entity \ No newline at end of file diff --git a/api/core/model_runtime/model_providers/sagemaker/sagemaker.py b/api/core/model_runtime/model_providers/sagemaker/sagemaker.py new file mode 100644 index 0000000000..02d05f406c --- /dev/null +++ b/api/core/model_runtime/model_providers/sagemaker/sagemaker.py @@ -0,0 +1,17 @@ +import logging + +from core.model_runtime.model_providers.__base.model_provider import ModelProvider + +logger = logging.getLogger(__name__) + + +class SageMakerProvider(ModelProvider): + def validate_provider_credentials(self, credentials: dict) -> None: + """ + Validate provider credentials + + if validate failed, raise exception + + :param credentials: provider credentials, credentials form defined in `provider_credential_schema`. + """ + pass diff --git a/api/core/model_runtime/model_providers/sagemaker/sagemaker.yaml b/api/core/model_runtime/model_providers/sagemaker/sagemaker.yaml new file mode 100644 index 0000000000..290cb0edab --- /dev/null +++ b/api/core/model_runtime/model_providers/sagemaker/sagemaker.yaml @@ -0,0 +1,125 @@ +provider: sagemaker +label: + zh_Hans: Sagemaker + en_US: Sagemaker +icon_small: + en_US: icon_s_en.png +icon_large: + en_US: icon_l_en.png +description: + en_US: Customized model on Sagemaker + zh_Hans: Sagemaker上的私有化部署的模型 +background: "#ECE9E3" +help: + title: + en_US: How to deploy customized model on Sagemaker + zh_Hans: 如何在Sagemaker上的私有化部署的模型 + url: + en_US: https://github.com/aws-samples/dify-aws-tool/blob/main/README.md#how-to-deploy-sagemaker-endpoint + zh_Hans: https://github.com/aws-samples/dify-aws-tool/blob/main/README_ZH.md#%E5%A6%82%E4%BD%95%E9%83%A8%E7%BD%B2sagemaker%E6%8E%A8%E7%90%86%E7%AB%AF%E7%82%B9 +supported_model_types: + - llm + - text-embedding + - rerank +configurate_methods: + - customizable-model +model_credential_schema: + model: + label: + en_US: Model Name + zh_Hans: 模型名称 + placeholder: + en_US: Enter your model name + zh_Hans: 输入模型名称 + credential_form_schemas: + - variable: mode + show_on: + - variable: __model_type + value: llm + label: + en_US: Completion mode + type: select + required: false + default: chat + placeholder: + zh_Hans: 选择对话类型 + en_US: Select completion mode + options: + - value: completion + label: + en_US: Completion + zh_Hans: 补全 + - value: chat + label: + en_US: Chat + zh_Hans: 对话 + - variable: sagemaker_endpoint + label: + en_US: sagemaker endpoint + type: text-input + required: true + placeholder: + zh_Hans: 请输出你的Sagemaker推理端点 + en_US: Enter your Sagemaker Inference endpoint + - variable: aws_access_key_id + required: false + label: + en_US: Access Key (If not provided, credentials are obtained from the running environment.) + zh_Hans: Access Key (如果未提供，凭证将从运行环境中获取。) + type: secret-input + placeholder: + en_US: Enter your Access Key + zh_Hans: 在此输入您的 Access Key + - variable: aws_secret_access_key + required: false + label: + en_US: Secret Access Key + zh_Hans: Secret Access Key + type: secret-input + placeholder: + en_US: Enter your Secret Access Key + zh_Hans: 在此输入您的 Secret Access Key + - variable: aws_region + required: false + label: + en_US: AWS Region + zh_Hans: AWS 地区 + type: select + default: us-east-1 + options: + - value: us-east-1 + label: + en_US: US East (N. Virginia) + zh_Hans: 美国东部 (弗吉尼亚北部) + - value: us-west-2 + label: + en_US: US West (Oregon) + zh_Hans: 美国西部 (俄勒冈州) + - value: ap-southeast-1 + label: + en_US: Asia Pacific (Singapore) + zh_Hans: 亚太地区 (新加坡) + - value: ap-northeast-1 + label: + en_US: Asia Pacific (Tokyo) + zh_Hans: 亚太地区 (东京) + - value: eu-central-1 + label: + en_US: Europe (Frankfurt) + zh_Hans: 欧洲 (法兰克福) + - value: us-gov-west-1 + label: + en_US: AWS GovCloud (US-West) + zh_Hans: AWS GovCloud (US-West) + - value: ap-southeast-2 + label: + en_US: Asia Pacific (Sydney) + zh_Hans: 亚太地区 (悉尼) + - value: cn-north-1 + label: + en_US: AWS Beijing (cn-north-1) + zh_Hans: 中国北京 (cn-north-1) + - value: cn-northwest-1 + label: + en_US: AWS Ningxia (cn-northwest-1) + zh_Hans: 中国宁夏 (cn-northwest-1) diff --git a/api/core/model_runtime/model_providers/sagemaker/text_embedding/__init__.py b/api/core/model_runtime/model_providers/sagemaker/text_embedding/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/model_runtime/model_providers/sagemaker/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/sagemaker/text_embedding/text_embedding.py new file mode 100644 index 0000000000..4b2858b1a2 --- /dev/null +++ b/api/core/model_runtime/model_providers/sagemaker/text_embedding/text_embedding.py @@ -0,0 +1,214 @@ +import itertools +import json +import logging +import time +from typing import Any, Optional + +import boto3 + +from core.model_runtime.entities.common_entities import I18nObject +from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelPropertyKey, ModelType, PriceType +from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult +from core.model_runtime.errors.invoke import ( + InvokeAuthorizationError, + InvokeBadRequestError, + InvokeConnectionError, + InvokeError, + InvokeRateLimitError, + InvokeServerUnavailableError, +) +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel + +BATCH_SIZE = 20 +CONTEXT_SIZE=8192 + +logger = logging.getLogger(__name__) + +def batch_generator(generator, batch_size): + while True: + batch = list(itertools.islice(generator, batch_size)) + if not batch: + break + yield batch + +class SageMakerEmbeddingModel(TextEmbeddingModel): + """ + Model class for Cohere text embedding model. + """ + sagemaker_client: Any = None + + def _sagemaker_embedding(self, sm_client, endpoint_name, content_list:list[str]): + response_model = sm_client.invoke_endpoint( + EndpointName=endpoint_name, + Body=json.dumps( + { + "inputs": content_list, + "parameters": {}, + "is_query" : False, + "instruction" : '' + } + ), + ContentType="application/json", + ) + json_str = response_model['Body'].read().decode('utf8') + json_obj = json.loads(json_str) + embeddings = json_obj['embeddings'] + return embeddings + + def _invoke(self, model: str, credentials: dict, + texts: list[str], user: Optional[str] = None) \ + -> TextEmbeddingResult: + """ + Invoke text embedding model + + :param model: model name + :param credentials: model credentials + :param texts: texts to embed + :param user: unique user id + :return: embeddings result + """ + # get model properties + try: + line = 1 + if not self.sagemaker_client: + access_key = credentials.get('aws_access_key_id') + secret_key = credentials.get('aws_secret_access_key') + aws_region = credentials.get('aws_region') + if aws_region: + if access_key and secret_key: + self.sagemaker_client = boto3.client("sagemaker-runtime", + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime", region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime") + + line = 2 + sagemaker_endpoint = credentials.get('sagemaker_endpoint') + + line = 3 + truncated_texts = [ item[:CONTEXT_SIZE] for item in texts ] + + batches = batch_generator((text for text in truncated_texts), batch_size=BATCH_SIZE) + all_embeddings = [] + + line = 4 + for batch in batches: + embeddings = self._sagemaker_embedding(self.sagemaker_client, sagemaker_endpoint, batch) + all_embeddings.extend(embeddings) + + line = 5 + # calc usage + usage = self._calc_response_usage( + model=model, + credentials=credentials, + tokens=0 # It's not SAAS API, usage is meaningless + ) + line = 6 + + return TextEmbeddingResult( + embeddings=all_embeddings, + usage=usage, + model=model + ) + + except Exception as e: + logger.exception(f'Exception {e}, line : {line}') + + def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + """ + Get number of tokens for given prompt messages + + :param model: model name + :param credentials: model credentials + :param texts: texts to embed + :return: + """ + return 0 + + def validate_credentials(self, model: str, credentials: dict) -> None: + """ + Validate model credentials + + :param model: model name + :param credentials: model credentials + :return: + """ + try: + print("validate_credentials ok....") + except Exception as ex: + raise CredentialsValidateFailedError(str(ex)) + + def _calc_response_usage(self, model: str, credentials: dict, tokens: int) -> EmbeddingUsage: + """ + Calculate response usage + + :param model: model name + :param credentials: model credentials + :param tokens: input tokens + :return: usage + """ + # get input price info + input_price_info = self.get_price( + model=model, + credentials=credentials, + price_type=PriceType.INPUT, + tokens=tokens + ) + + # transform usage + usage = EmbeddingUsage( + tokens=tokens, + total_tokens=tokens, + unit_price=input_price_info.unit_price, + price_unit=input_price_info.unit, + total_price=input_price_info.total_amount, + currency=input_price_info.currency, + latency=time.perf_counter() - self.started_at + ) + + return usage + + @property + def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + return { + InvokeConnectionError: [ + InvokeConnectionError + ], + InvokeServerUnavailableError: [ + InvokeServerUnavailableError + ], + InvokeRateLimitError: [ + InvokeRateLimitError + ], + InvokeAuthorizationError: [ + InvokeAuthorizationError + ], + InvokeBadRequestError: [ + KeyError + ] + } + + def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None: + """ + used to define customizable model schema + """ + + entity = AIModelEntity( + model=model, + label=I18nObject( + en_US=model + ), + fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, + model_type=ModelType.TEXT_EMBEDDING, + model_properties={ + ModelPropertyKey.CONTEXT_SIZE: CONTEXT_SIZE, + ModelPropertyKey.MAX_CHUNKS: BATCH_SIZE, + }, + parameter_rules=[] + ) + + return entity diff --git a/api/core/model_runtime/model_providers/stepfun/__init__.py b/api/core/model_runtime/model_providers/stepfun/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/model_runtime/model_providers/stepfun/_assets/icon_l_en.png b/api/core/model_runtime/model_providers/stepfun/_assets/icon_l_en.png new file mode 100644 index 0000000000..c118ea09bd Binary files /dev/null and b/api/core/model_runtime/model_providers/stepfun/_assets/icon_l_en.png differ diff --git a/api/core/model_runtime/model_providers/stepfun/_assets/icon_s_en.png b/api/core/model_runtime/model_providers/stepfun/_assets/icon_s_en.png new file mode 100644 index 0000000000..85b96d0c74 Binary files /dev/null and b/api/core/model_runtime/model_providers/stepfun/_assets/icon_s_en.png differ diff --git a/api/core/model_runtime/model_providers/stepfun/llm/__init__.py b/api/core/model_runtime/model_providers/stepfun/llm/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/model_runtime/model_providers/stepfun/llm/_position.yaml b/api/core/model_runtime/model_providers/stepfun/llm/_position.yaml new file mode 100644 index 0000000000..b34433e1d4 --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/_position.yaml @@ -0,0 +1,6 @@ +- step-1-8k +- step-1-32k +- step-1-128k +- step-1-256k +- step-1v-8k +- step-1v-32k diff --git a/api/core/model_runtime/model_providers/stepfun/llm/llm.py b/api/core/model_runtime/model_providers/stepfun/llm/llm.py new file mode 100644 index 0000000000..6f6ffc8faa --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/llm.py @@ -0,0 +1,328 @@ +import json +from collections.abc import Generator +from typing import Optional, Union, cast + +import requests + +from core.model_runtime.entities.common_entities import I18nObject +from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta +from core.model_runtime.entities.message_entities import ( + AssistantPromptMessage, + ImagePromptMessageContent, + PromptMessage, + PromptMessageContent, + PromptMessageContentType, + PromptMessageTool, + SystemPromptMessage, + ToolPromptMessage, + UserPromptMessage, +) +from core.model_runtime.entities.model_entities import ( + AIModelEntity, + FetchFrom, + ModelFeature, + ModelPropertyKey, + ModelType, + ParameterRule, + ParameterType, +) +from core.model_runtime.model_providers.openai_api_compatible.llm.llm import OAIAPICompatLargeLanguageModel + + +class StepfunLargeLanguageModel(OAIAPICompatLargeLanguageModel): + def _invoke(self, model: str, credentials: dict, + prompt_messages: list[PromptMessage], model_parameters: dict, + tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, + stream: bool = True, user: Optional[str] = None) \ + -> Union[LLMResult, Generator]: + self._add_custom_parameters(credentials) + self._add_function_call(model, credentials) + user = user[:32] if user else None + return super()._invoke(model, credentials, prompt_messages, model_parameters, tools, stop, stream, user) + + def validate_credentials(self, model: str, credentials: dict) -> None: + self._add_custom_parameters(credentials) + super().validate_credentials(model, credentials) + + def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None: + return AIModelEntity( + model=model, + label=I18nObject(en_US=model, zh_Hans=model), + model_type=ModelType.LLM, + features=[ModelFeature.TOOL_CALL, ModelFeature.MULTI_TOOL_CALL, ModelFeature.STREAM_TOOL_CALL] + if credentials.get('function_calling_type') == 'tool_call' + else [], + fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, + model_properties={ + ModelPropertyKey.CONTEXT_SIZE: int(credentials.get('context_size', 8000)), + ModelPropertyKey.MODE: LLMMode.CHAT.value, + }, + parameter_rules=[ + ParameterRule( + name='temperature', + use_template='temperature', + label=I18nObject(en_US='Temperature', zh_Hans='温度'), + type=ParameterType.FLOAT, + ), + ParameterRule( + name='max_tokens', + use_template='max_tokens', + default=512, + min=1, + max=int(credentials.get('max_tokens', 1024)), + label=I18nObject(en_US='Max Tokens', zh_Hans='最大标记'), + type=ParameterType.INT, + ), + ParameterRule( + name='top_p', + use_template='top_p', + label=I18nObject(en_US='Top P', zh_Hans='Top P'), + type=ParameterType.FLOAT, + ), + ] + ) + + def _add_custom_parameters(self, credentials: dict) -> None: + credentials['mode'] = 'chat' + credentials['endpoint_url'] = 'https://api.stepfun.com/v1' + + def _add_function_call(self, model: str, credentials: dict) -> None: + model_schema = self.get_model_schema(model, credentials) + if model_schema and { + ModelFeature.TOOL_CALL, ModelFeature.MULTI_TOOL_CALL + }.intersection(model_schema.features or []): + credentials['function_calling_type'] = 'tool_call' + + def _convert_prompt_message_to_dict(self, message: PromptMessage,credentials: Optional[dict] = None) -> dict: + """ + Convert PromptMessage to dict for OpenAI API format + """ + if isinstance(message, UserPromptMessage): + message = cast(UserPromptMessage, message) + if isinstance(message.content, str): + message_dict = {"role": "user", "content": message.content} + else: + sub_messages = [] + for message_content in message.content: + if message_content.type == PromptMessageContentType.TEXT: + message_content = cast(PromptMessageContent, message_content) + sub_message_dict = { + "type": "text", + "text": message_content.data + } + sub_messages.append(sub_message_dict) + elif message_content.type == PromptMessageContentType.IMAGE: + message_content = cast(ImagePromptMessageContent, message_content) + sub_message_dict = { + "type": "image_url", + "image_url": { + "url": message_content.data, + } + } + sub_messages.append(sub_message_dict) + message_dict = {"role": "user", "content": sub_messages} + elif isinstance(message, AssistantPromptMessage): + message = cast(AssistantPromptMessage, message) + message_dict = {"role": "assistant", "content": message.content} + if message.tool_calls: + message_dict["tool_calls"] = [] + for function_call in message.tool_calls: + message_dict["tool_calls"].append({ + "id": function_call.id, + "type": function_call.type, + "function": { + "name": function_call.function.name, + "arguments": function_call.function.arguments + } + }) + elif isinstance(message, ToolPromptMessage): + message = cast(ToolPromptMessage, message) + message_dict = {"role": "tool", "content": message.content, "tool_call_id": message.tool_call_id} + elif isinstance(message, SystemPromptMessage): + message = cast(SystemPromptMessage, message) + message_dict = {"role": "system", "content": message.content} + else: + raise ValueError(f"Got unknown type {message}") + + if message.name: + message_dict["name"] = message.name + + return message_dict + + def _extract_response_tool_calls(self, response_tool_calls: list[dict]) -> list[AssistantPromptMessage.ToolCall]: + """ + Extract tool calls from response + + :param response_tool_calls: response tool calls + :return: list of tool calls + """ + tool_calls = [] + if response_tool_calls: + for response_tool_call in response_tool_calls: + function = AssistantPromptMessage.ToolCall.ToolCallFunction( + name=response_tool_call["function"]["name"] if response_tool_call.get("function", {}).get("name") else "", + arguments=response_tool_call["function"]["arguments"] if response_tool_call.get("function", {}).get("arguments") else "" + ) + + tool_call = AssistantPromptMessage.ToolCall( + id=response_tool_call["id"] if response_tool_call.get("id") else "", + type=response_tool_call["type"] if response_tool_call.get("type") else "", + function=function + ) + tool_calls.append(tool_call) + + return tool_calls + + def _handle_generate_stream_response(self, model: str, credentials: dict, response: requests.Response, + prompt_messages: list[PromptMessage]) -> Generator: + """ + Handle llm stream response + + :param model: model name + :param credentials: model credentials + :param response: streamed response + :param prompt_messages: prompt messages + :return: llm response chunk generator + """ + full_assistant_content = '' + chunk_index = 0 + + def create_final_llm_result_chunk(index: int, message: AssistantPromptMessage, finish_reason: str) \ + -> LLMResultChunk: + # calculate num tokens + prompt_tokens = self._num_tokens_from_string(model, prompt_messages[0].content) + completion_tokens = self._num_tokens_from_string(model, full_assistant_content) + + # transform usage + usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens) + + return LLMResultChunk( + model=model, + prompt_messages=prompt_messages, + delta=LLMResultChunkDelta( + index=index, + message=message, + finish_reason=finish_reason, + usage=usage + ) + ) + + tools_calls: list[AssistantPromptMessage.ToolCall] = [] + finish_reason = "Unknown" + + def increase_tool_call(new_tool_calls: list[AssistantPromptMessage.ToolCall]): + def get_tool_call(tool_name: str): + if not tool_name: + return tools_calls[-1] + + tool_call = next((tool_call for tool_call in tools_calls if tool_call.function.name == tool_name), None) + if tool_call is None: + tool_call = AssistantPromptMessage.ToolCall( + id='', + type='', + function=AssistantPromptMessage.ToolCall.ToolCallFunction(name=tool_name, arguments="") + ) + tools_calls.append(tool_call) + + return tool_call + + for new_tool_call in new_tool_calls: + # get tool call + tool_call = get_tool_call(new_tool_call.function.name) + # update tool call + if new_tool_call.id: + tool_call.id = new_tool_call.id + if new_tool_call.type: + tool_call.type = new_tool_call.type + if new_tool_call.function.name: + tool_call.function.name = new_tool_call.function.name + if new_tool_call.function.arguments: + tool_call.function.arguments += new_tool_call.function.arguments + + for chunk in response.iter_lines(decode_unicode=True, delimiter="\n\n"): + if chunk: + # ignore sse comments + if chunk.startswith(':'): + continue + decoded_chunk = chunk.strip().lstrip('data: ').lstrip() + chunk_json = None + try: + chunk_json = json.loads(decoded_chunk) + # stream ended + except json.JSONDecodeError as e: + yield create_final_llm_result_chunk( + index=chunk_index + 1, + message=AssistantPromptMessage(content=""), + finish_reason="Non-JSON encountered." + ) + break + if not chunk_json or len(chunk_json['choices']) == 0: + continue + + choice = chunk_json['choices'][0] + finish_reason = chunk_json['choices'][0].get('finish_reason') + chunk_index += 1 + + if 'delta' in choice: + delta = choice['delta'] + delta_content = delta.get('content') + + assistant_message_tool_calls = delta.get('tool_calls', None) + # assistant_message_function_call = delta.delta.function_call + + # extract tool calls from response + if assistant_message_tool_calls: + tool_calls = self._extract_response_tool_calls(assistant_message_tool_calls) + increase_tool_call(tool_calls) + + if delta_content is None or delta_content == '': + continue + + # transform assistant message to prompt message + assistant_prompt_message = AssistantPromptMessage( + content=delta_content, + tool_calls=tool_calls if assistant_message_tool_calls else [] + ) + + full_assistant_content += delta_content + elif 'text' in choice: + choice_text = choice.get('text', '') + if choice_text == '': + continue + + # transform assistant message to prompt message + assistant_prompt_message = AssistantPromptMessage(content=choice_text) + full_assistant_content += choice_text + else: + continue + + # check payload indicator for completion + yield LLMResultChunk( + model=model, + prompt_messages=prompt_messages, + delta=LLMResultChunkDelta( + index=chunk_index, + message=assistant_prompt_message, + ) + ) + + chunk_index += 1 + + if tools_calls: + yield LLMResultChunk( + model=model, + prompt_messages=prompt_messages, + delta=LLMResultChunkDelta( + index=chunk_index, + message=AssistantPromptMessage( + tool_calls=tools_calls, + content="" + ), + ) + ) + + yield create_final_llm_result_chunk( + index=chunk_index, + message=AssistantPromptMessage(content=""), + finish_reason=finish_reason + ) \ No newline at end of file diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1-128k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1-128k.yaml new file mode 100644 index 0000000000..13f7b7fd26 --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1-128k.yaml @@ -0,0 +1,25 @@ +model: step-1-128k +label: + zh_Hans: step-1-128k + en_US: step-1-128k +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 128000 +pricing: + input: '0.04' + output: '0.20' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1-256k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1-256k.yaml new file mode 100644 index 0000000000..f80ec9851c --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1-256k.yaml @@ -0,0 +1,25 @@ +model: step-1-256k +label: + zh_Hans: step-1-256k + en_US: step-1-256k +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 256000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 256000 +pricing: + input: '0.095' + output: '0.300' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1-32k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1-32k.yaml new file mode 100644 index 0000000000..96132d14a8 --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1-32k.yaml @@ -0,0 +1,28 @@ +model: step-1-32k +label: + zh_Hans: step-1-32k + en_US: step-1-32k +model_type: llm +features: + - agent-thought + - tool-call + - multi-tool-call + - stream-tool-call +model_properties: + mode: chat + context_size: 32000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 32000 +pricing: + input: '0.015' + output: '0.070' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1-8k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1-8k.yaml new file mode 100644 index 0000000000..4a4ba8d178 --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1-8k.yaml @@ -0,0 +1,28 @@ +model: step-1-8k +label: + zh_Hans: step-1-8k + en_US: step-1-8k +model_type: llm +features: + - agent-thought + - tool-call + - multi-tool-call + - stream-tool-call +model_properties: + mode: chat + context_size: 8000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 8000 +pricing: + input: '0.005' + output: '0.020' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1v-32k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1v-32k.yaml new file mode 100644 index 0000000000..f878ee3e56 --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1v-32k.yaml @@ -0,0 +1,25 @@ +model: step-1v-32k +label: + zh_Hans: step-1v-32k + en_US: step-1v-32k +model_type: llm +features: + - vision +model_properties: + mode: chat + context_size: 32000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 32000 +pricing: + input: '0.015' + output: '0.070' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1v-8k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1v-8k.yaml new file mode 100644 index 0000000000..6c3cb61d2c --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1v-8k.yaml @@ -0,0 +1,25 @@ +model: step-1v-8k +label: + zh_Hans: step-1v-8k + en_US: step-1v-8k +model_type: llm +features: + - vision +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 8192 +pricing: + input: '0.005' + output: '0.020' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/stepfun.py b/api/core/model_runtime/model_providers/stepfun/stepfun.py new file mode 100644 index 0000000000..50b17392b5 --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/stepfun.py @@ -0,0 +1,30 @@ +import logging + +from core.model_runtime.entities.model_entities import ModelType +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.model_provider import ModelProvider + +logger = logging.getLogger(__name__) + + +class StepfunProvider(ModelProvider): + + def validate_provider_credentials(self, credentials: dict) -> None: + """ + Validate provider credentials + if validate failed, raise exception + + :param credentials: provider credentials, credentials form defined in `provider_credential_schema`. + """ + try: + model_instance = self.get_model_instance(ModelType.LLM) + + model_instance.validate_credentials( + model='step-1-8k', + credentials=credentials + ) + except CredentialsValidateFailedError as ex: + raise ex + except Exception as ex: + logger.exception(f'{self.get_provider_schema().provider} credentials validate failed') + raise ex diff --git a/api/core/model_runtime/model_providers/stepfun/stepfun.yaml b/api/core/model_runtime/model_providers/stepfun/stepfun.yaml new file mode 100644 index 0000000000..ccc8455adc --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/stepfun.yaml @@ -0,0 +1,81 @@ +provider: stepfun +label: + zh_Hans: 阶跃星辰 + en_US: Stepfun +description: + en_US: Models provided by stepfun, such as step-1-8k, step-1-32k、step-1v-8k、step-1v-32k, step-1-128k and step-1-256k + zh_Hans: 阶跃星辰提供的模型，例如 step-1-8k、step-1-32k、step-1v-8k、step-1v-32k、step-1-128k 和 step-1-256k。 +icon_small: + en_US: icon_s_en.png +icon_large: + en_US: icon_l_en.png +background: "#FFFFFF" +help: + title: + en_US: Get your API Key from stepfun + zh_Hans: 从 stepfun 获取 API Key + url: + en_US: https://platform.stepfun.com/interface-key +supported_model_types: + - llm +configurate_methods: + - predefined-model + - customizable-model +provider_credential_schema: + credential_form_schemas: + - variable: api_key + label: + en_US: API Key + type: secret-input + required: true + placeholder: + zh_Hans: 在此输入您的 API Key + en_US: Enter your API Key +model_credential_schema: + model: + label: + en_US: Model Name + zh_Hans: 模型名称 + placeholder: + en_US: Enter your model name + zh_Hans: 输入模型名称 + credential_form_schemas: + - variable: api_key + label: + en_US: API Key + type: secret-input + required: true + placeholder: + zh_Hans: 在此输入您的 API Key + en_US: Enter your API Key + - variable: context_size + label: + zh_Hans: 模型上下文长度 + en_US: Model context size + required: true + type: text-input + default: '8192' + placeholder: + zh_Hans: 在此输入您的模型上下文长度 + en_US: Enter your Model context size + - variable: max_tokens + label: + zh_Hans: 最大 token 上限 + en_US: Upper bound for max tokens + default: '8192' + type: text-input + - variable: function_calling_type + label: + en_US: Function calling + type: select + required: false + default: no_call + options: + - value: no_call + label: + en_US: Not supported + zh_Hans: 不支持 + - value: tool_call + label: + en_US: Tool Call + zh_Hans: Tool Call diff --git a/api/core/model_runtime/model_providers/tencent/__init__.py b/api/core/model_runtime/model_providers/tencent/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/model_runtime/model_providers/tencent/_assets/icon_l_en.svg b/api/core/model_runtime/model_providers/tencent/_assets/icon_l_en.svg new file mode 100644 index 0000000000..63c7c8f988 --- /dev/null +++ b/api/core/model_runtime/model_providers/tencent/_assets/icon_l_en.svg @@ -0,0 +1,13 @@ + + + + + tencent-cloud + + + \ No newline at end of file diff --git a/api/core/model_runtime/model_providers/tencent/_assets/icon_l_zh.svg b/api/core/model_runtime/model_providers/tencent/_assets/icon_l_zh.svg new file mode 100644 index 0000000000..63c7c8f988 --- /dev/null +++ b/api/core/model_runtime/model_providers/tencent/_assets/icon_l_zh.svg @@ -0,0 +1,13 @@ + + + + + tencent-cloud + + + \ No newline at end of file diff --git a/api/core/model_runtime/model_providers/tencent/_assets/icon_s_en.svg b/api/core/model_runtime/model_providers/tencent/_assets/icon_s_en.svg new file mode 100644 index 0000000000..a3299b9201 --- /dev/null +++ b/api/core/model_runtime/model_providers/tencent/_assets/icon_s_en.svg @@ -0,0 +1,11 @@ + + + + + tencent-cloud + + \ No newline at end of file diff --git a/api/core/model_runtime/model_providers/tencent/speech2text/__init__.py b/api/core/model_runtime/model_providers/tencent/speech2text/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/model_runtime/model_providers/tencent/speech2text/flash_recognizer.py b/api/core/model_runtime/model_providers/tencent/speech2text/flash_recognizer.py new file mode 100644 index 0000000000..c3e3b7c258 --- /dev/null +++ b/api/core/model_runtime/model_providers/tencent/speech2text/flash_recognizer.py @@ -0,0 +1,156 @@ +import base64 +import hashlib +import hmac +import time + +import requests + + +class Credential: + def __init__(self, secret_id, secret_key): + self.secret_id = secret_id + self.secret_key = secret_key + + +class FlashRecognitionRequest: + def __init__(self, voice_format="mp3", engine_type="16k_zh"): + self.engine_type = engine_type + self.speaker_diarization = 0 + self.hotword_id = "" + self.customization_id = "" + self.filter_dirty = 0 + self.filter_modal = 0 + self.filter_punc = 0 + self.convert_num_mode = 1 + self.word_info = 0 + self.voice_format = voice_format + self.first_channel_only = 1 + self.reinforce_hotword = 0 + self.sentence_max_length = 0 + + def set_first_channel_only(self, first_channel_only): + self.first_channel_only = first_channel_only + + def set_speaker_diarization(self, speaker_diarization): + self.speaker_diarization = speaker_diarization + + def set_filter_dirty(self, filter_dirty): + self.filter_dirty = filter_dirty + + def set_filter_modal(self, filter_modal): + self.filter_modal = filter_modal + + def set_filter_punc(self, filter_punc): + self.filter_punc = filter_punc + + def set_convert_num_mode(self, convert_num_mode): + self.convert_num_mode = convert_num_mode + + def set_word_info(self, word_info): + self.word_info = word_info + + def set_hotword_id(self, hotword_id): + self.hotword_id = hotword_id + + def set_customization_id(self, customization_id): + self.customization_id = customization_id + + def set_voice_format(self, voice_format): + self.voice_format = voice_format + + def set_sentence_max_length(self, sentence_max_length): + self.sentence_max_length = sentence_max_length + + def set_reinforce_hotword(self, reinforce_hotword): + self.reinforce_hotword = reinforce_hotword + + +class FlashRecognizer: + """ + reponse: + request_id string + status Integer + message String + audio_duration Integer + flash_result Result Array + + Result: + text String + channel_id Integer + sentence_list Sentence Array + + Sentence: + text String + start_time Integer + end_time Integer + speaker_id Integer + word_list Word Array + + Word: + word String + start_time Integer + end_time Integer + stable_flag： Integer + """ + + def __init__(self, appid, credential): + self.credential = credential + self.appid = appid + + def _format_sign_string(self, param): + signstr = "POSTasr.cloud.tencent.com/asr/flash/v1/" + for t in param: + if 'appid' in t: + signstr += str(t[1]) + break + signstr += "?" + for x in param: + tmp = x + if 'appid' in x: + continue + for t in tmp: + signstr += str(t) + signstr += "=" + signstr = signstr[:-1] + signstr += "&" + signstr = signstr[:-1] + return signstr + + def _build_header(self): + header = {"Host": "asr.cloud.tencent.com"} + return header + + def _sign(self, signstr, secret_key): + hmacstr = hmac.new(secret_key.encode('utf-8'), + signstr.encode('utf-8'), hashlib.sha1).digest() + s = base64.b64encode(hmacstr) + s = s.decode('utf-8') + return s + + def _build_req_with_signature(self, secret_key, params, header): + query = sorted(params.items(), key=lambda d: d[0]) + signstr = self._format_sign_string(query) + signature = self._sign(signstr, secret_key) + header["Authorization"] = signature + requrl = "https://" + requrl += signstr[4::] + return requrl + + def _create_query_arr(self, req): + return { + 'appid': self.appid, 'secretid': self.credential.secret_id, 'timestamp': str(int(time.time())), + 'engine_type': req.engine_type, 'voice_format': req.voice_format, + 'speaker_diarization': req.speaker_diarization, 'hotword_id': req.hotword_id, + 'customization_id': req.customization_id, 'filter_dirty': req.filter_dirty, + 'filter_modal': req.filter_modal, 'filter_punc': req.filter_punc, + 'convert_num_mode': req.convert_num_mode, 'word_info': req.word_info, + 'first_channel_only': req.first_channel_only, 'reinforce_hotword': req.reinforce_hotword, + 'sentence_max_length': req.sentence_max_length + } + + def recognize(self, req, data): + header = self._build_header() + query_arr = self._create_query_arr(req) + req_url = self._build_req_with_signature(self.credential.secret_key, query_arr, header) + r = requests.post(req_url, headers=header, data=data) + return r.text diff --git a/api/core/model_runtime/model_providers/tencent/speech2text/speech2text.py b/api/core/model_runtime/model_providers/tencent/speech2text/speech2text.py new file mode 100644 index 0000000000..00ec5aa9c8 --- /dev/null +++ b/api/core/model_runtime/model_providers/tencent/speech2text/speech2text.py @@ -0,0 +1,92 @@ +import json +from typing import IO, Optional + +import requests + +from core.model_runtime.errors.invoke import ( + InvokeAuthorizationError, + InvokeConnectionError, + InvokeError, +) +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.speech2text_model import Speech2TextModel +from core.model_runtime.model_providers.tencent.speech2text.flash_recognizer import ( + Credential, + FlashRecognitionRequest, + FlashRecognizer, +) + + +class TencentSpeech2TextModel(Speech2TextModel): + def _invoke(self, model: str, credentials: dict, + file: IO[bytes], user: Optional[str] = None) \ + -> str: + """ + Invoke speech2text model + + :param model: model name + :param credentials: model credentials + :param file: audio file + :param user: unique user id + :return: text for given audio file + """ + return self._speech2text_invoke(model, credentials, file) + + def validate_credentials(self, model: str, credentials: dict) -> None: + """ + Validate model credentials + + :param model: model name + :param credentials: model credentials + :return: + """ + try: + audio_file_path = self._get_demo_file_path() + + with open(audio_file_path, 'rb') as audio_file: + self._speech2text_invoke(model, credentials, audio_file) + except Exception as ex: + raise CredentialsValidateFailedError(str(ex)) + + def _speech2text_invoke(self, model: str, credentials: dict, file: IO[bytes]) -> str: + """ + Invoke speech2text model + + :param model: model name + :param credentials: model credentials + :param file: audio file + :return: text for given audio file + """ + app_id = credentials["app_id"] + secret_id = credentials["secret_id"] + secret_key = credentials["secret_key"] + voice_format = file.voice_format if hasattr(file, "voice_format") else "mp3" + tencent_voice_recognizer = FlashRecognizer(app_id, Credential(secret_id, secret_key)) + resp = tencent_voice_recognizer.recognize(FlashRecognitionRequest(voice_format), file) + resp = json.loads(resp) + code = resp["code"] + message = resp["message"] + if code == 4002: + raise CredentialsValidateFailedError(str(message)) + elif code != 0: + return f"Tencent ASR Recognition failed with code {code} and message {message}" + return "\n".join(item["text"] for item in resp["flash_result"]) + + @property + def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + """ + Map model invoke error to unified error + The key is the error type thrown to the caller + The value is the error type thrown by the model, + which needs to be converted into a unified error type for the caller. + + :return: Invoke error mapping + """ + return { + InvokeConnectionError: [ + requests.exceptions.ConnectionError + ], + InvokeAuthorizationError: [ + CredentialsValidateFailedError + ] + } diff --git a/api/core/model_runtime/model_providers/tencent/speech2text/tencent.yaml b/api/core/model_runtime/model_providers/tencent/speech2text/tencent.yaml new file mode 100644 index 0000000000..618d19ac7c --- /dev/null +++ b/api/core/model_runtime/model_providers/tencent/speech2text/tencent.yaml @@ -0,0 +1,5 @@ +model: tencent +model_type: speech2text +model_properties: + file_upload_limit: 25 + supported_file_extensions: flac,mp3,mp4,mpeg,mpga,m4a,ogg,wav,webm diff --git a/api/core/model_runtime/model_providers/tencent/tencent.py b/api/core/model_runtime/model_providers/tencent/tencent.py new file mode 100644 index 0000000000..dd9f90bb47 --- /dev/null +++ b/api/core/model_runtime/model_providers/tencent/tencent.py @@ -0,0 +1,29 @@ +import logging + +from core.model_runtime.entities.model_entities import ModelType +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.model_provider import ModelProvider + +logger = logging.getLogger(__name__) + + +class TencentProvider(ModelProvider): + def validate_provider_credentials(self, credentials: dict) -> None: + """ + Validate provider credentials + + if validate failed, raise exception + + :param credentials: provider credentials, credentials form defined in `provider_credential_schema`. + """ + try: + model_instance = self.get_model_instance(ModelType.SPEECH2TEXT) + model_instance.validate_credentials( + model='tencent', + credentials=credentials + ) + except CredentialsValidateFailedError as ex: + raise ex + except Exception as ex: + logger.exception(f'{self.get_provider_schema().provider} credentials validate failed') + raise ex diff --git a/api/core/model_runtime/model_providers/tencent/tencent.yaml b/api/core/model_runtime/model_providers/tencent/tencent.yaml new file mode 100644 index 0000000000..7d8d5a1866 --- /dev/null +++ b/api/core/model_runtime/model_providers/tencent/tencent.yaml @@ -0,0 +1,49 @@ +provider: tencent +label: + zh_Hans: 腾讯云 + en_US: Tencent +icon_small: + en_US: icon_s_en.svg +icon_large: + zh_Hans: icon_l_zh.svg + en_US: icon_l_en.svg +background: "#E5E7EB" +help: + title: + en_US: Get your API key from Tencent AI + zh_Hans: 从腾讯云获取 API Key + url: + en_US: https://cloud.tencent.com/product/asr +supported_model_types: + - speech2text +configurate_methods: + - predefined-model +provider_credential_schema: + credential_form_schemas: + - variable: app_id + label: + zh_Hans: APPID + en_US: APPID + type: text-input + required: true + placeholder: + zh_Hans: 在此输入您的腾讯语音识别服务的 APPID + en_US: Enter the APPID of your Tencent Cloud ASR service + - variable: secret_id + label: + zh_Hans: SecretId + en_US: SecretId + type: secret-input + required: true + placeholder: + zh_Hans: 在此输入您的腾讯语音识别服务的 SecretId + en_US: Enter the SecretId of your Tencent Cloud ASR service + - variable: secret_key + label: + zh_Hans: SecretKey + en_US: SecretKey + type: secret-input + required: true + placeholder: + zh_Hans: 在此输入您的腾讯语音识别服务的 SecretKey + en_US: Enter the SecretKey of your Tencent Cloud ASR service diff --git a/api/core/model_runtime/model_providers/tongyi/llm/llm.py b/api/core/model_runtime/model_providers/tongyi/llm/llm.py index 41d8f37aaf..6f768131fb 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/llm.py +++ b/api/core/model_runtime/model_providers/tongyi/llm/llm.py @@ -262,6 +262,10 @@ You should also complete the text started with ``` but not tell ``` directly. :param prompt_messages: prompt messages :return: llm response """ + if response.status_code != 200 and response.status_code != HTTPStatus.OK: + raise ServiceUnavailableError( + response.message + ) # transform assistant message to prompt message assistant_prompt_message = AssistantPromptMessage( content=response.output.choices[0].message.content, @@ -421,7 +425,7 @@ You should also complete the text started with ``` but not tell ``` directly. raise ValueError(f"Got unknown type {message}") return message_text - + def _convert_messages_to_prompt(self, messages: list[PromptMessage]) -> str: """ Format a list of messages into a full prompt for the Anthropic model @@ -496,6 +500,8 @@ You should also complete the text started with ``` but not tell ``` directly. tongyi_messages.append({ 'role': 'assistant', 'content': content if not rich_content else [{"text": content}], + 'tool_calls': [tool_call.model_dump() for tool_call in + prompt_message.tool_calls] if prompt_message.tool_calls else None }) elif isinstance(prompt_message, ToolPromptMessage): tongyi_messages.append({ diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-3.5-8k-0205.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-3.5-8k-0205.yaml index 34f73dccbb..b308abcb32 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie-3.5-8k-0205.yaml +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-3.5-8k-0205.yaml @@ -35,3 +35,4 @@ parameter_rules: zh_Hans: 禁用模型自行进行外部搜索。 en_US: Disable the model to perform external search. required: false +deprecated: true diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml index 50c82564f1..d23ae0dc48 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml @@ -1,4 +1,4 @@ -model: ernie-4.0-8k-Latest +model: ernie-4.0-8k-latest label: en_US: Ernie-4.0-8K-Latest model_type: llm diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k-preview b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k-preview.yaml similarity index 100% rename from api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k-preview rename to api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k-preview.yaml diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k.yaml new file mode 100644 index 0000000000..2887a510d0 --- /dev/null +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k.yaml @@ -0,0 +1,40 @@ +model: ernie-4.0-turbo-8k +label: + en_US: Ernie-4.0-turbo-8K +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + min: 0.1 + max: 1.0 + default: 0.8 + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 2 + max: 2048 + - name: presence_penalty + use_template: presence_penalty + default: 1.0 + min: 1.0 + max: 2.0 + - name: frequency_penalty + use_template: frequency_penalty + - name: response_format + use_template: response_format + - name: disable_search + label: + zh_Hans: 禁用搜索 + en_US: Disable Search + type: boolean + help: + zh_Hans: 禁用模型自行进行外部搜索。 + en_US: Disable the model to perform external search. + required: false diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k-0321.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k-0321.yaml index 52e1dc832d..74451ff9e3 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k-0321.yaml +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k-0321.yaml @@ -28,3 +28,4 @@ parameter_rules: default: 1.0 min: 1.0 max: 2.0 +deprecated: true diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k.yaml new file mode 100644 index 0000000000..4b11b3e895 --- /dev/null +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k.yaml @@ -0,0 +1,30 @@ +model: ernie-character-8k-0321 +label: + en_US: ERNIE-Character-8K +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + min: 0.1 + max: 1.0 + default: 0.95 + - name: top_p + use_template: top_p + min: 0 + max: 1.0 + default: 0.7 + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 2 + max: 1024 + - name: presence_penalty + use_template: presence_penalty + default: 1.0 + min: 1.0 + max: 2.0 diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0308.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0308.yaml index 78325c1d64..97ecb03f87 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0308.yaml +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0308.yaml @@ -28,3 +28,4 @@ parameter_rules: default: 1.0 min: 1.0 max: 2.0 +deprecated: true diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0922.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0922.yaml index ebb47417cc..7410ce51df 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0922.yaml +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0922.yaml @@ -28,3 +28,4 @@ parameter_rules: default: 1.0 min: 1.0 max: 2.0 +deprecated: true diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py b/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py index 9aeab04cd2..bc7f29cf6e 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py @@ -97,6 +97,7 @@ class BaiduAccessToken: baidu_access_tokens_lock.release() return token + class ErnieMessage: class Role(Enum): USER = 'user' @@ -137,7 +138,9 @@ class ErnieBotModel: 'ernie-speed-appbuilder': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ai_apaas', 'ernie-lite-8k-0922': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/eb-instant', 'ernie-lite-8k-0308': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-lite-8k', + 'ernie-character-8k': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-char-8k', 'ernie-character-8k-0321': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-char-8k', + 'ernie-4.0-tutbo-8k': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-4.0-turbo-8k', 'ernie-4.0-tutbo-8k-preview': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-4.0-turbo-8k-preview', } @@ -149,7 +152,8 @@ class ErnieBotModel: 'ernie-3.5-8k-1222', 'ernie-3.5-4k-0205', 'ernie-3.5-128k', - 'ernie-4.0-8k' + 'ernie-4.0-8k', + 'ernie-4.0-turbo-8k', 'ernie-4.0-turbo-8k-preview' ] diff --git a/api/core/model_runtime/model_providers/xinference/llm/llm.py b/api/core/model_runtime/model_providers/xinference/llm/llm.py index 0ef63f8e23..988bb0ce44 100644 --- a/api/core/model_runtime/model_providers/xinference/llm/llm.py +++ b/api/core/model_runtime/model_providers/xinference/llm/llm.py @@ -453,9 +453,11 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): if credentials['server_url'].endswith('/'): credentials['server_url'] = credentials['server_url'][:-1] + api_key = credentials.get('api_key') or "abc" + client = OpenAI( base_url=f'{credentials["server_url"]}/v1', - api_key='abc', + api_key=api_key, max_retries=3, timeout=60, ) diff --git a/api/core/model_runtime/model_providers/xinference/rerank/rerank.py b/api/core/model_runtime/model_providers/xinference/rerank/rerank.py index 17b85862c9..649898f47a 100644 --- a/api/core/model_runtime/model_providers/xinference/rerank/rerank.py +++ b/api/core/model_runtime/model_providers/xinference/rerank/rerank.py @@ -44,15 +44,23 @@ class XinferenceRerankModel(RerankModel): docs=[] ) - if credentials['server_url'].endswith('/'): - credentials['server_url'] = credentials['server_url'][:-1] + server_url = credentials['server_url'] + model_uid = credentials['model_uid'] + api_key = credentials.get('api_key') + if server_url.endswith('/'): + server_url = server_url[:-1] + auth_headers = {'Authorization': f'Bearer {api_key}'} if api_key else {} + + try: + handle = RESTfulRerankModelHandle(model_uid, server_url, auth_headers) + response = handle.rerank( + documents=docs, + query=query, + top_n=top_n, + ) + except RuntimeError as e: + raise InvokeServerUnavailableError(str(e)) - handle = RESTfulRerankModelHandle(credentials['model_uid'], credentials['server_url'],auth_headers={}) - response = handle.rerank( - documents=docs, - query=query, - top_n=top_n, - ) rerank_documents = [] for idx, result in enumerate(response['results']): @@ -102,7 +110,7 @@ class XinferenceRerankModel(RerankModel): if not isinstance(xinference_client, RESTfulRerankModelHandle): raise InvokeBadRequestError( 'please check model type, the model you want to invoke is not a rerank model') - + self.invoke( model=model, credentials=credentials, diff --git a/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py b/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py index f60d8d3443..9ee3621317 100644 --- a/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py +++ b/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py @@ -99,9 +99,9 @@ class XinferenceSpeech2TextModel(Speech2TextModel): } def _speech2text_invoke( - self, - model: str, - credentials: dict, + self, + model: str, + credentials: dict, file: IO[bytes], language: Optional[str] = None, prompt: Optional[str] = None, @@ -121,17 +121,24 @@ class XinferenceSpeech2TextModel(Speech2TextModel): :param temperature: The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output mor e random,while lower values like 0.2 will make it more focused and deterministic.If set to 0, the model wi ll use log probability to automatically increase the temperature until certain thresholds are hit. :return: text for given audio file """ - if credentials['server_url'].endswith('/'): - credentials['server_url'] = credentials['server_url'][:-1] + server_url = credentials['server_url'] + model_uid = credentials['model_uid'] + api_key = credentials.get('api_key') + if server_url.endswith('/'): + server_url = server_url[:-1] + auth_headers = {'Authorization': f'Bearer {api_key}'} if api_key else {} - handle = RESTfulAudioModelHandle(credentials['model_uid'],credentials['server_url'],auth_headers={}) - response = handle.transcriptions( - audio=file, - language = language, - prompt = prompt, - response_format = response_format, - temperature = temperature - ) + try: + handle = RESTfulAudioModelHandle(model_uid, server_url, auth_headers) + response = handle.transcriptions( + audio=file, + language=language, + prompt=prompt, + response_format=response_format, + temperature=temperature + ) + except RuntimeError as e: + raise InvokeServerUnavailableError(str(e)) return response["text"] diff --git a/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py index e8429cecd4..11f1e29cb3 100644 --- a/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py @@ -43,16 +43,17 @@ class XinferenceTextEmbeddingModel(TextEmbeddingModel): """ server_url = credentials['server_url'] model_uid = credentials['model_uid'] - + api_key = credentials.get('api_key') if server_url.endswith('/'): server_url = server_url[:-1] + auth_headers = {'Authorization': f'Bearer {api_key}'} if api_key else {} try: - handle = RESTfulEmbeddingModelHandle(model_uid, server_url, auth_headers={}) + handle = RESTfulEmbeddingModelHandle(model_uid, server_url, auth_headers) embeddings = handle.create_embedding(input=texts) except RuntimeError as e: - raise InvokeServerUnavailableError(e) - + raise InvokeServerUnavailableError(str(e)) + """ for convenience, the response json is like: class Embedding(TypedDict): @@ -106,7 +107,7 @@ class XinferenceTextEmbeddingModel(TextEmbeddingModel): try: if "/" in credentials['model_uid'] or "?" in credentials['model_uid'] or "#" in credentials['model_uid']: raise CredentialsValidateFailedError("model_uid should not contain /, ?, or #") - + server_url = credentials['server_url'] model_uid = credentials['model_uid'] extra_args = XinferenceHelper.get_xinference_extra_parameter(server_url=server_url, model_uid=model_uid) @@ -117,7 +118,7 @@ class XinferenceTextEmbeddingModel(TextEmbeddingModel): server_url = server_url[:-1] client = Client(base_url=server_url) - + try: handle = client.get_model(model_uid=model_uid) except RuntimeError as e: @@ -151,7 +152,7 @@ class XinferenceTextEmbeddingModel(TextEmbeddingModel): KeyError ] } - + def _calc_response_usage(self, model: str, credentials: dict, tokens: int) -> EmbeddingUsage: """ Calculate response usage @@ -186,7 +187,7 @@ class XinferenceTextEmbeddingModel(TextEmbeddingModel): """ used to define customizable model schema """ - + entity = AIModelEntity( model=model, label=I18nObject( diff --git a/api/core/model_runtime/model_providers/xinference/xinference.yaml b/api/core/model_runtime/model_providers/xinference/xinference.yaml index 28ffc0389e..9496c66fdd 100644 --- a/api/core/model_runtime/model_providers/xinference/xinference.yaml +++ b/api/core/model_runtime/model_providers/xinference/xinference.yaml @@ -46,3 +46,12 @@ model_credential_schema: placeholder: zh_Hans: 在此输入您的Model UID en_US: Enter the model uid + - variable: api_key + label: + zh_Hans: API密钥 + en_US: API key + type: text-input + required: false + placeholder: + zh_Hans: 在此输入您的API密钥 + en_US: Enter the api key diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai.py index b72b334c54..c517d2dba5 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai.py @@ -20,7 +20,7 @@ class ZhipuaiProvider(ModelProvider): model_instance = self.get_model_instance(ModelType.LLM) model_instance.validate_credentials( - model='chatglm_turbo', + model='glm-4', credentials=credentials ) except CredentialsValidateFailedError as ex: diff --git a/api/core/moderation/output_moderation.py b/api/core/moderation/output_moderation.py index 3587466952..4fd08ca021 100644 --- a/api/core/moderation/output_moderation.py +++ b/api/core/moderation/output_moderation.py @@ -6,6 +6,7 @@ from typing import Any, Optional from flask import Flask, current_app from pydantic import BaseModel, ConfigDict +from configs import dify_config from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom from core.app.entities.queue_entities import QueueMessageReplaceEvent from core.moderation.base import ModerationAction, ModerationOutputsResult @@ -76,7 +77,7 @@ class OutputModeration(BaseModel): return final_output def start_thread(self) -> threading.Thread: - buffer_size = int(current_app.config.get('MODERATION_BUFFER_SIZE', self.DEFAULT_BUFFER_SIZE)) + buffer_size = int(dify_config.config.MODERATION_BUFFER_SIZE) thread = threading.Thread(target=self.worker, kwargs={ 'flask_app': current_app._get_current_object(), 'buffer_size': buffer_size if buffer_size > 0 else self.DEFAULT_BUFFER_SIZE diff --git a/api/core/ops/utils.py b/api/core/ops/utils.py index 2b12db0f48..3b2e04abb7 100644 --- a/api/core/ops/utils.py +++ b/api/core/ops/utils.py @@ -25,7 +25,6 @@ def measure_time(): yield timing_info finally: timing_info['end'] = datetime.now() - print(f"Execution time: {timing_info['end'] - timing_info['start']}") def replace_text_with_content(data): diff --git a/api/core/rag/data_post_processor/data_post_processor.py b/api/core/rag/data_post_processor/data_post_processor.py index a0f2947784..2ed6d74187 100644 --- a/api/core/rag/data_post_processor/data_post_processor.py +++ b/api/core/rag/data_post_processor/data_post_processor.py @@ -5,15 +5,20 @@ from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.errors.invoke import InvokeAuthorizationError from core.rag.data_post_processor.reorder import ReorderRunner from core.rag.models.document import Document -from core.rag.rerank.rerank import RerankRunner +from core.rag.rerank.constants.rerank_mode import RerankMode +from core.rag.rerank.entity.weight import KeywordSetting, VectorSetting, Weights +from core.rag.rerank.rerank_model import RerankModelRunner +from core.rag.rerank.weight_rerank import WeightRerankRunner class DataPostProcessor: """Interface for data post-processing document. """ - def __init__(self, tenant_id: str, reranking_model: dict, reorder_enabled: bool = False): - self.rerank_runner = self._get_rerank_runner(reranking_model, tenant_id) + def __init__(self, tenant_id: str, reranking_mode: str, + reranking_model: Optional[dict] = None, weights: Optional[dict] = None, + reorder_enabled: bool = False): + self.rerank_runner = self._get_rerank_runner(reranking_mode, tenant_id, reranking_model, weights) self.reorder_runner = self._get_reorder_runner(reorder_enabled) def invoke(self, query: str, documents: list[Document], score_threshold: Optional[float] = None, @@ -26,19 +31,37 @@ class DataPostProcessor: return documents - def _get_rerank_runner(self, reranking_model: dict, tenant_id: str) -> Optional[RerankRunner]: - if reranking_model: - try: - model_manager = ModelManager() - rerank_model_instance = model_manager.get_model_instance( - tenant_id=tenant_id, - provider=reranking_model['reranking_provider_name'], - model_type=ModelType.RERANK, - model=reranking_model['reranking_model_name'] + def _get_rerank_runner(self, reranking_mode: str, tenant_id: str, reranking_model: Optional[dict] = None, + weights: Optional[dict] = None) -> Optional[RerankModelRunner | WeightRerankRunner]: + if reranking_mode == RerankMode.WEIGHTED_SCORE.value and weights: + return WeightRerankRunner( + tenant_id, + Weights( + weight_type=weights['weight_type'], + vector_setting=VectorSetting( + vector_weight=weights['vector_setting']['vector_weight'], + embedding_provider_name=weights['vector_setting']['embedding_provider_name'], + embedding_model_name=weights['vector_setting']['embedding_model_name'], + ), + keyword_setting=KeywordSetting( + keyword_weight=weights['keyword_setting']['keyword_weight'], + ) ) - except InvokeAuthorizationError: - return None - return RerankRunner(rerank_model_instance) + ) + elif reranking_mode == RerankMode.RERANKING_MODEL.value: + if reranking_model: + try: + model_manager = ModelManager() + rerank_model_instance = model_manager.get_model_instance( + tenant_id=tenant_id, + provider=reranking_model['reranking_provider_name'], + model_type=ModelType.RERANK, + model=reranking_model['reranking_model_name'] + ) + except InvokeAuthorizationError: + return None + return RerankModelRunner(rerank_model_instance) + return None return None def _get_reorder_runner(self, reorder_enabled) -> Optional[ReorderRunner]: diff --git a/api/core/rag/datasource/keyword/jieba/jieba_keyword_table_handler.py b/api/core/rag/datasource/keyword/jieba/jieba_keyword_table_handler.py index 5f862b8d18..ad669ef515 100644 --- a/api/core/rag/datasource/keyword/jieba/jieba_keyword_table_handler.py +++ b/api/core/rag/datasource/keyword/jieba/jieba_keyword_table_handler.py @@ -1,4 +1,5 @@ import re +from typing import Optional import jieba from jieba.analyse import default_tfidf @@ -11,7 +12,7 @@ class JiebaKeywordTableHandler: def __init__(self): default_tfidf.stop_words = STOPWORDS - def extract_keywords(self, text: str, max_keywords_per_chunk: int = 10) -> set[str]: + def extract_keywords(self, text: str, max_keywords_per_chunk: Optional[int] = 10) -> set[str]: """Extract keywords with JIEBA tfidf.""" keywords = jieba.analyse.extract_tags( sentence=text, diff --git a/api/core/rag/datasource/keyword/keyword_base.py b/api/core/rag/datasource/keyword/keyword_base.py index 02838cb1bd..b77c6562b2 100644 --- a/api/core/rag/datasource/keyword/keyword_base.py +++ b/api/core/rag/datasource/keyword/keyword_base.py @@ -38,7 +38,7 @@ class BaseKeyword(ABC): raise NotImplementedError def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: - for text in texts: + for text in texts[:]: doc_id = text.metadata['doc_id'] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py index 623b7a3123..abbf4a35a4 100644 --- a/api/core/rag/datasource/retrieval_service.py +++ b/api/core/rag/datasource/retrieval_service.py @@ -6,12 +6,13 @@ from flask import Flask, current_app from core.rag.data_post_processor.data_post_processor import DataPostProcessor from core.rag.datasource.keyword.keyword_factory import Keyword from core.rag.datasource.vdb.vector_factory import Vector +from core.rag.rerank.constants.rerank_mode import RerankMode from core.rag.retrieval.retrival_methods import RetrievalMethod from extensions.ext_database import db from models.dataset import Dataset default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -26,13 +27,19 @@ class RetrievalService: @classmethod def retrieve(cls, retrival_method: str, dataset_id: str, query: str, - top_k: int, score_threshold: Optional[float] = .0, reranking_model: Optional[dict] = None): + top_k: int, score_threshold: Optional[float] = .0, + reranking_model: Optional[dict] = None, reranking_mode: Optional[str] = None, + weights: Optional[dict] = None): dataset = db.session.query(Dataset).filter( Dataset.id == dataset_id ).first() if not dataset or dataset.available_document_count == 0 or dataset.available_segment_count == 0: return [] all_documents = [] + keyword_search_documents = [] + embedding_search_documents = [] + full_text_search_documents = [] + hybrid_search_documents = [] threads = [] exceptions = [] # retrieval_model source with keyword @@ -86,8 +93,9 @@ class RetrievalService: exception_message = ';\n'.join(exceptions) raise Exception(exception_message) - if retrival_method == RetrievalMethod.HYBRID_SEARCH: - data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False) + if retrival_method == RetrievalMethod.HYBRID_SEARCH.value: + data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_mode, + reranking_model, weights, False) all_documents = data_post_processor.invoke( query=query, documents=all_documents, @@ -110,7 +118,7 @@ class RetrievalService: ) documents = keyword.search( - query, + cls.escape_query_for_search(query), top_k=top_k ) all_documents.extend(documents) @@ -132,7 +140,7 @@ class RetrievalService: ) documents = vector.search_by_vector( - query, + cls.escape_query_for_search(query), search_type='similarity_score_threshold', top_k=top_k, score_threshold=score_threshold, @@ -142,8 +150,10 @@ class RetrievalService: ) if documents: - if reranking_model and retrival_method == RetrievalMethod.SEMANTIC_SEARCH: - data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False) + if reranking_model and retrival_method == RetrievalMethod.SEMANTIC_SEARCH.value: + data_post_processor = DataPostProcessor(str(dataset.tenant_id), + RerankMode.RERANKING_MODEL.value, + reranking_model, None, False) all_documents.extend(data_post_processor.invoke( query=query, documents=documents, @@ -170,12 +180,14 @@ class RetrievalService: ) documents = vector_processor.search_by_full_text( - query, + cls.escape_query_for_search(query), top_k=top_k ) if documents: - if reranking_model and retrival_method == RetrievalMethod.FULL_TEXT_SEARCH: - data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False) + if reranking_model and retrival_method == RetrievalMethod.FULL_TEXT_SEARCH.value: + data_post_processor = DataPostProcessor(str(dataset.tenant_id), + RerankMode.RERANKING_MODEL.value, + reranking_model, None, False) all_documents.extend(data_post_processor.invoke( query=query, documents=documents, @@ -186,3 +198,7 @@ class RetrievalService: all_documents.extend(documents) except Exception as e: exceptions.append(str(e)) + + @staticmethod + def escape_query_for_search(query: str) -> str: + return query.replace('"', '\\"') \ No newline at end of file diff --git a/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py b/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py index d7a5dd5dcc..442d71293f 100644 --- a/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py +++ b/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py @@ -7,8 +7,8 @@ _import_err_msg = ( "`alibabacloud_gpdb20160503` and `alibabacloud_tea_openapi` packages not found, " "please run `pip install alibabacloud_gpdb20160503 alibabacloud_tea_openapi`" ) -from flask import current_app +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -36,7 +36,7 @@ class AnalyticdbConfig(BaseModel): "region_id": self.region_id, "read_timeout": self.read_timeout, } - + class AnalyticdbVector(BaseVector): _instance = None _init = False @@ -45,7 +45,7 @@ class AnalyticdbVector(BaseVector): if cls._instance is None: cls._instance = super().__new__(cls) return cls._instance - + def __init__(self, collection_name: str, config: AnalyticdbConfig): # collection_name must be updated every time self._collection_name = collection_name.lower() @@ -105,7 +105,7 @@ class AnalyticdbVector(BaseVector): raise ValueError( f"failed to create namespace {self.config.namespace}: {e}" ) - + def _create_collection_if_not_exists(self, embedding_dimension: int): from alibabacloud_gpdb20160503 import models as gpdb_20160503_models from Tea.exceptions import TeaException @@ -149,7 +149,7 @@ class AnalyticdbVector(BaseVector): def get_type(self) -> str: return VectorType.ANALYTICDB - + def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs): dimension = len(embeddings[0]) self._create_collection_if_not_exists(dimension) @@ -199,7 +199,7 @@ class AnalyticdbVector(BaseVector): ) response = self._client.query_collection_data(request) return len(response.body.matches.match) > 0 - + def delete_by_ids(self, ids: list[str]) -> None: from alibabacloud_gpdb20160503 import models as gpdb_20160503_models ids_str = ",".join(f"'{id}'" for id in ids) @@ -260,7 +260,7 @@ class AnalyticdbVector(BaseVector): ) documents.append(doc) return documents - + def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: from alibabacloud_gpdb20160503 import models as gpdb_20160503_models score_threshold = ( @@ -291,17 +291,20 @@ class AnalyticdbVector(BaseVector): ) documents.append(doc) return documents - + def delete(self) -> None: - from alibabacloud_gpdb20160503 import models as gpdb_20160503_models - request = gpdb_20160503_models.DeleteCollectionRequest( - collection=self._collection_name, - dbinstance_id=self.config.instance_id, - namespace=self.config.namespace, - namespace_password=self.config.namespace_password, - region_id=self.config.region_id, - ) - self._client.delete_collection(request) + try: + from alibabacloud_gpdb20160503 import models as gpdb_20160503_models + request = gpdb_20160503_models.DeleteCollectionRequest( + collection=self._collection_name, + dbinstance_id=self.config.instance_id, + namespace=self.config.namespace, + namespace_password=self.config.namespace_password, + region_id=self.config.region_id, + ) + self._client.delete_collection(request) + except Exception as e: + raise e class AnalyticdbVectorFactory(AbstractVectorFactory): def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings): @@ -316,17 +319,18 @@ class AnalyticdbVectorFactory(AbstractVectorFactory): dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.ANALYTICDB, collection_name) ) - config = current_app.config + + # TODO handle optional params return AnalyticdbVector( collection_name, AnalyticdbConfig( - access_key_id=config.get("ANALYTICDB_KEY_ID"), - access_key_secret=config.get("ANALYTICDB_KEY_SECRET"), - region_id=config.get("ANALYTICDB_REGION_ID"), - instance_id=config.get("ANALYTICDB_INSTANCE_ID"), - account=config.get("ANALYTICDB_ACCOUNT"), - account_password=config.get("ANALYTICDB_PASSWORD"), - namespace=config.get("ANALYTICDB_NAMESPACE"), - namespace_password=config.get("ANALYTICDB_NAMESPACE_PASSWORD"), + access_key_id=dify_config.ANALYTICDB_KEY_ID, + access_key_secret=dify_config.ANALYTICDB_KEY_SECRET, + region_id=dify_config.ANALYTICDB_REGION_ID, + instance_id=dify_config.ANALYTICDB_INSTANCE_ID, + account=dify_config.ANALYTICDB_ACCOUNT, + account_password=dify_config.ANALYTICDB_PASSWORD, + namespace=dify_config.ANALYTICDB_NAMESPACE, + namespace_password=dify_config.ANALYTICDB_NAMESPACE_PASSWORD, ), - ) \ No newline at end of file + ) diff --git a/api/core/rag/datasource/vdb/chroma/chroma_vector.py b/api/core/rag/datasource/vdb/chroma/chroma_vector.py index 2d4e1975ea..3629887b44 100644 --- a/api/core/rag/datasource/vdb/chroma/chroma_vector.py +++ b/api/core/rag/datasource/vdb/chroma/chroma_vector.py @@ -3,9 +3,9 @@ from typing import Any, Optional import chromadb from chromadb import QueryResult, Settings -from flask import current_app from pydantic import BaseModel +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -111,7 +111,8 @@ class ChromaVector(BaseVector): metadata=metadata, ) docs.append(doc) - + # Sort the documents by score in descending order + docs = sorted(docs, key=lambda x: x.metadata['score'], reverse=True) return docs def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: @@ -133,15 +134,14 @@ class ChromaVectorFactory(AbstractVectorFactory): } dataset.index_struct = json.dumps(index_struct_dict) - config = current_app.config return ChromaVector( collection_name=collection_name, config=ChromaConfig( - host=config.get('CHROMA_HOST'), - port=int(config.get('CHROMA_PORT')), - tenant=config.get('CHROMA_TENANT', chromadb.DEFAULT_TENANT), - database=config.get('CHROMA_DATABASE', chromadb.DEFAULT_DATABASE), - auth_provider=config.get('CHROMA_AUTH_PROVIDER'), - auth_credentials=config.get('CHROMA_AUTH_CREDENTIALS'), + host=dify_config.CHROMA_HOST, + port=dify_config.CHROMA_PORT, + tenant=dify_config.CHROMA_TENANT or chromadb.DEFAULT_TENANT, + database=dify_config.CHROMA_DATABASE or chromadb.DEFAULT_DATABASE, + auth_provider=dify_config.CHROMA_AUTH_PROVIDER, + auth_credentials=dify_config.CHROMA_AUTH_CREDENTIALS, ), ) diff --git a/api/core/rag/datasource/vdb/milvus/milvus_vector.py b/api/core/rag/datasource/vdb/milvus/milvus_vector.py index 02b715d768..b679577c04 100644 --- a/api/core/rag/datasource/vdb/milvus/milvus_vector.py +++ b/api/core/rag/datasource/vdb/milvus/milvus_vector.py @@ -3,10 +3,10 @@ import logging from typing import Any, Optional from uuid import uuid4 -from flask import current_app from pydantic import BaseModel, model_validator from pymilvus import MilvusClient, MilvusException, connections +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.field import Field from core.rag.datasource.vdb.vector_base import BaseVector @@ -115,7 +115,8 @@ class MilvusVector(BaseVector): uri = "https://" + str(self._client_config.host) + ":" + str(self._client_config.port) else: uri = "http://" + str(self._client_config.host) + ":" + str(self._client_config.port) - connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password) + connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password, + db_name=self._client_config.database) from pymilvus import utility if utility.has_collection(self._collection_name, using=alias): @@ -130,7 +131,8 @@ class MilvusVector(BaseVector): uri = "https://" + str(self._client_config.host) + ":" + str(self._client_config.port) else: uri = "http://" + str(self._client_config.host) + ":" + str(self._client_config.port) - connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password) + connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password, + db_name=self._client_config.database) from pymilvus import utility if utility.has_collection(self._collection_name, using=alias): @@ -275,15 +277,14 @@ class MilvusVectorFactory(AbstractVectorFactory): dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.MILVUS, collection_name)) - config = current_app.config return MilvusVector( collection_name=collection_name, config=MilvusConfig( - host=config.get('MILVUS_HOST'), - port=config.get('MILVUS_PORT'), - user=config.get('MILVUS_USER'), - password=config.get('MILVUS_PASSWORD'), - secure=config.get('MILVUS_SECURE'), - database=config.get('MILVUS_DATABASE'), + host=dify_config.MILVUS_HOST, + port=dify_config.MILVUS_PORT, + user=dify_config.MILVUS_USER, + password=dify_config.MILVUS_PASSWORD, + secure=dify_config.MILVUS_SECURE, + database=dify_config.MILVUS_DATABASE, ) ) diff --git a/api/core/rag/datasource/vdb/myscale/myscale_vector.py b/api/core/rag/datasource/vdb/myscale/myscale_vector.py index 811b08818c..241b5a8414 100644 --- a/api/core/rag/datasource/vdb/myscale/myscale_vector.py +++ b/api/core/rag/datasource/vdb/myscale/myscale_vector.py @@ -5,9 +5,9 @@ from enum import Enum from typing import Any from clickhouse_connect import get_client -from flask import current_app from pydantic import BaseModel +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -156,15 +156,14 @@ class MyScaleVectorFactory(AbstractVectorFactory): dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.MYSCALE, collection_name)) - config = current_app.config return MyScaleVector( collection_name=collection_name, config=MyScaleConfig( - host=config.get("MYSCALE_HOST", "localhost"), - port=int(config.get("MYSCALE_PORT", 8123)), - user=config.get("MYSCALE_USER", "default"), - password=config.get("MYSCALE_PASSWORD", ""), - database=config.get("MYSCALE_DATABASE", "default"), - fts_params=config.get("MYSCALE_FTS_PARAMS", ""), + host=dify_config.MYSCALE_HOST, + port=dify_config.MYSCALE_PORT, + user=dify_config.MYSCALE_USER, + password=dify_config.MYSCALE_PASSWORD, + database=dify_config.MYSCALE_DATABASE, + fts_params=dify_config.MYSCALE_FTS_PARAMS, ), ) diff --git a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py index 744ff2d517..d834e8ce14 100644 --- a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py +++ b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py @@ -4,11 +4,11 @@ import ssl from typing import Any, Optional from uuid import uuid4 -from flask import current_app from opensearchpy import OpenSearch, helpers from opensearchpy.helpers import BulkIndexError from pydantic import BaseModel, model_validator +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.field import Field from core.rag.datasource.vdb.vector_base import BaseVector @@ -257,14 +257,13 @@ class OpenSearchVectorFactory(AbstractVectorFactory): dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.OPENSEARCH, collection_name)) - config = current_app.config open_search_config = OpenSearchConfig( - host=config.get('OPENSEARCH_HOST'), - port=config.get('OPENSEARCH_PORT'), - user=config.get('OPENSEARCH_USER'), - password=config.get('OPENSEARCH_PASSWORD'), - secure=config.get('OPENSEARCH_SECURE'), + host=dify_config.OPENSEARCH_HOST, + port=dify_config.OPENSEARCH_PORT, + user=dify_config.OPENSEARCH_USER, + password=dify_config.OPENSEARCH_PASSWORD, + secure=dify_config.OPENSEARCH_SECURE, ) return OpenSearchVector( diff --git a/api/core/rag/datasource/vdb/oracle/oraclevector.py b/api/core/rag/datasource/vdb/oracle/oraclevector.py index 5f7723508c..4bd09b331d 100644 --- a/api/core/rag/datasource/vdb/oracle/oraclevector.py +++ b/api/core/rag/datasource/vdb/oracle/oraclevector.py @@ -1,14 +1,18 @@ import array import json +import re import uuid from contextlib import contextmanager from typing import Any +import jieba.posseg as pseg +import nltk import numpy import oracledb -from flask import current_app +from nltk.corpus import stopwords from pydantic import BaseModel, model_validator +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -44,11 +48,16 @@ class OracleVectorConfig(BaseModel): SQL_CREATE_TABLE = """ CREATE TABLE IF NOT EXISTS {table_name} ( - id varchar2(100) + id varchar2(100) ,text CLOB NOT NULL ,meta JSON ,embedding vector NOT NULL -) +) +""" +SQL_CREATE_INDEX = """ +CREATE INDEX IF NOT EXISTS idx_docs_{table_name} ON {table_name}(text) +INDEXTYPE IS CTXSYS.CONTEXT PARAMETERS +('FILTER CTXSYS.NULL_FILTER SECTION GROUP CTXSYS.HTML_SECTION_GROUP LEXER sys.my_chinese_vgram_lexer') """ @@ -188,12 +197,58 @@ class OracleVector(BaseVector): return docs def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: - # do not support bm25 search + top_k = kwargs.get("top_k", 5) + # just not implement fetch by score_threshold now, may be later + score_threshold = kwargs.get("score_threshold") if kwargs.get("score_threshold") else 0.0 + if len(query) > 0: + # Check which language the query is in + zh_pattern = re.compile('[\u4e00-\u9fa5]+') + match = zh_pattern.search(query) + entities = [] + # match: query condition maybe is a chinese sentence, so using Jieba split,else using nltk split + if match: + words = pseg.cut(query) + current_entity = "" + for word, pos in words: + if pos == 'nr' or pos == 'Ng' or pos == 'eng' or pos == 'nz' or pos == 'n' or pos == 'ORG' or pos == 'v': # nr: 人名, ns: 地名, nt: 机构名 + current_entity += word + else: + if current_entity: + entities.append(current_entity) + current_entity = "" + if current_entity: + entities.append(current_entity) + else: + try: + nltk.data.find('tokenizers/punkt') + nltk.data.find('corpora/stopwords') + except LookupError: + nltk.download('punkt') + nltk.download('stopwords') + print("run download") + e_str = re.sub(r'[^\w ]', '', query) + all_tokens = nltk.word_tokenize(e_str) + stop_words = stopwords.words('english') + for token in all_tokens: + if token not in stop_words: + entities.append(token) + with self._get_cursor() as cur: + cur.execute( + f"select meta, text FROM {self.table_name} WHERE CONTAINS(text, :1, 1) > 0 order by score(1) desc fetch first {top_k} rows only", + [" ACCUM ".join(entities)] + ) + docs = [] + for record in cur: + metadata, text = record + docs.append(Document(page_content=text, metadata=metadata)) + return docs + else: + return [Document(page_content="", metadata="")] return [] def delete(self) -> None: with self._get_cursor() as cur: - cur.execute(f"DROP TABLE IF EXISTS {self.table_name}") + cur.execute(f"DROP TABLE IF EXISTS {self.table_name} cascade constraints") def _create_collection(self, dimension: int): cache_key = f"vector_indexing_{self._collection_name}" @@ -206,6 +261,8 @@ class OracleVector(BaseVector): with self._get_cursor() as cur: cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name)) redis_client.set(collection_exist_cache_key, 1, ex=3600) + with self._get_cursor() as cur: + cur.execute(SQL_CREATE_INDEX.format(table_name=self.table_name)) class OracleVectorFactory(AbstractVectorFactory): @@ -219,14 +276,13 @@ class OracleVectorFactory(AbstractVectorFactory): dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.ORACLE, collection_name)) - config = current_app.config return OracleVector( collection_name=collection_name, config=OracleVectorConfig( - host=config.get("ORACLE_HOST"), - port=config.get("ORACLE_PORT"), - user=config.get("ORACLE_USER"), - password=config.get("ORACLE_PASSWORD"), - database=config.get("ORACLE_DATABASE"), + host=dify_config.ORACLE_HOST, + port=dify_config.ORACLE_PORT, + user=dify_config.ORACLE_USER, + password=dify_config.ORACLE_PASSWORD, + database=dify_config.ORACLE_DATABASE, ), ) diff --git a/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py b/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py index 63c8edfbc3..82bdc5d4b9 100644 --- a/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py +++ b/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py @@ -3,7 +3,6 @@ import logging from typing import Any from uuid import UUID, uuid4 -from flask import current_app from numpy import ndarray from pgvecto_rs.sqlalchemy import Vector from pydantic import BaseModel, model_validator @@ -12,6 +11,7 @@ from sqlalchemy import text as sql_text from sqlalchemy.dialects import postgresql from sqlalchemy.orm import Mapped, Session, mapped_column +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.pgvecto_rs.collection import CollectionORM from core.rag.datasource.vdb.vector_base import BaseVector @@ -93,7 +93,7 @@ class PGVectoRS(BaseVector): text TEXT NOT NULL, meta JSONB NOT NULL, vector vector({dimension}) NOT NULL - ) using heap; + ) using heap; """) session.execute(create_statement) index_statement = sql_text(f""" @@ -233,15 +233,15 @@ class PGVectoRSFactory(AbstractVectorFactory): dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.WEAVIATE, collection_name)) dim = len(embeddings.embed_query("pgvecto_rs")) - config = current_app.config + return PGVectoRS( collection_name=collection_name, config=PgvectoRSConfig( - host=config.get('PGVECTO_RS_HOST'), - port=config.get('PGVECTO_RS_PORT'), - user=config.get('PGVECTO_RS_USER'), - password=config.get('PGVECTO_RS_PASSWORD'), - database=config.get('PGVECTO_RS_DATABASE'), + host=dify_config.PGVECTO_RS_HOST, + port=dify_config.PGVECTO_RS_PORT, + user=dify_config.PGVECTO_RS_USER, + password=dify_config.PGVECTO_RS_PASSWORD, + database=dify_config.PGVECTO_RS_DATABASE, ), dim=dim - ) \ No newline at end of file + ) diff --git a/api/core/rag/datasource/vdb/pgvector/pgvector.py b/api/core/rag/datasource/vdb/pgvector/pgvector.py index 72d0a85f8d..33ca5bc028 100644 --- a/api/core/rag/datasource/vdb/pgvector/pgvector.py +++ b/api/core/rag/datasource/vdb/pgvector/pgvector.py @@ -5,9 +5,9 @@ from typing import Any import psycopg2.extras import psycopg2.pool -from flask import current_app from pydantic import BaseModel, model_validator +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -45,7 +45,7 @@ CREATE TABLE IF NOT EXISTS {table_name} ( text TEXT NOT NULL, meta JSONB NOT NULL, embedding vector({dimension}) NOT NULL -) using heap; +) using heap; """ @@ -185,14 +185,13 @@ class PGVectorFactory(AbstractVectorFactory): dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.PGVECTOR, collection_name)) - config = current_app.config return PGVector( collection_name=collection_name, config=PGVectorConfig( - host=config.get("PGVECTOR_HOST"), - port=config.get("PGVECTOR_PORT"), - user=config.get("PGVECTOR_USER"), - password=config.get("PGVECTOR_PASSWORD"), - database=config.get("PGVECTOR_DATABASE"), + host=dify_config.PGVECTOR_HOST, + port=dify_config.PGVECTOR_PORT, + user=dify_config.PGVECTOR_USER, + password=dify_config.PGVECTOR_PASSWORD, + database=dify_config.PGVECTOR_DATABASE, ), - ) \ No newline at end of file + ) diff --git a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py index bccc3a39f6..77c3f6a271 100644 --- a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py +++ b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py @@ -19,6 +19,7 @@ from qdrant_client.http.models import ( ) from qdrant_client.local.qdrant_local import QdrantLocal +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.field import Field from core.rag.datasource.vdb.vector_base import BaseVector @@ -361,6 +362,8 @@ class QdrantVector(BaseVector): metadata=metadata, ) docs.append(doc) + # Sort the documents by score in descending order + docs = sorted(docs, key=lambda x: x.metadata['score'], reverse=True) return docs def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: @@ -393,9 +396,11 @@ class QdrantVector(BaseVector): documents = [] for result in results: if result: - documents.append(self._document_from_scored_point( + document = self._document_from_scored_point( result, Field.CONTENT_KEY.value, Field.METADATA_KEY.value - )) + ) + document.metadata['vector'] = result.vector + documents.append(document) return documents @@ -444,11 +449,11 @@ class QdrantVectorFactory(AbstractVectorFactory): collection_name=collection_name, group_id=dataset.id, config=QdrantConfig( - endpoint=config.get('QDRANT_URL'), - api_key=config.get('QDRANT_API_KEY'), + endpoint=dify_config.QDRANT_URL, + api_key=dify_config.QDRANT_API_KEY, root_path=config.root_path, - timeout=config.get('QDRANT_CLIENT_TIMEOUT'), - grpc_port=config.get('QDRANT_GRPC_PORT'), - prefer_grpc=config.get('QDRANT_GRPC_ENABLED') + timeout=dify_config.QDRANT_CLIENT_TIMEOUT, + grpc_port=dify_config.QDRANT_GRPC_PORT, + prefer_grpc=dify_config.QDRANT_GRPC_ENABLED ) ) diff --git a/api/core/rag/datasource/vdb/relyt/relyt_vector.py b/api/core/rag/datasource/vdb/relyt/relyt_vector.py index 4fe1df717a..2e0bd6f303 100644 --- a/api/core/rag/datasource/vdb/relyt/relyt_vector.py +++ b/api/core/rag/datasource/vdb/relyt/relyt_vector.py @@ -2,7 +2,6 @@ import json import uuid from typing import Any, Optional -from flask import current_app from pydantic import BaseModel, model_validator from sqlalchemy import Column, Sequence, String, Table, create_engine, insert from sqlalchemy import text as sql_text @@ -19,6 +18,7 @@ try: except ImportError: from sqlalchemy.ext.declarative import declarative_base +from configs import dify_config from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.models.document import Document from extensions.ext_redis import redis_client @@ -85,7 +85,7 @@ class RelytVector(BaseVector): document TEXT NOT NULL, metadata JSON NOT NULL, embedding vector({dimension}) NOT NULL - ) using heap; + ) using heap; """) session.execute(create_statement) index_statement = sql_text(f""" @@ -313,15 +313,14 @@ class RelytVectorFactory(AbstractVectorFactory): dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.RELYT, collection_name)) - config = current_app.config return RelytVector( collection_name=collection_name, config=RelytConfig( - host=config.get('RELYT_HOST'), - port=config.get('RELYT_PORT'), - user=config.get('RELYT_USER'), - password=config.get('RELYT_PASSWORD'), - database=config.get('RELYT_DATABASE'), + host=dify_config.RELYT_HOST, + port=dify_config.RELYT_PORT, + user=dify_config.RELYT_USER, + password=dify_config.RELYT_PASSWORD, + database=dify_config.RELYT_DATABASE, ), group_id=dataset.id ) diff --git a/api/core/rag/datasource/vdb/tencent/tencent_vector.py b/api/core/rag/datasource/vdb/tencent/tencent_vector.py index 3af85854d2..3325a1028e 100644 --- a/api/core/rag/datasource/vdb/tencent/tencent_vector.py +++ b/api/core/rag/datasource/vdb/tencent/tencent_vector.py @@ -1,13 +1,13 @@ import json from typing import Any, Optional -from flask import current_app from pydantic import BaseModel from tcvectordb import VectorDBClient from tcvectordb.model import document, enum from tcvectordb.model import index as vdb_index from tcvectordb.model.document import Filter +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -198,8 +198,6 @@ class TencentVector(BaseVector): self._db.drop_collection(name=self._collection_name) - - class TencentVectorFactory(AbstractVectorFactory): def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings) -> TencentVector: @@ -212,16 +210,15 @@ class TencentVectorFactory(AbstractVectorFactory): dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.TENCENT, collection_name)) - config = current_app.config return TencentVector( collection_name=collection_name, config=TencentConfig( - url=config.get('TENCENT_VECTOR_DB_URL'), - api_key=config.get('TENCENT_VECTOR_DB_API_KEY'), - timeout=config.get('TENCENT_VECTOR_DB_TIMEOUT'), - username=config.get('TENCENT_VECTOR_DB_USERNAME'), - database=config.get('TENCENT_VECTOR_DB_DATABASE'), - shard=config.get('TENCENT_VECTOR_DB_SHARD'), - replicas=config.get('TENCENT_VECTOR_DB_REPLICAS'), + url=dify_config.TENCENT_VECTOR_DB_URL, + api_key=dify_config.TENCENT_VECTOR_DB_API_KEY, + timeout=dify_config.TENCENT_VECTOR_DB_TIMEOUT, + username=dify_config.TENCENT_VECTOR_DB_USERNAME, + database=dify_config.TENCENT_VECTOR_DB_DATABASE, + shard=dify_config.TENCENT_VECTOR_DB_SHARD, + replicas=dify_config.TENCENT_VECTOR_DB_REPLICAS, ) - ) \ No newline at end of file + ) diff --git a/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py b/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py index 5922db1176..d3685c0991 100644 --- a/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py +++ b/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py @@ -3,12 +3,12 @@ import logging from typing import Any import sqlalchemy -from flask import current_app from pydantic import BaseModel, model_validator from sqlalchemy import JSON, TEXT, Column, DateTime, String, Table, create_engine, insert from sqlalchemy import text as sql_text from sqlalchemy.orm import Session, declarative_base +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -198,8 +198,8 @@ class TiDBVector(BaseVector): with Session(self._engine) as session: select_statement = sql_text( f"""SELECT meta, text, distance FROM ( - SELECT meta, text, {tidb_func}(vector, "{query_vector_str}") as distance - FROM {self._collection_name} + SELECT meta, text, {tidb_func}(vector, "{query_vector_str}") as distance + FROM {self._collection_name} ORDER BY distance LIMIT {top_k} ) t WHERE distance < {distance};""" @@ -234,15 +234,14 @@ class TiDBVectorFactory(AbstractVectorFactory): dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.TIDB_VECTOR, collection_name)) - config = current_app.config return TiDBVector( collection_name=collection_name, config=TiDBVectorConfig( - host=config.get('TIDB_VECTOR_HOST'), - port=config.get('TIDB_VECTOR_PORT'), - user=config.get('TIDB_VECTOR_USER'), - password=config.get('TIDB_VECTOR_PASSWORD'), - database=config.get('TIDB_VECTOR_DATABASE'), - program_name=config.get('APPLICATION_NAME'), + host=dify_config.TIDB_VECTOR_HOST, + port=dify_config.TIDB_VECTOR_PORT, + user=dify_config.TIDB_VECTOR_USER, + password=dify_config.TIDB_VECTOR_PASSWORD, + database=dify_config.TIDB_VECTOR_DATABASE, + program_name=dify_config.APPLICATION_NAME, ), - ) \ No newline at end of file + ) diff --git a/api/core/rag/datasource/vdb/vector_base.py b/api/core/rag/datasource/vdb/vector_base.py index dbd8b6284b..3f70e8b608 100644 --- a/api/core/rag/datasource/vdb/vector_base.py +++ b/api/core/rag/datasource/vdb/vector_base.py @@ -57,7 +57,7 @@ class BaseVector(ABC): raise NotImplementedError def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: - for text in texts: + for text in texts[:]: doc_id = text.metadata['doc_id'] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: @@ -67,3 +67,7 @@ class BaseVector(ABC): def _get_uuids(self, texts: list[Document]) -> list[str]: return [text.metadata['doc_id'] for text in texts] + + @property + def collection_name(self): + return self._collection_name diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index f8b58e1b9a..fad60ecf45 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -1,8 +1,7 @@ from abc import ABC, abstractmethod from typing import Any -from flask import current_app - +from configs import dify_config from core.embedding.cached_embedding import CacheEmbedding from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelType @@ -10,6 +9,7 @@ from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_type import VectorType from core.rag.models.document import Document +from extensions.ext_redis import redis_client from models.dataset import Dataset @@ -37,8 +37,7 @@ class Vector: self._vector_processor = self._init_vector() def _init_vector(self) -> BaseVector: - config = current_app.config - vector_type = config.get('VECTOR_STORE') + vector_type = dify_config.VECTOR_STORE if self._dataset.index_struct_dict: vector_type = self._dataset.index_struct_dict['type'] @@ -136,6 +135,10 @@ class Vector: def delete(self) -> None: self._vector_processor.delete() + # delete collection redis cache + if self._vector_processor.collection_name: + collection_exist_cache_key = 'vector_indexing_{}'.format(self._vector_processor.collection_name) + redis_client.delete(collection_exist_cache_key) def _get_embeddings(self) -> Embeddings: model_manager = ModelManager() @@ -150,7 +153,7 @@ class Vector: return CacheEmbedding(embedding_model) def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: - for text in texts: + for text in texts[:]: doc_id = text.metadata['doc_id'] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: diff --git a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py index b7c5c96a7d..87fc5ff158 100644 --- a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py +++ b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py @@ -4,9 +4,9 @@ from typing import Any, Optional import requests import weaviate -from flask import current_app from pydantic import BaseModel, model_validator +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.field import Field from core.rag.datasource.vdb.vector_base import BaseVector @@ -216,7 +216,8 @@ class WeaviateVector(BaseVector): if score > score_threshold: doc.metadata['score'] = score docs.append(doc) - + # Sort the documents by score in descending order + docs = sorted(docs, key=lambda x: x.metadata['score'], reverse=True) return docs def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: @@ -281,9 +282,9 @@ class WeaviateVectorFactory(AbstractVectorFactory): return WeaviateVector( collection_name=collection_name, config=WeaviateConfig( - endpoint=current_app.config.get('WEAVIATE_ENDPOINT'), - api_key=current_app.config.get('WEAVIATE_API_KEY'), - batch_size=int(current_app.config.get('WEAVIATE_BATCH_SIZE')) + endpoint=dify_config.WEAVIATE_ENDPOINT, + api_key=dify_config.WEAVIATE_API_KEY, + batch_size=dify_config.WEAVIATE_BATCH_SIZE ), attributes=attributes ) diff --git a/api/core/rag/docstore/__init__.py b/api/core/rag/docstore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/rag/extractor/excel_extractor.py b/api/core/rag/extractor/excel_extractor.py index 2b16275dc8..f0c302a619 100644 --- a/api/core/rag/extractor/excel_extractor.py +++ b/api/core/rag/extractor/excel_extractor.py @@ -3,6 +3,7 @@ import os from typing import Optional import pandas as pd +from openpyxl import load_workbook from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document @@ -28,26 +29,48 @@ class ExcelExtractor(BaseExtractor): self._autodetect_encoding = autodetect_encoding def extract(self) -> list[Document]: - """ Load from Excel file in xls or xlsx format using Pandas.""" + """ Load from Excel file in xls or xlsx format using Pandas and openpyxl.""" documents = [] - # Determine the file extension file_extension = os.path.splitext(self._file_path)[-1].lower() - # Read each worksheet of an Excel file using Pandas + if file_extension == '.xlsx': - excel_file = pd.ExcelFile(self._file_path, engine='openpyxl') + wb = load_workbook(self._file_path, data_only=True) + for sheet_name in wb.sheetnames: + sheet = wb[sheet_name] + data = sheet.values + cols = next(data) + df = pd.DataFrame(data, columns=cols) + + df.dropna(how='all', inplace=True) + + for index, row in df.iterrows(): + page_content = [] + for col_index, (k, v) in enumerate(row.items()): + if pd.notna(v): + cell = sheet.cell(row=index + 2, + column=col_index + 1) # +2 to account for header and 1-based index + if cell.hyperlink: + value = f"[{v}]({cell.hyperlink.target})" + page_content.append(f'"{k}":"{value}"') + else: + page_content.append(f'"{k}":"{v}"') + documents.append(Document(page_content=';'.join(page_content), + metadata={'source': self._file_path})) + elif file_extension == '.xls': excel_file = pd.ExcelFile(self._file_path, engine='xlrd') + for sheet_name in excel_file.sheet_names: + df = excel_file.parse(sheet_name=sheet_name) + df.dropna(how='all', inplace=True) + + for _, row in df.iterrows(): + page_content = [] + for k, v in row.items(): + if pd.notna(v): + page_content.append(f'"{k}":"{v}"') + documents.append(Document(page_content=';'.join(page_content), + metadata={'source': self._file_path})) else: raise ValueError(f"Unsupported file extension: {file_extension}") - for sheet_name in excel_file.sheet_names: - df: pd.DataFrame = excel_file.parse(sheet_name=sheet_name) - - # filter out rows with all NaN values - df.dropna(how='all', inplace=True) - - # transform each row into a Document - documents += [Document(page_content=';'.join(f'"{k}":"{v}"' for k, v in row.items() if pd.notna(v)), - metadata={'source': self._file_path}, - ) for _, row in df.iterrows()] return documents diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py index 909bfdc137..f7a08135f5 100644 --- a/api/core/rag/extractor/extract_processor.py +++ b/api/core/rag/extractor/extract_processor.py @@ -4,9 +4,8 @@ from pathlib import Path from typing import Union from urllib.parse import unquote -import requests -from flask import current_app - +from configs import dify_config +from core.helper import ssrf_proxy from core.rag.extractor.csv_extractor import CSVExtractor from core.rag.extractor.entity.datasource_type import DatasourceType from core.rag.extractor.entity.extract_setting import ExtractSetting @@ -51,7 +50,7 @@ class ExtractProcessor: @classmethod def load_from_url(cls, url: str, return_text: bool = False) -> Union[list[Document], str]: - response = requests.get(url, headers={ + response = ssrf_proxy.get(url, headers={ "User-Agent": USER_AGENT }) @@ -94,9 +93,9 @@ class ExtractProcessor: storage.download(upload_file.key, file_path) input_file = Path(file_path) file_extension = input_file.suffix.lower() - etl_type = current_app.config['ETL_TYPE'] - unstructured_api_url = current_app.config['UNSTRUCTURED_API_URL'] - unstructured_api_key = current_app.config['UNSTRUCTURED_API_KEY'] + etl_type = dify_config.ETL_TYPE + unstructured_api_url = dify_config.UNSTRUCTURED_API_URL + unstructured_api_key = dify_config.UNSTRUCTURED_API_KEY if etl_type == 'Unstructured': if file_extension == '.xlsx' or file_extension == '.xls': extractor = ExcelExtractor(file_path) diff --git a/api/core/rag/extractor/markdown_extractor.py b/api/core/rag/extractor/markdown_extractor.py index faa1e64057..b24cf2e170 100644 --- a/api/core/rag/extractor/markdown_extractor.py +++ b/api/core/rag/extractor/markdown_extractor.py @@ -54,8 +54,16 @@ class MarkdownExtractor(BaseExtractor): current_header = None current_text = "" + code_block_flag = False for line in lines: + if line.startswith("```"): + code_block_flag = not code_block_flag + current_text += line + "\n" + continue + if code_block_flag: + current_text += line + "\n" + continue header_match = re.match(r"^#+\s", line) if header_match: if current_header is not None: diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.py index 7c6101010e..9535455909 100644 --- a/api/core/rag/extractor/notion_extractor.py +++ b/api/core/rag/extractor/notion_extractor.py @@ -3,8 +3,8 @@ import logging from typing import Any, Optional import requests -from flask import current_app +from configs import dify_config from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document from extensions.ext_database import db @@ -49,7 +49,7 @@ class NotionExtractor(BaseExtractor): self._notion_access_token = self._get_access_token(tenant_id, self._notion_workspace_id) if not self._notion_access_token: - integration_token = current_app.config.get('NOTION_INTEGRATION_TOKEN') + integration_token = dify_config.NOTION_INTEGRATION_TOKEN if integration_token is None: raise ValueError( "Must specify `integration_token` or set environment " diff --git a/api/core/rag/extractor/word_extractor.py b/api/core/rag/extractor/word_extractor.py index 9045966da9..ac4a56319b 100644 --- a/api/core/rag/extractor/word_extractor.py +++ b/api/core/rag/extractor/word_extractor.py @@ -8,8 +8,8 @@ from urllib.parse import urlparse import requests from docx import Document as DocxDocument -from flask import current_app +from configs import dify_config from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document from extensions.ext_database import db @@ -96,10 +96,9 @@ class WordExtractor(BaseExtractor): storage.save(file_key, rel.target_part.blob) # save file to db - config = current_app.config upload_file = UploadFile( tenant_id=self.tenant_id, - storage_type=config['STORAGE_TYPE'], + storage_type=dify_config.STORAGE_TYPE, key=file_key, name=file_key, size=0, @@ -114,7 +113,7 @@ class WordExtractor(BaseExtractor): db.session.add(upload_file) db.session.commit() - image_map[rel.target_part] = f"![image]({current_app.config.get('CONSOLE_API_URL')}/files/{upload_file.id}/image-preview)" + image_map[rel.target_part] = f"![image]({dify_config.CONSOLE_API_URL}/files/{upload_file.id}/image-preview)" return image_map diff --git a/api/core/rag/index_processor/index_processor_base.py b/api/core/rag/index_processor/index_processor_base.py index edc16c821a..33e78ce8c5 100644 --- a/api/core/rag/index_processor/index_processor_base.py +++ b/api/core/rag/index_processor/index_processor_base.py @@ -2,8 +2,7 @@ from abc import ABC, abstractmethod from typing import Optional -from flask import current_app - +from configs import dify_config from core.model_manager import ModelInstance from core.rag.extractor.entity.extract_setting import ExtractSetting from core.rag.models.document import Document @@ -48,7 +47,7 @@ class BaseIndexProcessor(ABC): # The user-defined segmentation rule rules = processing_rule['rules'] segmentation = rules["segmentation"] - max_segmentation_tokens_length = int(current_app.config['INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH']) + max_segmentation_tokens_length = dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH if segmentation["max_tokens"] < 50 or segmentation["max_tokens"] > max_segmentation_tokens_length: raise ValueError(f"Custom segment length should be between 50 and {max_segmentation_tokens_length}.") diff --git a/api/core/rag/rerank/constants/rerank_mode.py b/api/core/rag/rerank/constants/rerank_mode.py new file mode 100644 index 0000000000..afbb9fd89d --- /dev/null +++ b/api/core/rag/rerank/constants/rerank_mode.py @@ -0,0 +1,8 @@ +from enum import Enum + + +class RerankMode(Enum): + + RERANKING_MODEL = 'reranking_model' + WEIGHTED_SCORE = 'weighted_score' + diff --git a/api/core/rag/rerank/entity/weight.py b/api/core/rag/rerank/entity/weight.py new file mode 100644 index 0000000000..36afc89a21 --- /dev/null +++ b/api/core/rag/rerank/entity/weight.py @@ -0,0 +1,23 @@ +from pydantic import BaseModel + + +class VectorSetting(BaseModel): + vector_weight: float + + embedding_provider_name: str + + embedding_model_name: str + + +class KeywordSetting(BaseModel): + keyword_weight: float + + +class Weights(BaseModel): + """Model for weighted rerank.""" + + weight_type: str + + vector_setting: VectorSetting + + keyword_setting: KeywordSetting diff --git a/api/core/rag/rerank/rerank.py b/api/core/rag/rerank/rerank_model.py similarity index 98% rename from api/core/rag/rerank/rerank.py rename to api/core/rag/rerank/rerank_model.py index 7000f4e0ad..d9067da288 100644 --- a/api/core/rag/rerank/rerank.py +++ b/api/core/rag/rerank/rerank_model.py @@ -4,7 +4,7 @@ from core.model_manager import ModelInstance from core.rag.models.document import Document -class RerankRunner: +class RerankModelRunner: def __init__(self, rerank_model_instance: ModelInstance) -> None: self.rerank_model_instance = rerank_model_instance diff --git a/api/core/rag/rerank/weight_rerank.py b/api/core/rag/rerank/weight_rerank.py new file mode 100644 index 0000000000..d07f94adb7 --- /dev/null +++ b/api/core/rag/rerank/weight_rerank.py @@ -0,0 +1,178 @@ +import math +from collections import Counter +from typing import Optional + +import numpy as np + +from core.embedding.cached_embedding import CacheEmbedding +from core.model_manager import ModelManager +from core.model_runtime.entities.model_entities import ModelType +from core.rag.datasource.keyword.jieba.jieba_keyword_table_handler import JiebaKeywordTableHandler +from core.rag.models.document import Document +from core.rag.rerank.entity.weight import VectorSetting, Weights + + +class WeightRerankRunner: + + def __init__(self, tenant_id: str, weights: Weights) -> None: + self.tenant_id = tenant_id + self.weights = weights + + def run(self, query: str, documents: list[Document], score_threshold: Optional[float] = None, + top_n: Optional[int] = None, user: Optional[str] = None) -> list[Document]: + """ + Run rerank model + :param query: search query + :param documents: documents for reranking + :param score_threshold: score threshold + :param top_n: top n + :param user: unique user id if needed + + :return: + """ + docs = [] + doc_id = [] + unique_documents = [] + for document in documents: + if document.metadata['doc_id'] not in doc_id: + doc_id.append(document.metadata['doc_id']) + docs.append(document.page_content) + unique_documents.append(document) + + documents = unique_documents + + rerank_documents = [] + query_scores = self._calculate_keyword_score(query, documents) + + query_vector_scores = self._calculate_cosine(self.tenant_id, query, documents, self.weights.vector_setting) + for document, query_score, query_vector_score in zip(documents, query_scores, query_vector_scores): + # format document + score = self.weights.vector_setting.vector_weight * query_vector_score + \ + self.weights.keyword_setting.keyword_weight * query_score + if score_threshold and score < score_threshold: + continue + document.metadata['score'] = score + rerank_documents.append(document) + rerank_documents = sorted(rerank_documents, key=lambda x: x.metadata['score'], reverse=True) + return rerank_documents[:top_n] if top_n else rerank_documents + + def _calculate_keyword_score(self, query: str, documents: list[Document]) -> list[float]: + """ + Calculate BM25 scores + :param query: search query + :param documents: documents for reranking + + :return: + """ + keyword_table_handler = JiebaKeywordTableHandler() + query_keywords = keyword_table_handler.extract_keywords(query, None) + documents_keywords = [] + for document in documents: + # get the document keywords + document_keywords = keyword_table_handler.extract_keywords(document.page_content, None) + document.metadata['keywords'] = document_keywords + documents_keywords.append(document_keywords) + + # Counter query keywords(TF) + query_keyword_counts = Counter(query_keywords) + + # total documents + total_documents = len(documents) + + # calculate all documents' keywords IDF + all_keywords = set() + for document_keywords in documents_keywords: + all_keywords.update(document_keywords) + + keyword_idf = {} + for keyword in all_keywords: + # calculate include query keywords' documents + doc_count_containing_keyword = sum(1 for doc_keywords in documents_keywords if keyword in doc_keywords) + # IDF + keyword_idf[keyword] = math.log((1 + total_documents) / (1 + doc_count_containing_keyword)) + 1 + + query_tfidf = {} + + for keyword, count in query_keyword_counts.items(): + tf = count + idf = keyword_idf.get(keyword, 0) + query_tfidf[keyword] = tf * idf + + # calculate all documents' TF-IDF + documents_tfidf = [] + for document_keywords in documents_keywords: + document_keyword_counts = Counter(document_keywords) + document_tfidf = {} + for keyword, count in document_keyword_counts.items(): + tf = count + idf = keyword_idf.get(keyword, 0) + document_tfidf[keyword] = tf * idf + documents_tfidf.append(document_tfidf) + + def cosine_similarity(vec1, vec2): + intersection = set(vec1.keys()) & set(vec2.keys()) + numerator = sum(vec1[x] * vec2[x] for x in intersection) + + sum1 = sum(vec1[x] ** 2 for x in vec1.keys()) + sum2 = sum(vec2[x] ** 2 for x in vec2.keys()) + denominator = math.sqrt(sum1) * math.sqrt(sum2) + + if not denominator: + return 0.0 + else: + return float(numerator) / denominator + + similarities = [] + for document_tfidf in documents_tfidf: + similarity = cosine_similarity(query_tfidf, document_tfidf) + similarities.append(similarity) + + # for idx, similarity in enumerate(similarities): + # print(f"Document {idx + 1} similarity: {similarity}") + + return similarities + + def _calculate_cosine(self, tenant_id: str, query: str, documents: list[Document], + vector_setting: VectorSetting) -> list[float]: + """ + Calculate Cosine scores + :param query: search query + :param documents: documents for reranking + + :return: + """ + query_vector_scores = [] + + model_manager = ModelManager() + + embedding_model = model_manager.get_model_instance( + tenant_id=tenant_id, + provider=vector_setting.embedding_provider_name, + model_type=ModelType.TEXT_EMBEDDING, + model=vector_setting.embedding_model_name + + ) + cache_embedding = CacheEmbedding(embedding_model) + query_vector = cache_embedding.embed_query(query) + for document in documents: + # calculate cosine similarity + if 'score' in document.metadata: + query_vector_scores.append(document.metadata['score']) + else: + content_vector = document.metadata['vector'] + # transform to NumPy + vec1 = np.array(query_vector) + vec2 = np.array(document.metadata['vector']) + + # calculate dot product + dot_product = np.dot(vec1, vec2) + + # calculate norm + norm_vec1 = np.linalg.norm(vec1) + norm_vec2 = np.linalg.norm(vec2) + + # calculate cosine similarity + cosine_sim = dot_product / (norm_vec1 * norm_vec2) + query_vector_scores.append(cosine_sim) + + return query_vector_scores diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index ea2a194a68..d51ea2942a 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -1,4 +1,6 @@ +import math import threading +from collections import Counter from typing import Optional, cast from flask import Flask, current_app @@ -14,9 +16,10 @@ from core.model_runtime.entities.model_entities import ModelFeature, ModelType from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.ops.ops_trace_manager import TraceQueueManager, TraceTask, TraceTaskName from core.ops.utils import measure_time +from core.rag.data_post_processor.data_post_processor import DataPostProcessor +from core.rag.datasource.keyword.jieba.jieba_keyword_table_handler import JiebaKeywordTableHandler from core.rag.datasource.retrieval_service import RetrievalService from core.rag.models.document import Document -from core.rag.rerank.rerank import RerankRunner from core.rag.retrieval.retrival_methods import RetrievalMethod from core.rag.retrieval.router.multi_dataset_function_call_router import FunctionCallMultiDatasetRouter from core.rag.retrieval.router.multi_dataset_react_route import ReactMultiDatasetRouter @@ -28,7 +31,7 @@ from models.dataset import Dataset, DatasetQuery, DocumentSegment from models.dataset import Document as DatasetDocument default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -132,8 +135,9 @@ class DatasetRetrieval: app_id, tenant_id, user_id, user_from, available_datasets, query, retrieve_config.top_k, retrieve_config.score_threshold, - retrieve_config.reranking_model.get('reranking_provider_name'), - retrieve_config.reranking_model.get('reranking_model_name'), + retrieve_config.rerank_mode, + retrieve_config.reranking_model, + retrieve_config.weights, message_id, ) @@ -272,7 +276,8 @@ class DatasetRetrieval: retrival_method=retrival_method, dataset_id=dataset.id, query=query, top_k=top_k, score_threshold=score_threshold, - reranking_model=reranking_model + reranking_model=reranking_model, + weights=retrieval_model_config.get('weights', None), ) self._on_query(query, [dataset_id], app_id, user_from, user_id) @@ -292,14 +297,18 @@ class DatasetRetrieval: query: str, top_k: int, score_threshold: float, - reranking_provider_name: str, - reranking_model_name: str, + reranking_mode: str, + reranking_model: Optional[dict] = None, + weights: Optional[dict] = None, + reranking_enable: bool = True, message_id: Optional[str] = None, ): threads = [] all_documents = [] dataset_ids = [dataset.id for dataset in available_datasets] + index_type = None for dataset in available_datasets: + index_type = dataset.indexing_technique retrieval_thread = threading.Thread(target=self._retriever, kwargs={ 'flask_app': current_app._get_current_object(), 'dataset_id': dataset.id, @@ -311,23 +320,24 @@ class DatasetRetrieval: retrieval_thread.start() for thread in threads: thread.join() - # do rerank for searched documents - model_manager = ModelManager() - rerank_model_instance = model_manager.get_model_instance( - tenant_id=tenant_id, - provider=reranking_provider_name, - model_type=ModelType.RERANK, - model=reranking_model_name - ) - rerank_runner = RerankRunner(rerank_model_instance) + if reranking_enable: + # do rerank for searched documents + data_post_processor = DataPostProcessor(tenant_id, reranking_mode, + reranking_model, weights, False) - with measure_time() as timer: - all_documents = rerank_runner.run( - query, all_documents, - score_threshold, - top_k - ) + with measure_time() as timer: + all_documents = data_post_processor.invoke( + query=query, + documents=all_documents, + score_threshold=score_threshold, + top_n=top_k + ) + else: + if index_type == "economy": + all_documents = self.calculate_keyword_score(query, all_documents, top_k) + elif index_type == "high_quality": + all_documents = self.calculate_vector_score(all_documents, top_k, score_threshold) self._on_query(query, dataset_ids, app_id, user_from, user_id) if all_documents: @@ -420,7 +430,8 @@ class DatasetRetrieval: score_threshold=retrieval_model['score_threshold'] if retrieval_model['score_threshold_enabled'] else None, reranking_model=retrieval_model['reranking_model'] - if retrieval_model['reranking_enable'] else None + if retrieval_model['reranking_enable'] else None, + weights=retrieval_model.get('weights', None), ) all_documents.extend(documents) @@ -464,7 +475,7 @@ class DatasetRetrieval: if retrieve_config.retrieve_strategy == DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE: # get retrieval model config default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -513,3 +524,94 @@ class DatasetRetrieval: tools.append(tool) return tools + + def calculate_keyword_score(self, query: str, documents: list[Document], top_k: int) -> list[Document]: + """ + Calculate keywords scores + :param query: search query + :param documents: documents for reranking + + :return: + """ + keyword_table_handler = JiebaKeywordTableHandler() + query_keywords = keyword_table_handler.extract_keywords(query, None) + documents_keywords = [] + for document in documents: + # get the document keywords + document_keywords = keyword_table_handler.extract_keywords(document.page_content, None) + document.metadata['keywords'] = document_keywords + documents_keywords.append(document_keywords) + + # Counter query keywords(TF) + query_keyword_counts = Counter(query_keywords) + + # total documents + total_documents = len(documents) + + # calculate all documents' keywords IDF + all_keywords = set() + for document_keywords in documents_keywords: + all_keywords.update(document_keywords) + + keyword_idf = {} + for keyword in all_keywords: + # calculate include query keywords' documents + doc_count_containing_keyword = sum(1 for doc_keywords in documents_keywords if keyword in doc_keywords) + # IDF + keyword_idf[keyword] = math.log((1 + total_documents) / (1 + doc_count_containing_keyword)) + 1 + + query_tfidf = {} + + for keyword, count in query_keyword_counts.items(): + tf = count + idf = keyword_idf.get(keyword, 0) + query_tfidf[keyword] = tf * idf + + # calculate all documents' TF-IDF + documents_tfidf = [] + for document_keywords in documents_keywords: + document_keyword_counts = Counter(document_keywords) + document_tfidf = {} + for keyword, count in document_keyword_counts.items(): + tf = count + idf = keyword_idf.get(keyword, 0) + document_tfidf[keyword] = tf * idf + documents_tfidf.append(document_tfidf) + + def cosine_similarity(vec1, vec2): + intersection = set(vec1.keys()) & set(vec2.keys()) + numerator = sum(vec1[x] * vec2[x] for x in intersection) + + sum1 = sum(vec1[x] ** 2 for x in vec1.keys()) + sum2 = sum(vec2[x] ** 2 for x in vec2.keys()) + denominator = math.sqrt(sum1) * math.sqrt(sum2) + + if not denominator: + return 0.0 + else: + return float(numerator) / denominator + + similarities = [] + for document_tfidf in documents_tfidf: + similarity = cosine_similarity(query_tfidf, document_tfidf) + similarities.append(similarity) + + for document, score in zip(documents, similarities): + # format document + document.metadata['score'] = score + documents = sorted(documents, key=lambda x: x.metadata['score'], reverse=True) + return documents[:top_k] if top_k else documents + + def calculate_vector_score(self, all_documents: list[Document], + top_k: int, score_threshold: float) -> list[Document]: + filter_documents = [] + for document in all_documents: + if document.metadata['score'] >= score_threshold: + filter_documents.append(document) + if not filter_documents: + return [] + filter_documents = sorted(filter_documents, key=lambda x: x.metadata['score'], reverse=True) + return filter_documents[:top_k] if top_k else filter_documents + + + diff --git a/api/core/rag/retrieval/retrival_methods.py b/api/core/rag/retrieval/retrival_methods.py index 9b7907013d..12aa28a51c 100644 --- a/api/core/rag/retrieval/retrival_methods.py +++ b/api/core/rag/retrieval/retrival_methods.py @@ -1,15 +1,15 @@ from enum import Enum -class RetrievalMethod(str, Enum): +class RetrievalMethod(Enum): SEMANTIC_SEARCH = 'semantic_search' FULL_TEXT_SEARCH = 'full_text_search' HYBRID_SEARCH = 'hybrid_search' @staticmethod def is_support_semantic_search(retrieval_method: str) -> bool: - return retrieval_method in {RetrievalMethod.SEMANTIC_SEARCH, RetrievalMethod.HYBRID_SEARCH} + return retrieval_method in {RetrievalMethod.SEMANTIC_SEARCH.value, RetrievalMethod.HYBRID_SEARCH.value} @staticmethod def is_support_fulltext_search(retrieval_method: str) -> bool: - return retrieval_method in {RetrievalMethod.FULL_TEXT_SEARCH, RetrievalMethod.HYBRID_SEARCH} + return retrieval_method in {RetrievalMethod.FULL_TEXT_SEARCH.value, RetrievalMethod.HYBRID_SEARCH.value} diff --git a/api/core/rag/splitter/__init__.py b/api/core/rag/splitter/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/tools/provider/_position.yaml b/api/core/tools/provider/_position.yaml index fa13629ef7..3a3ff64426 100644 --- a/api/core/tools/provider/_position.yaml +++ b/api/core/tools/provider/_position.yaml @@ -30,3 +30,4 @@ - feishu - feishu_base - slack +- tianditu diff --git a/api/core/tools/provider/builtin/cogview/tools/cogview3.py b/api/core/tools/provider/builtin/cogview/tools/cogview3.py index bb2720196f..89ffcf3347 100644 --- a/api/core/tools/provider/builtin/cogview/tools/cogview3.py +++ b/api/core/tools/provider/builtin/cogview/tools/cogview3.py @@ -30,7 +30,6 @@ class CogView3Tool(BuiltinTool): if not prompt: return self.create_text_message('Please input prompt') # get size - print(tool_parameters.get('prompt', 'square')) size = size_mapping[tool_parameters.get('size', 'square')] # get n n = tool_parameters.get('n', 1) @@ -58,8 +57,9 @@ class CogView3Tool(BuiltinTool): result = [] for image in response.data: result.append(self.create_image_message(image=image.url)) - result.append(self.create_text_message( - f'\nGenerate image source to Seed ID: {seed_id}')) + result.append(self.create_json_message({ + "url": image.url, + })) return result @staticmethod diff --git a/api/core/tools/provider/builtin/dalle/tools/dalle3.py b/api/core/tools/provider/builtin/dalle/tools/dalle3.py index 61609947fa..f985deade5 100644 --- a/api/core/tools/provider/builtin/dalle/tools/dalle3.py +++ b/api/core/tools/provider/builtin/dalle/tools/dalle3.py @@ -1,5 +1,5 @@ +import base64 import random -from base64 import b64decode from typing import Any, Union from openai import OpenAI @@ -69,11 +69,50 @@ class DallE3Tool(BuiltinTool): result = [] for image in response.data: - result.append(self.create_blob_message(blob=b64decode(image.b64_json), - meta={'mime_type': 'image/png'}, - save_as=self.VARIABLE_KEY.IMAGE.value)) + mime_type, blob_image = DallE3Tool._decode_image(image.b64_json) + blob_message = self.create_blob_message(blob=blob_image, + meta={'mime_type': mime_type}, + save_as=self.VARIABLE_KEY.IMAGE.value) + result.append(blob_message) return result + @staticmethod + def _decode_image(base64_image: str) -> tuple[str, bytes]: + """ + Decode a base64 encoded image. If the image is not prefixed with a MIME type, + it assumes 'image/png' as the default. + + :param base64_image: Base64 encoded image string + :return: A tuple containing the MIME type and the decoded image bytes + """ + if DallE3Tool._is_plain_base64(base64_image): + return 'image/png', base64.b64decode(base64_image) + else: + return DallE3Tool._extract_mime_and_data(base64_image) + + @staticmethod + def _is_plain_base64(encoded_str: str) -> bool: + """ + Check if the given encoded string is plain base64 without a MIME type prefix. + + :param encoded_str: Base64 encoded image string + :return: True if the string is plain base64, False otherwise + """ + return not encoded_str.startswith('data:image') + + @staticmethod + def _extract_mime_and_data(encoded_str: str) -> tuple[str, bytes]: + """ + Extract MIME type and image data from a base64 encoded string with a MIME type prefix. + + :param encoded_str: Base64 encoded image string with MIME type prefix + :return: A tuple containing the MIME type and the decoded image bytes + """ + mime_type = encoded_str.split(';')[0].split(':')[1] + image_data_base64 = encoded_str.split(',')[1] + decoded_data = base64.b64decode(image_data_base64) + return mime_type, decoded_data + @staticmethod def _generate_random_id(length=8): characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' diff --git a/api/core/tools/provider/builtin/firecrawl/firecrawl.py b/api/core/tools/provider/builtin/firecrawl/firecrawl.py index adcb7ebdd6..24dc35759d 100644 --- a/api/core/tools/provider/builtin/firecrawl/firecrawl.py +++ b/api/core/tools/provider/builtin/firecrawl/firecrawl.py @@ -1,22 +1,19 @@ from core.tools.errors import ToolProviderCredentialValidationError -from core.tools.provider.builtin.firecrawl.tools.crawl import CrawlTool +from core.tools.provider.builtin.firecrawl.tools.scrape import ScrapeTool from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController class FirecrawlProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: dict) -> None: try: - # Example validation using the Crawl tool - CrawlTool().fork_tool_runtime( + # Example validation using the ScrapeTool, only scraping title for minimize content + ScrapeTool().fork_tool_runtime( runtime={"credentials": credentials} ).invoke( user_id='', tool_parameters={ - "url": "https://example.com", - "includes": '', - "excludes": '', - "limit": 1, - "onlyMainContent": True, + "url": "https://google.com", + "onlyIncludeTags": 'title' } ) except Exception as e: diff --git a/api/core/tools/provider/builtin/firecrawl/firecrawl.yaml b/api/core/tools/provider/builtin/firecrawl/firecrawl.yaml index 613a0e4679..a48b9d9f54 100644 --- a/api/core/tools/provider/builtin/firecrawl/firecrawl.yaml +++ b/api/core/tools/provider/builtin/firecrawl/firecrawl.yaml @@ -31,8 +31,5 @@ credentials_for_provider: label: en_US: Firecrawl server's Base URL zh_Hans: Firecrawl服务器的API URL - pt_BR: Firecrawl server's Base URL placeholder: - en_US: https://www.firecrawl.dev - zh_HansL: https://www.firecrawl.dev - pt_BR: https://www.firecrawl.dev + en_US: https://api.firecrawl.dev diff --git a/api/core/tools/provider/builtin/firecrawl/firecrawl_appx.py b/api/core/tools/provider/builtin/firecrawl/firecrawl_appx.py index bfe3e7999d..3b3f78731b 100644 --- a/api/core/tools/provider/builtin/firecrawl/firecrawl_appx.py +++ b/api/core/tools/provider/builtin/firecrawl/firecrawl_appx.py @@ -1,3 +1,4 @@ +import json import logging import time from collections.abc import Mapping @@ -8,6 +9,7 @@ from requests.exceptions import HTTPError logger = logging.getLogger(__name__) + class FirecrawlApp: def __init__(self, api_key: str | None = None, base_url: str | None = None): self.api_key = api_key @@ -25,14 +27,16 @@ class FirecrawlApp: return headers def _request( - self, - method: str, - url: str, - data: Mapping[str, Any] | None = None, - headers: Mapping[str, str] | None = None, - retries: int = 3, - backoff_factor: float = 0.3, + self, + method: str, + url: str, + data: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + retries: int = 3, + backoff_factor: float = 0.3, ) -> Mapping[str, Any] | None: + if not headers: + headers = self._prepare_headers() for i in range(retries): try: response = requests.request(method, url, json=data, headers=headers) @@ -47,47 +51,51 @@ class FirecrawlApp: def scrape_url(self, url: str, **kwargs): endpoint = f'{self.base_url}/v0/scrape' - headers = self._prepare_headers() data = {'url': url, **kwargs} - response = self._request('POST', endpoint, data, headers) logger.debug(f"Sent request to {endpoint=} body={data}") + response = self._request('POST', endpoint, data) if response is None: raise HTTPError("Failed to scrape URL after multiple retries") return response def search(self, query: str, **kwargs): endpoint = f'{self.base_url}/v0/search' - headers = self._prepare_headers() data = {'query': query, **kwargs} - response = self._request('POST', endpoint, data, headers) logger.debug(f"Sent request to {endpoint=} body={data}") + response = self._request('POST', endpoint, data) if response is None: raise HTTPError("Failed to perform search after multiple retries") return response def crawl_url( - self, url: str, wait: bool = False, poll_interval: int = 5, idempotency_key: str | None = None, **kwargs + self, url: str, wait: bool = True, poll_interval: int = 5, idempotency_key: str | None = None, **kwargs ): endpoint = f'{self.base_url}/v0/crawl' headers = self._prepare_headers(idempotency_key) - data = {'url': url, **kwargs['params']} - response = self._request('POST', endpoint, data, headers) + data = {'url': url, **kwargs} logger.debug(f"Sent request to {endpoint=} body={data}") + response = self._request('POST', endpoint, data, headers) if response is None: raise HTTPError("Failed to initiate crawl after multiple retries") job_id: str = response['jobId'] if wait: return self._monitor_job_status(job_id=job_id, poll_interval=poll_interval) - return job_id + return response def check_crawl_status(self, job_id: str): endpoint = f'{self.base_url}/v0/crawl/status/{job_id}' - headers = self._prepare_headers() - response = self._request('GET', endpoint, headers=headers) + response = self._request('GET', endpoint) if response is None: raise HTTPError(f"Failed to check status for job {job_id} after multiple retries") return response + def cancel_crawl_job(self, job_id: str): + endpoint = f'{self.base_url}/v0/crawl/cancel/{job_id}' + response = self._request('DELETE', endpoint) + if response is None: + raise HTTPError(f"Failed to cancel job {job_id} after multiple retries") + return response + def _monitor_job_status(self, job_id: str, poll_interval: int): while True: status = self.check_crawl_status(job_id) @@ -96,3 +104,21 @@ class FirecrawlApp: elif status['status'] == 'failed': raise HTTPError(f'Job {job_id} failed: {status["error"]}') time.sleep(poll_interval) + + +def get_array_params(tool_parameters: dict[str, Any], key): + param = tool_parameters.get(key) + if param: + return param.split(',') + + +def get_json_params(tool_parameters: dict[str, Any], key): + param = tool_parameters.get(key) + if param: + try: + # support both single quotes and double quotes + param = param.replace("'", '"') + param = json.loads(param) + except: + raise ValueError(f"Invalid {key} format.") + return param diff --git a/api/core/tools/provider/builtin/firecrawl/tools/crawl.py b/api/core/tools/provider/builtin/firecrawl/tools/crawl.py index b000c1c6ce..08c40a4064 100644 --- a/api/core/tools/provider/builtin/firecrawl/tools/crawl.py +++ b/api/core/tools/provider/builtin/firecrawl/tools/crawl.py @@ -1,36 +1,48 @@ -import json -from typing import Any, Union +from typing import Any from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.provider.builtin.firecrawl.firecrawl_appx import FirecrawlApp +from core.tools.provider.builtin.firecrawl.firecrawl_appx import FirecrawlApp, get_array_params, get_json_params from core.tools.tool.builtin_tool import BuiltinTool class CrawlTool(BuiltinTool): - def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: - app = FirecrawlApp(api_key=self.runtime.credentials['firecrawl_api_key'], base_url=self.runtime.credentials['base_url']) + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage: + """ + the crawlerOptions and pageOptions comes from doc here: + https://docs.firecrawl.dev/api-reference/endpoint/crawl + """ + app = FirecrawlApp(api_key=self.runtime.credentials['firecrawl_api_key'], + base_url=self.runtime.credentials['base_url']) + crawlerOptions = {} + pageOptions = {} - options = { - 'crawlerOptions': { - 'excludes': tool_parameters.get('excludes', '').split(',') if tool_parameters.get('excludes') else [], - 'includes': tool_parameters.get('includes', '').split(',') if tool_parameters.get('includes') else [], - 'limit': tool_parameters.get('limit', 5) - }, - 'pageOptions': { - 'onlyMainContent': tool_parameters.get('onlyMainContent', False) - } - } + wait_for_results = tool_parameters.get('wait_for_results', True) + + crawlerOptions['excludes'] = get_array_params(tool_parameters, 'excludes') + crawlerOptions['includes'] = get_array_params(tool_parameters, 'includes') + crawlerOptions['returnOnlyUrls'] = tool_parameters.get('returnOnlyUrls', False) + crawlerOptions['maxDepth'] = tool_parameters.get('maxDepth') + crawlerOptions['mode'] = tool_parameters.get('mode') + crawlerOptions['ignoreSitemap'] = tool_parameters.get('ignoreSitemap', False) + crawlerOptions['limit'] = tool_parameters.get('limit', 5) + crawlerOptions['allowBackwardCrawling'] = tool_parameters.get('allowBackwardCrawling', False) + crawlerOptions['allowExternalContentLinks'] = tool_parameters.get('allowExternalContentLinks', False) + + pageOptions['headers'] = get_json_params(tool_parameters, 'headers') + pageOptions['includeHtml'] = tool_parameters.get('includeHtml', False) + pageOptions['includeRawHtml'] = tool_parameters.get('includeRawHtml', False) + pageOptions['onlyIncludeTags'] = get_array_params(tool_parameters, 'onlyIncludeTags') + pageOptions['removeTags'] = get_array_params(tool_parameters, 'removeTags') + pageOptions['onlyMainContent'] = tool_parameters.get('onlyMainContent', False) + pageOptions['replaceAllPathsWithAbsolutePaths'] = tool_parameters.get('replaceAllPathsWithAbsolutePaths', False) + pageOptions['screenshot'] = tool_parameters.get('screenshot', False) + pageOptions['waitFor'] = tool_parameters.get('waitFor', 0) crawl_result = app.crawl_url( - url=tool_parameters['url'], - params=options, - wait=True + url=tool_parameters['url'], + wait=wait_for_results, + crawlerOptions=crawlerOptions, + pageOptions=pageOptions ) - if not isinstance(crawl_result, str): - crawl_result = json.dumps(crawl_result, ensure_ascii=False, indent=4) - - if not crawl_result: - return self.create_text_message("Crawl request failed.") - - return self.create_text_message(crawl_result) + return self.create_json_message(crawl_result) diff --git a/api/core/tools/provider/builtin/firecrawl/tools/crawl.yaml b/api/core/tools/provider/builtin/firecrawl/tools/crawl.yaml index 3861670140..0c5399f973 100644 --- a/api/core/tools/provider/builtin/firecrawl/tools/crawl.yaml +++ b/api/core/tools/provider/builtin/firecrawl/tools/crawl.yaml @@ -3,76 +3,243 @@ identity: author: Richards Tu label: en_US: Crawl - zh_Hans: 爬取 + zh_Hans: 深度爬取 description: human: - en_US: Extract data from a website by crawling through a URL. - zh_Hans: 通过URL从网站中提取数据。 + en_US: Recursively search through a urls subdomains, and gather the content. + zh_Hans: 递归爬取一个网址的子域名，并收集内容。 llm: This tool initiates a web crawl to extract data from a specified URL. It allows configuring crawler options such as including or excluding URL patterns, generating alt text for images using LLMs (paid plan required), limiting the maximum number of pages to crawl, and returning only the main content of the page. The tool can return either a list of crawled documents or a list of URLs based on the provided options. parameters: - name: url type: string required: true label: - en_US: URL to crawl - zh_Hans: 要爬取的URL + en_US: Start URL + zh_Hans: 起始URL human_description: - en_US: The URL of the website to crawl and extract data from. - zh_Hans: 要爬取并提取数据的网站URL。 + en_US: The base URL to start crawling from. + zh_Hans: 要爬取网站的起始URL。 llm_description: The URL of the website that needs to be crawled. This is a required parameter. form: llm + - name: wait_for_results + type: boolean + default: true + label: + en_US: Wait For Results + zh_Hans: 等待爬取结果 + human_description: + en_US: If you choose not to wait, it will directly return a job ID. You can use this job ID to check the crawling results or cancel the crawling task, which is usually very useful for a large-scale crawling task. + zh_Hans: 如果选择不等待，则会直接返回一个job_id，可以通过job_id查询爬取结果或取消爬取任务，这通常对于一个大型爬取任务来说非常有用。 + form: form +############## Crawl Options ####################### - name: includes type: string required: false label: en_US: URL patterns to include zh_Hans: 要包含的URL模式 + placeholder: + en_US: Use commas to separate multiple tags + zh_Hans: 多个标签时使用半角逗号分隔 human_description: - en_US: Specify URL patterns to include during the crawl. Only pages matching these patterns will be crawled, you can use ',' to separate multiple patterns. - zh_Hans: 指定爬取过程中要包含的URL模式。只有与这些模式匹配的页面才会被爬取。 + en_US: | + Only pages matching these patterns will be crawled. Example: blog/*, about/* + zh_Hans: 只有与这些模式匹配的页面才会被爬取。示例：blog/*, about/* form: form - default: '' - name: excludes type: string - required: false label: en_US: URL patterns to exclude zh_Hans: 要排除的URL模式 + placeholder: + en_US: Use commas to separate multiple tags + zh_Hans: 多个标签时使用半角逗号分隔 human_description: - en_US: Specify URL patterns to exclude during the crawl. Pages matching these patterns will be skipped, you can use ',' to separate multiple patterns. - zh_Hans: 指定爬取过程中要排除的URL模式。匹配这些模式的页面将被跳过。 + en_US: | + Pages matching these patterns will be skipped. Example: blog/*, about/* + zh_Hans: 匹配这些模式的页面将被跳过。示例：blog/*, about/* + form: form + - name: returnOnlyUrls + type: boolean + default: false + label: + en_US: return Only Urls + zh_Hans: 仅返回URL + human_description: + en_US: | + If true, returns only the URLs as a list on the crawl status. Attention: the return response will be a list of URLs inside the data, not a list of documents. + zh_Hans: 只返回爬取到的网页链接，而不是网页内容本身。 + form: form + - name: maxDepth + type: number + label: + en_US: Maximum crawl depth + zh_Hans: 爬取深度 + human_description: + en_US: Maximum depth to crawl relative to the entered URL. A maxDepth of 0 scrapes only the entered URL. A maxDepth of 1 scrapes the entered URL and all pages one level deep. A maxDepth of 2 scrapes the entered URL and all pages up to two levels deep. Higher values follow the same pattern. + zh_Hans: 相对于输入的URL，爬取的最大深度。maxDepth为0时，仅抓取输入的URL。maxDepth为1时，抓取输入的URL以及所有一级深层页面。maxDepth为2时，抓取输入的URL以及所有两级深层页面。更高值遵循相同模式。 + form: form + min: 0 + - name: mode + type: select + required: false + form: form + options: + - value: default + label: + en_US: default + - value: fast + label: + en_US: fast + default: default + label: + en_US: Crawl Mode + zh_Hans: 爬取模式 + human_description: + en_US: The crawling mode to use. Fast mode crawls 4x faster websites without sitemap, but may not be as accurate and shouldn't be used in heavy js-rendered websites. + zh_Hans: 使用fast模式将不会使用其站点地图，比普通模式快4倍，但是可能不够准确，也不适用于大量js渲染的网站。 + - name: ignoreSitemap + type: boolean + default: false + label: + en_US: ignore Sitemap + zh_Hans: 忽略站点地图 + human_description: + en_US: Ignore the website sitemap when crawling. + zh_Hans: 爬取时忽略网站站点地图。 form: form - default: 'blog/*' - name: limit type: number required: false label: - en_US: Maximum number of pages to crawl + en_US: Maximum pages to crawl zh_Hans: 最大爬取页面数 human_description: en_US: Specify the maximum number of pages to crawl. The crawler will stop after reaching this limit. zh_Hans: 指定要爬取的最大页面数。爬虫将在达到此限制后停止。 form: form min: 1 - max: 20 default: 5 + - name: allowBackwardCrawling + type: boolean + default: false + label: + en_US: allow Backward Crawling + zh_Hans: 允许向后爬取 + human_description: + en_US: Enables the crawler to navigate from a specific URL to previously linked pages. For instance, from 'example.com/product/123' back to 'example.com/product' + zh_Hans: 使爬虫能够从特定URL导航到之前链接的页面。例如，从'example.com/product/123'返回到'example.com/product' + form: form + - name: allowExternalContentLinks + type: boolean + default: false + label: + en_US: allow External Content Links + zh_Hans: 允许爬取外链 + human_description: + en_US: Allows the crawler to follow links to external websites. + zh_Hans: + form: form +############## Page Options ####################### + - name: headers + type: string + label: + en_US: headers + zh_Hans: 请求头 + human_description: + en_US: | + Headers to send with the request. Can be used to send cookies, user-agent, etc. Example: {"cookies": "testcookies"} + zh_Hans: | + 随请求发送的头部。可以用来发送cookies、用户代理等。示例：{"cookies": "testcookies"} + placeholder: + en_US: Please enter an object that can be serialized in JSON + zh_Hans: 请输入可以json序列化的对象 + form: form + - name: includeHtml + type: boolean + default: false + label: + en_US: include Html + zh_Hans: 包含HTML + human_description: + en_US: Include the HTML version of the content on page. Will output a html key in the response. + zh_Hans: 返回中包含一个HTML版本的内容，将以html键返回。 + form: form + - name: includeRawHtml + type: boolean + default: false + label: + en_US: include Raw Html + zh_Hans: 包含原始HTML + human_description: + en_US: Include the raw HTML content of the page. Will output a rawHtml key in the response. + zh_Hans: 返回中包含一个原始HTML版本的内容，将以rawHtml键返回。 + form: form + - name: onlyIncludeTags + type: string + label: + en_US: only Include Tags + zh_Hans: 仅抓取这些标签 + placeholder: + en_US: Use commas to separate multiple tags + zh_Hans: 多个标签时使用半角逗号分隔 + human_description: + en_US: | + Only include tags, classes and ids from the page in the final output. Use comma separated values. Example: script, .ad, #footer + zh_Hans: | + 仅在最终输出中包含HTML页面的这些标签，可以通过标签名、类或ID来设定，使用逗号分隔值。示例：script, .ad, #footer + form: form - name: onlyMainContent type: boolean - required: false + default: false label: - en_US: Only return the main content of the page - zh_Hans: 仅返回页面的主要内容 + en_US: only Main Content + zh_Hans: 仅抓取主要内容 human_description: - en_US: If enabled, the crawler will only return the main content of the page, excluding headers, navigation, footers, etc. - zh_Hans: 如果启用，爬虫将仅返回页面的主要内容，不包括标题、导航、页脚等。 + en_US: Only return the main content of the page excluding headers, navs, footers, etc. + zh_Hans: 只返回页面的主要内容，不包括头部、导航栏、尾部等。 + form: form + - name: removeTags + type: string + label: + en_US: remove Tags + zh_Hans: 要移除这些标签 + human_description: + en_US: | + Tags, classes and ids to remove from the page. Use comma separated values. Example: script, .ad, #footer + zh_Hans: | + 要在最终输出中移除HTML页面的这些标签，可以通过标签名、类或ID来设定，使用逗号分隔值。示例：script, .ad, #footer + placeholder: + en_US: Use commas to separate multiple tags + zh_Hans: 多个标签时使用半角逗号分隔 + form: form + - name: replaceAllPathsWithAbsolutePaths + type: boolean + default: false + label: + en_US: All AbsolutePaths + zh_Hans: 使用绝对路径 + human_description: + en_US: Replace all relative paths with absolute paths for images and links. + zh_Hans: 将所有图片和链接的相对路径替换为绝对路径。 + form: form + - name: screenshot + type: boolean + default: false + label: + en_US: screenshot + zh_Hans: 截图 + human_description: + en_US: Include a screenshot of the top of the page that you are scraping. + zh_Hans: 提供正在抓取的页面的顶部的截图。 + form: form + - name: waitFor + type: number + min: 0 + label: + en_US: wait For + zh_Hans: 等待时间 + human_description: + en_US: Wait x amount of milliseconds for the page to load to fetch content. + zh_Hans: 等待x毫秒以使页面加载并获取内容。 form: form - options: - - value: 'true' - label: - en_US: 'Yes' - zh_Hans: 是 - - value: 'false' - label: - en_US: 'No' - zh_Hans: 否 - default: 'false' diff --git a/api/core/tools/provider/builtin/firecrawl/tools/crawl_job.py b/api/core/tools/provider/builtin/firecrawl/tools/crawl_job.py new file mode 100644 index 0000000000..fa6c1f87ee --- /dev/null +++ b/api/core/tools/provider/builtin/firecrawl/tools/crawl_job.py @@ -0,0 +1,20 @@ +from typing import Any + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.provider.builtin.firecrawl.firecrawl_appx import FirecrawlApp +from core.tools.tool.builtin_tool import BuiltinTool + + +class CrawlJobTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage: + app = FirecrawlApp(api_key=self.runtime.credentials['firecrawl_api_key'], + base_url=self.runtime.credentials['base_url']) + operation = tool_parameters.get('operation', 'get') + if operation == 'get': + result = app.check_crawl_status(job_id=tool_parameters['job_id']) + elif operation == 'cancel': + result = app.cancel_crawl_job(job_id=tool_parameters['job_id']) + else: + raise ValueError(f'Invalid operation: {operation}') + + return self.create_json_message(result) diff --git a/api/core/tools/provider/builtin/firecrawl/tools/crawl_job.yaml b/api/core/tools/provider/builtin/firecrawl/tools/crawl_job.yaml new file mode 100644 index 0000000000..78008e4ad4 --- /dev/null +++ b/api/core/tools/provider/builtin/firecrawl/tools/crawl_job.yaml @@ -0,0 +1,37 @@ +identity: + name: crawl_job + author: hjlarry + label: + en_US: Crawl Job + zh_Hans: 爬取任务处理 +description: + human: + en_US: Retrieve the scraping results based on the job ID, or cancel the scraping task. + zh_Hans: 根据爬取任务ID获取爬取结果，或者取消爬取任务 + llm: Retrieve the scraping results based on the job ID, or cancel the scraping task. +parameters: + - name: job_id + type: string + required: true + label: + en_US: Job ID + human_description: + en_US: Set wait_for_results to false in the Crawl tool can get the job ID. + zh_Hans: 在深度爬取工具中将等待爬取结果设置为否可以获取Job ID。 + llm_description: Set wait_for_results to false in the Crawl tool can get the job ID. + form: llm + - name: operation + type: select + required: true + options: + - value: get + label: + en_US: get crawl status + - value: cancel + label: + en_US: cancel crawl job + label: + en_US: operation + zh_Hans: 操作 + llm_description: choose the operation to perform. `get` is for getting the crawl status, `cancel` is for cancelling the crawl job. + form: llm diff --git a/api/core/tools/provider/builtin/firecrawl/tools/scrape.py b/api/core/tools/provider/builtin/firecrawl/tools/scrape.py index 3a78dce8d0..91412da548 100644 --- a/api/core/tools/provider/builtin/firecrawl/tools/scrape.py +++ b/api/core/tools/provider/builtin/firecrawl/tools/scrape.py @@ -1,26 +1,39 @@ -import json -from typing import Any, Union +from typing import Any from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.provider.builtin.firecrawl.firecrawl_appx import FirecrawlApp +from core.tools.provider.builtin.firecrawl.firecrawl_appx import FirecrawlApp, get_array_params, get_json_params from core.tools.tool.builtin_tool import BuiltinTool class ScrapeTool(BuiltinTool): - def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: - app = FirecrawlApp(api_key=self.runtime.credentials['firecrawl_api_key'], base_url=self.runtime.credentials['base_url']) - crawl_result = app.scrape_url( - url=tool_parameters['url'], - wait=True - ) + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage: + """ + the pageOptions and extractorOptions comes from doc here: + https://docs.firecrawl.dev/api-reference/endpoint/scrape + """ + app = FirecrawlApp(api_key=self.runtime.credentials['firecrawl_api_key'], + base_url=self.runtime.credentials['base_url']) - if isinstance(crawl_result, dict): - result_message = json.dumps(crawl_result, ensure_ascii=False, indent=4) - else: - result_message = str(crawl_result) + pageOptions = {} + extractorOptions = {} - if not crawl_result: - return self.create_text_message("Scrape request failed.") + pageOptions['headers'] = get_json_params(tool_parameters, 'headers') + pageOptions['includeHtml'] = tool_parameters.get('includeHtml', False) + pageOptions['includeRawHtml'] = tool_parameters.get('includeRawHtml', False) + pageOptions['onlyIncludeTags'] = get_array_params(tool_parameters, 'onlyIncludeTags') + pageOptions['removeTags'] = get_array_params(tool_parameters, 'removeTags') + pageOptions['onlyMainContent'] = tool_parameters.get('onlyMainContent', False) + pageOptions['replaceAllPathsWithAbsolutePaths'] = tool_parameters.get('replaceAllPathsWithAbsolutePaths', False) + pageOptions['screenshot'] = tool_parameters.get('screenshot', False) + pageOptions['waitFor'] = tool_parameters.get('waitFor', 0) - return self.create_text_message(result_message) + extractorOptions['mode'] = tool_parameters.get('mode', '') + extractorOptions['extractionPrompt'] = tool_parameters.get('extractionPrompt', '') + extractorOptions['extractionSchema'] = get_json_params(tool_parameters, 'extractionSchema') + + crawl_result = app.scrape_url(url=tool_parameters['url'], + pageOptions=pageOptions, + extractorOptions=extractorOptions) + + return self.create_json_message(crawl_result) diff --git a/api/core/tools/provider/builtin/firecrawl/tools/scrape.yaml b/api/core/tools/provider/builtin/firecrawl/tools/scrape.yaml index 29aa5991aa..598429de5e 100644 --- a/api/core/tools/provider/builtin/firecrawl/tools/scrape.yaml +++ b/api/core/tools/provider/builtin/firecrawl/tools/scrape.yaml @@ -3,7 +3,7 @@ identity: author: ahasasjeb label: en_US: Scrape - zh_Hans: 抓取 + zh_Hans: 单页面抓取 description: human: en_US: Extract data from a single URL. @@ -21,3 +21,160 @@ parameters: zh_Hans: 要抓取并提取数据的网站URL。 llm_description: The URL of the website that needs to be crawled. This is a required parameter. form: llm +############## Page Options ####################### + - name: headers + type: string + label: + en_US: headers + zh_Hans: 请求头 + human_description: + en_US: | + Headers to send with the request. Can be used to send cookies, user-agent, etc. Example: {"cookies": "testcookies"} + zh_Hans: | + 随请求发送的头部。可以用来发送cookies、用户代理等。示例：{"cookies": "testcookies"} + placeholder: + en_US: Please enter an object that can be serialized in JSON + zh_Hans: 请输入可以json序列化的对象 + form: form + - name: includeHtml + type: boolean + default: false + label: + en_US: include Html + zh_Hans: 包含HTML + human_description: + en_US: Include the HTML version of the content on page. Will output a html key in the response. + zh_Hans: 返回中包含一个HTML版本的内容，将以html键返回。 + form: form + - name: includeRawHtml + type: boolean + default: false + label: + en_US: include Raw Html + zh_Hans: 包含原始HTML + human_description: + en_US: Include the raw HTML content of the page. Will output a rawHtml key in the response. + zh_Hans: 返回中包含一个原始HTML版本的内容，将以rawHtml键返回。 + form: form + - name: onlyIncludeTags + type: string + label: + en_US: only Include Tags + zh_Hans: 仅抓取这些标签 + placeholder: + en_US: Use commas to separate multiple tags + zh_Hans: 多个标签时使用半角逗号分隔 + human_description: + en_US: | + Only include tags, classes and ids from the page in the final output. Use comma separated values. Example: script, .ad, #footer + zh_Hans: | + 仅在最终输出中包含HTML页面的这些标签，可以通过标签名、类或ID来设定，使用逗号分隔值。示例：script, .ad, #footer + form: form + - name: onlyMainContent + type: boolean + default: false + label: + en_US: only Main Content + zh_Hans: 仅抓取主要内容 + human_description: + en_US: Only return the main content of the page excluding headers, navs, footers, etc. + zh_Hans: 只返回页面的主要内容，不包括头部、导航栏、尾部等。 + form: form + - name: removeTags + type: string + label: + en_US: remove Tags + zh_Hans: 要移除这些标签 + human_description: + en_US: | + Tags, classes and ids to remove from the page. Use comma separated values. Example: script, .ad, #footer + zh_Hans: | + 要在最终输出中移除HTML页面的这些标签，可以通过标签名、类或ID来设定，使用逗号分隔值。示例：script, .ad, #footer + placeholder: + en_US: Use commas to separate multiple tags + zh_Hans: 多个标签时使用半角逗号分隔 + form: form + - name: replaceAllPathsWithAbsolutePaths + type: boolean + default: false + label: + en_US: All AbsolutePaths + zh_Hans: 使用绝对路径 + human_description: + en_US: Replace all relative paths with absolute paths for images and links. + zh_Hans: 将所有图片和链接的相对路径替换为绝对路径。 + form: form + - name: screenshot + type: boolean + default: false + label: + en_US: screenshot + zh_Hans: 截图 + human_description: + en_US: Include a screenshot of the top of the page that you are scraping. + zh_Hans: 提供正在抓取的页面的顶部的截图。 + form: form + - name: waitFor + type: number + min: 0 + label: + en_US: wait For + zh_Hans: 等待时间 + human_description: + en_US: Wait x amount of milliseconds for the page to load to fetch content. + zh_Hans: 等待x毫秒以使页面加载并获取内容。 + form: form +############## Extractor Options ####################### + - name: mode + type: select + options: + - value: markdown + label: + en_US: markdown + - value: llm-extraction + label: + en_US: llm-extraction + - value: llm-extraction-from-raw-html + label: + en_US: llm-extraction-from-raw-html + - value: llm-extraction-from-markdown + label: + en_US: llm-extraction-from-markdown + label: + en_US: Extractor Mode + zh_Hans: 提取模式 + human_description: + en_US: | + The extraction mode to use. 'markdown': Returns the scraped markdown content, does not perform LLM extraction. 'llm-extraction': Extracts information from the cleaned and parsed content using LLM. + zh_Hans: 使用的提取模式。“markdown”：返回抓取的markdown内容，不执行LLM提取。“llm-extractioin”：使用LLM按Extractor Schema从内容中提取信息。 + form: form + - name: extractionPrompt + type: string + label: + en_US: Extractor Prompt + zh_Hans: 提取时的提示词 + human_description: + en_US: A prompt describing what information to extract from the page, applicable for LLM extraction modes. + zh_Hans: 当使用LLM提取模式时，用于给LLM描述提取规则。 + form: form + - name: extractionSchema + type: string + label: + en_US: Extractor Schema + zh_Hans: 提取时的结构 + placeholder: + en_US: Please enter an object that can be serialized in JSON + human_description: + en_US: | + The schema for the data to be extracted, required only for LLM extraction modes. Example: { + "type": "object", + "properties": {"company_mission": {"type": "string"}}, + "required": ["company_mission"] + } + zh_Hans: | + 当使用LLM提取模式时，使用该结构去提取，示例：{ + "type": "object", + "properties": {"company_mission": {"type": "string"}}, + "required": ["company_mission"] + } + form: form diff --git a/api/core/tools/provider/builtin/firecrawl/tools/search.py b/api/core/tools/provider/builtin/firecrawl/tools/search.py index 0b118aa5f1..e2b2ac6b4d 100644 --- a/api/core/tools/provider/builtin/firecrawl/tools/search.py +++ b/api/core/tools/provider/builtin/firecrawl/tools/search.py @@ -1,5 +1,4 @@ -import json -from typing import Any, Union +from typing import Any from core.tools.entities.tool_entities import ToolInvokeMessage from core.tools.provider.builtin.firecrawl.firecrawl_appx import FirecrawlApp @@ -7,20 +6,23 @@ from core.tools.tool.builtin_tool import BuiltinTool class SearchTool(BuiltinTool): - def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: - app = FirecrawlApp(api_key=self.runtime.credentials['firecrawl_api_key'], base_url=self.runtime.credentials['base_url']) - - crawl_result = app.search( + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage: + """ + the pageOptions and searchOptions comes from doc here: + https://docs.firecrawl.dev/api-reference/endpoint/search + """ + app = FirecrawlApp(api_key=self.runtime.credentials['firecrawl_api_key'], + base_url=self.runtime.credentials['base_url']) + pageOptions = {} + pageOptions['onlyMainContent'] = tool_parameters.get('onlyMainContent', False) + pageOptions['fetchPageContent'] = tool_parameters.get('fetchPageContent', True) + pageOptions['includeHtml'] = tool_parameters.get('includeHtml', False) + pageOptions['includeRawHtml'] = tool_parameters.get('includeRawHtml', False) + searchOptions = {'limit': tool_parameters.get('limit')} + search_result = app.search( query=tool_parameters['keyword'], - wait=True + pageOptions=pageOptions, + searchOptions=searchOptions ) - if isinstance(crawl_result, dict): - result_message = json.dumps(crawl_result, ensure_ascii=False, indent=4) - else: - result_message = str(crawl_result) - - if not crawl_result: - return self.create_text_message("Search request failed.") - - return self.create_text_message(result_message) + return self.create_json_message(search_result) diff --git a/api/core/tools/provider/builtin/firecrawl/tools/search.yaml b/api/core/tools/provider/builtin/firecrawl/tools/search.yaml index b1513c914e..29df0cfaaa 100644 --- a/api/core/tools/provider/builtin/firecrawl/tools/search.yaml +++ b/api/core/tools/provider/builtin/firecrawl/tools/search.yaml @@ -21,3 +21,55 @@ parameters: zh_Hans: 输入关键词即可使用Firecrawl API进行搜索。 llm_description: Efficiently extract keywords from user text. form: llm +############## Page Options ####################### + - name: onlyMainContent + type: boolean + default: false + label: + en_US: only Main Content + zh_Hans: 仅抓取主要内容 + human_description: + en_US: Only return the main content of the page excluding headers, navs, footers, etc. + zh_Hans: 只返回页面的主要内容，不包括头部、导航栏、尾部等。 + form: form + - name: fetchPageContent + type: boolean + default: true + label: + en_US: fetch Page Content + zh_Hans: 抓取页面内容 + human_description: + en_US: Fetch the content of each page. If false, defaults to a basic fast serp API. + zh_Hans: 获取每个页面的内容。如果为否，则使用基本的快速搜索结果页面API。 + form: form + - name: includeHtml + type: boolean + default: false + label: + en_US: include Html + zh_Hans: 包含HTML + human_description: + en_US: Include the HTML version of the content on page. Will output a html key in the response. + zh_Hans: 返回中包含一个HTML版本的内容，将以html键返回。 + form: form + - name: includeRawHtml + type: boolean + default: false + label: + en_US: include Raw Html + zh_Hans: 包含原始HTML + human_description: + en_US: Include the raw HTML content of the page. Will output a rawHtml key in the response. + zh_Hans: 返回中包含一个原始HTML版本的内容，将以rawHtml键返回。 + form: form +############## Search Options ####################### + - name: limit + type: number + min: 0 + label: + en_US: Maximum results + zh_Hans: 最大结果数量 + human_description: + en_US: Maximum number of results. Max is 20 during beta. + zh_Hans: 最大结果数量。在测试阶段，最大为20。 + form: form diff --git a/api/core/tools/provider/builtin/getimgai/_assets/icon.svg b/api/core/tools/provider/builtin/getimgai/_assets/icon.svg new file mode 100644 index 0000000000..6b2513386d --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/_assets/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/api/core/tools/provider/builtin/getimgai/getimgai.py b/api/core/tools/provider/builtin/getimgai/getimgai.py new file mode 100644 index 0000000000..c81d5fa333 --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/getimgai.py @@ -0,0 +1,22 @@ +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.getimgai.tools.text2image import Text2ImageTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class GetImgAIProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict) -> None: + try: + # Example validation using the text2image tool + Text2ImageTool().fork_tool_runtime( + runtime={"credentials": credentials} + ).invoke( + user_id='', + tool_parameters={ + "prompt": "A fire egg", + "response_format": "url", + "style": "photorealism", + } + ) + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) + \ No newline at end of file diff --git a/api/core/tools/provider/builtin/getimgai/getimgai.yaml b/api/core/tools/provider/builtin/getimgai/getimgai.yaml new file mode 100644 index 0000000000..c9db0a9e22 --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/getimgai.yaml @@ -0,0 +1,29 @@ +identity: + author: Matri Qi + name: getimgai + label: + en_US: getimg.ai + zh_CN: getimg.ai + description: + en_US: GetImg API integration for image generation and scraping. + icon: icon.svg + tags: + - image +credentials_for_provider: + getimg_api_key: + type: secret-input + required: true + label: + en_US: getimg.ai API Key + placeholder: + en_US: Please input your getimg.ai API key + help: + en_US: Get your getimg.ai API key from your getimg.ai account settings. If you are using a self-hosted version, you may enter any key at your convenience. + url: https://dashboard.getimg.ai/api-keys + base_url: + type: text-input + required: false + label: + en_US: getimg.ai server's Base URL + placeholder: + en_US: https://api.getimg.ai/v1 diff --git a/api/core/tools/provider/builtin/getimgai/getimgai_appx.py b/api/core/tools/provider/builtin/getimgai/getimgai_appx.py new file mode 100644 index 0000000000..e28c57649c --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/getimgai_appx.py @@ -0,0 +1,59 @@ +import logging +import time +from collections.abc import Mapping +from typing import Any + +import requests +from requests.exceptions import HTTPError + +logger = logging.getLogger(__name__) + +class GetImgAIApp: + def __init__(self, api_key: str | None = None, base_url: str | None = None): + self.api_key = api_key + self.base_url = base_url or 'https://api.getimg.ai/v1' + if not self.api_key: + raise ValueError("API key is required") + + def _prepare_headers(self): + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self.api_key}' + } + return headers + + def _request( + self, + method: str, + url: str, + data: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + retries: int = 3, + backoff_factor: float = 0.3, + ) -> Mapping[str, Any] | None: + for i in range(retries): + try: + response = requests.request(method, url, json=data, headers=headers) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + if i < retries - 1 and isinstance(e, HTTPError) and e.response.status_code >= 500: + time.sleep(backoff_factor * (2 ** i)) + else: + raise + return None + + def text2image( + self, mode: str, **kwargs + ): + data = kwargs['params'] + if not data.get('prompt'): + raise ValueError("Prompt is required") + + endpoint = f'{self.base_url}/{mode}/text-to-image' + headers = self._prepare_headers() + logger.debug(f"Send request to {endpoint=} body={data}") + response = self._request('POST', endpoint, data, headers) + if response is None: + raise HTTPError("Failed to initiate getimg.ai after multiple retries") + return response diff --git a/api/core/tools/provider/builtin/getimgai/tools/text2image.py b/api/core/tools/provider/builtin/getimgai/tools/text2image.py new file mode 100644 index 0000000000..dad7314479 --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/tools/text2image.py @@ -0,0 +1,39 @@ +import json +from typing import Any, Union + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.provider.builtin.getimgai.getimgai_appx import GetImgAIApp +from core.tools.tool.builtin_tool import BuiltinTool + + +class Text2ImageTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + app = GetImgAIApp(api_key=self.runtime.credentials['getimg_api_key'], base_url=self.runtime.credentials['base_url']) + + options = { + 'style': tool_parameters.get('style'), + 'prompt': tool_parameters.get('prompt'), + 'aspect_ratio': tool_parameters.get('aspect_ratio'), + 'output_format': tool_parameters.get('output_format', 'jpeg'), + 'response_format': tool_parameters.get('response_format', 'url'), + 'width': tool_parameters.get('width'), + 'height': tool_parameters.get('height'), + 'steps': tool_parameters.get('steps'), + 'negative_prompt': tool_parameters.get('negative_prompt'), + 'prompt_2': tool_parameters.get('prompt_2'), + } + options = {k: v for k, v in options.items() if v} + + text2image_result = app.text2image( + mode=tool_parameters.get('mode', 'essential-v2'), + params=options, + wait=True + ) + + if not isinstance(text2image_result, str): + text2image_result = json.dumps(text2image_result, ensure_ascii=False, indent=4) + + if not text2image_result: + return self.create_text_message("getimg.ai request failed.") + + return self.create_text_message(text2image_result) diff --git a/api/core/tools/provider/builtin/getimgai/tools/text2image.yaml b/api/core/tools/provider/builtin/getimgai/tools/text2image.yaml new file mode 100644 index 0000000000..d972186f56 --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/tools/text2image.yaml @@ -0,0 +1,167 @@ +identity: + name: text2image + author: Matri Qi + label: + en_US: text2image + icon: icon.svg +description: + human: + en_US: Generate image via getimg.ai. + llm: This tool is used to generate image from prompt or image via https://getimg.ai. +parameters: + - name: prompt + type: string + required: true + label: + en_US: prompt + human_description: + en_US: The text prompt used to generate the image. The getimg.aier will generate an image based on this prompt. + llm_description: this prompt text will be used to generate image. + form: llm + - name: mode + type: select + required: false + label: + en_US: mode + human_description: + en_US: The getimg.ai mode to use. The mode determines the endpoint used to generate the image. + form: form + options: + - value: "essential-v2" + label: + en_US: essential-v2 + - value: stable-diffusion-xl + label: + en_US: stable-diffusion-xl + - value: stable-diffusion + label: + en_US: stable-diffusion + - value: latent-consistency + label: + en_US: latent-consistency + - name: style + type: select + required: false + label: + en_US: style + human_description: + en_US: The style preset to use. The style preset guides the generation towards a particular style. It's just efficient for `Essential V2` mode. + form: form + options: + - value: photorealism + label: + en_US: photorealism + - value: anime + label: + en_US: anime + - value: art + label: + en_US: art + - name: aspect_ratio + type: select + required: false + label: + en_US: "aspect ratio" + human_description: + en_US: The aspect ratio of the generated image. It's just efficient for `Essential V2` mode. + form: form + options: + - value: "1:1" + label: + en_US: "1:1" + - value: "4:5" + label: + en_US: "4:5" + - value: "5:4" + label: + en_US: "5:4" + - value: "2:3" + label: + en_US: "2:3" + - value: "3:2" + label: + en_US: "3:2" + - value: "4:7" + label: + en_US: "4:7" + - value: "7:4" + label: + en_US: "7:4" + - name: output_format + type: select + required: false + label: + en_US: "output format" + human_description: + en_US: The file format of the generated image. + form: form + options: + - value: jpeg + label: + en_US: jpeg + - value: png + label: + en_US: png + - name: response_format + type: select + required: false + label: + en_US: "response format" + human_description: + en_US: The format in which the generated images are returned. Must be one of url or b64. URLs are only valid for 1 hour after the image has been generated. + form: form + options: + - value: url + label: + en_US: url + - value: b64 + label: + en_US: b64 + - name: model + type: string + required: false + label: + en_US: model + human_description: + en_US: Model ID supported by this pipeline and family. It's just efficient for `Stable Diffusion XL`, `Stable Diffusion`, `Latent Consistency` mode. + form: form + - name: negative_prompt + type: string + required: false + label: + en_US: negative prompt + human_description: + en_US: Text input that will not guide the image generation. It's just efficient for `Stable Diffusion XL`, `Stable Diffusion`, `Latent Consistency` mode. + form: form + - name: prompt_2 + type: string + required: false + label: + en_US: prompt2 + human_description: + en_US: Prompt sent to second tokenizer and text encoder. If not defined, prompt is used in both text-encoders. It's just efficient for `Stable Diffusion XL` mode. + form: form + - name: width + type: number + required: false + label: + en_US: width + human_description: + en_US: he width of the generated image in pixels. Width needs to be multiple of 64. + form: form + - name: height + type: number + required: false + label: + en_US: height + human_description: + en_US: he height of the generated image in pixels. Height needs to be multiple of 64. + form: form + - name: steps + type: number + required: false + label: + en_US: steps + human_description: + en_US: The number of denoising steps. More steps usually can produce higher quality images, but take more time to generate. It's just efficient for `Stable Diffusion XL`, `Stable Diffusion`, `Latent Consistency` mode. + form: form diff --git a/api/core/tools/provider/builtin/google_translate/_assets/icon.svg b/api/core/tools/provider/builtin/google_translate/_assets/icon.svg new file mode 100644 index 0000000000..de69a9c5e5 --- /dev/null +++ b/api/core/tools/provider/builtin/google_translate/_assets/icon.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/api/core/tools/provider/builtin/google_translate/google_translate.py b/api/core/tools/provider/builtin/google_translate/google_translate.py new file mode 100644 index 0000000000..f6e1d65834 --- /dev/null +++ b/api/core/tools/provider/builtin/google_translate/google_translate.py @@ -0,0 +1,17 @@ +from typing import Any + +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.google_translate.tools.translate import GoogleTranslate +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class JsonExtractProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + try: + GoogleTranslate().invoke(user_id='', + tool_parameters={ + "content": "这是一段测试文本", + "dest": "en" + }) + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) diff --git a/api/core/tools/provider/builtin/google_translate/google_translate.yaml b/api/core/tools/provider/builtin/google_translate/google_translate.yaml new file mode 100644 index 0000000000..8bc821a3d5 --- /dev/null +++ b/api/core/tools/provider/builtin/google_translate/google_translate.yaml @@ -0,0 +1,12 @@ +identity: + author: Ron Liu + name: google_translate + label: + en_US: Google Translate + zh_Hans: 谷歌翻译 + description: + en_US: Translate text using Google + zh_Hans: 使用 Google 进行翻译 + icon: icon.svg + tags: + - utilities diff --git a/api/core/tools/provider/builtin/google_translate/tools/translate.py b/api/core/tools/provider/builtin/google_translate/tools/translate.py new file mode 100644 index 0000000000..4314182b06 --- /dev/null +++ b/api/core/tools/provider/builtin/google_translate/tools/translate.py @@ -0,0 +1,52 @@ +from typing import Any, Union + +import requests + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class GoogleTranslate(BuiltinTool): + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + invoke tools + """ + content = tool_parameters.get('content', '') + if not content: + return self.create_text_message('Invalid parameter content') + + dest = tool_parameters.get('dest', '') + if not dest: + return self.create_text_message('Invalid parameter destination language') + + try: + result = self._translate(content, dest) + return self.create_text_message(str(result)) + except Exception: + return self.create_text_message('Translation service error, please check the network') + + def _translate(self, content: str, dest: str) -> str: + try: + url = "https://translate.googleapis.com/translate_a/single" + params = { + "client": "gtx", + "sl": "auto", + "tl": dest, + "dt": "t", + "q": content + } + + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + } + + response_json = requests.get( + url, params=params, headers=headers).json() + result = response_json[0] + translated_text = ''.join([item[0] for item in result if item[0]]) + return str(translated_text) + except Exception as e: + return str(e) diff --git a/api/core/tools/provider/builtin/google_translate/tools/translate.yaml b/api/core/tools/provider/builtin/google_translate/tools/translate.yaml new file mode 100644 index 0000000000..a4189cd743 --- /dev/null +++ b/api/core/tools/provider/builtin/google_translate/tools/translate.yaml @@ -0,0 +1,215 @@ +identity: + name: translate + author: Ron Liu + label: + en_US: Translate + zh_Hans: 翻译 +description: + human: + en_US: A tool for Google Translate + zh_Hans: Google 翻译 + llm: A tool for Google Translate +parameters: + - name: content + type: string + required: true + label: + en_US: Text content + zh_Hans: 文本内容 + human_description: + en_US: Text content + zh_Hans: 需要翻译的文本内容 + llm_description: Text content + form: llm + - name: dest + type: select + required: true + label: + en_US: destination language + zh_Hans: 目标语言 + human_description: + en_US: The destination language you want to translate. + zh_Hans: 你想翻译的目标语言 + default: en + form: form + options: + - value: ar + label: + en_US: Arabic + zh_Hans: 阿拉伯语 + - value: bg + label: + en_US: Bulgarian + zh_Hans: 保加利亚语 + - value: ca + label: + en_US: Catalan + zh_Hans: 加泰罗尼亚语 + - value: zh-cn + label: + en_US: Chinese (Simplified) + zh_Hans: 中文（简体） + - value: zh-tw + label: + en_US: Chinese (Traditional) + zh_Hans: 中文（繁体） + - value: cs + label: + en_US: Czech + zh_Hans: 捷克语 + - value: da + label: + en_US: Danish + zh_Hans: 丹麦语 + - value: nl + label: + en_US: Dutch + zh_Hans: 荷兰语 + - value: en + label: + en_US: English + zh_Hans: 英语 + - value: et + label: + en_US: Estonian + zh_Hans: 爱沙尼亚语 + - value: fi + label: + en_US: Finnish + zh_Hans: 芬兰语 + - value: fr + label: + en_US: French + zh_Hans: 法语 + - value: de + label: + en_US: German + zh_Hans: 德语 + - value: el + label: + en_US: Greek + zh_Hans: 希腊语 + - value: iw + label: + en_US: Hebrew + zh_Hans: 希伯来语 + - value: hi + label: + en_US: Hindi + zh_Hans: 印地语 + - value: hu + label: + en_US: Hungarian + zh_Hans: 匈牙利语 + - value: id + label: + en_US: Indonesian + zh_Hans: 印尼语 + - value: it + label: + en_US: Italian + zh_Hans: 意大利语 + - value: ja + label: + en_US: Japanese + zh_Hans: 日语 + - value: kn + label: + en_US: Kannada + zh_Hans: 卡纳达语 + - value: ko + label: + en_US: Korean + zh_Hans: 韩语 + - value: lv + label: + en_US: Latvian + zh_Hans: 拉脱维亚语 + - value: lt + label: + en_US: Lithuanian + zh_Hans: 立陶宛语 + - value: my + label: + en_US: Malay + zh_Hans: 马来语 + - value: ml + label: + en_US: Malayalam + zh_Hans: 马拉雅拉姆语 + - value: mr + label: + en_US: Marathi + zh_Hans: 马拉地语 + - value: "no" + label: + en_US: Norwegian + zh_Hans: 挪威语 + - value: pl + label: + en_US: Polish + zh_Hans: 波兰语 + - value: pt-br + label: + en_US: Portuguese (Brazil) + zh_Hans: 葡萄牙语（巴西） + - value: pt-pt + label: + en_US: Portuguese (Portugal) + zh_Hans: 葡萄牙语（葡萄牙） + - value: pa + label: + en_US: Punjabi + zh_Hans: 旁遮普语 + - value: ro + label: + en_US: Romanian + zh_Hans: 罗马尼亚语 + - value: ru + label: + en_US: Russian + zh_Hans: 俄语 + - value: sr + label: + en_US: Serbian + zh_Hans: 塞尔维亚语 + - value: sk + label: + en_US: Slovak + zh_Hans: 斯洛伐克语 + - value: sl + label: + en_US: Slovenian + zh_Hans: 斯洛文尼亚语 + - value: es + label: + en_US: Spanish + zh_Hans: 西班牙语 + - value: sv + label: + en_US: Swedish + zh_Hans: 瑞典语 + - value: ta + label: + en_US: Tamil + zh_Hans: 泰米尔语 + - value: te + label: + en_US: Telugu + zh_Hans: 泰卢固语 + - value: th + label: + en_US: Thai + zh_Hans: 泰语 + - value: tr + label: + en_US: Turkish + zh_Hans: 土耳其语 + - value: uk + label: + en_US: Ukrainian + zh_Hans: 乌克兰语 + - value: vi + label: + en_US: Vietnamese + zh_Hans: 越南语 diff --git a/api/core/tools/provider/builtin/hap/_assets/icon.svg b/api/core/tools/provider/builtin/hap/_assets/icon.svg new file mode 100644 index 0000000000..0fa6f0886f --- /dev/null +++ b/api/core/tools/provider/builtin/hap/_assets/icon.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/api/core/tools/provider/builtin/hap/hap.py b/api/core/tools/provider/builtin/hap/hap.py new file mode 100644 index 0000000000..e0a48e05a5 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/hap.py @@ -0,0 +1,8 @@ +from typing import Any + +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class HapProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + pass \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/hap.yaml b/api/core/tools/provider/builtin/hap/hap.yaml new file mode 100644 index 0000000000..25b473cf9d --- /dev/null +++ b/api/core/tools/provider/builtin/hap/hap.yaml @@ -0,0 +1,15 @@ +identity: + author: Mingdao + name: hap + label: + en_US: HAP + zh_Hans: HAP + pt_BR: HAP + description: + en_US: "Hyper application platform that is particularly friendly to AI" + zh_Hans: "对 AI 特别友好的超级应用平台" + pt_BR: "Plataforma de aplicação hiper que é particularmente amigável à IA" + icon: icon.svg + tags: + - productivity +credentials_for_provider: diff --git a/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.py b/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.py new file mode 100644 index 0000000000..0e101dc67d --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.py @@ -0,0 +1,53 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class AddWorksheetRecordTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + record_data = tool_parameters.get('record_data', '') + if not record_data: + return self.create_text_message('Invalid parameter Record Row Data') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not host.startswith(("http://", "https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url = f"{host}/v2/open/worksheet/addRow" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id} + + try: + payload['controls'] = json.loads(record_data) + res = httpx.post(url, headers=headers, json=payload, timeout=60) + res.raise_for_status() + res_json = res.json() + if res_json.get('error_code') != 1: + return self.create_text_message(f"Failed to add the new record. {res_json['error_msg']}") + return self.create_text_message(f"New record added successfully. The record ID is {res_json['data']}.") + except httpx.RequestError as e: + return self.create_text_message(f"Failed to add the new record, request error: {e}") + except json.JSONDecodeError as e: + return self.create_text_message(f"Failed to parse JSON response: {e}") + except Exception as e: + return self.create_text_message(f"Failed to add the new record, unexpected error: {e}") diff --git a/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.yaml b/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.yaml new file mode 100644 index 0000000000..add7742cd7 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.yaml @@ -0,0 +1,78 @@ +identity: + name: add_worksheet_record + author: Ryan Tian + label: + en_US: Add Worksheet Record + zh_Hans: 新增一条工作表记录 +description: + human: + en_US: Adds a new record to the specified worksheet + zh_Hans: 向指定的工作表新增一条记录数据 + llm: A tool to append a new data entry into a specified worksheet. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the specified worksheet + zh_Hans: 要获取字段信息的工作表 ID + llm_description: The ID of the specified worksheet which to get the fields information. + form: llm + + - name: record_data + type: string + required: true + label: + en_US: Record Row Data + zh_Hans: 记录数据 + human_description: + en_US: The fields with data of the specified record + zh_Hans: 要新增的记录数据，JSON 对象数组格式。数组元素属性：controlId-字段ID，value-字段值 + llm_description: | + The fields with data of the specified record which to be created. It is in the format of an array of JSON objects, and the structure is defined as follows: + ``` + type RowData = { + controlId: string; // Field ID to be updated + value: string; // Field value to be updated + }[]; + ``` + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.py b/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.py new file mode 100644 index 0000000000..ba25952c9f --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.py @@ -0,0 +1,49 @@ +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class DeleteWorksheetRecordTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + row_id = tool_parameters.get('row_id', '') + if not row_id: + return self.create_text_message('Invalid parameter Record Row ID') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not host.startswith(("http://", "https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url = f"{host}/v2/open/worksheet/deleteRow" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id, "rowId": row_id} + + try: + res = httpx.post(url, headers=headers, json=payload, timeout=30) + res.raise_for_status() + res_json = res.json() + if res_json.get('error_code') != 1: + return self.create_text_message(f"Failed to delete the record. {res_json['error_msg']}") + return self.create_text_message("Successfully deleted the record.") + except httpx.RequestError as e: + return self.create_text_message(f"Failed to delete the record, request error: {e}") + except Exception as e: + return self.create_text_message(f"Failed to delete the record, unexpected error: {e}") \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.yaml b/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.yaml new file mode 100644 index 0000000000..7c0c2a6439 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.yaml @@ -0,0 +1,71 @@ +identity: + name: delete_worksheet_record + author: Ryan Tian + label: + en_US: Delete Worksheet Record + zh_Hans: 删除指定的一条工作表记录 +description: + human: + en_US: Deletes a single record from a worksheet based on the specified record row ID + zh_Hans: 根据指定的记录ID删除一条工作表记录数据 + llm: A tool to remove a particular record from a worksheet by specifying its unique record identifier. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the specified worksheet + zh_Hans: 要获取字段信息的工作表 ID + llm_description: The ID of the specified worksheet which to get the fields information. + form: llm + + - name: row_id + type: string + required: true + label: + en_US: Record Row ID + zh_Hans: 记录 ID + human_description: + en_US: The row ID of the specified record + zh_Hans: 要删除的记录 ID + llm_description: The row ID of the specified record which to be deleted. + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py new file mode 100644 index 0000000000..2c46d9dd4e --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py @@ -0,0 +1,148 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class GetWorksheetFieldsTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not host.startswith(("http://", "https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url = f"{host}/v2/open/worksheet/getWorksheetInfo" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id} + + try: + res = httpx.post(url, headers=headers, json=payload, timeout=60) + res.raise_for_status() + res_json = res.json() + if res_json.get('error_code') != 1: + return self.create_text_message(f"Failed to get the worksheet information. {res_json['error_msg']}") + + fields_json, fields_table = self.get_controls(res_json['data']['controls']) + result_type = tool_parameters.get('result_type', 'table') + return self.create_text_message( + text=json.dumps(fields_json, ensure_ascii=False) if result_type == 'json' else fields_table + ) + except httpx.RequestError as e: + return self.create_text_message(f"Failed to get the worksheet information, request error: {e}") + except json.JSONDecodeError as e: + return self.create_text_message(f"Failed to parse JSON response: {e}") + except Exception as e: + return self.create_text_message(f"Failed to get the worksheet information, unexpected error: {e}") + + def get_field_type_by_id(self, field_type_id: int) -> str: + field_type_map = { + 2: "Text", + 3: "Text-Phone", + 4: "Text-Phone", + 5: "Text-Email", + 6: "Number", + 7: "Text", + 8: "Number", + 9: "Option-Single Choice", + 10: "Option-Multiple Choices", + 11: "Option-Single Choice", + 15: "Date", + 16: "Date", + 24: "Option-Region", + 25: "Text", + 26: "Option-Member", + 27: "Option-Department", + 28: "Number", + 29: "Option-Linked Record", + 30: "Unknown Type", + 31: "Number", + 32: "Text", + 33: "Text", + 35: "Option-Linked Record", + 36: "Number-Yes1/No0", + 37: "Number", + 38: "Date", + 40: "Location", + 41: "Text", + 46: "Time", + 48: "Option-Organizational Role", + 50: "Text", + 51: "Query Record", + } + return field_type_map.get(field_type_id, '') + + def get_controls(self, controls: list) -> dict: + fields = [] + fields_list = ['|fieldId|fieldName|fieldType|fieldTypeId|description|options|','|'+'---|'*6] + for control in controls: + if control['type'] in self._get_ignore_types(): + continue + field_type_id = control['type'] + field_type = self.get_field_type_by_id(control['type']) + if field_type_id == 30: + source_type = control['sourceControl']['type'] + if source_type in self._get_ignore_types(): + continue + else: + field_type_id = source_type + field_type = self.get_field_type_by_id(source_type) + field = { + 'id': control['controlId'], + 'name': control['controlName'], + 'type': field_type, + 'typeId': field_type_id, + 'description': control['remark'].replace('\n', ' ').replace('\t', ' '), + 'options': self._extract_options(control), + } + fields.append(field) + fields_list.append(f"|{field['id']}|{field['name']}|{field['type']}|{field['typeId']}|{field['description']}|{field['options'] if field['options'] else ''}|") + + fields.append({ + 'id': 'ctime', + 'name': 'Created Time', + 'type': self.get_field_type_by_id(16), + 'typeId': 16, + 'description': '', + 'options': [] + }) + fields_list.append("|ctime|Created Time|Date|16|||") + return fields, '\n'.join(fields_list) + + def _extract_options(self, control: dict) -> list: + options = [] + if control['type'] in [9, 10, 11]: + options.extend([{"key": opt['key'], "value": opt['value']} for opt in control.get('options', [])]) + elif control['type'] in [28, 36]: + itemnames = control['advancedSetting'].get('itemnames') + if itemnames and itemnames.startswith('[{'): + try: + options = json.loads(itemnames) + except json.JSONDecodeError: + pass + elif control['type'] == 30: + source_type = control['sourceControl']['type'] + if source_type not in self._get_ignore_types(): + options.extend([{"key": opt['key'], "value": opt['value']} for opt in control.get('options', [])]) + return options + + def _get_ignore_types(self): + return {14, 21, 22, 34, 42, 43, 45, 47, 49, 10010} \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.yaml b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.yaml new file mode 100644 index 0000000000..f0d4973e85 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.yaml @@ -0,0 +1,80 @@ +identity: + name: get_worksheet_fields + author: Ryan Tian + label: + en_US: Get Worksheet Fields + zh_Hans: 获取工作表字段结构 +description: + human: + en_US: Get fields information of the worksheet + zh_Hans: 获取指定工作表的所有字段结构信息 + llm: A tool to get fields information of the specific worksheet. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the specified worksheet + zh_Hans: 要获取字段信息的工作表 ID + llm_description: The ID of the specified worksheet which to get the fields information. + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form + + - name: result_type + type: select + required: true + options: + - value: table + label: + en_US: table text + zh_Hans: 表格文本 + - value: json + label: + en_US: json text + zh_Hans: JSON文本 + default: table + label: + en_US: Result type + zh_Hans: 结果类型 + human_description: + en_US: used for selecting the result type, table styled text or json text + zh_Hans: 用于选择结果类型，使用表格格式文本还是JSON格式文本 + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.py b/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.py new file mode 100644 index 0000000000..6bf1caa65e --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.py @@ -0,0 +1,130 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class GetWorksheetPivotDataTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + x_column_fields = tool_parameters.get('x_column_fields', '') + if not x_column_fields or not x_column_fields.startswith('['): + return self.create_text_message('Invalid parameter Column Fields') + y_row_fields = tool_parameters.get('y_row_fields', '') + if y_row_fields and not y_row_fields.strip().startswith('['): + return self.create_text_message('Invalid parameter Row Fields') + elif not y_row_fields: + y_row_fields = '[]' + value_fields = tool_parameters.get('value_fields', '') + if not value_fields or not value_fields.strip().startswith('['): + return self.create_text_message('Invalid parameter Value Fields') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not host.startswith(("http://", "https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url = f"{host}/report/getPivotData" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id, "options": {"showTotal": True}} + + try: + x_column_fields = json.loads(x_column_fields) + payload['columns'] = x_column_fields + y_row_fields = json.loads(y_row_fields) + if y_row_fields: payload['rows'] = y_row_fields + value_fields = json.loads(value_fields) + payload['values'] = value_fields + sort_fields = tool_parameters.get('sort_fields', '') + if not sort_fields: sort_fields = '[]' + sort_fields = json.loads(sort_fields) + if sort_fields: payload['options']['sort'] = sort_fields + res = httpx.post(url, headers=headers, json=payload, timeout=60) + res.raise_for_status() + res_json = res.json() + if res_json.get('status') != 1: + return self.create_text_message(f"Failed to get the worksheet pivot data. {res_json['msg']}") + + pivot_json = self.generate_pivot_json(res_json['data']) + pivot_table = self.generate_pivot_table(res_json['data']) + result_type = tool_parameters.get('result_type', '') + text = pivot_table if result_type == 'table' else json.dumps(pivot_json, ensure_ascii=False) + return self.create_text_message(text) + except httpx.RequestError as e: + return self.create_text_message(f"Failed to get the worksheet pivot data, request error: {e}") + except json.JSONDecodeError as e: + return self.create_text_message(f"Failed to parse JSON response: {e}") + except Exception as e: + return self.create_text_message(f"Failed to get the worksheet pivot data, unexpected error: {e}") + + def generate_pivot_table(self, data: dict[str, Any]) -> str: + columns = data['metadata']['columns'] + rows = data['metadata']['rows'] + values = data['metadata']['values'] + + rows_data = data['data'] + + header = ([row['displayName'] for row in rows] if rows else []) + [column['displayName'] for column in columns] + [value['displayName'] for value in values] + line = (['---'] * len(rows) if rows else []) + ['---'] * len(columns) + ['--:'] * len(values) + + table = [header, line] + for row in rows_data: + row_data = [self.replace_pipe(row['rows'][r['controlId']]) for r in rows] if rows else [] + row_data.extend([self.replace_pipe(row['columns'][column['controlId']]) for column in columns]) + row_data.extend([self.replace_pipe(str(row['values'][value['controlId']])) for value in values]) + table.append(row_data) + + return '\n'.join([('|'+'|'.join(row) +'|') for row in table]) + + def replace_pipe(self, text: str) -> str: + return text.replace('|', '▏').replace('\n', ' ') + + def generate_pivot_json(self, data: dict[str, Any]) -> dict: + fields = { + "x-axis": [ + {"fieldId": column["controlId"], "fieldName": column["displayName"]} + for column in data["metadata"]["columns"] + ], + "y-axis": [ + {"fieldId": row["controlId"], "fieldName": row["displayName"]} + for row in data["metadata"]["rows"] + ] if data["metadata"]["rows"] else [], + "values": [ + {"fieldId": value["controlId"], "fieldName": value["displayName"]} + for value in data["metadata"]["values"] + ] + } + # fields = ([ + # {"fieldId": row["controlId"], "fieldName": row["displayName"]} + # for row in data["metadata"]["rows"] + # ] if data["metadata"]["rows"] else []) + [ + # {"fieldId": column["controlId"], "fieldName": column["displayName"]} + # for column in data["metadata"]["columns"] + # ] + [ + # {"fieldId": value["controlId"], "fieldName": value["displayName"]} + # for value in data["metadata"]["values"] + # ] + rows = [] + for row in data["data"]: + row_data = row["rows"] if row["rows"] else {} + row_data.update(row["columns"]) + row_data.update(row["values"]) + rows.append(row_data) + return {"fields": fields, "rows": rows, "summary": data["metadata"]["totalRow"]} \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.yaml b/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.yaml new file mode 100644 index 0000000000..cf8c57b262 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.yaml @@ -0,0 +1,248 @@ +identity: + name: get_worksheet_pivot_data + author: Ryan Tian + label: + en_US: Get Worksheet Pivot Data + zh_Hans: 获取工作表统计透视数据 +description: + human: + en_US: Retrieve statistical pivot table data from a specified worksheet + zh_Hans: 从指定的工作表中检索统计透视表数据 + llm: A tool for extracting statistical pivot table data from a specific worksheet, providing summarized information for analysis and reporting purposes. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the specified worksheet + zh_Hans: 要获取字段信息的工作表 ID + llm_description: The ID of the specified worksheet which to get the fields information. + form: llm + + - name: x_column_fields + type: string + required: true + label: + en_US: Columns (X-axis) + zh_Hans: 统计列字段（X轴） + human_description: + en_US: The column fields that make up the pivot table's X-axis groups or other dimensions for the X-axis in pivot charts + zh_Hans: 组成透视表的统计列或者统计图表的X轴分组及X轴其它维度。JSON 对象数组格式，数组元素属性：controlId-列ID，displayName-显示名称，particleSize(可选)-字段类型是日期或者地区时，通过此参数设置统计维度（日期时间：1-日，2-周，3-月；地区：1-全国，2-省，3-市） + llm_description: | + This parameter allows you to specify the columns that make up the pivot table's X-axis groups or other dimensions for the X-axis in pivot charts. It is formatted as a JSON array, with its structure defined as follows: + ``` + type XColumnFields = { // X-axis or column object array + controlId: string; // fieldId + displayName: string; // displayName + particleSize?: number; // field type is date or area, set the statistical dimension (date time: 1-day, 2-week, 3-month; area: 1-nation, 2-province, 3-city) + }[]; + ``` + form: llm + + - name: y_row_fields + type: string + required: false + label: + en_US: Rows (Y-axis) + zh_Hans: 统计行字段（Y轴） + human_description: + en_US: The row fields that make up the pivot table's Y-axis groups or other dimensions for the Y-axis in pivot charts + zh_Hans: 组成透视表的统计行或者统计图表的Y轴分组及Y轴其它维度。JSON 对象数组格式，数组元素属性：controlId-列ID，displayName-显示名称，particleSize(可选)-字段类型是日期或者地区时，通过此参数设置统计维度（日期时间：1-日，2-周，3-月；地区：1-全国，2-省，3-市） + llm_description: | + This parameter allows you to specify the rows that make up the pivot table's Y-axis groups or other dimensions for the Y-axis in pivot charts. It is formatted as a JSON array, with its structure defined as follows: + ``` + type YRowFields = { // Y-axis or row object array + controlId: string; // fieldId + displayName: string; // displayName + particleSize?: number; // field type is date or area, set the statistical dimension (date time: 1-day, 2-week, 3-month; area: 1-nation, 2-province, 3-city) + }[]; + ``` + form: llm + + - name: value_fields + type: string + required: true + label: + en_US: Aggregated Values + zh_Hans: 统计值字段 + human_description: + en_US: The aggregated value fields in the pivot table + zh_Hans: 透视表中经过聚合计算后的统计值字段。JSON 对象数组格式，数组元素属性：controlId-列ID，displayName-显示名称，aggregation-聚合方式（SUM,AVG,MIN,MAX,COUNT） + llm_description: | + This parameter allows you to specify the aggregated value fields in the pivot table. It is formatted as a JSON array, with its structure defined as follows: + ``` + type ValueFields = { // aggregated value object array + controlId: string; // fieldId + displayName: string; // displayName + aggregation: string; // aggregation method, e.g.: SUM, AVG, MIN, MAX, COUNT + }[]; + ``` + form: llm + + - name: filters + type: string + required: false + label: + en_US: Filter Set + zh_Hans: 筛选器组合 + human_description: + en_US: A combination of filters applied to query records, formatted as a JSON array. See the application's API documentation for details on its structure and usage. + zh_Hans: 查询记录的筛选条件组合，格式为 JSON 数组，可以从应用 API 文档中了解参数结构详情 + llm_description: | + This parameter allows you to specify a set of conditions that records must meet to be included in the result set. It is formatted as a JSON array, with its structure defined as follows: + ``` + type Filters = { // filter object array + controlId: string; // fieldId + dataType: number; // fieldTypeId + spliceType: number; // condition concatenation method, 1: And, 2: Or + filterType: number; // expression type, refer to the for enumerable values + values?: string[]; // values in the condition, for option-type fields, multiple values can be passed + value?: string; // value in the condition, a single value can be passed according to the field type + dateRange?: number; // date range, mandatory when filterType is 17 or 18, refer to the for enumerable values + minValue?: string; // minimum value for custom range + maxValue?: string; // maximum value for custom range + isAsc?: boolean; // ascending order, false: descending, true: ascending + }[]; + ``` + For option-type fields, if this option field has `options`, then you need to get the corresponding `key` value from the `options` in the current field information via `value`, and pass it into `values` in array format. Do not use the `options` value of other fields as input conditions. + + ### FilterTypeEnum Reference + ``` + Enum Value, Enum Character, Description + 1, Like, Contains + 2, Eq, Is (Equal) + 3, Start, Starts With + 4, End, Ends With + 5, NotLike, Does Not Contain + 6, Ne, Is Not (Not Equal) + 7, IsEmpty, Empty + 8, HasValue, Not Empty + 11, Between, Within Range + 12, NotBetween, Outside Range + 13, Gt, Greater Than + 14, Gte, Greater Than or Equal To + 15, Lt, Less Than + 16, Lte, Less Than or Equal To + 17, DateEnum, Date Is + 18, NotDateEnum, Date Is Not + 21, MySelf, Owned by Me + 22, UnRead, Unread + 23, Sub, Owned by Subordinate + 24, RCEq, Associated Field Is + 25, RCNe, Associated Field Is Not + 26, ArrEq, Array Equals + 27, ArrNe, Array Does Not Equal + 31, DateBetween, Date Within Range (can only be used with minValue and maxValue) + 32, DateNotBetween, Date Not Within Range (can only be used with minValue and maxValue) + 33, DateGt, Date Later Than + 34, DateGte, Date Later Than or Equal To + 35, DateLt, Date Earlier Than + 36, DateLte, Date Earlier Than or Equal To + ``` + + ### DateRangeEnum Reference + ``` + Enum Value, Enum Character, Description + 1, Today, Today + 2, Yesterday, Yesterday + 3, Tomorrow, Tomorrow + 4, ThisWeek, This Week + 5, LastWeek, Last Week + 6, NextWeek, Next Week + 7, ThisMonth, This Month + 8, LastMonth, Last Month + 9, NextMonth, Next Month + 12, ThisQuarter, This Quarter + 13, LastQuarter, Last Quarter + 14, NextQuarter, Next Quarter + 15, ThisYear, This Year + 16, LastYear, Last Year + 17, NextYear, Next Year + 18, Customize, Custom + 21, Last7Day, Past 7 Days + 22, Last14Day, Past 14 Days + 23, Last30Day, Past 30 Days + 31, Next7Day, Next 7 Days + 32, Next14Day, Next 14 Days + 33, Next33Day, Next 33 Days + ``` + form: llm + + - name: sort_fields + type: string + required: false + label: + en_US: Sort Fields + zh_Hans: 排序字段 + human_description: + en_US: The fields to used for sorting + zh_Hans: 用于确定排序的字段，不超过3个 + llm_description: | + This optional parameter specifies the unique identifier of the fields that will be used to sort the results. It is in the format of an array of JSON objects, and its structure is defined as follows: + ``` + type SortByFields = { + controlId: string; // Field ID used for sorting + isAsc: boolean; // Sorting direction, true indicates ascending order, false indicates descending order + }[]; + ``` + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form + + - name: result_type + type: select + required: true + options: + - value: table + label: + en_US: table text + zh_Hans: 表格文本 + - value: json + label: + en_US: json text + zh_Hans: JSON文本 + default: table + label: + en_US: Result type + zh_Hans: 结果类型 + human_description: + en_US: used for selecting the result type, table styled text or json text + zh_Hans: 用于选择结果类型，使用表格格式文本还是JSON格式文本 + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py new file mode 100644 index 0000000000..dddc041cc1 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py @@ -0,0 +1,209 @@ +import json +import re +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class ListWorksheetRecordsTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not (host.startswith("http://") or host.startswith("https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url_fields = f"{host}/v2/open/worksheet/getWorksheetInfo" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id} + + field_ids = tool_parameters.get('field_ids', '') + + try: + res = httpx.post(url_fields, headers=headers, json=payload, timeout=30) + res_json = res.json() + if res.is_success: + if res_json['error_code'] != 1: + return self.create_text_message("Failed to get the worksheet information. {}".format(res_json['error_msg'])) + else: + worksheet_name = res_json['data']['name'] + fields, schema, table_header = self.get_schema(res_json['data']['controls'], field_ids) + else: + return self.create_text_message( + f"Failed to get the worksheet information, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to get the worksheet information, something went wrong: {}".format(e)) + + if field_ids: + payload['controls'] = [v.strip() for v in field_ids.split(',')] if field_ids else [] + filters = tool_parameters.get('filters', '') + if filters: + payload['filters'] = json.loads(filters) + sort_id = tool_parameters.get('sort_id', '') + sort_is_asc = tool_parameters.get('sort_is_asc', False) + if sort_id: + payload['sortId'] = sort_id + payload['isAsc'] = sort_is_asc + limit = tool_parameters.get('limit', 50) + payload['pageSize'] = limit + page_index = tool_parameters.get('page_index', 1) + payload['pageIndex'] = page_index + payload['useControlId'] = True + payload['listType'] = 1 + + url = f"{host}/v2/open/worksheet/getFilterRows" + try: + res = httpx.post(url, headers=headers, json=payload, timeout=90) + res_json = res.json() + if res.is_success: + if res_json['error_code'] != 1: + return self.create_text_message("Failed to get the records. {}".format(res_json['error_msg'])) + else: + result = { + "fields": fields, + "rows": [], + "total": res_json.get("data", {}).get("total"), + "payload": {key: payload[key] for key in ['worksheetId', 'controls', 'filters', 'sortId', 'isAsc', 'pageSize', 'pageIndex'] if key in payload} + } + rows = res_json.get("data", {}).get("rows", []) + result_type = tool_parameters.get('result_type', '') + if not result_type: result_type = 'table' + if result_type == 'json': + for row in rows: + result['rows'].append(self.get_row_field_value(row, schema)) + return self.create_text_message(json.dumps(result, ensure_ascii=False)) + else: + result_text = f"Found {result['total']} rows in worksheet \"{worksheet_name}\"." + if result['total'] > 0: + result_text += f" The following are {result['total'] if result['total'] < limit else limit} pieces of data presented in a table format:\n\n{table_header}" + for row in rows: + result_values = [] + for f in fields: + result_values.append(self.handle_value_type(row[f['fieldId']], schema[f['fieldId']])) + result_text += '\n|'+'|'.join(result_values)+'|' + return self.create_text_message(result_text) + else: + return self.create_text_message( + f"Failed to get the records, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to get the records, something went wrong: {}".format(e)) + + + def get_row_field_value(self, row: dict, schema: dict): + row_value = {"rowid": row["rowid"]} + for field in schema: + row_value[field] = self.handle_value_type(row[field], schema[field]) + return row_value + + + def get_schema(self, controls: list, fieldids: str): + allow_fields = {v.strip() for v in fieldids.split(',')} if fieldids else set() + fields = [] + schema = {} + field_names = [] + for control in controls: + control_type_id = self.get_real_type_id(control) + if (control_type_id in self._get_ignore_types()) or (allow_fields and not control['controlId'] in allow_fields): + continue + else: + fields.append({'fieldId': control['controlId'], 'fieldName': control['controlName']}) + schema[control['controlId']] = {'typeId': control_type_id, 'options': self.set_option(control)} + field_names.append(control['controlName']) + if (not allow_fields or ('ctime' in allow_fields)): + fields.append({'fieldId': 'ctime', 'fieldName': 'Created Time'}) + schema['ctime'] = {'typeId': 16, 'options': {}} + field_names.append("Created Time") + fields.append({'fieldId':'rowid', 'fieldName': 'Record Row ID'}) + schema['rowid'] = {'typeId': 2, 'options': {}} + field_names.append("Record Row ID") + return fields, schema, '|'+'|'.join(field_names)+'|\n|'+'---|'*len(field_names) + + def get_real_type_id(self, control: dict) -> int: + return control['sourceControlType'] if control['type'] == 30 else control['type'] + + def set_option(self, control: dict) -> dict: + options = {} + if control.get('options'): + options = {option['key']: option['value'] for option in control['options']} + elif control.get('advancedSetting', {}).get('itemnames'): + try: + itemnames = json.loads(control['advancedSetting']['itemnames']) + options = {item['key']: item['value'] for item in itemnames} + except json.JSONDecodeError: + pass + return options + + def _get_ignore_types(self): + return {14, 21, 22, 34, 42, 43, 45, 47, 49, 10010} + + def handle_value_type(self, value, field): + type_id = field.get("typeId") + if type_id == 10: + value = value if isinstance(value, str) else "、".join(value) + elif type_id in [28, 36]: + value = field.get("options", {}).get(value, value) + elif type_id in [26, 27, 48, 14]: + value = self.process_value(value) + elif type_id in [35, 29]: + value = self.parse_cascade_or_associated(field, value) + elif type_id == 40: + value = self.parse_location(value) + return self.rich_text_to_plain_text(value) if value else '' + + def process_value(self, value): + if isinstance(value, str): + if value.startswith("[{\"accountId\""): + value = json.loads(value) + value = ', '.join([item['fullname'] for item in value]) + elif value.startswith("[{\"departmentId\""): + value = json.loads(value) + value = '、'.join([item['departmentName'] for item in value]) + elif value.startswith("[{\"organizeId\""): + value = json.loads(value) + value = '、'.join([item['organizeName'] for item in value]) + elif value.startswith("[{\"file_id\""): + value = '' + elif value == '[]': + value = '' + elif hasattr(value, 'accountId'): + value = value['fullname'] + return value + + def parse_cascade_or_associated(self, field, value): + if (field['typeId'] == 35 and value.startswith('[')) or (field['typeId'] == 29 and value.startswith('[{')): + value = json.loads(value) + value = value[0]['name'] if len(value) > 0 else '' + else: + value = '' + return value + + def parse_location(self, value): + if len(value) > 10: + parsed_value = json.loads(value) + value = parsed_value.get("address", "") + else: + value = "" + return value + + def rich_text_to_plain_text(self, rich_text): + text = re.sub(r'<[^>]+>', '', rich_text) if '<' in rich_text else rich_text + return text.replace("|", "▏").replace("\n", " ") \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.yaml b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.yaml new file mode 100644 index 0000000000..3c37746b92 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.yaml @@ -0,0 +1,226 @@ +identity: + name: list_worksheet_records + author: Ryan Tian + label: + en_US: List Worksheet Records + zh_Hans: 查询工作表记录数据 +description: + human: + en_US: List records from the worksheet + zh_Hans: 查询工作表的记录列表数据，一次最多1000行，可分页获取 + llm: A tool to retrieve record data from the specific worksheet. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the worksheet from which to retrieve record data + zh_Hans: 要获取记录数据的工作表 ID + llm_description: This parameter specifies the ID of the worksheet where the records are stored. + form: llm + + - name: field_ids + type: string + required: false + label: + en_US: Field IDs + zh_Hans: 字段 ID 列表 + human_description: + en_US: A comma-separated list of field IDs whose data to retrieve. If not provided, all fields' data will be fetched + zh_Hans: 要获取记录数据的字段 ID，多个 ID 间用英文逗号隔开，不传此参数则将获取所有字段的数据 + llm_description: This optional parameter lets you specify a comma-separated list of field IDs. Unless the user explicitly requests to output the specified field in the question, this parameter should usually be omitted. If this parameter is omitted, the API will return data for all fields by default. When provided, only the data associated with these fields will be included in the response. + form: llm + + - name: filters + type: string + required: false + label: + en_US: Filter Set + zh_Hans: 筛选器组合 + human_description: + en_US: A combination of filters applied to query records, formatted as a JSON array. See the application's API documentation for details on its structure and usage. + zh_Hans: 查询记录的筛选条件组合，格式为 JSON 数组，可以从应用 API 文档中了解参数结构详情 + llm_description: | + This parameter allows you to specify a set of conditions that records must meet to be included in the result set. It is formatted as a JSON array, with its structure defined as follows: + ``` + type Filters = { // filter object array + controlId: string; // fieldId + dataType: number; // fieldTypeId + spliceType: number; // condition concatenation method, 1: And, 2: Or + filterType: number; // expression type, refer to the for enumerable values + values?: string[]; // values in the condition, for option-type fields, multiple values can be passed + value?: string; // value in the condition, a single value can be passed according to the field type + dateRange?: number; // date range, mandatory when filterType is 17 or 18, refer to the for enumerable values + minValue?: string; // minimum value for custom range + maxValue?: string; // maximum value for custom range + isAsc?: boolean; // ascending order, false: descending, true: ascending + }[]; + ``` + For option-type fields, if this option field has `options`, then you need to get the corresponding `key` value from the `options` in the current field information via `value`, and pass it into `values` in array format. Do not use the `options` value of other fields as input conditions. + + ### FilterTypeEnum Reference + ``` + Enum Value, Enum Character, Description + 1, Like, Contains(Include) + 2, Eq, Is (Equal) + 3, Start, Starts With + 4, End, Ends With + 5, NotLike, Does Not Contain(Not Include) + 6, Ne, Is Not (Not Equal) + 7, IsEmpty, Empty + 8, HasValue, Not Empty + 11, Between, Within Range(Belong to) + 12, NotBetween, Outside Range(Not belong to) + 13, Gt, Greater Than + 14, Gte, Greater Than or Equal To + 15, Lt, Less Than + 16, Lte, Less Than or Equal To + 17, DateEnum, Date Is + 18, NotDateEnum, Date Is Not + 24, RCEq, Associated Field Is + 25, RCNe, Associated Field Is Not + 26, ArrEq, Array Equals + 27, ArrNe, Array Does Not Equal + 31, DateBetween, Date Within Range (can only be used with minValue and maxValue) + 32, DateNotBetween, Date Not Within Range (can only be used with minValue and maxValue) + 33, DateGt, Date Later Than + 34, DateGte, Date Later Than or Equal To + 35, DateLt, Date Earlier Than + 36, DateLte, Date Earlier Than or Equal To + ``` + + ### DateRangeEnum Reference + ``` + Enum Value, Enum Character, Description + 1, Today, Today + 2, Yesterday, Yesterday + 3, Tomorrow, Tomorrow + 4, ThisWeek, This Week + 5, LastWeek, Last Week + 6, NextWeek, Next Week + 7, ThisMonth, This Month + 8, LastMonth, Last Month + 9, NextMonth, Next Month + 12, ThisQuarter, This Quarter + 13, LastQuarter, Last Quarter + 14, NextQuarter, Next Quarter + 15, ThisYear, This Year + 16, LastYear, Last Year + 17, NextYear, Next Year + 18, Customize, Custom + 21, Last7Day, Past 7 Days + 22, Last14Day, Past 14 Days + 23, Last30Day, Past 30 Days + 31, Next7Day, Next 7 Days + 32, Next14Day, Next 14 Days + 33, Next33Day, Next 33 Days + ``` + form: llm + + - name: sort_id + type: string + required: false + label: + en_US: Sort Field ID + zh_Hans: 排序字段 ID + human_description: + en_US: The ID of the field used for sorting + zh_Hans: 用以排序的字段 ID + llm_description: This optional parameter specifies the unique identifier of the field that will be used to sort the results. It should be set to the ID of an existing field within your data structure. + form: llm + + - name: sort_is_asc + type: boolean + required: false + label: + en_US: Ascending Order + zh_Hans: 是否升序排列 + human_description: + en_US: Determines whether the sorting is in ascending (true) or descending (false) order + zh_Hans: 排序字段的排序方式：true-升序，false-降序 + llm_description: This optional parameter controls the direction of the sort. If set to true, the results will be sorted in ascending order; if false, they will be sorted in descending order. + form: llm + + - name: limit + type: number + required: false + label: + en_US: Record Limit + zh_Hans: 记录数量限制 + human_description: + en_US: The maximum number of records to retrieve + zh_Hans: 要获取的记录数量限制条数 + llm_description: This optional parameter allows you to specify the maximum number of records that should be returned in the result set. When retrieving paginated record data, this parameter indicates the number of rows to fetch per page, and must be used in conjunction with the `page_index` parameter. + form: llm + + - name: page_index + type: number + required: false + label: + en_US: Page Index + zh_Hans: 页码 + human_description: + en_US: The page number when paginating through a list of records + zh_Hans: 分页读取记录列表时的页码 + llm_description: This parameter is used when you need to paginate through a large set of records. The default value is 1, which refers to the first page. When it is used, the meaning of the `limit` parameter becomes the number of records per page. + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form + + - name: result_type + type: select + required: true + options: + - value: table + label: + en_US: table text + zh_Hans: 表格文本 + - value: json + label: + en_US: json text + zh_Hans: JSON文本 + default: table + label: + en_US: Result type + zh_Hans: 结果类型 + human_description: + en_US: used for selecting the result type, table styled text or json text + zh_Hans: 用于选择结果类型，使用表格格式文本还是JSON格式文本 + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/list_worksheets.py b/api/core/tools/provider/builtin/hap/tools/list_worksheets.py new file mode 100644 index 0000000000..960cbd10ac --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/list_worksheets.py @@ -0,0 +1,82 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class ListWorksheetsTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not (host.startswith("http://") or host.startswith("https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + url = f"{host}/v1/open/app/get" + + result_type = tool_parameters.get('result_type', '') + if not result_type: + result_type = 'table' + + headers = { 'Content-Type': 'application/json' } + params = { "appKey": appkey, "sign": sign, } + try: + res = httpx.get(url, headers=headers, params=params, timeout=30) + res_json = res.json() + if res.is_success: + if res_json['error_code'] != 1: + return self.create_text_message("Failed to access the application. {}".format(res_json['error_msg'])) + else: + if result_type == 'json': + worksheets = [] + for section in res_json['data']['sections']: + worksheets.extend(self._extract_worksheets(section, result_type)) + return self.create_text_message(text=json.dumps(worksheets, ensure_ascii=False)) + else: + worksheets = '|worksheetId|worksheetName|description|\n|---|---|---|' + for section in res_json['data']['sections']: + worksheets += self._extract_worksheets(section, result_type) + return self.create_text_message(worksheets) + + else: + return self.create_text_message( + f"Failed to list worksheets, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to list worksheets, something went wrong: {}".format(e)) + + def _extract_worksheets(self, section, type): + items = [] + tables = '' + for item in section.get('items', []): + if item.get('type') == 0 and (not 'notes' in item or item.get('notes') != 'NO'): + if type == 'json': + filtered_item = { + 'id': item['id'], + 'name': item['name'], + 'notes': item.get('notes', '') + } + items.append(filtered_item) + else: + tables += f"\n|{item['id']}|{item['name']}|{item.get('notes', '')}|" + + for child_section in section.get('childSections', []): + if type == 'json': + items.extend(self._extract_worksheets(child_section, 'json')) + else: + tables += self._extract_worksheets(child_section, 'table') + + return items if type == 'json' else tables \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/tools/list_worksheets.yaml b/api/core/tools/provider/builtin/hap/tools/list_worksheets.yaml new file mode 100644 index 0000000000..935b72a895 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/list_worksheets.yaml @@ -0,0 +1,68 @@ +identity: + name: list_worksheets + author: Ryan Tian + label: + en_US: List Worksheets + zh_Hans: 获取应用下所有工作表 +description: + human: + en_US: List worksheets within an application + zh_Hans: 获取应用下的所有工作表和说明信息 + llm: A tool to list worksheets info within an application, imported parameter is AppKey and Sign of the application. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form + + - name: result_type + type: select + required: true + options: + - value: table + label: + en_US: table text + zh_Hans: 表格文本 + - value: json + label: + en_US: json text + zh_Hans: JSON文本 + default: table + label: + en_US: Result type + zh_Hans: 结果类型 + human_description: + en_US: used for selecting the result type, table styled text or json text + zh_Hans: 用于选择结果类型，使用表格格式文本还是JSON格式文本 + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.py b/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.py new file mode 100644 index 0000000000..6ca1b98d90 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.py @@ -0,0 +1,56 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class UpdateWorksheetRecordTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + row_id = tool_parameters.get('row_id', '') + if not row_id: + return self.create_text_message('Invalid parameter Record Row ID') + record_data = tool_parameters.get('record_data', '') + if not record_data: + return self.create_text_message('Invalid parameter Record Row Data') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not host.startswith(("http://", "https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url = f"{host}/v2/open/worksheet/editRow" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id, "rowId": row_id} + + try: + payload['controls'] = json.loads(record_data) + res = httpx.post(url, headers=headers, json=payload, timeout=60) + res.raise_for_status() + res_json = res.json() + if res_json.get('error_code') != 1: + return self.create_text_message(f"Failed to update the record. {res_json['error_msg']}") + return self.create_text_message("Record updated successfully.") + except httpx.RequestError as e: + return self.create_text_message(f"Failed to update the record, request error: {e}") + except json.JSONDecodeError as e: + return self.create_text_message(f"Failed to parse JSON response: {e}") + except Exception as e: + return self.create_text_message(f"Failed to update the record, unexpected error: {e}") diff --git a/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.yaml b/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.yaml new file mode 100644 index 0000000000..fe1f8f671a --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.yaml @@ -0,0 +1,90 @@ +identity: + name: update_worksheet_record + author: Ryan Tian + label: + en_US: Update Worksheet Record + zh_Hans: 更新指定的一条工作表记录 +description: + human: + en_US: Updates a single record in a worksheet based on the specified record row ID + zh_Hans: 根据指定的记录ID更新一条工作表记录数据 + llm: A tool to modify existing information within a particular record of a worksheet by referencing its unique identifier. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the specified worksheet + zh_Hans: 要获取字段信息的工作表 ID + llm_description: The ID of the specified worksheet which to get the fields information. + form: llm + + - name: row_id + type: string + required: true + label: + en_US: Record Row ID + zh_Hans: 记录 ID + human_description: + en_US: The row ID of the specified record + zh_Hans: 要更新的记录 ID + llm_description: The row ID of the specified record which to be updated. + form: llm + + - name: record_data + type: string + required: true + label: + en_US: Record Row Data + zh_Hans: 记录数据 + human_description: + en_US: The fields with data of the specified record + zh_Hans: 要更新的记录数据，JSON 对象数组格式。数组元素属性：controlId-字段ID，value-字段值 + llm_description: | + The fields with data of the specified record which to be updated. It is in the format of an array of JSON objects, and the structure is defined as follows: + ``` + type RowData = { + controlId: string; // Field ID to be updated + value: string; // Field value to be updated + }[]; + ``` + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form diff --git a/api/core/tools/provider/builtin/jina/tools/jina_reader.py b/api/core/tools/provider/builtin/jina/tools/jina_reader.py index 8409129833..cee46cee23 100644 --- a/api/core/tools/provider/builtin/jina/tools/jina_reader.py +++ b/api/core/tools/provider/builtin/jina/tools/jina_reader.py @@ -60,11 +60,13 @@ class JinaReaderTool(BuiltinTool): if tool_parameters.get('no_cache', False): headers['X-No-Cache'] = 'true' + max_retries = tool_parameters.get('max_retries', 3) response = ssrf_proxy.get( str(URL(self._jina_reader_endpoint + url)), headers=headers, params=request_params, timeout=(10, 60), + max_retries=max_retries ) if tool_parameters.get('summary', False): diff --git a/api/core/tools/provider/builtin/jina/tools/jina_reader.yaml b/api/core/tools/provider/builtin/jina/tools/jina_reader.yaml index 072e7f0528..58ad6d8694 100644 --- a/api/core/tools/provider/builtin/jina/tools/jina_reader.yaml +++ b/api/core/tools/provider/builtin/jina/tools/jina_reader.yaml @@ -150,3 +150,17 @@ parameters: pt_BR: Habilitar resumo para a saída llm_description: enable summary form: form + - name: max_retries + type: number + required: false + default: 3 + label: + en_US: Retry + zh_Hans: 重试 + pt_BR: Repetir + human_description: + en_US: Number of times to retry the request if it fails + zh_Hans: 请求失败时重试的次数 + pt_BR: Número de vezes para repetir a solicitação se falhar + llm_description: Number of times to retry the request if it fails + form: form diff --git a/api/core/tools/provider/builtin/jina/tools/jina_search.py b/api/core/tools/provider/builtin/jina/tools/jina_search.py index e6bc08147f..d4a81cd096 100644 --- a/api/core/tools/provider/builtin/jina/tools/jina_search.py +++ b/api/core/tools/provider/builtin/jina/tools/jina_search.py @@ -40,10 +40,12 @@ class JinaSearchTool(BuiltinTool): if tool_parameters.get('no_cache', False): headers['X-No-Cache'] = 'true' + max_retries = tool_parameters.get('max_retries', 3) response = ssrf_proxy.get( str(URL(self._jina_search_endpoint + query)), headers=headers, - timeout=(10, 60) + timeout=(10, 60), + max_retries=max_retries ) return self.create_text_message(response.text) diff --git a/api/core/tools/provider/builtin/jina/tools/jina_search.yaml b/api/core/tools/provider/builtin/jina/tools/jina_search.yaml index da0a300c6c..2bc70e1be1 100644 --- a/api/core/tools/provider/builtin/jina/tools/jina_search.yaml +++ b/api/core/tools/provider/builtin/jina/tools/jina_search.yaml @@ -91,3 +91,17 @@ parameters: pt_BR: Ignorar o cache llm_description: bypass the cache form: form + - name: max_retries + type: number + required: false + default: 3 + label: + en_US: Retry + zh_Hans: 重试 + pt_BR: Repetir + human_description: + en_US: Number of times to retry the request if it fails + zh_Hans: 请求失败时重试的次数 + pt_BR: Número de vezes para repetir a solicitação se falhar + llm_description: Number of times to retry the request if it fails + form: form diff --git a/api/core/tools/provider/builtin/json_process/tools/delete.py b/api/core/tools/provider/builtin/json_process/tools/delete.py index b09e494881..1b49cfe2f3 100644 --- a/api/core/tools/provider/builtin/json_process/tools/delete.py +++ b/api/core/tools/provider/builtin/json_process/tools/delete.py @@ -19,28 +19,29 @@ class JSONDeleteTool(BuiltinTool): content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # Get query query = tool_parameters.get('query', '') if not query: return self.create_text_message('Invalid parameter query') - + + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: - result = self._delete(content, query) + result = self._delete(content, query, ensure_ascii) return self.create_text_message(str(result)) except Exception as e: return self.create_text_message(f'Failed to delete JSON content: {str(e)}') - def _delete(self, origin_json: str, query: str) -> str: + def _delete(self, origin_json: str, query: str, ensure_ascii: bool) -> str: try: input_data = json.loads(origin_json) expr = parse('$.' + query.lstrip('$.')) # Ensure query path starts with $ - + matches = expr.find(input_data) - + if not matches: - return json.dumps(input_data, ensure_ascii=True) # No changes if no matches found - + return json.dumps(input_data, ensure_ascii=ensure_ascii) # No changes if no matches found + for match in matches: if isinstance(match.context.value, dict): # Delete key from dictionary @@ -53,7 +54,7 @@ class JSONDeleteTool(BuiltinTool): parent = match.context.parent if parent: del parent.value[match.path.fields[-1]] - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: - raise Exception(f"Delete operation failed: {str(e)}") \ No newline at end of file + raise Exception(f"Delete operation failed: {str(e)}") diff --git a/api/core/tools/provider/builtin/json_process/tools/delete.yaml b/api/core/tools/provider/builtin/json_process/tools/delete.yaml index 4cfa90b861..4d390e40d1 100644 --- a/api/core/tools/provider/builtin/json_process/tools/delete.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/delete.yaml @@ -38,3 +38,15 @@ parameters: pt_BR: JSONPath query to locate the element to delete llm_description: JSONPath query to locate the element to delete form: llm + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form diff --git a/api/core/tools/provider/builtin/json_process/tools/insert.py b/api/core/tools/provider/builtin/json_process/tools/insert.py index aa5986e2b4..27e34f1ff3 100644 --- a/api/core/tools/provider/builtin/json_process/tools/insert.py +++ b/api/core/tools/provider/builtin/json_process/tools/insert.py @@ -19,31 +19,31 @@ class JSONParseTool(BuiltinTool): content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # get query query = tool_parameters.get('query', '') if not query: return self.create_text_message('Invalid parameter query') - + # get new value new_value = tool_parameters.get('new_value', '') if not new_value: return self.create_text_message('Invalid parameter new_value') - + # get insert position index = tool_parameters.get('index') - + # get create path create_path = tool_parameters.get('create_path', False) - + + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: - result = self._insert(content, query, new_value, index, create_path) + result = self._insert(content, query, new_value, ensure_ascii, index, create_path) return self.create_text_message(str(result)) except Exception: return self.create_text_message('Failed to insert JSON content') - - def _insert(self, origin_json, query, new_value, index=None, create_path=False): + def _insert(self, origin_json, query, new_value, ensure_ascii: bool, index=None, create_path=False): try: input_data = json.loads(origin_json) expr = parse(query) @@ -51,9 +51,9 @@ class JSONParseTool(BuiltinTool): new_value = json.loads(new_value) except json.JSONDecodeError: new_value = new_value - + matches = expr.find(input_data) - + if not matches and create_path: # create new path path_parts = query.strip('$').strip('.').split('.') @@ -91,7 +91,7 @@ class JSONParseTool(BuiltinTool): else: # replace old value with new value match.full_path.update(input_data, new_value) - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: - return str(e) \ No newline at end of file + return str(e) diff --git a/api/core/tools/provider/builtin/json_process/tools/insert.yaml b/api/core/tools/provider/builtin/json_process/tools/insert.yaml index 66a6ff9929..63e7816455 100644 --- a/api/core/tools/provider/builtin/json_process/tools/insert.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/insert.yaml @@ -75,3 +75,15 @@ parameters: zh_Hans: 否 pt_BR: "No" form: form + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form diff --git a/api/core/tools/provider/builtin/json_process/tools/parse.py b/api/core/tools/provider/builtin/json_process/tools/parse.py index b246afc07e..ecd39113ae 100644 --- a/api/core/tools/provider/builtin/json_process/tools/parse.py +++ b/api/core/tools/provider/builtin/json_process/tools/parse.py @@ -19,33 +19,34 @@ class JSONParseTool(BuiltinTool): content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # get json filter json_filter = tool_parameters.get('json_filter', '') if not json_filter: return self.create_text_message('Invalid parameter json_filter') + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: - result = self._extract(content, json_filter) + result = self._extract(content, json_filter, ensure_ascii) return self.create_text_message(str(result)) except Exception: return self.create_text_message('Failed to extract JSON content') # Extract data from JSON content - def _extract(self, content: str, json_filter: str) -> str: + def _extract(self, content: str, json_filter: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(json_filter) result = [match.value for match in expr.find(input_data)] - + if len(result) == 1: result = result[0] - + if isinstance(result, dict | list): - return json.dumps(result, ensure_ascii=True) + return json.dumps(result, ensure_ascii=ensure_ascii) elif isinstance(result, str | int | float | bool) or result is None: return str(result) else: return repr(result) except Exception as e: - return str(e) \ No newline at end of file + return str(e) diff --git a/api/core/tools/provider/builtin/json_process/tools/parse.yaml b/api/core/tools/provider/builtin/json_process/tools/parse.yaml index b619dcde94..c35f4eac07 100644 --- a/api/core/tools/provider/builtin/json_process/tools/parse.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/parse.yaml @@ -38,3 +38,15 @@ parameters: pt_BR: JSON fields to be parsed llm_description: JSON fields to be parsed form: llm + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form diff --git a/api/core/tools/provider/builtin/json_process/tools/replace.py b/api/core/tools/provider/builtin/json_process/tools/replace.py index 9f127b9d06..be696bce0e 100644 --- a/api/core/tools/provider/builtin/json_process/tools/replace.py +++ b/api/core/tools/provider/builtin/json_process/tools/replace.py @@ -19,61 +19,62 @@ class JSONReplaceTool(BuiltinTool): content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # get query query = tool_parameters.get('query', '') if not query: return self.create_text_message('Invalid parameter query') - + # get replace value replace_value = tool_parameters.get('replace_value', '') if not replace_value: return self.create_text_message('Invalid parameter replace_value') - + # get replace model replace_model = tool_parameters.get('replace_model', '') if not replace_model: return self.create_text_message('Invalid parameter replace_model') + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: if replace_model == 'pattern': # get replace pattern replace_pattern = tool_parameters.get('replace_pattern', '') if not replace_pattern: return self.create_text_message('Invalid parameter replace_pattern') - result = self._replace_pattern(content, query, replace_pattern, replace_value) + result = self._replace_pattern(content, query, replace_pattern, replace_value, ensure_ascii) elif replace_model == 'key': - result = self._replace_key(content, query, replace_value) + result = self._replace_key(content, query, replace_value, ensure_ascii) elif replace_model == 'value': - result = self._replace_value(content, query, replace_value) + result = self._replace_value(content, query, replace_value, ensure_ascii) return self.create_text_message(str(result)) except Exception: return self.create_text_message('Failed to replace JSON content') # Replace pattern - def _replace_pattern(self, content: str, query: str, replace_pattern: str, replace_value: str) -> str: + def _replace_pattern(self, content: str, query: str, replace_pattern: str, replace_value: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(query) - + matches = expr.find(input_data) - + for match in matches: new_value = match.value.replace(replace_pattern, replace_value) match.full_path.update(input_data, new_value) - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: return str(e) - + # Replace key - def _replace_key(self, content: str, query: str, replace_value: str) -> str: + def _replace_key(self, content: str, query: str, replace_value: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(query) - + matches = expr.find(input_data) - + for match in matches: parent = match.context.value if isinstance(parent, dict): @@ -86,21 +87,21 @@ class JSONReplaceTool(BuiltinTool): if isinstance(item, dict) and old_key in item: value = item.pop(old_key) item[replace_value] = value - return json.dumps(input_data, ensure_ascii=True) + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: return str(e) - + # Replace value - def _replace_value(self, content: str, query: str, replace_value: str) -> str: + def _replace_value(self, content: str, query: str, replace_value: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(query) - + matches = expr.find(input_data) - + for match in matches: match.full_path.update(input_data, replace_value) - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: - return str(e) \ No newline at end of file + return str(e) diff --git a/api/core/tools/provider/builtin/json_process/tools/replace.yaml b/api/core/tools/provider/builtin/json_process/tools/replace.yaml index 556be5e8b2..cf4b1dc63f 100644 --- a/api/core/tools/provider/builtin/json_process/tools/replace.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/replace.yaml @@ -93,3 +93,15 @@ parameters: zh_Hans: 字符串替换 pt_BR: replace string form: form + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form diff --git a/api/core/tools/provider/builtin/spider/_assets/icon.svg b/api/core/tools/provider/builtin/spider/_assets/icon.svg new file mode 100644 index 0000000000..604a09d01d --- /dev/null +++ b/api/core/tools/provider/builtin/spider/_assets/icon.svg @@ -0,0 +1 @@ +Spider v1 Logo diff --git a/api/core/tools/provider/builtin/spider/spider.py b/api/core/tools/provider/builtin/spider/spider.py new file mode 100644 index 0000000000..6fa431b6bb --- /dev/null +++ b/api/core/tools/provider/builtin/spider/spider.py @@ -0,0 +1,14 @@ +from typing import Any + +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.spider.spiderApp import Spider +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class SpiderProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + try: + app = Spider(api_key=credentials["spider_api_key"]) + app.scrape_url(url="https://spider.cloud") + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) diff --git a/api/core/tools/provider/builtin/spider/spider.yaml b/api/core/tools/provider/builtin/spider/spider.yaml new file mode 100644 index 0000000000..45702c85dd --- /dev/null +++ b/api/core/tools/provider/builtin/spider/spider.yaml @@ -0,0 +1,27 @@ +identity: + author: William Espegren + name: spider + label: + en_US: Spider + zh_CN: Spider + description: + en_US: Spider API integration, returning LLM-ready data by scraping & crawling websites. + zh_CN: Spider API 集成，通过爬取和抓取网站返回 LLM-ready 数据。 + icon: icon.svg + tags: + - search + - utilities +credentials_for_provider: + spider_api_key: + type: secret-input + required: true + label: + en_US: Spider API Key + zh_CN: Spider API 密钥 + placeholder: + en_US: Please input your Spider API key + zh_CN: 请输入您的 Spider API 密钥 + help: + en_US: Get your Spider API key from your Spider dashboard + zh_CN: 从您的 Spider 仪表板中获取 Spider API 密钥。 + url: https://spider.cloud/ diff --git a/api/core/tools/provider/builtin/spider/spiderApp.py b/api/core/tools/provider/builtin/spider/spiderApp.py new file mode 100644 index 0000000000..82c0df19ca --- /dev/null +++ b/api/core/tools/provider/builtin/spider/spiderApp.py @@ -0,0 +1,237 @@ +import os +from typing import Literal, Optional, TypedDict + +import requests + + +class RequestParamsDict(TypedDict, total=False): + url: Optional[str] + request: Optional[Literal["http", "chrome", "smart"]] + limit: Optional[int] + return_format: Optional[Literal["raw", "markdown", "html2text", "text", "bytes"]] + tld: Optional[bool] + depth: Optional[int] + cache: Optional[bool] + budget: Optional[dict[str, int]] + locale: Optional[str] + cookies: Optional[str] + stealth: Optional[bool] + headers: Optional[dict[str, str]] + anti_bot: Optional[bool] + metadata: Optional[bool] + viewport: Optional[dict[str, int]] + encoding: Optional[str] + subdomains: Optional[bool] + user_agent: Optional[str] + store_data: Optional[bool] + gpt_config: Optional[list[str]] + fingerprint: Optional[bool] + storageless: Optional[bool] + readability: Optional[bool] + proxy_enabled: Optional[bool] + respect_robots: Optional[bool] + query_selector: Optional[str] + full_resources: Optional[bool] + request_timeout: Optional[int] + run_in_background: Optional[bool] + skip_config_checks: Optional[bool] + + +class Spider: + def __init__(self, api_key: Optional[str] = None): + """ + Initialize the Spider with an API key. + + :param api_key: A string of the API key for Spider. Defaults to the SPIDER_API_KEY environment variable. + :raises ValueError: If no API key is provided. + """ + self.api_key = api_key or os.getenv("SPIDER_API_KEY") + if self.api_key is None: + raise ValueError("No API key provided") + + def api_post( + self, + endpoint: str, + data: dict, + stream: bool, + content_type: str = "application/json", + ): + """ + Send a POST request to the specified API endpoint. + + :param endpoint: The API endpoint to which the POST request is sent. + :param data: The data (dictionary) to be sent in the POST request. + :param stream: Boolean indicating if the response should be streamed. + :return: The JSON response or the raw response stream if stream is True. + """ + headers = self._prepare_headers(content_type) + response = self._post_request( + f"https://api.spider.cloud/v1/{endpoint}", data, headers, stream + ) + + if stream: + return response + elif response.status_code == 200: + return response.json() + else: + self._handle_error(response, f"post to {endpoint}") + + def api_get( + self, endpoint: str, stream: bool, content_type: str = "application/json" + ): + """ + Send a GET request to the specified endpoint. + + :param endpoint: The API endpoint from which to retrieve data. + :return: The JSON decoded response. + """ + headers = self._prepare_headers(content_type) + response = self._get_request( + f"https://api.spider.cloud/v1/{endpoint}", headers, stream + ) + if response.status_code == 200: + return response.json() + else: + self._handle_error(response, f"get from {endpoint}") + + def get_credits(self): + """ + Retrieve the account's remaining credits. + + :return: JSON response containing the number of credits left. + """ + return self.api_get("credits", stream=False) + + def scrape_url( + self, + url: str, + params: Optional[RequestParamsDict] = None, + stream: bool = False, + content_type: str = "application/json", + ): + """ + Scrape data from the specified URL. + + :param url: The URL from which to scrape data. + :param params: Optional dictionary of additional parameters for the scrape request. + :return: JSON response containing the scraping results. + """ + + # Add { "return_format": "markdown" } to the params if not already present + if "return_format" not in params: + params["return_format"] = "markdown" + + # Set limit to 1 + params["limit"] = 1 + + return self.api_post( + "crawl", {"url": url, **(params or {})}, stream, content_type + ) + + def crawl_url( + self, + url: str, + params: Optional[RequestParamsDict] = None, + stream: bool = False, + content_type: str = "application/json", + ): + """ + Start crawling at the specified URL. + + :param url: The URL to begin crawling. + :param params: Optional dictionary with additional parameters to customize the crawl. + :param stream: Boolean indicating if the response should be streamed. Defaults to False. + :return: JSON response or the raw response stream if streaming enabled. + """ + + # Add { "return_format": "markdown" } to the params if not already present + if "return_format" not in params: + params["return_format"] = "markdown" + + return self.api_post( + "crawl", {"url": url, **(params or {})}, stream, content_type + ) + + def links( + self, + url: str, + params: Optional[RequestParamsDict] = None, + stream: bool = False, + content_type: str = "application/json", + ): + """ + Retrieve links from the specified URL. + + :param url: The URL from which to extract links. + :param params: Optional parameters for the link retrieval request. + :return: JSON response containing the links. + """ + return self.api_post( + "links", {"url": url, **(params or {})}, stream, content_type + ) + + def extract_contacts( + self, + url: str, + params: Optional[RequestParamsDict] = None, + stream: bool = False, + content_type: str = "application/json", + ): + """ + Extract contact information from the specified URL. + + :param url: The URL from which to extract contact information. + :param params: Optional parameters for the contact extraction. + :return: JSON response containing extracted contact details. + """ + return self.api_post( + "pipeline/extract-contacts", + {"url": url, **(params or {})}, + stream, + content_type, + ) + + def label( + self, + url: str, + params: Optional[RequestParamsDict] = None, + stream: bool = False, + content_type: str = "application/json", + ): + """ + Apply labeling to data extracted from the specified URL. + + :param url: The URL to label data from. + :param params: Optional parameters to guide the labeling process. + :return: JSON response with labeled data. + """ + return self.api_post( + "pipeline/label", {"url": url, **(params or {})}, stream, content_type + ) + + def _prepare_headers(self, content_type: str = "application/json"): + return { + "Content-Type": content_type, + "Authorization": f"Bearer {self.api_key}", + "User-Agent": "Spider-Client/0.0.27", + } + + def _post_request(self, url: str, data, headers, stream=False): + return requests.post(url, headers=headers, json=data, stream=stream) + + def _get_request(self, url: str, headers, stream=False): + return requests.get(url, headers=headers, stream=stream) + + def _delete_request(self, url: str, headers, stream=False): + return requests.delete(url, headers=headers, stream=stream) + + def _handle_error(self, response, action): + if response.status_code in [402, 409, 500]: + error_message = response.json().get("error", "Unknown error occurred") + raise Exception( + f"Failed to {action}. Status code: {response.status_code}. Error: {error_message}" + ) + else: + raise Exception( + f"Unexpected error occurred while trying to {action}. Status code: {response.status_code}" + ) diff --git a/api/core/tools/provider/builtin/spider/tools/scraper_crawler.py b/api/core/tools/provider/builtin/spider/tools/scraper_crawler.py new file mode 100644 index 0000000000..64bbcc10cc --- /dev/null +++ b/api/core/tools/provider/builtin/spider/tools/scraper_crawler.py @@ -0,0 +1,47 @@ +from typing import Any, Union + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.provider.builtin.spider.spiderApp import Spider +from core.tools.tool.builtin_tool import BuiltinTool + + +class ScrapeTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + # initialize the app object with the api key + app = Spider(api_key=self.runtime.credentials['spider_api_key']) + + url = tool_parameters['url'] + mode = tool_parameters['mode'] + + options = { + 'limit': tool_parameters.get('limit', 0), + 'depth': tool_parameters.get('depth', 0), + 'blacklist': tool_parameters.get('blacklist', '').split(',') if tool_parameters.get('blacklist') else [], + 'whitelist': tool_parameters.get('whitelist', '').split(',') if tool_parameters.get('whitelist') else [], + 'readability': tool_parameters.get('readability', False), + } + + result = "" + + try: + if mode == 'scrape': + scrape_result = app.scrape_url( + url=url, + params=options, + ) + + for i in scrape_result: + result += "URL: " + i.get('url', '') + "\n" + result += "CONTENT: " + i.get('content', '') + "\n\n" + elif mode == 'crawl': + crawl_result = app.crawl_url( + url=tool_parameters['url'], + params=options, + ) + for i in crawl_result: + result += "URL: " + i.get('url', '') + "\n" + result += "CONTENT: " + i.get('content', '') + "\n\n" + except Exception as e: + return self.create_text_message("An error occured", str(e)) + + return self.create_text_message(result) diff --git a/api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml b/api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml new file mode 100644 index 0000000000..5b20c2fc2f --- /dev/null +++ b/api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml @@ -0,0 +1,102 @@ +identity: + name: scraper_crawler + author: William Espegren + label: + en_US: Web Scraper & Crawler + zh_Hans: 网页抓取与爬虫 +description: + human: + en_US: A tool for scraping & crawling webpages. Input should be a url. + zh_Hans: 用于抓取和爬取网页的工具。输入应该是一个网址。 + llm: A tool for scraping & crawling webpages. Input should be a url. +parameters: + - name: url + type: string + required: true + label: + en_US: URL + zh_Hans: 网址 + human_description: + en_US: url to be scraped or crawled + zh_Hans: 要抓取或爬取的网址 + llm_description: url to either be scraped or crawled + form: llm + - name: mode + type: select + required: true + options: + - value: scrape + label: + en_US: scrape + zh_Hans: 抓取 + - value: crawl + label: + en_US: crawl + zh_Hans: 爬取 + default: crawl + label: + en_US: Mode + zh_Hans: 模式 + human_description: + en_US: used for selecting to either scrape the website or crawl the entire website following subpages + zh_Hans: 用于选择抓取网站或爬取整个网站及其子页面 + form: form + - name: limit + type: number + required: false + label: + en_US: maximum number of pages to crawl + zh_Hans: 最大爬取页面数 + human_description: + en_US: specify the maximum number of pages to crawl per website. the crawler will stop after reaching this limit. + zh_Hans: 指定每个网站要爬取的最大页面数。爬虫将在达到此限制后停止。 + form: form + min: 0 + default: 0 + - name: depth + type: number + required: false + label: + en_US: maximum depth of pages to crawl + zh_Hans: 最大爬取深度 + human_description: + en_US: the crawl limit for maximum depth. + zh_Hans: 最大爬取深度的限制。 + form: form + min: 0 + default: 0 + - name: blacklist + type: string + required: false + label: + en_US: url patterns to exclude + zh_Hans: 要排除的URL模式 + human_description: + en_US: blacklist a set of paths that you do not want to crawl. you can use regex patterns to help with the list. + zh_Hans: 指定一组不想爬取的路径。您可以使用正则表达式模式来帮助定义列表。 + placeholder: + en_US: /blog/*, /about + form: form + - name: whitelist + type: string + required: false + label: + en_US: URL patterns to include + zh_Hans: 要包含的URL模式 + human_description: + en_US: Whitelist a set of paths that you want to crawl, ignoring all other routes that do not match the patterns. You can use regex patterns to help with the list. + zh_Hans: 指定一组要爬取的路径，忽略所有不匹配模式的其他路由。您可以使用正则表达式模式来帮助定义列表。 + placeholder: + en_US: /blog/*, /about + form: form + - name: readability + type: boolean + required: false + label: + en_US: Pre-process the content for LLM usage + zh_Hans: 仅返回页面的主要内容 + human_description: + en_US: Use Mozilla's readability to pre-process the content for reading. This may drastically improve the content for LLM usage. + zh_Hans: 如果启用，爬虫将仅返回页面的主要内容，不包括标题、导航、页脚等。 + form: form + default: false diff --git a/api/core/tools/provider/builtin/tianditu/_assets/icon.svg b/api/core/tools/provider/builtin/tianditu/_assets/icon.svg new file mode 100644 index 0000000000..749d4bda26 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/_assets/icon.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/api/core/tools/provider/builtin/tianditu/tianditu.py b/api/core/tools/provider/builtin/tianditu/tianditu.py new file mode 100644 index 0000000000..1f96be06b0 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tianditu.py @@ -0,0 +1,21 @@ +from typing import Any + +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.tianditu.tools.poisearch import PoiSearchTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class TiandituProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + try: + PoiSearchTool().fork_tool_runtime( + runtime={ + "credentials": credentials, + } + ).invoke(user_id='', + tool_parameters={ + 'content': '北京', + 'specify': '156110000', + }) + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) diff --git a/api/core/tools/provider/builtin/tianditu/tianditu.yaml b/api/core/tools/provider/builtin/tianditu/tianditu.yaml new file mode 100644 index 0000000000..77af834bdc --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tianditu.yaml @@ -0,0 +1,32 @@ +identity: + author: Listeng + name: tianditu + label: + en_US: Tianditu + zh_Hans: 天地图 + pt_BR: Tianditu + description: + en_US: The Tianditu tool provided the functions of place name search, geocoding, static maps generation, etc. in China region. + zh_Hans: 天地图工具可以调用天地图的接口，实现中国区域内的地名搜索、地理编码、静态地图等功能。 + pt_BR: The Tianditu tool provided the functions of place name search, geocoding, static maps generation, etc. in China region. + icon: icon.svg + tags: + - utilities + - travel +credentials_for_provider: + tianditu_api_key: + type: secret-input + required: true + label: + en_US: Tianditu API Key + zh_Hans: 天地图Key + pt_BR: Tianditu API key + placeholder: + en_US: Please input your Tianditu API key + zh_Hans: 请输入你的天地图Key + pt_BR: Please input your Tianditu API key + help: + en_US: Get your Tianditu API key from Tianditu + zh_Hans: 获取您的天地图Key + pt_BR: Get your Tianditu API key from Tianditu + url: http://lbs.tianditu.gov.cn/home.html diff --git a/api/core/tools/provider/builtin/tianditu/tools/geocoder.py b/api/core/tools/provider/builtin/tianditu/tools/geocoder.py new file mode 100644 index 0000000000..484a3768c8 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/geocoder.py @@ -0,0 +1,33 @@ +import json +from typing import Any, Union + +import requests + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class GeocoderTool(BuiltinTool): + + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + invoke tools + """ + base_url = 'http://api.tianditu.gov.cn/geocoder' + + keyword = tool_parameters.get('keyword', '') + if not keyword: + return self.create_text_message('Invalid parameter keyword') + + tk = self.runtime.credentials['tianditu_api_key'] + + params = { + 'keyWord': keyword, + } + + result = requests.get(base_url + '?ds=' + json.dumps(params, ensure_ascii=False) + '&tk=' + tk).json() + + return self.create_json_message(result) diff --git a/api/core/tools/provider/builtin/tianditu/tools/geocoder.yaml b/api/core/tools/provider/builtin/tianditu/tools/geocoder.yaml new file mode 100644 index 0000000000..d6a168f950 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/geocoder.yaml @@ -0,0 +1,26 @@ +identity: + name: geocoder + author: Listeng + label: + en_US: Get coords converted from address name + zh_Hans: 地理编码 + pt_BR: Get coords converted from address name +description: + human: + en_US: Geocoder + zh_Hans: 中国区域地理编码查询 + pt_BR: Geocoder + llm: A tool for geocoder in China +parameters: + - name: keyword + type: string + required: true + label: + en_US: keyword + zh_Hans: 搜索的关键字 + pt_BR: keyword + human_description: + en_US: keyword + zh_Hans: 搜索的关键字 + pt_BR: keyword + form: llm diff --git a/api/core/tools/provider/builtin/tianditu/tools/poisearch.py b/api/core/tools/provider/builtin/tianditu/tools/poisearch.py new file mode 100644 index 0000000000..08a5b8ef42 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/poisearch.py @@ -0,0 +1,45 @@ +import json +from typing import Any, Union + +import requests + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class PoiSearchTool(BuiltinTool): + + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + invoke tools + """ + geocoder_base_url = 'http://api.tianditu.gov.cn/geocoder' + base_url = 'http://api.tianditu.gov.cn/v2/search' + + keyword = tool_parameters.get('keyword', '') + if not keyword: + return self.create_text_message('Invalid parameter keyword') + + baseAddress = tool_parameters.get('baseAddress', '') + if not baseAddress: + return self.create_text_message('Invalid parameter baseAddress') + + tk = self.runtime.credentials['tianditu_api_key'] + + base_coords = requests.get(geocoder_base_url + '?ds=' + json.dumps({'keyWord': baseAddress,}, ensure_ascii=False) + '&tk=' + tk).json() + + params = { + 'keyWord': keyword, + 'queryRadius': 5000, + 'queryType': 3, + 'pointLonlat': base_coords['location']['lon'] + ',' + base_coords['location']['lat'], + 'start': 0, + 'count': 100, + } + + result = requests.get(base_url + '?postStr=' + json.dumps(params, ensure_ascii=False) + '&type=query&tk=' + tk).json() + + return self.create_json_message(result) diff --git a/api/core/tools/provider/builtin/tianditu/tools/poisearch.yaml b/api/core/tools/provider/builtin/tianditu/tools/poisearch.yaml new file mode 100644 index 0000000000..01289d24e3 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/poisearch.yaml @@ -0,0 +1,38 @@ +identity: + name: point_of_interest_search + author: Listeng + label: + en_US: Point of Interest search + zh_Hans: 兴趣点搜索 + pt_BR: Point of Interest search +description: + human: + en_US: Search for certain types of points of interest around a location + zh_Hans: 搜索某个位置周边的5公里内某种类型的兴趣点 + pt_BR: Search for certain types of points of interest around a location + llm: A tool for searching for certain types of points of interest around a location +parameters: + - name: keyword + type: string + required: true + label: + en_US: poi keyword + zh_Hans: 兴趣点的关键字 + pt_BR: poi keyword + human_description: + en_US: poi keyword + zh_Hans: 兴趣点的关键字 + pt_BR: poi keyword + form: llm + - name: baseAddress + type: string + required: true + label: + en_US: base current point + zh_Hans: 当前位置的关键字 + pt_BR: base current point + human_description: + en_US: base current point + zh_Hans: 当前位置的关键字 + pt_BR: base current point + form: llm diff --git a/api/core/tools/provider/builtin/tianditu/tools/staticmap.py b/api/core/tools/provider/builtin/tianditu/tools/staticmap.py new file mode 100644 index 0000000000..ecac4404ca --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/staticmap.py @@ -0,0 +1,36 @@ +import json +from typing import Any, Union + +import requests + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class PoiSearchTool(BuiltinTool): + + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + invoke tools + """ + + geocoder_base_url = 'http://api.tianditu.gov.cn/geocoder' + base_url = 'http://api.tianditu.gov.cn/staticimage' + + keyword = tool_parameters.get('keyword', '') + if not keyword: + return self.create_text_message('Invalid parameter keyword') + + tk = self.runtime.credentials['tianditu_api_key'] + + keyword_coords = requests.get(geocoder_base_url + '?ds=' + json.dumps({'keyWord': keyword,}, ensure_ascii=False) + '&tk=' + tk).json() + coords = keyword_coords['location']['lon'] + ',' + keyword_coords['location']['lat'] + + result = requests.get(base_url + '?center=' + coords + '&markers=' + coords + '&width=400&height=300&zoom=14&tk=' + tk).content + + return self.create_blob_message(blob=result, + meta={'mime_type': 'image/png'}, + save_as=self.VARIABLE_KEY.IMAGE.value) diff --git a/api/core/tools/provider/builtin/tianditu/tools/staticmap.yaml b/api/core/tools/provider/builtin/tianditu/tools/staticmap.yaml new file mode 100644 index 0000000000..fc54c42806 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/staticmap.yaml @@ -0,0 +1,26 @@ +identity: + name: generate_static_map + author: Listeng + label: + en_US: Generate a static map + zh_Hans: 生成静态地图 + pt_BR: Generate a static map +description: + human: + en_US: Generate a static map + zh_Hans: 生成静态地图 + pt_BR: Generate a static map + llm: A tool for generate a static map +parameters: + - name: keyword + type: string + required: true + label: + en_US: keyword + zh_Hans: 搜索的关键字 + pt_BR: keyword + human_description: + en_US: keyword + zh_Hans: 搜索的关键字 + pt_BR: keyword + form: llm diff --git a/api/core/tools/provider/builtin_tool_provider.py b/api/core/tools/provider/builtin_tool_provider.py index 47e33b70c9..bcf41c90ed 100644 --- a/api/core/tools/provider/builtin_tool_provider.py +++ b/api/core/tools/provider/builtin_tool_provider.py @@ -27,7 +27,7 @@ class BuiltinToolProviderController(ToolProviderController): provider = self.__class__.__module__.split('.')[-1] yaml_path = path.join(path.dirname(path.realpath(__file__)), 'builtin', provider, f'{provider}.yaml') try: - provider_yaml = load_yaml_file(yaml_path) + provider_yaml = load_yaml_file(yaml_path, ignore_error=False) except Exception as e: raise ToolProviderNotFoundError(f'can not load provider yaml for {provider}: {e}') @@ -58,7 +58,7 @@ class BuiltinToolProviderController(ToolProviderController): for tool_file in tool_files: # get tool name tool_name = tool_file.split(".")[0] - tool = load_yaml_file(path.join(tool_path, tool_file)) + tool = load_yaml_file(path.join(tool_path, tool_file), ignore_error=False) # get tool class, import the module assistant_tool_class = load_single_subclass_from_source( diff --git a/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py b/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py index 5b053678f3..1a0933af16 100644 --- a/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py +++ b/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py @@ -7,14 +7,14 @@ from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCa from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelType from core.rag.datasource.retrieval_service import RetrievalService -from core.rag.rerank.rerank import RerankRunner +from core.rag.rerank.rerank_model import RerankModelRunner from core.rag.retrieval.retrival_methods import RetrievalMethod from core.tools.tool.dataset_retriever.dataset_retriever_base_tool import DatasetRetrieverBaseTool from extensions.ext_database import db from models.dataset import Dataset, Document, DocumentSegment default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -72,7 +72,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool): model=self.reranking_model_name ) - rerank_runner = RerankRunner(rerank_model_instance) + rerank_runner = RerankModelRunner(rerank_model_instance) all_documents = rerank_runner.run(query, all_documents, self.score_threshold, self.top_k) for hit_callback in self.hit_callbacks: @@ -180,7 +180,8 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool): score_threshold=retrieval_model['score_threshold'] if retrieval_model['score_threshold_enabled'] else None, reranking_model=retrieval_model['reranking_model'] - if retrieval_model['reranking_enable'] else None + if retrieval_model['reranking_enable'] else None, + weights=retrieval_model.get('weights', None), ) all_documents.extend(documents) \ No newline at end of file diff --git a/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py b/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py index de2ce5858a..397ff7966e 100644 --- a/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py +++ b/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py @@ -8,7 +8,7 @@ from extensions.ext_database import db from models.dataset import Dataset, Document, DocumentSegment default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -78,7 +78,8 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool): score_threshold=retrieval_model['score_threshold'] if retrieval_model['score_threshold_enabled'] else None, reranking_model=retrieval_model['reranking_model'] - if retrieval_model['reranking_enable'] else None + if retrieval_model['reranking_enable'] else None, + weights=retrieval_model.get('weights', None), ) else: documents = [] diff --git a/api/core/tools/tool/tool.py b/api/core/tools/tool/tool.py index 291cac5ee3..e679434bde 100644 --- a/api/core/tools/tool/tool.py +++ b/api/core/tools/tool/tool.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from collections.abc import Generator +from collections.abc import Generator, Mapping from copy import deepcopy from enum import Enum from typing import Any, Optional, Union @@ -191,8 +191,9 @@ class Tool(BaseModel, ABC): return result - def invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Generator[ToolInvokeMessage]: + def invoke(self, user_id: str, tool_parameters: Mapping[str, Any]) -> Generator[ToolInvokeMessage]: # update tool_parameters + # TODO: Fix type error. if self.runtime.runtime_parameters: tool_parameters.update(self.runtime.runtime_parameters) @@ -206,7 +207,7 @@ class Tool(BaseModel, ABC): return result - def _transform_tool_parameters_type(self, tool_parameters: dict[str, Any]) -> dict[str, Any]: + def _transform_tool_parameters_type(self, tool_parameters: Mapping[str, Any]) -> dict[str, Any]: """ Transform tool parameters type """ @@ -239,7 +240,7 @@ class Tool(BaseModel, ABC): :return: the runtime parameters """ - return self.parameters + return self.parameters or [] def get_all_runtime_parameters(self) -> list[ToolParameter]: """ diff --git a/api/core/tools/tool_engine.py b/api/core/tools/tool_engine.py index 7d94eedc5f..9397f22494 100644 --- a/api/core/tools/tool_engine.py +++ b/api/core/tools/tool_engine.py @@ -1,5 +1,5 @@ import json -from collections.abc import Generator +from collections.abc import Generator, Mapping from copy import deepcopy from datetime import datetime, timezone from mimetypes import guess_type @@ -48,7 +48,7 @@ class ToolEngine: if isinstance(tool_parameters, str): # check if this tool has only one parameter parameters = [ - parameter for parameter in tool.get_runtime_parameters() + parameter for parameter in tool.get_runtime_parameters() or [] if parameter.form == ToolParameter.ToolParameterForm.LLM ] if parameters and len(parameters) == 1: @@ -136,8 +136,8 @@ class ToolEngine: return error_response, [], ToolInvokeMeta.error_instance(error_response) @staticmethod - def workflow_invoke(tool: Tool, tool_parameters: dict, - user_id: str, workflow_id: str, + def workflow_invoke(tool: Tool, tool_parameters: Mapping[str, Any], + user_id: str, workflow_tool_callback: DifyWorkflowCallbackHandler, workflow_call_depth: int, ) -> Generator[ToolInvokeMessage, None, None]: @@ -154,7 +154,9 @@ class ToolEngine: if isinstance(tool, WorkflowTool): tool.workflow_call_depth = workflow_call_depth + 1 - response = tool.invoke(user_id, tool_parameters) + if tool.runtime and tool.runtime.runtime_parameters: + tool_parameters = {**tool.runtime.runtime_parameters, **tool_parameters} + response = tool.invoke(user_id=user_id, tool_parameters=tool_parameters) # hit the callback handler workflow_tool_callback.on_tool_end( diff --git a/api/core/tools/tool_file_manager.py b/api/core/tools/tool_file_manager.py index 207f009eed..f9f7c7d78a 100644 --- a/api/core/tools/tool_file_manager.py +++ b/api/core/tools/tool_file_manager.py @@ -9,9 +9,9 @@ from mimetypes import guess_extension, guess_type from typing import Optional, Union from uuid import uuid4 -from flask import current_app from httpx import get +from configs import dify_config from extensions.ext_database import db from extensions.ext_storage import storage from models.model import MessageFile @@ -26,25 +26,25 @@ class ToolFileManager: """ sign file to get a temporary url """ - base_url = current_app.config.get('FILES_URL') + base_url = dify_config.FILES_URL file_preview_url = f'{base_url}/files/tools/{tool_file_id}{extension}' timestamp = str(int(time.time())) nonce = os.urandom(16).hex() - data_to_sign = f"file-preview|{tool_file_id}|{timestamp}|{nonce}" - secret_key = current_app.config['SECRET_KEY'].encode() + data_to_sign = f'file-preview|{tool_file_id}|{timestamp}|{nonce}' + secret_key = dify_config.SECRET_KEY.encode() if dify_config.SECRET_KEY else b'' sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest() encoded_sign = base64.urlsafe_b64encode(sign).decode() - return f"{file_preview_url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}" + return f'{file_preview_url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}' @staticmethod def verify_file(file_id: str, timestamp: str, nonce: str, sign: str) -> bool: """ verify signature """ - data_to_sign = f"file-preview|{file_id}|{timestamp}|{nonce}" - secret_key = current_app.config['SECRET_KEY'].encode() + data_to_sign = f'file-preview|{file_id}|{timestamp}|{nonce}' + secret_key = dify_config.SECRET_KEY.encode() if dify_config.SECRET_KEY else b'' recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest() recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode() @@ -53,23 +53,23 @@ class ToolFileManager: return False current_time = int(time.time()) - return current_time - int(timestamp) <= current_app.config.get('FILES_ACCESS_TIMEOUT') + return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT @staticmethod - def create_file_by_raw(user_id: str, tenant_id: str, - conversation_id: Optional[str], file_binary: bytes, - mimetype: str - ) -> ToolFile: + def create_file_by_raw( + user_id: str, tenant_id: str, conversation_id: Optional[str], file_binary: bytes, mimetype: str + ) -> ToolFile: """ create file """ extension = guess_extension(mimetype) or '.bin' unique_name = uuid4().hex - filename = f"tools/{tenant_id}/{unique_name}{extension}" + filename = f'tools/{tenant_id}/{unique_name}{extension}' storage.save(filename, file_binary) - tool_file = ToolFile(user_id=user_id, tenant_id=tenant_id, - conversation_id=conversation_id, file_key=filename, mimetype=mimetype) + tool_file = ToolFile( + user_id=user_id, tenant_id=tenant_id, conversation_id=conversation_id, file_key=filename, mimetype=mimetype + ) db.session.add(tool_file) db.session.commit() @@ -77,9 +77,12 @@ class ToolFileManager: return tool_file @staticmethod - def create_file_by_url(user_id: str, tenant_id: str, - conversation_id: str, file_url: str, - ) -> ToolFile: + def create_file_by_url( + user_id: str, + tenant_id: str, + conversation_id: str, + file_url: str, + ) -> ToolFile: """ create file """ @@ -90,12 +93,17 @@ class ToolFileManager: mimetype = guess_type(file_url)[0] or 'octet/stream' extension = guess_extension(mimetype) or '.bin' unique_name = uuid4().hex - filename = f"tools/{tenant_id}/{unique_name}{extension}" + filename = f'tools/{tenant_id}/{unique_name}{extension}' storage.save(filename, blob) - tool_file = ToolFile(user_id=user_id, tenant_id=tenant_id, - conversation_id=conversation_id, file_key=filename, - mimetype=mimetype, original_url=file_url) + tool_file = ToolFile( + user_id=user_id, + tenant_id=tenant_id, + conversation_id=conversation_id, + file_key=filename, + mimetype=mimetype, + original_url=file_url, + ) db.session.add(tool_file) db.session.commit() @@ -103,15 +111,15 @@ class ToolFileManager: return tool_file @staticmethod - def create_file_by_key(user_id: str, tenant_id: str, - conversation_id: str, file_key: str, - mimetype: str - ) -> ToolFile: + def create_file_by_key( + user_id: str, tenant_id: str, conversation_id: str, file_key: str, mimetype: str + ) -> ToolFile: """ create file """ - tool_file = ToolFile(user_id=user_id, tenant_id=tenant_id, - conversation_id=conversation_id, file_key=file_key, mimetype=mimetype) + tool_file = ToolFile( + user_id=user_id, tenant_id=tenant_id, conversation_id=conversation_id, file_key=file_key, mimetype=mimetype + ) return tool_file @staticmethod @@ -123,9 +131,13 @@ class ToolFileManager: :return: the binary of the file, mime type """ - tool_file: ToolFile = db.session.query(ToolFile).filter( - ToolFile.id == id, - ).first() + tool_file: ToolFile = ( + db.session.query(ToolFile) + .filter( + ToolFile.id == id, + ) + .first() + ) if not tool_file: return None @@ -143,18 +155,31 @@ class ToolFileManager: :return: the binary of the file, mime type """ - message_file: MessageFile = db.session.query(MessageFile).filter( - MessageFile.id == id, - ).first() + message_file: MessageFile = ( + db.session.query(MessageFile) + .filter( + MessageFile.id == id, + ) + .first() + ) - # get tool file id - tool_file_id = message_file.url.split('/')[-1] - # trim extension - tool_file_id = tool_file_id.split('.')[0] + # Check if message_file is not None + if message_file is not None: + # get tool file id + tool_file_id = message_file.url.split('/')[-1] + # trim extension + tool_file_id = tool_file_id.split('.')[0] + else: + tool_file_id = None - tool_file: ToolFile = db.session.query(ToolFile).filter( - ToolFile.id == tool_file_id, - ).first() + + tool_file: ToolFile = ( + db.session.query(ToolFile) + .filter( + ToolFile.id == tool_file_id, + ) + .first() + ) if not tool_file: return None @@ -172,9 +197,13 @@ class ToolFileManager: :return: the binary of the file, mime type """ - tool_file: ToolFile = db.session.query(ToolFile).filter( - ToolFile.id == tool_file_id, - ).first() + tool_file: ToolFile = ( + db.session.query(ToolFile) + .filter( + ToolFile.id == tool_file_id, + ) + .first() + ) if not tool_file: return None diff --git a/api/core/tools/tool_manager.py b/api/core/tools/tool_manager.py index 5822841db7..8c5cc4c540 100644 --- a/api/core/tools/tool_manager.py +++ b/api/core/tools/tool_manager.py @@ -6,8 +6,7 @@ from os import listdir, path from threading import Lock from typing import Any, Union -from flask import current_app - +from configs import dify_config from core.agent.entities import AgentToolEntity from core.app.entities.app_invoke_entities import InvokeFrom from core.helper.module_import_helper import load_single_subclass_from_source @@ -541,8 +540,10 @@ class ToolManager: :param provider_id: the id of the provider :return: """ + provider_type = provider_type + provider_id = provider_id if provider_type == ToolProviderType.BUILT_IN: - return (current_app.config.get("CONSOLE_API_URL") + return (dify_config.CONSOLE_API_URL + "/console/api/workspaces/current/tool-provider/builtin/" + provider_id + "/icon") @@ -551,7 +552,7 @@ class ToolManager: provider: ApiToolProvider = db.session.query(ApiToolProvider).filter( ApiToolProvider.tenant_id == tenant_id, ApiToolProvider.id == provider_id - ) + ).first() return json.loads(provider.icon) except: return { @@ -570,4 +571,4 @@ class ToolManager: else: raise ValueError(f"provider type {provider_type} not found") -ToolManager.load_builtin_providers_cache() \ No newline at end of file +ToolManager.load_builtin_providers_cache() diff --git a/api/core/tools/utils/tool_parameter_converter.py b/api/core/tools/utils/tool_parameter_converter.py index 0c4ec00ec6..6f88eeaa0a 100644 --- a/api/core/tools/utils/tool_parameter_converter.py +++ b/api/core/tools/utils/tool_parameter_converter.py @@ -53,7 +53,7 @@ class ToolParameterConverter: case ToolParameter.ToolParameterType.NUMBER: if isinstance(value, int) | isinstance(value, float): return value - elif isinstance(value, str): + elif isinstance(value, str) and value != '': if '.' in value: return float(value) else: diff --git a/api/core/tools/utils/web_reader_tool.py b/api/core/tools/utils/web_reader_tool.py index 1e7eb129a7..f6f04271d6 100644 --- a/api/core/tools/utils/web_reader_tool.py +++ b/api/core/tools/utils/web_reader_tool.py @@ -10,11 +10,11 @@ import unicodedata from contextlib import contextmanager from urllib.parse import unquote -import requests +import cloudscraper from bs4 import BeautifulSoup, CData, Comment, NavigableString -from newspaper import Article from regex import regex +from core.helper import ssrf_proxy from core.rag.extractor import extract_processor from core.rag.extractor.extract_processor import ExtractProcessor @@ -44,35 +44,41 @@ def get_url(url: str, user_agent: str = None) -> str: main_content_type = None supported_content_types = extract_processor.SUPPORT_URL_CONTENT_TYPES + ["text/html"] - response = requests.head(url, headers=headers, allow_redirects=True, timeout=(5, 10)) + response = ssrf_proxy.head(url, headers=headers, follow_redirects=True, timeout=(5, 10)) + + if response.status_code == 200: + # check content-type + content_type = response.headers.get('Content-Type') + if content_type: + main_content_type = response.headers.get('Content-Type').split(';')[0].strip() + else: + content_disposition = response.headers.get('Content-Disposition', '') + filename_match = re.search(r'filename="([^"]+)"', content_disposition) + if filename_match: + filename = unquote(filename_match.group(1)) + extension = re.search(r'\.(\w+)$', filename) + if extension: + main_content_type = mimetypes.guess_type(filename)[0] + + if main_content_type not in supported_content_types: + return "Unsupported content-type [{}] of URL.".format(main_content_type) + + if main_content_type in extract_processor.SUPPORT_URL_CONTENT_TYPES: + return ExtractProcessor.load_from_url(url, return_text=True) + + response = ssrf_proxy.get(url, headers=headers, follow_redirects=True, timeout=(120, 300)) + elif response.status_code == 403: + scraper = cloudscraper.create_scraper() + scraper.perform_request = ssrf_proxy.make_request + response = scraper.get(url, headers=headers, follow_redirects=True, timeout=(120, 300)) if response.status_code != 200: return "URL returned status code {}.".format(response.status_code) - # check content-type - content_type = response.headers.get('Content-Type') - if content_type: - main_content_type = response.headers.get('Content-Type').split(';')[0].strip() - else: - content_disposition = response.headers.get('Content-Disposition', '') - filename_match = re.search(r'filename="([^"]+)"', content_disposition) - if filename_match: - filename = unquote(filename_match.group(1)) - extension = re.search(r'\.(\w+)$', filename) - if extension: - main_content_type = mimetypes.guess_type(filename)[0] - - if main_content_type not in supported_content_types: - return "Unsupported content-type [{}] of URL.".format(main_content_type) - - if main_content_type in extract_processor.SUPPORT_URL_CONTENT_TYPES: - return ExtractProcessor.load_from_url(url, return_text=True) - - response = requests.get(url, headers=headers, allow_redirects=True, timeout=(120, 300)) a = extract_using_readabilipy(response.text) if not a['plain_text'] or not a['plain_text'].strip(): - return get_url_from_newspaper3k(url) + return '' res = FULL_TEMPLATE.format( title=a['title'], @@ -85,23 +91,6 @@ def get_url(url: str, user_agent: str = None) -> str: return res -def get_url_from_newspaper3k(url: str) -> str: - - a = Article(url) - a.download() - a.parse() - - res = FULL_TEMPLATE.format( - title=a.title, - authors=a.authors, - publish_date=a.publish_date, - top_image=a.top_image, - text=a.text, - ) - - return res - - def extract_using_readabilipy(html): with tempfile.NamedTemporaryFile(delete=False, mode='w+') as f_html: f_html.write(html) diff --git a/api/core/tools/utils/yaml_utils.py b/api/core/tools/utils/yaml_utils.py index 3526647b4f..21155a6960 100644 --- a/api/core/tools/utils/yaml_utils.py +++ b/api/core/tools/utils/yaml_utils.py @@ -1,35 +1,32 @@ import logging -import os +from typing import Any import yaml from yaml import YAMLError logger = logging.getLogger(__name__) -def load_yaml_file(file_path: str, ignore_error: bool = False) -> dict: + +def load_yaml_file(file_path: str, ignore_error: bool = True, default_value: Any = {}) -> Any: """ - Safe loading a YAML file to a dict + Safe loading a YAML file :param file_path: the path of the YAML file :param ignore_error: - if True, return empty dict if error occurs and the error will be logged in warning level + if True, return default_value if error occurs and the error will be logged in debug level if False, raise error if error occurs - :return: a dict of the YAML content + :param default_value: the value returned when errors ignored + :return: an object of the YAML content """ try: - if not file_path or not os.path.exists(file_path): - raise FileNotFoundError(f'Failed to load YAML file {file_path}: file not found') - - with open(file_path, encoding='utf-8') as file: + with open(file_path, encoding='utf-8') as yaml_file: try: - return yaml.safe_load(file) + yaml_content = yaml.safe_load(yaml_file) + return yaml_content if yaml_content else default_value except Exception as e: raise YAMLError(f'Failed to load YAML file {file_path}: {e}') - except FileNotFoundError as e: - logger.debug(f'Failed to load YAML file {file_path}: {e}') - return {} except Exception as e: if ignore_error: - logger.warning(f'Failed to load YAML file {file_path}: {e}') - return {} + logger.debug(f'Failed to load YAML file {file_path}: {e}') + return default_value else: raise e diff --git a/api/core/workflow/callbacks/base_workflow_callback.py b/api/core/workflow/callbacks/base_workflow_callback.py index 3b0d51d868..6db8adf4c2 100644 --- a/api/core/workflow/callbacks/base_workflow_callback.py +++ b/api/core/workflow/callbacks/base_workflow_callback.py @@ -6,7 +6,7 @@ from core.workflow.entities.base_node_data_entities import BaseNodeData from core.workflow.entities.node_entities import NodeType -class BaseWorkflowCallback(ABC): +class WorkflowCallback(ABC): @abstractmethod def on_workflow_run_started(self) -> None: """ @@ -78,7 +78,7 @@ class BaseWorkflowCallback(ABC): node_type: NodeType, node_run_index: int = 1, node_data: Optional[BaseNodeData] = None, - inputs: dict = None, + inputs: Optional[dict] = None, predecessor_node_id: Optional[str] = None, metadata: Optional[dict] = None) -> None: """ diff --git a/api/core/workflow/entities/node_entities.py b/api/core/workflow/entities/node_entities.py index ae86463407..996aae94c2 100644 --- a/api/core/workflow/entities/node_entities.py +++ b/api/core/workflow/entities/node_entities.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from enum import Enum from typing import Any, Optional @@ -82,9 +83,9 @@ class NodeRunResult(BaseModel): """ status: WorkflowNodeExecutionStatus = WorkflowNodeExecutionStatus.RUNNING - inputs: Optional[dict] = None # node inputs + inputs: Optional[Mapping[str, Any]] = None # node inputs process_data: Optional[dict] = None # process data - outputs: Optional[dict] = None # node outputs + outputs: Optional[Mapping[str, Any]] = None # node outputs metadata: Optional[dict[NodeRunMetadataKey, Any]] = None # node metadata edge_source_handle: Optional[str] = None # source handle id of node with multiple branches diff --git a/api/core/workflow/entities/variable_pool.py b/api/core/workflow/entities/variable_pool.py index c04770616c..a27b4261e4 100644 --- a/api/core/workflow/entities/variable_pool.py +++ b/api/core/workflow/entities/variable_pool.py @@ -1,101 +1,134 @@ -from enum import Enum -from typing import Any, Optional, Union +from collections import defaultdict +from collections.abc import Mapping, Sequence +from typing import Any, Union +from typing_extensions import deprecated + +from core.app.segments import Segment, Variable, factory from core.file.file_obj import FileVar from core.workflow.entities.node_entities import SystemVariable VariableValue = Union[str, int, float, dict, list, FileVar] -class ValueType(Enum): - """ - Value Type Enum - """ - STRING = "string" - NUMBER = "number" - OBJECT = "object" - ARRAY_STRING = "array[string]" - ARRAY_NUMBER = "array[number]" - ARRAY_OBJECT = "array[object]" - ARRAY_FILE = "array[file]" - FILE = "file" +SYSTEM_VARIABLE_NODE_ID = 'sys' +ENVIRONMENT_VARIABLE_NODE_ID = 'env' class VariablePool: - - def __init__(self, system_variables: dict[SystemVariable, Any], - user_inputs: dict) -> None: + def __init__( + self, + system_variables: Mapping[SystemVariable, Any], + user_inputs: Mapping[str, Any], + environment_variables: Sequence[Variable], + ) -> None: # system variables # for example: # { # 'query': 'abc', # 'files': [] # } - self.variables_mapping = {} + + # Varaible dictionary is a dictionary for looking up variables by their selector. + # The first element of the selector is the node id, it's the first-level key in the dictionary. + # Other elements of the selector are the keys in the second-level dictionary. To get the key, we hash the + # elements of the selector except the first one. + self._variable_dictionary: dict[str, dict[int, Segment]] = defaultdict(dict) + + # TODO: This user inputs is not used for pool. self.user_inputs = user_inputs + + # Add system variables to the variable pool self.system_variables = system_variables - for system_variable, value in system_variables.items(): - self.append_variable('sys', [system_variable.value], value) + for key, value in system_variables.items(): + self.add((SYSTEM_VARIABLE_NODE_ID, key.value), value) - def append_variable(self, node_id: str, variable_key_list: list[str], value: VariableValue) -> None: + # Add environment variables to the variable pool + for var in environment_variables or []: + self.add((ENVIRONMENT_VARIABLE_NODE_ID, var.name), var) + + def add(self, selector: Sequence[str], value: Any, /) -> None: """ - Append variable - :param node_id: node id - :param variable_key_list: variable key list, like: ['result', 'text'] - :param value: value - :return: + Adds a variable to the variable pool. + + Args: + selector (Sequence[str]): The selector for the variable. + value (VariableValue): The value of the variable. + + Raises: + ValueError: If the selector is invalid. + + Returns: + None """ - if node_id not in self.variables_mapping: - self.variables_mapping[node_id] = {} + if len(selector) < 2: + raise ValueError('Invalid selector') - variable_key_list_hash = hash(tuple(variable_key_list)) + if value is None: + return - self.variables_mapping[node_id][variable_key_list_hash] = value + if isinstance(value, Segment): + v = value + else: + v = factory.build_segment(value) - def get_variable_value(self, variable_selector: list[str], - target_value_type: Optional[ValueType] = None) -> Optional[VariableValue]: + hash_key = hash(tuple(selector[1:])) + self._variable_dictionary[selector[0]][hash_key] = v + + def get(self, selector: Sequence[str], /) -> Segment | None: """ - Get variable - :param variable_selector: include node_id and variables - :param target_value_type: target value type - :return: + Retrieves the value from the variable pool based on the given selector. + + Args: + selector (Sequence[str]): The selector used to identify the variable. + + Returns: + Any: The value associated with the given selector. + + Raises: + ValueError: If the selector is invalid. """ - if len(variable_selector) < 2: - raise ValueError('Invalid value selector') - - node_id = variable_selector[0] - if node_id not in self.variables_mapping: - return None - - # fetch variable keys, pop node_id - variable_key_list = variable_selector[1:] - - variable_key_list_hash = hash(tuple(variable_key_list)) - - value = self.variables_mapping[node_id].get(variable_key_list_hash) - - if target_value_type: - if target_value_type == ValueType.STRING: - return str(value) - elif target_value_type == ValueType.NUMBER: - return int(value) - elif target_value_type == ValueType.OBJECT: - if not isinstance(value, dict): - raise ValueError('Invalid value type: object') - elif target_value_type in [ValueType.ARRAY_STRING, - ValueType.ARRAY_NUMBER, - ValueType.ARRAY_OBJECT, - ValueType.ARRAY_FILE]: - if not isinstance(value, list): - raise ValueError(f'Invalid value type: {target_value_type.value}') + if len(selector) < 2: + raise ValueError('Invalid selector') + hash_key = hash(tuple(selector[1:])) + value = self._variable_dictionary[selector[0]].get(hash_key) return value - def clear_node_variables(self, node_id: str) -> None: + @deprecated('This method is deprecated, use `get` instead.') + def get_any(self, selector: Sequence[str], /) -> Any | None: """ - Clear node variables - :param node_id: node id - :return: + Retrieves the value from the variable pool based on the given selector. + + Args: + selector (Sequence[str]): The selector used to identify the variable. + + Returns: + Any: The value associated with the given selector. + + Raises: + ValueError: If the selector is invalid. """ - if node_id in self.variables_mapping: - self.variables_mapping.pop(node_id) \ No newline at end of file + if len(selector) < 2: + raise ValueError('Invalid selector') + hash_key = hash(tuple(selector[1:])) + value = self._variable_dictionary[selector[0]].get(hash_key) + return value.to_object() if value else None + + def remove(self, selector: Sequence[str], /): + """ + Remove variables from the variable pool based on the given selector. + + Args: + selector (Sequence[str]): A sequence of strings representing the selector. + + Returns: + None + """ + if not selector: + return + if len(selector) == 1: + self._variable_dictionary[selector[0]] = {} + return + hash_key = hash(tuple(selector[1:])) + self._variable_dictionary[selector[0]].pop(hash_key, None) diff --git a/api/core/workflow/nodes/answer/answer_node.py b/api/core/workflow/nodes/answer/answer_node.py index e8f1678ecb..5bae27092f 100644 --- a/api/core/workflow/nodes/answer/answer_node.py +++ b/api/core/workflow/nodes/answer/answer_node.py @@ -1,7 +1,5 @@ -import json from typing import cast -from core.file.file_obj import FileVar from core.prompt.utils.prompt_template_parser import PromptTemplateParser from core.workflow.entities.base_node_data_entities import BaseNodeData from core.workflow.entities.node_entities import NodeRunResult, NodeType @@ -19,7 +17,7 @@ from models.workflow import WorkflowNodeExecutionStatus class AnswerNode(BaseNode): _node_data_cls = AnswerNodeData - node_type = NodeType.ANSWER + _node_type: NodeType = NodeType.ANSWER def _run(self, variable_pool: VariablePool) -> NodeRunResult: """ @@ -28,7 +26,7 @@ class AnswerNode(BaseNode): :return: """ node_data = self.node_data - node_data = cast(self._node_data_cls, node_data) + node_data = cast(AnswerNodeData, node_data) # generate routes generate_routes = self.extract_generate_route_from_node_data(node_data) @@ -38,31 +36,9 @@ class AnswerNode(BaseNode): if part.type == "var": part = cast(VarGenerateRouteChunk, part) value_selector = part.value_selector - value = variable_pool.get_variable_value( - variable_selector=value_selector - ) - - text = '' - if isinstance(value, str | int | float): - text = str(value) - elif isinstance(value, dict): - # other types - text = json.dumps(value, ensure_ascii=False) - elif isinstance(value, FileVar): - # convert file to markdown - text = value.to_markdown() - elif isinstance(value, list): - for item in value: - if isinstance(item, FileVar): - text += item.to_markdown() + ' ' - - text = text.strip() - - if not text and value: - # other types - text = json.dumps(value, ensure_ascii=False) - - answer += text + value = variable_pool.get(value_selector) + if value: + answer += value.markdown else: part = cast(TextGenerateRouteChunk, part) answer += part.text @@ -82,7 +58,7 @@ class AnswerNode(BaseNode): :return: """ node_data = cls._node_data_cls(**config.get("data", {})) - node_data = cast(cls._node_data_cls, node_data) + node_data = cast(AnswerNodeData, node_data) return cls.extract_generate_route_from_node_data(node_data) @@ -143,7 +119,7 @@ class AnswerNode(BaseNode): :return: """ node_data = node_data - node_data = cast(cls._node_data_cls, node_data) + node_data = cast(AnswerNodeData, node_data) variable_template_parser = VariableTemplateParser(template=node_data.answer) variable_selectors = variable_template_parser.extract_variable_selectors() diff --git a/api/core/workflow/nodes/base_node.py b/api/core/workflow/nodes/base_node.py index fa7d6424f1..f42cee4ccd 100644 --- a/api/core/workflow/nodes/base_node.py +++ b/api/core/workflow/nodes/base_node.py @@ -1,9 +1,10 @@ from abc import ABC, abstractmethod +from collections.abc import Mapping, Sequence from enum import Enum -from typing import Optional +from typing import Any, Optional from core.app.entities.app_invoke_entities import InvokeFrom -from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.base_node_data_entities import BaseIterationState, BaseNodeData from core.workflow.entities.node_entities import NodeRunResult, NodeType from core.workflow.entities.variable_pool import VariablePool @@ -46,7 +47,7 @@ class BaseNode(ABC): node_data: BaseNodeData node_run_result: Optional[NodeRunResult] = None - callbacks: list[BaseWorkflowCallback] + callbacks: Sequence[WorkflowCallback] def __init__(self, tenant_id: str, app_id: str, @@ -54,8 +55,8 @@ class BaseNode(ABC): user_id: str, user_from: UserFrom, invoke_from: InvokeFrom, - config: dict, - callbacks: list[BaseWorkflowCallback] = None, + config: Mapping[str, Any], + callbacks: Sequence[WorkflowCallback] | None = None, workflow_call_depth: int = 0) -> None: self.tenant_id = tenant_id self.app_id = app_id @@ -65,7 +66,8 @@ class BaseNode(ABC): self.invoke_from = invoke_from self.workflow_call_depth = workflow_call_depth - self.node_id = config.get("id") + # TODO: May need to check if key exists. + self.node_id = config["id"] if not self.node_id: raise ValueError("Node ID is required.") @@ -113,7 +115,7 @@ class BaseNode(ABC): ) @classmethod - def extract_variable_selector_to_variable_mapping(cls, config: dict) -> dict[str, list[str]]: + def extract_variable_selector_to_variable_mapping(cls, config: dict): """ Extract variable selector to variable mapping :param config: node config @@ -123,14 +125,13 @@ class BaseNode(ABC): return cls._extract_variable_selector_to_variable_mapping(node_data) @classmethod - @abstractmethod - def _extract_variable_selector_to_variable_mapping(cls, node_data: BaseNodeData) -> dict[str, list[str]]: + def _extract_variable_selector_to_variable_mapping(cls, node_data: BaseNodeData) -> Mapping[str, Sequence[str]]: """ Extract variable selector to variable mapping :param node_data: node data :return: """ - raise NotImplementedError + return {} @classmethod def get_default_config(cls, filters: Optional[dict] = None) -> dict: diff --git a/api/core/workflow/nodes/code/code_node.py b/api/core/workflow/nodes/code/code_node.py index e15c1c6f87..fafd43e5bc 100644 --- a/api/core/workflow/nodes/code/code_node.py +++ b/api/core/workflow/nodes/code/code_node.py @@ -59,9 +59,7 @@ class CodeNode(BaseNode): variables = {} for variable_selector in node_data.variables: variable = variable_selector.variable - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector - ) + value = variable_pool.get_any(variable_selector.value_selector) variables[variable] = value # Run code diff --git a/api/core/workflow/nodes/end/end_node.py b/api/core/workflow/nodes/end/end_node.py index 08d55d5576..440dfa2f27 100644 --- a/api/core/workflow/nodes/end/end_node.py +++ b/api/core/workflow/nodes/end/end_node.py @@ -10,7 +10,7 @@ from models.workflow import WorkflowNodeExecutionStatus class EndNode(BaseNode): _node_data_cls = EndNodeData - node_type = NodeType.END + _node_type = NodeType.END def _run(self, variable_pool: VariablePool) -> NodeRunResult: """ @@ -19,15 +19,12 @@ class EndNode(BaseNode): :return: """ node_data = self.node_data - node_data = cast(self._node_data_cls, node_data) + node_data = cast(EndNodeData, node_data) output_variables = node_data.outputs outputs = {} for variable_selector in output_variables: - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector - ) - + value = variable_pool.get_any(variable_selector.value_selector) outputs[variable_selector.variable] = value return NodeRunResult( @@ -45,7 +42,7 @@ class EndNode(BaseNode): :return: """ node_data = cls._node_data_cls(**config.get("data", {})) - node_data = cast(cls._node_data_cls, node_data) + node_data = cast(EndNodeData, node_data) return cls.extract_generate_nodes_from_node_data(graph, node_data) @@ -57,7 +54,7 @@ class EndNode(BaseNode): :param node_data: node data object :return: """ - nodes = graph.get('nodes') + nodes = graph.get('nodes', []) node_mapping = {node.get('id'): node for node in nodes} variable_selectors = node_data.outputs diff --git a/api/core/workflow/nodes/http_request/entities.py b/api/core/workflow/nodes/http_request/entities.py index 65451452c8..90d644e0e2 100644 --- a/api/core/workflow/nodes/http_request/entities.py +++ b/api/core/workflow/nodes/http_request/entities.py @@ -58,4 +58,3 @@ class HttpRequestNodeData(BaseNodeData): params: str body: Optional[HttpRequestNodeBody] = None timeout: Optional[HttpRequestNodeTimeout] = None - mask_authorization_header: Optional[bool] = True diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index 3736c67fb7..3c24c0a018 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -9,7 +9,7 @@ import httpx import core.helper.ssrf_proxy as ssrf_proxy from configs import dify_config from core.workflow.entities.variable_entities import VariableSelector -from core.workflow.entities.variable_pool import ValueType, VariablePool +from core.workflow.entities.variable_pool import VariablePool from core.workflow.nodes.http_request.entities import ( HttpRequestNodeAuthorization, HttpRequestNodeBody, @@ -212,13 +212,11 @@ class HttpExecutor: raise ValueError('self.authorization config is required') if authorization.config is None: raise ValueError('authorization config is required') - if authorization.config.type != 'bearer' and authorization.config.header is None: - raise ValueError('authorization config header is required') if self.authorization.config.api_key is None: raise ValueError('api_key is required') - if not self.authorization.config.header: + if not authorization.config.header: authorization.config.header = 'Authorization' if self.authorization.config.type == 'bearer': @@ -283,7 +281,7 @@ class HttpExecutor: # validate response return self._validate_and_parse_response(response) - def to_raw_request(self, mask_authorization_header: Optional[bool] = True) -> str: + def to_raw_request(self) -> str: """ convert to raw request """ @@ -295,16 +293,15 @@ class HttpExecutor: headers = self._assembling_headers() for k, v in headers.items(): - if mask_authorization_header: - # get authorization header - if self.authorization.type == 'api-key': - authorization_header = 'Authorization' - if self.authorization.config and self.authorization.config.header: - authorization_header = self.authorization.config.header + # get authorization header + if self.authorization.type == 'api-key': + authorization_header = 'Authorization' + if self.authorization.config and self.authorization.config.header: + authorization_header = self.authorization.config.header - if k.lower() == authorization_header.lower(): - raw_request += f'{k}: {"*" * len(v)}\n' - continue + if k.lower() == authorization_header.lower(): + raw_request += f'{k}: {"*" * len(v)}\n' + continue raw_request += f'{k}: {v}\n' @@ -336,16 +333,13 @@ class HttpExecutor: if variable_pool: variable_value_mapping = {} for variable_selector in variable_selectors: - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector, target_value_type=ValueType.STRING - ) - - if value is None: + variable = variable_pool.get_any(variable_selector.value_selector) + if variable is None: raise ValueError(f'Variable {variable_selector.variable} not found') - - if escape_quotes and isinstance(value, str): - value = value.replace('"', '\\"') - + if escape_quotes and isinstance(variable, str): + value = variable.replace('"', '\\"') + else: + value = variable variable_value_mapping[variable_selector.variable] = value return variable_template_parser.format(variable_value_mapping), variable_selectors diff --git a/api/core/workflow/nodes/http_request/http_request_node.py b/api/core/workflow/nodes/http_request/http_request_node.py index 24acf984f2..bbe5f9ad43 100644 --- a/api/core/workflow/nodes/http_request/http_request_node.py +++ b/api/core/workflow/nodes/http_request/http_request_node.py @@ -3,6 +3,7 @@ from mimetypes import guess_extension from os import path from typing import cast +from core.app.segments import parser from core.file.file_obj import FileTransferMethod, FileType, FileVar from core.tools.tool_file_manager import ToolFileManager from core.workflow.entities.base_node_data_entities import BaseNodeData @@ -51,6 +52,9 @@ class HttpRequestNode(BaseNode): def _run(self, variable_pool: VariablePool) -> NodeRunResult: node_data: HttpRequestNodeData = cast(HttpRequestNodeData, self.node_data) + # TODO: Switch to use segment directly + if node_data.authorization.config and node_data.authorization.config.api_key: + node_data.authorization.config.api_key = parser.convert_template(template=node_data.authorization.config.api_key, variable_pool=variable_pool).text # init http executor http_executor = None @@ -65,9 +69,7 @@ class HttpRequestNode(BaseNode): process_data = {} if http_executor: process_data = { - 'request': http_executor.to_raw_request( - mask_authorization_header=node_data.mask_authorization_header - ), + 'request': http_executor.to_raw_request(), } return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, @@ -86,9 +88,7 @@ class HttpRequestNode(BaseNode): 'files': files, }, process_data={ - 'request': http_executor.to_raw_request( - mask_authorization_header=node_data.mask_authorization_header, - ), + 'request': http_executor.to_raw_request(), }, ) diff --git a/api/core/workflow/nodes/if_else/if_else_node.py b/api/core/workflow/nodes/if_else/if_else_node.py index 6176a75201..c6d235627f 100644 --- a/api/core/workflow/nodes/if_else/if_else_node.py +++ b/api/core/workflow/nodes/if_else/if_else_node.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from typing import Optional, cast from core.workflow.entities.base_node_data_entities import BaseNodeData @@ -11,7 +12,7 @@ from models.workflow import WorkflowNodeExecutionStatus class IfElseNode(BaseNode): _node_data_cls = IfElseNodeData - node_type = NodeType.IF_ELSE + _node_type = NodeType.IF_ELSE def _run(self, variable_pool: VariablePool) -> NodeRunResult: """ @@ -20,7 +21,7 @@ class IfElseNode(BaseNode): :return: """ node_data = self.node_data - node_data = cast(self._node_data_cls, node_data) + node_data = cast(IfElseNodeData, node_data) node_inputs = { "conditions": [] @@ -138,14 +139,12 @@ class IfElseNode(BaseNode): else: raise ValueError(f"Invalid comparison operator: {comparison_operator}") - def process_conditions(self, variable_pool: VariablePool, conditions: list[Condition]): + def process_conditions(self, variable_pool: VariablePool, conditions: Sequence[Condition]): input_conditions = [] group_result = [] for condition in conditions: - actual_value = variable_pool.get_variable_value( - variable_selector=condition.variable_selector - ) + actual_variable = variable_pool.get_any(condition.variable_selector) if condition.value is not None: variable_template_parser = VariableTemplateParser(template=condition.value) @@ -153,9 +152,7 @@ class IfElseNode(BaseNode): variable_selectors = variable_template_parser.extract_variable_selectors() if variable_selectors: for variable_selector in variable_selectors: - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector - ) + value = variable_pool.get_any(variable_selector.value_selector) expected_value = variable_template_parser.format({variable_selector.variable: value}) else: expected_value = condition.value @@ -165,13 +162,13 @@ class IfElseNode(BaseNode): comparison_operator = condition.comparison_operator input_conditions.append( { - "actual_value": actual_value, + "actual_value": actual_variable, "expected_value": expected_value, "comparison_operator": comparison_operator } ) - result = self.evaluate_condition(actual_value, expected_value, comparison_operator) + result = self.evaluate_condition(actual_variable, expected_value, comparison_operator) group_result.append(result) return input_conditions, group_result diff --git a/api/core/workflow/nodes/iteration/iteration_node.py b/api/core/workflow/nodes/iteration/iteration_node.py index 12d792f297..54dfe8b7f4 100644 --- a/api/core/workflow/nodes/iteration/iteration_node.py +++ b/api/core/workflow/nodes/iteration/iteration_node.py @@ -20,7 +20,8 @@ class IterationNode(BaseIterationNode): """ Run the node. """ - iterator = variable_pool.get_variable_value(cast(IterationNodeData, self.node_data).iterator_selector) + self.node_data = cast(IterationNodeData, self.node_data) + iterator = variable_pool.get_any(self.node_data.iterator_selector) if not isinstance(iterator, list): raise ValueError(f"Invalid iterator value: {iterator}, please provide a list.") @@ -63,15 +64,15 @@ class IterationNode(BaseIterationNode): """ node_data = cast(IterationNodeData, self.node_data) - variable_pool.append_variable(self.node_id, ['index'], state.index) + variable_pool.add((self.node_id, 'index'), state.index) # get the iterator value - iterator = variable_pool.get_variable_value(node_data.iterator_selector) + iterator = variable_pool.get_any(node_data.iterator_selector) if iterator is None or not isinstance(iterator, list): return if state.index < len(iterator): - variable_pool.append_variable(self.node_id, ['item'], iterator[state.index]) + variable_pool.add((self.node_id, 'item'), iterator[state.index]) def _next_iteration(self, variable_pool: VariablePool, state: IterationState): """ @@ -87,7 +88,7 @@ class IterationNode(BaseIterationNode): :return: True if iteration limit is reached, False otherwise """ node_data = cast(IterationNodeData, self.node_data) - iterator = variable_pool.get_variable_value(node_data.iterator_selector) + iterator = variable_pool.get_any(node_data.iterator_selector) if iterator is None or not isinstance(iterator, list): return True @@ -100,12 +101,16 @@ class IterationNode(BaseIterationNode): :param variable_pool: variable pool """ output_selector = cast(IterationNodeData, self.node_data).output_selector - output = variable_pool.get_variable_value(output_selector) + output = variable_pool.get_any(output_selector) # clear the output for this iteration - variable_pool.append_variable(self.node_id, output_selector[1:], None) + variable_pool.remove([self.node_id] + output_selector[1:]) state.current_output = output if output is not None: - state.outputs.append(output) + # NOTE: This is a temporary patch to process double nested list (for example, DALL-E output in iteration). + if isinstance(output, list): + state.outputs.extend(output) + else: + state.outputs.append(output) @classmethod def _extract_variable_selector_to_variable_mapping(cls, node_data: IterationNodeData) -> dict[str, list[str]]: diff --git a/api/core/workflow/nodes/knowledge_retrieval/entities.py b/api/core/workflow/nodes/knowledge_retrieval/entities.py index c8874ff22c..5758b895f3 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/entities.py +++ b/api/core/workflow/nodes/knowledge_retrieval/entities.py @@ -13,13 +13,41 @@ class RerankingModelConfig(BaseModel): model: str +class VectorSetting(BaseModel): + """ + Vector Setting. + """ + vector_weight: float + embedding_provider_name: str + embedding_model_name: str + + +class KeywordSetting(BaseModel): + """ + Keyword Setting. + """ + keyword_weight: float + + +class WeightedScoreConfig(BaseModel): + """ + Weighted score Config. + """ + weight_type: str + vector_setting: VectorSetting + keyword_setting: KeywordSetting + + class MultipleRetrievalConfig(BaseModel): """ Multiple Retrieval Config. """ top_k: int score_threshold: Optional[float] = None - reranking_model: RerankingModelConfig + reranking_mode: str = 'reranking_model' + reranking_enable: bool = True + reranking_model: Optional[RerankingModelConfig] = None + weights: Optional[WeightedScoreConfig] = None class ModelConfig(BaseModel): diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index 9e29bd9ea1..7a2b3c8512 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -22,7 +22,7 @@ from models.dataset import Dataset, Document, DocumentSegment from models.workflow import WorkflowNodeExecutionStatus default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -41,7 +41,8 @@ class KnowledgeRetrievalNode(BaseNode): node_data: KnowledgeRetrievalNodeData = cast(self._node_data_cls, self.node_data) # extract variables - query = variable_pool.get_variable_value(variable_selector=node_data.query_variable_selector) + variable = variable_pool.get_any(node_data.query_variable_selector) + query = variable variables = { 'query': query } @@ -137,13 +138,38 @@ class KnowledgeRetrievalNode(BaseNode): planning_strategy=planning_strategy ) elif node_data.retrieval_mode == DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE.value: + if node_data.multiple_retrieval_config.reranking_mode == 'reranking_model': + reranking_model = { + 'reranking_provider_name': node_data.multiple_retrieval_config.reranking_model.provider, + 'reranking_model_name': node_data.multiple_retrieval_config.reranking_model.model + } + weights = None + elif node_data.multiple_retrieval_config.reranking_mode == 'weighted_score': + reranking_model = None + weights = { + 'weight_type': node_data.multiple_retrieval_config.weights.weight_type, + 'vector_setting': { + "vector_weight": node_data.multiple_retrieval_config.weights.vector_setting.vector_weight, + "embedding_provider_name": node_data.multiple_retrieval_config.weights.vector_setting.embedding_provider_name, + "embedding_model_name": node_data.multiple_retrieval_config.weights.vector_setting.embedding_model_name, + }, + 'keyword_setting': { + "keyword_weight": node_data.multiple_retrieval_config.weights.keyword_setting.keyword_weight + } + } + else: + reranking_model = None + weights = None all_documents = dataset_retrieval.multiple_retrieve(self.app_id, self.tenant_id, self.user_id, self.user_from.value, available_datasets, query, node_data.multiple_retrieval_config.top_k, node_data.multiple_retrieval_config.score_threshold, - node_data.multiple_retrieval_config.reranking_model.provider, - node_data.multiple_retrieval_config.reranking_model.model) + node_data.multiple_retrieval_config.reranking_mode, + reranking_model, + weights, + node_data.multiple_retrieval_config.reranking_enable, + ) context_list = [] if all_documents: diff --git a/api/core/workflow/nodes/llm/llm_node.py b/api/core/workflow/nodes/llm/llm_node.py index af928517d9..4431259a57 100644 --- a/api/core/workflow/nodes/llm/llm_node.py +++ b/api/core/workflow/nodes/llm/llm_node.py @@ -41,7 +41,7 @@ from models.workflow import WorkflowNodeExecutionStatus class LLMNode(BaseNode): _node_data_cls = LLMNodeData - node_type = NodeType.LLM + _node_type = NodeType.LLM def _run(self, variable_pool: VariablePool) -> NodeRunResult: """ @@ -90,7 +90,7 @@ class LLMNode(BaseNode): # fetch prompt messages prompt_messages, stop = self._fetch_prompt_messages( node_data=node_data, - query=variable_pool.get_variable_value(['sys', SystemVariable.QUERY.value]) + query=variable_pool.get_any(['sys', SystemVariable.QUERY.value]) if node_data.memory else None, query_prompt_template=node_data.memory.query_prompt_template if node_data.memory else None, inputs=inputs, @@ -238,8 +238,8 @@ class LLMNode(BaseNode): for variable_selector in node_data.prompt_config.jinja2_variables or []: variable = variable_selector.variable - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector + value = variable_pool.get_any( + variable_selector.value_selector ) def parse_dict(d: dict) -> str: @@ -302,7 +302,7 @@ class LLMNode(BaseNode): variable_selectors = variable_template_parser.extract_variable_selectors() for variable_selector in variable_selectors: - variable_value = variable_pool.get_variable_value(variable_selector.value_selector) + variable_value = variable_pool.get_any(variable_selector.value_selector) if variable_value is None: raise ValueError(f'Variable {variable_selector.variable} not found') @@ -313,7 +313,7 @@ class LLMNode(BaseNode): query_variable_selectors = (VariableTemplateParser(template=memory.query_prompt_template) .extract_variable_selectors()) for variable_selector in query_variable_selectors: - variable_value = variable_pool.get_variable_value(variable_selector.value_selector) + variable_value = variable_pool.get_any(variable_selector.value_selector) if variable_value is None: raise ValueError(f'Variable {variable_selector.variable} not found') @@ -331,7 +331,7 @@ class LLMNode(BaseNode): if not node_data.vision.enabled: return [] - files = variable_pool.get_variable_value(['sys', SystemVariable.FILES.value]) + files = variable_pool.get_any(['sys', SystemVariable.FILES.value]) if not files: return [] @@ -350,7 +350,7 @@ class LLMNode(BaseNode): if not node_data.context.variable_selector: return None - context_value = variable_pool.get_variable_value(node_data.context.variable_selector) + context_value = variable_pool.get_any(node_data.context.variable_selector) if context_value: if isinstance(context_value, str): return context_value @@ -496,7 +496,7 @@ class LLMNode(BaseNode): return None # get conversation id - conversation_id = variable_pool.get_variable_value(['sys', SystemVariable.CONVERSATION_ID.value]) + conversation_id = variable_pool.get_any(['sys', SystemVariable.CONVERSATION_ID.value]) if conversation_id is None: return None diff --git a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py index d219156026..2876695a82 100644 --- a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py +++ b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py @@ -71,9 +71,10 @@ class ParameterExtractorNode(LLMNode): Run the node. """ node_data = cast(ParameterExtractorNodeData, self.node_data) - query = variable_pool.get_variable_value(node_data.query) - if not query: + variable = variable_pool.get_any(node_data.query) + if not variable: raise ValueError("Input variable content not found or is empty") + query = variable inputs = { 'query': query, @@ -564,7 +565,8 @@ class ParameterExtractorNode(LLMNode): variable_template_parser = VariableTemplateParser(instruction) inputs = {} for selector in variable_template_parser.extract_variable_selectors(): - inputs[selector.variable] = variable_pool.get_variable_value(selector.value_selector) + variable = variable_pool.get_any(selector.value_selector) + inputs[selector.variable] = variable return variable_template_parser.format(inputs) diff --git a/api/core/workflow/nodes/question_classifier/question_classifier_node.py b/api/core/workflow/nodes/question_classifier/question_classifier_node.py index 76f3dec836..2e1464efce 100644 --- a/api/core/workflow/nodes/question_classifier/question_classifier_node.py +++ b/api/core/workflow/nodes/question_classifier/question_classifier_node.py @@ -41,7 +41,8 @@ class QuestionClassifierNode(LLMNode): node_data = cast(QuestionClassifierNodeData, node_data) # extract variables - query = variable_pool.get_variable_value(variable_selector=node_data.query_variable_selector) + variable = variable_pool.get(node_data.query_variable_selector) + query = variable.value if variable else None variables = { 'query': query } @@ -294,7 +295,8 @@ class QuestionClassifierNode(LLMNode): variable_template_parser = VariableTemplateParser(template=instruction) variable_selectors.extend(variable_template_parser.extract_variable_selectors()) for variable_selector in variable_selectors: - variable_value = variable_pool.get_variable_value(variable_selector.value_selector) + variable = variable_pool.get(variable_selector.value_selector) + variable_value = variable.value if variable else None if variable_value is None: raise ValueError(f'Variable {variable_selector.variable} not found') diff --git a/api/core/workflow/nodes/start/start_node.py b/api/core/workflow/nodes/start/start_node.py index fd51a6c476..661b403d32 100644 --- a/api/core/workflow/nodes/start/start_node.py +++ b/api/core/workflow/nodes/start/start_node.py @@ -9,7 +9,7 @@ from models.workflow import WorkflowNodeExecutionStatus class StartNode(BaseNode): _node_data_cls = StartNodeData - node_type = NodeType.START + _node_type = NodeType.START def _run(self, variable_pool: VariablePool) -> NodeRunResult: """ @@ -18,7 +18,7 @@ class StartNode(BaseNode): :return: """ # Get cleaned inputs - cleaned_inputs = variable_pool.user_inputs + cleaned_inputs = dict(variable_pool.user_inputs) for var in variable_pool.system_variables: cleaned_inputs['sys.' + var.value] = variable_pool.system_variables[var] diff --git a/api/core/workflow/nodes/template_transform/template_transform_node.py b/api/core/workflow/nodes/template_transform/template_transform_node.py index 2c4a2257f5..21f71db6c5 100644 --- a/api/core/workflow/nodes/template_transform/template_transform_node.py +++ b/api/core/workflow/nodes/template_transform/template_transform_node.py @@ -44,12 +44,9 @@ class TemplateTransformNode(BaseNode): # Get variables variables = {} for variable_selector in node_data.variables: - variable = variable_selector.variable - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector - ) - - variables[variable] = value + variable_name = variable_selector.variable + value = variable_pool.get_any(variable_selector.value_selector) + variables[variable_name] = value # Run code try: result = CodeExecutor.execute_workflow_code_template( diff --git a/api/core/workflow/nodes/tool/entities.py b/api/core/workflow/nodes/tool/entities.py index bc9bfde4db..335a41061c 100644 --- a/api/core/workflow/nodes/tool/entities.py +++ b/api/core/workflow/nodes/tool/entities.py @@ -30,6 +30,7 @@ class ToolEntity(BaseModel): class ToolNodeData(BaseNodeData, ToolEntity): class ToolInput(BaseModel): + # TODO: check this type value: Union[Any, list[str]] type: Literal['mixed', 'variable', 'constant'] diff --git a/api/core/workflow/nodes/tool/tool_node.py b/api/core/workflow/nodes/tool/tool_node.py index f77ccd9bd6..15c05e785c 100644 --- a/api/core/workflow/nodes/tool/tool_node.py +++ b/api/core/workflow/nodes/tool/tool_node.py @@ -1,10 +1,11 @@ +from collections.abc import Mapping, Sequence from os import path -from typing import Optional, cast +from typing import Any, cast +from core.app.segments import parser from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler from core.file.file_obj import FileTransferMethod, FileType, FileVar from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter -from core.tools.tool.tool import Tool from core.tools.tool_engine import ToolEngine from core.tools.tool_manager import ToolManager from core.tools.utils.message_transformer import ToolFileMessageTransformer @@ -20,6 +21,7 @@ class ToolNode(BaseNode): """ Tool Node """ + _node_data_cls = ToolNodeData _node_type = NodeType.TOOL @@ -50,23 +52,24 @@ class ToolNode(BaseNode): }, error=f'Failed to get tool runtime: {str(e)}' ) - + # get parameters - parameters = self._generate_parameters(variable_pool, node_data, tool_runtime) + tool_parameters = tool_runtime.get_runtime_parameters() or [] + parameters = self._generate_parameters(tool_parameters=tool_parameters, variable_pool=variable_pool, node_data=node_data) + parameters_for_log = self._generate_parameters(tool_parameters=tool_parameters, variable_pool=variable_pool, node_data=node_data, for_log=True) try: messages = ToolEngine.workflow_invoke( tool=tool_runtime, tool_parameters=parameters, user_id=self.user_id, - workflow_id=self.workflow_id, workflow_tool_callback=DifyWorkflowCallbackHandler(), workflow_call_depth=self.workflow_call_depth, ) except Exception as e: return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, - inputs=parameters, + inputs=parameters_for_log, metadata={ NodeRunMetadataKey.TOOL_INFO: tool_info }, @@ -86,21 +89,34 @@ class ToolNode(BaseNode): metadata={ NodeRunMetadataKey.TOOL_INFO: tool_info }, - inputs=parameters + inputs=parameters_for_log ) - def _generate_parameters(self, variable_pool: VariablePool, node_data: ToolNodeData, tool_runtime: Tool) -> dict: + def _generate_parameters( + self, + *, + tool_parameters: Sequence[ToolParameter], + variable_pool: VariablePool, + node_data: ToolNodeData, + for_log: bool = False, + ) -> Mapping[str, Any]: """ - Generate parameters - """ - tool_parameters = tool_runtime.get_all_runtime_parameters() + Generate parameters based on the given tool parameters, variable pool, and node data. - def fetch_parameter(name: str) -> Optional[ToolParameter]: - return next((parameter for parameter in tool_parameters if parameter.name == name), None) + Args: + tool_parameters (Sequence[ToolParameter]): The list of tool parameters. + variable_pool (VariablePool): The variable pool containing the variables. + node_data (ToolNodeData): The data associated with the tool node. + + Returns: + Mapping[str, Any]: A dictionary containing the generated parameters. + + """ + tool_parameters_dictionary = {parameter.name: parameter for parameter in tool_parameters} result = {} for parameter_name in node_data.tool_parameters: - parameter = fetch_parameter(parameter_name) + parameter = tool_parameters_dictionary.get(parameter_name) if not parameter: continue if parameter.type == ToolParameter.ToolParameterType.FILE: @@ -108,35 +124,26 @@ class ToolNode(BaseNode): v.to_dict() for v in self._fetch_files(variable_pool) ] else: - input = node_data.tool_parameters[parameter_name] - if input.type == 'mixed': - result[parameter_name] = self._format_variable_template(input.value, variable_pool) - elif input.type == 'variable': - result[parameter_name] = variable_pool.get_variable_value(input.value) - elif input.type == 'constant': - result[parameter_name] = input.value + tool_input = node_data.tool_parameters[parameter_name] + if tool_input.type == 'variable': + # TODO: check if the variable exists in the variable pool + parameter_value = variable_pool.get(tool_input.value).value + else: + segment_group = parser.convert_template( + template=str(tool_input.value), + variable_pool=variable_pool, + ) + parameter_value = segment_group.log if for_log else segment_group.text + result[parameter_name] = parameter_value return result - - def _format_variable_template(self, template: str, variable_pool: VariablePool) -> str: - """ - Format variable template - """ - inputs = {} - template_parser = VariableTemplateParser(template) - for selector in template_parser.extract_variable_selectors(): - inputs[selector.variable] = variable_pool.get_variable_value(selector.value_selector) - - return template_parser.format(inputs) - - def _fetch_files(self, variable_pool: VariablePool) -> list[FileVar]: - files = variable_pool.get_variable_value(['sys', SystemVariable.FILES.value]) - if not files: - return [] - - return files - def _convert_tool_messages(self, messages: list[ToolInvokeMessage]) -> tuple[str, list[FileVar]]: + def _fetch_files(self, variable_pool: VariablePool) -> list[FileVar]: + # FIXME: ensure this is a ArrayVariable contains FileVariable. + variable = variable_pool.get(['sys', SystemVariable.FILES.value]) + return [file_var.value for file_var in variable.value] if variable else [] + + def _convert_tool_messages(self, messages: list[ToolInvokeMessage]): """ Convert ToolInvokeMessages into tuple[plain_text, files] """ diff --git a/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py b/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py index 63ce790625..885f7d7617 100644 --- a/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py +++ b/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py @@ -19,28 +19,27 @@ class VariableAggregatorNode(BaseNode): inputs = {} if not node_data.advanced_settings or not node_data.advanced_settings.group_enabled: - for variable in node_data.variables: - value = variable_pool.get_variable_value(variable) - - if value is not None: + for selector in node_data.variables: + variable = variable_pool.get_any(selector) + if variable is not None: outputs = { - "output": value + "output": variable } inputs = { - '.'.join(variable[1:]): value + '.'.join(selector[1:]): variable } break else: for group in node_data.advanced_settings.groups: - for variable in group.variables: - value = variable_pool.get_variable_value(variable) + for selector in group.variables: + variable = variable_pool.get_any(selector) - if value is not None: + if variable is not None: outputs[group.group_name] = { - 'output': value + 'output': variable } - inputs['.'.join(variable[1:])] = value + inputs['.'.join(selector[1:])] = variable break return NodeRunResult( diff --git a/api/core/workflow/utils/variable_template_parser.py b/api/core/workflow/utils/variable_template_parser.py index 925c31a6aa..c43fde172c 100644 --- a/api/core/workflow/utils/variable_template_parser.py +++ b/api/core/workflow/utils/variable_template_parser.py @@ -1,12 +1,48 @@ import re +from collections.abc import Mapping +from typing import Any from core.workflow.entities.variable_entities import VariableSelector +from core.workflow.entities.variable_pool import VariablePool -REGEX = re.compile(r"\{\{(#[a-zA-Z0-9_]{1,50}(\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10}#)\}\}") +REGEX = re.compile(r'\{\{(#[a-zA-Z0-9_]{1,50}(\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10}#)\}\}') + + +def parse_mixed_template(*, template: str, variable_pool: VariablePool) -> str: + """ + This is an alternative to the VariableTemplateParser class, + offering the same functionality but with better readability and ease of use. + """ + variable_keys = [match[0] for match in re.findall(REGEX, template)] + variable_keys = list(set(variable_keys)) + + # This key_selector is a tuple of (key, selector) where selector is a list of keys + # e.g. ('#node_id.query.name#', ['node_id', 'query', 'name']) + key_selectors = filter( + lambda t: len(t[1]) >= 2, + ((key, selector.replace('#', '').split('.')) for key, selector in zip(variable_keys, variable_keys)), + ) + inputs = {key: variable_pool.get_any(selector) for key, selector in key_selectors} + + def replacer(match): + key = match.group(1) + # return original matched string if key not found + value = inputs.get(key, match.group(0)) + if value is None: + value = '' + value = str(value) + # remove template variables if required + return re.sub(REGEX, r'{\1}', value) + + result = re.sub(REGEX, replacer, template) + result = re.sub(r'<\|.*?\|>', '', result) + return result class VariableTemplateParser: """ + !NOTE: Consider to use the new `segments` module instead of this class. + A class for parsing and manipulating template variables in a string. Rules: @@ -70,14 +106,11 @@ class VariableTemplateParser: if len(split_result) < 2: continue - variable_selectors.append(VariableSelector( - variable=variable_key, - value_selector=split_result - )) + variable_selectors.append(VariableSelector(variable=variable_key, value_selector=split_result)) return variable_selectors - def format(self, inputs: dict, remove_template_variables: bool = True) -> str: + def format(self, inputs: Mapping[str, Any]) -> str: """ Formats the template string by replacing the template variables with their corresponding values. @@ -88,17 +121,19 @@ class VariableTemplateParser: Returns: The formatted string with template variables replaced by their values. """ + def replacer(match): key = match.group(1) value = inputs.get(key, match.group(0)) # return original matched string if key not found + + if value is None: + value = '' # convert the value to string if isinstance(value, list | dict | bool | int | float): value = str(value) - + # remove template variables if required - if remove_template_variables: - return VariableTemplateParser.remove_template_variables(value) - return value + return VariableTemplateParser.remove_template_variables(value) prompt = re.sub(REGEX, replacer, self.template) return re.sub(r'<\|.*?\|>', '', prompt) diff --git a/api/core/workflow/workflow_engine_manager.py b/api/core/workflow/workflow_engine_manager.py index 22deafb8a3..32f0dbba06 100644 --- a/api/core/workflow/workflow_engine_manager.py +++ b/api/core/workflow/workflow_engine_manager.py @@ -1,15 +1,15 @@ import logging import time -from typing import Optional, cast - -from flask import current_app +from collections.abc import Mapping, Sequence +from typing import Any, Optional, cast +from configs import dify_config from core.app.app_config.entities import FileExtraConfig from core.app.apps.base_app_queue_manager import GenerateTaskStoppedException from core.app.entities.app_invoke_entities import InvokeFrom from core.file.file_obj import FileTransferMethod, FileType, FileVar -from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback -from core.workflow.entities.node_entities import NodeRunMetadataKey, NodeRunResult, NodeType +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback +from core.workflow.entities.node_entities import NodeRunMetadataKey, NodeRunResult, NodeType, SystemVariable from core.workflow.entities.variable_pool import VariablePool, VariableValue from core.workflow.entities.workflow_entities import WorkflowNodeAndResult, WorkflowRunState from core.workflow.errors import WorkflowNodeRunFailedError @@ -36,7 +36,7 @@ from models.workflow import ( WorkflowNodeExecutionStatus, ) -node_classes = { +node_classes: Mapping[NodeType, type[BaseNode]] = { NodeType.START: StartNode, NodeType.END: EndNode, NodeType.ANSWER: AnswerNode, @@ -87,15 +87,18 @@ class WorkflowEngineManager: return default_config - def run_workflow(self, workflow: Workflow, - user_id: str, - user_from: UserFrom, - invoke_from: InvokeFrom, - user_inputs: dict, - system_inputs: Optional[dict] = None, - callbacks: list[BaseWorkflowCallback] = None, - call_depth: Optional[int] = 0, - variable_pool: Optional[VariablePool] = None) -> None: + def run_workflow( + self, + *, + workflow: Workflow, + user_id: str, + user_from: UserFrom, + invoke_from: InvokeFrom, + user_inputs: Mapping[str, Any], + system_inputs: Mapping[SystemVariable, Any], + callbacks: Sequence[WorkflowCallback], + call_depth: int = 0 + ) -> None: """ :param workflow: Workflow instance :param user_id: user id @@ -118,15 +121,15 @@ class WorkflowEngineManager: if not isinstance(graph.get('edges'), list): raise ValueError('edges in workflow graph must be a list') - - # init variable pool - if not variable_pool: - variable_pool = VariablePool( - system_variables=system_inputs, - user_inputs=user_inputs - ) - workflow_call_max_depth = current_app.config.get("WORKFLOW_CALL_MAX_DEPTH") + # init variable pool + variable_pool = VariablePool( + system_variables=system_inputs, + user_inputs=user_inputs, + environment_variables=workflow.environment_variables, + ) + + workflow_call_max_depth = dify_config.WORKFLOW_CALL_MAX_DEPTH if call_depth > workflow_call_max_depth: raise ValueError('Max workflow call depth {} reached.'.format(workflow_call_max_depth)) @@ -155,7 +158,7 @@ class WorkflowEngineManager: def _run_workflow(self, workflow: Workflow, workflow_run_state: WorkflowRunState, - callbacks: list[BaseWorkflowCallback] = None, + callbacks: Sequence[WorkflowCallback], start_at: Optional[str] = None, end_at: Optional[str] = None) -> None: """ @@ -174,11 +177,11 @@ class WorkflowEngineManager: graph = workflow.graph_dict try: - predecessor_node: BaseNode = None - current_iteration_node: BaseIterationNode = None + predecessor_node: BaseNode | None = None + current_iteration_node: BaseIterationNode | None = None has_entry_node = False - max_execution_steps = current_app.config.get("WORKFLOW_MAX_EXECUTION_STEPS") - max_execution_time = current_app.config.get("WORKFLOW_MAX_EXECUTION_TIME") + max_execution_steps = dify_config.WORKFLOW_MAX_EXECUTION_STEPS + max_execution_time = dify_config.WORKFLOW_MAX_EXECUTION_TIME while True: # get next node, multiple target nodes in the future next_node = self._get_next_overall_node( @@ -236,8 +239,8 @@ class WorkflowEngineManager: # move to next iteration next_node_id = next_iteration # get next id - next_node = self._get_node(workflow_run_state, graph, next_node_id, callbacks) - + next_node = self._get_node(workflow_run_state=workflow_run_state, graph=graph, node_id=next_node_id, callbacks=callbacks) + if not next_node: break @@ -296,7 +299,7 @@ class WorkflowEngineManager: workflow_run_state.current_iteration_state = None continue else: - next_node = self._get_node(workflow_run_state, graph, next_node_id, callbacks) + next_node = self._get_node(workflow_run_state=workflow_run_state, graph=graph, node_id=next_node_id, callbacks=callbacks) # run workflow, run multiple target nodes in the future self._run_workflow_node( @@ -382,14 +385,14 @@ class WorkflowEngineManager: # init variable pool variable_pool = VariablePool( system_variables={}, - user_inputs={} + user_inputs={}, + environment_variables=workflow.environment_variables, ) + if node_cls is None: + raise ValueError('Node class not found') # variable selector to variable mapping - try: - variable_mapping = node_cls.extract_variable_selector_to_variable_mapping(node_config) - except NotImplementedError: - variable_mapping = {} + variable_mapping = node_cls.extract_variable_selector_to_variable_mapping(node_config) self._mapping_user_inputs_to_variable_pool( variable_mapping=variable_mapping, @@ -398,7 +401,7 @@ class WorkflowEngineManager: tenant_id=workflow.tenant_id, node_instance=node_instance ) - + # run node node_run_result = node_instance.run( variable_pool=variable_pool @@ -420,7 +423,7 @@ class WorkflowEngineManager: node_id: str, user_id: str, user_inputs: dict, - callbacks: list[BaseWorkflowCallback] = None, + callbacks: Sequence[WorkflowCallback], ) -> None: """ Single iteration run workflow node @@ -443,16 +446,17 @@ class WorkflowEngineManager: node_config = node else: raise ValueError('node id is not an iteration node') - + # init variable pool variable_pool = VariablePool( system_variables={}, - user_inputs={} + user_inputs={}, + environment_variables=workflow.environment_variables, ) # variable selector to variable mapping iteration_nested_nodes = [ - node for node in nodes + node for node in nodes if node.get('data', {}).get('iteration_id') == node_id or node.get('id') == node_id ] iteration_nested_node_ids = [node.get('id') for node in iteration_nested_nodes] @@ -468,20 +472,19 @@ class WorkflowEngineManager: for node_config in iteration_nested_nodes: # mapping user inputs to variable pool node_cls = node_classes.get(NodeType.value_of(node_config.get('data', {}).get('type'))) - try: - variable_mapping = node_cls.extract_variable_selector_to_variable_mapping(node_config) - except NotImplementedError: - variable_mapping = {} + if node_cls is None: + raise ValueError('Node class not found') + variable_mapping = node_cls.extract_variable_selector_to_variable_mapping(node_config) # remove iteration variables variable_mapping = { - f'{node_config.get("id")}.{key}': value for key, value in variable_mapping.items() + f'{node_config.get("id")}.{key}': value for key, value in variable_mapping.items() if value[0] != node_id } # remove variable out from iteration variable_mapping = { - key: value for key, value in variable_mapping.items() + key: value for key, value in variable_mapping.items() if value[0] not in iteration_nested_node_ids } @@ -536,7 +539,7 @@ class WorkflowEngineManager: end_at=end_node_id ) - def _workflow_run_success(self, callbacks: list[BaseWorkflowCallback] = None) -> None: + def _workflow_run_success(self, callbacks: Sequence[WorkflowCallback]) -> None: """ Workflow run success :param callbacks: workflow callbacks @@ -548,7 +551,7 @@ class WorkflowEngineManager: callback.on_workflow_run_succeeded() def _workflow_run_failed(self, error: str, - callbacks: list[BaseWorkflowCallback] = None) -> None: + callbacks: Sequence[WorkflowCallback]) -> None: """ Workflow run failed :param error: error message @@ -561,11 +564,11 @@ class WorkflowEngineManager: error=error ) - def _workflow_iteration_started(self, graph: dict, + def _workflow_iteration_started(self, *, graph: Mapping[str, Any], current_iteration_node: BaseIterationNode, workflow_run_state: WorkflowRunState, predecessor_node_id: Optional[str] = None, - callbacks: list[BaseWorkflowCallback] = None) -> None: + callbacks: Sequence[WorkflowCallback]) -> None: """ Workflow iteration started :param current_iteration_node: current iteration node @@ -598,10 +601,10 @@ class WorkflowEngineManager: # add steps workflow_run_state.workflow_node_steps += 1 - def _workflow_iteration_next(self, graph: dict, + def _workflow_iteration_next(self, *, graph: Mapping[str, Any], current_iteration_node: BaseIterationNode, - workflow_run_state: WorkflowRunState, - callbacks: list[BaseWorkflowCallback] = None) -> None: + workflow_run_state: WorkflowRunState, + callbacks: Sequence[WorkflowCallback]) -> None: """ Workflow iteration next :param workflow_run_state: workflow run state @@ -628,11 +631,11 @@ class WorkflowEngineManager: nodes = [node for node in nodes if node.get('data', {}).get('iteration_id') == current_iteration_node.node_id] for node in nodes: - workflow_run_state.variable_pool.clear_node_variables(node_id=node.get('id')) - - def _workflow_iteration_completed(self, current_iteration_node: BaseIterationNode, - workflow_run_state: WorkflowRunState, - callbacks: list[BaseWorkflowCallback] = None) -> None: + workflow_run_state.variable_pool.remove((node.get('id'),)) + + def _workflow_iteration_completed(self, *, current_iteration_node: BaseIterationNode, + workflow_run_state: WorkflowRunState, + callbacks: Sequence[WorkflowCallback]) -> None: if callbacks: if isinstance(workflow_run_state.current_iteration_state, IterationState): for callback in callbacks: @@ -645,10 +648,10 @@ class WorkflowEngineManager: } ) - def _get_next_overall_node(self, workflow_run_state: WorkflowRunState, - graph: dict, + def _get_next_overall_node(self, *, workflow_run_state: WorkflowRunState, + graph: Mapping[str, Any], predecessor_node: Optional[BaseNode] = None, - callbacks: list[BaseWorkflowCallback] = None, + callbacks: Sequence[WorkflowCallback], start_at: Optional[str] = None, end_at: Optional[str] = None) -> Optional[BaseNode]: """ @@ -684,7 +687,7 @@ class WorkflowEngineManager: callbacks=callbacks, workflow_call_depth=workflow_run_state.workflow_call_depth ) - + else: edges = graph.get('edges') source_node_id = predecessor_node.node_id @@ -738,11 +741,11 @@ class WorkflowEngineManager: callbacks=callbacks, workflow_call_depth=workflow_run_state.workflow_call_depth ) - - def _get_node(self, workflow_run_state: WorkflowRunState, - graph: dict, + + def _get_node(self, workflow_run_state: WorkflowRunState, + graph: Mapping[str, Any], node_id: str, - callbacks: list[BaseWorkflowCallback]) -> Optional[BaseNode]: + callbacks: Sequence[WorkflowCallback]): """ Get node from graph by node id """ @@ -753,7 +756,7 @@ class WorkflowEngineManager: for node_config in nodes: if node_config.get('id') == node_id: node_type = NodeType.value_of(node_config.get('data', {}).get('type')) - node_cls = node_classes.get(node_type) + node_cls = node_classes[node_type] return node_cls( tenant_id=workflow_run_state.tenant_id, app_id=workflow_run_state.app_id, @@ -766,8 +769,6 @@ class WorkflowEngineManager: workflow_call_depth=workflow_run_state.workflow_call_depth ) - return None - def _is_timed_out(self, start_at: float, max_execution_time: int) -> bool: """ Check timeout @@ -786,10 +787,10 @@ class WorkflowEngineManager: if node_and_result.node_id == node_id ]) - def _run_workflow_node(self, workflow_run_state: WorkflowRunState, + def _run_workflow_node(self, *, workflow_run_state: WorkflowRunState, node: BaseNode, predecessor_node: Optional[BaseNode] = None, - callbacks: list[BaseWorkflowCallback] = None) -> None: + callbacks: Sequence[WorkflowCallback]) -> None: if callbacks: for callback in callbacks: callback.on_workflow_node_execute_started( @@ -895,10 +896,8 @@ class WorkflowEngineManager: :param variable_value: variable value :return: """ - variable_pool.append_variable( - node_id=node_id, - variable_key_list=variable_key_list, - value=variable_value + variable_pool.add( + [node_id] + variable_key_list, variable_value ) # if variable_value is a dict, then recursively append variables @@ -940,14 +939,14 @@ class WorkflowEngineManager: return new_value - def _mapping_user_inputs_to_variable_pool(self, - variable_mapping: dict, + def _mapping_user_inputs_to_variable_pool(self, + variable_mapping: Mapping[str, Sequence[str]], user_inputs: dict, variable_pool: VariablePool, tenant_id: str, node_instance: BaseNode): for variable_key, variable_selector in variable_mapping.items(): - if variable_key not in user_inputs: + if variable_key not in user_inputs and not variable_pool.get(variable_selector): raise ValueError(f'Variable key {variable_key} not found in user inputs.') # fetch variable node id from variable selector @@ -957,7 +956,7 @@ class WorkflowEngineManager: # get value value = user_inputs.get(variable_key) - # temp fix for image type + # FIXME: temp fix for image type if node_instance.node_type == NodeType.LLM: new_value = [] if isinstance(value, list): @@ -984,8 +983,4 @@ class WorkflowEngineManager: value = new_value # append variable and value to variable pool - variable_pool.append_variable( - node_id=variable_node_id, - variable_key_list=variable_key_list, - value=value - ) \ No newline at end of file + variable_pool.add([variable_node_id]+variable_key_list, value) diff --git a/api/docker/entrypoint.sh b/api/docker/entrypoint.sh index e74c6c2406..a53d84c6e9 100755 --- a/api/docker/entrypoint.sh +++ b/api/docker/entrypoint.sh @@ -8,15 +8,15 @@ if [[ "${MIGRATION_ENABLED}" == "true" ]]; then fi if [[ "${MODE}" == "worker" ]]; then - celery -A app.celery worker -P ${CELERY_WORKER_CLASS:-gevent} -c ${CELERY_WORKER_AMOUNT:-1} --loglevel INFO \ + exec celery -A app.celery worker -P ${CELERY_WORKER_CLASS:-gevent} -c ${CELERY_WORKER_AMOUNT:-1} --loglevel INFO \ -Q ${CELERY_QUEUES:-dataset,generation,mail,ops_trace,app_deletion} elif [[ "${MODE}" == "beat" ]]; then - celery -A app.celery beat --loglevel INFO + exec celery -A app.celery beat --loglevel INFO else if [[ "${DEBUG}" == "true" ]]; then - flask run --host=${DIFY_BIND_ADDRESS:-0.0.0.0} --port=${DIFY_PORT:-5001} --debug + exec flask run --host=${DIFY_BIND_ADDRESS:-0.0.0.0} --port=${DIFY_PORT:-5001} --debug else - gunicorn \ + exec gunicorn \ --bind "${DIFY_BIND_ADDRESS:-0.0.0.0}:${DIFY_PORT:-5001}" \ --workers ${SERVER_WORKER_AMOUNT:-1} \ --worker-class ${SERVER_WORKER_CLASS:-gevent} \ @@ -24,4 +24,4 @@ else --preload \ app:app fi -fi \ No newline at end of file +fi diff --git a/api/events/event_handlers/clean_when_document_deleted.py b/api/events/event_handlers/clean_when_document_deleted.py index d0bec667a9..24022da15f 100644 --- a/api/events/event_handlers/clean_when_document_deleted.py +++ b/api/events/event_handlers/clean_when_document_deleted.py @@ -7,4 +7,5 @@ def handle(sender, **kwargs): document_id = sender dataset_id = kwargs.get('dataset_id') doc_form = kwargs.get('doc_form') - clean_document_task.delay(document_id, dataset_id, doc_form) + file_id = kwargs.get('file_id') + clean_document_task.delay(document_id, dataset_id, doc_form, file_id) diff --git a/api/extensions/ext_celery.py b/api/extensions/ext_celery.py index bd4755e768..ae9a075340 100644 --- a/api/extensions/ext_celery.py +++ b/api/extensions/ext_celery.py @@ -43,15 +43,15 @@ def init_app(app: Flask) -> Celery: "schedule.clean_embedding_cache_task", "schedule.clean_unused_datasets_task", ] - + day = app.config["CELERY_BEAT_SCHEDULER_TIME"] beat_schedule = { 'clean_embedding_cache_task': { 'task': 'schedule.clean_embedding_cache_task.clean_embedding_cache_task', - 'schedule': timedelta(days=1), + 'schedule': timedelta(days=day), }, 'clean_unused_datasets_task': { 'task': 'schedule.clean_unused_datasets_task.clean_unused_datasets_task', - 'schedule': timedelta(days=1), + 'schedule': timedelta(days=day), } } celery_app.conf.update( diff --git a/api/extensions/storage/tencent_storage.py b/api/extensions/storage/tencent_storage.py index 6d9fb80f5e..e2c1ca55e3 100644 --- a/api/extensions/storage/tencent_storage.py +++ b/api/extensions/storage/tencent_storage.py @@ -32,8 +32,7 @@ class TencentStorage(BaseStorage): def load_stream(self, filename: str) -> Generator: def generate(filename: str = filename) -> Generator: response = self.client.get_object(Bucket=self.bucket_name, Key=filename) - while chunk := response['Body'].get_stream(chunk_size=4096): - yield chunk + yield from response['Body'].get_stream(chunk_size=4096) return generate() diff --git a/api/fields/dataset_fields.py b/api/fields/dataset_fields.py index 50c5f43540..120b66a92d 100644 --- a/api/fields/dataset_fields.py +++ b/api/fields/dataset_fields.py @@ -18,10 +18,28 @@ reranking_model_fields = { 'reranking_model_name': fields.String } +keyword_setting_fields = { + 'keyword_weight': fields.Float +} + +vector_setting_fields = { + 'vector_weight': fields.Float, + 'embedding_model_name': fields.String, + 'embedding_provider_name': fields.String, +} + +weighted_score_fields = { + 'weight_type': fields.String, + 'keyword_setting': fields.Nested(keyword_setting_fields), + 'vector_setting': fields.Nested(vector_setting_fields), +} + dataset_retrieval_model_fields = { 'search_method': fields.String, 'reranking_enable': fields.Boolean, + 'reranking_mode': fields.String, 'reranking_model': fields.Nested(reranking_model_fields), + 'weights': fields.Nested(weighted_score_fields, allow_null=True), 'top_k': fields.Integer, 'score_threshold_enabled': fields.Boolean, 'score_threshold': fields.Float diff --git a/api/fields/workflow_fields.py b/api/fields/workflow_fields.py index 54d7ed55f8..ff33a97ff2 100644 --- a/api/fields/workflow_fields.py +++ b/api/fields/workflow_fields.py @@ -1,8 +1,44 @@ from flask_restful import fields +from core.app.segments import SecretVariable, SegmentType, Variable +from core.helper import encrypter from fields.member_fields import simple_account_fields from libs.helper import TimestampField +ENVIRONMENT_VARIABLE_SUPPORTED_TYPES = (SegmentType.STRING, SegmentType.NUMBER, SegmentType.SECRET) + + +class EnvironmentVariableField(fields.Raw): + def format(self, value): + # Mask secret variables values in environment_variables + if isinstance(value, SecretVariable): + return { + 'id': value.id, + 'name': value.name, + 'value': encrypter.obfuscated_token(value.value), + 'value_type': value.value_type.value, + } + if isinstance(value, Variable): + return { + 'id': value.id, + 'name': value.name, + 'value': value.value, + 'value_type': value.value_type.value, + } + if isinstance(value, dict): + value_type = value.get('value_type') + if value_type not in ENVIRONMENT_VARIABLE_SUPPORTED_TYPES: + raise ValueError(f'Unsupported environment variable value type: {value_type}') + return value + + +environment_variable_fields = { + 'id': fields.String, + 'name': fields.String, + 'value': fields.Raw, + 'value_type': fields.String(attribute='value_type.value'), +} + workflow_fields = { 'id': fields.String, 'graph': fields.Raw(attribute='graph_dict'), @@ -13,4 +49,5 @@ workflow_fields = { 'updated_by': fields.Nested(simple_account_fields, attribute='updated_by_account', allow_null=True), 'updated_at': TimestampField, 'tool_published': fields.Boolean, + 'environment_variables': fields.List(EnvironmentVariableField()), } diff --git a/api/libs/helper.py b/api/libs/helper.py index 335c6688f4..15cd65dd6a 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -15,6 +15,7 @@ from zoneinfo import available_timezones from flask import Response, current_app, stream_with_context from flask_restful import fields +from core.app.features.rate_limiting.rate_limit import RateLimitGenerator from extensions.ext_redis import redis_client from models.account import Account @@ -159,7 +160,7 @@ def generate_text_hash(text: str) -> str: return sha256(hash_text.encode()).hexdigest() -def compact_generate_response(response: Union[dict, Generator]) -> Response: +def compact_generate_response(response: Union[dict, RateLimitGenerator]) -> Response: if isinstance(response, dict): return Response(response=json.dumps(response), status=200, mimetype='application/json') else: diff --git a/api/migrations/versions/53bf8af60645_update_model.py b/api/migrations/versions/53bf8af60645_update_model.py new file mode 100644 index 0000000000..3d0928d013 --- /dev/null +++ b/api/migrations/versions/53bf8af60645_update_model.py @@ -0,0 +1,41 @@ +"""update model + +Revision ID: 53bf8af60645 +Revises: 8e5588e6412e +Create Date: 2024-07-24 08:06:55.291031 + +""" +import sqlalchemy as sa +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = '53bf8af60645' +down_revision = '8e5588e6412e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('embeddings', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.VARCHAR(length=40), + type_=sa.String(length=255), + existing_nullable=False, + existing_server_default=sa.text("''::character varying")) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('embeddings', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.String(length=255), + type_=sa.VARCHAR(length=40), + existing_nullable=False, + existing_server_default=sa.text("''::character varying")) + + # ### end Alembic commands ### diff --git a/api/migrations/versions/6e957a32015b_add_embedding_cache_created_at_index.py b/api/migrations/versions/6e957a32015b_add_embedding_cache_created_at_index.py new file mode 100644 index 0000000000..7445f664cd --- /dev/null +++ b/api/migrations/versions/6e957a32015b_add_embedding_cache_created_at_index.py @@ -0,0 +1,32 @@ +"""add-embedding-cache-created_at_index + +Revision ID: 6e957a32015b +Revises: fecff1c3da27 +Create Date: 2024-07-19 17:21:34.414705 + +""" +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = '6e957a32015b' +down_revision = 'fecff1c3da27' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('embeddings', schema=None) as batch_op: + batch_op.create_index('created_at_idx', ['created_at'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('embeddings', schema=None) as batch_op: + batch_op.drop_index('created_at_idx') + + # ### end Alembic commands ### diff --git a/api/migrations/versions/8e5588e6412e_add_environment_variable_to_workflow_.py b/api/migrations/versions/8e5588e6412e_add_environment_variable_to_workflow_.py new file mode 100644 index 0000000000..ec2336da4d --- /dev/null +++ b/api/migrations/versions/8e5588e6412e_add_environment_variable_to_workflow_.py @@ -0,0 +1,33 @@ +"""add environment variable to workflow model + +Revision ID: 8e5588e6412e +Revises: 6e957a32015b +Create Date: 2024-07-22 03:27:16.042533 + +""" +import sqlalchemy as sa +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = '8e5588e6412e' +down_revision = '6e957a32015b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('workflows', schema=None) as batch_op: + batch_op.add_column(sa.Column('environment_variables', sa.Text(), server_default='{}', nullable=False)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('workflows', schema=None) as batch_op: + batch_op.drop_column('environment_variables') + + # ### end Alembic commands ### diff --git a/api/migrations/versions/eeb2e349e6ac_increase_max_model_name_length.py b/api/migrations/versions/eeb2e349e6ac_increase_max_model_name_length.py new file mode 100644 index 0000000000..434531b6c8 --- /dev/null +++ b/api/migrations/versions/eeb2e349e6ac_increase_max_model_name_length.py @@ -0,0 +1,53 @@ +"""increase max model_name length + +Revision ID: eeb2e349e6ac +Revises: 53bf8af60645 +Create Date: 2024-07-26 12:02:00.750358 + +""" +import sqlalchemy as sa +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = 'eeb2e349e6ac' +down_revision = '53bf8af60645' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('dataset_collection_bindings', schema=None) as batch_op: + batch_op.alter_column('model_name', + existing_type=sa.VARCHAR(length=40), + type_=sa.String(length=255), + existing_nullable=False) + + with op.batch_alter_table('embeddings', schema=None) as batch_op: + batch_op.alter_column('model_name', + existing_type=sa.VARCHAR(length=40), + type_=sa.String(length=255), + existing_nullable=False, + existing_server_default=sa.text("'text-embedding-ada-002'::character varying")) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('embeddings', schema=None) as batch_op: + batch_op.alter_column('model_name', + existing_type=sa.String(length=255), + type_=sa.VARCHAR(length=40), + existing_nullable=False, + existing_server_default=sa.text("'text-embedding-ada-002'::character varying")) + + with op.batch_alter_table('dataset_collection_bindings', schema=None) as batch_op: + batch_op.alter_column('model_name', + existing_type=sa.String(length=255), + type_=sa.VARCHAR(length=40), + existing_nullable=False) + + # ### end Alembic commands ### diff --git a/api/migrations/versions/fecff1c3da27_remove_extra_tracing_app_config_table .py b/api/migrations/versions/fecff1c3da27_remove_extra_tracing_app_config_table .py new file mode 100644 index 0000000000..271b2490de --- /dev/null +++ b/api/migrations/versions/fecff1c3da27_remove_extra_tracing_app_config_table .py @@ -0,0 +1,54 @@ +"""remove extra tracing app config table and add idx_dataset_permissions_tenant_id + +Revision ID: fecff1c3da27 +Revises: 408176b91ad3 +Create Date: 2024-07-19 12:03:21.217463 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'fecff1c3da27' +down_revision = '408176b91ad3' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('tracing_app_configs') + + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.drop_index('tracing_app_config_app_id_idx') + + # idx_dataset_permissions_tenant_id + with op.batch_alter_table('dataset_permissions', schema=None) as batch_op: + batch_op.create_index('idx_dataset_permissions_tenant_id', ['tenant_id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'tracing_app_configs', + sa.Column('id', postgresql.UUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('app_id', postgresql.UUID(), nullable=False), + sa.Column('tracing_provider', sa.String(length=255), nullable=True), + sa.Column('tracing_config', postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column( + 'created_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False + ), + sa.Column( + 'updated_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False + ), + sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + ) + + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.create_index('tracing_app_config_app_id_idx', ['app_id']) + + with op.batch_alter_table('dataset_permissions', schema=None) as batch_op: + batch_op.drop_index('idx_dataset_permissions_tenant_id') + # ### end Alembic commands ### diff --git a/api/models/account.py b/api/models/account.py index 23e7528d22..d36b2b9fda 100644 --- a/api/models/account.py +++ b/api/models/account.py @@ -48,7 +48,7 @@ class Account(UserMixin, db.Model): return self._current_tenant @current_tenant.setter - def current_tenant(self, value): + def current_tenant(self, value: "Tenant"): tenant = value ta = TenantAccountJoin.query.filter_by(tenant_id=tenant.id, account_id=self.id).first() if ta: @@ -62,7 +62,7 @@ class Account(UserMixin, db.Model): return self._current_tenant.id @current_tenant_id.setter - def current_tenant_id(self, value): + def current_tenant_id(self, value: str): try: tenant_account_join = db.session.query(Tenant, TenantAccountJoin) \ .filter(Tenant.id == value) \ diff --git a/api/models/dataset.py b/api/models/dataset.py index 02d49380bd..40f9f4cf83 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -9,10 +9,10 @@ import re import time from json import JSONDecodeError -from flask import current_app from sqlalchemy import func from sqlalchemy.dialects.postgresql import JSONB +from configs import dify_config from core.rag.retrieval.retrival_methods import RetrievalMethod from extensions.ext_database import db from extensions.ext_storage import storage @@ -68,7 +68,7 @@ class Dataset(db.Model): @property def created_by_account(self): - return Account.query.get(self.created_by) + return db.session.get(Account, self.created_by) @property def latest_process_rule(self): @@ -117,7 +117,7 @@ class Dataset(db.Model): @property def retrieval_model_dict(self): default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -336,7 +336,7 @@ class Document(db.Model): @property def dataset_process_rule(self): if self.dataset_process_rule_id: - return DatasetProcessRule.query.get(self.dataset_process_rule_id) + return db.session.get(DatasetProcessRule, self.dataset_process_rule_id) return None @property @@ -528,7 +528,7 @@ class DocumentSegment(db.Model): nonce = os.urandom(16).hex() timestamp = str(int(time.time())) data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}" - secret_key = current_app.config['SECRET_KEY'].encode() + secret_key = dify_config.SECRET_KEY.encode() if dify_config.SECRET_KEY else b'' sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest() encoded_sign = base64.urlsafe_b64encode(sign).decode() @@ -560,7 +560,7 @@ class AppDatasetJoin(db.Model): @property def app(self): - return App.query.get(self.app_id) + return db.session.get(App, self.app_id) class DatasetQuery(db.Model): @@ -630,16 +630,17 @@ class Embedding(db.Model): __tablename__ = 'embeddings' __table_args__ = ( db.PrimaryKeyConstraint('id', name='embedding_pkey'), - db.UniqueConstraint('model_name', 'hash', 'provider_name', name='embedding_hash_idx') + db.UniqueConstraint('model_name', 'hash', 'provider_name', name='embedding_hash_idx'), + db.Index('created_at_idx', 'created_at') ) id = db.Column(StringUUID, primary_key=True, server_default=db.text('uuid_generate_v4()')) - model_name = db.Column(db.String(40), nullable=False, + model_name = db.Column(db.String(255), nullable=False, server_default=db.text("'text-embedding-ada-002'::character varying")) hash = db.Column(db.String(64), nullable=False) embedding = db.Column(db.LargeBinary, nullable=False) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) - provider_name = db.Column(db.String(40), nullable=False, + provider_name = db.Column(db.String(255), nullable=False, server_default=db.text("''::character varying")) def set_embedding(self, embedding_data: list[float]): @@ -659,7 +660,7 @@ class DatasetCollectionBinding(db.Model): id = db.Column(StringUUID, primary_key=True, server_default=db.text('uuid_generate_v4()')) provider_name = db.Column(db.String(40), nullable=False) - model_name = db.Column(db.String(40), nullable=False) + model_name = db.Column(db.String(255), nullable=False) type = db.Column(db.String(40), server_default=db.text("'dataset'::character varying"), nullable=False) collection_name = db.Column(db.String(64), nullable=False) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) diff --git a/api/models/model.py b/api/models/model.py index 4d67272c1a..a6f517ea6b 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -4,10 +4,11 @@ import uuid from enum import Enum from typing import Optional -from flask import current_app, request +from flask import request from flask_login import UserMixin from sqlalchemy import Float, func, text +from configs import dify_config from core.file.tool_file_parser import ToolFileParser from core.file.upload_file_parser import UploadFileParser from extensions.ext_database import db @@ -111,7 +112,7 @@ class App(db.Model): @property def api_base_url(self): - return (current_app.config['SERVICE_API_URL'] if current_app.config['SERVICE_API_URL'] + return (dify_config.SERVICE_API_URL if dify_config.SERVICE_API_URL else request.host_url.rstrip('/')) + '/v1' @property @@ -327,7 +328,9 @@ class AppModelConfig(db.Model): return {'retrieval_model': 'single'} else: return dataset_configs - return {'retrieval_model': 'single'} + return { + 'retrieval_model': 'multiple', + } @property def file_upload_dict(self) -> dict: @@ -1113,7 +1116,7 @@ class Site(db.Model): @property def app_base_url(self): return ( - current_app.config['APP_WEB_URL'] if current_app.config['APP_WEB_URL'] else request.host_url.rstrip('/')) + dify_config.APP_WEB_URL if dify_config.APP_WEB_URL else request.host_url.rstrip('/')) class ApiToken(db.Model): @@ -1382,7 +1385,7 @@ class TraceAppConfig(db.Model): __tablename__ = 'trace_app_config' __table_args__ = ( db.PrimaryKeyConstraint('id', name='tracing_app_config_pkey'), - db.Index('tracing_app_config_app_id_idx', 'app_id'), + db.Index('trace_app_config_app_id_idx', 'app_id'), ) id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) diff --git a/api/models/workflow.py b/api/models/workflow.py index 2d6491032b..df2269cd0f 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -1,7 +1,16 @@ import json +from collections.abc import Mapping, Sequence from enum import Enum -from typing import Optional, Union +from typing import Any, Optional, Union +import contexts +from constants import HIDDEN_VALUE +from core.app.segments import ( + SecretVariable, + Variable, + factory, +) +from core.helper import encrypter from extensions.ext_database import db from libs import helper from models import StringUUID @@ -112,21 +121,22 @@ class Workflow(db.Model): created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) updated_by = db.Column(StringUUID) updated_at = db.Column(db.DateTime) + _environment_variables = db.Column('environment_variables', db.Text, nullable=False, server_default='{}') @property def created_by_account(self): - return Account.query.get(self.created_by) + return db.session.get(Account, self.created_by) @property def updated_by_account(self): - return Account.query.get(self.updated_by) if self.updated_by else None + return db.session.get(Account, self.updated_by) if self.updated_by else None @property - def graph_dict(self): - return json.loads(self.graph) if self.graph else None + def graph_dict(self) -> Mapping[str, Any]: + return json.loads(self.graph) if self.graph else {} @property - def features_dict(self): + def features_dict(self) -> Mapping[str, Any]: return json.loads(self.features) if self.features else {} def user_input_form(self, to_old_structure: bool = False) -> list: @@ -177,6 +187,72 @@ class Workflow(db.Model): WorkflowToolProvider.app_id == self.app_id ).first() is not None + @property + def environment_variables(self) -> Sequence[Variable]: + # TODO: find some way to init `self._environment_variables` when instance created. + if self._environment_variables is None: + self._environment_variables = '{}' + + tenant_id = contexts.tenant_id.get() + + environment_variables_dict: dict[str, Any] = json.loads(self._environment_variables) + results = [factory.build_variable_from_mapping(v) for v in environment_variables_dict.values()] + + # decrypt secret variables value + decrypt_func = ( + lambda var: var.model_copy( + update={'value': encrypter.decrypt_token(tenant_id=tenant_id, token=var.value)} + ) + if isinstance(var, SecretVariable) + else var + ) + results = list(map(decrypt_func, results)) + return results + + @environment_variables.setter + def environment_variables(self, value: Sequence[Variable]): + tenant_id = contexts.tenant_id.get() + + value = list(value) + if any(var for var in value if not var.id): + raise ValueError('environment variable require a unique id') + + # Compare inputs and origin variables, if the value is HIDDEN_VALUE, use the origin variable value (only update `name`). + origin_variables_dictionary = {var.id: var for var in self.environment_variables} + for i, variable in enumerate(value): + if variable.id in origin_variables_dictionary and variable.value == HIDDEN_VALUE: + value[i] = origin_variables_dictionary[variable.id].model_copy(update={'name': variable.name}) + + # encrypt secret variables value + encrypt_func = ( + lambda var: var.model_copy( + update={'value': encrypter.encrypt_token(tenant_id=tenant_id, token=var.value)} + ) + if isinstance(var, SecretVariable) + else var + ) + encrypted_vars = list(map(encrypt_func, value)) + environment_variables_json = json.dumps( + {var.name: var.model_dump() for var in encrypted_vars}, + ensure_ascii=False, + ) + self._environment_variables = environment_variables_json + + def to_dict(self, *, include_secret: bool = False) -> Mapping[str, Any]: + environment_variables = list(self.environment_variables) + environment_variables = [ + v if not isinstance(v, SecretVariable) or include_secret else v.model_copy(update={'value': ''}) + for v in environment_variables + ] + + result = { + 'graph': self.graph_dict, + 'features': self.features_dict, + 'environment_variables': [var.model_dump(mode='json') for var in environment_variables], + } + return result + + class WorkflowRunTriggeredFrom(Enum): """ Workflow Run Triggered From Enum @@ -290,14 +366,14 @@ class WorkflowRun(db.Model): @property def created_by_account(self): created_by_role = CreatedByRole.value_of(self.created_by_role) - return Account.query.get(self.created_by) \ + return db.session.get(Account, self.created_by) \ if created_by_role == CreatedByRole.ACCOUNT else None @property def created_by_end_user(self): from models.model import EndUser created_by_role = CreatedByRole.value_of(self.created_by_role) - return EndUser.query.get(self.created_by) \ + return db.session.get(EndUser, self.created_by) \ if created_by_role == CreatedByRole.END_USER else None @property @@ -500,14 +576,14 @@ class WorkflowNodeExecution(db.Model): @property def created_by_account(self): created_by_role = CreatedByRole.value_of(self.created_by_role) - return Account.query.get(self.created_by) \ + return db.session.get(Account, self.created_by) \ if created_by_role == CreatedByRole.ACCOUNT else None @property def created_by_end_user(self): from models.model import EndUser created_by_role = CreatedByRole.value_of(self.created_by_role) - return EndUser.query.get(self.created_by) \ + return db.session.get(EndUser, self.created_by) \ if created_by_role == CreatedByRole.END_USER else None @property @@ -612,17 +688,17 @@ class WorkflowAppLog(db.Model): @property def workflow_run(self): - return WorkflowRun.query.get(self.workflow_run_id) + return db.session.get(WorkflowRun, self.workflow_run_id) @property def created_by_account(self): created_by_role = CreatedByRole.value_of(self.created_by_role) - return Account.query.get(self.created_by) \ + return db.session.get(Account, self.created_by) \ if created_by_role == CreatedByRole.ACCOUNT else None @property def created_by_end_user(self): from models.model import EndUser created_by_role = CreatedByRole.value_of(self.created_by_role) - return EndUser.query.get(self.created_by) \ + return db.session.get(EndUser, self.created_by) \ if created_by_role == CreatedByRole.END_USER else None diff --git a/api/poetry.lock b/api/poetry.lock index ae7cdbb32d..0abd2d2175 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -184,13 +184,13 @@ alibabacloud_credentials = ">=0.2.0,<1.0.0" [[package]] name = "alibabacloud-gpdb20160503" -version = "3.8.2" +version = "3.8.3" description = "Alibaba Cloud AnalyticDB for PostgreSQL (20160503) SDK Library for Python" optional = false python-versions = ">=3.6" files = [ - {file = "alibabacloud_gpdb20160503-3.8.2-py3-none-any.whl", hash = "sha256:081977cdd4174c786b303f3c5651026297d84baa0256386be8215ee997cd5c75"}, - {file = "alibabacloud_gpdb20160503-3.8.2.tar.gz", hash = "sha256:c964ca721a05e440a1065e33aa74d456eafe2c8b17f6e0d960d5bb44dfe4bd9c"}, + {file = "alibabacloud_gpdb20160503-3.8.3-py3-none-any.whl", hash = "sha256:06e1c46ce5e4e9d1bcae76e76e51034196c625799d06b2efec8d46a7df323fe8"}, + {file = "alibabacloud_gpdb20160503-3.8.3.tar.gz", hash = "sha256:4dfcc0d9cff5a921d529d76f4bf97e2ceb9dc2fa53f00ab055f08509423d8e30"}, ] [package.dependencies] @@ -311,12 +311,12 @@ alibabacloud_tea_xml = ">=0.0.2,<1.0.0" [[package]] name = "alibabacloud-tea-util" -version = "0.3.12" +version = "0.3.13" description = "The tea-util module of alibabaCloud Python SDK." optional = false python-versions = ">=3.6" files = [ - {file = "alibabacloud_tea_util-0.3.12.tar.gz", hash = "sha256:72a2f5a046e5b977ade4202eb4f65b3d70ad707a548e29aacd4a572c2d18d06b"}, + {file = "alibabacloud_tea_util-0.3.13.tar.gz", hash = "sha256:8cbdfd2a03fbbf622f901439fa08643898290dd40e1d928347f6346e43f63c90"}, ] [package.dependencies] @@ -646,38 +646,38 @@ files = [ [[package]] name = "bcrypt" -version = "4.1.3" +version = "4.2.0" description = "Modern password hashing for your software and your servers" optional = false python-versions = ">=3.7" files = [ - {file = "bcrypt-4.1.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:48429c83292b57bf4af6ab75809f8f4daf52aa5d480632e53707805cc1ce9b74"}, - {file = "bcrypt-4.1.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a8bea4c152b91fd8319fef4c6a790da5c07840421c2b785084989bf8bbb7455"}, - {file = "bcrypt-4.1.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d3b317050a9a711a5c7214bf04e28333cf528e0ed0ec9a4e55ba628d0f07c1a"}, - {file = "bcrypt-4.1.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:094fd31e08c2b102a14880ee5b3d09913ecf334cd604af27e1013c76831f7b05"}, - {file = "bcrypt-4.1.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4fb253d65da30d9269e0a6f4b0de32bd657a0208a6f4e43d3e645774fb5457f3"}, - {file = "bcrypt-4.1.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:193bb49eeeb9c1e2db9ba65d09dc6384edd5608d9d672b4125e9320af9153a15"}, - {file = "bcrypt-4.1.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:8cbb119267068c2581ae38790e0d1fbae65d0725247a930fc9900c285d95725d"}, - {file = "bcrypt-4.1.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6cac78a8d42f9d120b3987f82252bdbeb7e6e900a5e1ba37f6be6fe4e3848286"}, - {file = "bcrypt-4.1.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:01746eb2c4299dd0ae1670234bf77704f581dd72cc180f444bfe74eb80495b64"}, - {file = "bcrypt-4.1.3-cp37-abi3-win32.whl", hash = "sha256:037c5bf7c196a63dcce75545c8874610c600809d5d82c305dd327cd4969995bf"}, - {file = "bcrypt-4.1.3-cp37-abi3-win_amd64.whl", hash = "sha256:8a893d192dfb7c8e883c4576813bf18bb9d59e2cfd88b68b725990f033f1b978"}, - {file = "bcrypt-4.1.3-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0d4cf6ef1525f79255ef048b3489602868c47aea61f375377f0d00514fe4a78c"}, - {file = "bcrypt-4.1.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5698ce5292a4e4b9e5861f7e53b1d89242ad39d54c3da451a93cac17b61921a"}, - {file = "bcrypt-4.1.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec3c2e1ca3e5c4b9edb94290b356d082b721f3f50758bce7cce11d8a7c89ce84"}, - {file = "bcrypt-4.1.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3a5be252fef513363fe281bafc596c31b552cf81d04c5085bc5dac29670faa08"}, - {file = "bcrypt-4.1.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5f7cd3399fbc4ec290378b541b0cf3d4398e4737a65d0f938c7c0f9d5e686611"}, - {file = "bcrypt-4.1.3-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:c4c8d9b3e97209dd7111bf726e79f638ad9224b4691d1c7cfefa571a09b1b2d6"}, - {file = "bcrypt-4.1.3-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:31adb9cbb8737a581a843e13df22ffb7c84638342de3708a98d5c986770f2834"}, - {file = "bcrypt-4.1.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:551b320396e1d05e49cc18dd77d970accd52b322441628aca04801bbd1d52a73"}, - {file = "bcrypt-4.1.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6717543d2c110a155e6821ce5670c1f512f602eabb77dba95717ca76af79867d"}, - {file = "bcrypt-4.1.3-cp39-abi3-win32.whl", hash = "sha256:6004f5229b50f8493c49232b8e75726b568535fd300e5039e255d919fc3a07f2"}, - {file = "bcrypt-4.1.3-cp39-abi3-win_amd64.whl", hash = "sha256:2505b54afb074627111b5a8dc9b6ae69d0f01fea65c2fcaea403448c503d3991"}, - {file = "bcrypt-4.1.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:cb9c707c10bddaf9e5ba7cdb769f3e889e60b7d4fea22834b261f51ca2b89fed"}, - {file = "bcrypt-4.1.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:9f8ea645eb94fb6e7bea0cf4ba121c07a3a182ac52876493870033141aa687bc"}, - {file = "bcrypt-4.1.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:f44a97780677e7ac0ca393bd7982b19dbbd8d7228c1afe10b128fd9550eef5f1"}, - {file = "bcrypt-4.1.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d84702adb8f2798d813b17d8187d27076cca3cd52fe3686bb07a9083930ce650"}, - {file = "bcrypt-4.1.3.tar.gz", hash = "sha256:2ee15dd749f5952fe3f0430d0ff6b74082e159c50332a1413d51b5689cf06623"}, + {file = "bcrypt-4.2.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:096a15d26ed6ce37a14c1ac1e48119660f21b24cba457f160a4b830f3fe6b5cb"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c02d944ca89d9b1922ceb8a46460dd17df1ba37ab66feac4870f6862a1533c00"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d84cf6d877918620b687b8fd1bf7781d11e8a0998f576c7aa939776b512b98d"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1bb429fedbe0249465cdd85a58e8376f31bb315e484f16e68ca4c786dcc04291"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:655ea221910bcac76ea08aaa76df427ef8625f92e55a8ee44fbf7753dbabb328"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:1ee38e858bf5d0287c39b7a1fc59eec64bbf880c7d504d3a06a96c16e14058e7"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0da52759f7f30e83f1e30a888d9163a81353ef224d82dc58eb5bb52efcabc399"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3698393a1b1f1fd5714524193849d0c6d524d33523acca37cd28f02899285060"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:762a2c5fb35f89606a9fde5e51392dad0cd1ab7ae64149a8b935fe8d79dd5ed7"}, + {file = "bcrypt-4.2.0-cp37-abi3-win32.whl", hash = "sha256:5a1e8aa9b28ae28020a3ac4b053117fb51c57a010b9f969603ed885f23841458"}, + {file = "bcrypt-4.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:8f6ede91359e5df88d1f5c1ef47428a4420136f3ce97763e31b86dd8280fbdf5"}, + {file = "bcrypt-4.2.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:c52aac18ea1f4a4f65963ea4f9530c306b56ccd0c6f8c8da0c06976e34a6e841"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bbbfb2734f0e4f37c5136130405332640a1e46e6b23e000eeff2ba8d005da68"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3413bd60460f76097ee2e0a493ccebe4a7601918219c02f503984f0a7ee0aebe"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8d7bb9c42801035e61c109c345a28ed7e84426ae4865511eb82e913df18f58c2"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3d3a6d28cb2305b43feac298774b997e372e56c7c7afd90a12b3dc49b189151c"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9c1c4ad86351339c5f320ca372dfba6cb6beb25e8efc659bedd918d921956bae"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:27fe0f57bb5573104b5a6de5e4153c60814c711b29364c10a75a54bb6d7ff48d"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8ac68872c82f1add6a20bd489870c71b00ebacd2e9134a8aa3f98a0052ab4b0e"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cb2a8ec2bc07d3553ccebf0746bbf3d19426d1c6d1adbd4fa48925f66af7b9e8"}, + {file = "bcrypt-4.2.0-cp39-abi3-win32.whl", hash = "sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34"}, + {file = "bcrypt-4.2.0-cp39-abi3-win_amd64.whl", hash = "sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9"}, + {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:39e1d30c7233cfc54f5c3f2c825156fe044efdd3e0b9d309512cc514a263ec2a"}, + {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f4f4acf526fcd1c34e7ce851147deedd4e26e6402369304220250598b26448db"}, + {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:1ff39b78a52cf03fdf902635e4c81e544714861ba3f0efc56558979dd4f09170"}, + {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:373db9abe198e8e2c70d12b479464e0d5092cc122b20ec504097b5f2297ed184"}, + {file = "bcrypt-4.2.0.tar.gz", hash = "sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221"}, ] [package.extras] @@ -745,13 +745,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.139" +version = "1.34.147" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.139-py3-none-any.whl", hash = "sha256:dd1e085d4caa2a4c1b7d83e3bc51416111c8238a35d498e9d3b04f3b63b086ba"}, - {file = "botocore-1.34.139.tar.gz", hash = "sha256:df023d8cf8999d574214dad4645cb90f9d2ccd1494f6ee2b57b1ab7522f6be77"}, + {file = "botocore-1.34.147-py3-none-any.whl", hash = "sha256:be94a2f4874b1d1705cae2bd512c475047497379651678593acb6c61c50d91de"}, + {file = "botocore-1.34.147.tar.gz", hash = "sha256:2e8f000b77e4ca345146cb2edab6403769a517b564f627bb084ab335417f3dbe"}, ] [package.dependencies] @@ -1279,13 +1279,13 @@ numpy = "*" [[package]] name = "chromadb" -version = "0.5.3" +version = "0.5.1" description = "Chroma." optional = false python-versions = ">=3.8" files = [ - {file = "chromadb-0.5.3-py3-none-any.whl", hash = "sha256:b3874f08356e291c68c6d2e177db472cd51f22f3af7b9746215b748fd1e29982"}, - {file = "chromadb-0.5.3.tar.gz", hash = "sha256:05d887f56a46b2e0fc6ac5ab979503a27b9ee50d5ca9e455f83b2fb9840cd026"}, + {file = "chromadb-0.5.1-py3-none-any.whl", hash = "sha256:61f1f75a672b6edce7f1c8875c67e2aaaaf130dc1c1684431fbc42ad7240d01d"}, + {file = "chromadb-0.5.1.tar.gz", hash = "sha256:e2b2b6a34c2a949bedcaa42fa7775f40c7f6667848fc8094dcbf97fc0d30bee7"}, ] [package.dependencies] @@ -1610,6 +1610,22 @@ lz4 = ["clickhouse-cityhash (>=1.0.2.1)", "lz4", "lz4 (<=3.0.1)"] numpy = ["numpy (>=1.12.0)", "pandas (>=0.24.0)"] zstd = ["clickhouse-cityhash (>=1.0.2.1)", "zstd"] +[[package]] +name = "cloudscraper" +version = "1.2.71" +description = "A Python module to bypass Cloudflare's anti-bot page." +optional = false +python-versions = "*" +files = [ + {file = "cloudscraper-1.2.71-py2.py3-none-any.whl", hash = "sha256:76f50ca529ed2279e220837befdec892626f9511708e200d48d5bb76ded679b0"}, + {file = "cloudscraper-1.2.71.tar.gz", hash = "sha256:429c6e8aa6916d5bad5c8a5eac50f3ea53c9ac22616f6cb21b18dcc71517d0d3"}, +] + +[package.dependencies] +pyparsing = ">=2.4.7" +requests = ">=2.9.2" +requests-toolbelt = ">=0.9.1" + [[package]] name = "cohere" version = "5.2.6" @@ -1822,43 +1838,38 @@ files = [ [[package]] name = "cryptography" -version = "42.0.8" +version = "43.0.0" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, - {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, - {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, - {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, - {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, - {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, - {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, + {file = "cryptography-43.0.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:64c3f16e2a4fc51c0d06af28441881f98c5d91009b8caaff40cf3548089e9c74"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3dcdedae5c7710b9f97ac6bba7e1052b95c7083c9d0e9df96e02a1932e777895"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d9a1eca329405219b605fac09ecfc09ac09e595d6def650a437523fcd08dd22"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ea9e57f8ea880eeea38ab5abf9fbe39f923544d7884228ec67d666abd60f5a47"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9a8d6802e0825767476f62aafed40532bd435e8a5f7d23bd8b4f5fd04cc80ecf"}, + {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cc70b4b581f28d0a254d006f26949245e3657d40d8857066c2ae22a61222ef55"}, + {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4a997df8c1c2aae1e1e5ac49c2e4f610ad037fc5a3aadc7b64e39dea42249431"}, + {file = "cryptography-43.0.0-cp37-abi3-win32.whl", hash = "sha256:6e2b11c55d260d03a8cf29ac9b5e0608d35f08077d8c087be96287f43af3ccdc"}, + {file = "cryptography-43.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:31e44a986ceccec3d0498e16f3d27b2ee5fdf69ce2ab89b52eaad1d2f33d8778"}, + {file = "cryptography-43.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:7b3f5fe74a5ca32d4d0f302ffe6680fcc5c28f8ef0dc0ae8f40c0f3a1b4fca66"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac1955ce000cb29ab40def14fd1bbfa7af2017cca696ee696925615cafd0dce5"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:299d3da8e00b7e2b54bb02ef58d73cd5f55fb31f33ebbf33bd00d9aa6807df7e"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ee0c405832ade84d4de74b9029bedb7b31200600fa524d218fc29bfa371e97f5"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cb013933d4c127349b3948aa8aaf2f12c0353ad0eccd715ca789c8a0f671646f"}, + {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fdcb265de28585de5b859ae13e3846a8e805268a823a12a4da2597f1f5afc9f0"}, + {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2905ccf93a8a2a416f3ec01b1a7911c3fe4073ef35640e7ee5296754e30b762b"}, + {file = "cryptography-43.0.0-cp39-abi3-win32.whl", hash = "sha256:47ca71115e545954e6c1d207dd13461ab81f4eccfcb1345eac874828b5e3eaaf"}, + {file = "cryptography-43.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:0663585d02f76929792470451a5ba64424acc3cd5227b03921dab0e2f27b1709"}, + {file = "cryptography-43.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2c6d112bf61c5ef44042c253e4859b3cbbb50df2f78fa8fae6747a7814484a70"}, + {file = "cryptography-43.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:844b6d608374e7d08f4f6e6f9f7b951f9256db41421917dfb2d003dde4cd6b66"}, + {file = "cryptography-43.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:51956cf8730665e2bdf8ddb8da0056f699c1a5715648c1b0144670c1ba00b48f"}, + {file = "cryptography-43.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:aae4d918f6b180a8ab8bf6511a419473d107df4dbb4225c7b48c5c9602c38c7f"}, + {file = "cryptography-43.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:232ce02943a579095a339ac4b390fbbe97f5b5d5d107f8a08260ea2768be8cc2"}, + {file = "cryptography-43.0.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5bcb8a5620008a8034d39bce21dc3e23735dfdb6a33a06974739bfa04f853947"}, + {file = "cryptography-43.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:08a24a7070b2b6804c1940ff0f910ff728932a9d0e80e7814234269f9d46d069"}, + {file = "cryptography-43.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:e9c5266c432a1e23738d178e51c2c7a5e2ddf790f248be939448c0ba2021f9d1"}, + {file = "cryptography-43.0.0.tar.gz", hash = "sha256:b88075ada2d51aa9f18283532c9f60e72170041bba88d7f37e49cbb10275299e"}, ] [package.dependencies] @@ -1871,7 +1882,7 @@ nox = ["nox"] pep8test = ["check-sdist", "click", "mypy", "ruff"] sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi", "cryptography-vectors (==43.0.0)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] [[package]] @@ -2122,21 +2133,21 @@ files = [ [[package]] name = "duckduckgo-search" -version = "6.1.9" +version = "6.2.1" description = "Search for words, documents, images, news, maps and text translation using the DuckDuckGo.com search engine." optional = false python-versions = ">=3.8" files = [ - {file = "duckduckgo_search-6.1.9-py3-none-any.whl", hash = "sha256:a208babf87b971290b1afed9908bc5ab6ac6c1738b90b48ad613267f7630cb77"}, - {file = "duckduckgo_search-6.1.9.tar.gz", hash = "sha256:0d7d746e003d6b3bcd0d0dc11927c9a69b6fa271f3b3f65df6f01ea4d9d2689d"}, + {file = "duckduckgo_search-6.2.1-py3-none-any.whl", hash = "sha256:1a03f799b85fdfa08d5e6478624683f373b9dc35e6f145544b9cab72a4f575fa"}, + {file = "duckduckgo_search-6.2.1.tar.gz", hash = "sha256:d664ec096193e3fb43bdfae4b0ad9c04e44094b58f41998adcdd20a86ee1ed74"}, ] [package.dependencies] click = ">=8.1.7" -pyreqwest-impersonate = ">=0.4.9" +pyreqwest-impersonate = ">=0.5.0" [package.extras] -dev = ["mypy (>=1.10.1)", "pytest (>=8.2.2)", "pytest-asyncio (>=0.23.7)", "ruff (>=0.5.0)"] +dev = ["mypy (>=1.10.1)", "pytest (>=8.2.2)", "pytest-asyncio (>=0.23.7)", "ruff (>=0.5.2)"] lxml = ["lxml (>=5.2.2)"] [[package]] @@ -2205,13 +2216,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.2.1" +version = "1.2.2" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, - {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, ] [package.extras] @@ -2219,13 +2230,13 @@ test = ["pytest (>=6)"] [[package]] name = "fastapi" -version = "0.111.0" +version = "0.111.1" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.8" files = [ - {file = "fastapi-0.111.0-py3-none-any.whl", hash = "sha256:97ecbf994be0bcbdadedf88c3150252bed7b2087075ac99735403b1b76cc8fc0"}, - {file = "fastapi-0.111.0.tar.gz", hash = "sha256:b9db9dd147c91cb8b769f7183535773d8741dd46f9dc6676cd82eab510228cd7"}, + {file = "fastapi-0.111.1-py3-none-any.whl", hash = "sha256:4f51cfa25d72f9fbc3280832e84b32494cf186f50158d364a8765aabf22587bf"}, + {file = "fastapi-0.111.1.tar.gz", hash = "sha256:ddd1ac34cb1f76c2e2d7f8545a4bcb5463bce4834e81abf0b189e0c359ab2413"}, ] [package.dependencies] @@ -2233,12 +2244,10 @@ email_validator = ">=2.0.0" fastapi-cli = ">=0.0.2" httpx = ">=0.23.0" jinja2 = ">=2.11.2" -orjson = ">=3.2.1" pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" python-multipart = ">=0.0.7" starlette = ">=0.37.2,<0.38.0" typing-extensions = ">=4.8.0" -ujson = ">=4.0.1,<4.0.2 || >4.0.2,<4.1.0 || >4.1.0,<4.2.0 || >4.2.0,<4.3.0 || >4.3.0,<5.0.0 || >5.0.0,<5.1.0 || >5.1.0" uvicorn = {version = ">=0.12.0", extras = ["standard"]} [package.extras] @@ -2263,42 +2272,42 @@ standard = ["fastapi", "uvicorn[standard] (>=0.15.0)"] [[package]] name = "fastavro" -version = "1.9.4" +version = "1.9.5" description = "Fast read/write of AVRO files" optional = false python-versions = ">=3.8" files = [ - {file = "fastavro-1.9.4-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:60cb38f07462a7fb4e4440ed0de67d3d400ae6b3d780f81327bebde9aa55faef"}, - {file = "fastavro-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:063d01d197fc929c20adc09ca9f0ca86d33ac25ee0963ce0b438244eee8315ae"}, - {file = "fastavro-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87a9053fcfbc895f2a16a4303af22077e3a8fdcf1cd5d6ed47ff2ef22cbba2f0"}, - {file = "fastavro-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:02bf1276b7326397314adf41b34a4890f6ffa59cf7e0eb20b9e4ab0a143a1598"}, - {file = "fastavro-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56bed9eca435389a8861e6e2d631ec7f8f5dda5b23f93517ac710665bd34ca29"}, - {file = "fastavro-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:0cd2099c8c672b853e0b20c13e9b62a69d3fbf67ee7c59c7271ba5df1680310d"}, - {file = "fastavro-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:af8c6d8c43a02b5569c093fc5467469541ac408c79c36a5b0900d3dd0b3ba838"}, - {file = "fastavro-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4a138710bd61580324d23bc5e3df01f0b82aee0a76404d5dddae73d9e4c723f"}, - {file = "fastavro-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:903d97418120ca6b6a7f38a731166c1ccc2c4344ee5e0470d09eb1dc3687540a"}, - {file = "fastavro-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c443eeb99899d062dbf78c525e4614dd77e041a7688fa2710c224f4033f193ae"}, - {file = "fastavro-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ac26ab0774d1b2b7af6d8f4300ad20bbc4b5469e658a02931ad13ce23635152f"}, - {file = "fastavro-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:cf7247874c22be856ba7d1f46a0f6e0379a6025f1a48a7da640444cbac6f570b"}, - {file = "fastavro-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:68912f2020e1b3d70557260b27dd85fb49a4fc6bfab18d384926127452c1da4c"}, - {file = "fastavro-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6925ce137cdd78e109abdb0bc33aad55de6c9f2d2d3036b65453128f2f5f5b92"}, - {file = "fastavro-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b928cd294e36e35516d0deb9e104b45be922ba06940794260a4e5dbed6c192a"}, - {file = "fastavro-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:90c9838bc4c991ffff5dd9d88a0cc0030f938b3fdf038cdf6babde144b920246"}, - {file = "fastavro-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:eca6e54da571b06a3c5a72dbb7212073f56c92a6fbfbf847b91c347510f8a426"}, - {file = "fastavro-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a4b02839ac261100cefca2e2ad04cdfedc556cb66b5ec735e0db428e74b399de"}, - {file = "fastavro-1.9.4-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:4451ee9a305a73313a1558d471299f3130e4ecc10a88bf5742aa03fb37e042e6"}, - {file = "fastavro-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8524fccfb379565568c045d29b2ebf71e1f2c0dd484aeda9fe784ef5febe1a8"}, - {file = "fastavro-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33d0a00a6e09baa20f6f038d7a2ddcb7eef0e7a9980e947a018300cb047091b8"}, - {file = "fastavro-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:23d7e5b29c9bf6f26e8be754b2c8b919838e506f78ef724de7d22881696712fc"}, - {file = "fastavro-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2e6ab3ee53944326460edf1125b2ad5be2fadd80f7211b13c45fa0c503b4cf8d"}, - {file = "fastavro-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:64d335ec2004204c501f8697c385d0a8f6b521ac82d5b30696f789ff5bc85f3c"}, - {file = "fastavro-1.9.4-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:7e05f44c493e89e73833bd3ff3790538726906d2856f59adc8103539f4a1b232"}, - {file = "fastavro-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:253c63993250bff4ee7b11fb46cf3a4622180a783bedc82a24c6fdcd1b10ca2a"}, - {file = "fastavro-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24d6942eb1db14640c2581e0ecd1bbe0afc8a83731fcd3064ae7f429d7880cb7"}, - {file = "fastavro-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d47bb66be6091cd48cfe026adcad11c8b11d7d815a2949a1e4ccf03df981ca65"}, - {file = "fastavro-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c293897f12f910e58a1024f9c77f565aa8e23b36aafda6ad8e7041accc57a57f"}, - {file = "fastavro-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:f05d2afcb10a92e2a9e580a3891f090589b3e567fdc5641f8a46a0b084f120c3"}, - {file = "fastavro-1.9.4.tar.gz", hash = "sha256:56b8363e360a1256c94562393dc7f8611f3baf2b3159f64fb2b9c6b87b14e876"}, + {file = "fastavro-1.9.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:61253148e95dd2b6457247b441b7555074a55de17aef85f5165bfd5facf600fc"}, + {file = "fastavro-1.9.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b604935d671ad47d888efc92a106f98e9440874108b444ac10e28d643109c937"}, + {file = "fastavro-1.9.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0adbf4956fd53bd74c41e7855bb45ccce953e0eb0e44f5836d8d54ad843f9944"}, + {file = "fastavro-1.9.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:53d838e31457db8bf44460c244543f75ed307935d5fc1d93bc631cc7caef2082"}, + {file = "fastavro-1.9.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:07b6288e8681eede16ff077632c47395d4925c2f51545cd7a60f194454db2211"}, + {file = "fastavro-1.9.5-cp310-cp310-win_amd64.whl", hash = "sha256:ef08cf247fdfd61286ac0c41854f7194f2ad05088066a756423d7299b688d975"}, + {file = "fastavro-1.9.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c52d7bb69f617c90935a3e56feb2c34d4276819a5c477c466c6c08c224a10409"}, + {file = "fastavro-1.9.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85e05969956003df8fa4491614bc62fe40cec59e94d06e8aaa8d8256ee3aab82"}, + {file = "fastavro-1.9.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06e6df8527493a9f0d9a8778df82bab8b1aa6d80d1b004e5aec0a31dc4dc501c"}, + {file = "fastavro-1.9.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:27820da3b17bc01cebb6d1687c9d7254b16d149ef458871aaa207ed8950f3ae6"}, + {file = "fastavro-1.9.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:195a5b8e33eb89a1a9b63fa9dce7a77d41b3b0cd785bac6044df619f120361a2"}, + {file = "fastavro-1.9.5-cp311-cp311-win_amd64.whl", hash = "sha256:be612c109efb727bfd36d4d7ed28eb8e0506617b7dbe746463ebbf81e85eaa6b"}, + {file = "fastavro-1.9.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b133456c8975ec7d2a99e16a7e68e896e45c821b852675eac4ee25364b999c14"}, + {file = "fastavro-1.9.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf586373c3d1748cac849395aad70c198ee39295f92e7c22c75757b5c0300fbe"}, + {file = "fastavro-1.9.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:724ef192bc9c55d5b4c7df007f56a46a21809463499856349d4580a55e2b914c"}, + {file = "fastavro-1.9.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bfd11fe355a8f9c0416803afac298960eb4c603a23b1c74ff9c1d3e673ea7185"}, + {file = "fastavro-1.9.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9827d1654d7bcb118ef5efd3e5b2c9ab2a48d44dac5e8c6a2327bc3ac3caa828"}, + {file = "fastavro-1.9.5-cp312-cp312-win_amd64.whl", hash = "sha256:d84b69dca296667e6137ae7c9a96d060123adbc0c00532cc47012b64d38b47e9"}, + {file = "fastavro-1.9.5-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:fb744e9de40fb1dc75354098c8db7da7636cba50a40f7bef3b3fb20f8d189d88"}, + {file = "fastavro-1.9.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:240df8bacd13ff5487f2465604c007d686a566df5cbc01d0550684eaf8ff014a"}, + {file = "fastavro-1.9.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3bb35c25bbc3904e1c02333bc1ae0173e0a44aa37a8e95d07e681601246e1f1"}, + {file = "fastavro-1.9.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:b47a54a9700de3eabefd36dabfb237808acae47bc873cada6be6990ef6b165aa"}, + {file = "fastavro-1.9.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:48c7b5e6d2f3bf7917af301c275b05c5be3dd40bb04e80979c9e7a2ab31a00d1"}, + {file = "fastavro-1.9.5-cp38-cp38-win_amd64.whl", hash = "sha256:05d13f98d4e325be40387e27da9bd60239968862fe12769258225c62ec906f04"}, + {file = "fastavro-1.9.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5b47948eb196263f6111bf34e1cd08d55529d4ed46eb50c1bc8c7c30a8d18868"}, + {file = "fastavro-1.9.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85b7a66ad521298ad9373dfe1897a6ccfc38feab54a47b97922e213ae5ad8870"}, + {file = "fastavro-1.9.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44cb154f863ad80e41aea72a709b12e1533b8728c89b9b1348af91a6154ab2f5"}, + {file = "fastavro-1.9.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:b5f7f2b1fe21231fd01f1a2a90e714ae267fe633cd7ce930c0aea33d1c9f4901"}, + {file = "fastavro-1.9.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:88fbbe16c61d90a89d78baeb5a34dc1c63a27b115adccdbd6b1fb6f787deacf2"}, + {file = "fastavro-1.9.5-cp39-cp39-win_amd64.whl", hash = "sha256:753f5eedeb5ca86004e23a9ce9b41c5f25eb64a876f95edcc33558090a7f3e4b"}, + {file = "fastavro-1.9.5.tar.gz", hash = "sha256:6419ebf45f88132a9945c51fe555d4f10bb97c236288ed01894f957c6f914553"}, ] [package.extras] @@ -2486,18 +2495,18 @@ docs = ["sphinx"] [[package]] name = "flask-sqlalchemy" -version = "3.0.5" +version = "3.1.1" description = "Add SQLAlchemy support to your Flask application." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "flask_sqlalchemy-3.0.5-py3-none-any.whl", hash = "sha256:cabb6600ddd819a9f859f36515bb1bd8e7dbf30206cc679d2b081dff9e383283"}, - {file = "flask_sqlalchemy-3.0.5.tar.gz", hash = "sha256:c5765e58ca145401b52106c0f46178569243c5da25556be2c231ecc60867c5b1"}, + {file = "flask_sqlalchemy-3.1.1-py3-none-any.whl", hash = "sha256:4ba4be7f419dc72f4efd8802d69974803c37259dd42f3913b0dcf75c9447e0a0"}, + {file = "flask_sqlalchemy-3.1.1.tar.gz", hash = "sha256:e4b68bb881802dda1a7d878b2fc84c06d1ee57fb40b874d3dc97dabfa36b8312"}, ] [package.dependencies] flask = ">=2.2.5" -sqlalchemy = ">=1.4.18" +sqlalchemy = ">=2.0.16" [[package]] name = "flatbuffers" @@ -3069,13 +3078,13 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-resource-manager" -version = "1.12.3" +version = "1.12.4" description = "Google Cloud Resource Manager API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-resource-manager-1.12.3.tar.gz", hash = "sha256:809851824119834e4f2310b2c4f38621c1d16b2bb14d5b9f132e69c79d355e7f"}, - {file = "google_cloud_resource_manager-1.12.3-py2.py3-none-any.whl", hash = "sha256:92be7d6959927b76d90eafc4028985c37975a46ded5466a018f02e8649e113d4"}, + {file = "google-cloud-resource-manager-1.12.4.tar.gz", hash = "sha256:3eda914a925e92465ef80faaab7e0f7a9312d486dd4e123d2c76e04bac688ff0"}, + {file = "google_cloud_resource_manager-1.12.4-py2.py3-none-any.whl", hash = "sha256:0b6663585f7f862166c0fb4c55fdda721fce4dc2dc1d5b52d03ee4bf2653a85f"}, ] [package.dependencies] @@ -3083,7 +3092,7 @@ google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extr google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev" grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" proto-plus = ">=1.22.3,<2.0.0dev" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" +protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev" [[package]] name = "google-cloud-storage" @@ -3514,120 +3523,105 @@ hyperframe = ">=6.0,<7" [[package]] name = "hiredis" -version = "2.3.2" +version = "3.0.0" description = "Python wrapper for hiredis" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "hiredis-2.3.2-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:742093f33d374098aa21c1696ac6e4874b52658c870513a297a89265a4d08fe5"}, - {file = "hiredis-2.3.2-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:9e14fb70ca4f7efa924f508975199353bf653f452e4ef0a1e47549e208f943d7"}, - {file = "hiredis-2.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d7302b4b17fcc1cc727ce84ded7f6be4655701e8d58744f73b09cb9ed2b13df"}, - {file = "hiredis-2.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed63e8b75c193c5e5a8288d9d7b011da076cc314fafc3bfd59ec1d8a750d48c8"}, - {file = "hiredis-2.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b4edee59dc089bc3948f4f6fba309f51aa2ccce63902364900aa0a553a85e97"}, - {file = "hiredis-2.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6481c3b7673a86276220140456c2a6fbfe8d1fb5c613b4728293c8634134824"}, - {file = "hiredis-2.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684840b014ce83541a087fcf2d48227196576f56ae3e944d4dfe14c0a3e0ccb7"}, - {file = "hiredis-2.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c4c0bcf786f0eac9593367b6279e9b89534e008edbf116dcd0de956524702c8"}, - {file = "hiredis-2.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66ab949424ac6504d823cba45c4c4854af5c59306a1531edb43b4dd22e17c102"}, - {file = "hiredis-2.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:322c668ee1c12d6c5750a4b1057e6b4feee2a75b3d25d630922a463cfe5e7478"}, - {file = "hiredis-2.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bfa73e3f163c6e8b2ec26f22285d717a5f77ab2120c97a2605d8f48b26950dac"}, - {file = "hiredis-2.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:7f39f28ffc65de577c3bc0c7615f149e35bc927802a0f56e612db9b530f316f9"}, - {file = "hiredis-2.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:55ce31bf4711da879b96d511208efb65a6165da4ba91cb3a96d86d5a8d9d23e6"}, - {file = "hiredis-2.3.2-cp310-cp310-win32.whl", hash = "sha256:3dd63d0bbbe75797b743f35d37a4cca7ca7ba35423a0de742ae2985752f20c6d"}, - {file = "hiredis-2.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:ea002656a8d974daaf6089863ab0a306962c8b715db6b10879f98b781a2a5bf5"}, - {file = "hiredis-2.3.2-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:adfbf2e9c38b77d0db2fb32c3bdaea638fa76b4e75847283cd707521ad2475ef"}, - {file = "hiredis-2.3.2-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:80b02d27864ebaf9b153d4b99015342382eeaed651f5591ce6f07e840307c56d"}, - {file = "hiredis-2.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd40d2e2f82a483de0d0a6dfd8c3895a02e55e5c9949610ecbded18188fd0a56"}, - {file = "hiredis-2.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfa904045d7cebfb0f01dad51352551cce1d873d7c3f80c7ded7d42f8cac8f89"}, - {file = "hiredis-2.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:28bd184b33e0dd6d65816c16521a4ba1ffbe9ff07d66873c42ea4049a62fed83"}, - {file = "hiredis-2.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f70481213373d44614148f0f2e38e7905be3f021902ae5167289413196de4ba4"}, - {file = "hiredis-2.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8797b528c1ff81eef06713623562b36db3dafa106b59f83a6468df788ff0d1"}, - {file = "hiredis-2.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:02fc71c8333586871602db4774d3a3e403b4ccf6446dc4603ec12df563127cee"}, - {file = "hiredis-2.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0da56915bda1e0a49157191b54d3e27689b70960f0685fdd5c415dacdee2fbed"}, - {file = "hiredis-2.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e2674a5a3168349435b08fa0b82998ed2536eb9acccf7087efe26e4cd088a525"}, - {file = "hiredis-2.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:dc1c3fd49930494a67dcec37d0558d99d84eca8eb3f03b17198424538f2608d7"}, - {file = "hiredis-2.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:14c7b43205e515f538a9defb4e411e0f0576caaeeda76bb9993ed505486f7562"}, - {file = "hiredis-2.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7bac7e02915b970c3723a7a7c5df4ba7a11a3426d2a3f181e041aa506a1ff028"}, - {file = "hiredis-2.3.2-cp311-cp311-win32.whl", hash = "sha256:63a090761ddc3c1f7db5e67aa4e247b4b3bb9890080bdcdadd1b5200b8b89ac4"}, - {file = "hiredis-2.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d226ab0306a5b8d408235cabe51d4bf3554c9e8a72d53ce0b3c5c84cf78881"}, - {file = "hiredis-2.3.2-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:5c614552c6bd1d0d907f448f75550f6b24fb56cbfce80c094908b7990cad9702"}, - {file = "hiredis-2.3.2-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9c431431abf55b64347ddc8df68b3ef840269cb0aa5bc2d26ad9506eb4b1b866"}, - {file = "hiredis-2.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a45857e87e9d2b005e81ddac9d815a33efd26ec67032c366629f023fe64fb415"}, - {file = "hiredis-2.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e138d141ec5a6ec800b6d01ddc3e5561ce1c940215e0eb9960876bfde7186aae"}, - {file = "hiredis-2.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:387f655444d912a963ab68abf64bf6e178a13c8e4aa945cb27388fd01a02e6f1"}, - {file = "hiredis-2.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4852f4bf88f0e2d9bdf91279892f5740ed22ae368335a37a52b92a5c88691140"}, - {file = "hiredis-2.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d711c107e83117129b7f8bd08e9820c43ceec6204fff072a001fd82f6d13db9f"}, - {file = "hiredis-2.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92830c16885f29163e1c2da1f3c1edb226df1210ec7e8711aaabba3dd0d5470a"}, - {file = "hiredis-2.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:16b01d9ceae265d4ab9547be0cd628ecaff14b3360357a9d30c029e5ae8b7e7f"}, - {file = "hiredis-2.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:5986fb5f380169270a0293bebebd95466a1c85010b4f1afc2727e4d17c452512"}, - {file = "hiredis-2.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:49532d7939cc51f8e99efc326090c54acf5437ed88b9c904cc8015b3c4eda9c9"}, - {file = "hiredis-2.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:8f34801b251ca43ad70691fb08b606a2e55f06b9c9fb1fc18fd9402b19d70f7b"}, - {file = "hiredis-2.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7298562a49d95570ab1c7fc4051e72824c6a80e907993a21a41ba204223e7334"}, - {file = "hiredis-2.3.2-cp312-cp312-win32.whl", hash = "sha256:e1d86b75de787481b04d112067a4033e1ecfda2a060e50318a74e4e1c9b2948c"}, - {file = "hiredis-2.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:6dbfe1887ffa5cf3030451a56a8f965a9da2fa82b7149357752b67a335a05fc6"}, - {file = "hiredis-2.3.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:4fc242e9da4af48714199216eb535b61e8f8d66552c8819e33fc7806bd465a09"}, - {file = "hiredis-2.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e81aa4e9a1fcf604c8c4b51aa5d258e195a6ba81efe1da82dea3204443eba01c"}, - {file = "hiredis-2.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:419780f8583ddb544ffa86f9d44a7fcc183cd826101af4e5ffe535b6765f5f6b"}, - {file = "hiredis-2.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6871306d8b98a15e53a5f289ec1106a3a1d43e7ab6f4d785f95fcef9a7bd9504"}, - {file = "hiredis-2.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88cb0b35b63717ef1e41d62f4f8717166f7c6245064957907cfe177cc144357c"}, - {file = "hiredis-2.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c490191fa1218851f8a80c5a21a05a6f680ac5aebc2e688b71cbfe592f8fec6"}, - {file = "hiredis-2.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:4baf4b579b108062e91bd2a991dc98b9dc3dc06e6288db2d98895eea8acbac22"}, - {file = "hiredis-2.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e627d8ef5e100556e09fb44c9571a432b10e11596d3c4043500080ca9944a91a"}, - {file = "hiredis-2.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:ba3dc0af0def8c21ce7d903c59ea1e8ec4cb073f25ece9edaec7f92a286cd219"}, - {file = "hiredis-2.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:56e9b7d6051688ca94e68c0c8a54a243f8db841911b683cedf89a29d4de91509"}, - {file = "hiredis-2.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:380e029bb4b1d34cf560fcc8950bf6b57c2ef0c9c8b7c7ac20b7c524a730fadd"}, - {file = "hiredis-2.3.2-cp37-cp37m-win32.whl", hash = "sha256:948d9f2ca7841794dd9b204644963a4bcd69ced4e959b0d4ecf1b8ce994a6daa"}, - {file = "hiredis-2.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:cfa67afe2269b2d203cd1389c00c5bc35a287cd57860441fb0e53b371ea6a029"}, - {file = "hiredis-2.3.2-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:bcbe47da0aebc00a7cfe3ebdcff0373b86ce2b1856251c003e3d69c9db44b5a7"}, - {file = "hiredis-2.3.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f2c9c0d910dd3f7df92f0638e7f65d8edd7f442203caf89c62fc79f11b0b73f8"}, - {file = "hiredis-2.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:01b6c24c0840ac7afafbc4db236fd55f56a9a0919a215c25a238f051781f4772"}, - {file = "hiredis-2.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1f567489f422d40c21e53212a73bef4638d9f21043848150f8544ef1f3a6ad1"}, - {file = "hiredis-2.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:28adecb308293e705e44087a1c2d557a816f032430d8a2a9bb7873902a1c6d48"}, - {file = "hiredis-2.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:27e9619847e9dc70b14b1ad2d0fb4889e7ca18996585c3463cff6c951fd6b10b"}, - {file = "hiredis-2.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a0026cfbf29f07649b0e34509091a2a6016ff8844b127de150efce1c3aff60b"}, - {file = "hiredis-2.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9de7586522e5da6bee83c9cf0dcccac0857a43249cb4d721a2e312d98a684d1"}, - {file = "hiredis-2.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e58494f282215fc461b06709e9a195a24c12ba09570f25bdf9efb036acc05101"}, - {file = "hiredis-2.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:de3a32b4b76d46f1eb42b24a918d51d8ca52411a381748196241d59a895f7c5c"}, - {file = "hiredis-2.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1979334ccab21a49c544cd1b8d784ffb2747f99a51cb0bd0976eebb517628382"}, - {file = "hiredis-2.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:0c0773266e1c38a06e7593bd08870ac1503f5f0ce0f5c63f2b4134b090b5d6a4"}, - {file = "hiredis-2.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bd1cee053416183adcc8e6134704c46c60c3f66b8faaf9e65bf76191ca59a2f7"}, - {file = "hiredis-2.3.2-cp38-cp38-win32.whl", hash = "sha256:5341ce3d01ef3c7418a72e370bf028c7aeb16895e79e115fe4c954fff990489e"}, - {file = "hiredis-2.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:8fc7197ff33047ce43a67851ccf190acb5b05c52fd4a001bb55766358f04da68"}, - {file = "hiredis-2.3.2-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:f47775e27388b58ce52f4f972f80e45b13c65113e9e6b6bf60148f893871dc9b"}, - {file = "hiredis-2.3.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:9412a06b8a8e09abd6313d96864b6d7713c6003a365995a5c70cfb9209df1570"}, - {file = "hiredis-2.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3020b60e3fc96d08c2a9b011f1c2e2a6bdcc09cb55df93c509b88be5cb791df"}, - {file = "hiredis-2.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:53d0f2c59bce399b8010a21bc779b4f8c32d0f582b2284ac8c98dc7578b27bc4"}, - {file = "hiredis-2.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:57c0d0c7e308ed5280a4900d4468bbfec51f0e1b4cde1deae7d4e639bc6b7766"}, - {file = "hiredis-2.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1d63318ca189fddc7e75f6a4af8eae9c0545863619fb38cfba5f43e81280b286"}, - {file = "hiredis-2.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e741ffe4e2db78a1b9dd6e5d29678ce37fbaaf65dfe132e5b82a794413302ef1"}, - {file = "hiredis-2.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb98038ccd368e0d88bd92ee575c58cfaf33e77f788c36b2a89a84ee1936dc6b"}, - {file = "hiredis-2.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:eae62ed60d53b3561148bcd8c2383e430af38c0deab9f2dd15f8874888ffd26f"}, - {file = "hiredis-2.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ca33c175c1cf60222d9c6d01c38fc17ec3a484f32294af781de30226b003e00f"}, - {file = "hiredis-2.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0c5f6972d2bdee3cd301d5c5438e31195cf1cabf6fd9274491674d4ceb46914d"}, - {file = "hiredis-2.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:a6b54dabfaa5dbaa92f796f0c32819b4636e66aa8e9106c3d421624bd2a2d676"}, - {file = "hiredis-2.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e96cd35df012a17c87ae276196ea8f215e77d6eeca90709eb03999e2d5e3fd8a"}, - {file = "hiredis-2.3.2-cp39-cp39-win32.whl", hash = "sha256:63b99b5ea9fe4f21469fb06a16ca5244307678636f11917359e3223aaeca0b67"}, - {file = "hiredis-2.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:a50c8af811b35b8a43b1590cf890b61ff2233225257a3cad32f43b3ec7ff1b9f"}, - {file = "hiredis-2.3.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7e8bf4444b09419b77ce671088db9f875b26720b5872d97778e2545cd87dba4a"}, - {file = "hiredis-2.3.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bd42d0d45ea47a2f96babd82a659fbc60612ab9423a68e4a8191e538b85542a"}, - {file = "hiredis-2.3.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80441b55edbef868e2563842f5030982b04349408396e5ac2b32025fb06b5212"}, - {file = "hiredis-2.3.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec444ab8f27562a363672d6a7372bc0700a1bdc9764563c57c5f9efa0e592b5f"}, - {file = "hiredis-2.3.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f9f606e810858207d4b4287b4ef0dc622c2aa469548bf02b59dcc616f134f811"}, - {file = "hiredis-2.3.2-pp37-pypy37_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c3dde4ca00fe9eee3b76209711f1941bb86db42b8a75d7f2249ff9dfc026ab0e"}, - {file = "hiredis-2.3.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4dd676107a1d3c724a56a9d9db38166ad4cf44f924ee701414751bd18a784a0"}, - {file = "hiredis-2.3.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce42649e2676ad783186264d5ffc788a7612ecd7f9effb62d51c30d413a3eefe"}, - {file = "hiredis-2.3.2-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e3f8b1733078ac663dad57e20060e16389a60ab542f18a97931f3a2a2dd64a4"}, - {file = "hiredis-2.3.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:532a84a82156a82529ec401d1c25d677c6543c791e54a263aa139541c363995f"}, - {file = "hiredis-2.3.2-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4d59f88c4daa36b8c38e59ac7bffed6f5d7f68eaccad471484bf587b28ccc478"}, - {file = "hiredis-2.3.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a91a14dd95e24dc078204b18b0199226ee44644974c645dc54ee7b00c3157330"}, - {file = "hiredis-2.3.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb777a38797c8c7df0444533119570be18d1a4ce5478dffc00c875684df7bfcb"}, - {file = "hiredis-2.3.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d47c915897a99d0d34a39fad4be97b4b709ab3d0d3b779ebccf2b6024a8c681e"}, - {file = "hiredis-2.3.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:333b5e04866758b11bda5f5315b4e671d15755fc6ed3b7969721bc6311d0ee36"}, - {file = "hiredis-2.3.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c8937f1100435698c18e4da086968c4b5d70e86ea718376f833475ab3277c9aa"}, - {file = "hiredis-2.3.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa45f7d771094b8145af10db74704ab0f698adb682fbf3721d8090f90e42cc49"}, - {file = "hiredis-2.3.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33d5ebc93c39aed4b5bc769f8ce0819bc50e74bb95d57a35f838f1c4378978e0"}, - {file = "hiredis-2.3.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a797d8c7df9944314d309b0d9e1b354e2fa4430a05bb7604da13b6ad291bf959"}, - {file = "hiredis-2.3.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:e15a408f71a6c8c87b364f1f15a6cd9c1baca12bbc47a326ac8ab99ec7ad3c64"}, - {file = "hiredis-2.3.2.tar.gz", hash = "sha256:733e2456b68f3f126ddaf2cd500a33b25146c3676b97ea843665717bda0c5d43"}, + {file = "hiredis-3.0.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:4b182791c41c5eb1d9ed736f0ff81694b06937ca14b0d4dadde5dadba7ff6dae"}, + {file = "hiredis-3.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:13c275b483a052dd645eb2cb60d6380f1f5215e4c22d6207e17b86be6dd87ffa"}, + {file = "hiredis-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1018cc7f12824506f165027eabb302735b49e63af73eb4d5450c66c88f47026"}, + {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83a29cc7b21b746cb6a480189e49f49b2072812c445e66a9e38d2004d496b81c"}, + {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e241fab6332e8fb5f14af00a4a9c6aefa22f19a336c069b7ddbf28ef8341e8d6"}, + {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1fb8de899f0145d6c4d5d4bd0ee88a78eb980a7ffabd51e9889251b8f58f1785"}, + {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b23291951959141173eec10f8573538e9349fa27f47a0c34323d1970bf891ee5"}, + {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e421ac9e4b5efc11705a0d5149e641d4defdc07077f748667f359e60dc904420"}, + {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:77c8006c12154c37691b24ff293c077300c22944018c3ff70094a33e10c1d795"}, + {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:41afc0d3c18b59eb50970479a9c0e5544fb4b95e3a79cf2fbaece6ddefb926fe"}, + {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:04ccae6dcd9647eae6025425ab64edb4d79fde8b9e6e115ebfabc6830170e3b2"}, + {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:fe91d62b0594db5ea7d23fc2192182b1a7b6973f628a9b8b2e0a42a2be721ac6"}, + {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:99516d99316062824a24d145d694f5b0d030c80da693ea6f8c4ecf71a251d8bb"}, + {file = "hiredis-3.0.0-cp310-cp310-win32.whl", hash = "sha256:562eaf820de045eb487afaa37e6293fe7eceb5b25e158b5a1974b7e40bf04543"}, + {file = "hiredis-3.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:a1c81c89ed765198da27412aa21478f30d54ef69bf5e4480089d9c3f77b8f882"}, + {file = "hiredis-3.0.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:4664dedcd5933364756d7251a7ea86d60246ccf73a2e00912872dacbfcef8978"}, + {file = "hiredis-3.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:47de0bbccf4c8a9f99d82d225f7672b9dd690d8fd872007b933ef51a302c9fa6"}, + {file = "hiredis-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e43679eca508ba8240d016d8cca9d27342d70184773c15bea78a23c87a1922f1"}, + {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13c345e7278c210317e77e1934b27b61394fee0dec2e8bd47e71570900f75823"}, + {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00018f22f38530768b73ea86c11f47e8d4df65facd4e562bd78773bd1baef35e"}, + {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ea3a86405baa8eb0d3639ced6926ad03e07113de54cb00fd7510cb0db76a89d"}, + {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c073848d2b1d5561f3903879ccf4e1a70c9b1e7566c7bdcc98d082fa3e7f0a1d"}, + {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a8dffb5f5b3415a4669d25de48b617fd9d44b0bccfc4c2ab24b06406ecc9ecb"}, + {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:22c17c96143c2a62dfd61b13803bc5de2ac526b8768d2141c018b965d0333b66"}, + {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c3ece960008dab66c6b8bb3a1350764677ee7c74ccd6270aaf1b1caf9ccebb46"}, + {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f75999ae00a920f7dce6ecae76fa5e8674a3110e5a75f12c7a2c75ae1af53396"}, + {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e069967cbd5e1900aafc4b5943888f6d34937fc59bf8918a1a546cb729b4b1e4"}, + {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0aacc0a78e1d94d843a6d191f224a35893e6bdfeb77a4a89264155015c65f126"}, + {file = "hiredis-3.0.0-cp311-cp311-win32.whl", hash = "sha256:719c32147ba29528cb451f037bf837dcdda4ff3ddb6cdb12c4216b0973174718"}, + {file = "hiredis-3.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:bdc144d56333c52c853c31b4e2e52cfbdb22d3da4374c00f5f3d67c42158970f"}, + {file = "hiredis-3.0.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:484025d2eb8f6348f7876fc5a2ee742f568915039fcb31b478fd5c242bb0fe3a"}, + {file = "hiredis-3.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:fcdb552ffd97151dab8e7bc3ab556dfa1512556b48a367db94b5c20253a35ee1"}, + {file = "hiredis-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bb6f9fd92f147ba11d338ef5c68af4fd2908739c09e51f186e1d90958c68cc1"}, + {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa86bf9a0ed339ec9e8a9a9d0ae4dccd8671625c83f9f9f2640729b15e07fbfd"}, + {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e194a0d5df9456995d8f510eab9f529213e7326af6b94770abf8f8b7952ddcaa"}, + {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c8a1df39d74ec507d79c7a82c8063eee60bf80537cdeee652f576059b9cdd15c"}, + {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f91456507427ba36fd81b2ca11053a8e112c775325acc74e993201ea912d63e9"}, + {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9862db92ef67a8a02e0d5370f07d380e14577ecb281b79720e0d7a89aedb9ee5"}, + {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d10fcd9e0eeab835f492832b2a6edb5940e2f1230155f33006a8dfd3bd2c94e4"}, + {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:48727d7d405d03977d01885f317328dc21d639096308de126c2c4e9950cbd3c9"}, + {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e0bb6102ebe2efecf8a3292c6660a0e6fac98176af6de67f020bea1c2343717"}, + {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:df274e3abb4df40f4c7274dd3e587dfbb25691826c948bc98d5fead019dfb001"}, + {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:034925b5fb514f7b11aac38cd55b3fd7e9d3af23bd6497f3f20aa5b8ba58e232"}, + {file = "hiredis-3.0.0-cp312-cp312-win32.whl", hash = "sha256:120f2dda469b28d12ccff7c2230225162e174657b49cf4cd119db525414ae281"}, + {file = "hiredis-3.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:e584fe5f4e6681d8762982be055f1534e0170f6308a7a90f58d737bab12ff6a8"}, + {file = "hiredis-3.0.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:122171ff47d96ed8dd4bba6c0e41d8afaba3e8194949f7720431a62aa29d8895"}, + {file = "hiredis-3.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:ba9fc605ac558f0de67463fb588722878641e6fa1dabcda979e8e69ff581d0bd"}, + {file = "hiredis-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a631e2990b8be23178f655cae8ac6c7422af478c420dd54e25f2e26c29e766f1"}, + {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63482db3fadebadc1d01ad33afa6045ebe2ea528eb77ccaabd33ee7d9c2bad48"}, + {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f669212c390eebfbe03c4e20181f5970b82c5d0a0ad1df1785f7ffbe7d61150"}, + {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a49ef161739f8018c69b371528bdb47d7342edfdee9ddc75a4d8caddf45a6e"}, + {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98a152052b8878e5e43a2e3a14075218adafc759547c98668a21e9485882696c"}, + {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50a196af0ce657fcde9bf8a0bbe1032e22c64d8fcec2bc926a35e7ff68b3a166"}, + {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f2f312eef8aafc2255e3585dcf94d5da116c43ef837db91db9ecdc1bc930072d"}, + {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:6ca41fa40fa019cde42c21add74aadd775e71458051a15a352eabeb12eb4d084"}, + {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:6eecb343c70629f5af55a8b3e53264e44fa04e155ef7989de13668a0cb102a90"}, + {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:c3fdad75e7837a475900a1d3a5cc09aa024293c3b0605155da2d42f41bc0e482"}, + {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:8854969e7480e8d61ed7549eb232d95082a743e94138d98d7222ba4e9f7ecacd"}, + {file = "hiredis-3.0.0-cp38-cp38-win32.whl", hash = "sha256:f114a6c86edbf17554672b050cce72abf489fe58d583c7921904d5f1c9691605"}, + {file = "hiredis-3.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:7d99b91e42217d7b4b63354b15b41ce960e27d216783e04c4a350224d55842a4"}, + {file = "hiredis-3.0.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:4c6efcbb5687cf8d2aedcc2c3ed4ac6feae90b8547427d417111194873b66b06"}, + {file = "hiredis-3.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5b5cff42a522a0d81c2ae7eae5e56d0ee7365e0c4ad50c4de467d8957aff4414"}, + {file = "hiredis-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:82f794d564f4bc76b80c50b03267fe5d6589e93f08e66b7a2f674faa2fa76ebc"}, + {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7a4c1791d7aa7e192f60fe028ae409f18ccdd540f8b1e6aeb0df7816c77e4a4"}, + {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2537b2cd98192323fce4244c8edbf11f3cac548a9d633dbbb12b48702f379f4"}, + {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fed69bbaa307040c62195a269f82fc3edf46b510a17abb6b30a15d7dab548df"}, + {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:869f6d5537d243080f44253491bb30aa1ec3c21754003b3bddeadedeb65842b0"}, + {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d435ae89073d7cd51e6b6bf78369c412216261c9c01662e7008ff00978153729"}, + {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:204b79b30a0e6be0dc2301a4d385bb61472809f09c49f400497f1cdd5a165c66"}, + {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3ea635101b739c12effd189cc19b2671c268abb03013fd1f6321ca29df3ca625"}, + {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:f359175197fd833c8dd7a8c288f1516be45415bb5c939862ab60c2918e1e1943"}, + {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac6d929cb33dd12ad3424b75725975f0a54b5b12dbff95f2a2d660c510aa106d"}, + {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:100431e04d25a522ef2c3b94f294c4219c4de3bfc7d557b6253296145a144c11"}, + {file = "hiredis-3.0.0-cp39-cp39-win32.whl", hash = "sha256:e1a9c14ae9573d172dc050a6f63a644457df5d01ec4d35a6a0f097f812930f83"}, + {file = "hiredis-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:54a6dd7b478e6eb01ce15b3bb5bf771e108c6c148315bf194eb2ab776a3cac4d"}, + {file = "hiredis-3.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:50da7a9edf371441dfcc56288d790985ee9840d982750580710a9789b8f4a290"}, + {file = "hiredis-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9b285ef6bf1581310b0d5e8f6ce64f790a1c40e89c660e1320b35f7515433672"}, + {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dcfa684966f25b335072115de2f920228a3c2caf79d4bfa2b30f6e4f674a948"}, + {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a41be8af1fd78ca97bc948d789a09b730d1e7587d07ca53af05758f31f4b985d"}, + {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:038756db735e417ab36ee6fd7725ce412385ed2bd0767e8179a4755ea11b804f"}, + {file = "hiredis-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:fcecbd39bd42cef905c0b51c9689c39d0cc8b88b1671e7f40d4fb213423aef3a"}, + {file = "hiredis-3.0.0-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:a131377493a59fb0f5eaeb2afd49c6540cafcfba5b0b3752bed707be9e7c4eaf"}, + {file = "hiredis-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:3d22c53f0ec5c18ecb3d92aa9420563b1c5d657d53f01356114978107b00b860"}, + {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8a91e9520fbc65a799943e5c970ffbcd67905744d8becf2e75f9f0a5e8414f0"}, + {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3dc8043959b50141df58ab4f398e8ae84c6f9e673a2c9407be65fc789138f4a6"}, + {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:51b99cfac514173d7b8abdfe10338193e8a0eccdfe1870b646009d2fb7cbe4b5"}, + {file = "hiredis-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:fa1fcad89d8a41d8dc10b1e54951ec1e161deabd84ed5a2c95c3c7213bdb3514"}, + {file = "hiredis-3.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:898636a06d9bf575d2c594129085ad6b713414038276a4bfc5db7646b8a5be78"}, + {file = "hiredis-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:466f836dbcf86de3f9692097a7a01533dc9926986022c6617dc364a402b265c5"}, + {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23142a8af92a13fc1e3f2ca1d940df3dcf2af1d176be41fe8d89e30a837a0b60"}, + {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:793c80a3d6b0b0e8196a2d5de37a08330125668c8012922685e17aa9108c33ac"}, + {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:467d28112c7faa29b7db743f40803d927c8591e9da02b6ce3d5fadc170a542a2"}, + {file = "hiredis-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:dc384874a719c767b50a30750f937af18842ee5e288afba95a5a3ed703b1515a"}, + {file = "hiredis-3.0.0.tar.gz", hash = "sha256:fed8581ae26345dea1f1e0d1a96e05041a727a45e7d8d459164583e23c6ac441"}, ] [[package]] @@ -4184,13 +4178,13 @@ six = "*" [[package]] name = "langfuse" -version = "2.38.0" +version = "2.39.3" description = "A client library for accessing langfuse" optional = false python-versions = "<4.0,>=3.8.1" files = [ - {file = "langfuse-2.38.0-py3-none-any.whl", hash = "sha256:9e81757b88d26acb8949dbd1d25153df49f4ce8da39e5347c4aa23c3b9d4559c"}, - {file = "langfuse-2.38.0.tar.gz", hash = "sha256:0022a805a167d2e436759ac48b5efb45db2910b02e2f934e47eaf481ad3b21f5"}, + {file = "langfuse-2.39.3-py3-none-any.whl", hash = "sha256:24b12cbb23f866b22706c1ea9631781f99fe37b0b15889d241198c4d1c07516b"}, + {file = "langfuse-2.39.3.tar.gz", hash = "sha256:4d2df8f9344572370703db103ddf97176df518699593254e6d6c2b8ca3bf2f12"}, ] [package.dependencies] @@ -4209,13 +4203,13 @@ openai = ["openai (>=0.27.8)"] [[package]] name = "langsmith" -version = "0.1.83" +version = "0.1.93" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.83-py3-none-any.whl", hash = "sha256:f54d8cd8479b648b6339f3f735d19292c3516d080f680933ecdca3eab4b67ed3"}, - {file = "langsmith-0.1.83.tar.gz", hash = "sha256:5cdd947212c8ad19adb992c06471c860185a777daa6859bb47150f90daf64bf3"}, + {file = "langsmith-0.1.93-py3-none-any.whl", hash = "sha256:811210b9d5f108f36431bd7b997eb9476a9ecf5a2abd7ddbb606c1cdcf0f43ce"}, + {file = "langsmith-0.1.93.tar.gz", hash = "sha256:285b6ad3a54f50fa8eb97b5f600acc57d0e37e139dd8cf2111a117d0435ba9b4"}, ] [package.dependencies] @@ -4761,13 +4755,13 @@ tests = ["pytest (>=4.6)"] [[package]] name = "msal" -version = "1.29.0" +version = "1.30.0" description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." optional = false python-versions = ">=3.7" files = [ - {file = "msal-1.29.0-py3-none-any.whl", hash = "sha256:6b301e63f967481f0cc1a3a3bac0cf322b276855bc1b0955468d9deb3f33d511"}, - {file = "msal-1.29.0.tar.gz", hash = "sha256:8f6725f099752553f9b2fe84125e2a5ebe47b49f92eacca33ebedd3a9ebaae25"}, + {file = "msal-1.30.0-py3-none-any.whl", hash = "sha256:423872177410cb61683566dc3932db7a76f661a5d2f6f52f02a047f101e1c1de"}, + {file = "msal-1.30.0.tar.gz", hash = "sha256:b4bf00850092e465157d814efa24a18f788284c9a479491024d62903085ea2fb"}, ] [package.dependencies] @@ -5674,12 +5668,12 @@ xml = ["lxml (>=4.9.2)"] [[package]] name = "peewee" -version = "3.17.5" +version = "3.17.6" description = "a little orm" optional = false python-versions = "*" files = [ - {file = "peewee-3.17.5.tar.gz", hash = "sha256:e1b6a64192207fd3ddb4e1188054820f42aef0aadfa749e3981af3c119a76420"}, + {file = "peewee-3.17.6.tar.gz", hash = "sha256:cea5592c6f4da1592b7cff8eaf655be6648a1f5857469e30037bf920c03fb8fb"}, ] [[package]] @@ -5830,13 +5824,13 @@ type = ["mypy (>=1.8)"] [[package]] name = "plotly" -version = "5.22.0" +version = "5.23.0" description = "An open-source, interactive data visualization library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "plotly-5.22.0-py3-none-any.whl", hash = "sha256:68fc1901f098daeb233cc3dd44ec9dc31fb3ca4f4e53189344199c43496ed006"}, - {file = "plotly-5.22.0.tar.gz", hash = "sha256:859fdadbd86b5770ae2466e542b761b247d1c6b49daed765b95bb8c7063e7469"}, + {file = "plotly-5.23.0-py3-none-any.whl", hash = "sha256:76cbe78f75eddc10c56f5a4ee3e7ccaade7c0a57465546f02098c0caed6c2d1a"}, + {file = "plotly-5.23.0.tar.gz", hash = "sha256:89e57d003a116303a34de6700862391367dd564222ab71f8531df70279fc0193"}, ] [package.dependencies] @@ -5871,13 +5865,13 @@ files = [ [[package]] name = "portalocker" -version = "2.10.0" +version = "2.10.1" description = "Wraps the portalocker recipe for easy usage" optional = false python-versions = ">=3.8" files = [ - {file = "portalocker-2.10.0-py3-none-any.whl", hash = "sha256:48944147b2cd42520549bc1bb8fe44e220296e56f7c3d551bc6ecce69d9b0de1"}, - {file = "portalocker-2.10.0.tar.gz", hash = "sha256:49de8bc0a2f68ca98bf9e219c81a3e6b27097c7bf505a87c5a112ce1aaeb9b81"}, + {file = "portalocker-2.10.1-py3-none-any.whl", hash = "sha256:53a5984ebc86a025552264b459b46a2086e269b21823cb572f8f28ee759e45bf"}, + {file = "portalocker-2.10.1.tar.gz", hash = "sha256:ef1bf844e878ab08aee7e40184156e1151f228f103aa5c6bd0724cc330960f8f"}, ] [package.dependencies] @@ -6056,52 +6050,42 @@ files = [ [[package]] name = "pyarrow" -version = "16.1.0" +version = "17.0.0" description = "Python library for Apache Arrow" optional = false python-versions = ">=3.8" files = [ - {file = "pyarrow-16.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:17e23b9a65a70cc733d8b738baa6ad3722298fa0c81d88f63ff94bf25eaa77b9"}, - {file = "pyarrow-16.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4740cc41e2ba5d641071d0ab5e9ef9b5e6e8c7611351a5cb7c1d175eaf43674a"}, - {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98100e0268d04e0eec47b73f20b39c45b4006f3c4233719c3848aa27a03c1aef"}, - {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f68f409e7b283c085f2da014f9ef81e885d90dcd733bd648cfba3ef265961848"}, - {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:a8914cd176f448e09746037b0c6b3a9d7688cef451ec5735094055116857580c"}, - {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:48be160782c0556156d91adbdd5a4a7e719f8d407cb46ae3bb4eaee09b3111bd"}, - {file = "pyarrow-16.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:9cf389d444b0f41d9fe1444b70650fea31e9d52cfcb5f818b7888b91b586efff"}, - {file = "pyarrow-16.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:d0ebea336b535b37eee9eee31761813086d33ed06de9ab6fc6aaa0bace7b250c"}, - {file = "pyarrow-16.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e73cfc4a99e796727919c5541c65bb88b973377501e39b9842ea71401ca6c1c"}, - {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf9251264247ecfe93e5f5a0cd43b8ae834f1e61d1abca22da55b20c788417f6"}, - {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddf5aace92d520d3d2a20031d8b0ec27b4395cab9f74e07cc95edf42a5cc0147"}, - {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:25233642583bf658f629eb230b9bb79d9af4d9f9229890b3c878699c82f7d11e"}, - {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a33a64576fddfbec0a44112eaf844c20853647ca833e9a647bfae0582b2ff94b"}, - {file = "pyarrow-16.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:185d121b50836379fe012753cf15c4ba9638bda9645183ab36246923875f8d1b"}, - {file = "pyarrow-16.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:2e51ca1d6ed7f2e9d5c3c83decf27b0d17bb207a7dea986e8dc3e24f80ff7d6f"}, - {file = "pyarrow-16.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06ebccb6f8cb7357de85f60d5da50e83507954af617d7b05f48af1621d331c9a"}, - {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b04707f1979815f5e49824ce52d1dceb46e2f12909a48a6a753fe7cafbc44a0c"}, - {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d32000693deff8dc5df444b032b5985a48592c0697cb6e3071a5d59888714e2"}, - {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:8785bb10d5d6fd5e15d718ee1d1f914fe768bf8b4d1e5e9bf253de8a26cb1628"}, - {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:e1369af39587b794873b8a307cc6623a3b1194e69399af0efd05bb202195a5a7"}, - {file = "pyarrow-16.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:febde33305f1498f6df85e8020bca496d0e9ebf2093bab9e0f65e2b4ae2b3444"}, - {file = "pyarrow-16.1.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b5f5705ab977947a43ac83b52ade3b881eb6e95fcc02d76f501d549a210ba77f"}, - {file = "pyarrow-16.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0d27bf89dfc2576f6206e9cd6cf7a107c9c06dc13d53bbc25b0bd4556f19cf5f"}, - {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d07de3ee730647a600037bc1d7b7994067ed64d0eba797ac74b2bc77384f4c2"}, - {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbef391b63f708e103df99fbaa3acf9f671d77a183a07546ba2f2c297b361e83"}, - {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:19741c4dbbbc986d38856ee7ddfdd6a00fc3b0fc2d928795b95410d38bb97d15"}, - {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:f2c5fb249caa17b94e2b9278b36a05ce03d3180e6da0c4c3b3ce5b2788f30eed"}, - {file = "pyarrow-16.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:e6b6d3cd35fbb93b70ade1336022cc1147b95ec6af7d36906ca7fe432eb09710"}, - {file = "pyarrow-16.1.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:18da9b76a36a954665ccca8aa6bd9f46c1145f79c0bb8f4f244f5f8e799bca55"}, - {file = "pyarrow-16.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99f7549779b6e434467d2aa43ab2b7224dd9e41bdde486020bae198978c9e05e"}, - {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f07fdffe4fd5b15f5ec15c8b64584868d063bc22b86b46c9695624ca3505b7b4"}, - {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddfe389a08ea374972bd4065d5f25d14e36b43ebc22fc75f7b951f24378bf0b5"}, - {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:3b20bd67c94b3a2ea0a749d2a5712fc845a69cb5d52e78e6449bbd295611f3aa"}, - {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:ba8ac20693c0bb0bf4b238751d4409e62852004a8cf031c73b0e0962b03e45e3"}, - {file = "pyarrow-16.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:31a1851751433d89a986616015841977e0a188662fcffd1a5677453f1df2de0a"}, - {file = "pyarrow-16.1.0.tar.gz", hash = "sha256:15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315"}, + {file = "pyarrow-17.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a5c8b238d47e48812ee577ee20c9a2779e6a5904f1708ae240f53ecbee7c9f07"}, + {file = "pyarrow-17.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db023dc4c6cae1015de9e198d41250688383c3f9af8f565370ab2b4cb5f62655"}, + {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da1e060b3876faa11cee287839f9cc7cdc00649f475714b8680a05fd9071d545"}, + {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c06d4624c0ad6674364bb46ef38c3132768139ddec1c56582dbac54f2663e2"}, + {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:fa3c246cc58cb5a4a5cb407a18f193354ea47dd0648194e6265bd24177982fe8"}, + {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:f7ae2de664e0b158d1607699a16a488de3d008ba99b3a7aa5de1cbc13574d047"}, + {file = "pyarrow-17.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:5984f416552eea15fd9cee03da53542bf4cddaef5afecefb9aa8d1010c335087"}, + {file = "pyarrow-17.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:1c8856e2ef09eb87ecf937104aacfa0708f22dfeb039c363ec99735190ffb977"}, + {file = "pyarrow-17.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e19f569567efcbbd42084e87f948778eb371d308e137a0f97afe19bb860ccb3"}, + {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b244dc8e08a23b3e352899a006a26ae7b4d0da7bb636872fa8f5884e70acf15"}, + {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b72e87fe3e1db343995562f7fff8aee354b55ee83d13afba65400c178ab2597"}, + {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dc5c31c37409dfbc5d014047817cb4ccd8c1ea25d19576acf1a001fe07f5b420"}, + {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e3343cb1e88bc2ea605986d4b94948716edc7a8d14afd4e2c097232f729758b4"}, + {file = "pyarrow-17.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a27532c38f3de9eb3e90ecab63dfda948a8ca859a66e3a47f5f42d1e403c4d03"}, + {file = "pyarrow-17.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9b8a823cea605221e61f34859dcc03207e52e409ccf6354634143e23af7c8d22"}, + {file = "pyarrow-17.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1e70de6cb5790a50b01d2b686d54aaf73da01266850b05e3af2a1bc89e16053"}, + {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0071ce35788c6f9077ff9ecba4858108eebe2ea5a3f7cf2cf55ebc1dbc6ee24a"}, + {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:757074882f844411fcca735e39aae74248a1531367a7c80799b4266390ae51cc"}, + {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9ba11c4f16976e89146781a83833df7f82077cdab7dc6232c897789343f7891a"}, + {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b0c6ac301093b42d34410b187bba560b17c0330f64907bfa4f7f7f2444b0cf9b"}, + {file = "pyarrow-17.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:392bc9feabc647338e6c89267635e111d71edad5fcffba204425a7c8d13610d7"}, + {file = "pyarrow-17.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:af5ff82a04b2171415f1410cff7ebb79861afc5dae50be73ce06d6e870615204"}, + {file = "pyarrow-17.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:edca18eaca89cd6382dfbcff3dd2d87633433043650c07375d095cd3517561d8"}, ] [package.dependencies] numpy = ">=1.16.6" +[package.extras] +test = ["cffi", "hypothesis", "pandas", "pytest", "pytz"] + [[package]] name = "pyasn1" version = "0.6.0" @@ -6181,109 +6165,122 @@ files = [ [[package]] name = "pydantic" -version = "2.7.4" +version = "2.8.2" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.7.4-py3-none-any.whl", hash = "sha256:ee8538d41ccb9c0a9ad3e0e5f07bf15ed8015b481ced539a1759d8cc89ae90d0"}, - {file = "pydantic-2.7.4.tar.gz", hash = "sha256:0c84efd9548d545f63ac0060c1e4d39bb9b14db8b3c0652338aecc07b5adec52"}, + {file = "pydantic-2.8.2-py3-none-any.whl", hash = "sha256:73ee9fddd406dc318b885c7a2eab8a6472b68b8fb5ba8150949fc3db939f23c8"}, + {file = "pydantic-2.8.2.tar.gz", hash = "sha256:6f62c13d067b0755ad1c21a34bdd06c0c12625a22b0fc09c6b149816604f7c2a"}, ] [package.dependencies] annotated-types = ">=0.4.0" -pydantic-core = "2.18.4" -typing-extensions = ">=4.6.1" +pydantic-core = "2.20.1" +typing-extensions = [ + {version = ">=4.6.1", markers = "python_version < \"3.13\""}, + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, +] [package.extras] email = ["email-validator (>=2.0.0)"] [[package]] name = "pydantic-core" -version = "2.18.4" +version = "2.20.1" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.18.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:f76d0ad001edd426b92233d45c746fd08f467d56100fd8f30e9ace4b005266e4"}, - {file = "pydantic_core-2.18.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:59ff3e89f4eaf14050c8022011862df275b552caef8082e37b542b066ce1ff26"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a55b5b16c839df1070bc113c1f7f94a0af4433fcfa1b41799ce7606e5c79ce0a"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4d0dcc59664fcb8974b356fe0a18a672d6d7cf9f54746c05f43275fc48636851"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8951eee36c57cd128f779e641e21eb40bc5073eb28b2d23f33eb0ef14ffb3f5d"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4701b19f7e3a06ea655513f7938de6f108123bf7c86bbebb1196eb9bd35cf724"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e00a3f196329e08e43d99b79b286d60ce46bed10f2280d25a1718399457e06be"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:97736815b9cc893b2b7f663628e63f436018b75f44854c8027040e05230eeddb"}, - {file = "pydantic_core-2.18.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6891a2ae0e8692679c07728819b6e2b822fb30ca7445f67bbf6509b25a96332c"}, - {file = "pydantic_core-2.18.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bc4ff9805858bd54d1a20efff925ccd89c9d2e7cf4986144b30802bf78091c3e"}, - {file = "pydantic_core-2.18.4-cp310-none-win32.whl", hash = "sha256:1b4de2e51bbcb61fdebd0ab86ef28062704f62c82bbf4addc4e37fa4b00b7cbc"}, - {file = "pydantic_core-2.18.4-cp310-none-win_amd64.whl", hash = "sha256:6a750aec7bf431517a9fd78cb93c97b9b0c496090fee84a47a0d23668976b4b0"}, - {file = "pydantic_core-2.18.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:942ba11e7dfb66dc70f9ae66b33452f51ac7bb90676da39a7345e99ffb55402d"}, - {file = "pydantic_core-2.18.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b2ebef0e0b4454320274f5e83a41844c63438fdc874ea40a8b5b4ecb7693f1c4"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a642295cd0c8df1b86fc3dced1d067874c353a188dc8e0f744626d49e9aa51c4"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f09baa656c904807e832cf9cce799c6460c450c4ad80803517032da0cd062e2"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98906207f29bc2c459ff64fa007afd10a8c8ac080f7e4d5beff4c97086a3dabd"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19894b95aacfa98e7cb093cd7881a0c76f55731efad31073db4521e2b6ff5b7d"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fbbdc827fe5e42e4d196c746b890b3d72876bdbf160b0eafe9f0334525119c8"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f85d05aa0918283cf29a30b547b4df2fbb56b45b135f9e35b6807cb28bc47951"}, - {file = "pydantic_core-2.18.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e85637bc8fe81ddb73fda9e56bab24560bdddfa98aa64f87aaa4e4b6730c23d2"}, - {file = "pydantic_core-2.18.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2f5966897e5461f818e136b8451d0551a2e77259eb0f73a837027b47dc95dab9"}, - {file = "pydantic_core-2.18.4-cp311-none-win32.whl", hash = "sha256:44c7486a4228413c317952e9d89598bcdfb06399735e49e0f8df643e1ccd0558"}, - {file = "pydantic_core-2.18.4-cp311-none-win_amd64.whl", hash = "sha256:8a7164fe2005d03c64fd3b85649891cd4953a8de53107940bf272500ba8a788b"}, - {file = "pydantic_core-2.18.4-cp311-none-win_arm64.whl", hash = "sha256:4e99bc050fe65c450344421017f98298a97cefc18c53bb2f7b3531eb39bc7805"}, - {file = "pydantic_core-2.18.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6f5c4d41b2771c730ea1c34e458e781b18cc668d194958e0112455fff4e402b2"}, - {file = "pydantic_core-2.18.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2fdf2156aa3d017fddf8aea5adfba9f777db1d6022d392b682d2a8329e087cef"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4748321b5078216070b151d5271ef3e7cc905ab170bbfd27d5c83ee3ec436695"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847a35c4d58721c5dc3dba599878ebbdfd96784f3fb8bb2c356e123bdcd73f34"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c40d4eaad41f78e3bbda31b89edc46a3f3dc6e171bf0ecf097ff7a0ffff7cb1"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:21a5e440dbe315ab9825fcd459b8814bb92b27c974cbc23c3e8baa2b76890077"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01dd777215e2aa86dfd664daed5957704b769e726626393438f9c87690ce78c3"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4b06beb3b3f1479d32befd1f3079cc47b34fa2da62457cdf6c963393340b56e9"}, - {file = "pydantic_core-2.18.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:564d7922e4b13a16b98772441879fcdcbe82ff50daa622d681dd682175ea918c"}, - {file = "pydantic_core-2.18.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0eb2a4f660fcd8e2b1c90ad566db2b98d7f3f4717c64fe0a83e0adb39766d5b8"}, - {file = "pydantic_core-2.18.4-cp312-none-win32.whl", hash = "sha256:8b8bab4c97248095ae0c4455b5a1cd1cdd96e4e4769306ab19dda135ea4cdb07"}, - {file = "pydantic_core-2.18.4-cp312-none-win_amd64.whl", hash = "sha256:14601cdb733d741b8958224030e2bfe21a4a881fb3dd6fbb21f071cabd48fa0a"}, - {file = "pydantic_core-2.18.4-cp312-none-win_arm64.whl", hash = "sha256:c1322d7dd74713dcc157a2b7898a564ab091ca6c58302d5c7b4c07296e3fd00f"}, - {file = "pydantic_core-2.18.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:823be1deb01793da05ecb0484d6c9e20baebb39bd42b5d72636ae9cf8350dbd2"}, - {file = "pydantic_core-2.18.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ebef0dd9bf9b812bf75bda96743f2a6c5734a02092ae7f721c048d156d5fabae"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae1d6df168efb88d7d522664693607b80b4080be6750c913eefb77e34c12c71a"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9899c94762343f2cc2fc64c13e7cae4c3cc65cdfc87dd810a31654c9b7358cc"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99457f184ad90235cfe8461c4d70ab7dd2680e28821c29eca00252ba90308c78"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18f469a3d2a2fdafe99296a87e8a4c37748b5080a26b806a707f25a902c040a8"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7cdf28938ac6b8b49ae5e92f2735056a7ba99c9b110a474473fd71185c1af5d"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:938cb21650855054dc54dfd9120a851c974f95450f00683399006aa6e8abb057"}, - {file = "pydantic_core-2.18.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:44cd83ab6a51da80fb5adbd9560e26018e2ac7826f9626bc06ca3dc074cd198b"}, - {file = "pydantic_core-2.18.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:972658f4a72d02b8abfa2581d92d59f59897d2e9f7e708fdabe922f9087773af"}, - {file = "pydantic_core-2.18.4-cp38-none-win32.whl", hash = "sha256:1d886dc848e60cb7666f771e406acae54ab279b9f1e4143babc9c2258213daa2"}, - {file = "pydantic_core-2.18.4-cp38-none-win_amd64.whl", hash = "sha256:bb4462bd43c2460774914b8525f79b00f8f407c945d50881568f294c1d9b4443"}, - {file = "pydantic_core-2.18.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:44a688331d4a4e2129140a8118479443bd6f1905231138971372fcde37e43528"}, - {file = "pydantic_core-2.18.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a2fdd81edd64342c85ac7cf2753ccae0b79bf2dfa063785503cb85a7d3593223"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86110d7e1907ab36691f80b33eb2da87d780f4739ae773e5fc83fb272f88825f"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46387e38bd641b3ee5ce247563b60c5ca098da9c56c75c157a05eaa0933ed154"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:123c3cec203e3f5ac7b000bd82235f1a3eced8665b63d18be751f115588fea30"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dc1803ac5c32ec324c5261c7209e8f8ce88e83254c4e1aebdc8b0a39f9ddb443"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53db086f9f6ab2b4061958d9c276d1dbe3690e8dd727d6abf2321d6cce37fa94"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abc267fa9837245cc28ea6929f19fa335f3dc330a35d2e45509b6566dc18be23"}, - {file = "pydantic_core-2.18.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0d829524aaefdebccb869eed855e2d04c21d2d7479b6cada7ace5448416597b"}, - {file = "pydantic_core-2.18.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:509daade3b8649f80d4e5ff21aa5673e4ebe58590b25fe42fac5f0f52c6f034a"}, - {file = "pydantic_core-2.18.4-cp39-none-win32.whl", hash = "sha256:ca26a1e73c48cfc54c4a76ff78df3727b9d9f4ccc8dbee4ae3f73306a591676d"}, - {file = "pydantic_core-2.18.4-cp39-none-win_amd64.whl", hash = "sha256:c67598100338d5d985db1b3d21f3619ef392e185e71b8d52bceacc4a7771ea7e"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:574d92eac874f7f4db0ca653514d823a0d22e2354359d0759e3f6a406db5d55d"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1f4d26ceb5eb9eed4af91bebeae4b06c3fb28966ca3a8fb765208cf6b51102ab"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77450e6d20016ec41f43ca4a6c63e9fdde03f0ae3fe90e7c27bdbeaece8b1ed4"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d323a01da91851a4f17bf592faf46149c9169d68430b3146dcba2bb5e5719abc"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43d447dd2ae072a0065389092a231283f62d960030ecd27565672bd40746c507"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:578e24f761f3b425834f297b9935e1ce2e30f51400964ce4801002435a1b41ef"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:81b5efb2f126454586d0f40c4d834010979cb80785173d1586df845a632e4e6d"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ab86ce7c8f9bea87b9d12c7f0af71102acbf5ecbc66c17796cff45dae54ef9a5"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:90afc12421df2b1b4dcc975f814e21bc1754640d502a2fbcc6d41e77af5ec312"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:51991a89639a912c17bef4b45c87bd83593aee0437d8102556af4885811d59f5"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:293afe532740370aba8c060882f7d26cfd00c94cae32fd2e212a3a6e3b7bc15e"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b48ece5bde2e768197a2d0f6e925f9d7e3e826f0ad2271120f8144a9db18d5c8"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eae237477a873ab46e8dd748e515c72c0c804fb380fbe6c85533c7de51f23a8f"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:834b5230b5dfc0c1ec37b2fda433b271cbbc0e507560b5d1588e2cc1148cf1ce"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e858ac0a25074ba4bce653f9b5d0a85b7456eaddadc0ce82d3878c22489fa4ee"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2fd41f6eff4c20778d717af1cc50eca52f5afe7805ee530a4fbd0bae284f16e9"}, - {file = "pydantic_core-2.18.4.tar.gz", hash = "sha256:ec3beeada09ff865c344ff3bc2f427f5e6c26401cc6113d77e372c3fdac73864"}, + {file = "pydantic_core-2.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3acae97ffd19bf091c72df4d726d552c473f3576409b2a7ca36b2f535ffff4a3"}, + {file = "pydantic_core-2.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:41f4c96227a67a013e7de5ff8f20fb496ce573893b7f4f2707d065907bffdbd6"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f239eb799a2081495ea659d8d4a43a8f42cd1fe9ff2e7e436295c38a10c286a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53e431da3fc53360db73eedf6f7124d1076e1b4ee4276b36fb25514544ceb4a3"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1f62b2413c3a0e846c3b838b2ecd6c7a19ec6793b2a522745b0869e37ab5bc1"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d41e6daee2813ecceea8eda38062d69e280b39df793f5a942fa515b8ed67953"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d482efec8b7dc6bfaedc0f166b2ce349df0011f5d2f1f25537ced4cfc34fd98"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e93e1a4b4b33daed65d781a57a522ff153dcf748dee70b40c7258c5861e1768a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e7c4ea22b6739b162c9ecaaa41d718dfad48a244909fe7ef4b54c0b530effc5a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4f2790949cf385d985a31984907fecb3896999329103df4e4983a4a41e13e840"}, + {file = "pydantic_core-2.20.1-cp310-none-win32.whl", hash = "sha256:5e999ba8dd90e93d57410c5e67ebb67ffcaadcea0ad973240fdfd3a135506250"}, + {file = "pydantic_core-2.20.1-cp310-none-win_amd64.whl", hash = "sha256:512ecfbefef6dac7bc5eaaf46177b2de58cdf7acac8793fe033b24ece0b9566c"}, + {file = "pydantic_core-2.20.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d2a8fa9d6d6f891f3deec72f5cc668e6f66b188ab14bb1ab52422fe8e644f312"}, + {file = "pydantic_core-2.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:175873691124f3d0da55aeea1d90660a6ea7a3cfea137c38afa0a5ffabe37b88"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37eee5b638f0e0dcd18d21f59b679686bbd18917b87db0193ae36f9c23c355fc"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25e9185e2d06c16ee438ed39bf62935ec436474a6ac4f9358524220f1b236e43"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:150906b40ff188a3260cbee25380e7494ee85048584998c1e66df0c7a11c17a6"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ad4aeb3e9a97286573c03df758fc7627aecdd02f1da04516a86dc159bf70121"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3f3ed29cd9f978c604708511a1f9c2fdcb6c38b9aae36a51905b8811ee5cbf1"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0dae11d8f5ded51699c74d9548dcc5938e0804cc8298ec0aa0da95c21fff57b"}, + {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:faa6b09ee09433b87992fb5a2859efd1c264ddc37280d2dd5db502126d0e7f27"}, + {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9dc1b507c12eb0481d071f3c1808f0529ad41dc415d0ca11f7ebfc666e66a18b"}, + {file = "pydantic_core-2.20.1-cp311-none-win32.whl", hash = "sha256:fa2fddcb7107e0d1808086ca306dcade7df60a13a6c347a7acf1ec139aa6789a"}, + {file = "pydantic_core-2.20.1-cp311-none-win_amd64.whl", hash = "sha256:40a783fb7ee353c50bd3853e626f15677ea527ae556429453685ae32280c19c2"}, + {file = "pydantic_core-2.20.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:595ba5be69b35777474fa07f80fc260ea71255656191adb22a8c53aba4479231"}, + {file = "pydantic_core-2.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a4f55095ad087474999ee28d3398bae183a66be4823f753cd7d67dd0153427c9"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9aa05d09ecf4c75157197f27cdc9cfaeb7c5f15021c6373932bf3e124af029f"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e97fdf088d4b31ff4ba35db26d9cc472ac7ef4a2ff2badeabf8d727b3377fc52"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc633a9fe1eb87e250b5c57d389cf28998e4292336926b0b6cdaee353f89a237"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d573faf8eb7e6b1cbbcb4f5b247c60ca8be39fe2c674495df0eb4318303137fe"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26dc97754b57d2fd00ac2b24dfa341abffc380b823211994c4efac7f13b9e90e"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:33499e85e739a4b60c9dac710c20a08dc73cb3240c9a0e22325e671b27b70d24"}, + {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bebb4d6715c814597f85297c332297c6ce81e29436125ca59d1159b07f423eb1"}, + {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:516d9227919612425c8ef1c9b869bbbee249bc91912c8aaffb66116c0b447ebd"}, + {file = "pydantic_core-2.20.1-cp312-none-win32.whl", hash = "sha256:469f29f9093c9d834432034d33f5fe45699e664f12a13bf38c04967ce233d688"}, + {file = "pydantic_core-2.20.1-cp312-none-win_amd64.whl", hash = "sha256:035ede2e16da7281041f0e626459bcae33ed998cca6a0a007a5ebb73414ac72d"}, + {file = "pydantic_core-2.20.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:0827505a5c87e8aa285dc31e9ec7f4a17c81a813d45f70b1d9164e03a813a686"}, + {file = "pydantic_core-2.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19c0fa39fa154e7e0b7f82f88ef85faa2a4c23cc65aae2f5aea625e3c13c735a"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa223cd1e36b642092c326d694d8bf59b71ddddc94cdb752bbbb1c5c91d833b"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c336a6d235522a62fef872c6295a42ecb0c4e1d0f1a3e500fe949415761b8a19"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7eb6a0587eded33aeefea9f916899d42b1799b7b14b8f8ff2753c0ac1741edac"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70c8daf4faca8da5a6d655f9af86faf6ec2e1768f4b8b9d0226c02f3d6209703"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9fa4c9bf273ca41f940bceb86922a7667cd5bf90e95dbb157cbb8441008482c"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:11b71d67b4725e7e2a9f6e9c0ac1239bbc0c48cce3dc59f98635efc57d6dac83"}, + {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:270755f15174fb983890c49881e93f8f1b80f0b5e3a3cc1394a255706cabd203"}, + {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c81131869240e3e568916ef4c307f8b99583efaa60a8112ef27a366eefba8ef0"}, + {file = "pydantic_core-2.20.1-cp313-none-win32.whl", hash = "sha256:b91ced227c41aa29c672814f50dbb05ec93536abf8f43cd14ec9521ea09afe4e"}, + {file = "pydantic_core-2.20.1-cp313-none-win_amd64.whl", hash = "sha256:65db0f2eefcaad1a3950f498aabb4875c8890438bc80b19362cf633b87a8ab20"}, + {file = "pydantic_core-2.20.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4745f4ac52cc6686390c40eaa01d48b18997cb130833154801a442323cc78f91"}, + {file = "pydantic_core-2.20.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8ad4c766d3f33ba8fd692f9aa297c9058970530a32c728a2c4bfd2616d3358b"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41e81317dd6a0127cabce83c0c9c3fbecceae981c8391e6f1dec88a77c8a569a"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04024d270cf63f586ad41fff13fde4311c4fc13ea74676962c876d9577bcc78f"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eaad4ff2de1c3823fddf82f41121bdf453d922e9a238642b1dedb33c4e4f98ad"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26ab812fa0c845df815e506be30337e2df27e88399b985d0bb4e3ecfe72df31c"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c5ebac750d9d5f2706654c638c041635c385596caf68f81342011ddfa1e5598"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2aafc5a503855ea5885559eae883978c9b6d8c8993d67766ee73d82e841300dd"}, + {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4868f6bd7c9d98904b748a2653031fc9c2f85b6237009d475b1008bfaeb0a5aa"}, + {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aa2f457b4af386254372dfa78a2eda2563680d982422641a85f271c859df1987"}, + {file = "pydantic_core-2.20.1-cp38-none-win32.whl", hash = "sha256:225b67a1f6d602de0ce7f6c1c3ae89a4aa25d3de9be857999e9124f15dab486a"}, + {file = "pydantic_core-2.20.1-cp38-none-win_amd64.whl", hash = "sha256:6b507132dcfc0dea440cce23ee2182c0ce7aba7054576efc65634f080dbe9434"}, + {file = "pydantic_core-2.20.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b03f7941783b4c4a26051846dea594628b38f6940a2fdc0df00b221aed39314c"}, + {file = "pydantic_core-2.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1eedfeb6089ed3fad42e81a67755846ad4dcc14d73698c120a82e4ccf0f1f9f6"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:635fee4e041ab9c479e31edda27fcf966ea9614fff1317e280d99eb3e5ab6fe2"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:77bf3ac639c1ff567ae3b47f8d4cc3dc20f9966a2a6dd2311dcc055d3d04fb8a"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ed1b0132f24beeec5a78b67d9388656d03e6a7c837394f99257e2d55b461611"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6514f963b023aeee506678a1cf821fe31159b925c4b76fe2afa94cc70b3222b"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10d4204d8ca33146e761c79f83cc861df20e7ae9f6487ca290a97702daf56006"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d036c7187b9422ae5b262badb87a20a49eb6c5238b2004e96d4da1231badef1"}, + {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ebfef07dbe1d93efb94b4700f2d278494e9162565a54f124c404a5656d7ff09"}, + {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6b9d9bb600328a1ce523ab4f454859e9d439150abb0906c5a1983c146580ebab"}, + {file = "pydantic_core-2.20.1-cp39-none-win32.whl", hash = "sha256:784c1214cb6dd1e3b15dd8b91b9a53852aed16671cc3fbe4786f4f1db07089e2"}, + {file = "pydantic_core-2.20.1-cp39-none-win_amd64.whl", hash = "sha256:d2fe69c5434391727efa54b47a1e7986bb0186e72a41b203df8f5b0a19a4f669"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a45f84b09ac9c3d35dfcf6a27fd0634d30d183205230a0ebe8373a0e8cfa0906"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d02a72df14dfdbaf228424573a07af10637bd490f0901cee872c4f434a735b94"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2b27e6af28f07e2f195552b37d7d66b150adbaa39a6d327766ffd695799780f"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084659fac3c83fd674596612aeff6041a18402f1e1bc19ca39e417d554468482"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:242b8feb3c493ab78be289c034a1f659e8826e2233786e36f2893a950a719bb6"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:38cf1c40a921d05c5edc61a785c0ddb4bed67827069f535d794ce6bcded919fc"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e0bbdd76ce9aa5d4209d65f2b27fc6e5ef1312ae6c5333c26db3f5ade53a1e99"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:254ec27fdb5b1ee60684f91683be95e5133c994cc54e86a0b0963afa25c8f8a6"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:407653af5617f0757261ae249d3fba09504d7a71ab36ac057c938572d1bc9331"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c693e916709c2465b02ca0ad7b387c4f8423d1db7b4649c551f27a529181c5ad"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b5ff4911aea936a47d9376fd3ab17e970cc543d1b68921886e7f64bd28308d1"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177f55a886d74f1808763976ac4efd29b7ed15c69f4d838bbd74d9d09cf6fa86"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:964faa8a861d2664f0c7ab0c181af0bea66098b1919439815ca8803ef136fc4e"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4dd484681c15e6b9a977c785a345d3e378d72678fd5f1f3c0509608da24f2ac0"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f6d6cff3538391e8486a431569b77921adfcdef14eb18fbf19b7c0a5294d4e6a"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a6d511cc297ff0883bc3708b465ff82d7560193169a8b93260f74ecb0a5e08a7"}, + {file = "pydantic_core-2.20.1.tar.gz", hash = "sha256:26ca695eeee5f9f1aeeb211ffc12f10bcb6f71e2989988fda61dabd65db878d4"}, ] [package.dependencies] @@ -6291,24 +6288,25 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pydantic-extra-types" -version = "2.8.2" +version = "2.9.0" description = "Extra Pydantic types." optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_extra_types-2.8.2-py3-none-any.whl", hash = "sha256:f2400b3c3553fb7fa09a131967b4edf2d53f01ad9fa89d158784653f2e5c13d1"}, - {file = "pydantic_extra_types-2.8.2.tar.gz", hash = "sha256:4d2b3c52c1e2e4dfa31bf1d5a37b841b09e3c5a08ec2bffca0e07fc2ad7d5c4a"}, + {file = "pydantic_extra_types-2.9.0-py3-none-any.whl", hash = "sha256:f0bb975508572ba7bf3390b7337807588463b7248587e69f43b1ad7c797530d0"}, + {file = "pydantic_extra_types-2.9.0.tar.gz", hash = "sha256:e061c01636188743bb69f368dcd391f327b8cfbfede2fe1cbb1211b06601ba3b"}, ] [package.dependencies] pydantic = ">=2.5.2" [package.extras] -all = ["pendulum (>=3.0.0,<4.0.0)", "phonenumbers (>=8,<9)", "pycountry (>=23)", "python-ulid (>=1,<2)", "python-ulid (>=1,<3)"] +all = ["pendulum (>=3.0.0,<4.0.0)", "phonenumbers (>=8,<9)", "pycountry (>=23)", "python-ulid (>=1,<2)", "python-ulid (>=1,<3)", "pytz (>=2024.1)", "semver (>=3.0.2)", "tzdata (>=2024.1)"] pendulum = ["pendulum (>=3.0.0,<4.0.0)"] phonenumbers = ["phonenumbers (>=8,<9)"] pycountry = ["pycountry (>=23)"] python-ulid = ["python-ulid (>=1,<2)", "python-ulid (>=1,<3)"] +semver = ["semver (>=3.0.2)"] [[package]] name = "pydantic-settings" @@ -6500,59 +6498,19 @@ files = [ [[package]] name = "pyreqwest-impersonate" -version = "0.4.9" +version = "0.5.3" description = "HTTP client that can impersonate web browsers, mimicking their headers and `TLS/JA3/JA4/HTTP2` fingerprints" optional = false python-versions = ">=3.8" files = [ - {file = "pyreqwest_impersonate-0.4.9-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a229f56575d992df0c520d93408b4b6b660b304387af06208e7b97d739cce2ff"}, - {file = "pyreqwest_impersonate-0.4.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c00dbfd0ed878bed231384cd0c823d71a42220ae73c6d982b6fe77d2870338ca"}, - {file = "pyreqwest_impersonate-0.4.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d4e6ce0e48b73740f08b1aa69cdbded5d66f4eec327d5eaf2ac42a4fce1a008"}, - {file = "pyreqwest_impersonate-0.4.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:690a5c5615b33cbab340e3a4247256ede157ebf39a02f563cff5356bf61c0c51"}, - {file = "pyreqwest_impersonate-0.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7231511ee14faee27b90a84ec74e15515b7e2d1c389710698660765eaed6e2fd"}, - {file = "pyreqwest_impersonate-0.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2fdbe8e146e595c02fa0afb776d70f9e3b351122e2de9af15934b83f3a548ecd"}, - {file = "pyreqwest_impersonate-0.4.9-cp310-none-win_amd64.whl", hash = "sha256:982b0e53db24c084675a056944dd769aa07cd1378abf972927f3f1afb23e08b0"}, - {file = "pyreqwest_impersonate-0.4.9-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:60b1102b8aec7bbf91e0f7b8bbc3507776731a9acc6844de764911e8d64f7dd2"}, - {file = "pyreqwest_impersonate-0.4.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37150478157e683374517d4c0eae0f991b8f5280067a8ee042b6a72fec088843"}, - {file = "pyreqwest_impersonate-0.4.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ccc77cd1cdae22dad7549a4e9a1a4630619c2ff443add1b28c7d607accda81eb"}, - {file = "pyreqwest_impersonate-0.4.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83e99e627d13f1f60d71ce2c2a2b03e1c7f57e8f6a73bde2827ff97cb96f1683"}, - {file = "pyreqwest_impersonate-0.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:72d1adb73264db8c5e24d073d558a895d6690d13a5e38fd857b8b01c33fcbabf"}, - {file = "pyreqwest_impersonate-0.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6253bd8a104316bbece0e6c658d28292f0bf37a99cccfaf476470b98252d185b"}, - {file = "pyreqwest_impersonate-0.4.9-cp311-none-win_amd64.whl", hash = "sha256:7e25628a900236fc76320e790fce90e5502371994523c476af2b1c938382f5fa"}, - {file = "pyreqwest_impersonate-0.4.9-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:57e1e7f3bfc175c3229947cdd2b26564bcea2923135b8dec8ab157609e201a7c"}, - {file = "pyreqwest_impersonate-0.4.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3aeb1c834f54fe685d3c7c0bec65dd981bd988fa3725ee3c7b5656eb7c44a1f7"}, - {file = "pyreqwest_impersonate-0.4.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27bc384f18099573817d7ed68d12eb67d33dfc5d2b30ab2ac5a69cdf19c22b6f"}, - {file = "pyreqwest_impersonate-0.4.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd604444ddf86ed222b49dd5e3f050c4c0e980dd7be0b3ea0f208fb70544c4b6"}, - {file = "pyreqwest_impersonate-0.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5206dc7311081accf5b7d021c9e0e68219fd7bb35b0cd755b2d72c3ebfa41842"}, - {file = "pyreqwest_impersonate-0.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:76802f0738c2d00bb4e057938ec048c4c7c4efc5c44f04b9d877ad4419b21ee8"}, - {file = "pyreqwest_impersonate-0.4.9-cp312-none-win_amd64.whl", hash = "sha256:7cf94f6365bc144f787658e844f94dad38107fb9ff61d65079fb6510447777fe"}, - {file = "pyreqwest_impersonate-0.4.9-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:66e92bd868146028dac1ef9cd2b4aac57e7e6cbd8806fa8a4c77ac5becf396e1"}, - {file = "pyreqwest_impersonate-0.4.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:dc3ff7ac332879e40301d737b3ec1f3691b1de7492728bea26e75e26d05f89ec"}, - {file = "pyreqwest_impersonate-0.4.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c9e9eba83620852d4253023e50e3436726aee16e2de94afbd468da4373830dc"}, - {file = "pyreqwest_impersonate-0.4.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d6b47d403c63b461a97efa2aed668f0f06ed26cf61c23d7d6dab4f5a0c81ffc"}, - {file = "pyreqwest_impersonate-0.4.9-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:88f695a01e8699ec3a1547d793617b9fd00f810c05c2b4dc0d1472c7f12eed97"}, - {file = "pyreqwest_impersonate-0.4.9-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:abb4fbfaa1a3c3adeb7f46baa1d67663af85ab24f2b4cdd15a668ddc6be3a375"}, - {file = "pyreqwest_impersonate-0.4.9-cp38-none-win_amd64.whl", hash = "sha256:884c1399fe0157dcd0a5a71e3600910df50faa0108c64602d47c15e75b32e60b"}, - {file = "pyreqwest_impersonate-0.4.9-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:bf5cd99276207510d64b48eff5602e12f049754d3b0f1194a024e1a080a61d3d"}, - {file = "pyreqwest_impersonate-0.4.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:029eea1d386d12856da767d685169835f0b0c025ae312c1ee7bc0d8cb47a7d3d"}, - {file = "pyreqwest_impersonate-0.4.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1bfb8795fe0a46aee883abcf510a9ecdb4e9acf75c3a5a23571276f555f5e88"}, - {file = "pyreqwest_impersonate-0.4.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fe35ce48e7e6b570304ee15915da0e6fab82dcae2b7a1d1a92593b522ebe852"}, - {file = "pyreqwest_impersonate-0.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dfa377a842bd2e73d1f201bfc33194dd98c148372409d376f6d57efe338ff0eb"}, - {file = "pyreqwest_impersonate-0.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d46880e68eb779cd071648e94a7ec50b3b77a28210f218be407eda1b0c8df343"}, - {file = "pyreqwest_impersonate-0.4.9-cp39-none-win_amd64.whl", hash = "sha256:ac431e4a94f8529a19a396750d78c66cc4fa11a8cc61d4bed7f0e0295a9394a9"}, - {file = "pyreqwest_impersonate-0.4.9-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12fd04d8da4d23ab5720402fd9f3b6944fb388c19952f2ec9121b46ac1f74616"}, - {file = "pyreqwest_impersonate-0.4.9-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b52df560d78681cde2fbc39bee547a42a79c8fd33655b79618835ecc412e6933"}, - {file = "pyreqwest_impersonate-0.4.9-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:e1a2828942f9d589ee6161496444a380d3305e78bda25ff63e4f993b0545b193"}, - {file = "pyreqwest_impersonate-0.4.9-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:beebedf6d8c0d5fdee9ae15bc64a74e51b35f98eb0d049bf2db067702fbf4e53"}, - {file = "pyreqwest_impersonate-0.4.9-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a3d47dea1f46410b58ab60795b5818c8c99d901f6c93fbb6a9d23fa55adb2b1"}, - {file = "pyreqwest_impersonate-0.4.9-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bb871adc5d12b2bcbb5af167384d49fc4e7e5e07d12cf912b931149163df724"}, - {file = "pyreqwest_impersonate-0.4.9-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:d1b0b5556d2bd14a4ffa32654291fe2a9ef1eaac35b5514d9220e7e333a6c727"}, - {file = "pyreqwest_impersonate-0.4.9-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:5d50feaec78c06d51e1dd65cdbe80a1fc62ff93c8114555482f8a8cc5fe14895"}, - {file = "pyreqwest_impersonate-0.4.9-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2a9cfc41917200d8eee61b87a5668abe7d1f924a55b7437065540edf613beed"}, - {file = "pyreqwest_impersonate-0.4.9-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8106e3df0c1dca4df99e0f998f0e085ea3e1facfaa5afc268160a496ddf7256f"}, - {file = "pyreqwest_impersonate-0.4.9-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:ff66bb7dc6b1f52cf950b5e9cb0e53baffd1a15da595fd1ef933cd9e36396403"}, - {file = "pyreqwest_impersonate-0.4.9-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:39f2a3ed17cf08098dc637459e88fb86d3fa7bdf9502659c82218be75651649c"}, - {file = "pyreqwest_impersonate-0.4.9.tar.gz", hash = "sha256:4ec8df7fe813e89f61e814c5ef75f6fd71164c8e26299c1a42dcd0d42f0bc96c"}, + {file = "pyreqwest_impersonate-0.5.3-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f15922496f728769fb9e1b116d5d9d7ba5525d0f2f7a76a41a1daef8b2e0c6c3"}, + {file = "pyreqwest_impersonate-0.5.3-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:77533133ae73020e59bc56d776eea3fe3af4ac41d763a89f39c495436da0f4cf"}, + {file = "pyreqwest_impersonate-0.5.3-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:436055fa3eeb3e01e2e8efd42a9f6c4ab62fd643eddc7c66d0e671b71605f273"}, + {file = "pyreqwest_impersonate-0.5.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e9d2e981a525fb72c1521f454e5581d2c7a3b1fcf1c97c0acfcb7a923d8cf3e"}, + {file = "pyreqwest_impersonate-0.5.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a6bf986d4a165f6976b3e862111e2a46091883cb55e9e6325150f5aea2644229"}, + {file = "pyreqwest_impersonate-0.5.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b7397f6dad3d5ae158e0b272cb3eafe8382e71775d829b286ae9c21cb5a879ff"}, + {file = "pyreqwest_impersonate-0.5.3-cp38-abi3-win_amd64.whl", hash = "sha256:6026e4751b5912aec1e45238c07daf1e2c9126b3b32b33396b72885021e8990c"}, + {file = "pyreqwest_impersonate-0.5.3.tar.gz", hash = "sha256:f21c10609958ff5be18df0c329eed42d2b3ba8a339b65dc5f96ab74537231692"}, ] [package.extras] @@ -6637,98 +6595,111 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] [[package]] name = "python-calamine" -version = "0.2.0" +version = "0.2.3" description = "Python binding for Rust's library for reading excel and odf file - calamine" optional = false python-versions = ">=3.8" files = [ - {file = "python_calamine-0.2.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:da0c64292f011853046e4ce833e3319d11ab4b0ac1dec5ace1887e1f1630a3e0"}, - {file = "python_calamine-0.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7adbe5cbcdc04bd850beba71f51208fff69e03071ee2db556c227f992fc5b7ce"}, - {file = "python_calamine-0.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:984e34a17f91970aa72dc0809339d401943c11a4650c16c79eb9bf5fb25b21f2"}, - {file = "python_calamine-0.2.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:216e30810f1b1c9a4b8413fdcfa062e3f8a1eafc544966c89e0bfbace258300f"}, - {file = "python_calamine-0.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bd8d63b506a4759d6bb92029938ea725a58553f5e7d0cfeb8c94fd25dd30f71b"}, - {file = "python_calamine-0.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9f21143b105dcad70f4ee45bda1ab1d76a97ae520f60fe65f27fbbb8a6f70a27"}, - {file = "python_calamine-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35e1e0535d22117b2c720b5c0fb6c1ca4bbcecfb4b06a259e52567a21a365575"}, - {file = "python_calamine-0.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:70fc2b4d4188f459639a8ab6ea618fb6941a66b1c7aabe8be71eb3a2c5b8537a"}, - {file = "python_calamine-0.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f4a055c0f47d20c2017e5eb75a388900e2a58beca96e54ecd95647b2c8c97855"}, - {file = "python_calamine-0.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:212637c9b32d0ea00135f280b70b5d950cef270f38fa1b73037d15ed16dafdaa"}, - {file = "python_calamine-0.2.0-cp310-none-win32.whl", hash = "sha256:f53fdaa2fa53512e8e0c9a2e9079e2c77f4e5653ac1af86ce73f90577c186f19"}, - {file = "python_calamine-0.2.0-cp310-none-win_amd64.whl", hash = "sha256:fcc1cc57df8b3527fc083aa707ba5ee6613bb6096014243d2cce4bac17dde142"}, - {file = "python_calamine-0.2.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:765123dc19b4ea7dfcb8bc50dc836dc516cabc09d4bc7ddb34276c159b76dc55"}, - {file = "python_calamine-0.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afefc3764eda1dd334008956b25ff3086b4949ad22f693293d8551b61ab74c2d"}, - {file = "python_calamine-0.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb52052aedb35387d2d0bfbdd1e8d491b721143abaeb33a9defae89963cd6d37"}, - {file = "python_calamine-0.2.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38348374b7f76e5334d107f547dad16fd4c6a32d0241166c0fd68084eace2558"}, - {file = "python_calamine-0.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ef2b0fdd00c1b0e7c8861912ef78a9ea772186ace7f018c0d6ccddf7c59adf70"}, - {file = "python_calamine-0.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cd33e553a03bd0f7c746df601a95e9d700bda9d691f1d5d9b516535976738cb"}, - {file = "python_calamine-0.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27a494518b55acebab195727d0aa5e35c48aacd2cd85c81e31575cfa690dfd2a"}, - {file = "python_calamine-0.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7f99599821e685042abc5fc550e8dbe3bfc1b7acb327542c86fe4c08505af311"}, - {file = "python_calamine-0.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7416a8ea27fe4954ce2c38d9e9d8c8678844eace612c58c28d1db3b896c9e52e"}, - {file = "python_calamine-0.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0b7b4e47ff5624418290c269a62474047751bcb0263b25e9ad1104c8ceed580d"}, - {file = "python_calamine-0.2.0-cp311-none-win32.whl", hash = "sha256:a620a8ef94db07b292c1e12b46b8b74f3cdfab9cf6221de94fcd8e35a534b613"}, - {file = "python_calamine-0.2.0-cp311-none-win_amd64.whl", hash = "sha256:8ad572df76c2a3593cfbef6ae715b4b9bbc2dd0d54d384c2e9ae93d6e5e00489"}, - {file = "python_calamine-0.2.0-cp311-none-win_arm64.whl", hash = "sha256:747326551bf826a305a0b0c57adf36d08b440f00af63de9c36a170212770eea8"}, - {file = "python_calamine-0.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b9266a850a317b18ff7c3a8364ee18eb63f0db089cb605f67da2e4578f07cfa7"}, - {file = "python_calamine-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b30c5b9b5405f98ddb1f276a231d3f63456ded4e3a3d88eebab61c23584a5b87"}, - {file = "python_calamine-0.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c8e245cc6189b2931dc49b1750f9416420b73fdcf789a5bf8b7df6128285b2c"}, - {file = "python_calamine-0.2.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:654f8beda2036fa3dcf707f3772f838bd5d6f0d5df99980b04492cc37d9dfae1"}, - {file = "python_calamine-0.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3d1fd1f1d51fb720aca1cc1b2be7224ddaeb0b4e15276504669118419b6f4b90"}, - {file = "python_calamine-0.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6dd7efdfa07ffbc048b3374b79c13aa9fc0aee8eccd0e9a8c6b5f17d8316c08"}, - {file = "python_calamine-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7875433fa6050d042c31adf3588dbc1718d7d10ee9c7b0e51ed2e5ee0f6e68e"}, - {file = "python_calamine-0.2.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:86c3ebc805df4e864fbbb669b868ad72c78f2f2e27d39eb1709267228a36bdcb"}, - {file = "python_calamine-0.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:93e97f3496fed357072acb00ff7145b930cb060a6aa2ac2589187a647db8211f"}, - {file = "python_calamine-0.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ac94d26a7cdf3ead932a1352f4112efaa84ec913d1cb6934176428393c6e8b58"}, - {file = "python_calamine-0.2.0-cp312-none-win32.whl", hash = "sha256:91fb49823df1ef80b615a8f1b6b4bbd40bb523f949333c7e41d75efa6cbe88f9"}, - {file = "python_calamine-0.2.0-cp312-none-win_amd64.whl", hash = "sha256:3d167ea8b2d929c1edcc69f0bad0ff57529673ba72cd9448f20b8e1107993268"}, - {file = "python_calamine-0.2.0-cp312-none-win_arm64.whl", hash = "sha256:7d7f829ee9db104e3b1c751414c62ed124aa7186c2887112722e6eb90a376db4"}, - {file = "python_calamine-0.2.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6430c39fe87f4816ede3311706580fd5fbf7c300ba49086acc5bd1cad400eb38"}, - {file = "python_calamine-0.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4aaf84cf22b0b8906106cc4aa260dafbbcb8d1b4f91cddcace5aa153fbe5f421"}, - {file = "python_calamine-0.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:842272e3f9813b956761214460c757576d4b7a81623743e70ab1aecd667a4752"}, - {file = "python_calamine-0.2.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b11fc2283c747899eeb66a0354b91423a9d6a35df9372c6330bde6eeed7c852a"}, - {file = "python_calamine-0.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6722258b934c12ae153f4789e440825d155a428148f2f9d71821e9a515592cb6"}, - {file = "python_calamine-0.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b294ad2a85be8fe161a33e7baa178b381763242c9fcbd16e4d36ee88485706c"}, - {file = "python_calamine-0.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06369b17b5f3a8e1e3b1988fdbe978301ff94d9c40f3ad2cb4ba814503344abb"}, - {file = "python_calamine-0.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c9d543bbc7b340dcaaee8d4e8e9a934fe75f71f1bfda224e35f7f7f39ad733bf"}, - {file = "python_calamine-0.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5381ee79f12625dca1dccc1d305e6b4948d61e68e208747594c39742a4a7145a"}, - {file = "python_calamine-0.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e8f77ee228ce1ea983c273ea1c17130a334d1049abbf28607de36168b65d676d"}, - {file = "python_calamine-0.2.0-cp38-none-win32.whl", hash = "sha256:c5323089868399ad49175126cdfdc1d5c3f8be122faaf7d497b10f23f6cd41ca"}, - {file = "python_calamine-0.2.0-cp38-none-win_amd64.whl", hash = "sha256:f00ae29d9062c9a490949478bf8880e4dc3551b18b335d4c44dac861ab7a6003"}, - {file = "python_calamine-0.2.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:e114069a5189702a3bf2358caa4e349178ec29f2dab711762a48b622e47aa9b9"}, - {file = "python_calamine-0.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4fb854c110643dab3514e5f54e65090789c56df53fa24f477498b95a3e834a1a"}, - {file = "python_calamine-0.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3622fa8ae26a01e9a934fdadee00222c1231d56b9343f6f947e0aa8ba4ccda35"}, - {file = "python_calamine-0.2.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c221573ab5a8b642345371f3e6b854c3d293fbad1d7d9be2c7bff2ec8de076b9"}, - {file = "python_calamine-0.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5830829753b1e62353b1c1971992dc9764f5d88d2a78b6386b5077efa98dab8"}, - {file = "python_calamine-0.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e76c87a40d1a5b22ba85ea88c86d2d89311eaa45b4df1afbc121b552c2efede7"}, - {file = "python_calamine-0.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c67b1468b527fc43038d1e095a62a9cc0db009d7780cd6092cf9ef427bd8a9c"}, - {file = "python_calamine-0.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b9fc8fd1cbf994bdd54a2a48c32ea8cce050a27f0859f0508cf44ffba2def465"}, - {file = "python_calamine-0.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f6924afb342ff9e1264e97835f26a56b542be06a4b5d461fb228c6358a52b24e"}, - {file = "python_calamine-0.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:19e143a8c6f16dcc9a994567b06766a84b5cd6869656efe1d39902d41a99b00a"}, - {file = "python_calamine-0.2.0-cp39-none-win32.whl", hash = "sha256:4e5bfcef754de1553b3ac4f91af2c471aa720ee467d4bad49144be25af724fad"}, - {file = "python_calamine-0.2.0-cp39-none-win_amd64.whl", hash = "sha256:a0e76379128ae6d8fba65eaee01a7c02dad258b11bffccdf2c1a74ee84c90d49"}, - {file = "python_calamine-0.2.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d767fb8db8527d6ed8d3602fb9e8778a32ae0db66b1081541fbb1f804796bd87"}, - {file = "python_calamine-0.2.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:74dce7c8dc863dba44d52effb1290505610564be574924fc92770728bdd38393"}, - {file = "python_calamine-0.2.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36beae8347b414a321276a7d065f672883f075ff40fa1c789e841bf6962b96fa"}, - {file = "python_calamine-0.2.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:caa39238ff71d47cb4ed556e7080829e8c9dae3fc0d2a01c6bb0114a1f313cdb"}, - {file = "python_calamine-0.2.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3e42b156ef7485a846195fc01013eab2816841bc8b87f0393dc8f1c19f2e4f12"}, - {file = "python_calamine-0.2.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fdb4a5b9e719668054cd9d4b210b38ae0eeb056340125704eb3665555e2f6e04"}, - {file = "python_calamine-0.2.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e6636349742c73944a3cb05baeb99a2d72371712b35abf7f0b22d9de6b85b8fd"}, - {file = "python_calamine-0.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:cb6f2a65444e01032564f7c5fdcfea43b6506b82f2d5308edd7d1d030bb01a13"}, - {file = "python_calamine-0.2.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e92f84ea190d881bec8852c6b89fa3ca93ce514094a827be2d66c31cec063838"}, - {file = "python_calamine-0.2.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:a2bda72687d6fce125ccf373ad760f5f50561219e91664baf51ab36ec0dbcaf2"}, - {file = "python_calamine-0.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7450398d1212bb668e88e42a5a3c1239b55371d1514c2d9164ecdb89c8b02426"}, - {file = "python_calamine-0.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed12d1780d0235c4ff5a5a2f9fab846498ee90e739def6aa27052895a860e2a5"}, - {file = "python_calamine-0.2.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:978431179238f2a61046696bbe93aff9f4c9f7c2d054b84b8963c246a837702f"}, - {file = "python_calamine-0.2.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:97b38e1f9c3d6dd921662d327ebbc1ef912faa970f28066784777745a4b9094a"}, - {file = "python_calamine-0.2.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c296857b22568a381cb1cf908928ad8bb18ea19590c1ff3f67a51010a29cebb3"}, - {file = "python_calamine-0.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b85f3156c87f870e3f48765ce46347c67c13fd0fc7e3de993285fd428f99b282"}, - {file = "python_calamine-0.2.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:8f0fc394191aea88efc0ff0e3cb1fdd4d15c33bce13f075187c3caccc995e964"}, - {file = "python_calamine-0.2.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:3c08312a1f8a99cf525714c53854f8c5a9bf12d7c04adaf71a00c88ed8228704"}, - {file = "python_calamine-0.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:183a78c284597c0b25df15a7fb74baa90df71fcc71721190c9c93229958f5809"}, - {file = "python_calamine-0.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91089a7f4dba9c9312b873ea2f47cd0dc868db2a4ae829b720e86a761d6daded"}, - {file = "python_calamine-0.2.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:25b4ee9cc67248ab69e46920a9f10a89efc27bcfe1586ca299136ba0bee16c0d"}, - {file = "python_calamine-0.2.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2d3b010c9824cddc426cac51654521fda9a33c902c965fb4b2c8fe8d75839e22"}, - {file = "python_calamine-0.2.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:06bfd869d4e1d0c48273c4c262359aa912eaa5d40ee1b5b653dac3e321a61e4c"}, - {file = "python_calamine-0.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:91b67d4284de11046c5ab20afebba17ea72415e7773089e12c0a3dac390e5e73"}, - {file = "python_calamine-0.2.0.tar.gz", hash = "sha256:3cc81fefbd697050bed7d9968be3562ab41390bf8fd0a79e97eed6056be3d332"}, + {file = "python_calamine-0.2.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:f292a03591b1cab1537424851b74baa33b0a55affc315248a7592ba3de1c3e83"}, + {file = "python_calamine-0.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6cfbd23d1147f53fd70fddfb38af2a98896ecad069c9a4120e77358a6fc43b39"}, + {file = "python_calamine-0.2.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:847373d0152bafd92b739c911de8c2d23e32ea93d9358bf32b58ed4ace382ae7"}, + {file = "python_calamine-0.2.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1e0dcdc796eb4b4907618392c4b71146812774ca30bf6162a711b63e54214912"}, + {file = "python_calamine-0.2.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2ee8250638ad174aa22a3776ebd41500cf88af62346f1c857505158d2685852"}, + {file = "python_calamine-0.2.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9ac718eb8e9753b986f329aec5dea964005a79115c622a2671fccd0c563d345a"}, + {file = "python_calamine-0.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa1baf404027779cb298d15939a5268eb3d477c86a7a8f4cad0734ea513876c2"}, + {file = "python_calamine-0.2.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dc36a85f1a182e49fc318b3e91f06f390d3889ce8c843721cb03a68ca4c7e4ce"}, + {file = "python_calamine-0.2.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:11e2a74da47adc502c776e399972864802a20d358001a1cfaefb13c36a5116c0"}, + {file = "python_calamine-0.2.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f19c8eb9f2182cca54c274145b6c8409776b7c08ee5be8a61d44f0448dc55192"}, + {file = "python_calamine-0.2.3-cp310-none-win32.whl", hash = "sha256:37367f85282d87c0d9453cb3caec5a74f2720252bfbc1365d627e9fe12251e56"}, + {file = "python_calamine-0.2.3-cp310-none-win_amd64.whl", hash = "sha256:6d73ef3131b3a7c3894a533857b02fc50198fb65528cbf869742555d1497ee52"}, + {file = "python_calamine-0.2.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:e5a36cca8b447295e9edddbe055857bdfdec56cb78554455a03bacd78e3c45a0"}, + {file = "python_calamine-0.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7b5b0803c70269d93b67c42f03e5711a7ba02166fd473a6cb89ef71632167154"}, + {file = "python_calamine-0.2.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73766349215f69854afb092ef891cb1ff253f4b6611342566c469b46516c6ada"}, + {file = "python_calamine-0.2.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3bf4cf41518541016b9442082360a83f3579955a872cfca5cec50acc3101cce5"}, + {file = "python_calamine-0.2.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f1f6dab7b44deed8cf7b45a6d6d2743b622ba5e21a8b73f52ef1064cc5e3638"}, + {file = "python_calamine-0.2.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1991261d40be3d577ce48c0884c6403aefd1cbef5dcc451e039746aa1d185931"}, + {file = "python_calamine-0.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f675e7f45d9e3f1430f3114701133432c279aba06442e743220f6b648023b5ee"}, + {file = "python_calamine-0.2.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8bb7444454cff2c1ad44e7f1a1be776845cbad8f1210d868c7058d2183b3da74"}, + {file = "python_calamine-0.2.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7a604306cd5ceca720f0426deb49192f2ede5eedd1597b7ff4fa9659a36dc462"}, + {file = "python_calamine-0.2.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b95afd1a1cd3871d472aa117537b8731c1609756347874b251300cff152176a5"}, + {file = "python_calamine-0.2.3-cp311-none-win32.whl", hash = "sha256:a0ae5a740c9d97b2842d948a91f926a0fab278d247d816fe786219b94507c5a2"}, + {file = "python_calamine-0.2.3-cp311-none-win_amd64.whl", hash = "sha256:a32c64e74673fb0203ad877c6ba4832de7976fd31c79c637552b567d295ff6b5"}, + {file = "python_calamine-0.2.3-cp311-none-win_arm64.whl", hash = "sha256:f8c4c9e7ade09b4122c59e3e0da7e5fba872a0e47d3076702185a4ffdf99dec4"}, + {file = "python_calamine-0.2.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:40e5f75c4a7bb2105e3bd65e7b4656e085c6d86e46af1c56468a2f87c2ed639a"}, + {file = "python_calamine-0.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3557bdd36060db4929f42bf4c2c728a76af60ccc95d5c98f2110331d993a7299"}, + {file = "python_calamine-0.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa75b28686f9dc727d26a97b41c6a2a6ca1d2c679139b6199edbae2782e7c77"}, + {file = "python_calamine-0.2.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d2c8577b00e13f5f43b1c03a2eca01848c3b24467ebaf597729d1e483613c110"}, + {file = "python_calamine-0.2.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4639255202380251833a9ab75c077e687ebbef2120f54030b2dc46eb6ce43105"}, + {file = "python_calamine-0.2.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:583656c6a6e8efac8951cd72459e2d84eea5f2617214ebc7e1c96217b44a0fa1"}, + {file = "python_calamine-0.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68fc61b34a1d82d3eee2109d323268dd455107dfb639b027aa5c388e2781273c"}, + {file = "python_calamine-0.2.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:64bb1f212275ed0288f578ee817e5cad4a063cfe5c38bf4c4dc6968957cb95b0"}, + {file = "python_calamine-0.2.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a7da299c1676dc34cd5f0adf93e92139afbfb832722d5d50a696ac180885aabb"}, + {file = "python_calamine-0.2.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:599752629ab0c5231159c5bea4f94795dd9b11a36c02dd5bd0613cf257ecd710"}, + {file = "python_calamine-0.2.3-cp312-none-win32.whl", hash = "sha256:fc73da2863c3251862583d64c0d07fe907f489a86a205e2b6ac94a39a1df1b42"}, + {file = "python_calamine-0.2.3-cp312-none-win_amd64.whl", hash = "sha256:a8d1662b4767f863c17ea4c1afc3c3fe3174d7b007ae77349d481e6792d142fe"}, + {file = "python_calamine-0.2.3-cp312-none-win_arm64.whl", hash = "sha256:87af11076364ade6f3da9e33993b6f55ec8dfd5f017129de688fd6d94d7bc24a"}, + {file = "python_calamine-0.2.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1ae98e1db1d3e74df08291f66d872bf7a4c47d96d39f8f589bff5dab873fbd13"}, + {file = "python_calamine-0.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bc270e8827191e7125600c97b61b3c78ec17d394820c2607c801f93c3475a0aa"}, + {file = "python_calamine-0.2.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c25b18eca7976aac0748fc122fa5109be66801d94b77a7676125fb825a8b67b9"}, + {file = "python_calamine-0.2.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:484330c0a917879afc615dc15e5ad925953a726f1a839ce3c35504a5befdae0c"}, + {file = "python_calamine-0.2.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c15ccb20f49eb6f824664ca8ec741edf09679977c2d41d13a02f0532f71a318b"}, + {file = "python_calamine-0.2.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19421a1b8a808333c39b03e007b74c85220700ceed1229449a21d51803d0671b"}, + {file = "python_calamine-0.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0cd8e3069c57a26eea5e6d3addb3dab812cc39b70f0cd11246d6f6592b7f293"}, + {file = "python_calamine-0.2.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d13822a6669a00da497394719a1fa63033ab79858fd653d330a6a7a681a5f6ce"}, + {file = "python_calamine-0.2.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:767db722eeb9c4d3847a87e4c3c4c9cc3e48938efaed4c507a5dd538a6bc5910"}, + {file = "python_calamine-0.2.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:4cac4095c25c64ef091fd994f62c5169f3ab0eec39c5bdbd0f319cac633b8183"}, + {file = "python_calamine-0.2.3-cp313-none-win32.whl", hash = "sha256:79aab3dc2c54525896b24002756e12fe09ec573efc2787285c244520bc17c39f"}, + {file = "python_calamine-0.2.3-cp313-none-win_amd64.whl", hash = "sha256:bd6606c893493eb555db5e63aef85b87fd806e6a0aa59bad0dbb591b88db2a0d"}, + {file = "python_calamine-0.2.3-cp313-none-win_arm64.whl", hash = "sha256:9f7b93851c941efba8387bb3c004437541230e8253230868204a079f1dacc21a"}, + {file = "python_calamine-0.2.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:5fa0395816ecff641b5df7ee3a2a953fb0f449a88f780e1c8b762b94578fdb9c"}, + {file = "python_calamine-0.2.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7397213b734e71434be06c3391ba9c23660215dc5e1c5601b8141f9f623fef84"}, + {file = "python_calamine-0.2.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be628b380f190b4140801731786f14d59d5a25c54398a724543181e6f46e71d3"}, + {file = "python_calamine-0.2.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d7fc182ebd15dd629d5c355207b125fd2301f109bc6cd2d91b1e67626fdbec1f"}, + {file = "python_calamine-0.2.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ae983b57379225f44102e0ff2f3724428174d0156ac42b1b69ed7f63ce105b1"}, + {file = "python_calamine-0.2.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98592f79f46cd2d74cd7f4e69ef2031a51138159a5852efe56fa5bc289c106b4"}, + {file = "python_calamine-0.2.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660347ae698f63f4a495b60411e913cfa448b149e7f51434934782559df6158f"}, + {file = "python_calamine-0.2.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fef87aa0b533c15e22ddb1bd6c257b3de9616c7a4ed3ca00c3c19e4cd8825d08"}, + {file = "python_calamine-0.2.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:06ab4232827eed11f6a40ddca5dd9015fe73a10c1cf71a4ab2aa26e63f3d1ffb"}, + {file = "python_calamine-0.2.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a6f64365bfc2cf6acefc3a618c7f25f64c317be3187d50dba3a2ccdbf405f911"}, + {file = "python_calamine-0.2.3-cp38-none-win32.whl", hash = "sha256:08b4b35d5943574ab44e87e4ccc2250f14ce7e8b34ad437ff95c1ae845823d0e"}, + {file = "python_calamine-0.2.3-cp38-none-win_amd64.whl", hash = "sha256:cd9b57326453be8ab52807cde90f3a61a008ed22a69489b41e9edbf66fb86a68"}, + {file = "python_calamine-0.2.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b439270ac6283a2e00abaae167ed35dececaa73f394bf5be8bf8631f3c9757fc"}, + {file = "python_calamine-0.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:38b6d1c315feaacfa95336f7d8d82bdc9fc75854ceae3dd003f075a4cf943582"}, + {file = "python_calamine-0.2.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:411812b0ffcf042be71408ae82b6fcc8dd70e2ee9ba8e8024a70242f7bce305e"}, + {file = "python_calamine-0.2.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4086c857d2cd1bf388bab6f18ca6ae453fb6618b8f3547e76447dc759b9a3a2a"}, + {file = "python_calamine-0.2.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c6b43b8d0b556cb6e9fa9280cc6a61945fcef0005622590c45fa1471705476b5"}, + {file = "python_calamine-0.2.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ce29ebf7b8bd978ef7aaf7755489f67f056327a53ef112a9b24c7a90970f9467"}, + {file = "python_calamine-0.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:042385ce2ba386ef72bd678ed44ee6d4a5de20c9561c3cd1ecd2a57bfdc874cc"}, + {file = "python_calamine-0.2.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9e55fd471afd1c50ad88b442ef20c57d7efd38c7c300992708aa2cff943a29b9"}, + {file = "python_calamine-0.2.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4972a653bd54a4513e9419c26576429b391cdb4b417e7afa46469089ee7c10ee"}, + {file = "python_calamine-0.2.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:206524d140eb7d2999791afd4dfd62ceed531af3cfa487ff2b8b8fdc4b7c2b50"}, + {file = "python_calamine-0.2.3-cp39-none-win32.whl", hash = "sha256:e5a2c540d631343ba9f16be2afbb7b9fa187b3ced1b292ecc4cfcd51b8859bef"}, + {file = "python_calamine-0.2.3-cp39-none-win_amd64.whl", hash = "sha256:af65a13551d6575468d7cfcc61028df5d4218796dc4886419049e136148694e6"}, + {file = "python_calamine-0.2.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:10f28b56fb84bd622e23f32881fd17b07ab039e7f2cacdfb6101dce702e77970"}, + {file = "python_calamine-0.2.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d00cef2e12e4b6660b5fab13f936194263e7e11f707f7951b1867995278051df"}, + {file = "python_calamine-0.2.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7aebcbd105e49516dd1831f05a0ffca7c9b85f855bf3a9c68f9bc509a212e381"}, + {file = "python_calamine-0.2.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d5a9182590f5ad12e08a0ba9b72dfe0e6b1780ff95153926e2f4564a6018a14"}, + {file = "python_calamine-0.2.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2af3805806088acc7b4d766b58b03d08947a7100e1ef26e55509161adbb36201"}, + {file = "python_calamine-0.2.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5283e049cc36a0e2442f72d0c2c156dc1e7dc7ca48cba02d52c5cb223525b5c3"}, + {file = "python_calamine-0.2.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:9b7d0ef322f073099ea69e4a3db8c31ff4c4f7cdf4cd333f0577ab0c9320eaf5"}, + {file = "python_calamine-0.2.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0bcd07be6953efb08340ccb19b9ae0732b104a9e672edf1ffd2d6b3cc226d815"}, + {file = "python_calamine-0.2.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a8b12de6e2329643dd6b0a56570b853b94149ca7b1b323db3f69a06f61ec1e2"}, + {file = "python_calamine-0.2.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:cad27b0e491060dc72653ccd9288301120b23261e3e374f2401cc133547615d4"}, + {file = "python_calamine-0.2.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:303e2f2a1bdfaf428db7aca50d954667078c0cdf1b585ff090dfca2fac9107d7"}, + {file = "python_calamine-0.2.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a21187b6ebcdabdfe2113df11c2a522b9adc02bcf54bd3ba424ca8c6762cd9b"}, + {file = "python_calamine-0.2.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2773094cc62602f6bcc2acd8e905b3e2292daf6a6c24ddbc85f41065604fd9d4"}, + {file = "python_calamine-0.2.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6de5646a9ec3d24b5089ed174f4dcee13620e65e20dc463097c00e803c81f86f"}, + {file = "python_calamine-0.2.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e976c948ab18e9fee589994b68878381e1e393d870362babf9634258deb4f13b"}, + {file = "python_calamine-0.2.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:00fdfd24d13d8b04619dd933be4888bc6a70427e217fb179f3a1f71f2e377219"}, + {file = "python_calamine-0.2.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ab7d60482520508ebf00476cde1b97011084a2e73ac49b2ca32003547e7444c9"}, + {file = "python_calamine-0.2.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:00c915fc67b0b4e1ddd000d374bd808d947f2ecb0f6051a4669a77abada4b7b8"}, + {file = "python_calamine-0.2.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c869fe1b568a2a970b13dd59a58a13a81a667aff2f365a95a577555585ff14bc"}, + {file = "python_calamine-0.2.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:602ebad70b176a41f22547d6bb99a6d32a531a11dbf74720f3984e6bf98c94ab"}, + {file = "python_calamine-0.2.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f6a7c4eb79803ee7cdfd00a0b8267c60c33f25da8bb9275f6168a4dd1a54db76"}, + {file = "python_calamine-0.2.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:68275fed9dcbe90a9185c9919980933e4feea925db178461f0cdb336a2587021"}, + {file = "python_calamine-0.2.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5efc667fd002db9482a7b9f2c70b41fa69c86e18206132be1a0adcad3c998c17"}, + {file = "python_calamine-0.2.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d2d845cbcd767c7b85c616849f0c6cd619662adb98d86af2a3fd8630d6acc48d"}, + {file = "python_calamine-0.2.3.tar.gz", hash = "sha256:d6b3858c3756629d9b4a166de0facfa6c8033fa0b73dcddd3d82144f3170c0dc"}, ] [[package]] @@ -6983,6 +6954,23 @@ maintainer = ["zest.releaser[recommended]"] pil = ["pillow (>=9.1.0)"] test = ["coverage", "pytest"] +[[package]] +name = "rank-bm25" +version = "0.2.2" +description = "Various BM25 algorithms for document ranking" +optional = false +python-versions = "*" +files = [ + {file = "rank_bm25-0.2.2-py3-none-any.whl", hash = "sha256:7bd4a95571adadfc271746fa146a4bcfd89c0cf731e49c3d1ad863290adbe8ae"}, + {file = "rank_bm25-0.2.2.tar.gz", hash = "sha256:096ccef76f8188563419aaf384a02f0ea459503fdf77901378d4fd9d87e5e51d"}, +] + +[package.dependencies] +numpy = "*" + +[package.extras] +dev = ["pytest"] + [[package]] name = "rapidfuzz" version = "3.9.4" @@ -7290,6 +7278,20 @@ requests = ">=2.0.0" [package.extras] rsa = ["oauthlib[signedtoken] (>=3.0.0)"] +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +description = "A utility belt for advanced users of python-requests" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, + {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, +] + +[package.dependencies] +requests = ">=2.0.1,<3.0.0" + [[package]] name = "resend" version = "0.7.2" @@ -7338,29 +7340,29 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.5.1" +version = "0.5.4" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.5.1-py3-none-linux_armv6l.whl", hash = "sha256:6ecf968fcf94d942d42b700af18ede94b07521bd188aaf2cd7bc898dd8cb63b6"}, - {file = "ruff-0.5.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:204fb0a472f00f2e6280a7c8c7c066e11e20e23a37557d63045bf27a616ba61c"}, - {file = "ruff-0.5.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d235968460e8758d1e1297e1de59a38d94102f60cafb4d5382033c324404ee9d"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38beace10b8d5f9b6bdc91619310af6d63dd2019f3fb2d17a2da26360d7962fa"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e478d2f09cf06add143cf8c4540ef77b6599191e0c50ed976582f06e588c994"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0368d765eec8247b8550251c49ebb20554cc4e812f383ff9f5bf0d5d94190b0"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:3a9a9a1b582e37669b0138b7c1d9d60b9edac880b80eb2baba6d0e566bdeca4d"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bdd9f723e16003623423affabcc0a807a66552ee6a29f90eddad87a40c750b78"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be9fd62c1e99539da05fcdc1e90d20f74aec1b7a1613463ed77870057cd6bd96"}, - {file = "ruff-0.5.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e216fc75a80ea1fbd96af94a6233d90190d5b65cc3d5dfacf2bd48c3e067d3e1"}, - {file = "ruff-0.5.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c4c2112e9883a40967827d5c24803525145e7dab315497fae149764979ac7929"}, - {file = "ruff-0.5.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dfaf11c8a116394da3b65cd4b36de30d8552fa45b8119b9ef5ca6638ab964fa3"}, - {file = "ruff-0.5.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d7ceb9b2fe700ee09a0c6b192c5ef03c56eb82a0514218d8ff700f6ade004108"}, - {file = "ruff-0.5.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bac6288e82f6296f82ed5285f597713acb2a6ae26618ffc6b429c597b392535c"}, - {file = "ruff-0.5.1-py3-none-win32.whl", hash = "sha256:5c441d9c24ec09e1cb190a04535c5379b36b73c4bc20aa180c54812c27d1cca4"}, - {file = "ruff-0.5.1-py3-none-win_amd64.whl", hash = "sha256:b1789bf2cd3d1b5a7d38397cac1398ddf3ad7f73f4de01b1e913e2abc7dfc51d"}, - {file = "ruff-0.5.1-py3-none-win_arm64.whl", hash = "sha256:2875b7596a740cbbd492f32d24be73e545a4ce0a3daf51e4f4e609962bfd3cd2"}, - {file = "ruff-0.5.1.tar.gz", hash = "sha256:3164488aebd89b1745b47fd00604fb4358d774465f20d1fcd907f9c0fc1b0655"}, + {file = "ruff-0.5.4-py3-none-linux_armv6l.whl", hash = "sha256:82acef724fc639699b4d3177ed5cc14c2a5aacd92edd578a9e846d5b5ec18ddf"}, + {file = "ruff-0.5.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:da62e87637c8838b325e65beee485f71eb36202ce8e3cdbc24b9fcb8b99a37be"}, + {file = "ruff-0.5.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e98ad088edfe2f3b85a925ee96da652028f093d6b9b56b76fc242d8abb8e2059"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c55efbecc3152d614cfe6c2247a3054cfe358cefbf794f8c79c8575456efe19"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9b85eaa1f653abd0a70603b8b7008d9e00c9fa1bbd0bf40dad3f0c0bdd06793"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cf497a47751be8c883059c4613ba2f50dd06ec672692de2811f039432875278"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:09c14ed6a72af9ccc8d2e313d7acf7037f0faff43cde4b507e66f14e812e37f7"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:628f6b8f97b8bad2490240aa84f3e68f390e13fabc9af5c0d3b96b485921cd60"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3520a00c0563d7a7a7c324ad7e2cde2355733dafa9592c671fb2e9e3cd8194c1"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93789f14ca2244fb91ed481456f6d0bb8af1f75a330e133b67d08f06ad85b516"}, + {file = "ruff-0.5.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:029454e2824eafa25b9df46882f7f7844d36fd8ce51c1b7f6d97e2615a57bbcc"}, + {file = "ruff-0.5.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9492320eed573a13a0bc09a2957f17aa733fff9ce5bf00e66e6d4a88ec33813f"}, + {file = "ruff-0.5.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a6e1f62a92c645e2919b65c02e79d1f61e78a58eddaebca6c23659e7c7cb4ac7"}, + {file = "ruff-0.5.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:768fa9208df2bec4b2ce61dbc7c2ddd6b1be9fb48f1f8d3b78b3332c7d71c1ff"}, + {file = "ruff-0.5.4-py3-none-win32.whl", hash = "sha256:e1e7393e9c56128e870b233c82ceb42164966f25b30f68acbb24ed69ce9c3a4e"}, + {file = "ruff-0.5.4-py3-none-win_amd64.whl", hash = "sha256:58b54459221fd3f661a7329f177f091eb35cf7a603f01d9eb3eb11cc348d38c4"}, + {file = "ruff-0.5.4-py3-none-win_arm64.whl", hash = "sha256:bd53da65f1085fb5b307c38fd3c0829e76acf7b2a912d8d79cadcdb4875c1eb7"}, + {file = "ruff-0.5.4.tar.gz", hash = "sha256:2795726d5f71c4f4e70653273d1c23a8182f07dd8e48c12de5d867bfb7557eed"}, ] [[package]] @@ -7502,15 +7504,102 @@ tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools-rust (>=1.5.2)"] torch = ["safetensors[numpy]", "torch (>=1.10)"] +[[package]] +name = "scikit-learn" +version = "1.5.1" +description = "A set of python modules for machine learning and data mining" +optional = false +python-versions = ">=3.9" +files = [ + {file = "scikit_learn-1.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:781586c414f8cc58e71da4f3d7af311e0505a683e112f2f62919e3019abd3745"}, + {file = "scikit_learn-1.5.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:f5b213bc29cc30a89a3130393b0e39c847a15d769d6e59539cd86b75d276b1a7"}, + {file = "scikit_learn-1.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ff4ba34c2abff5ec59c803ed1d97d61b036f659a17f55be102679e88f926fac"}, + {file = "scikit_learn-1.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:161808750c267b77b4a9603cf9c93579c7a74ba8486b1336034c2f1579546d21"}, + {file = "scikit_learn-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:10e49170691514a94bb2e03787aa921b82dbc507a4ea1f20fd95557862c98dc1"}, + {file = "scikit_learn-1.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:154297ee43c0b83af12464adeab378dee2d0a700ccd03979e2b821e7dd7cc1c2"}, + {file = "scikit_learn-1.5.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b5e865e9bd59396220de49cb4a57b17016256637c61b4c5cc81aaf16bc123bbe"}, + {file = "scikit_learn-1.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:909144d50f367a513cee6090873ae582dba019cb3fca063b38054fa42704c3a4"}, + {file = "scikit_learn-1.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:689b6f74b2c880276e365fe84fe4f1befd6a774f016339c65655eaff12e10cbf"}, + {file = "scikit_learn-1.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:9a07f90846313a7639af6a019d849ff72baadfa4c74c778821ae0fad07b7275b"}, + {file = "scikit_learn-1.5.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5944ce1faada31c55fb2ba20a5346b88e36811aab504ccafb9f0339e9f780395"}, + {file = "scikit_learn-1.5.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:0828673c5b520e879f2af6a9e99eee0eefea69a2188be1ca68a6121b809055c1"}, + {file = "scikit_learn-1.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:508907e5f81390e16d754e8815f7497e52139162fd69c4fdbd2dfa5d6cc88915"}, + {file = "scikit_learn-1.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97625f217c5c0c5d0505fa2af28ae424bd37949bb2f16ace3ff5f2f81fb4498b"}, + {file = "scikit_learn-1.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:da3f404e9e284d2b0a157e1b56b6566a34eb2798205cba35a211df3296ab7a74"}, + {file = "scikit_learn-1.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:88e0672c7ac21eb149d409c74cc29f1d611d5158175846e7a9c2427bd12b3956"}, + {file = "scikit_learn-1.5.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:7b073a27797a283187a4ef4ee149959defc350b46cbf63a84d8514fe16b69855"}, + {file = "scikit_learn-1.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b59e3e62d2be870e5c74af4e793293753565c7383ae82943b83383fdcf5cc5c1"}, + {file = "scikit_learn-1.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bd8d3a19d4bd6dc5a7d4f358c8c3a60934dc058f363c34c0ac1e9e12a31421d"}, + {file = "scikit_learn-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:5f57428de0c900a98389c4a433d4a3cf89de979b3aa24d1c1d251802aa15e44d"}, + {file = "scikit_learn-1.5.1.tar.gz", hash = "sha256:0ea5d40c0e3951df445721927448755d3fe1d80833b0b7308ebff5d2a45e6414"}, +] + +[package.dependencies] +joblib = ">=1.2.0" +numpy = ">=1.19.5" +scipy = ">=1.6.0" +threadpoolctl = ">=3.1.0" + +[package.extras] +benchmark = ["matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "pandas (>=1.1.5)"] +build = ["cython (>=3.0.10)", "meson-python (>=0.16.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)"] +docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "polars (>=0.20.23)", "pooch (>=1.6.0)", "pydata-sphinx-theme (>=0.15.3)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)", "sphinx (>=7.3.7)", "sphinx-copybutton (>=0.5.2)", "sphinx-design (>=0.5.0)", "sphinx-gallery (>=0.16.0)", "sphinx-prompt (>=1.4.0)", "sphinx-remove-toctrees (>=1.0.0.post1)", "sphinxcontrib-sass (>=0.3.4)", "sphinxext-opengraph (>=0.9.1)"] +examples = ["matplotlib (>=3.3.4)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)"] +install = ["joblib (>=1.2.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)", "threadpoolctl (>=3.1.0)"] +maintenance = ["conda-lock (==2.5.6)"] +tests = ["black (>=24.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.9)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.20.23)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.2.1)", "scikit-image (>=0.17.2)"] + +[[package]] +name = "scipy" +version = "1.14.0" +description = "Fundamental algorithms for scientific computing in Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "scipy-1.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7e911933d54ead4d557c02402710c2396529540b81dd554fc1ba270eb7308484"}, + {file = "scipy-1.14.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:687af0a35462402dd851726295c1a5ae5f987bd6e9026f52e9505994e2f84ef6"}, + {file = "scipy-1.14.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:07e179dc0205a50721022344fb85074f772eadbda1e1b3eecdc483f8033709b7"}, + {file = "scipy-1.14.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:6a9c9a9b226d9a21e0a208bdb024c3982932e43811b62d202aaf1bb59af264b1"}, + {file = "scipy-1.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:076c27284c768b84a45dcf2e914d4000aac537da74236a0d45d82c6fa4b7b3c0"}, + {file = "scipy-1.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42470ea0195336df319741e230626b6225a740fd9dce9642ca13e98f667047c0"}, + {file = "scipy-1.14.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:176c6f0d0470a32f1b2efaf40c3d37a24876cebf447498a4cefb947a79c21e9d"}, + {file = "scipy-1.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:ad36af9626d27a4326c8e884917b7ec321d8a1841cd6dacc67d2a9e90c2f0359"}, + {file = "scipy-1.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6d056a8709ccda6cf36cdd2eac597d13bc03dba38360f418560a93050c76a16e"}, + {file = "scipy-1.14.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:f0a50da861a7ec4573b7c716b2ebdcdf142b66b756a0d392c236ae568b3a93fb"}, + {file = "scipy-1.14.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:94c164a9e2498e68308e6e148646e486d979f7fcdb8b4cf34b5441894bdb9caf"}, + {file = "scipy-1.14.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:a7d46c3e0aea5c064e734c3eac5cf9eb1f8c4ceee756262f2c7327c4c2691c86"}, + {file = "scipy-1.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eee2989868e274aae26125345584254d97c56194c072ed96cb433f32f692ed8"}, + {file = "scipy-1.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e3154691b9f7ed73778d746da2df67a19d046a6c8087c8b385bc4cdb2cfca74"}, + {file = "scipy-1.14.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c40003d880f39c11c1edbae8144e3813904b10514cd3d3d00c277ae996488cdb"}, + {file = "scipy-1.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:5b083c8940028bb7e0b4172acafda6df762da1927b9091f9611b0bcd8676f2bc"}, + {file = "scipy-1.14.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bff2438ea1330e06e53c424893ec0072640dac00f29c6a43a575cbae4c99b2b9"}, + {file = "scipy-1.14.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:bbc0471b5f22c11c389075d091d3885693fd3f5e9a54ce051b46308bc787e5d4"}, + {file = "scipy-1.14.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:64b2ff514a98cf2bb734a9f90d32dc89dc6ad4a4a36a312cd0d6327170339eb0"}, + {file = "scipy-1.14.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:7d3da42fbbbb860211a811782504f38ae7aaec9de8764a9bef6b262de7a2b50f"}, + {file = "scipy-1.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d91db2c41dd6c20646af280355d41dfa1ec7eead235642178bd57635a3f82209"}, + {file = "scipy-1.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a01cc03bcdc777c9da3cfdcc74b5a75caffb48a6c39c8450a9a05f82c4250a14"}, + {file = "scipy-1.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:65df4da3c12a2bb9ad52b86b4dcf46813e869afb006e58be0f516bc370165159"}, + {file = "scipy-1.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:4c4161597c75043f7154238ef419c29a64ac4a7c889d588ea77690ac4d0d9b20"}, + {file = "scipy-1.14.0.tar.gz", hash = "sha256:b5923f48cb840380f9854339176ef21763118a7300a88203ccd0bdd26e58527b"}, +] + +[package.dependencies] +numpy = ">=1.23.5,<2.3" + +[package.extras] +dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] +doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.13.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"] +test = ["Cython", "array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + [[package]] name = "sentry-sdk" -version = "1.39.2" +version = "1.44.1" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.39.2.tar.gz", hash = "sha256:24c83b0b41c887d33328a9166f5950dc37ad58f01c9f2fbff6b87a6f1094170c"}, - {file = "sentry_sdk-1.39.2-py2.py3-none-any.whl", hash = "sha256:acaf597b30258fc7663063b291aa99e58f3096e91fe1e6634f4b79f9c1943e8e"}, + {file = "sentry-sdk-1.44.1.tar.gz", hash = "sha256:24e6a53eeabffd2f95d952aa35ca52f0f4201d17f820ac9d3ff7244c665aaf68"}, + {file = "sentry_sdk-1.44.1-py2.py3-none-any.whl", hash = "sha256:5f75eb91d8ab6037c754a87b8501cc581b2827e923682f593bed3539ce5b3999"}, ] [package.dependencies] @@ -7527,6 +7616,7 @@ asyncpg = ["asyncpg (>=0.23)"] beam = ["apache-beam (>=2.12)"] bottle = ["bottle (>=0.12.13)"] celery = ["celery (>=3)"] +celery-redbeat = ["celery-redbeat (>=2)"] chalice = ["chalice (>=1.16.0)"] clickhouse-driver = ["clickhouse-driver (>=0.2.0)"] django = ["django (>=1.8)"] @@ -7537,9 +7627,10 @@ grpcio = ["grpcio (>=1.21.1)"] httpx = ["httpx (>=0.16.0)"] huey = ["huey (>=2)"] loguru = ["loguru (>=0.5)"] +openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] opentelemetry-experimental = ["opentelemetry-distro (>=0.40b0,<1.0)", "opentelemetry-instrumentation-aiohttp-client (>=0.40b0,<1.0)", "opentelemetry-instrumentation-django (>=0.40b0,<1.0)", "opentelemetry-instrumentation-fastapi (>=0.40b0,<1.0)", "opentelemetry-instrumentation-flask (>=0.40b0,<1.0)", "opentelemetry-instrumentation-requests (>=0.40b0,<1.0)", "opentelemetry-instrumentation-sqlite3 (>=0.40b0,<1.0)", "opentelemetry-instrumentation-urllib (>=0.40b0,<1.0)"] -pure-eval = ["asttokens", "executing", "pure_eval"] +pure-eval = ["asttokens", "executing", "pure-eval"] pymongo = ["pymongo (>=3.1)"] pyspark = ["pyspark (>=2.4.4)"] quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] @@ -7552,18 +7643,19 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "70.2.0" +version = "71.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.2.0-py3-none-any.whl", hash = "sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05"}, - {file = "setuptools-70.2.0.tar.gz", hash = "sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1"}, + {file = "setuptools-71.1.0-py3-none-any.whl", hash = "sha256:33874fdc59b3188304b2e7c80d9029097ea31627180896fb549c578ceb8a0855"}, + {file = "setuptools-71.1.0.tar.gz", hash = "sha256:032d42ee9fb536e33087fb66cac5f840eb9391ed05637b3f2a76a7c8fb477936"}, ] [package.extras] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "sgmllib3k" @@ -7577,52 +7669,47 @@ files = [ [[package]] name = "shapely" -version = "2.0.4" +version = "2.0.5" description = "Manipulation and analysis of geometric objects" optional = false python-versions = ">=3.7" files = [ - {file = "shapely-2.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:011b77153906030b795791f2fdfa2d68f1a8d7e40bce78b029782ade3afe4f2f"}, - {file = "shapely-2.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9831816a5d34d5170aa9ed32a64982c3d6f4332e7ecfe62dc97767e163cb0b17"}, - {file = "shapely-2.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5c4849916f71dc44e19ed370421518c0d86cf73b26e8656192fcfcda08218fbd"}, - {file = "shapely-2.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:841f93a0e31e4c64d62ea570d81c35de0f6cea224568b2430d832967536308e6"}, - {file = "shapely-2.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b4431f522b277c79c34b65da128029a9955e4481462cbf7ebec23aab61fc58"}, - {file = "shapely-2.0.4-cp310-cp310-win32.whl", hash = "sha256:92a41d936f7d6743f343be265ace93b7c57f5b231e21b9605716f5a47c2879e7"}, - {file = "shapely-2.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:30982f79f21bb0ff7d7d4a4e531e3fcaa39b778584c2ce81a147f95be1cd58c9"}, - {file = "shapely-2.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:de0205cb21ad5ddaef607cda9a3191eadd1e7a62a756ea3a356369675230ac35"}, - {file = "shapely-2.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7d56ce3e2a6a556b59a288771cf9d091470116867e578bebced8bfc4147fbfd7"}, - {file = "shapely-2.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:58b0ecc505bbe49a99551eea3f2e8a9b3b24b3edd2a4de1ac0dc17bc75c9ec07"}, - {file = "shapely-2.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:790a168a808bd00ee42786b8ba883307c0e3684ebb292e0e20009588c426da47"}, - {file = "shapely-2.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4310b5494271e18580d61022c0857eb85d30510d88606fa3b8314790df7f367d"}, - {file = "shapely-2.0.4-cp311-cp311-win32.whl", hash = "sha256:63f3a80daf4f867bd80f5c97fbe03314348ac1b3b70fb1c0ad255a69e3749879"}, - {file = "shapely-2.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:c52ed79f683f721b69a10fb9e3d940a468203f5054927215586c5d49a072de8d"}, - {file = "shapely-2.0.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:5bbd974193e2cc274312da16b189b38f5f128410f3377721cadb76b1e8ca5328"}, - {file = "shapely-2.0.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:41388321a73ba1a84edd90d86ecc8bfed55e6a1e51882eafb019f45895ec0f65"}, - {file = "shapely-2.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0776c92d584f72f1e584d2e43cfc5542c2f3dd19d53f70df0900fda643f4bae6"}, - {file = "shapely-2.0.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c75c98380b1ede1cae9a252c6dc247e6279403fae38c77060a5e6186c95073ac"}, - {file = "shapely-2.0.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3e700abf4a37b7b8b90532fa6ed5c38a9bfc777098bc9fbae5ec8e618ac8f30"}, - {file = "shapely-2.0.4-cp312-cp312-win32.whl", hash = "sha256:4f2ab0faf8188b9f99e6a273b24b97662194160cc8ca17cf9d1fb6f18d7fb93f"}, - {file = "shapely-2.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:03152442d311a5e85ac73b39680dd64a9892fa42bb08fd83b3bab4fe6999bfa0"}, - {file = "shapely-2.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:994c244e004bc3cfbea96257b883c90a86e8cbd76e069718eb4c6b222a56f78b"}, - {file = "shapely-2.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05ffd6491e9e8958b742b0e2e7c346635033d0a5f1a0ea083547fcc854e5d5cf"}, - {file = "shapely-2.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbdc1140a7d08faa748256438291394967aa54b40009f54e8d9825e75ef6113"}, - {file = "shapely-2.0.4-cp37-cp37m-win32.whl", hash = "sha256:5af4cd0d8cf2912bd95f33586600cac9c4b7c5053a036422b97cfe4728d2eb53"}, - {file = "shapely-2.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:464157509ce4efa5ff285c646a38b49f8c5ef8d4b340f722685b09bb033c5ccf"}, - {file = "shapely-2.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:489c19152ec1f0e5c5e525356bcbf7e532f311bff630c9b6bc2db6f04da6a8b9"}, - {file = "shapely-2.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b79bbd648664aa6f44ef018474ff958b6b296fed5c2d42db60078de3cffbc8aa"}, - {file = "shapely-2.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:674d7baf0015a6037d5758496d550fc1946f34bfc89c1bf247cabdc415d7747e"}, - {file = "shapely-2.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cd4ccecc5ea5abd06deeaab52fcdba372f649728050c6143cc405ee0c166679"}, - {file = "shapely-2.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb5cdcbbe3080181498931b52a91a21a781a35dcb859da741c0345c6402bf00c"}, - {file = "shapely-2.0.4-cp38-cp38-win32.whl", hash = "sha256:55a38dcd1cee2f298d8c2ebc60fc7d39f3b4535684a1e9e2f39a80ae88b0cea7"}, - {file = "shapely-2.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:ec555c9d0db12d7fd777ba3f8b75044c73e576c720a851667432fabb7057da6c"}, - {file = "shapely-2.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3f9103abd1678cb1b5f7e8e1af565a652e036844166c91ec031eeb25c5ca8af0"}, - {file = "shapely-2.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:263bcf0c24d7a57c80991e64ab57cba7a3906e31d2e21b455f493d4aab534aaa"}, - {file = "shapely-2.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ddf4a9bfaac643e62702ed662afc36f6abed2a88a21270e891038f9a19bc08fc"}, - {file = "shapely-2.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:485246fcdb93336105c29a5cfbff8a226949db37b7473c89caa26c9bae52a242"}, - {file = "shapely-2.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8de4578e838a9409b5b134a18ee820730e507b2d21700c14b71a2b0757396acc"}, - {file = "shapely-2.0.4-cp39-cp39-win32.whl", hash = "sha256:9dab4c98acfb5fb85f5a20548b5c0abe9b163ad3525ee28822ffecb5c40e724c"}, - {file = "shapely-2.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:31c19a668b5a1eadab82ff070b5a260478ac6ddad3a5b62295095174a8d26398"}, - {file = "shapely-2.0.4.tar.gz", hash = "sha256:5dc736127fac70009b8d309a0eeb74f3e08979e530cf7017f2f507ef62e6cfb8"}, + {file = "shapely-2.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:89d34787c44f77a7d37d55ae821f3a784fa33592b9d217a45053a93ade899375"}, + {file = "shapely-2.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:798090b426142df2c5258779c1d8d5734ec6942f778dab6c6c30cfe7f3bf64ff"}, + {file = "shapely-2.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45211276900c4790d6bfc6105cbf1030742da67594ea4161a9ce6812a6721e68"}, + {file = "shapely-2.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e119444bc27ca33e786772b81760f2028d930ac55dafe9bc50ef538b794a8e1"}, + {file = "shapely-2.0.5-cp310-cp310-win32.whl", hash = "sha256:9a4492a2b2ccbeaebf181e7310d2dfff4fdd505aef59d6cb0f217607cb042fb3"}, + {file = "shapely-2.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:1e5cb5ee72f1bc7ace737c9ecd30dc174a5295fae412972d3879bac2e82c8fae"}, + {file = "shapely-2.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5bbfb048a74cf273db9091ff3155d373020852805a37dfc846ab71dde4be93ec"}, + {file = "shapely-2.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93be600cbe2fbaa86c8eb70656369f2f7104cd231f0d6585c7d0aa555d6878b8"}, + {file = "shapely-2.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f8e71bb9a46814019f6644c4e2560a09d44b80100e46e371578f35eaaa9da1c"}, + {file = "shapely-2.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5251c28a29012e92de01d2e84f11637eb1d48184ee8f22e2df6c8c578d26760"}, + {file = "shapely-2.0.5-cp311-cp311-win32.whl", hash = "sha256:35110e80070d664781ec7955c7de557456b25727a0257b354830abb759bf8311"}, + {file = "shapely-2.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:6c6b78c0007a34ce7144f98b7418800e0a6a5d9a762f2244b00ea560525290c9"}, + {file = "shapely-2.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:03bd7b5fa5deb44795cc0a503999d10ae9d8a22df54ae8d4a4cd2e8a93466195"}, + {file = "shapely-2.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ff9521991ed9e201c2e923da014e766c1aa04771bc93e6fe97c27dcf0d40ace"}, + {file = "shapely-2.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b65365cfbf657604e50d15161ffcc68de5cdb22a601bbf7823540ab4918a98d"}, + {file = "shapely-2.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21f64e647a025b61b19585d2247137b3a38a35314ea68c66aaf507a1c03ef6fe"}, + {file = "shapely-2.0.5-cp312-cp312-win32.whl", hash = "sha256:3ac7dc1350700c139c956b03d9c3df49a5b34aaf91d024d1510a09717ea39199"}, + {file = "shapely-2.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:30e8737983c9d954cd17feb49eb169f02f1da49e24e5171122cf2c2b62d65c95"}, + {file = "shapely-2.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ff7731fea5face9ec08a861ed351734a79475631b7540ceb0b66fb9732a5f529"}, + {file = "shapely-2.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff9e520af0c5a578e174bca3c18713cd47a6c6a15b6cf1f50ac17dc8bb8db6a2"}, + {file = "shapely-2.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49b299b91557b04acb75e9732645428470825061f871a2edc36b9417d66c1fc5"}, + {file = "shapely-2.0.5-cp37-cp37m-win32.whl", hash = "sha256:b5870633f8e684bf6d1ae4df527ddcb6f3895f7b12bced5c13266ac04f47d231"}, + {file = "shapely-2.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:401cb794c5067598f50518e5a997e270cd7642c4992645479b915c503866abed"}, + {file = "shapely-2.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e91ee179af539100eb520281ba5394919067c6b51824e6ab132ad4b3b3e76dd0"}, + {file = "shapely-2.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8af6f7260f809c0862741ad08b1b89cb60c130ae30efab62320bbf4ee9cc71fa"}, + {file = "shapely-2.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5456dd522800306ba3faef77c5ba847ec30a0bd73ab087a25e0acdd4db2514f"}, + {file = "shapely-2.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b714a840402cde66fd7b663bb08cacb7211fa4412ea2a209688f671e0d0631fd"}, + {file = "shapely-2.0.5-cp38-cp38-win32.whl", hash = "sha256:7e8cf5c252fac1ea51b3162be2ec3faddedc82c256a1160fc0e8ddbec81b06d2"}, + {file = "shapely-2.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:4461509afdb15051e73ab178fae79974387f39c47ab635a7330d7fee02c68a3f"}, + {file = "shapely-2.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7545a39c55cad1562be302d74c74586f79e07b592df8ada56b79a209731c0219"}, + {file = "shapely-2.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4c83a36f12ec8dee2066946d98d4d841ab6512a6ed7eb742e026a64854019b5f"}, + {file = "shapely-2.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89e640c2cd37378480caf2eeda9a51be64201f01f786d127e78eaeff091ec897"}, + {file = "shapely-2.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06efe39beafde3a18a21dde169d32f315c57da962826a6d7d22630025200c5e6"}, + {file = "shapely-2.0.5-cp39-cp39-win32.whl", hash = "sha256:8203a8b2d44dcb366becbc8c3d553670320e4acf0616c39e218c9561dd738d92"}, + {file = "shapely-2.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:7fed9dbfbcfec2682d9a047b9699db8dcc890dfca857ecba872c42185fc9e64e"}, + {file = "shapely-2.0.5.tar.gz", hash = "sha256:bff2366bc786bfa6cb353d6b47d0443c570c32776612e527ee47b6df63fcfe32"}, ] [package.dependencies] @@ -7793,13 +7880,13 @@ sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sqlparse" -version = "0.5.0" +version = "0.5.1" description = "A non-validating SQL parser." optional = false python-versions = ">=3.8" files = [ - {file = "sqlparse-0.5.0-py3-none-any.whl", hash = "sha256:c204494cd97479d0e39f28c93d46c0b2d5959c7b9ab904762ea6c7af211c8663"}, - {file = "sqlparse-0.5.0.tar.gz", hash = "sha256:714d0a4932c059d16189f58ef5411ec2287a4360f17cdd0edd2d09d4c5087c93"}, + {file = "sqlparse-0.5.1-py3-none-any.whl", hash = "sha256:773dcbf9a5ab44a090f3441e2180efe2560220203dc2f8c0b0fa141e18b505e4"}, + {file = "sqlparse-0.5.1.tar.gz", hash = "sha256:bb6b4df465655ef332548e24f08e205afc81b9ab86cb1c45657a7ff173a3a00e"}, ] [package.extras] @@ -7883,13 +7970,13 @@ test = ["pytest", "tornado (>=4.5)", "typeguard"] [[package]] name = "tencentcloud-sdk-python-common" -version = "3.0.1183" +version = "3.0.1196" description = "Tencent Cloud Common SDK for Python" optional = false python-versions = "*" files = [ - {file = "tencentcloud-sdk-python-common-3.0.1183.tar.gz", hash = "sha256:59f8175cd2be20badfbed035637794d1d827071dd4e9d746543689254a9eae47"}, - {file = "tencentcloud_sdk_python_common-3.0.1183-py2.py3-none-any.whl", hash = "sha256:9deb38d80f7d8fbaf45b46f201f8c0c324a78cc0cb6c5034c1da84a06116af88"}, + {file = "tencentcloud-sdk-python-common-3.0.1196.tar.gz", hash = "sha256:a8acd14f7480987ff0fd1d961ad934b2b7533ab1937d7e3adb74d95dc49954bd"}, + {file = "tencentcloud_sdk_python_common-3.0.1196-py2.py3-none-any.whl", hash = "sha256:5ed438bc3e2818ca8e84b3896aaa2746798fba981bd94b27528eb36efa5b4a30"}, ] [package.dependencies] @@ -7897,17 +7984,28 @@ requests = ">=2.16.0" [[package]] name = "tencentcloud-sdk-python-hunyuan" -version = "3.0.1183" +version = "3.0.1196" description = "Tencent Cloud Hunyuan SDK for Python" optional = false python-versions = "*" files = [ - {file = "tencentcloud-sdk-python-hunyuan-3.0.1183.tar.gz", hash = "sha256:5648994f0124c694ee75dd498d991ca632c8dc8d55b6d349d8cc7fd5bc33b1bd"}, - {file = "tencentcloud_sdk_python_hunyuan-3.0.1183-py2.py3-none-any.whl", hash = "sha256:01bfdf33ea04ed791931636c3eafa569a0387f623967ef880ff220b3c548e6f5"}, + {file = "tencentcloud-sdk-python-hunyuan-3.0.1196.tar.gz", hash = "sha256:ced26497ae5f1b8fcc6cbd12238109274251e82fa1cfedfd6700df776306a36c"}, + {file = "tencentcloud_sdk_python_hunyuan-3.0.1196-py2.py3-none-any.whl", hash = "sha256:d18a19cffeaf4ff8a60670dc2bdb644f3d7ae6a51c30d21b50ded24a9c542248"}, ] [package.dependencies] -tencentcloud-sdk-python-common = "3.0.1183" +tencentcloud-sdk-python-common = "3.0.1196" + +[[package]] +name = "threadpoolctl" +version = "3.5.0" +description = "threadpoolctl" +optional = false +python-versions = ">=3.8" +files = [ + {file = "threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"}, + {file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"}, +] [[package]] name = "tidb-vector" @@ -8282,13 +8380,13 @@ typing-extensions = ">=3.7.4.3" [[package]] name = "types-requests" -version = "2.32.0.20240622" +version = "2.32.0.20240712" description = "Typing stubs for requests" optional = false python-versions = ">=3.8" files = [ - {file = "types-requests-2.32.0.20240622.tar.gz", hash = "sha256:ed5e8a412fcc39159d6319385c009d642845f250c63902718f605cd90faade31"}, - {file = "types_requests-2.32.0.20240622-py3-none-any.whl", hash = "sha256:97bac6b54b5bd4cf91d407e62f0932a74821bc2211f22116d9ee1dd643826caf"}, + {file = "types-requests-2.32.0.20240712.tar.gz", hash = "sha256:90c079ff05e549f6bf50e02e910210b98b8ff1ebdd18e19c873cd237737c1358"}, + {file = "types_requests-2.32.0.20240712-py3-none-any.whl", hash = "sha256:f754283e152c752e46e70942fa2a146b5bc70393522257bb85bd1ef7e019dcc3"}, ] [package.dependencies] @@ -8548,13 +8646,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvicorn" -version = "0.30.1" +version = "0.30.3" description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.8" files = [ - {file = "uvicorn-0.30.1-py3-none-any.whl", hash = "sha256:cd17daa7f3b9d7a24de3617820e634d0933b69eed8e33a516071174427238c81"}, - {file = "uvicorn-0.30.1.tar.gz", hash = "sha256:d46cd8e0fd80240baffbcd9ec1012a712938754afcf81bce56c024c1656aece8"}, + {file = "uvicorn-0.30.3-py3-none-any.whl", hash = "sha256:94a3608da0e530cea8f69683aa4126364ac18e3826b6630d1a65f4638aade503"}, + {file = "uvicorn-0.30.3.tar.gz", hash = "sha256:0d114d0831ff1adbf231d358cbf42f17333413042552a624ea6a9b4c33dcfd81"}, ] [package.dependencies] @@ -9201,13 +9299,13 @@ multidict = ">=4.0" [[package]] name = "yfinance" -version = "0.2.40" +version = "0.2.41" description = "Download market data from Yahoo! Finance API" optional = false python-versions = "*" files = [ - {file = "yfinance-0.2.40-py2.py3-none-any.whl", hash = "sha256:328176b5690de7aa192456a15b351c20ddde31b35d479f8179f5325bd340fc0b"}, - {file = "yfinance-0.2.40.tar.gz", hash = "sha256:b053ac31229b5dc7f49a17a057f66aa7f688de2f5ddeb95c2455ec13cd89511a"}, + {file = "yfinance-0.2.41-py2.py3-none-any.whl", hash = "sha256:2ed7b453cb8568773eb2dbb4d87cc37ff02e5d133f7723ec3e219ab0b86b56d8"}, + {file = "yfinance-0.2.41.tar.gz", hash = "sha256:f94409a1ed4d596b9da8d2dbb498faaabfcf593d5870e1412e17669a212bb345"}, ] [package.dependencies] @@ -9332,57 +9430,108 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [[package]] name = "zstandard" -version = "0.22.0" +version = "0.23.0" description = "Zstandard bindings for Python" optional = false python-versions = ">=3.8" files = [ - {file = "zstandard-0.22.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:275df437ab03f8c033b8a2c181e51716c32d831082d93ce48002a5227ec93019"}, - {file = "zstandard-0.22.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ac9957bc6d2403c4772c890916bf181b2653640da98f32e04b96e4d6fb3252a"}, - {file = "zstandard-0.22.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe3390c538f12437b859d815040763abc728955a52ca6ff9c5d4ac707c4ad98e"}, - {file = "zstandard-0.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1958100b8a1cc3f27fa21071a55cb2ed32e9e5df4c3c6e661c193437f171cba2"}, - {file = "zstandard-0.22.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93e1856c8313bc688d5df069e106a4bc962eef3d13372020cc6e3ebf5e045202"}, - {file = "zstandard-0.22.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1a90ba9a4c9c884bb876a14be2b1d216609385efb180393df40e5172e7ecf356"}, - {file = "zstandard-0.22.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3db41c5e49ef73641d5111554e1d1d3af106410a6c1fb52cf68912ba7a343a0d"}, - {file = "zstandard-0.22.0-cp310-cp310-win32.whl", hash = "sha256:d8593f8464fb64d58e8cb0b905b272d40184eac9a18d83cf8c10749c3eafcd7e"}, - {file = "zstandard-0.22.0-cp310-cp310-win_amd64.whl", hash = "sha256:f1a4b358947a65b94e2501ce3e078bbc929b039ede4679ddb0460829b12f7375"}, - {file = "zstandard-0.22.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:589402548251056878d2e7c8859286eb91bd841af117dbe4ab000e6450987e08"}, - {file = "zstandard-0.22.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a97079b955b00b732c6f280d5023e0eefe359045e8b83b08cf0333af9ec78f26"}, - {file = "zstandard-0.22.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:445b47bc32de69d990ad0f34da0e20f535914623d1e506e74d6bc5c9dc40bb09"}, - {file = "zstandard-0.22.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33591d59f4956c9812f8063eff2e2c0065bc02050837f152574069f5f9f17775"}, - {file = "zstandard-0.22.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:888196c9c8893a1e8ff5e89b8f894e7f4f0e64a5af4d8f3c410f0319128bb2f8"}, - {file = "zstandard-0.22.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:53866a9d8ab363271c9e80c7c2e9441814961d47f88c9bc3b248142c32141d94"}, - {file = "zstandard-0.22.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4ac59d5d6910b220141c1737b79d4a5aa9e57466e7469a012ed42ce2d3995e88"}, - {file = "zstandard-0.22.0-cp311-cp311-win32.whl", hash = "sha256:2b11ea433db22e720758cba584c9d661077121fcf60ab43351950ded20283440"}, - {file = "zstandard-0.22.0-cp311-cp311-win_amd64.whl", hash = "sha256:11f0d1aab9516a497137b41e3d3ed4bbf7b2ee2abc79e5c8b010ad286d7464bd"}, - {file = "zstandard-0.22.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6c25b8eb733d4e741246151d895dd0308137532737f337411160ff69ca24f93a"}, - {file = "zstandard-0.22.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f9b2cde1cd1b2a10246dbc143ba49d942d14fb3d2b4bccf4618d475c65464912"}, - {file = "zstandard-0.22.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a88b7df61a292603e7cd662d92565d915796b094ffb3d206579aaebac6b85d5f"}, - {file = "zstandard-0.22.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:466e6ad8caefb589ed281c076deb6f0cd330e8bc13c5035854ffb9c2014b118c"}, - {file = "zstandard-0.22.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1d67d0d53d2a138f9e29d8acdabe11310c185e36f0a848efa104d4e40b808e4"}, - {file = "zstandard-0.22.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:39b2853efc9403927f9065cc48c9980649462acbdf81cd4f0cb773af2fd734bc"}, - {file = "zstandard-0.22.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8a1b2effa96a5f019e72874969394edd393e2fbd6414a8208fea363a22803b45"}, - {file = "zstandard-0.22.0-cp312-cp312-win32.whl", hash = "sha256:88c5b4b47a8a138338a07fc94e2ba3b1535f69247670abfe422de4e0b344aae2"}, - {file = "zstandard-0.22.0-cp312-cp312-win_amd64.whl", hash = "sha256:de20a212ef3d00d609d0b22eb7cc798d5a69035e81839f549b538eff4105d01c"}, - {file = "zstandard-0.22.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d75f693bb4e92c335e0645e8845e553cd09dc91616412d1d4650da835b5449df"}, - {file = "zstandard-0.22.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:36a47636c3de227cd765e25a21dc5dace00539b82ddd99ee36abae38178eff9e"}, - {file = "zstandard-0.22.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68953dc84b244b053c0d5f137a21ae8287ecf51b20872eccf8eaac0302d3e3b0"}, - {file = "zstandard-0.22.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2612e9bb4977381184bb2463150336d0f7e014d6bb5d4a370f9a372d21916f69"}, - {file = "zstandard-0.22.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23d2b3c2b8e7e5a6cb7922f7c27d73a9a615f0a5ab5d0e03dd533c477de23004"}, - {file = "zstandard-0.22.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d43501f5f31e22baf822720d82b5547f8a08f5386a883b32584a185675c8fbf"}, - {file = "zstandard-0.22.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a493d470183ee620a3df1e6e55b3e4de8143c0ba1b16f3ded83208ea8ddfd91d"}, - {file = "zstandard-0.22.0-cp38-cp38-win32.whl", hash = "sha256:7034d381789f45576ec3f1fa0e15d741828146439228dc3f7c59856c5bcd3292"}, - {file = "zstandard-0.22.0-cp38-cp38-win_amd64.whl", hash = "sha256:d8fff0f0c1d8bc5d866762ae95bd99d53282337af1be9dc0d88506b340e74b73"}, - {file = "zstandard-0.22.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2fdd53b806786bd6112d97c1f1e7841e5e4daa06810ab4b284026a1a0e484c0b"}, - {file = "zstandard-0.22.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:73a1d6bd01961e9fd447162e137ed949c01bdb830dfca487c4a14e9742dccc93"}, - {file = "zstandard-0.22.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9501f36fac6b875c124243a379267d879262480bf85b1dbda61f5ad4d01b75a3"}, - {file = "zstandard-0.22.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48f260e4c7294ef275744210a4010f116048e0c95857befb7462e033f09442fe"}, - {file = "zstandard-0.22.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:959665072bd60f45c5b6b5d711f15bdefc9849dd5da9fb6c873e35f5d34d8cfb"}, - {file = "zstandard-0.22.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d22fdef58976457c65e2796e6730a3ea4a254f3ba83777ecfc8592ff8d77d303"}, - {file = "zstandard-0.22.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a7ccf5825fd71d4542c8ab28d4d482aace885f5ebe4b40faaa290eed8e095a4c"}, - {file = "zstandard-0.22.0-cp39-cp39-win32.whl", hash = "sha256:f058a77ef0ece4e210bb0450e68408d4223f728b109764676e1a13537d056bb0"}, - {file = "zstandard-0.22.0-cp39-cp39-win_amd64.whl", hash = "sha256:e9e9d4e2e336c529d4c435baad846a181e39a982f823f7e4495ec0b0ec8538d2"}, - {file = "zstandard-0.22.0.tar.gz", hash = "sha256:8226a33c542bcb54cd6bd0a366067b610b41713b64c9abec1bc4533d69f51e70"}, + {file = "zstandard-0.23.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bf0a05b6059c0528477fba9054d09179beb63744355cab9f38059548fedd46a9"}, + {file = "zstandard-0.23.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fc9ca1c9718cb3b06634c7c8dec57d24e9438b2aa9a0f02b8bb36bf478538880"}, + {file = "zstandard-0.23.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77da4c6bfa20dd5ea25cbf12c76f181a8e8cd7ea231c673828d0386b1740b8dc"}, + {file = "zstandard-0.23.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2170c7e0367dde86a2647ed5b6f57394ea7f53545746104c6b09fc1f4223573"}, + {file = "zstandard-0.23.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c16842b846a8d2a145223f520b7e18b57c8f476924bda92aeee3a88d11cfc391"}, + {file = "zstandard-0.23.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:157e89ceb4054029a289fb504c98c6a9fe8010f1680de0201b3eb5dc20aa6d9e"}, + {file = "zstandard-0.23.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:203d236f4c94cd8379d1ea61db2fce20730b4c38d7f1c34506a31b34edc87bdd"}, + {file = "zstandard-0.23.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:dc5d1a49d3f8262be192589a4b72f0d03b72dcf46c51ad5852a4fdc67be7b9e4"}, + {file = "zstandard-0.23.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:752bf8a74412b9892f4e5b58f2f890a039f57037f52c89a740757ebd807f33ea"}, + {file = "zstandard-0.23.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:80080816b4f52a9d886e67f1f96912891074903238fe54f2de8b786f86baded2"}, + {file = "zstandard-0.23.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:84433dddea68571a6d6bd4fbf8ff398236031149116a7fff6f777ff95cad3df9"}, + {file = "zstandard-0.23.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ab19a2d91963ed9e42b4e8d77cd847ae8381576585bad79dbd0a8837a9f6620a"}, + {file = "zstandard-0.23.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:59556bf80a7094d0cfb9f5e50bb2db27fefb75d5138bb16fb052b61b0e0eeeb0"}, + {file = "zstandard-0.23.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:27d3ef2252d2e62476389ca8f9b0cf2bbafb082a3b6bfe9d90cbcbb5529ecf7c"}, + {file = "zstandard-0.23.0-cp310-cp310-win32.whl", hash = "sha256:5d41d5e025f1e0bccae4928981e71b2334c60f580bdc8345f824e7c0a4c2a813"}, + {file = "zstandard-0.23.0-cp310-cp310-win_amd64.whl", hash = "sha256:519fbf169dfac1222a76ba8861ef4ac7f0530c35dd79ba5727014613f91613d4"}, + {file = "zstandard-0.23.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:34895a41273ad33347b2fc70e1bff4240556de3c46c6ea430a7ed91f9042aa4e"}, + {file = "zstandard-0.23.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:77ea385f7dd5b5676d7fd943292ffa18fbf5c72ba98f7d09fc1fb9e819b34c23"}, + {file = "zstandard-0.23.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:983b6efd649723474f29ed42e1467f90a35a74793437d0bc64a5bf482bedfa0a"}, + {file = "zstandard-0.23.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80a539906390591dd39ebb8d773771dc4db82ace6372c4d41e2d293f8e32b8db"}, + {file = "zstandard-0.23.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:445e4cb5048b04e90ce96a79b4b63140e3f4ab5f662321975679b5f6360b90e2"}, + {file = "zstandard-0.23.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd30d9c67d13d891f2360b2a120186729c111238ac63b43dbd37a5a40670b8ca"}, + {file = "zstandard-0.23.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d20fd853fbb5807c8e84c136c278827b6167ded66c72ec6f9a14b863d809211c"}, + {file = "zstandard-0.23.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ed1708dbf4d2e3a1c5c69110ba2b4eb6678262028afd6c6fbcc5a8dac9cda68e"}, + {file = "zstandard-0.23.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:be9b5b8659dff1f913039c2feee1aca499cfbc19e98fa12bc85e037c17ec6ca5"}, + {file = "zstandard-0.23.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:65308f4b4890aa12d9b6ad9f2844b7ee42c7f7a4fd3390425b242ffc57498f48"}, + {file = "zstandard-0.23.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:98da17ce9cbf3bfe4617e836d561e433f871129e3a7ac16d6ef4c680f13a839c"}, + {file = "zstandard-0.23.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:8ed7d27cb56b3e058d3cf684d7200703bcae623e1dcc06ed1e18ecda39fee003"}, + {file = "zstandard-0.23.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:b69bb4f51daf461b15e7b3db033160937d3ff88303a7bc808c67bbc1eaf98c78"}, + {file = "zstandard-0.23.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:034b88913ecc1b097f528e42b539453fa82c3557e414b3de9d5632c80439a473"}, + {file = "zstandard-0.23.0-cp311-cp311-win32.whl", hash = "sha256:f2d4380bf5f62daabd7b751ea2339c1a21d1c9463f1feb7fc2bdcea2c29c3160"}, + {file = "zstandard-0.23.0-cp311-cp311-win_amd64.whl", hash = "sha256:62136da96a973bd2557f06ddd4e8e807f9e13cbb0bfb9cc06cfe6d98ea90dfe0"}, + {file = "zstandard-0.23.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b4567955a6bc1b20e9c31612e615af6b53733491aeaa19a6b3b37f3b65477094"}, + {file = "zstandard-0.23.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e172f57cd78c20f13a3415cc8dfe24bf388614324d25539146594c16d78fcc8"}, + {file = "zstandard-0.23.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0e166f698c5a3e914947388c162be2583e0c638a4703fc6a543e23a88dea3c1"}, + {file = "zstandard-0.23.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12a289832e520c6bd4dcaad68e944b86da3bad0d339ef7989fb7e88f92e96072"}, + {file = "zstandard-0.23.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d50d31bfedd53a928fed6707b15a8dbeef011bb6366297cc435accc888b27c20"}, + {file = "zstandard-0.23.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72c68dda124a1a138340fb62fa21b9bf4848437d9ca60bd35db36f2d3345f373"}, + {file = "zstandard-0.23.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53dd9d5e3d29f95acd5de6802e909ada8d8d8cfa37a3ac64836f3bc4bc5512db"}, + {file = "zstandard-0.23.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:6a41c120c3dbc0d81a8e8adc73312d668cd34acd7725f036992b1b72d22c1772"}, + {file = "zstandard-0.23.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:40b33d93c6eddf02d2c19f5773196068d875c41ca25730e8288e9b672897c105"}, + {file = "zstandard-0.23.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9206649ec587e6b02bd124fb7799b86cddec350f6f6c14bc82a2b70183e708ba"}, + {file = "zstandard-0.23.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:76e79bc28a65f467e0409098fa2c4376931fd3207fbeb6b956c7c476d53746dd"}, + {file = "zstandard-0.23.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:66b689c107857eceabf2cf3d3fc699c3c0fe8ccd18df2219d978c0283e4c508a"}, + {file = "zstandard-0.23.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9c236e635582742fee16603042553d276cca506e824fa2e6489db04039521e90"}, + {file = "zstandard-0.23.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a8fffdbd9d1408006baaf02f1068d7dd1f016c6bcb7538682622c556e7b68e35"}, + {file = "zstandard-0.23.0-cp312-cp312-win32.whl", hash = "sha256:dc1d33abb8a0d754ea4763bad944fd965d3d95b5baef6b121c0c9013eaf1907d"}, + {file = "zstandard-0.23.0-cp312-cp312-win_amd64.whl", hash = "sha256:64585e1dba664dc67c7cdabd56c1e5685233fbb1fc1966cfba2a340ec0dfff7b"}, + {file = "zstandard-0.23.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:576856e8594e6649aee06ddbfc738fec6a834f7c85bf7cadd1c53d4a58186ef9"}, + {file = "zstandard-0.23.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:38302b78a850ff82656beaddeb0bb989a0322a8bbb1bf1ab10c17506681d772a"}, + {file = "zstandard-0.23.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2240ddc86b74966c34554c49d00eaafa8200a18d3a5b6ffbf7da63b11d74ee2"}, + {file = "zstandard-0.23.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ef230a8fd217a2015bc91b74f6b3b7d6522ba48be29ad4ea0ca3a3775bf7dd5"}, + {file = "zstandard-0.23.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:774d45b1fac1461f48698a9d4b5fa19a69d47ece02fa469825b442263f04021f"}, + {file = "zstandard-0.23.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f77fa49079891a4aab203d0b1744acc85577ed16d767b52fc089d83faf8d8ed"}, + {file = "zstandard-0.23.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac184f87ff521f4840e6ea0b10c0ec90c6b1dcd0bad2f1e4a9a1b4fa177982ea"}, + {file = "zstandard-0.23.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c363b53e257246a954ebc7c488304b5592b9c53fbe74d03bc1c64dda153fb847"}, + {file = "zstandard-0.23.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e7792606d606c8df5277c32ccb58f29b9b8603bf83b48639b7aedf6df4fe8171"}, + {file = "zstandard-0.23.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a0817825b900fcd43ac5d05b8b3079937073d2b1ff9cf89427590718b70dd840"}, + {file = "zstandard-0.23.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:9da6bc32faac9a293ddfdcb9108d4b20416219461e4ec64dfea8383cac186690"}, + {file = "zstandard-0.23.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fd7699e8fd9969f455ef2926221e0233f81a2542921471382e77a9e2f2b57f4b"}, + {file = "zstandard-0.23.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d477ed829077cd945b01fc3115edd132c47e6540ddcd96ca169facff28173057"}, + {file = "zstandard-0.23.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ce8b52c5987b3e34d5674b0ab529a4602b632ebab0a93b07bfb4dfc8f8a33"}, + {file = "zstandard-0.23.0-cp313-cp313-win32.whl", hash = "sha256:a9b07268d0c3ca5c170a385a0ab9fb7fdd9f5fd866be004c4ea39e44edce47dd"}, + {file = "zstandard-0.23.0-cp313-cp313-win_amd64.whl", hash = "sha256:f3513916e8c645d0610815c257cbfd3242adfd5c4cfa78be514e5a3ebb42a41b"}, + {file = "zstandard-0.23.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2ef3775758346d9ac6214123887d25c7061c92afe1f2b354f9388e9e4d48acfc"}, + {file = "zstandard-0.23.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4051e406288b8cdbb993798b9a45c59a4896b6ecee2f875424ec10276a895740"}, + {file = "zstandard-0.23.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2d1a054f8f0a191004675755448d12be47fa9bebbcffa3cdf01db19f2d30a54"}, + {file = "zstandard-0.23.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f83fa6cae3fff8e98691248c9320356971b59678a17f20656a9e59cd32cee6d8"}, + {file = "zstandard-0.23.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32ba3b5ccde2d581b1e6aa952c836a6291e8435d788f656fe5976445865ae045"}, + {file = "zstandard-0.23.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f146f50723defec2975fb7e388ae3a024eb7151542d1599527ec2aa9cacb152"}, + {file = "zstandard-0.23.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1bfe8de1da6d104f15a60d4a8a768288f66aa953bbe00d027398b93fb9680b26"}, + {file = "zstandard-0.23.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:29a2bc7c1b09b0af938b7a8343174b987ae021705acabcbae560166567f5a8db"}, + {file = "zstandard-0.23.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:61f89436cbfede4bc4e91b4397eaa3e2108ebe96d05e93d6ccc95ab5714be512"}, + {file = "zstandard-0.23.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:53ea7cdc96c6eb56e76bb06894bcfb5dfa93b7adcf59d61c6b92674e24e2dd5e"}, + {file = "zstandard-0.23.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:a4ae99c57668ca1e78597d8b06d5af837f377f340f4cce993b551b2d7731778d"}, + {file = "zstandard-0.23.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:379b378ae694ba78cef921581ebd420c938936a153ded602c4fea612b7eaa90d"}, + {file = "zstandard-0.23.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:50a80baba0285386f97ea36239855f6020ce452456605f262b2d33ac35c7770b"}, + {file = "zstandard-0.23.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:61062387ad820c654b6a6b5f0b94484fa19515e0c5116faf29f41a6bc91ded6e"}, + {file = "zstandard-0.23.0-cp38-cp38-win32.whl", hash = "sha256:b8c0bd73aeac689beacd4e7667d48c299f61b959475cdbb91e7d3d88d27c56b9"}, + {file = "zstandard-0.23.0-cp38-cp38-win_amd64.whl", hash = "sha256:a05e6d6218461eb1b4771d973728f0133b2a4613a6779995df557f70794fd60f"}, + {file = "zstandard-0.23.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3aa014d55c3af933c1315eb4bb06dd0459661cc0b15cd61077afa6489bec63bb"}, + {file = "zstandard-0.23.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7f0804bb3799414af278e9ad51be25edf67f78f916e08afdb983e74161b916"}, + {file = "zstandard-0.23.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb2b1ecfef1e67897d336de3a0e3f52478182d6a47eda86cbd42504c5cbd009a"}, + {file = "zstandard-0.23.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:837bb6764be6919963ef41235fd56a6486b132ea64afe5fafb4cb279ac44f259"}, + {file = "zstandard-0.23.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1516c8c37d3a053b01c1c15b182f3b5f5eef19ced9b930b684a73bad121addf4"}, + {file = "zstandard-0.23.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48ef6a43b1846f6025dde6ed9fee0c24e1149c1c25f7fb0a0585572b2f3adc58"}, + {file = "zstandard-0.23.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11e3bf3c924853a2d5835b24f03eeba7fc9b07d8ca499e247e06ff5676461a15"}, + {file = "zstandard-0.23.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2fb4535137de7e244c230e24f9d1ec194f61721c86ebea04e1581d9d06ea1269"}, + {file = "zstandard-0.23.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8c24f21fa2af4bb9f2c492a86fe0c34e6d2c63812a839590edaf177b7398f700"}, + {file = "zstandard-0.23.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a8c86881813a78a6f4508ef9daf9d4995b8ac2d147dcb1a450448941398091c9"}, + {file = "zstandard-0.23.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:fe3b385d996ee0822fd46528d9f0443b880d4d05528fd26a9119a54ec3f91c69"}, + {file = "zstandard-0.23.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:82d17e94d735c99621bf8ebf9995f870a6b3e6d14543b99e201ae046dfe7de70"}, + {file = "zstandard-0.23.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c7c517d74bea1a6afd39aa612fa025e6b8011982a0897768a2f7c8ab4ebb78a2"}, + {file = "zstandard-0.23.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fd7e0f1cfb70eb2f95a19b472ee7ad6d9a0a992ec0ae53286870c104ca939e5"}, + {file = "zstandard-0.23.0-cp39-cp39-win32.whl", hash = "sha256:43da0f0092281bf501f9c5f6f3b4c975a8a0ea82de49ba3f7100e64d422a1274"}, + {file = "zstandard-0.23.0-cp39-cp39-win_amd64.whl", hash = "sha256:f8346bfa098532bc1fb6c7ef06783e969d87a99dd1d2a5a18a892c1d7a643c58"}, + {file = "zstandard-0.23.0.tar.gz", hash = "sha256:b2d8c62d08e7255f68f7a740bae85b3c9b8e5466baa9cbf7f57f1cde0ac6bc09"}, ] [package.dependencies] @@ -9394,4 +9543,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "76374a3483905c3219821ec31cefd75c64e0ccb3f0c5424acf1b8a1322587411" +content-hash = "9619ddabdd67710981c13dcfa3ddae0a48497c9f694afc81b820e882440c1265" diff --git a/api/pyproject.toml b/api/pyproject.toml index 88721b5a35..430e3d79c4 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -121,7 +121,7 @@ flask-cors = "~4.0.0" flask-login = "~0.6.3" flask-migrate = "~4.0.5" flask-restful = "~0.3.10" -flask-sqlalchemy = "~3.0.5" +Flask-SQLAlchemy = "~3.1.1" gevent = "~23.9.1" gmpy2 = "~2.1.5" google-ai-generativelanguage = "0.6.1" @@ -148,9 +148,9 @@ oss2 = "2.18.5" pandas = { version = "~2.2.2", extras = ["performance", "excel"] } psycopg2-binary = "~2.9.6" pycryptodome = "3.19.1" -pydantic = "~2.7.4" -pydantic-settings = "~2.3.3" -pydantic_extra_types = "~2.8.1" +pydantic = "~2.8.2" +pydantic-settings = "~2.3.4" +pydantic_extra_types = "~2.9.0" pydub = "~0.25.1" pyjwt = "~2.8.0" pypdfium2 = "~4.17.0" @@ -163,7 +163,8 @@ redis = { version = "~5.0.3", extras = ["hiredis"] } replicate = "~0.22.0" resend = "~0.7.0" safetensors = "~0.4.3" -sentry-sdk = { version = "~1.39.2", extras = ["flask"] } +scikit-learn = "^1.5.1" +sentry-sdk = { version = "~1.44.1", extras = ["flask"] } sqlalchemy = "~2.0.29" tencentcloud-sdk-python-hunyuan = "~3.0.1158" tiktoken = "~0.7.0" @@ -175,7 +176,8 @@ werkzeug = "~3.0.1" xinference-client = "0.9.4" yarl = "~1.9.4" zhipuai = "1.0.7" - +rank-bm25 = "~0.2.2" +openpyxl = "^3.1.5" ############################################################ # Tool dependencies required by tool implementations ############################################################ @@ -193,13 +195,14 @@ twilio = "~9.0.4" vanna = { version = "0.5.5", extras = ["postgres", "mysql", "clickhouse", "duckdb"] } wikipedia = "1.4.0" yfinance = "~0.2.40" +cloudscraper = "1.2.71" ############################################################ # VDB dependencies required by vector store clients ############################################################ [tool.poetry.group.vdb.dependencies] -chromadb = "~0.5.1" +chromadb = "0.5.1" oracledb = "~2.2.1" pgvecto-rs = "0.1.4" pgvector = "0.2.5" diff --git a/api/schedule/clean_embedding_cache_task.py b/api/schedule/clean_embedding_cache_task.py index 3d49b487c6..ccc1062266 100644 --- a/api/schedule/clean_embedding_cache_task.py +++ b/api/schedule/clean_embedding_cache_task.py @@ -2,6 +2,7 @@ import datetime import time import click +from sqlalchemy import text from werkzeug.exceptions import NotFound import app @@ -16,16 +17,21 @@ def clean_embedding_cache_task(): clean_days = int(dify_config.CLEAN_DAY_SETTING) start_at = time.perf_counter() thirty_days_ago = datetime.datetime.now() - datetime.timedelta(days=clean_days) - page = 1 while True: try: - embeddings = db.session.query(Embedding).filter(Embedding.created_at < thirty_days_ago) \ - .order_by(Embedding.created_at.desc()).paginate(page=page, per_page=100) + embedding_ids = db.session.query(Embedding.id).filter(Embedding.created_at < thirty_days_ago) \ + .order_by(Embedding.created_at.desc()).limit(100).all() + embedding_ids = [embedding_id[0] for embedding_id in embedding_ids] except NotFound: break - for embedding in embeddings: - db.session.delete(embedding) - db.session.commit() - page += 1 + if embedding_ids: + for embedding_id in embedding_ids: + db.session.execute(text( + "DELETE FROM embeddings WHERE id = :embedding_id" + ), {'embedding_id': embedding_id}) + + db.session.commit() + else: + break end_at = time.perf_counter() click.echo(click.style('Cleaned embedding cache from db success latency: {}'.format(end_at - start_at), fg='green')) diff --git a/api/schedule/clean_unused_datasets_task.py b/api/schedule/clean_unused_datasets_task.py index 2033791ace..b2b2f82b78 100644 --- a/api/schedule/clean_unused_datasets_task.py +++ b/api/schedule/clean_unused_datasets_task.py @@ -2,6 +2,7 @@ import datetime import time import click +from sqlalchemy import func from werkzeug.exceptions import NotFound import app @@ -14,16 +15,52 @@ from models.dataset import Dataset, DatasetQuery, Document @app.celery.task(queue='dataset') def clean_unused_datasets_task(): click.echo(click.style('Start clean unused datasets indexes.', fg='green')) - clean_days = int(dify_config.CLEAN_DAY_SETTING) + clean_days = dify_config.CLEAN_DAY_SETTING start_at = time.perf_counter() thirty_days_ago = datetime.datetime.now() - datetime.timedelta(days=clean_days) page = 1 while True: try: - datasets = db.session.query(Dataset).filter(Dataset.created_at < thirty_days_ago) \ - .order_by(Dataset.created_at.desc()).paginate(page=page, per_page=50) + # Subquery for counting new documents + document_subquery_new = db.session.query( + Document.dataset_id, + func.count(Document.id).label('document_count') + ).filter( + Document.indexing_status == 'completed', + Document.enabled == True, + Document.archived == False, + Document.updated_at > thirty_days_ago + ).group_by(Document.dataset_id).subquery() + + # Subquery for counting old documents + document_subquery_old = db.session.query( + Document.dataset_id, + func.count(Document.id).label('document_count') + ).filter( + Document.indexing_status == 'completed', + Document.enabled == True, + Document.archived == False, + Document.updated_at < thirty_days_ago + ).group_by(Document.dataset_id).subquery() + + # Main query with join and filter + datasets = (db.session.query(Dataset) + .outerjoin( + document_subquery_new, Dataset.id == document_subquery_new.c.dataset_id + ).outerjoin( + document_subquery_old, Dataset.id == document_subquery_old.c.dataset_id + ).filter( + Dataset.created_at < thirty_days_ago, + func.coalesce(document_subquery_new.c.document_count, 0) == 0, + func.coalesce(document_subquery_old.c.document_count, 0) > 0 + ).order_by( + Dataset.created_at.desc() + ).paginate(page=page, per_page=50)) + except NotFound: break + if datasets.items is None or len(datasets.items) == 0: + break page += 1 for dataset in datasets: dataset_query = db.session.query(DatasetQuery).filter( @@ -31,31 +68,23 @@ def clean_unused_datasets_task(): DatasetQuery.dataset_id == dataset.id ).all() if not dataset_query or len(dataset_query) == 0: - documents = db.session.query(Document).filter( - Document.dataset_id == dataset.id, - Document.indexing_status == 'completed', - Document.enabled == True, - Document.archived == False, - Document.updated_at > thirty_days_ago - ).all() - if not documents or len(documents) == 0: - try: - # remove index - index_processor = IndexProcessorFactory(dataset.doc_form).init_index_processor() - index_processor.clean(dataset, None) + try: + # remove index + index_processor = IndexProcessorFactory(dataset.doc_form).init_index_processor() + index_processor.clean(dataset, None) - # update document - update_params = { - Document.enabled: False - } + # update document + update_params = { + Document.enabled: False + } - Document.query.filter_by(dataset_id=dataset.id).update(update_params) - db.session.commit() - click.echo(click.style('Cleaned unused dataset {} from db success!'.format(dataset.id), - fg='green')) - except Exception as e: - click.echo( - click.style('clean dataset index error: {} {}'.format(e.__class__.__name__, str(e)), - fg='red')) + Document.query.filter_by(dataset_id=dataset.id).update(update_params) + db.session.commit() + click.echo(click.style('Cleaned unused dataset {} from db success!'.format(dataset.id), + fg='green')) + except Exception as e: + click.echo( + click.style('clean dataset index error: {} {}'.format(e.__class__.__name__, str(e)), + fg='red')) end_at = time.perf_counter() click.echo(click.style('Cleaned unused dataset from db success latency: {}'.format(end_at - start_at), fg='green')) diff --git a/api/services/account_service.py b/api/services/account_service.py index 0bcbe8b2c0..d73cec2697 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -47,7 +47,7 @@ class AccountService: ) @staticmethod - def load_user(user_id: str) -> Account: + def load_user(user_id: str) -> None | Account: account = Account.query.filter_by(id=user_id).first() if not account: return None @@ -55,7 +55,7 @@ class AccountService: if account.status in [AccountStatus.BANNED.value, AccountStatus.CLOSED.value]: raise Unauthorized("Account is banned or closed.") - current_tenant = TenantAccountJoin.query.filter_by(account_id=account.id, current=True).first() + current_tenant: TenantAccountJoin = TenantAccountJoin.query.filter_by(account_id=account.id, current=True).first() if current_tenant: account.current_tenant_id = current_tenant.tenant_id else: diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py new file mode 100644 index 0000000000..3764166333 --- /dev/null +++ b/api/services/app_dsl_service.py @@ -0,0 +1,411 @@ +import logging + +import httpx +import yaml # type: ignore + +from core.app.segments import factory +from events.app_event import app_model_config_was_updated, app_was_created +from extensions.ext_database import db +from models.account import Account +from models.model import App, AppMode, AppModelConfig +from models.workflow import Workflow +from services.workflow_service import WorkflowService + +logger = logging.getLogger(__name__) + +current_dsl_version = "0.1.0" +dsl_to_dify_version_mapping: dict[str, str] = { + "0.1.0": "0.6.0", # dsl version -> from dify version +} + + +class AppDslService: + @classmethod + def import_and_create_new_app_from_url(cls, tenant_id: str, url: str, args: dict, account: Account) -> App: + """ + Import app dsl from url and create new app + :param tenant_id: tenant id + :param url: import url + :param args: request args + :param account: Account instance + """ + try: + max_size = 10 * 1024 * 1024 # 10MB + timeout = httpx.Timeout(10.0) + with httpx.stream("GET", url.strip(), follow_redirects=True, timeout=timeout) as response: + response.raise_for_status() + total_size = 0 + content = b"" + for chunk in response.iter_bytes(): + total_size += len(chunk) + if total_size > max_size: + raise ValueError("File size exceeds the limit of 10MB") + content += chunk + except httpx.HTTPStatusError as http_err: + raise ValueError(f"HTTP error occurred: {http_err}") + except httpx.RequestError as req_err: + raise ValueError(f"Request error occurred: {req_err}") + except Exception as e: + raise ValueError(f"Failed to fetch DSL from URL: {e}") + + if not content: + raise ValueError("Empty content from url") + + try: + data = content.decode("utf-8") + except UnicodeDecodeError as e: + raise ValueError(f"Error decoding content: {e}") + + return cls.import_and_create_new_app(tenant_id, data, args, account) + + @classmethod + def import_and_create_new_app(cls, tenant_id: str, data: str, args: dict, account: Account) -> App: + """ + Import app dsl and create new app + :param tenant_id: tenant id + :param data: import data + :param args: request args + :param account: Account instance + """ + try: + import_data = yaml.safe_load(data) + except yaml.YAMLError: + raise ValueError("Invalid YAML format in data argument.") + + # check or repair dsl version + import_data = cls._check_or_fix_dsl(import_data) + + app_data = import_data.get('app') + if not app_data: + raise ValueError("Missing app in data argument") + + # get app basic info + name = args.get("name") if args.get("name") else app_data.get('name') + description = args.get("description") if args.get("description") else app_data.get('description', '') + icon = args.get("icon") if args.get("icon") else app_data.get('icon') + icon_background = args.get("icon_background") if args.get("icon_background") \ + else app_data.get('icon_background') + + # import dsl and create app + app_mode = AppMode.value_of(app_data.get('mode')) + if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: + app = cls._import_and_create_new_workflow_based_app( + tenant_id=tenant_id, + app_mode=app_mode, + workflow_data=import_data.get('workflow'), + account=account, + name=name, + description=description, + icon=icon, + icon_background=icon_background + ) + elif app_mode in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION]: + app = cls._import_and_create_new_model_config_based_app( + tenant_id=tenant_id, + app_mode=app_mode, + model_config_data=import_data.get('model_config'), + account=account, + name=name, + description=description, + icon=icon, + icon_background=icon_background + ) + else: + raise ValueError("Invalid app mode") + + return app + + @classmethod + def import_and_overwrite_workflow(cls, app_model: App, data: str, account: Account) -> Workflow: + """ + Import app dsl and overwrite workflow + :param app_model: App instance + :param data: import data + :param account: Account instance + """ + try: + import_data = yaml.safe_load(data) + except yaml.YAMLError: + raise ValueError("Invalid YAML format in data argument.") + + # check or repair dsl version + import_data = cls._check_or_fix_dsl(import_data) + + app_data = import_data.get('app') + if not app_data: + raise ValueError("Missing app in data argument") + + # import dsl and overwrite app + app_mode = AppMode.value_of(app_data.get('mode')) + if app_mode not in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: + raise ValueError("Only support import workflow in advanced-chat or workflow app.") + + if app_data.get('mode') != app_model.mode: + raise ValueError( + f"App mode {app_data.get('mode')} is not matched with current app mode {app_mode.value}") + + return cls._import_and_overwrite_workflow_based_app( + app_model=app_model, + workflow_data=import_data.get('workflow'), + account=account, + ) + + @classmethod + def export_dsl(cls, app_model: App, include_secret:bool = False) -> str: + """ + Export app + :param app_model: App instance + :return: + """ + app_mode = AppMode.value_of(app_model.mode) + + export_data = { + "version": current_dsl_version, + "kind": "app", + "app": { + "name": app_model.name, + "mode": app_model.mode, + "icon": app_model.icon, + "icon_background": app_model.icon_background, + "description": app_model.description + } + } + + if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: + cls._append_workflow_export_data(export_data=export_data, app_model=app_model, include_secret=include_secret) + else: + cls._append_model_config_export_data(export_data, app_model) + + return yaml.dump(export_data) + + @classmethod + def _check_or_fix_dsl(cls, import_data: dict) -> dict: + """ + Check or fix dsl + + :param import_data: import data + """ + if not import_data.get('version'): + import_data['version'] = "0.1.0" + + if not import_data.get('kind') or import_data.get('kind') != "app": + import_data['kind'] = "app" + + if import_data.get('version') != current_dsl_version: + # Currently only one DSL version, so no difference checks or compatibility fixes will be performed. + logger.warning(f"DSL version {import_data.get('version')} is not compatible " + f"with current version {current_dsl_version}, related to " + f"Dify version {dsl_to_dify_version_mapping.get(current_dsl_version)}.") + + return import_data + + @classmethod + def _import_and_create_new_workflow_based_app(cls, + tenant_id: str, + app_mode: AppMode, + workflow_data: dict, + account: Account, + name: str, + description: str, + icon: str, + icon_background: str) -> App: + """ + Import app dsl and create new workflow based app + + :param tenant_id: tenant id + :param app_mode: app mode + :param workflow_data: workflow data + :param account: Account instance + :param name: app name + :param description: app description + :param icon: app icon + :param icon_background: app icon background + """ + if not workflow_data: + raise ValueError("Missing workflow in data argument " + "when app mode is advanced-chat or workflow") + + app = cls._create_app( + tenant_id=tenant_id, + app_mode=app_mode, + account=account, + name=name, + description=description, + icon=icon, + icon_background=icon_background + ) + + # init draft workflow + environment_variables_list = workflow_data.get('environment_variables') or [] + environment_variables = [factory.build_variable_from_mapping(obj) for obj in environment_variables_list] + workflow_service = WorkflowService() + draft_workflow = workflow_service.sync_draft_workflow( + app_model=app, + graph=workflow_data.get('graph', {}), + features=workflow_data.get('../core/app/features', {}), + unique_hash=None, + account=account, + environment_variables=environment_variables, + ) + workflow_service.publish_workflow( + app_model=app, + account=account, + draft_workflow=draft_workflow + ) + + return app + + @classmethod + def _import_and_overwrite_workflow_based_app(cls, + app_model: App, + workflow_data: dict, + account: Account) -> Workflow: + """ + Import app dsl and overwrite workflow based app + + :param app_model: App instance + :param workflow_data: workflow data + :param account: Account instance + """ + if not workflow_data: + raise ValueError("Missing workflow in data argument " + "when app mode is advanced-chat or workflow") + + # fetch draft workflow by app_model + workflow_service = WorkflowService() + current_draft_workflow = workflow_service.get_draft_workflow(app_model=app_model) + if current_draft_workflow: + unique_hash = current_draft_workflow.unique_hash + else: + unique_hash = None + + # sync draft workflow + environment_variables_list = workflow_data.get('environment_variables') or [] + environment_variables = [factory.build_variable_from_mapping(obj) for obj in environment_variables_list] + draft_workflow = workflow_service.sync_draft_workflow( + app_model=app_model, + graph=workflow_data.get('graph', {}), + features=workflow_data.get('features', {}), + unique_hash=unique_hash, + account=account, + environment_variables=environment_variables, + ) + + return draft_workflow + + @classmethod + def _import_and_create_new_model_config_based_app(cls, + tenant_id: str, + app_mode: AppMode, + model_config_data: dict, + account: Account, + name: str, + description: str, + icon: str, + icon_background: str) -> App: + """ + Import app dsl and create new model config based app + + :param tenant_id: tenant id + :param app_mode: app mode + :param model_config_data: model config data + :param account: Account instance + :param name: app name + :param description: app description + :param icon: app icon + :param icon_background: app icon background + """ + if not model_config_data: + raise ValueError("Missing model_config in data argument " + "when app mode is chat, agent-chat or completion") + + app = cls._create_app( + tenant_id=tenant_id, + app_mode=app_mode, + account=account, + name=name, + description=description, + icon=icon, + icon_background=icon_background + ) + + app_model_config = AppModelConfig() + app_model_config = app_model_config.from_model_config_dict(model_config_data) + app_model_config.app_id = app.id + + db.session.add(app_model_config) + db.session.commit() + + app.app_model_config_id = app_model_config.id + + app_model_config_was_updated.send( + app, + app_model_config=app_model_config + ) + + return app + + @classmethod + def _create_app(cls, + tenant_id: str, + app_mode: AppMode, + account: Account, + name: str, + description: str, + icon: str, + icon_background: str) -> App: + """ + Create new app + + :param tenant_id: tenant id + :param app_mode: app mode + :param account: Account instance + :param name: app name + :param description: app description + :param icon: app icon + :param icon_background: app icon background + """ + app = App( + tenant_id=tenant_id, + mode=app_mode.value, + name=name, + description=description, + icon=icon, + icon_background=icon_background, + enable_site=True, + enable_api=True + ) + + db.session.add(app) + db.session.commit() + + app_was_created.send(app, account=account) + + return app + + @classmethod + def _append_workflow_export_data(cls, *, export_data: dict, app_model: App, include_secret: bool) -> None: + """ + Append workflow export data + :param export_data: export data + :param app_model: App instance + """ + workflow_service = WorkflowService() + workflow = workflow_service.get_draft_workflow(app_model) + if not workflow: + raise ValueError("Missing draft workflow configuration, please check.") + + export_data['workflow'] = workflow.to_dict(include_secret=include_secret) + + @classmethod + def _append_model_config_export_data(cls, export_data: dict, app_model: App) -> None: + """ + Append model config export data + :param export_data: export data + :param app_model: App instance + """ + app_model_config = app_model.app_model_config + if not app_model_config: + raise ValueError("Missing app configuration, please check.") + + export_data['model_config'] = app_model_config.to_dict() diff --git a/api/services/app_generate_service.py b/api/services/app_generate_service.py index e894570b97..cff4ba8af9 100644 --- a/api/services/app_generate_service.py +++ b/api/services/app_generate_service.py @@ -21,7 +21,7 @@ class AppGenerateService: args: Any, invoke_from: InvokeFrom, streaming: bool = True, - ) -> Union[dict, Generator[dict, None, None]]: + ): """ App Content Generate :param app_model: app model diff --git a/api/services/app_service.py b/api/services/app_service.py index ca3c8d4fdc..e433bb59bb 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -3,7 +3,6 @@ import logging from datetime import datetime, timezone from typing import cast -import yaml from flask_login import current_user from flask_sqlalchemy.pagination import Pagination @@ -17,13 +16,12 @@ from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelTy from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.tools.tool_manager import ToolManager from core.tools.utils.configuration import ToolParameterConfigurationManager -from events.app_event import app_model_config_was_updated, app_was_created +from events.app_event import app_was_created from extensions.ext_database import db from models.account import Account from models.model import App, AppMode, AppModelConfig from models.tools import ApiToolProvider from services.tag_service import TagService -from services.workflow_service import WorkflowService from tasks.remove_app_and_related_data_task import remove_app_and_related_data_task @@ -100,7 +98,7 @@ class AppService: model_instance = None if model_instance: - if model_instance.model == default_model_config['model']['name']: + if model_instance.model == default_model_config['model']['name'] and model_instance.provider == default_model_config['model']['provider']: default_model_dict = default_model_config['model'] else: llm_model = cast(LargeLanguageModel, model_instance.model_type_instance) @@ -144,120 +142,6 @@ class AppService: return app - def import_app(self, tenant_id: str, data: str, args: dict, account: Account) -> App: - """ - Import app - :param tenant_id: tenant id - :param data: import data - :param args: request args - :param account: Account instance - """ - try: - import_data = yaml.safe_load(data) - except yaml.YAMLError as e: - raise ValueError("Invalid YAML format in data argument.") - - app_data = import_data.get('app') - model_config_data = import_data.get('model_config') - workflow = import_data.get('workflow') - - if not app_data: - raise ValueError("Missing app in data argument") - - app_mode = AppMode.value_of(app_data.get('mode')) - if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: - if not workflow: - raise ValueError("Missing workflow in data argument " - "when app mode is advanced-chat or workflow") - elif app_mode in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION]: - if not model_config_data: - raise ValueError("Missing model_config in data argument " - "when app mode is chat, agent-chat or completion") - else: - raise ValueError("Invalid app mode") - - app = App( - tenant_id=tenant_id, - mode=app_data.get('mode'), - name=args.get("name") if args.get("name") else app_data.get('name'), - description=args.get("description") if args.get("description") else app_data.get('description', ''), - icon=args.get("icon") if args.get("icon") else app_data.get('icon'), - icon_background=args.get("icon_background") if args.get("icon_background") \ - else app_data.get('icon_background'), - enable_site=True, - enable_api=True - ) - - db.session.add(app) - db.session.commit() - - app_was_created.send(app, account=account) - - if workflow: - # init draft workflow - workflow_service = WorkflowService() - draft_workflow = workflow_service.sync_draft_workflow( - app_model=app, - graph=workflow.get('graph'), - features=workflow.get('features'), - unique_hash=None, - account=account - ) - workflow_service.publish_workflow( - app_model=app, - account=account, - draft_workflow=draft_workflow - ) - - if model_config_data: - app_model_config = AppModelConfig() - app_model_config = app_model_config.from_model_config_dict(model_config_data) - app_model_config.app_id = app.id - - db.session.add(app_model_config) - db.session.commit() - - app.app_model_config_id = app_model_config.id - - app_model_config_was_updated.send( - app, - app_model_config=app_model_config - ) - - return app - - def export_app(self, app: App) -> str: - """ - Export app - :param app: App instance - :return: - """ - app_mode = AppMode.value_of(app.mode) - - export_data = { - "app": { - "name": app.name, - "mode": app.mode, - "icon": app.icon, - "icon_background": app.icon_background, - "description": app.description - } - } - - if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: - workflow_service = WorkflowService() - workflow = workflow_service.get_draft_workflow(app) - export_data['workflow'] = { - "graph": workflow.graph_dict, - "features": workflow.features_dict - } - else: - app_model_config = app.app_model_config - - export_data['model_config'] = app_model_config.to_dict() - - return yaml.dump(export_data) - def get_app(self, app: App) -> App: """ Get App @@ -403,8 +287,12 @@ class AppService: """ db.session.delete(app) db.session.commit() + # Trigger asynchronous deletion of app and related data - remove_app_and_related_data_task.delay(app.id) + remove_app_and_related_data_task.delay( + tenant_id=app.tenant_id, + app_id=app.id + ) def get_app_meta(self, app_model: App) -> dict: """ @@ -462,7 +350,7 @@ class AppService: try: provider: ApiToolProvider = db.session.query(ApiToolProvider).filter( ApiToolProvider.id == provider_id - ) + ).first() meta['tool_icons'][tool_name] = json.loads(provider.icon) except: meta['tool_icons'][tool_name] = { diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index fbaf44c9a4..d5a54ba731 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -524,7 +524,14 @@ class DocumentService: @staticmethod def delete_document(document): # trigger document_was_deleted signal - document_was_deleted.send(document.id, dataset_id=document.dataset_id, doc_form=document.doc_form) + file_id = None + if document.data_source_type == 'upload_file': + if document.data_source_info: + data_source_info = document.data_source_info_dict + if data_source_info and 'upload_file_id' in data_source_info: + file_id = data_source_info['upload_file_id'] + document_was_deleted.send(document.id, dataset_id=document.dataset_id, + doc_form=document.doc_form, file_id=file_id) db.session.delete(document) db.session.commit() @@ -681,7 +688,7 @@ class DocumentService: dataset.collection_binding_id = dataset_collection_binding.id if not dataset.retrieval_model: default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -838,13 +845,17 @@ class DocumentService: 'only_main_content': website_info.get('only_main_content', False), 'mode': 'crawl', } + if len(url) > 255: + document_name = url[:200] + '...' + else: + document_name = url document = DocumentService.build_document( dataset, dataset_process_rule.id, document_data["data_source"]["type"], document_data["doc_form"], document_data["doc_language"], data_source_info, created_from, position, - account, url, batch + account, document_name, batch ) db.session.add(document) db.session.flush() @@ -1052,7 +1063,7 @@ class DocumentService: retrieval_model = document_data['retrieval_model'] else: default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', diff --git a/api/services/hit_testing_service.py b/api/services/hit_testing_service.py index 9bcf828712..69274dff09 100644 --- a/api/services/hit_testing_service.py +++ b/api/services/hit_testing_service.py @@ -9,7 +9,7 @@ from models.account import Account from models.dataset import Dataset, DatasetQuery, DocumentSegment default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -38,14 +38,16 @@ class HitTestingService: if not retrieval_model: retrieval_model = dataset.retrieval_model if dataset.retrieval_model else default_retrieval_model - all_documents = RetrievalService.retrieve(retrival_method=retrieval_model['search_method'], + all_documents = RetrievalService.retrieve(retrival_method=retrieval_model.get('search_method', 'semantic_search'), dataset_id=dataset.id, - query=query, - top_k=retrieval_model['top_k'], + query=cls.escape_query_for_search(query), + top_k=retrieval_model.get('top_k', 2), score_threshold=retrieval_model['score_threshold'] if retrieval_model['score_threshold_enabled'] else None, reranking_model=retrieval_model['reranking_model'] - if retrieval_model['reranking_enable'] else None + if retrieval_model['reranking_enable'] else None, + reranking_mode=retrieval_model.get('reranking_mode', None), + weights=retrieval_model.get('weights', None), ) end = time.perf_counter() @@ -104,3 +106,7 @@ class HitTestingService: if not query or len(query) > 250: raise ValueError('Query is required and cannot exceed 250 characters') + + @staticmethod + def escape_query_for_search(query: str) -> str: + return query.replace('"', '\\"') diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index c684c2862b..0983839996 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -131,7 +131,7 @@ class ModelLoadBalancingService: load_balancing_configs.insert(0, inherit_config) else: # move the inherit configuration to the first - for i, load_balancing_config in enumerate(load_balancing_configs): + for i, load_balancing_config in enumerate(load_balancing_configs[:]): if load_balancing_config.name == '__inherit__': inherit_config = load_balancing_configs.pop(i) load_balancing_configs.insert(0, inherit_config) diff --git a/api/services/recommended_app_service.py b/api/services/recommended_app_service.py index c4733b6d3f..1c1c5be17c 100644 --- a/api/services/recommended_app_service.py +++ b/api/services/recommended_app_service.py @@ -4,12 +4,13 @@ from os import path from typing import Optional import requests +from flask import current_app from configs import dify_config from constants.languages import languages from extensions.ext_database import db from models.model import App, RecommendedApp -from services.app_service import AppService +from services.app_dsl_service import AppDslService logger = logging.getLogger(__name__) @@ -186,16 +187,13 @@ class RecommendedAppService: if not app_model or not app_model.is_public: return None - app_service = AppService() - export_str = app_service.export_app(app_model) - return { 'id': app_model.id, 'name': app_model.name, 'icon': app_model.icon, 'icon_background': app_model.icon_background, 'mode': app_model.mode, - 'export_data': export_str + 'export_data': AppDslService.export_dsl(app_model=app_model) } @classmethod diff --git a/api/services/workflow/workflow_converter.py b/api/services/workflow/workflow_converter.py index 010d53389a..06b129be69 100644 --- a/api/services/workflow/workflow_converter.py +++ b/api/services/workflow/workflow_converter.py @@ -199,7 +199,8 @@ class WorkflowConverter: version='draft', graph=json.dumps(graph), features=json.dumps(features), - created_by=account_id + created_by=account_id, + environment_variables=[], ) db.session.add(workflow) diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 025c1090b4..d868255f96 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -1,12 +1,12 @@ import json import time +from collections.abc import Sequence from datetime import datetime, timezone from typing import Optional -import yaml - from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager +from core.app.segments import Variable from core.model_runtime.utils.encoders import jsonable_encoder from core.workflow.entities.node_entities import NodeType from core.workflow.errors import WorkflowNodeRunFailedError @@ -63,11 +63,16 @@ class WorkflowService: return workflow - def sync_draft_workflow(self, app_model: App, - graph: dict, - features: dict, - unique_hash: Optional[str], - account: Account) -> Workflow: + def sync_draft_workflow( + self, + *, + app_model: App, + graph: dict, + features: dict, + unique_hash: Optional[str], + account: Account, + environment_variables: Sequence[Variable], + ) -> Workflow: """ Sync draft workflow :raises WorkflowHashNotEqualError @@ -75,10 +80,8 @@ class WorkflowService: # fetch draft workflow by app_model workflow = self.get_draft_workflow(app_model=app_model) - if workflow: - # validate unique hash - if workflow.unique_hash != unique_hash: - raise WorkflowHashNotEqualError() + if workflow and workflow.unique_hash != unique_hash: + raise WorkflowHashNotEqualError() # validate features structure self.validate_features_structure( @@ -95,7 +98,8 @@ class WorkflowService: version='draft', graph=json.dumps(graph), features=json.dumps(features), - created_by=account.id + created_by=account.id, + environment_variables=environment_variables ) db.session.add(workflow) # update draft workflow if found @@ -104,6 +108,7 @@ class WorkflowService: workflow.features = json.dumps(features) workflow.updated_by = account.id workflow.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) + workflow.environment_variables = environment_variables # commit db session changes db.session.commit() @@ -114,56 +119,6 @@ class WorkflowService: # return draft workflow return workflow - def import_draft_workflow(self, app_model: App, - data: str, - account: Account) -> Workflow: - """ - Import draft workflow - :param app_model: App instance - :param data: import data - :param account: Account instance - :return: - """ - try: - import_data = yaml.safe_load(data) - except yaml.YAMLError as e: - raise ValueError("Invalid YAML format in data argument.") - - app_data = import_data.get('app') - workflow = import_data.get('workflow') - - if not app_data: - raise ValueError("Missing app in data argument") - - app_mode = AppMode.value_of(app_data.get('mode')) - if app_mode not in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: - raise ValueError("Only support import workflow in advanced-chat or workflow app.") - - if app_data.get('mode') != app_model.mode: - raise ValueError(f"App mode {app_data.get('mode')} is not matched with current app mode {app_model.mode}") - - if not workflow: - raise ValueError("Missing workflow in data argument " - "when app mode is advanced-chat or workflow") - - # fetch draft workflow by app_model - current_draft_workflow = self.get_draft_workflow(app_model=app_model) - if current_draft_workflow: - unique_hash = current_draft_workflow.unique_hash - else: - unique_hash = None - - # sync draft workflow - draft_workflow = self.sync_draft_workflow( - app_model=app_model, - graph=workflow.get('graph'), - features=workflow.get('features'), - unique_hash=unique_hash, - account=account - ) - - return draft_workflow - def publish_workflow(self, app_model: App, account: Account, draft_workflow: Optional[Workflow] = None) -> Workflow: @@ -189,7 +144,8 @@ class WorkflowService: version=str(datetime.now(timezone.utc).replace(tzinfo=None)), graph=draft_workflow.graph, features=draft_workflow.features, - created_by=account.id + created_by=account.id, + environment_variables=draft_workflow.environment_variables ) # commit db session changes diff --git a/api/tasks/clean_dataset_task.py b/api/tasks/clean_dataset_task.py index 4de587d26a..1f26c966c4 100644 --- a/api/tasks/clean_dataset_task.py +++ b/api/tasks/clean_dataset_task.py @@ -6,6 +6,7 @@ from celery import shared_task from core.rag.index_processor.index_processor_factory import IndexProcessorFactory from extensions.ext_database import db +from extensions.ext_storage import storage from models.dataset import ( AppDatasetJoin, Dataset, @@ -14,6 +15,7 @@ from models.dataset import ( Document, DocumentSegment, ) +from models.model import UploadFile # Add import statement for ValueError @@ -65,8 +67,27 @@ def clean_dataset_task(dataset_id: str, tenant_id: str, indexing_technique: str, db.session.query(DatasetQuery).filter(DatasetQuery.dataset_id == dataset_id).delete() db.session.query(AppDatasetJoin).filter(AppDatasetJoin.dataset_id == dataset_id).delete() - db.session.commit() + # delete files + if documents: + for document in documents: + try: + if document.data_source_type == 'upload_file': + if document.data_source_info: + data_source_info = document.data_source_info_dict + if data_source_info and 'upload_file_id' in data_source_info: + file_id = data_source_info['upload_file_id'] + file = db.session.query(UploadFile).filter( + UploadFile.tenant_id == document.tenant_id, + UploadFile.id == file_id + ).first() + if not file: + continue + storage.delete(file.key) + db.session.delete(file) + except Exception: + continue + db.session.commit() end_at = time.perf_counter() logging.info( click.style('Cleaned dataset when dataset deleted: {} latency: {}'.format(dataset_id, end_at - start_at), fg='green')) diff --git a/api/tasks/clean_document_task.py b/api/tasks/clean_document_task.py index 71ebad1da4..0fd05615b6 100644 --- a/api/tasks/clean_document_task.py +++ b/api/tasks/clean_document_task.py @@ -1,21 +1,25 @@ import logging import time +from typing import Optional import click from celery import shared_task from core.rag.index_processor.index_processor_factory import IndexProcessorFactory from extensions.ext_database import db +from extensions.ext_storage import storage from models.dataset import Dataset, DocumentSegment +from models.model import UploadFile @shared_task(queue='dataset') -def clean_document_task(document_id: str, dataset_id: str, doc_form: str): +def clean_document_task(document_id: str, dataset_id: str, doc_form: str, file_id: Optional[str]): """ Clean document when document deleted. :param document_id: document id :param dataset_id: dataset id :param doc_form: doc_form + :param file_id: file id Usage: clean_document_task.delay(document_id, dataset_id) """ @@ -39,8 +43,20 @@ def clean_document_task(document_id: str, dataset_id: str, doc_form: str): db.session.delete(segment) db.session.commit() - end_at = time.perf_counter() - logging.info( - click.style('Cleaned document when document deleted: {} latency: {}'.format(document_id, end_at - start_at), fg='green')) + if file_id: + file = db.session.query(UploadFile).filter( + UploadFile.id == file_id + ).first() + if file: + try: + storage.delete(file.key) + except Exception: + logging.exception("Delete file failed when document deleted, file_id: {}".format(file_id)) + db.session.delete(file) + db.session.commit() + + end_at = time.perf_counter() + logging.info( + click.style('Cleaned document when document deleted: {} latency: {}'.format(document_id, end_at - start_at), fg='green')) except Exception: logging.exception("Cleaned document when document deleted failed") diff --git a/api/tasks/document_indexing_update_task.py b/api/tasks/document_indexing_update_task.py index b27274be37..f129d93de8 100644 --- a/api/tasks/document_indexing_update_task.py +++ b/api/tasks/document_indexing_update_task.py @@ -46,14 +46,15 @@ def document_indexing_update_task(dataset_id: str, document_id: str): index_processor = IndexProcessorFactory(index_type).init_index_processor() segments = db.session.query(DocumentSegment).filter(DocumentSegment.document_id == document_id).all() - index_node_ids = [segment.index_node_id for segment in segments] + if segments: + index_node_ids = [segment.index_node_id for segment in segments] - # delete from vector index - index_processor.clean(dataset, index_node_ids) + # delete from vector index + index_processor.clean(dataset, index_node_ids) - for segment in segments: - db.session.delete(segment) - db.session.commit() + for segment in segments: + db.session.delete(segment) + db.session.commit() end_at = time.perf_counter() logging.info( click.style('Cleaned document when document update data source or process rule: {} latency: {}'.format(document_id, end_at - start_at), fg='green')) diff --git a/api/tasks/remove_app_and_related_data_task.py b/api/tasks/remove_app_and_related_data_task.py index 117ce8d923..378756e68c 100644 --- a/api/tasks/remove_app_and_related_data_task.py +++ b/api/tasks/remove_app_and_related_data_task.py @@ -3,7 +3,6 @@ import time import click from celery import shared_task -from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError from extensions.ext_database import db @@ -25,6 +24,7 @@ from models.model import ( RecommendedApp, Site, TagBinding, + TraceAppConfig, ) from models.tools import WorkflowToolProvider from models.web import PinnedConversation, SavedMessage @@ -32,122 +32,287 @@ from models.workflow import Workflow, WorkflowAppLog, WorkflowNodeExecution, Wor @shared_task(queue='app_deletion', bind=True, max_retries=3) -def remove_app_and_related_data_task(self, app_id: str): - logging.info(click.style(f'Start deleting app and related data: {app_id}', fg='green')) +def remove_app_and_related_data_task(self, tenant_id: str, app_id: str): + logging.info(click.style(f'Start deleting app and related data: {tenant_id}:{app_id}', fg='green')) start_at = time.perf_counter() try: - # Use a transaction to ensure all deletions succeed or none do - with db.session.begin_nested(): - # Delete related data - _delete_app_model_configs(app_id) - _delete_app_site(app_id) - _delete_app_api_tokens(app_id) - _delete_installed_apps(app_id) - _delete_recommended_apps(app_id) - _delete_app_annotation_data(app_id) - _delete_app_dataset_joins(app_id) - _delete_app_workflows(app_id) - _delete_app_conversations(app_id) - _delete_app_messages(app_id) - _delete_workflow_tool_providers(app_id) - _delete_app_tag_bindings(app_id) - _delete_end_users(app_id) - - # If we reach here, the transaction was successful - db.session.commit() + # Delete related data + _delete_app_model_configs(tenant_id, app_id) + _delete_app_site(tenant_id, app_id) + _delete_app_api_tokens(tenant_id, app_id) + _delete_installed_apps(tenant_id, app_id) + _delete_recommended_apps(tenant_id, app_id) + _delete_app_annotation_data(tenant_id, app_id) + _delete_app_dataset_joins(tenant_id, app_id) + _delete_app_workflows(tenant_id, app_id) + _delete_app_workflow_runs(tenant_id, app_id) + _delete_app_workflow_node_executions(tenant_id, app_id) + _delete_app_workflow_app_logs(tenant_id, app_id) + _delete_app_conversations(tenant_id, app_id) + _delete_app_messages(tenant_id, app_id) + _delete_workflow_tool_providers(tenant_id, app_id) + _delete_app_tag_bindings(tenant_id, app_id) + _delete_end_users(tenant_id, app_id) + _delete_trace_app_configs(tenant_id, app_id) end_at = time.perf_counter() logging.info(click.style(f'App and related data deleted: {app_id} latency: {end_at - start_at}', fg='green')) - except SQLAlchemyError as e: - db.session.rollback() logging.exception( click.style(f"Database error occurred while deleting app {app_id} and related data", fg='red')) raise self.retry(exc=e, countdown=60) # Retry after 60 seconds - except Exception as e: logging.exception(click.style(f"Error occurred while deleting app {app_id} and related data", fg='red')) raise self.retry(exc=e, countdown=60) # Retry after 60 seconds -def _delete_app_model_configs(app_id: str): - db.session.query(AppModelConfig).filter(AppModelConfig.app_id == app_id).delete() +def _delete_app_model_configs(tenant_id: str, app_id: str): + def del_model_config(model_config_id: str): + db.session.query(AppModelConfig).filter(AppModelConfig.id == model_config_id).delete(synchronize_session=False) + + _delete_records( + """select id from app_model_configs where app_id=:app_id limit 1000""", + {"app_id": app_id}, + del_model_config, + "app model config" + ) -def _delete_app_site(app_id: str): - db.session.query(Site).filter(Site.app_id == app_id).delete() +def _delete_app_site(tenant_id: str, app_id: str): + def del_site(site_id: str): + db.session.query(Site).filter(Site.id == site_id).delete(synchronize_session=False) + + _delete_records( + """select id from sites where app_id=:app_id limit 1000""", + {"app_id": app_id}, + del_site, + "site" + ) -def _delete_app_api_tokens(app_id: str): - db.session.query(ApiToken).filter(ApiToken.app_id == app_id).delete() +def _delete_app_api_tokens(tenant_id: str, app_id: str): + def del_api_token(api_token_id: str): + db.session.query(ApiToken).filter(ApiToken.id == api_token_id).delete(synchronize_session=False) + + _delete_records( + """select id from api_tokens where app_id=:app_id limit 1000""", + {"app_id": app_id}, + del_api_token, + "api token" + ) -def _delete_installed_apps(app_id: str): - db.session.query(InstalledApp).filter(InstalledApp.app_id == app_id).delete() +def _delete_installed_apps(tenant_id: str, app_id: str): + def del_installed_app(installed_app_id: str): + db.session.query(InstalledApp).filter(InstalledApp.id == installed_app_id).delete(synchronize_session=False) + + _delete_records( + """select id from installed_apps where tenant_id=:tenant_id and app_id=:app_id limit 1000""", + {"tenant_id": tenant_id, "app_id": app_id}, + del_installed_app, + "installed app" + ) -def _delete_recommended_apps(app_id: str): - db.session.query(RecommendedApp).filter(RecommendedApp.app_id == app_id).delete() +def _delete_recommended_apps(tenant_id: str, app_id: str): + def del_recommended_app(recommended_app_id: str): + db.session.query(RecommendedApp).filter(RecommendedApp.id == recommended_app_id).delete( + synchronize_session=False) + + _delete_records( + """select id from recommended_apps where app_id=:app_id limit 1000""", + {"app_id": app_id}, + del_recommended_app, + "recommended app" + ) -def _delete_app_annotation_data(app_id: str): - db.session.query(AppAnnotationHitHistory).filter(AppAnnotationHitHistory.app_id == app_id).delete() - db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app_id).delete() +def _delete_app_annotation_data(tenant_id: str, app_id: str): + def del_annotation_hit_history(annotation_hit_history_id: str): + db.session.query(AppAnnotationHitHistory).filter( + AppAnnotationHitHistory.id == annotation_hit_history_id).delete(synchronize_session=False) + + _delete_records( + """select id from app_annotation_hit_histories where app_id=:app_id limit 1000""", + {"app_id": app_id}, + del_annotation_hit_history, + "annotation hit history" + ) + + def del_annotation_setting(annotation_setting_id: str): + db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.id == annotation_setting_id).delete( + synchronize_session=False) + + _delete_records( + """select id from app_annotation_settings where app_id=:app_id limit 1000""", + {"app_id": app_id}, + del_annotation_setting, + "annotation setting" + ) -def _delete_app_dataset_joins(app_id: str): - db.session.query(AppDatasetJoin).filter(AppDatasetJoin.app_id == app_id).delete() +def _delete_app_dataset_joins(tenant_id: str, app_id: str): + def del_dataset_join(dataset_join_id: str): + db.session.query(AppDatasetJoin).filter(AppDatasetJoin.id == dataset_join_id).delete(synchronize_session=False) + + _delete_records( + """select id from app_dataset_joins where app_id=:app_id limit 1000""", + {"app_id": app_id}, + del_dataset_join, + "dataset join" + ) -def _delete_app_workflows(app_id: str): - db.session.query(WorkflowRun).filter( - WorkflowRun.workflow_id.in_( - db.session.query(Workflow.id).filter(Workflow.app_id == app_id) - ) - ).delete(synchronize_session=False) - db.session.query(WorkflowNodeExecution).filter( - WorkflowNodeExecution.workflow_id.in_( - db.session.query(Workflow.id).filter(Workflow.app_id == app_id) - ) - ).delete(synchronize_session=False) - db.session.query(WorkflowAppLog).filter(WorkflowAppLog.app_id == app_id).delete(synchronize_session=False) - db.session.query(Workflow).filter(Workflow.app_id == app_id).delete(synchronize_session=False) +def _delete_app_workflows(tenant_id: str, app_id: str): + def del_workflow(workflow_id: str): + db.session.query(Workflow).filter(Workflow.id == workflow_id).delete(synchronize_session=False) + + _delete_records( + """select id from workflows where tenant_id=:tenant_id and app_id=:app_id limit 1000""", + {"tenant_id": tenant_id, "app_id": app_id}, + del_workflow, + "workflow" + ) -def _delete_app_conversations(app_id: str): - db.session.query(PinnedConversation).filter( - PinnedConversation.conversation_id.in_( - db.session.query(Conversation.id).filter(Conversation.app_id == app_id) - ) - ).delete(synchronize_session=False) - db.session.query(Conversation).filter(Conversation.app_id == app_id).delete() +def _delete_app_workflow_runs(tenant_id: str, app_id: str): + def del_workflow_run(workflow_run_id: str): + db.session.query(WorkflowRun).filter(WorkflowRun.id == workflow_run_id).delete(synchronize_session=False) + + _delete_records( + """select id from workflow_runs where tenant_id=:tenant_id and app_id=:app_id limit 1000""", + {"tenant_id": tenant_id, "app_id": app_id}, + del_workflow_run, + "workflow run" + ) -def _delete_app_messages(app_id: str): - message_ids = select(Message.id).filter(Message.app_id == app_id).scalar_subquery() - db.session.query(MessageFeedback).filter(MessageFeedback.message_id.in_(message_ids)).delete( - synchronize_session=False) - db.session.query(MessageAnnotation).filter(MessageAnnotation.message_id.in_(message_ids)).delete( - synchronize_session=False) - db.session.query(MessageChain).filter(MessageChain.message_id.in_(message_ids)).delete(synchronize_session=False) - db.session.query(MessageAgentThought).filter(MessageAgentThought.message_id.in_(message_ids)).delete( - synchronize_session=False) - db.session.query(MessageFile).filter(MessageFile.message_id.in_(message_ids)).delete(synchronize_session=False) - db.session.query(SavedMessage).filter(SavedMessage.message_id.in_(message_ids)).delete(synchronize_session=False) - db.session.query(Message).filter(Message.app_id == app_id).delete(synchronize_session=False) +def _delete_app_workflow_node_executions(tenant_id: str, app_id: str): + def del_workflow_node_execution(workflow_node_execution_id: str): + db.session.query(WorkflowNodeExecution).filter( + WorkflowNodeExecution.id == workflow_node_execution_id).delete(synchronize_session=False) + + _delete_records( + """select id from workflow_node_executions where tenant_id=:tenant_id and app_id=:app_id limit 1000""", + {"tenant_id": tenant_id, "app_id": app_id}, + del_workflow_node_execution, + "workflow node execution" + ) -def _delete_workflow_tool_providers(app_id: str): - db.session.query(WorkflowToolProvider).filter( - WorkflowToolProvider.app_id == app_id - ).delete(synchronize_session=False) +def _delete_app_workflow_app_logs(tenant_id: str, app_id: str): + def del_workflow_app_log(workflow_app_log_id: str): + db.session.query(WorkflowAppLog).filter(WorkflowAppLog.id == workflow_app_log_id).delete(synchronize_session=False) + + _delete_records( + """select id from workflow_app_logs where tenant_id=:tenant_id and app_id=:app_id limit 1000""", + {"tenant_id": tenant_id, "app_id": app_id}, + del_workflow_app_log, + "workflow app log" + ) -def _delete_app_tag_bindings(app_id: str): - db.session.query(TagBinding).filter( - TagBinding.target_id == app_id - ).delete(synchronize_session=False) +def _delete_app_conversations(tenant_id: str, app_id: str): + def del_conversation(conversation_id: str): + db.session.query(PinnedConversation).filter(PinnedConversation.conversation_id == conversation_id).delete( + synchronize_session=False) + db.session.query(Conversation).filter(Conversation.id == conversation_id).delete(synchronize_session=False) + + _delete_records( + """select id from conversations where app_id=:app_id limit 1000""", + {"app_id": app_id}, + del_conversation, + "conversation" + ) -def _delete_end_users(app_id: str): - db.session.query(EndUser).filter(EndUser.app_id == app_id).delete() +def _delete_app_messages(tenant_id: str, app_id: str): + def del_message(message_id: str): + db.session.query(MessageFeedback).filter(MessageFeedback.message_id == message_id).delete( + synchronize_session=False) + db.session.query(MessageAnnotation).filter(MessageAnnotation.message_id == message_id).delete( + synchronize_session=False) + db.session.query(MessageChain).filter(MessageChain.message_id == message_id).delete( + synchronize_session=False) + db.session.query(MessageAgentThought).filter(MessageAgentThought.message_id == message_id).delete( + synchronize_session=False) + db.session.query(MessageFile).filter(MessageFile.message_id == message_id).delete(synchronize_session=False) + db.session.query(SavedMessage).filter(SavedMessage.message_id == message_id).delete( + synchronize_session=False) + db.session.query(Message).filter(Message.id == message_id).delete() + + _delete_records( + """select id from messages where app_id=:app_id limit 1000""", + {"app_id": app_id}, + del_message, + "message" + ) + + +def _delete_workflow_tool_providers(tenant_id: str, app_id: str): + def del_tool_provider(tool_provider_id: str): + db.session.query(WorkflowToolProvider).filter(WorkflowToolProvider.id == tool_provider_id).delete( + synchronize_session=False) + + _delete_records( + """select id from tool_workflow_providers where tenant_id=:tenant_id and app_id=:app_id limit 1000""", + {"tenant_id": tenant_id, "app_id": app_id}, + del_tool_provider, + "tool workflow provider" + ) + + +def _delete_app_tag_bindings(tenant_id: str, app_id: str): + def del_tag_binding(tag_binding_id: str): + db.session.query(TagBinding).filter(TagBinding.id == tag_binding_id).delete(synchronize_session=False) + + _delete_records( + """select id from tag_bindings where tenant_id=:tenant_id and target_id=:app_id limit 1000""", + {"tenant_id": tenant_id, "app_id": app_id}, + del_tag_binding, + "tag binding" + ) + + +def _delete_end_users(tenant_id: str, app_id: str): + def del_end_user(end_user_id: str): + db.session.query(EndUser).filter(EndUser.id == end_user_id).delete(synchronize_session=False) + + _delete_records( + """select id from end_users where tenant_id=:tenant_id and app_id=:app_id limit 1000""", + {"tenant_id": tenant_id, "app_id": app_id}, + del_end_user, + "end user" + ) + + +def _delete_trace_app_configs(tenant_id: str, app_id: str): + def del_trace_app_config(trace_app_config_id: str): + db.session.query(TraceAppConfig).filter(TraceAppConfig.id == trace_app_config_id).delete( + synchronize_session=False) + + _delete_records( + """select id from trace_app_config where app_id=:app_id limit 1000""", + {"app_id": app_id}, + del_trace_app_config, + "trace app config" + ) + + +def _delete_records(query_sql: str, params: dict, delete_func: callable, name: str) -> None: + while True: + with db.engine.begin() as conn: + rs = conn.execute(db.text(query_sql), params) + if rs.rowcount == 0: + break + + for i in rs: + record_id = str(i.id) + try: + delete_func(record_id) + db.session.commit() + logging.info(click.style(f"Deleted {name} {record_id}", fg='green')) + except Exception: + logging.exception(f"Error occurred while deleting {name} {record_id}") + continue + rs.close() diff --git a/api/tests/integration_tests/model_runtime/sagemaker/__init__.py b/api/tests/integration_tests/model_runtime/sagemaker/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/integration_tests/model_runtime/sagemaker/test_provider.py b/api/tests/integration_tests/model_runtime/sagemaker/test_provider.py new file mode 100644 index 0000000000..639227e745 --- /dev/null +++ b/api/tests/integration_tests/model_runtime/sagemaker/test_provider.py @@ -0,0 +1,19 @@ +import os + +import pytest + +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.sagemaker.sagemaker import SageMakerProvider + + +def test_validate_provider_credentials(): + provider = SageMakerProvider() + + with pytest.raises(CredentialsValidateFailedError): + provider.validate_provider_credentials( + credentials={} + ) + + provider.validate_provider_credentials( + credentials={} + ) diff --git a/api/tests/integration_tests/model_runtime/sagemaker/test_rerank.py b/api/tests/integration_tests/model_runtime/sagemaker/test_rerank.py new file mode 100644 index 0000000000..c67849dd79 --- /dev/null +++ b/api/tests/integration_tests/model_runtime/sagemaker/test_rerank.py @@ -0,0 +1,55 @@ +import os + +import pytest + +from core.model_runtime.entities.rerank_entities import RerankResult +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.sagemaker.rerank.rerank import SageMakerRerankModel + + +def test_validate_credentials(): + model = SageMakerRerankModel() + + with pytest.raises(CredentialsValidateFailedError): + model.validate_credentials( + model='bge-m3-rerank-v2', + credentials={ + "aws_region": os.getenv("AWS_REGION"), + "aws_access_key": os.getenv("AWS_ACCESS_KEY"), + "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY") + }, + query="What is the capital of the United States?", + docs=[ + "Carson City is the capital city of the American state of Nevada. At the 2010 United States " + "Census, Carson City had a population of 55,274.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that " + "are a political division controlled by the United States. Its capital is Saipan.", + ], + score_threshold=0.8 + ) + + +def test_invoke_model(): + model = SageMakerRerankModel() + + result = model.invoke( + model='bge-m3-rerank-v2', + credentials={ + "aws_region": os.getenv("AWS_REGION"), + "aws_access_key": os.getenv("AWS_ACCESS_KEY"), + "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY") + }, + query="What is the capital of the United States?", + docs=[ + "Carson City is the capital city of the American state of Nevada. At the 2010 United States " + "Census, Carson City had a population of 55,274.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that " + "are a political division controlled by the United States. Its capital is Saipan.", + ], + score_threshold=0.8 + ) + + assert isinstance(result, RerankResult) + assert len(result.docs) == 1 + assert result.docs[0].index == 1 + assert result.docs[0].score >= 0.8 diff --git a/api/tests/integration_tests/model_runtime/sagemaker/test_text_embedding.py b/api/tests/integration_tests/model_runtime/sagemaker/test_text_embedding.py new file mode 100644 index 0000000000..e817e8f04a --- /dev/null +++ b/api/tests/integration_tests/model_runtime/sagemaker/test_text_embedding.py @@ -0,0 +1,55 @@ +import os + +import pytest + +from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.sagemaker.text_embedding.text_embedding import SageMakerEmbeddingModel + + +def test_validate_credentials(): + model = SageMakerEmbeddingModel() + + with pytest.raises(CredentialsValidateFailedError): + model.validate_credentials( + model='bge-m3', + credentials={ + } + ) + + model.validate_credentials( + model='bge-m3-embedding', + credentials={ + } + ) + + +def test_invoke_model(): + model = SageMakerEmbeddingModel() + + result = model.invoke( + model='bge-m3-embedding', + credentials={ + }, + texts=[ + "hello", + "world" + ], + user="abc-123" + ) + + assert isinstance(result, TextEmbeddingResult) + assert len(result.embeddings) == 2 + +def test_get_num_tokens(): + model = SageMakerEmbeddingModel() + + num_tokens = model.get_num_tokens( + model='bge-m3-embedding', + credentials={ + }, + texts=[ + ] + ) + + assert num_tokens == 0 diff --git a/api/tests/integration_tests/model_runtime/stepfun/__init__.py b/api/tests/integration_tests/model_runtime/stepfun/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/integration_tests/model_runtime/stepfun/test_llm.py b/api/tests/integration_tests/model_runtime/stepfun/test_llm.py new file mode 100644 index 0000000000..d703147d63 --- /dev/null +++ b/api/tests/integration_tests/model_runtime/stepfun/test_llm.py @@ -0,0 +1,176 @@ +import os +from collections.abc import Generator + +import pytest + +from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta +from core.model_runtime.entities.message_entities import ( + AssistantPromptMessage, + ImagePromptMessageContent, + PromptMessageTool, + SystemPromptMessage, + TextPromptMessageContent, + UserPromptMessage, +) +from core.model_runtime.entities.model_entities import AIModelEntity, ModelType +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.stepfun.llm.llm import StepfunLargeLanguageModel + + +def test_validate_credentials(): + model = StepfunLargeLanguageModel() + + with pytest.raises(CredentialsValidateFailedError): + model.validate_credentials( + model='step-1-8k', + credentials={ + 'api_key': 'invalid_key' + } + ) + + model.validate_credentials( + model='step-1-8k', + credentials={ + 'api_key': os.environ.get('STEPFUN_API_KEY') + } + ) + +def test_invoke_model(): + model = StepfunLargeLanguageModel() + + response = model.invoke( + model='step-1-8k', + credentials={ + 'api_key': os.environ.get('STEPFUN_API_KEY') + }, + prompt_messages=[ + UserPromptMessage( + content='Hello World!' + ) + ], + model_parameters={ + 'temperature': 0.9, + 'top_p': 0.7 + }, + stop=['Hi'], + stream=False, + user="abc-123" + ) + + assert isinstance(response, LLMResult) + assert len(response.message.content) > 0 + + +def test_invoke_stream_model(): + model = StepfunLargeLanguageModel() + + response = model.invoke( + model='step-1-8k', + credentials={ + 'api_key': os.environ.get('STEPFUN_API_KEY') + }, + prompt_messages=[ + SystemPromptMessage( + content='You are a helpful AI assistant.', + ), + UserPromptMessage( + content='Hello World!' + ) + ], + model_parameters={ + 'temperature': 0.9, + 'top_p': 0.7 + }, + stream=True, + user="abc-123" + ) + + assert isinstance(response, Generator) + + for chunk in response: + assert isinstance(chunk, LLMResultChunk) + assert isinstance(chunk.delta, LLMResultChunkDelta) + assert isinstance(chunk.delta.message, AssistantPromptMessage) + assert len(chunk.delta.message.content) > 0 if chunk.delta.finish_reason is None else True + + +def test_get_customizable_model_schema(): + model = StepfunLargeLanguageModel() + + schema = model.get_customizable_model_schema( + model='step-1-8k', + credentials={ + 'api_key': os.environ.get('STEPFUN_API_KEY') + } + ) + assert isinstance(schema, AIModelEntity) + + +def test_invoke_chat_model_with_tools(): + model = StepfunLargeLanguageModel() + + result = model.invoke( + model='step-1-8k', + credentials={ + 'api_key': os.environ.get('STEPFUN_API_KEY') + }, + prompt_messages=[ + SystemPromptMessage( + content='You are a helpful AI assistant.', + ), + UserPromptMessage( + content="what's the weather today in Shanghai?", + ) + ], + model_parameters={ + 'temperature': 0.9, + 'max_tokens': 100 + }, + tools=[ + PromptMessageTool( + name='get_weather', + description='Determine weather in my location', + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": [ + "c", + "f" + ] + } + }, + "required": [ + "location" + ] + } + ), + PromptMessageTool( + name='get_stock_price', + description='Get the current stock price', + parameters={ + "type": "object", + "properties": { + "symbol": { + "type": "string", + "description": "The stock symbol" + } + }, + "required": [ + "symbol" + ] + } + ) + ], + stream=False, + user="abc-123" + ) + + assert isinstance(result, LLMResult) + assert isinstance(result.message, AssistantPromptMessage) + assert len(result.message.tool_calls) > 0 \ No newline at end of file diff --git a/api/tests/integration_tests/workflow/nodes/test_code.py b/api/tests/integration_tests/workflow/nodes/test_code.py index 15cf5367d3..5c95258520 100644 --- a/api/tests/integration_tests/workflow/nodes/test_code.py +++ b/api/tests/integration_tests/workflow/nodes/test_code.py @@ -55,9 +55,9 @@ def test_execute_code(setup_code_executor_mock): ) # construct variable pool - pool = VariablePool(system_variables={}, user_inputs={}) - pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1) - pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=2) + pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) + pool.add(['1', '123', 'args1'], 1) + pool.add(['1', '123', 'args2'], 2) # execute node result = node.run(pool) @@ -109,9 +109,9 @@ def test_execute_code_output_validator(setup_code_executor_mock): ) # construct variable pool - pool = VariablePool(system_variables={}, user_inputs={}) - pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1) - pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=2) + pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) + pool.add(['1', '123', 'args1'], 1) + pool.add(['1', '123', 'args2'], 2) # execute node result = node.run(pool) diff --git a/api/tests/integration_tests/workflow/nodes/test_http.py b/api/tests/integration_tests/workflow/nodes/test_http.py index eaed24e56c..a1354bd6a5 100644 --- a/api/tests/integration_tests/workflow/nodes/test_http.py +++ b/api/tests/integration_tests/workflow/nodes/test_http.py @@ -18,9 +18,9 @@ BASIC_NODE_DATA = { } # construct variable pool -pool = VariablePool(system_variables={}, user_inputs={}) -pool.append_variable(node_id='a', variable_key_list=['b123', 'args1'], value=1) -pool.append_variable(node_id='a', variable_key_list=['b123', 'args2'], value=2) +pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) +pool.add(['a', 'b123', 'args1'], 1) +pool.add(['a', 'b123', 'args2'], 2) @pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True) @@ -43,7 +43,6 @@ def test_get(setup_http_mock): 'headers': 'X-Header:123', 'params': 'A:b', 'body': None, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -52,7 +51,6 @@ def test_get(setup_http_mock): data = result.process_data.get('request', '') assert '?A=b' in data - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data @@ -103,7 +101,6 @@ def test_custom_authorization_header(setup_http_mock): 'headers': 'X-Header:123', 'params': 'A:b', 'body': None, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -113,7 +110,6 @@ def test_custom_authorization_header(setup_http_mock): assert '?A=b' in data assert 'X-Header: 123' in data - assert 'X-Auth: Auth' in data @pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True) @@ -136,7 +132,6 @@ def test_template(setup_http_mock): 'headers': 'X-Header:123\nX-Header2:{{#a.b123.args2#}}', 'params': 'A:b\nTemplate:{{#a.b123.args2#}}', 'body': None, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -145,7 +140,6 @@ def test_template(setup_http_mock): assert '?A=b' in data assert 'Template=2' in data - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data assert 'X-Header2: 2' in data @@ -173,7 +167,6 @@ def test_json(setup_http_mock): 'type': 'json', 'data': '{"a": "{{#a.b123.args1#}}"}' }, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -181,7 +174,6 @@ def test_json(setup_http_mock): data = result.process_data.get('request', '') assert '{"a": "1"}' in data - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data @@ -207,7 +199,6 @@ def test_x_www_form_urlencoded(setup_http_mock): 'type': 'x-www-form-urlencoded', 'data': 'a:{{#a.b123.args1#}}\nb:{{#a.b123.args2#}}' }, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -215,7 +206,6 @@ def test_x_www_form_urlencoded(setup_http_mock): data = result.process_data.get('request', '') assert 'a=1&b=2' in data - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data @@ -241,7 +231,6 @@ def test_form_data(setup_http_mock): 'type': 'form-data', 'data': 'a:{{#a.b123.args1#}}\nb:{{#a.b123.args2#}}' }, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -252,7 +241,6 @@ def test_form_data(setup_http_mock): assert '1' in data assert 'form-data; name="b"' in data assert '2' in data - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data @@ -278,14 +266,12 @@ def test_none_data(setup_http_mock): 'type': 'none', 'data': '123123123' }, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) result = node.run(pool) data = result.process_data.get('request', '') - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data assert '123123123' not in data @@ -305,7 +291,6 @@ def test_mock_404(setup_http_mock): 'body': None, 'params': '', 'headers': 'X-Header:123', - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -334,7 +319,6 @@ def test_multi_colons_parse(setup_http_mock): 'type': 'form-data', 'data': 'Referer:http://example5.com\nRedirect:http://example6.com' }, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) diff --git a/api/tests/integration_tests/workflow/nodes/test_llm.py b/api/tests/integration_tests/workflow/nodes/test_llm.py index d7a6c1224f..ac704e4eaf 100644 --- a/api/tests/integration_tests/workflow/nodes/test_llm.py +++ b/api/tests/integration_tests/workflow/nodes/test_llm.py @@ -70,8 +70,8 @@ def test_execute_llm(setup_openai_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) - pool.append_variable(node_id='abc', variable_key_list=['output'], value='sunny') + }, user_inputs={}, environment_variables=[]) + pool.add(['abc', 'output'], 'sunny') credentials = { 'openai_api_key': os.environ.get('OPENAI_API_KEY') @@ -185,8 +185,8 @@ def test_execute_llm_with_jinja2(setup_code_executor_mock, setup_openai_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) - pool.append_variable(node_id='abc', variable_key_list=['output'], value='sunny') + }, user_inputs={}, environment_variables=[]) + pool.add(['abc', 'output'], 'sunny') credentials = { 'openai_api_key': os.environ.get('OPENAI_API_KEY') diff --git a/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py b/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py index e5fd2bc1fd..312ad47026 100644 --- a/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py +++ b/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py @@ -123,7 +123,7 @@ def test_function_calling_parameter_extractor(setup_openai_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) + }, user_inputs={}, environment_variables=[]) result = node.run(pool) @@ -181,7 +181,7 @@ def test_instructions(setup_openai_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) + }, user_inputs={}, environment_variables=[]) result = node.run(pool) @@ -247,7 +247,7 @@ def test_chat_parameter_extractor(setup_anthropic_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) + }, user_inputs={}, environment_variables=[]) result = node.run(pool) @@ -311,7 +311,7 @@ def test_completion_parameter_extractor(setup_openai_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) + }, user_inputs={}, environment_variables=[]) result = node.run(pool) @@ -424,7 +424,7 @@ def test_chat_parameter_extractor_with_memory(setup_anthropic_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) + }, user_inputs={}, environment_variables=[]) result = node.run(pool) diff --git a/api/tests/integration_tests/workflow/nodes/test_template_transform.py b/api/tests/integration_tests/workflow/nodes/test_template_transform.py index 02999bf0a2..781dfbc50f 100644 --- a/api/tests/integration_tests/workflow/nodes/test_template_transform.py +++ b/api/tests/integration_tests/workflow/nodes/test_template_transform.py @@ -38,9 +38,9 @@ def test_execute_code(setup_code_executor_mock): ) # construct variable pool - pool = VariablePool(system_variables={}, user_inputs={}) - pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1) - pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=3) + pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) + pool.add(['1', '123', 'args1'], 1) + pool.add(['1', '123', 'args2'], 3) # execute node result = node.run(pool) diff --git a/api/tests/integration_tests/workflow/nodes/test_tool.py b/api/tests/integration_tests/workflow/nodes/test_tool.py index fffd074457..01d62280e8 100644 --- a/api/tests/integration_tests/workflow/nodes/test_tool.py +++ b/api/tests/integration_tests/workflow/nodes/test_tool.py @@ -6,8 +6,8 @@ from models.workflow import WorkflowNodeExecutionStatus def test_tool_variable_invoke(): - pool = VariablePool(system_variables={}, user_inputs={}) - pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value='1+1') + pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) + pool.add(['1', '123', 'args1'], '1+1') node = ToolNode( tenant_id='1', @@ -45,8 +45,8 @@ def test_tool_variable_invoke(): assert result.outputs['files'] == [] def test_tool_mixed_invoke(): - pool = VariablePool(system_variables={}, user_inputs={}) - pool.append_variable(node_id='1', variable_key_list=['args1'], value='1+1') + pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) + pool.add(['1', 'args1'], '1+1') node = ToolNode( tenant_id='1', diff --git a/api/tests/unit_tests/configs/test_dify_config.py b/api/tests/unit_tests/configs/test_dify_config.py index 50bb2b75ac..949a5a1769 100644 --- a/api/tests/unit_tests/configs/test_dify_config.py +++ b/api/tests/unit_tests/configs/test_dify_config.py @@ -1,3 +1,4 @@ +import os from textwrap import dedent import pytest @@ -48,7 +49,9 @@ def test_dify_config(example_env_file): # This is due to `pymilvus` loading all the variables from the `.env` file into `os.environ`. def test_flask_configs(example_env_file): flask_app = Flask('app') - flask_app.config.from_mapping(DifyConfig(_env_file=example_env_file).model_dump()) + # clear system environment variables + os.environ.clear() + flask_app.config.from_mapping(DifyConfig(_env_file=example_env_file).model_dump()) # pyright: ignore config = flask_app.config # configs read from pydantic-settings diff --git a/api/tests/unit_tests/core/app/segments/test_factory.py b/api/tests/unit_tests/core/app/segments/test_factory.py new file mode 100644 index 0000000000..85321ee374 --- /dev/null +++ b/api/tests/unit_tests/core/app/segments/test_factory.py @@ -0,0 +1,307 @@ +from uuid import uuid4 + +import pytest + +from core.app.segments import ( + ArrayFileVariable, + ArrayNumberVariable, + ArrayObjectVariable, + ArrayStringVariable, + FileVariable, + FloatVariable, + IntegerVariable, + NoneSegment, + ObjectSegment, + SecretVariable, + StringVariable, + factory, +) + + +def test_string_variable(): + test_data = {'value_type': 'string', 'name': 'test_text', 'value': 'Hello, World!'} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, StringVariable) + + +def test_integer_variable(): + test_data = {'value_type': 'number', 'name': 'test_int', 'value': 42} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, IntegerVariable) + + +def test_float_variable(): + test_data = {'value_type': 'number', 'name': 'test_float', 'value': 3.14} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, FloatVariable) + + +def test_secret_variable(): + test_data = {'value_type': 'secret', 'name': 'test_secret', 'value': 'secret_value'} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, SecretVariable) + + +def test_invalid_value_type(): + test_data = {'value_type': 'unknown', 'name': 'test_invalid', 'value': 'value'} + with pytest.raises(ValueError): + factory.build_variable_from_mapping(test_data) + + +def test_build_a_blank_string(): + result = factory.build_variable_from_mapping( + { + 'value_type': 'string', + 'name': 'blank', + 'value': '', + } + ) + assert isinstance(result, StringVariable) + assert result.value == '' + + +def test_build_a_object_variable_with_none_value(): + var = factory.build_segment( + { + 'key1': None, + } + ) + assert isinstance(var, ObjectSegment) + assert isinstance(var.value['key1'], NoneSegment) + + +def test_object_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'object', + 'name': 'test_object', + 'description': 'Description of the variable.', + 'value': { + 'key1': { + 'id': str(uuid4()), + 'value_type': 'string', + 'name': 'text', + 'value': 'text', + 'description': 'Description of the variable.', + }, + 'key2': { + 'id': str(uuid4()), + 'value_type': 'number', + 'name': 'number', + 'value': 1, + 'description': 'Description of the variable.', + }, + }, + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, ObjectSegment) + assert isinstance(variable.value['key1'], StringVariable) + assert isinstance(variable.value['key2'], IntegerVariable) + + +def test_array_string_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'array[string]', + 'name': 'test_array', + 'description': 'Description of the variable.', + 'value': [ + { + 'id': str(uuid4()), + 'value_type': 'string', + 'name': 'text', + 'value': 'text', + 'description': 'Description of the variable.', + }, + { + 'id': str(uuid4()), + 'value_type': 'string', + 'name': 'text', + 'value': 'text', + 'description': 'Description of the variable.', + }, + ], + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, ArrayStringVariable) + assert isinstance(variable.value[0], StringVariable) + assert isinstance(variable.value[1], StringVariable) + + +def test_array_number_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'array[number]', + 'name': 'test_array', + 'description': 'Description of the variable.', + 'value': [ + { + 'id': str(uuid4()), + 'value_type': 'number', + 'name': 'number', + 'value': 1, + 'description': 'Description of the variable.', + }, + { + 'id': str(uuid4()), + 'value_type': 'number', + 'name': 'number', + 'value': 2.0, + 'description': 'Description of the variable.', + }, + ], + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, ArrayNumberVariable) + assert isinstance(variable.value[0], IntegerVariable) + assert isinstance(variable.value[1], FloatVariable) + + +def test_array_object_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'array[object]', + 'name': 'test_array', + 'description': 'Description of the variable.', + 'value': [ + { + 'id': str(uuid4()), + 'value_type': 'object', + 'name': 'object', + 'description': 'Description of the variable.', + 'value': { + 'key1': { + 'id': str(uuid4()), + 'value_type': 'string', + 'name': 'text', + 'value': 'text', + 'description': 'Description of the variable.', + }, + 'key2': { + 'id': str(uuid4()), + 'value_type': 'number', + 'name': 'number', + 'value': 1, + 'description': 'Description of the variable.', + }, + }, + }, + { + 'id': str(uuid4()), + 'value_type': 'object', + 'name': 'object', + 'description': 'Description of the variable.', + 'value': { + 'key1': { + 'id': str(uuid4()), + 'value_type': 'string', + 'name': 'text', + 'value': 'text', + 'description': 'Description of the variable.', + }, + 'key2': { + 'id': str(uuid4()), + 'value_type': 'number', + 'name': 'number', + 'value': 1, + 'description': 'Description of the variable.', + }, + }, + }, + ], + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, ArrayObjectVariable) + assert isinstance(variable.value[0], ObjectSegment) + assert isinstance(variable.value[1], ObjectSegment) + assert isinstance(variable.value[0].value['key1'], StringVariable) + assert isinstance(variable.value[0].value['key2'], IntegerVariable) + assert isinstance(variable.value[1].value['key1'], StringVariable) + assert isinstance(variable.value[1].value['key2'], IntegerVariable) + + +def test_file_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'file', + 'name': 'test_file', + 'description': 'Description of the variable.', + 'value': { + 'id': str(uuid4()), + 'tenant_id': 'tenant_id', + 'type': 'image', + 'transfer_method': 'local_file', + 'url': 'url', + 'related_id': 'related_id', + 'extra_config': { + 'image_config': { + 'width': 100, + 'height': 100, + }, + }, + 'filename': 'filename', + 'extension': 'extension', + 'mime_type': 'mime_type', + }, + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, FileVariable) + + +def test_array_file_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'array[file]', + 'name': 'test_array_file', + 'description': 'Description of the variable.', + 'value': [ + { + 'id': str(uuid4()), + 'name': 'file', + 'value_type': 'file', + 'value': { + 'id': str(uuid4()), + 'tenant_id': 'tenant_id', + 'type': 'image', + 'transfer_method': 'local_file', + 'url': 'url', + 'related_id': 'related_id', + 'extra_config': { + 'image_config': { + 'width': 100, + 'height': 100, + }, + }, + 'filename': 'filename', + 'extension': 'extension', + 'mime_type': 'mime_type', + }, + }, + { + 'id': str(uuid4()), + 'name': 'file', + 'value_type': 'file', + 'value': { + 'id': str(uuid4()), + 'tenant_id': 'tenant_id', + 'type': 'image', + 'transfer_method': 'local_file', + 'url': 'url', + 'related_id': 'related_id', + 'extra_config': { + 'image_config': { + 'width': 100, + 'height': 100, + }, + }, + 'filename': 'filename', + 'extension': 'extension', + 'mime_type': 'mime_type', + }, + }, + ], + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, ArrayFileVariable) + assert isinstance(variable.value[0], FileVariable) + assert isinstance(variable.value[1], FileVariable) diff --git a/api/tests/unit_tests/core/app/segments/test_segment.py b/api/tests/unit_tests/core/app/segments/test_segment.py new file mode 100644 index 0000000000..414404b7d0 --- /dev/null +++ b/api/tests/unit_tests/core/app/segments/test_segment.py @@ -0,0 +1,54 @@ +from core.app.segments import SecretVariable, StringSegment, parser +from core.helper import encrypter +from core.workflow.entities.node_entities import SystemVariable +from core.workflow.entities.variable_pool import VariablePool + + +def test_segment_group_to_text(): + variable_pool = VariablePool( + system_variables={ + SystemVariable('user_id'): 'fake-user-id', + }, + user_inputs={}, + environment_variables=[ + SecretVariable(name='secret_key', value='fake-secret-key'), + ], + ) + variable_pool.add(('node_id', 'custom_query'), 'fake-user-query') + template = ( + 'Hello, {{#sys.user_id#}}! Your query is {{#node_id.custom_query#}}. And your key is {{#env.secret_key#}}.' + ) + segments_group = parser.convert_template(template=template, variable_pool=variable_pool) + + assert segments_group.text == 'Hello, fake-user-id! Your query is fake-user-query. And your key is fake-secret-key.' + assert ( + segments_group.log + == f"Hello, fake-user-id! Your query is fake-user-query. And your key is {encrypter.obfuscated_token('fake-secret-key')}." + ) + + +def test_convert_constant_to_segment_group(): + variable_pool = VariablePool( + system_variables={}, + user_inputs={}, + environment_variables=[], + ) + template = 'Hello, world!' + segments_group = parser.convert_template(template=template, variable_pool=variable_pool) + assert segments_group.text == 'Hello, world!' + assert segments_group.log == 'Hello, world!' + + +def test_convert_variable_to_segment_group(): + variable_pool = VariablePool( + system_variables={ + SystemVariable('user_id'): 'fake-user-id', + }, + user_inputs={}, + environment_variables=[], + ) + template = '{{#sys.user_id#}}' + segments_group = parser.convert_template(template=template, variable_pool=variable_pool) + assert segments_group.text == 'fake-user-id' + assert segments_group.log == 'fake-user-id' + assert segments_group.value == [StringSegment(value='fake-user-id')] diff --git a/api/tests/unit_tests/core/app/segments/test_variables.py b/api/tests/unit_tests/core/app/segments/test_variables.py new file mode 100644 index 0000000000..e3f513971a --- /dev/null +++ b/api/tests/unit_tests/core/app/segments/test_variables.py @@ -0,0 +1,94 @@ +import pytest +from pydantic import ValidationError + +from core.app.segments import ( + ArrayAnyVariable, + FloatVariable, + IntegerVariable, + ObjectVariable, + SecretVariable, + SegmentType, + StringVariable, +) + + +def test_frozen_variables(): + var = StringVariable(name='text', value='text') + with pytest.raises(ValidationError): + var.value = 'new value' + + int_var = IntegerVariable(name='integer', value=42) + with pytest.raises(ValidationError): + int_var.value = 100 + + float_var = FloatVariable(name='float', value=3.14) + with pytest.raises(ValidationError): + float_var.value = 2.718 + + secret_var = SecretVariable(name='secret', value='secret_value') + with pytest.raises(ValidationError): + secret_var.value = 'new_secret_value' + + +def test_variable_value_type_immutable(): + with pytest.raises(ValidationError): + StringVariable(value_type=SegmentType.ARRAY_ANY, name='text', value='text') + + with pytest.raises(ValidationError): + StringVariable.model_validate({'value_type': 'not text', 'name': 'text', 'value': 'text'}) + + var = IntegerVariable(name='integer', value=42) + with pytest.raises(ValidationError): + IntegerVariable(value_type=SegmentType.ARRAY_ANY, name=var.name, value=var.value) + + var = FloatVariable(name='float', value=3.14) + with pytest.raises(ValidationError): + FloatVariable(value_type=SegmentType.ARRAY_ANY, name=var.name, value=var.value) + + var = SecretVariable(name='secret', value='secret_value') + with pytest.raises(ValidationError): + SecretVariable(value_type=SegmentType.ARRAY_ANY, name=var.name, value=var.value) + + +def test_object_variable_to_object(): + var = ObjectVariable( + name='object', + value={ + 'key1': ObjectVariable( + name='object', + value={ + 'key2': StringVariable(name='key2', value='value2'), + }, + ), + 'key2': ArrayAnyVariable( + name='array', + value=[ + StringVariable(name='key5_1', value='value5_1'), + IntegerVariable(name='key5_2', value=42), + ObjectVariable(name='key5_3', value={}), + ], + ), + }, + ) + + assert var.to_object() == { + 'key1': { + 'key2': 'value2', + }, + 'key2': [ + 'value5_1', + 42, + {}, + ], + } + + +def test_variable_to_object(): + var = StringVariable(name='text', value='text') + assert var.to_object() == 'text' + var = IntegerVariable(name='integer', value=42) + assert var.to_object() == 42 + var = FloatVariable(name='float', value=3.14) + assert var.to_object() == 3.14 + var = SecretVariable(name='secret', value='secret_value') + assert var.to_object() == 'secret_value' diff --git a/api/tests/unit_tests/core/helper/__init__.py b/api/tests/unit_tests/core/helper/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/core/helper/test_ssrf_proxy.py b/api/tests/unit_tests/core/helper/test_ssrf_proxy.py new file mode 100644 index 0000000000..d917bb1003 --- /dev/null +++ b/api/tests/unit_tests/core/helper/test_ssrf_proxy.py @@ -0,0 +1,52 @@ +import random +from unittest.mock import MagicMock, patch + +from core.helper.ssrf_proxy import SSRF_DEFAULT_MAX_RETRIES, STATUS_FORCELIST, make_request + + +@patch('httpx.request') +def test_successful_request(mock_request): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_request.return_value = mock_response + + response = make_request('GET', 'http://example.com') + assert response.status_code == 200 + + +@patch('httpx.request') +def test_retry_exceed_max_retries(mock_request): + mock_response = MagicMock() + mock_response.status_code = 500 + + side_effects = [mock_response] * SSRF_DEFAULT_MAX_RETRIES + mock_request.side_effect = side_effects + + try: + make_request('GET', 'http://example.com', max_retries=SSRF_DEFAULT_MAX_RETRIES - 1) + raise AssertionError("Expected Exception not raised") + except Exception as e: + assert str(e) == f"Reached maximum retries ({SSRF_DEFAULT_MAX_RETRIES - 1}) for URL http://example.com" + + +@patch('httpx.request') +def test_retry_logic_success(mock_request): + side_effects = [] + + for _ in range(SSRF_DEFAULT_MAX_RETRIES): + status_code = random.choice(STATUS_FORCELIST) + mock_response = MagicMock() + mock_response.status_code = status_code + side_effects.append(mock_response) + + mock_response_200 = MagicMock() + mock_response_200.status_code = 200 + side_effects.append(mock_response_200) + + mock_request.side_effect = side_effects + + response = make_request('GET', 'http://example.com', max_retries=SSRF_DEFAULT_MAX_RETRIES) + + assert response.status_code == 200 + assert mock_request.call_count == SSRF_DEFAULT_MAX_RETRIES + 1 + assert mock_request.call_args_list[0][1].get('method') == 'GET' diff --git a/api/tests/unit_tests/core/workflow/nodes/test_answer.py b/api/tests/unit_tests/core/workflow/nodes/test_answer.py index 102711b4b6..3a32829e37 100644 --- a/api/tests/unit_tests/core/workflow/nodes/test_answer.py +++ b/api/tests/unit_tests/core/workflow/nodes/test_answer.py @@ -31,9 +31,9 @@ def test_execute_answer(): pool = VariablePool(system_variables={ SystemVariable.FILES: [], SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) - pool.append_variable(node_id='start', variable_key_list=['weather'], value='sunny') - pool.append_variable(node_id='llm', variable_key_list=['text'], value='You are a helpful AI.') + }, user_inputs={}, environment_variables=[]) + pool.add(['start', 'weather'], 'sunny') + pool.add(['llm', 'text'], 'You are a helpful AI.') # Mock db.session.close() db.session.close = MagicMock() diff --git a/api/tests/unit_tests/core/workflow/nodes/test_if_else.py b/api/tests/unit_tests/core/workflow/nodes/test_if_else.py index 6860b2fd97..4662c5ff2b 100644 --- a/api/tests/unit_tests/core/workflow/nodes/test_if_else.py +++ b/api/tests/unit_tests/core/workflow/nodes/test_if_else.py @@ -121,24 +121,24 @@ def test_execute_if_else_result_true(): pool = VariablePool(system_variables={ SystemVariable.FILES: [], SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) - pool.append_variable(node_id='start', variable_key_list=['array_contains'], value=['ab', 'def']) - pool.append_variable(node_id='start', variable_key_list=['array_not_contains'], value=['ac', 'def']) - pool.append_variable(node_id='start', variable_key_list=['contains'], value='cabcde') - pool.append_variable(node_id='start', variable_key_list=['not_contains'], value='zacde') - pool.append_variable(node_id='start', variable_key_list=['start_with'], value='abc') - pool.append_variable(node_id='start', variable_key_list=['end_with'], value='zzab') - pool.append_variable(node_id='start', variable_key_list=['is'], value='ab') - pool.append_variable(node_id='start', variable_key_list=['is_not'], value='aab') - pool.append_variable(node_id='start', variable_key_list=['empty'], value='') - pool.append_variable(node_id='start', variable_key_list=['not_empty'], value='aaa') - pool.append_variable(node_id='start', variable_key_list=['equals'], value=22) - pool.append_variable(node_id='start', variable_key_list=['not_equals'], value=23) - pool.append_variable(node_id='start', variable_key_list=['greater_than'], value=23) - pool.append_variable(node_id='start', variable_key_list=['less_than'], value=21) - pool.append_variable(node_id='start', variable_key_list=['greater_than_or_equal'], value=22) - pool.append_variable(node_id='start', variable_key_list=['less_than_or_equal'], value=21) - pool.append_variable(node_id='start', variable_key_list=['not_null'], value='1212') + }, user_inputs={}, environment_variables=[]) + pool.add(['start', 'array_contains'], ['ab', 'def']) + pool.add(['start', 'array_not_contains'], ['ac', 'def']) + pool.add(['start', 'contains'], 'cabcde') + pool.add(['start', 'not_contains'], 'zacde') + pool.add(['start', 'start_with'], 'abc') + pool.add(['start', 'end_with'], 'zzab') + pool.add(['start', 'is'], 'ab') + pool.add(['start', 'is_not'], 'aab') + pool.add(['start', 'empty'], '') + pool.add(['start', 'not_empty'], 'aaa') + pool.add(['start', 'equals'], 22) + pool.add(['start', 'not_equals'], 23) + pool.add(['start', 'greater_than'], 23) + pool.add(['start', 'less_than'], 21) + pool.add(['start', 'greater_than_or_equal'], 22) + pool.add(['start', 'less_than_or_equal'], 21) + pool.add(['start', 'not_null'], '1212') # Mock db.session.close() db.session.close = MagicMock() @@ -184,9 +184,9 @@ def test_execute_if_else_result_false(): pool = VariablePool(system_variables={ SystemVariable.FILES: [], SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) - pool.append_variable(node_id='start', variable_key_list=['array_contains'], value=['1ab', 'def']) - pool.append_variable(node_id='start', variable_key_list=['array_not_contains'], value=['ab', 'def']) + }, user_inputs={}, environment_variables=[]) + pool.add(['start', 'array_contains'], ['1ab', 'def']) + pool.add(['start', 'array_not_contains'], ['ab', 'def']) # Mock db.session.close() db.session.close = MagicMock() diff --git a/api/tests/unit_tests/models/test_workflow.py b/api/tests/unit_tests/models/test_workflow.py new file mode 100644 index 0000000000..facea34b5b --- /dev/null +++ b/api/tests/unit_tests/models/test_workflow.py @@ -0,0 +1,95 @@ +from unittest import mock +from uuid import uuid4 + +import contexts +from constants import HIDDEN_VALUE +from core.app.segments import FloatVariable, IntegerVariable, SecretVariable, StringVariable +from models.workflow import Workflow + + +def test_environment_variables(): + contexts.tenant_id.set('tenant_id') + + # Create a Workflow instance + workflow = Workflow() + + # Create some EnvironmentVariable instances + variable1 = StringVariable.model_validate({'name': 'var1', 'value': 'value1', 'id': str(uuid4())}) + variable2 = IntegerVariable.model_validate({'name': 'var2', 'value': 123, 'id': str(uuid4())}) + variable3 = SecretVariable.model_validate({'name': 'var3', 'value': 'secret', 'id': str(uuid4())}) + variable4 = FloatVariable.model_validate({'name': 'var4', 'value': 3.14, 'id': str(uuid4())}) + + with ( + mock.patch('core.helper.encrypter.encrypt_token', return_value='encrypted_token'), + mock.patch('core.helper.encrypter.decrypt_token', return_value='secret'), + ): + # Set the environment_variables property of the Workflow instance + variables = [variable1, variable2, variable3, variable4] + workflow.environment_variables = variables + + # Get the environment_variables property and assert its value + assert workflow.environment_variables == variables + + +def test_update_environment_variables(): + contexts.tenant_id.set('tenant_id') + + # Create a Workflow instance + workflow = Workflow() + + # Create some EnvironmentVariable instances + variable1 = StringVariable.model_validate({'name': 'var1', 'value': 'value1', 'id': str(uuid4())}) + variable2 = IntegerVariable.model_validate({'name': 'var2', 'value': 123, 'id': str(uuid4())}) + variable3 = SecretVariable.model_validate({'name': 'var3', 'value': 'secret', 'id': str(uuid4())}) + variable4 = FloatVariable.model_validate({'name': 'var4', 'value': 3.14, 'id': str(uuid4())}) + + with ( + mock.patch('core.helper.encrypter.encrypt_token', return_value='encrypted_token'), + mock.patch('core.helper.encrypter.decrypt_token', return_value='secret'), + ): + variables = [variable1, variable2, variable3, variable4] + + # Set the environment_variables property of the Workflow instance + workflow.environment_variables = variables + assert workflow.environment_variables == [variable1, variable2, variable3, variable4] + + # Update the name of variable3 and keep the value as it is + variables[2] = variable3.model_copy( + update={ + 'name': 'new name', + 'value': HIDDEN_VALUE, + } + ) + + workflow.environment_variables = variables + assert workflow.environment_variables[2].name == 'new name' + assert workflow.environment_variables[2].value == variable3.value + + +def test_to_dict(): + contexts.tenant_id.set('tenant_id') + + # Create a Workflow instance + workflow = Workflow() + workflow.graph = '{}' + workflow.features = '{}' + + # Create some EnvironmentVariable instances + + with ( + mock.patch('core.helper.encrypter.encrypt_token', return_value='encrypted_token'), + mock.patch('core.helper.encrypter.decrypt_token', return_value='secret'), + ): + # Set the environment_variables property of the Workflow instance + workflow.environment_variables = [ + SecretVariable.model_validate({'name': 'secret', 'value': 'secret', 'id': str(uuid4())}), + StringVariable.model_validate({'name': 'text', 'value': 'text', 'id': str(uuid4())}), + ] + + workflow_dict = workflow.to_dict() + assert workflow_dict['environment_variables'][0]['value'] == '' + assert workflow_dict['environment_variables'][1]['value'] == 'text' + + workflow_dict = workflow.to_dict(include_secret=True) + assert workflow_dict['environment_variables'][0]['value'] == 'secret' + assert workflow_dict['environment_variables'][1]['value'] == 'text' diff --git a/api/tests/unit_tests/utils/position_helper/test_position_helper.py b/api/tests/unit_tests/utils/position_helper/test_position_helper.py index c389461454..2237319904 100644 --- a/api/tests/unit_tests/utils/position_helper/test_position_helper.py +++ b/api/tests/unit_tests/utils/position_helper/test_position_helper.py @@ -21,6 +21,20 @@ def prepare_example_positions_yaml(tmp_path, monkeypatch) -> str: return str(tmp_path) +@pytest.fixture +def prepare_empty_commented_positions_yaml(tmp_path, monkeypatch) -> str: + monkeypatch.chdir(tmp_path) + tmp_path.joinpath("example_positions_all_commented.yaml").write_text(dedent( + """\ + # - commented1 + # - commented2 + - + - + + """)) + return str(tmp_path) + + def test_position_helper(prepare_example_positions_yaml): position_map = get_position_map( folder_path=prepare_example_positions_yaml, @@ -32,3 +46,10 @@ def test_position_helper(prepare_example_positions_yaml): 'third': 2, 'forth': 3, } + + +def test_position_helper_with_all_commented(prepare_empty_commented_positions_yaml): + position_map = get_position_map( + folder_path=prepare_empty_commented_positions_yaml, + file_name='example_positions_all_commented.yaml') + assert position_map == {} diff --git a/api/tests/unit_tests/utils/yaml/test_yaml_utils.py b/api/tests/unit_tests/utils/yaml/test_yaml_utils.py index 446588cde1..c0452b4e4d 100644 --- a/api/tests/unit_tests/utils/yaml/test_yaml_utils.py +++ b/api/tests/unit_tests/utils/yaml/test_yaml_utils.py @@ -53,6 +53,9 @@ def test_load_yaml_non_existing_file(): assert load_yaml_file(file_path=NON_EXISTING_YAML_FILE) == {} assert load_yaml_file(file_path='') == {} + with pytest.raises(FileNotFoundError): + load_yaml_file(file_path=NON_EXISTING_YAML_FILE, ignore_error=False) + def test_load_valid_yaml_file(prepare_example_yaml_file): yaml_data = load_yaml_file(file_path=prepare_example_yaml_file) @@ -68,7 +71,7 @@ def test_load_valid_yaml_file(prepare_example_yaml_file): def test_load_invalid_yaml_file(prepare_invalid_yaml_file): # yaml syntax error with pytest.raises(YAMLError): - load_yaml_file(file_path=prepare_invalid_yaml_file) + load_yaml_file(file_path=prepare_invalid_yaml_file, ignore_error=False) # ignore error - assert load_yaml_file(file_path=prepare_invalid_yaml_file, ignore_error=True) == {} + assert load_yaml_file(file_path=prepare_invalid_yaml_file) == {} diff --git a/dev/pytest/pytest_vdb.sh b/dev/pytest/pytest_vdb.sh index cb8aae6740..c954c528fb 100755 --- a/dev/pytest/pytest_vdb.sh +++ b/dev/pytest/pytest_vdb.sh @@ -3,7 +3,6 @@ set -x pytest api/tests/integration_tests/vdb/chroma \ api/tests/integration_tests/vdb/milvus \ - api/tests/integration_tests/vdb/myscale \ api/tests/integration_tests/vdb/pgvecto_rs \ api/tests/integration_tests/vdb/pgvector \ api/tests/integration_tests/vdb/qdrant \ diff --git a/docker-legacy/docker-compose.yaml b/docker-legacy/docker-compose.yaml index 416a1b8c52..9d7039df2c 100644 --- a/docker-legacy/docker-compose.yaml +++ b/docker-legacy/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3' services: # API service api: - image: langgenius/dify-api:0.6.13 + image: langgenius/dify-api:0.6.15 restart: always environment: # Startup mode, 'api' starts the API server. @@ -224,7 +224,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.6.13 + image: langgenius/dify-api:0.6.15 restart: always environment: CONSOLE_WEB_URL: '' @@ -390,7 +390,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.6.13 + image: langgenius/dify-web:0.6.15 restart: always environment: # The base URL of console application api server, refers to the Console base URL of WEB service if console domain is diff --git a/docker/.env.example b/docker/.env.example index 4f7e13e823..2f8ec358f4 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -124,6 +124,10 @@ GUNICORN_TIMEOUT=360 # The number of Celery workers. The default is 1, and can be set as needed. CELERY_WORKER_AMOUNT= +# API Tool configuration +API_TOOL_DEFAULT_CONNECT_TIMEOUT=10 +API_TOOL_DEFAULT_READ_TIMEOUT=60 + # ------------------------------ # Database Configuration # The database uses PostgreSQL. Please use the public schema. diff --git a/docker/docker-compose.middleware.yaml b/docker/docker-compose.middleware.yaml index 3dee6efb7c..6ab003ceab 100644 --- a/docker/docker-compose.middleware.yaml +++ b/docker/docker-compose.middleware.yaml @@ -48,7 +48,7 @@ services: # ssrf_proxy server # for more information, please refer to - # https://docs.dify.ai/getting-started/install-self-hosted/install-faq#id-16.-why-is-ssrf_proxy-needed + # https://docs.dify.ai/learn-more/faq/self-host-faq#id-18.-why-is-ssrf_proxy-needed ssrf_proxy: image: ubuntu/squid:latest restart: always diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index beca104a68..a9b7b8acb0 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -22,6 +22,8 @@ x-shared-env: &shared-api-worker-env CELERY_WORKER_CLASS: ${CELERY_WORKER_CLASS:-} GUNICORN_TIMEOUT: ${GUNICORN_TIMEOUT:-360} CELERY_WORKER_AMOUNT: ${CELERY_WORKER_AMOUNT:-} + API_TOOL_DEFAULT_CONNECT_TIMEOUT: ${API_TOOL_DEFAULT_CONNECT_TIMEOUT:-10} + API_TOOL_DEFAULT_READ_TIMEOUT: ${API_TOOL_DEFAULT_READ_TIMEOUT:-60} DB_USERNAME: ${DB_USERNAME:-postgres} DB_PASSWORD: ${DB_PASSWORD:-difyai123456} DB_HOST: ${DB_HOST:-db} @@ -177,7 +179,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:0.6.13 + image: langgenius/dify-api:0.6.15 restart: always environment: # Use the shared environment variables. @@ -197,7 +199,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.6.13 + image: langgenius/dify-api:0.6.15 restart: always environment: # Use the shared environment variables. @@ -216,7 +218,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.6.13 + image: langgenius/dify-web:0.6.15 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} diff --git a/docker/startupscripts/init.sh b/docker/startupscripts/init.sh new file mode 100755 index 0000000000..ee7600850a --- /dev/null +++ b/docker/startupscripts/init.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +DB_INITIALISED="/opt/oracle/oradata/dbinit" +#[ -f ${DB_INITIALISED} ] && exit +#touch ${DB_INITIALISED} +if [ -f ${DB_INITIALISED} ]; then + echo 'File exists. Standards for have been Init' + exit +else + echo 'File does not exist. Standards for first time Strart up this DB' + "$ORACLE_HOME"/bin/sqlplus -s "/ as sysdba" @"/opt/oracle/scripts/startup/init_user.script"; + touch ${DB_INITIALISED} +fi diff --git a/docker/startupscripts/create_user.sql b/docker/startupscripts/init_user.script similarity index 70% rename from docker/startupscripts/create_user.sql rename to docker/startupscripts/init_user.script index b80e19c3b0..7aa7c28049 100755 --- a/docker/startupscripts/create_user.sql +++ b/docker/startupscripts/init_user.script @@ -3,3 +3,8 @@ ALTER SYSTEM SET PROCESSES=500 SCOPE=SPFILE; alter session set container= freepdb1; create user dify identified by dify DEFAULT TABLESPACE users quota unlimited on users; grant DB_DEVELOPER_ROLE to dify; + +BEGIN +CTX_DDL.CREATE_PREFERENCE('my_chinese_vgram_lexer','CHINESE_VGRAM_LEXER'); +END; +/ diff --git a/sdks/python-client/dify_client/client.py b/sdks/python-client/dify_client/client.py index 53880c1000..22d4e6189a 100644 --- a/sdks/python-client/dify_client/client.py +++ b/sdks/python-client/dify_client/client.py @@ -2,9 +2,9 @@ import requests class DifyClient: - def __init__(self, api_key): + def __init__(self, api_key, base_url: str = 'https://api.dify.ai/v1'): self.api_key = api_key - self.base_url = "https://api.dify.ai/v1" + self.base_url = base_url def _send_request(self, method, endpoint, json=None, params=None, stream=False): headers = { diff --git a/web/.vscode/extensions.json b/web/.vscode/extensions.json new file mode 100644 index 0000000000..d7680d74a5 --- /dev/null +++ b/web/.vscode/extensions.json @@ -0,0 +1,6 @@ +{ + "recommendations": [ + "bradlc.vscode-tailwindcss", + "firsttris.vscode-jest-runner" + ] +} diff --git a/web/README.md b/web/README.md index 2ecba1c8ff..867d822e27 100644 --- a/web/README.md +++ b/web/README.md @@ -74,6 +74,25 @@ npm run start --port=3001 --host=0.0.0.0 If your IDE is VSCode, rename `web/.vscode/settings.example.json` to `web/.vscode/settings.json` for lint code setting. +## Test + +We start to use [Jest](https://jestjs.io/) and [React Testing Library](https://testing-library.com/docs/react-testing-library/intro/) for Unit Testing. + +You can create a test file with a suffix of `.spec` beside the file that to be tested. For example, if you want to test a file named `util.ts`. The test file name should be `util.spec.ts`. + +Run test: + +```bash +npm run test +``` + +If you are not familiar with writing tests, here is some code to refer to: +* [classnames.spec.ts](./utils/classnames.spec.ts) +* [index.spec.tsx](./app/components/base/button/index.spec.tsx) + + + + ## Documentation Visit to view the full documentation. diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx index 86bee98bcd..09569df8bf 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx @@ -40,7 +40,7 @@ const AppDetailLayout: FC = (props) => { const pathname = usePathname() const media = useBreakpoints() const isMobile = media === MediaType.mobile - const { isCurrentWorkspaceManager, isCurrentWorkspaceEditor } = useAppContext() + const { isCurrentWorkspaceEditor } = useAppContext() const { appDetail, setAppDetail, setAppSiderbarExpand } = useStore(useShallow(state => ({ appDetail: state.appDetail, setAppDetail: state.setAppDetail, @@ -53,7 +53,7 @@ const AppDetailLayout: FC = (props) => { selectedIcon: NavIcon }>>([]) - const getNavigations = useCallback((appId: string, isCurrentWorkspaceManager: boolean, isCurrentWorkspaceEditor: boolean, mode: string) => { + const getNavigations = useCallback((appId: string, isCurrentWorkspaceEditor: boolean, mode: string) => { const navs = [ ...(isCurrentWorkspaceEditor ? [{ @@ -70,7 +70,7 @@ const AppDetailLayout: FC = (props) => { icon: RiTerminalBoxLine, selectedIcon: RiTerminalBoxFill, }, - ...(isCurrentWorkspaceManager + ...(isCurrentWorkspaceEditor ? [{ name: mode !== 'workflow' ? t('common.appMenus.logAndAnn') @@ -115,13 +115,13 @@ const AppDetailLayout: FC = (props) => { } else { setAppDetail(res) - setNavigation(getNavigations(appId, isCurrentWorkspaceManager, isCurrentWorkspaceEditor, res.mode)) + setNavigation(getNavigations(appId, isCurrentWorkspaceEditor, res.mode)) } }).catch((e: any) => { if (e.status === 404) router.replace('/apps') }) - }, [appId, isCurrentWorkspaceManager, isCurrentWorkspaceEditor]) + }, [appId, isCurrentWorkspaceEditor]) useUnmount(() => { setAppDetail() diff --git a/web/app/(commonLayout)/apps/AppCard.tsx b/web/app/(commonLayout)/apps/AppCard.tsx index 53b31af7f0..34279f816e 100644 --- a/web/app/(commonLayout)/apps/AppCard.tsx +++ b/web/app/(commonLayout)/apps/AppCard.tsx @@ -28,6 +28,9 @@ import EditAppModal from '@/app/components/explore/create-app-modal' import SwitchAppModal from '@/app/components/app/switch-app-modal' import type { Tag } from '@/app/components/base/tag-management/constant' import TagSelector from '@/app/components/base/tag-management/selector' +import type { EnvironmentVariable } from '@/app/components/workflow/types' +import DSLExportConfirmModal from '@/app/components/workflow/dsl-export-confirm-modal' +import { fetchWorkflowDraft } from '@/service/workflow' export type AppCardProps = { app: App @@ -50,6 +53,7 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { const [showDuplicateModal, setShowDuplicateModal] = useState(false) const [showSwitchModal, setShowSwitchModal] = useState(false) const [showConfirmDelete, setShowConfirmDelete] = useState(false) + const [secretEnvList, setSecretEnvList] = useState([]) const onConfirmDelete = useCallback(async () => { try { @@ -123,9 +127,12 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { } } - const onExport = async () => { + const onExport = async (include = false) => { try { - const { data } = await exportAppConfig(app.id) + const { data } = await exportAppConfig({ + appID: app.id, + include, + }) const a = document.createElement('a') const file = new Blob([data], { type: 'application/yaml' }) a.href = URL.createObjectURL(file) @@ -137,6 +144,25 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { } } + const exportCheck = async () => { + if (app.mode !== 'workflow' && app.mode !== 'advanced-chat') { + onExport() + return + } + try { + const workflowDraft = await fetchWorkflowDraft(`/apps/${app.id}/workflows/draft`) + const list = (workflowDraft.environment_variables || []).filter(env => env.value_type === 'secret') + if (list.length === 0) { + onExport() + return + } + setSecretEnvList(list) + } + catch (e) { + notify({ type: 'error', message: t('app.exportFailed') }) + } + } + const onSwitch = () => { if (onRefresh) onRefresh() @@ -164,7 +190,7 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { e.stopPropagation() props.onClick?.() e.preventDefault() - onExport() + exportCheck() } const onClickSwitch = async (e: React.MouseEvent) => { e.stopPropagation() @@ -371,6 +397,13 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { onCancel={() => setShowConfirmDelete(false)} /> )} + {secretEnvList.length > 0 && ( + setSecretEnvList([])} + /> + )} ) } diff --git a/web/app/(commonLayout)/apps/NewAppCard.tsx b/web/app/(commonLayout)/apps/NewAppCard.tsx index 16f6ee26f0..c0dffa99ab 100644 --- a/web/app/(commonLayout)/apps/NewAppCard.tsx +++ b/web/app/(commonLayout)/apps/NewAppCard.tsx @@ -1,10 +1,14 @@ 'use client' -import { forwardRef, useState } from 'react' +import { forwardRef, useMemo, useState } from 'react' +import { + useRouter, + useSearchParams, +} from 'next/navigation' import { useTranslation } from 'react-i18next' import CreateAppTemplateDialog from '@/app/components/app/create-app-dialog' import CreateAppModal from '@/app/components/app/create-app-modal' -import CreateFromDSLModal from '@/app/components/app/create-from-dsl-modal' +import CreateFromDSLModal, { CreateFromDSLModalTab } from '@/app/components/app/create-from-dsl-modal' import { useProviderContext } from '@/context/provider-context' import { FileArrow01, FilePlus01, FilePlus02 } from '@/app/components/base/icons/src/vender/line/files' @@ -16,10 +20,21 @@ export type CreateAppCardProps = { const CreateAppCard = forwardRef(({ onSuccess }, ref) => { const { t } = useTranslation() const { onPlanInfoChanged } = useProviderContext() + const searchParams = useSearchParams() + const { replace } = useRouter() + const dslUrl = searchParams.get('remoteInstallUrl') || undefined const [showNewAppTemplateDialog, setShowNewAppTemplateDialog] = useState(false) const [showNewAppModal, setShowNewAppModal] = useState(false) - const [showCreateFromDSLModal, setShowCreateFromDSLModal] = useState(false) + const [showCreateFromDSLModal, setShowCreateFromDSLModal] = useState(!!dslUrl) + + const activeTab = useMemo(() => { + if (dslUrl) + return CreateFromDSLModalTab.FROM_URL + + return undefined + }, [dslUrl]) + return ( (({ onSuc /> setShowCreateFromDSLModal(false)} + onClose={() => { + setShowCreateFromDSLModal(false) + + if (dslUrl) + replace('/') + }} + activeTab={activeTab} + dslUrl={dslUrl} onSuccess={() => { onPlanInfoChanged() if (onSuccess) diff --git a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx index a1543230a9..11893ec9de 100644 --- a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx +++ b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx @@ -166,8 +166,8 @@ const ExtraInfo = ({ isMobile, relatedApps }: IExtraInfoProps) => { className='inline-flex items-center text-xs text-primary-600 mt-2 cursor-pointer' href={ locale === LanguagesSupported[1] - ? 'https://docs.dify.ai/v/zh-hans/guides/application-design/prompt-engineering' - : 'https://docs.dify.ai/user-guide/creating-dify-apps/prompt-engineering' + ? 'https://docs.dify.ai/v/zh-hans/guides/knowledge-base/integrate_knowledge_within_application' + : 'https://docs.dify.ai/guides/knowledge-base/integrate-knowledge-within-application' } target='_blank' rel='noopener noreferrer' > diff --git a/web/app/(commonLayout)/layout.tsx b/web/app/(commonLayout)/layout.tsx index ceeb164315..af36d4d961 100644 --- a/web/app/(commonLayout)/layout.tsx +++ b/web/app/(commonLayout)/layout.tsx @@ -3,7 +3,7 @@ import type { ReactNode } from 'react' import SwrInitor from '@/app/components/swr-initor' import { AppContextProvider } from '@/context/app-context' import GA, { GaType } from '@/app/components/base/ga' -import HeaderWrapper from '@/app/components/header/HeaderWrapper' +import HeaderWrapper from '@/app/components/header/header-wrapper' import Header from '@/app/components/header' import { EventEmitterContextProvider } from '@/context/event-emitter' import { ProviderContextProvider } from '@/context/provider-context' diff --git a/web/app/(commonLayout)/tools/page.tsx b/web/app/(commonLayout)/tools/page.tsx index 4e64d8c0df..1b08d54ba3 100644 --- a/web/app/(commonLayout)/tools/page.tsx +++ b/web/app/(commonLayout)/tools/page.tsx @@ -12,15 +12,16 @@ const Layout: FC = () => { const { isCurrentWorkspaceDatasetOperator } = useAppContext() useEffect(() => { - document.title = `${t('tools.title')} - Dify` + if (typeof window !== 'undefined') + document.title = `${t('tools.title')} - Dify` if (isCurrentWorkspaceDatasetOperator) return router.replace('/datasets') - }, []) + }, [isCurrentWorkspaceDatasetOperator, router, t]) useEffect(() => { if (isCurrentWorkspaceDatasetOperator) return router.replace('/datasets') - }, [isCurrentWorkspaceDatasetOperator]) + }, [isCurrentWorkspaceDatasetOperator, router]) return } diff --git a/web/app/components/app-sidebar/app-info.tsx b/web/app/components/app-sidebar/app-info.tsx index c931afbe7f..3f52e9d6f5 100644 --- a/web/app/components/app-sidebar/app-info.tsx +++ b/web/app/components/app-sidebar/app-info.tsx @@ -28,6 +28,9 @@ import type { CreateAppModalProps } from '@/app/components/explore/create-app-mo import { NEED_REFRESH_APP_LIST_KEY } from '@/config' import { getRedirection } from '@/utils/app-redirection' import UpdateDSLModal from '@/app/components/workflow/update-dsl-modal' +import type { EnvironmentVariable } from '@/app/components/workflow/types' +import DSLExportConfirmModal from '@/app/components/workflow/dsl-export-confirm-modal' +import { fetchWorkflowDraft } from '@/service/workflow' export type IAppInfoProps = { expand: boolean @@ -47,6 +50,7 @@ const AppInfo = ({ expand }: IAppInfoProps) => { const [showSwitchTip, setShowSwitchTip] = useState('') const [showSwitchModal, setShowSwitchModal] = useState(false) const [showImportDSLModal, setShowImportDSLModal] = useState(false) + const [secretEnvList, setSecretEnvList] = useState([]) const mutateApps = useContextSelector( AppsContext, @@ -108,11 +112,14 @@ const AppInfo = ({ expand }: IAppInfoProps) => { } } - const onExport = async () => { + const onExport = async (include = false) => { if (!appDetail) return try { - const { data } = await exportAppConfig(appDetail.id) + const { data } = await exportAppConfig({ + appID: appDetail.id, + include, + }) const a = document.createElement('a') const file = new Blob([data], { type: 'application/yaml' }) a.href = URL.createObjectURL(file) @@ -124,6 +131,27 @@ const AppInfo = ({ expand }: IAppInfoProps) => { } } + const exportCheck = async () => { + if (!appDetail) + return + if (appDetail.mode !== 'workflow' && appDetail.mode !== 'advanced-chat') { + onExport() + return + } + try { + const workflowDraft = await fetchWorkflowDraft(`/apps/${appDetail.id}/workflows/draft`) + const list = (workflowDraft.environment_variables || []).filter(env => env.value_type === 'secret') + if (list.length === 0) { + onExport() + return + } + setSecretEnvList(list) + } + catch (e) { + notify({ type: 'error', message: t('app.exportFailed') }) + } + } + const onConfirmDelete = useCallback(async () => { if (!appDetail) return @@ -190,7 +218,7 @@ const AppInfo = ({ expand }: IAppInfoProps) => { {expand && (
-
+
{appDetail.name}
{isCurrentWorkspaceEditor && }
@@ -314,7 +342,7 @@ const AppInfo = ({ expand }: IAppInfoProps) => { )} -
+
{t('app.export')}
{ @@ -403,14 +431,19 @@ const AppInfo = ({ expand }: IAppInfoProps) => { onCancel={() => setShowConfirmDelete(false)} /> )} - { - showImportDSLModal && ( - setShowImportDSLModal(false)} - onBackup={onExport} - /> - ) - } + {showImportDSLModal && ( + setShowImportDSLModal(false)} + onBackup={exportCheck} + /> + )} + {secretEnvList.length > 0 && ( + setSecretEnvList([])} + /> + )}
) diff --git a/web/app/components/app-sidebar/index.tsx b/web/app/components/app-sidebar/index.tsx index 9a78cd7e6e..5d5d407dc0 100644 --- a/web/app/components/app-sidebar/index.tsx +++ b/web/app/components/app-sidebar/index.tsx @@ -49,7 +49,7 @@ const AppDetailNav = ({ title, desc, icon, icon_background, navigation, extraInf return (
@@ -60,7 +60,7 @@ const AppDetailNav = ({ title, desc, icon, icon_background, navigation, extraInf `} > {iconType === 'app' && ( - + )} {iconType !== 'app' && ( {!expand && ( -
+
)}
)} + hideResize={noResize} > void } @@ -26,7 +30,11 @@ const Prompt: FC = ({ mode, promptTemplate, promptVariables, + noTitle, + gradientBorder, readonly = false, + editorHeight, + noResize, onChange, }) => { const { t } = useTranslation() @@ -99,6 +107,10 @@ const Prompt: FC = ({ promptVariables={promptVariables} readonly={readonly} onChange={onChange} + noTitle={noTitle} + gradientBorder={gradientBorder} + editorHeight={editorHeight} + noResize={noResize} /> ) } @@ -121,6 +133,7 @@ const Prompt: FC = ({ promptVariables={promptVariables} isContextMissing={isContextMissing && !isHideContextMissTip} onHideContextMissingTip={() => setIsHideContextMissTip(true)} + noResize={noResize} /> )) ) @@ -136,6 +149,7 @@ const Prompt: FC = ({ promptVariables={promptVariables} isContextMissing={isContextMissing && !isHideContextMissTip} onHideContextMissingTip={() => setIsHideContextMissTip(true)} + noResize={noResize} /> ) } diff --git a/web/app/components/app/configuration/config-prompt/simple-prompt-input.tsx b/web/app/components/app/configuration/config-prompt/simple-prompt-input.tsx index a15f538227..2811c42402 100644 --- a/web/app/components/app/configuration/config-prompt/simple-prompt-input.tsx +++ b/web/app/components/app/configuration/config-prompt/simple-prompt-input.tsx @@ -14,6 +14,7 @@ import PromptEditorHeightResizeWrap from './prompt-editor-height-resize-wrap' import cn from '@/utils/classnames' import { type PromptVariable } from '@/models/debug' import Tooltip from '@/app/components/base/tooltip' +import type { CompletionParams } from '@/types/app' import { AppType } from '@/types/app' import { getNewVar, getVars } from '@/utils/var' import AutomaticBtn from '@/app/components/app/configuration/config/automatic/automatic-btn' @@ -28,6 +29,7 @@ import { useEventEmitterContextContext } from '@/context/event-emitter' import { ADD_EXTERNAL_DATA_TOOL } from '@/app/components/app/configuration/config-var' import { INSERT_VARIABLE_VALUE_BLOCK_COMMAND } from '@/app/components/base/prompt-editor/plugins/variable-block' import { PROMPT_EDITOR_UPDATE_VALUE_BY_EVENT_EMITTER } from '@/app/components/base/prompt-editor/plugins/update-block' +import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' export type ISimplePromptInput = { mode: AppType @@ -35,6 +37,10 @@ export type ISimplePromptInput = { promptVariables: PromptVariable[] readonly?: boolean onChange?: (promp: string, promptVariables: PromptVariable[]) => void + noTitle?: boolean + gradientBorder?: boolean + editorHeight?: number + noResize?: boolean } const Prompt: FC = ({ @@ -43,11 +49,19 @@ const Prompt: FC = ({ promptVariables, readonly = false, onChange, + noTitle, + gradientBorder, + editorHeight: initEditorHeight, + noResize, }) => { const { t } = useTranslation() + const media = useBreakpoints() + const isMobile = media === MediaType.mobile + const { eventEmitter } = useEventEmitterContextContext() const { modelConfig, + completionParams, dataSets, setModelConfig, setPrevPromptConfig, @@ -116,6 +130,11 @@ const Prompt: FC = ({ const [showAutomatic, { setTrue: showAutomaticTrue, setFalse: showAutomaticFalse }] = useBoolean(false) const handleAutomaticRes = (res: AutomaticRes) => { + // put eventEmitter in first place to prevent overwrite the configs.prompt_variables.But another problem is that prompt won't hight the prompt_variables. + eventEmitter?.emit({ + type: PROMPT_EDITOR_UPDATE_VALUE_BY_EVENT_EMITTER, + payload: res.prompt, + } as any) const newModelConfig = produce(modelConfig, (draft) => { draft.configs.prompt_template = res.prompt draft.configs.prompt_variables = res.variables.map(key => ({ key, name: key, type: 'string', required: true })) @@ -125,41 +144,41 @@ const Prompt: FC = ({ if (mode !== AppType.completion) setIntroduction(res.opening_statement) showAutomaticFalse() - eventEmitter?.emit({ - type: PROMPT_EDITOR_UPDATE_VALUE_BY_EVENT_EMITTER, - payload: res.prompt, - } as any) } - const minHeight = 228 + const minHeight = initEditorHeight || 228 const [editorHeight, setEditorHeight] = useState(minHeight) return ( -
+
-
-
-
{mode !== AppType.completion ? t('appDebug.chatSubTitle') : t('appDebug.completionSubTitle')}
- {!readonly && ( - - {t('appDebug.promptTip')} -
} - selector='config-prompt-tooltip'> - - - )} + {!noTitle && ( +
+
+
{mode !== AppType.completion ? t('appDebug.chatSubTitle') : t('appDebug.completionSubTitle')}
+ {!readonly && ( + + {t('appDebug.promptTip')} +
} + selector='config-prompt-tooltip'> + + + )} +
+
+ {!isAgent && !readonly && !isMobile && ( + + )} +
-
- {!isAgent && !readonly && ( - - )} -
-
+ )} +
{promptTemplate.length}
@@ -216,6 +235,7 @@ const Prompt: FC = ({ onBlur={() => { handleChange(promptTemplate, getVars(promptTemplate)) }} + editable={!readonly} />
@@ -232,6 +252,14 @@ const Prompt: FC = ({ {showAutomatic && ( = ({ promptVariables, readonly, onPromptVar ...rest, type: type === InputVarType.textInput ? 'string' : type, key: variable, - name: label, + name: label as string, } if (payload.type === InputVarType.textInput) diff --git a/web/app/components/app/configuration/config-voice/param-config-content.tsx b/web/app/components/app/configuration/config-voice/param-config-content.tsx index cced3b0458..9b0d5bbb69 100644 --- a/web/app/components/app/configuration/config-voice/param-config-content.tsx +++ b/web/app/components/app/configuration/config-voice/param-config-content.tsx @@ -31,12 +31,12 @@ const VoiceParamConfig: FC = () => { let languageItem = languages.find(item => item.value === textToSpeechConfig.language) const localLanguagePlaceholder = languageItem?.name || t('common.placeholder.select') - if (languages && !languageItem) + if (languages && !languageItem && languages.length > 0) languageItem = languages[0] const language = languageItem?.value const voiceItems = useSWR({ appId, language }, fetchAppVoices).data let voiceItem = voiceItems?.find(item => item.value === textToSpeechConfig.voice) - if (voiceItems && !voiceItem) + if (voiceItems && !voiceItem && voiceItems.length > 0) voiceItem = voiceItems[0] const localVoicePlaceholder = voiceItem?.name || t('common.placeholder.select') @@ -125,9 +125,11 @@ const VoiceParamConfig: FC = () => {
{t('appDebug.voice.voiceSettings.voice')}
{ + if (!value.value) + return setTextToSpeechConfig({ ...textToSpeechConfig, voice: String(value.value), diff --git a/web/app/components/app/configuration/config/automatic/automatic-btn.tsx b/web/app/components/app/configuration/config/automatic/automatic-btn.tsx index 40a9b9d799..f70976082d 100644 --- a/web/app/components/app/configuration/config/automatic/automatic-btn.tsx +++ b/web/app/components/app/configuration/config/automatic/automatic-btn.tsx @@ -2,29 +2,21 @@ import type { FC } from 'react' import React from 'react' import { useTranslation } from 'react-i18next' +import { Generator } from '@/app/components/base/icons/src/vender/other' export type IAutomaticBtnProps = { onClick: () => void } - -const leftIcon = ( - - - - - - -) const AutomaticBtn: FC = ({ onClick, }) => { const { t } = useTranslation() return ( -
- {leftIcon} + {t('appDebug.operation.automatic')}
) diff --git a/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx b/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx index fa58253cac..1939dd3ad7 100644 --- a/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx +++ b/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx @@ -1,70 +1,126 @@ 'use client' import type { FC } from 'react' -import React from 'react' +import React, { useCallback } from 'react' import { useTranslation } from 'react-i18next' import { useBoolean } from 'ahooks' +import { + RiDatabase2Line, + RiFileExcel2Line, + RiGitCommitLine, + RiNewspaperLine, + RiPresentationLine, + RiRoadMapLine, + RiTerminalBoxLine, + RiTranslate, + RiUser2Line, +} from '@remixicon/react' +import cn from 'classnames' +import s from './style.module.css' import Modal from '@/app/components/base/modal' import Button from '@/app/components/base/button' import Toast from '@/app/components/base/toast' import { generateRule } from '@/service/debug' import ConfigPrompt from '@/app/components/app/configuration/config-prompt' +import type { Model } from '@/types/app' import { AppType } from '@/types/app' import ConfigVar from '@/app/components/app/configuration/config-var' import OpeningStatement from '@/app/components/app/configuration/features/chat-group/opening-statement' import GroupName from '@/app/components/app/configuration/base/group-name' import Loading from '@/app/components/base/loading' import Confirm from '@/app/components/base/confirm' + // type import type { AutomaticRes } from '@/service/debug' -import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' - -const noDataIcon = ( - - - -) +import { Generator } from '@/app/components/base/icons/src/vender/other' export type IGetAutomaticResProps = { mode: AppType + model: Model isShow: boolean onClose: () => void onFinished: (res: AutomaticRes) => void + isInLLMNode?: boolean } -const genIcon = ( - - - - - -) +const TryLabel: FC<{ + Icon: any + text: string + onClick: () => void +}> = ({ Icon, text, onClick }) => { + return ( +
+ +
{text}
+
+ ) +} const GetAutomaticRes: FC = ({ mode, + model, isShow, onClose, - // appId, + isInLLMNode, onFinished, }) => { const { t } = useTranslation() - const media = useBreakpoints() - const isMobile = media === MediaType.mobile + const tryList = [ + { + icon: RiTerminalBoxLine, + key: 'pythonDebugger', + }, + { + icon: RiTranslate, + key: 'translation', + }, + { + icon: RiPresentationLine, + key: 'meetingTakeaways', + }, + { + icon: RiNewspaperLine, + key: 'writingsPolisher', + }, + { + icon: RiUser2Line, + key: 'professionalAnalyst', + }, + { + icon: RiFileExcel2Line, + key: 'excelFormulaExpert', + }, + { + icon: RiRoadMapLine, + key: 'travelPlanning', + }, + { + icon: RiDatabase2Line, + key: 'SQLSorcerer', + }, + { + icon: RiGitCommitLine, + key: 'GitGud', + }, + ] - const [audiences, setAudiences] = React.useState('') - const [hopingToSolve, setHopingToSolve] = React.useState('') - const isValid = () => { - if (audiences.trim() === '') { - Toast.notify({ - type: 'error', - message: t('appDebug.automatic.audiencesRequired'), - }) - return false + const [instruction, setInstruction] = React.useState('') + const handleChooseTemplate = useCallback((key: string) => { + return () => { + const template = t(`appDebug.generate.template.${key}.instruction`) + setInstruction(template) } - if (hopingToSolve.trim() === '') { + }, [t]) + const isValid = () => { + if (instruction.trim() === '') { Toast.notify({ type: 'error', - message: t('appDebug.automatic.problemRequired'), + message: t('common.errorMsg.fieldRequired', { + field: t('appDebug.generate.instruction'), + }), }) return false } @@ -76,14 +132,17 @@ const GetAutomaticRes: FC = ({ const renderLoading = (
-
{t('appDebug.automatic.loading')}
+
{t('appDebug.generate.loading')}
) const renderNoData = (
- {noDataIcon} -
{t('appDebug.automatic.noData')}
+ +
+
{t('appDebug.generate.noDataLine1')}
+
{t('appDebug.generate.noDataLine2')}
+
) @@ -94,11 +153,18 @@ const GetAutomaticRes: FC = ({ return setLoadingTrue() try { - const res = await generateRule({ - audiences, - hoping_to_solve: hopingToSolve, + const { error, ...res } = await generateRule({ + instruction, + model_config: model, + no_variable: !!isInLLMNode, }) setRes(res) + if (error) { + Toast.notify({ + type: 'error', + message: error, + }) + } } finally { setLoadingFalse() @@ -107,24 +173,7 @@ const GetAutomaticRes: FC = ({ const [showConfirmOverwrite, setShowConfirmOverwrite] = React.useState(false) - const isShowAutoPromptInput = () => { - if (isMobile) { - // hide prompt panel on mobile if it is loading or has had result - if (isLoading || res) - return false - return true - } - - // always display prompt panel on desktop mode - return true - } - const isShowAutoPromptResPlaceholder = () => { - if (isMobile) { - // hide placeholder panel on mobile - return false - } - return !isLoading && !res } @@ -132,75 +181,97 @@ const GetAutomaticRes: FC = ({ -
- {isShowAutoPromptInput() &&
-
-
{t('appDebug.automatic.title')}
-
{t('appDebug.automatic.description')}
+
+
+
+
{t('appDebug.generate.title')}
+
{t('appDebug.generate.description')}
+
+
+
+
{t('appDebug.generate.tryIt')}
+
+
+
+ {tryList.map(item => ( + + ))} +
{/* inputs */} -
-
-
{t('appDebug.automatic.intendedAudience')}
- setAudiences(e.target.value)} /> -
-
-
{t('appDebug.automatic.solveProblem')}
-