From 77789016300b0c0e9070812639e60d98132128b2 Mon Sep 17 00:00:00 2001 From: takatost Date: Tue, 19 Mar 2024 17:49:17 +0800 Subject: [PATCH 01/21] fix tool image render --- .../advanced_chat/generate_task_pipeline.py | 45 +++++++++++-------- .../task_pipeline/workflow_cycle_manage.py | 2 + 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index 639d1c98ec..b4ed123475 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -529,31 +529,40 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc text = '' if isinstance(value, str | int | float): text = str(value) - elif isinstance(value, dict): - # other types - text = json.dumps(value, ensure_ascii=False) elif isinstance(value, FileVar): # convert file to markdown text = value.to_markdown() + elif isinstance(value, dict): + # handle files + file_vars = self._fetch_files_from_variable_value(value) + if file_vars: + file_var = file_vars[0] + try: + file_var_obj = FileVar(**file_var) + + # convert file to markdown + text = file_var_obj.to_markdown() + except Exception as e: + logger.error(f'Error creating file var: {e}') + + if not text: + # other types + text = json.dumps(value, ensure_ascii=False) elif isinstance(value, list): - for item in value: - if isinstance(item, FileVar): - text += item.to_markdown() + ' ' + # handle files + file_vars = self._fetch_files_from_variable_value(value) + for file_var in file_vars: + try: + file_var_obj = FileVar(**file_var) + except Exception as e: + logger.error(f'Error creating file var: {e}') + continue + + # convert file to markdown + text = file_var_obj.to_markdown() + ' ' text = text.strip() - # # handle files - # file_vars = self._fetch_files_from_variable_value(value) - # for file_var in file_vars: - # try: - # file_var_obj = FileVar(**file_var) - # except Exception as e: - # logger.error(f'Error creating file var: {e}') - # continue - # - # # convert file to markdown - # text = file_var_obj.to_markdown() - if not text and value: # other types text = json.dumps(value, ensure_ascii=False) diff --git a/api/core/app/task_pipeline/workflow_cycle_manage.py b/api/core/app/task_pipeline/workflow_cycle_manage.py index 89da11b76c..fc8afa8c70 100644 --- a/api/core/app/task_pipeline/workflow_cycle_manage.py +++ b/api/core/app/task_pipeline/workflow_cycle_manage.py @@ -533,5 +533,7 @@ class WorkflowCycleManage: if isinstance(value, dict): if '__variant' in value and value['__variant'] == FileVar.__name__: return value + elif isinstance(value, FileVar): + return value.to_dict() return None From 17b7426cc6c19c6d4519444b7112b44c9dac2bfa Mon Sep 17 00:00:00 2001 From: takatost Date: Tue, 19 Mar 2024 17:58:33 +0800 Subject: [PATCH 02/21] fix external_data_tools bug --- api/core/app/app_config/easy_ui_based_app/variables/manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/core/app/app_config/easy_ui_based_app/variables/manager.py b/api/core/app/app_config/easy_ui_based_app/variables/manager.py index 1237da502b..3eb006b46e 100644 --- a/api/core/app/app_config/easy_ui_based_app/variables/manager.py +++ b/api/core/app/app_config/easy_ui_based_app/variables/manager.py @@ -34,6 +34,9 @@ class BasicVariablesConfigManager: typ = list(variable.keys())[0] if typ == 'external_data_tool': val = variable[typ] + if 'config' not in val: + continue + external_data_variables.append( ExternalDataVariableEntity( variable=val['variable'], From 55d2417906339092293a94f730ab8bab45c02bb5 Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Tue, 19 Mar 2024 18:12:50 +0800 Subject: [PATCH 03/21] fix: invalid http header --- api/core/workflow/nodes/http_request/http_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index 6474a6259e..0b11454d3d 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -213,7 +213,7 @@ class HttpExecutor: else: raise ValueError(f'Invalid headers {kv}') - self.headers[k] = v + self.headers[k.strip()] = v.strip() # extract all template in body if node_data.body: From 2f16b3600ce46ebfb027c93e2648e48b209d7806 Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Tue, 19 Mar 2024 18:13:30 +0800 Subject: [PATCH 04/21] fix: avoid space in http key --- api/core/workflow/nodes/http_request/http_executor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index 0b11454d3d..daa36bd380 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -187,7 +187,7 @@ class HttpExecutor: else: raise ValueError(f'Invalid params {kv}') - self.params[k] = v + self.params[k.strip()] = v # extract all template in headers header_template = re.findall(r'{{(.*?)}}', node_data.headers) or [] @@ -239,9 +239,9 @@ class HttpExecutor: continue kv = kv.split(':') if len(kv) == 2: - body[kv[0]] = kv[1] + body[kv[0].strip()] = kv[1] elif len(kv) == 1: - body[kv[0]] = '' + body[kv[0].strip()] = '' else: raise ValueError(f'Invalid body {kv}') From b17e30b1c23741db82d4d41452de20e7a56614ac Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Tue, 19 Mar 2024 18:30:13 +0800 Subject: [PATCH 05/21] fix: form-data --- api/core/workflow/nodes/http_request/http_executor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index daa36bd380..fbbd9a1b55 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -361,12 +361,12 @@ class HttpExecutor: # if files, use multipart/form-data with boundary if self.files: boundary = self.boundary + raw_request += f'--{boundary}' for k, v in self.files.items(): - raw_request += f'Content-Disposition: form-data; name="{k}"; filename="{v[0]}"\n' - raw_request += f'Content-Type: {v[1]}\n\n' - raw_request += v[1] + '\n' - raw_request += f'{boundary}\n' - raw_request += '--\n' + raw_request += f'\nContent-Disposition: form-data; name="{k}"\n\n' + raw_request += f'{v[1]}\n' + raw_request += f'--{boundary}' + raw_request += '--' else: raw_request += self.body or '' From c61f51dc5d033c2e700fad5f0a7390aa4482c688 Mon Sep 17 00:00:00 2001 From: Su Yang Date: Mon, 18 Mar 2024 18:16:36 +0800 Subject: [PATCH 06/21] feat: AWS Bedrock Claude3 (#2864) Co-authored-by: crazywoola <427733928@qq.com> Co-authored-by: Chenhe Gu --- .../bedrock/llm/_position.yaml | 2 + .../llm/anthropic.claude-3-haiku-v1.yaml | 57 ++++ .../llm/anthropic.claude-3-sonnet-v1.yaml | 56 ++++ .../model_providers/bedrock/llm/llm.py | 294 +++++++++++++++++- api/requirements.txt | 2 +- 5 files changed, 407 insertions(+), 4 deletions(-) create mode 100644 api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-haiku-v1.yaml create mode 100644 api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-sonnet-v1.yaml diff --git a/api/core/model_runtime/model_providers/bedrock/llm/_position.yaml b/api/core/model_runtime/model_providers/bedrock/llm/_position.yaml index c4be732f2e..a4cfbd171e 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/_position.yaml +++ b/api/core/model_runtime/model_providers/bedrock/llm/_position.yaml @@ -4,6 +4,8 @@ - anthropic.claude-v1 - anthropic.claude-v2 - anthropic.claude-v2:1 +- anthropic.claude-3-sonnet-v1:0 +- anthropic.claude-3-haiku-v1:0 - cohere.command-light-text-v14 - cohere.command-text-v14 - meta.llama2-13b-chat-v1 diff --git a/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-haiku-v1.yaml b/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-haiku-v1.yaml new file mode 100644 index 0000000000..73fe5567fc --- /dev/null +++ b/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-haiku-v1.yaml @@ -0,0 +1,57 @@ +model: anthropic.claude-3-haiku-20240307-v1:0 +label: + en_US: Claude 3 Haiku +model_type: llm +features: + - agent-thought + - vision +model_properties: + mode: chat + context_size: 200000 +# docs: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html +parameter_rules: + - name: max_tokens + use_template: max_tokens + required: true + type: int + default: 4096 + min: 1 + max: 4096 + help: + zh_Hans: 停止前生成的最大令牌数。请注意，Anthropic Claude 模型可能会在达到 max_tokens 的值之前停止生成令牌。不同的 Anthropic Claude 模型对此参数具有不同的最大值。 + en_US: The maximum number of tokens to generate before stopping. Note that Anthropic Claude models might stop generating tokens before reaching the value of max_tokens. Different Anthropic Claude models have different maximum values for this parameter. + # docs: https://docs.anthropic.com/claude/docs/system-prompts + - name: temperature + use_template: temperature + required: false + type: float + default: 1 + min: 0.0 + max: 1.0 + help: + zh_Hans: 生成内容的随机性。 + en_US: The amount of randomness injected into the response. + - name: top_p + required: false + type: float + default: 0.999 + min: 0.000 + max: 1.000 + help: + zh_Hans: 在核采样中，Anthropic Claude 按概率递减顺序计算每个后续标记的所有选项的累积分布，并在达到 top_p 指定的特定概率时将其切断。您应该更改温度或top_p，但不能同时更改两者。 + en_US: In nucleus sampling, Anthropic Claude computes the cumulative distribution over all the options for each subsequent token in decreasing probability order and cuts it off once it reaches a particular probability specified by top_p. You should alter either temperature or top_p, but not both. + - name: top_k + required: false + type: int + default: 0 + min: 0 + # tip docs from aws has error, max value is 500 + max: 500 + help: + zh_Hans: 对于每个后续标记，仅从前 K 个选项中进行采样。使用 top_k 删除长尾低概率响应。 + en_US: Only sample from the top K options for each subsequent token. Use top_k to remove long tail low probability responses. +pricing: + input: '0.003' + output: '0.015' + unit: '0.001' + currency: USD diff --git a/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-sonnet-v1.yaml b/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-sonnet-v1.yaml new file mode 100644 index 0000000000..cb11df0b60 --- /dev/null +++ b/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-sonnet-v1.yaml @@ -0,0 +1,56 @@ +model: anthropic.claude-3-sonnet-20240229-v1:0 +label: + en_US: Claude 3 Sonnet +model_type: llm +features: + - agent-thought + - vision +model_properties: + mode: chat + context_size: 200000 +# docs: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html +parameter_rules: + - name: max_tokens + use_template: max_tokens + required: true + type: int + default: 4096 + min: 1 + max: 4096 + help: + zh_Hans: 停止前生成的最大令牌数。请注意，Anthropic Claude 模型可能会在达到 max_tokens 的值之前停止生成令牌。不同的 Anthropic Claude 模型对此参数具有不同的最大值。 + en_US: The maximum number of tokens to generate before stopping. Note that Anthropic Claude models might stop generating tokens before reaching the value of max_tokens. Different Anthropic Claude models have different maximum values for this parameter. + - name: temperature + use_template: temperature + required: false + type: float + default: 1 + min: 0.0 + max: 1.0 + help: + zh_Hans: 生成内容的随机性。 + en_US: The amount of randomness injected into the response. + - name: top_p + required: false + type: float + default: 0.999 + min: 0.000 + max: 1.000 + help: + zh_Hans: 在核采样中，Anthropic Claude 按概率递减顺序计算每个后续标记的所有选项的累积分布，并在达到 top_p 指定的特定概率时将其切断。您应该更改温度或top_p，但不能同时更改两者。 + en_US: In nucleus sampling, Anthropic Claude computes the cumulative distribution over all the options for each subsequent token in decreasing probability order and cuts it off once it reaches a particular probability specified by top_p. You should alter either temperature or top_p, but not both. + - name: top_k + required: false + type: int + default: 0 + min: 0 + # tip docs from aws has error, max value is 500 + max: 500 + help: + zh_Hans: 对于每个后续标记，仅从前 K 个选项中进行采样。使用 top_k 删除长尾低概率响应。 + en_US: Only sample from the top K options for each subsequent token. Use top_k to remove long tail low probability responses. +pricing: + input: '0.00025' + output: '0.00125' + unit: '0.001' + currency: USD diff --git a/api/core/model_runtime/model_providers/bedrock/llm/llm.py b/api/core/model_runtime/model_providers/bedrock/llm/llm.py index c6aaa24ade..5745721ae8 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/llm.py +++ b/api/core/model_runtime/model_providers/bedrock/llm/llm.py @@ -1,9 +1,22 @@ +import base64 import json import logging +import mimetypes +import time from collections.abc import Generator -from typing import Optional, Union +from typing import Optional, Union, cast import boto3 +import requests +from anthropic import AnthropicBedrock, Stream +from anthropic.types import ( + ContentBlockDeltaEvent, + Message, + MessageDeltaEvent, + MessageStartEvent, + MessageStopEvent, + MessageStreamEvent, +) from botocore.config import Config from botocore.exceptions import ( ClientError, @@ -13,14 +26,18 @@ from botocore.exceptions import ( UnknownServiceError, ) -from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta +from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage from core.model_runtime.entities.message_entities import ( AssistantPromptMessage, + ImagePromptMessageContent, PromptMessage, + PromptMessageContentType, PromptMessageTool, SystemPromptMessage, + TextPromptMessageContent, UserPromptMessage, ) +from core.model_runtime.entities.model_entities import PriceType from core.model_runtime.errors.invoke import ( InvokeAuthorizationError, InvokeBadRequestError, @@ -54,9 +71,268 @@ class BedrockLargeLanguageModel(LargeLanguageModel): :param user: unique user id :return: full response or stream response chunk generator result """ + + # invoke claude 3 models via anthropic official SDK + if "anthropic.claude-3" in model: + return self._invoke_claude3(model, credentials, prompt_messages, model_parameters, stop, stream) # invoke model return self._generate(model, credentials, prompt_messages, model_parameters, stop, stream, user) + def _invoke_claude3(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], model_parameters: dict, + stop: Optional[list[str]] = None, stream: bool = True) -> Union[LLMResult, Generator]: + """ + Invoke Claude3 large language model + + :param model: model name + :param credentials: model credentials + :param prompt_messages: prompt messages + :param model_parameters: model parameters + :param stop: stop words + :param stream: is stream response + :return: full response or stream response chunk generator result + """ + # use Anthropic official SDK references + # - https://docs.anthropic.com/claude/reference/claude-on-amazon-bedrock + # - https://github.com/anthropics/anthropic-sdk-python + client = AnthropicBedrock( + aws_access_key=credentials["aws_access_key_id"], + aws_secret_key=credentials["aws_secret_access_key"], + aws_region=credentials["aws_region"], + ) + + system, prompt_message_dicts = self._convert_claude3_prompt_messages(prompt_messages) + + response = client.messages.create( + model=model, + messages=prompt_message_dicts, + stop_sequences=stop if stop else [], + system=system, + stream=stream, + **model_parameters, + ) + + if stream is False: + return self._handle_claude3_response(model, credentials, response, prompt_messages) + else: + return self._handle_claude3_stream_response(model, credentials, response, prompt_messages) + + def _handle_claude3_response(self, model: str, credentials: dict, response: Message, + prompt_messages: list[PromptMessage]) -> LLMResult: + """ + Handle llm chat response + + :param model: model name + :param credentials: credentials + :param response: response + :param prompt_messages: prompt messages + :return: full response chunk generator result + """ + + # transform assistant message to prompt message + assistant_prompt_message = AssistantPromptMessage( + content=response.content[0].text + ) + + # calculate num tokens + if response.usage: + # transform usage + prompt_tokens = response.usage.input_tokens + completion_tokens = response.usage.output_tokens + else: + # calculate num tokens + prompt_tokens = self.get_num_tokens(model, credentials, prompt_messages) + completion_tokens = self.get_num_tokens(model, credentials, [assistant_prompt_message]) + + # transform usage + usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens) + + # transform response + response = LLMResult( + model=response.model, + prompt_messages=prompt_messages, + message=assistant_prompt_message, + usage=usage + ) + + return response + + def _handle_claude3_stream_response(self, model: str, credentials: dict, response: Stream[MessageStreamEvent], + prompt_messages: list[PromptMessage], ) -> Generator: + """ + Handle llm chat stream response + + :param model: model name + :param credentials: credentials + :param response: response + :param prompt_messages: prompt messages + :return: full response or stream response chunk generator result + """ + + try: + full_assistant_content = '' + return_model = None + input_tokens = 0 + output_tokens = 0 + finish_reason = None + index = 0 + + for chunk in response: + if isinstance(chunk, MessageStartEvent): + return_model = chunk.message.model + input_tokens = chunk.message.usage.input_tokens + elif isinstance(chunk, MessageDeltaEvent): + output_tokens = chunk.usage.output_tokens + finish_reason = chunk.delta.stop_reason + elif isinstance(chunk, MessageStopEvent): + usage = self._calc_response_usage(model, credentials, input_tokens, output_tokens) + yield LLMResultChunk( + model=return_model, + prompt_messages=prompt_messages, + delta=LLMResultChunkDelta( + index=index + 1, + message=AssistantPromptMessage( + content='' + ), + finish_reason=finish_reason, + usage=usage + ) + ) + elif isinstance(chunk, ContentBlockDeltaEvent): + chunk_text = chunk.delta.text if chunk.delta.text else '' + full_assistant_content += chunk_text + assistant_prompt_message = AssistantPromptMessage( + content=chunk_text if chunk_text else '', + ) + index = chunk.index + yield LLMResultChunk( + model=model, + prompt_messages=prompt_messages, + delta=LLMResultChunkDelta( + index=index, + message=assistant_prompt_message, + ) + ) + except Exception as ex: + raise InvokeError(str(ex)) + + def _calc_claude3_response_usage(self, model: str, credentials: dict, prompt_tokens: int, completion_tokens: int) -> LLMUsage: + """ + Calculate response usage + + :param model: model name + :param credentials: model credentials + :param prompt_tokens: prompt tokens + :param completion_tokens: completion tokens + :return: usage + """ + # get prompt price info + prompt_price_info = self.get_price( + model=model, + credentials=credentials, + price_type=PriceType.INPUT, + tokens=prompt_tokens, + ) + + # get completion price info + completion_price_info = self.get_price( + model=model, + credentials=credentials, + price_type=PriceType.OUTPUT, + tokens=completion_tokens + ) + + # transform usage + usage = LLMUsage( + prompt_tokens=prompt_tokens, + prompt_unit_price=prompt_price_info.unit_price, + prompt_price_unit=prompt_price_info.unit, + prompt_price=prompt_price_info.total_amount, + completion_tokens=completion_tokens, + completion_unit_price=completion_price_info.unit_price, + completion_price_unit=completion_price_info.unit, + completion_price=completion_price_info.total_amount, + total_tokens=prompt_tokens + completion_tokens, + total_price=prompt_price_info.total_amount + completion_price_info.total_amount, + currency=prompt_price_info.currency, + latency=time.perf_counter() - self.started_at + ) + + return usage + + def _convert_claude3_prompt_messages(self, prompt_messages: list[PromptMessage]) -> tuple[str, list[dict]]: + """ + Convert prompt messages to dict list and system + """ + system = "" + prompt_message_dicts = [] + + for message in prompt_messages: + if isinstance(message, SystemPromptMessage): + system += message.content + ("\n" if not system else "") + else: + prompt_message_dicts.append(self._convert_claude3_prompt_message_to_dict(message)) + + return system, prompt_message_dicts + + def _convert_claude3_prompt_message_to_dict(self, message: PromptMessage) -> dict: + """ + Convert PromptMessage to dict + """ + if isinstance(message, UserPromptMessage): + message = cast(UserPromptMessage, message) + if isinstance(message.content, str): + message_dict = {"role": "user", "content": message.content} + else: + sub_messages = [] + for message_content in message.content: + if message_content.type == PromptMessageContentType.TEXT: + message_content = cast(TextPromptMessageContent, message_content) + sub_message_dict = { + "type": "text", + "text": message_content.data + } + sub_messages.append(sub_message_dict) + elif message_content.type == PromptMessageContentType.IMAGE: + message_content = cast(ImagePromptMessageContent, message_content) + if not message_content.data.startswith("data:"): + # fetch image data from url + try: + image_content = requests.get(message_content.data).content + mime_type, _ = mimetypes.guess_type(message_content.data) + base64_data = base64.b64encode(image_content).decode('utf-8') + except Exception as ex: + raise ValueError(f"Failed to fetch image data from url {message_content.data}, {ex}") + else: + data_split = message_content.data.split(";base64,") + mime_type = data_split[0].replace("data:", "") + base64_data = data_split[1] + + if mime_type not in ["image/jpeg", "image/png", "image/gif", "image/webp"]: + raise ValueError(f"Unsupported image type {mime_type}, " + f"only support image/jpeg, image/png, image/gif, and image/webp") + + sub_message_dict = { + "type": "image", + "source": { + "type": "base64", + "media_type": mime_type, + "data": base64_data + } + } + sub_messages.append(sub_message_dict) + + message_dict = {"role": "user", "content": sub_messages} + elif isinstance(message, AssistantPromptMessage): + message = cast(AssistantPromptMessage, message) + message_dict = {"role": "assistant", "content": message.content} + elif isinstance(message, SystemPromptMessage): + message = cast(SystemPromptMessage, message) + message_dict = {"role": "system", "content": message.content} + else: + raise ValueError(f"Got unknown type {message}") + + return message_dict + def get_num_tokens(self, model: str, credentials: dict, messages: list[PromptMessage] | str, tools: Optional[list[PromptMessageTool]] = None) -> int: """ @@ -101,7 +377,19 @@ class BedrockLargeLanguageModel(LargeLanguageModel): :param credentials: model credentials :return: """ - + + if "anthropic.claude-3" in model: + try: + self._invoke_claude3(model=model, + credentials=credentials, + prompt_messages=[{"role": "user", "content": "ping"}], + model_parameters={}, + stop=None, + stream=False) + + except Exception as ex: + raise CredentialsValidateFailedError(str(ex)) + try: ping_message = UserPromptMessage(content="ping") self._generate(model=model, diff --git a/api/requirements.txt b/api/requirements.txt index 7edd95a893..b8714291a9 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -36,7 +36,7 @@ python-docx~=1.1.0 pypdfium2==4.16.0 resend~=0.7.0 pyjwt~=2.8.0 -anthropic~=0.17.0 +anthropic~=0.20.0 newspaper3k==0.2.8 google-api-python-client==2.90.0 wikipedia==1.4.0 From 758b8bf812aeccb8cc50a9a23ae6aa6713aaf2f5 Mon Sep 17 00:00:00 2001 From: Su Yang Date: Tue, 19 Mar 2024 00:57:19 +0800 Subject: [PATCH 07/21] i18n: update bedrock label (#2879) --- .../model_runtime/model_providers/bedrock/bedrock.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/core/model_runtime/model_providers/bedrock/bedrock.yaml b/api/core/model_runtime/model_providers/bedrock/bedrock.yaml index 05cd402d4e..e1923f8f8a 100644 --- a/api/core/model_runtime/model_providers/bedrock/bedrock.yaml +++ b/api/core/model_runtime/model_providers/bedrock/bedrock.yaml @@ -48,23 +48,23 @@ provider_credential_schema: - value: us-east-1 label: en_US: US East (N. Virginia) - zh_Hans: US East (N. Virginia) + zh_Hans: 美国东部 (弗吉尼亚北部) - value: us-west-2 label: en_US: US West (Oregon) - zh_Hans: US West (Oregon) + zh_Hans: 美国西部 (俄勒冈州) - value: ap-southeast-1 label: en_US: Asia Pacific (Singapore) - zh_Hans: Asia Pacific (Singapore) + zh_Hans: 亚太地区 (新加坡) - value: ap-northeast-1 label: en_US: Asia Pacific (Tokyo) - zh_Hans: Asia Pacific (Tokyo) + zh_Hans: 亚太地区 (东京) - value: eu-central-1 label: en_US: Europe (Frankfurt) - zh_Hans: Europe (Frankfurt) + zh_Hans: 欧洲 (法兰克福) - value: us-gov-west-1 label: en_US: AWS GovCloud (US-West) From 779f77ccd6ab64dd90e95e98936c9742229d5552 Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:53:21 +0800 Subject: [PATCH 08/21] feat: add icons for 01.ai (#2883) --- .../model_providers/yi/_assets/icon_l_en.svg | 32 +++++++------------ .../model_providers/yi/_assets/icon_l_zh.svg | 20 ------------ .../model_providers/yi/_assets/icon_s_en.svg | 15 +++++---- .../model_runtime/model_providers/yi/yi.yaml | 2 +- 4 files changed, 21 insertions(+), 48 deletions(-) delete mode 100644 api/core/model_runtime/model_providers/yi/_assets/icon_l_zh.svg diff --git a/api/core/model_runtime/model_providers/yi/_assets/icon_l_en.svg b/api/core/model_runtime/model_providers/yi/_assets/icon_l_en.svg index 0efce4e85b..9ce3baddaa 100644 --- a/api/core/model_runtime/model_providers/yi/_assets/icon_l_en.svg +++ b/api/core/model_runtime/model_providers/yi/_assets/icon_l_en.svg @@ -1,20 +1,12 @@ - - - - - - - - - - - - - - - - - - 01.AI - - + + + + + + + + + + + + \ No newline at end of file diff --git a/api/core/model_runtime/model_providers/yi/_assets/icon_l_zh.svg b/api/core/model_runtime/model_providers/yi/_assets/icon_l_zh.svg deleted file mode 100644 index 951842da55..0000000000 --- a/api/core/model_runtime/model_providers/yi/_assets/icon_l_zh.svg +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - 零一万物 - - diff --git a/api/core/model_runtime/model_providers/yi/_assets/icon_s_en.svg b/api/core/model_runtime/model_providers/yi/_assets/icon_s_en.svg index a813274466..eb0395a21c 100644 --- a/api/core/model_runtime/model_providers/yi/_assets/icon_s_en.svg +++ b/api/core/model_runtime/model_providers/yi/_assets/icon_s_en.svg @@ -1,7 +1,8 @@ - - - - - - - \ No newline at end of file + + + + + + + + \ No newline at end of file diff --git a/api/core/model_runtime/model_providers/yi/yi.yaml b/api/core/model_runtime/model_providers/yi/yi.yaml index 368c715456..a8c0d857b6 100644 --- a/api/core/model_runtime/model_providers/yi/yi.yaml +++ b/api/core/model_runtime/model_providers/yi/yi.yaml @@ -9,7 +9,7 @@ icon_small: en_US: icon_s_en.svg icon_large: en_US: icon_l_en.svg -background: "#EFFDFD" +background: "#E9F1EC" help: title: en_US: Get your API Key from 01.ai From faf936416f1995e1900181d19550940b71618c76 Mon Sep 17 00:00:00 2001 From: Su Yang Date: Tue, 19 Mar 2024 13:56:22 +0800 Subject: [PATCH 09/21] fix: Fix the problem of system not working (#2884) --- .../model_providers/bedrock/llm/llm.py | 47 ++++++++++++++----- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/api/core/model_runtime/model_providers/bedrock/llm/llm.py b/api/core/model_runtime/model_providers/bedrock/llm/llm.py index 5745721ae8..b274cec35f 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/llm.py +++ b/api/core/model_runtime/model_providers/bedrock/llm/llm.py @@ -74,12 +74,12 @@ class BedrockLargeLanguageModel(LargeLanguageModel): # invoke claude 3 models via anthropic official SDK if "anthropic.claude-3" in model: - return self._invoke_claude3(model, credentials, prompt_messages, model_parameters, stop, stream) + return self._invoke_claude3(model, credentials, prompt_messages, model_parameters, stop, stream, user) # invoke model return self._generate(model, credentials, prompt_messages, model_parameters, stop, stream, user) def _invoke_claude3(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], model_parameters: dict, - stop: Optional[list[str]] = None, stream: bool = True) -> Union[LLMResult, Generator]: + stop: Optional[list[str]] = None, stream: bool = True, user: Optional[str] = None) -> Union[LLMResult, Generator]: """ Invoke Claude3 large language model @@ -100,22 +100,38 @@ class BedrockLargeLanguageModel(LargeLanguageModel): aws_region=credentials["aws_region"], ) + extra_model_kwargs = {} + if stop: + extra_model_kwargs['stop_sequences'] = stop + + # Notice: If you request the current version of the SDK to the bedrock server, + # you will get the following error message and you need to wait for the service or SDK to be updated. + # Response: Error code: 400 + # {'message': 'Malformed input request: #: subject must not be valid against schema + # {"required":["messages"]}#: extraneous key [metadata] is not permitted, please reformat your input and try again.'} + # TODO: Open in the future when the interface is properly supported + # if user: + # ref: https://github.com/anthropics/anthropic-sdk-python/blob/e84645b07ca5267066700a104b4d8d6a8da1383d/src/anthropic/resources/messages.py#L465 + # extra_model_kwargs['metadata'] = message_create_params.Metadata(user_id=user) + system, prompt_message_dicts = self._convert_claude3_prompt_messages(prompt_messages) + if system: + extra_model_kwargs['system'] = system + response = client.messages.create( model=model, messages=prompt_message_dicts, - stop_sequences=stop if stop else [], - system=system, stream=stream, **model_parameters, + **extra_model_kwargs ) - if stream is False: - return self._handle_claude3_response(model, credentials, response, prompt_messages) - else: + if stream: return self._handle_claude3_stream_response(model, credentials, response, prompt_messages) + return self._handle_claude3_response(model, credentials, response, prompt_messages) + def _handle_claude3_response(self, model: str, credentials: dict, response: Message, prompt_messages: list[PromptMessage]) -> LLMResult: """ @@ -263,13 +279,22 @@ class BedrockLargeLanguageModel(LargeLanguageModel): """ Convert prompt messages to dict list and system """ - system = "" - prompt_message_dicts = [] + system = "" + first_loop = True for message in prompt_messages: if isinstance(message, SystemPromptMessage): - system += message.content + ("\n" if not system else "") - else: + message.content=message.content.strip() + if first_loop: + system=message.content + first_loop=False + else: + system+="\n" + system+=message.content + + prompt_message_dicts = [] + for message in prompt_messages: + if not isinstance(message, SystemPromptMessage): prompt_message_dicts.append(self._convert_claude3_prompt_message_to_dict(message)) return system, prompt_message_dicts From 10237c99e4f76f309390443a811b0caf7da0dae9 Mon Sep 17 00:00:00 2001 From: Su Yang Date: Tue, 19 Mar 2024 15:50:02 +0800 Subject: [PATCH 10/21] fix: anthropic system prompt not working (#2885) --- .../model_providers/anthropic/llm/llm.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/api/core/model_runtime/model_providers/anthropic/llm/llm.py b/api/core/model_runtime/model_providers/anthropic/llm/llm.py index 1e88bd87d9..724a0401b7 100644 --- a/api/core/model_runtime/model_providers/anthropic/llm/llm.py +++ b/api/core/model_runtime/model_providers/anthropic/llm/llm.py @@ -345,16 +345,15 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): first_loop = True for message in prompt_messages: if isinstance(message, SystemPromptMessage): - message.content = message.content.strip() + message.content=message.content.strip() if first_loop: - system = message.content - first_loop = False + system=message.content + first_loop=False else: - system += "\n" - system += message.content + system+="\n" + system+=message.content prompt_message_dicts = [] - for message in prompt_messages: if not isinstance(message, SystemPromptMessage): prompt_message_dicts.append(self._convert_prompt_message_to_dict(message)) From 3f13c47b9b9bd42f3f2b78b2c2cb7200041ff89b Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Tue, 19 Mar 2024 16:31:46 +0800 Subject: [PATCH 11/21] Bump tiktoken to 0.6.0 to support text-embedding-3-* in encoding_for_model (#2891) --- api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index b8714291a9..886d7e42d0 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -12,7 +12,7 @@ gunicorn~=21.2.0 gevent~=23.9.1 langchain==0.0.250 openai~=1.13.3 -tiktoken~=0.5.2 +tiktoken~=0.6.0 psycopg2-binary~=2.9.6 pycryptodome==3.19.1 python-dotenv==1.0.0 From 4e24e116aabaf91209b1603f844e56fc0f053a8c Mon Sep 17 00:00:00 2001 From: Su Yang Date: Tue, 19 Mar 2024 16:32:06 +0800 Subject: [PATCH 12/21] chore: use API Key instead of APIKey (#2888) --- api/core/model_runtime/model_providers/tongyi/tongyi.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/core/model_runtime/model_providers/tongyi/tongyi.yaml b/api/core/model_runtime/model_providers/tongyi/tongyi.yaml index 441d833f70..b251391e34 100644 --- a/api/core/model_runtime/model_providers/tongyi/tongyi.yaml +++ b/api/core/model_runtime/model_providers/tongyi/tongyi.yaml @@ -24,9 +24,9 @@ provider_credential_schema: credential_form_schemas: - variable: dashscope_api_key label: - en_US: APIKey + en_US: API Key type: secret-input required: true placeholder: - zh_Hans: 在此输入您的 APIKey - en_US: Enter your APIKey + zh_Hans: 在此输入您的 API Key + en_US: Enter your API Key From 66538d8cbdda97fb5afcbf34e0f2899c32484e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=86=E8=90=8C=E9=97=B7=E6=B2=B9=E7=93=B6?= <253605712@qq.com> Date: Tue, 19 Mar 2024 16:32:26 +0800 Subject: [PATCH 13/21] feat:support azure openai llm 0125 version (#2889) --- .../model_providers/azure_openai/_constant.py | 134 ++++++++++++++++++ .../azure_openai/azure_openai.yaml | 12 ++ 2 files changed, 146 insertions(+) diff --git a/api/core/model_runtime/model_providers/azure_openai/_constant.py b/api/core/model_runtime/model_providers/azure_openai/_constant.py index 4aa767fa1d..e81a120fa0 100644 --- a/api/core/model_runtime/model_providers/azure_openai/_constant.py +++ b/api/core/model_runtime/model_providers/azure_openai/_constant.py @@ -123,6 +123,65 @@ LLM_BASE_MODELS = [ ) ) ), + AzureBaseModel( + base_model_name='gpt-35-turbo-0125', + entity=AIModelEntity( + model='fake-deployment-name', + label=I18nObject( + en_US='fake-deployment-name-label', + ), + model_type=ModelType.LLM, + features=[ + ModelFeature.AGENT_THOUGHT, + ModelFeature.MULTI_TOOL_CALL, + ModelFeature.STREAM_TOOL_CALL, + ], + fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, + model_properties={ + ModelPropertyKey.MODE: LLMMode.CHAT.value, + ModelPropertyKey.CONTEXT_SIZE: 16385, + }, + parameter_rules=[ + ParameterRule( + name='temperature', + **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TEMPERATURE], + ), + ParameterRule( + name='top_p', + **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TOP_P], + ), + ParameterRule( + name='presence_penalty', + **PARAMETER_RULE_TEMPLATE[DefaultParameterName.PRESENCE_PENALTY], + ), + ParameterRule( + name='frequency_penalty', + **PARAMETER_RULE_TEMPLATE[DefaultParameterName.FREQUENCY_PENALTY], + ), + _get_max_tokens(default=512, min_val=1, max_val=4096), + ParameterRule( + name='response_format', + label=I18nObject( + zh_Hans='回复格式', + en_US='response_format' + ), + type='string', + help=I18nObject( + zh_Hans='指定模型必须输出的格式', + en_US='specifying the format that the model must output' + ), + required=False, + options=['text', 'json_object'] + ), + ], + pricing=PriceConfig( + input=0.0005, + output=0.0015, + unit=0.001, + currency='USD', + ) + ) + ), AzureBaseModel( base_model_name='gpt-4', entity=AIModelEntity( @@ -273,6 +332,81 @@ LLM_BASE_MODELS = [ ) ) ), + AzureBaseModel( + base_model_name='gpt-4-0125-preview', + entity=AIModelEntity( + model='fake-deployment-name', + label=I18nObject( + en_US='fake-deployment-name-label', + ), + model_type=ModelType.LLM, + features=[ + ModelFeature.AGENT_THOUGHT, + ModelFeature.MULTI_TOOL_CALL, + ModelFeature.STREAM_TOOL_CALL, + ], + fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, + model_properties={ + ModelPropertyKey.MODE: LLMMode.CHAT.value, + ModelPropertyKey.CONTEXT_SIZE: 128000, + }, + parameter_rules=[ + ParameterRule( + name='temperature', + **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TEMPERATURE], + ), + ParameterRule( + name='top_p', + **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TOP_P], + ), + ParameterRule( + name='presence_penalty', + **PARAMETER_RULE_TEMPLATE[DefaultParameterName.PRESENCE_PENALTY], + ), + ParameterRule( + name='frequency_penalty', + **PARAMETER_RULE_TEMPLATE[DefaultParameterName.FREQUENCY_PENALTY], + ), + _get_max_tokens(default=512, min_val=1, max_val=4096), + ParameterRule( + name='seed', + label=I18nObject( + zh_Hans='种子', + en_US='Seed' + ), + type='int', + help=I18nObject( + zh_Hans='如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。', + en_US='If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.' + ), + required=False, + precision=2, + min=0, + max=1, + ), + ParameterRule( + name='response_format', + label=I18nObject( + zh_Hans='回复格式', + en_US='response_format' + ), + type='string', + help=I18nObject( + zh_Hans='指定模型必须输出的格式', + en_US='specifying the format that the model must output' + ), + required=False, + options=['text', 'json_object'] + ), + ], + pricing=PriceConfig( + input=0.01, + output=0.03, + unit=0.001, + currency='USD', + ) + ) + ), AzureBaseModel( base_model_name='gpt-4-1106-preview', entity=AIModelEntity( diff --git a/api/core/model_runtime/model_providers/azure_openai/azure_openai.yaml b/api/core/model_runtime/model_providers/azure_openai/azure_openai.yaml index 224f2a08a1..792d051d94 100644 --- a/api/core/model_runtime/model_providers/azure_openai/azure_openai.yaml +++ b/api/core/model_runtime/model_providers/azure_openai/azure_openai.yaml @@ -75,6 +75,12 @@ model_credential_schema: show_on: - variable: __model_type value: llm + - label: + en_US: gpt-35-turbo-0125 + value: gpt-35-turbo-0125 + show_on: + - variable: __model_type + value: llm - label: en_US: gpt-35-turbo-16k value: gpt-35-turbo-16k @@ -93,6 +99,12 @@ model_credential_schema: show_on: - variable: __model_type value: llm + - label: + en_US: gpt-4-0125-preview + value: gpt-4-0125-preview + show_on: + - variable: __model_type + value: llm - label: en_US: gpt-4-1106-preview value: gpt-4-1106-preview From b84d4bdb85748a75efddcfa0a05b49527210143d Mon Sep 17 00:00:00 2001 From: Su Yang Date: Tue, 19 Mar 2024 16:32:42 +0800 Subject: [PATCH 14/21] chore: Update TongYi models prices (#2890) --- .../model_providers/tongyi/llm/qwen-max-1201.yaml | 5 +++++ .../model_providers/tongyi/llm/qwen-max-longcontext.yaml | 5 +++++ .../model_runtime/model_providers/tongyi/llm/qwen-max.yaml | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-1201.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-1201.yaml index 3461863e67..e0ba6fe4a8 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-1201.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-1201.yaml @@ -59,3 +59,8 @@ parameter_rules: required: false - name: response_format use_template: response_format +pricing: + input: '0.12' + output: '0.12' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-longcontext.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-longcontext.yaml index 9089c5904a..e2a291cc59 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-longcontext.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-longcontext.yaml @@ -59,3 +59,8 @@ parameter_rules: required: false - name: response_format use_template: response_format +pricing: + input: '0.12' + output: '0.12' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max.yaml index eb1e8ac09b..8260b5081d 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max.yaml @@ -59,3 +59,8 @@ parameter_rules: required: false - name: response_format use_template: response_format +pricing: + input: '0.12' + output: '0.12' + unit: '0.001' + currency: RMB From e7895cdc537e8ee1d4048571883c165d9c38ae2b Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Tue, 19 Mar 2024 17:24:57 +0800 Subject: [PATCH 15/21] chore: update pr template (#2893) --- .github/pull_request_template.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 21ec0d5fa4..965831ebe3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -12,6 +12,8 @@ Please delete options that are not relevant. - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] This change requires a documentation update, included: [Dify Document](https://github.com/langgenius/dify-docs) +- [ ] Improvement，including but not limited to code refactoring, performance optimization, and UI/UX improvement +- [ ] Dependency upgrade # How Has This Been Tested? From 53507539051553ddb3fcfb027c221969fc409702 Mon Sep 17 00:00:00 2001 From: Su Yang Date: Tue, 19 Mar 2024 18:13:32 +0800 Subject: [PATCH 16/21] chore: update Qwen model params (#2892) --- .../tongyi/llm/qwen-max-1201.yaml | 53 +++++++++++------- .../tongyi/llm/qwen-max-longcontext.yaml | 55 +++++++++++-------- .../model_providers/tongyi/llm/qwen-max.yaml | 53 +++++++++++------- .../model_providers/tongyi/llm/qwen-plus.yaml | 54 +++++++++++------- .../tongyi/llm/qwen-turbo.yaml | 53 +++++++++++------- 5 files changed, 162 insertions(+), 106 deletions(-) diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-1201.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-1201.yaml index e0ba6fe4a8..691347e701 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-1201.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-1201.yaml @@ -8,55 +8,66 @@ model_properties: parameter_rules: - name: temperature use_template: temperature - default: 1.0 + type: float + default: 0.85 min: 0.0 max: 2.0 help: zh_Hans: 用于控制随机性和多样性的程度。具体来说，temperature值控制了生成文本时对每个候选词的概率分布进行平滑的程度。较高的temperature值会降低概率分布的峰值，使得更多的低概率词被选择，生成结果更加多样化；而较低的temperature值则会增强概率分布的峰值，使得高概率词更容易被选择，生成结果更加确定。 en_US: Used to control the degree of randomness and diversity. Specifically, the temperature value controls the degree to which the probability distribution of each candidate word is smoothed when generating text. A higher temperature value will reduce the peak value of the probability distribution, allowing more low-probability words to be selected, and the generated results will be more diverse; while a lower temperature value will enhance the peak value of the probability distribution, making it easier for high-probability words to be selected. , the generated results are more certain. + - name: max_tokens + use_template: max_tokens + type: int + default: 2000 + min: 1 + max: 2000 + help: + zh_Hans: 用于指定模型在生成内容时token的最大数量，它定义了生成的上限，但不保证每次都会生成到这个数量。 + en_US: It is used to specify the maximum number of tokens when the model generates content. It defines the upper limit of generation, but does not guarantee that this number will be generated every time. - name: top_p use_template: top_p + type: float default: 0.8 min: 0.1 max: 0.9 help: zh_Hans: 生成过程中核采样方法概率阈值，例如，取值为0.8时，仅保留概率加起来大于等于0.8的最可能token的最小集合作为候选集。取值范围为（0,1.0)，取值越大，生成的随机性越高；取值越低，生成的确定性越高。 en_US: The probability threshold of the kernel sampling method during the generation process. For example, when the value is 0.8, only the smallest set of the most likely tokens with a sum of probabilities greater than or equal to 0.8 is retained as the candidate set. The value range is (0,1.0). The larger the value, the higher the randomness generated; the lower the value, the higher the certainty generated. - - name: max_tokens - use_template: max_tokens - default: 1500 - min: 1 - max: 6000 - help: - zh_Hans: 用于限制模型生成token的数量，max_tokens设置的是生成上限，并不表示一定会生成这么多的token数量。 - en_US: It is used to limit the number of tokens generated by the model. max_tokens sets the upper limit of generation, which does not mean that so many tokens will be generated. - name: top_k + type: int + min: 0 + max: 99 label: zh_Hans: 取样数量 en_US: Top k - type: int help: - zh_Hans: 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。默认不传递该参数，取值为None或当top_k大于100时，表示不启用top_k策略，此时，仅有top_p策略生效。 - en_US: The size of the sample candidate set when generated. For example, when the value is 50, only the 50 highest-scoring tokens in a single generation form a randomly sampled candidate set. The larger the value, the higher the randomness generated; the smaller the value, the higher the certainty generated. This parameter is not passed by default. The value is None or when top_k is greater than 100, it means that the top_k policy is not enabled. At this time, only the top_p policy takes effect. - required: false + zh_Hans: 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。 + en_US: The size of the sample candidate set when generated. For example, when the value is 50, only the 50 highest-scoring tokens in a single generation form a randomly sampled candidate set. The larger the value, the higher the randomness generated; the smaller the value, the higher the certainty generated. - name: seed + required: false + type: int + default: 1234 label: zh_Hans: 随机种子 en_US: Random seed - type: int help: - zh_Hans: 生成时，随机数的种子，用于控制模型生成的随机性。如果使用相同的种子，每次运行生成的结果都将相同；当需要复现模型的生成结果时，可以使用相同的种子。seed参数支持无符号64位整数类型。 - en_US: When generating, the random number seed is used to control the randomness of model generation. If you use the same seed, the results generated by each run will be the same; when you need to reproduce the results of the model, you can use the same seed. The seed parameter supports unsigned 64-bit integer types. - required: false + zh_Hans: 生成时使用的随机数种子，用户控制模型生成内容的随机性。支持无符号64位整数，默认值为 1234。在使用seed时，模型将尽可能生成相同或相似的结果，但目前不保证每次生成的结果完全相同。 + en_US: The random number seed used when generating, the user controls the randomness of the content generated by the model. Supports unsigned 64-bit integers, default value is 1234. When using seed, the model will try its best to generate the same or similar results, but there is currently no guarantee that the results will be exactly the same every time. - name: repetition_penalty - label: - en_US: Repetition penalty + required: false type: float default: 1.1 + label: + en_US: Repetition penalty help: zh_Hans: 用于控制模型生成时的重复度。提高repetition_penalty时可以降低模型生成的重复度。1.0表示不做惩罚。 - en_US: Used to control the repetition of model generation. Increasing the repetition_penalty can reduce the repetition of model generation. 1.0 means no punishment. - required: false + en_US: Used to control the repeatability when generating models. Increasing repetition_penalty can reduce the duplication of model generation. 1.0 means no punishment. + - name: enable_search + type: boolean + default: false + help: + zh_Hans: 模型内置了互联网搜索服务，该参数控制模型在生成文本时是否参考使用互联网搜索结果。启用互联网搜索，模型会将搜索结果作为文本生成过程中的参考信息，但模型会基于其内部逻辑“自行判断”是否使用互联网搜索结果。 + en_US: The model has a built-in Internet search service. This parameter controls whether the model refers to Internet search results when generating text. When Internet search is enabled, the model will use the search results as reference information in the text generation process, but the model will "judge" whether to use Internet search results based on its internal logic. - name: response_format use_template: response_format pricing: diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-longcontext.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-longcontext.yaml index e2a291cc59..91129d37dd 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-longcontext.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max-longcontext.yaml @@ -4,59 +4,70 @@ label: model_type: llm model_properties: mode: chat - context_size: 30000 + context_size: 32768 parameter_rules: - name: temperature use_template: temperature - default: 1.0 + type: float + default: 0.85 min: 0.0 max: 2.0 help: zh_Hans: 用于控制随机性和多样性的程度。具体来说，temperature值控制了生成文本时对每个候选词的概率分布进行平滑的程度。较高的temperature值会降低概率分布的峰值，使得更多的低概率词被选择，生成结果更加多样化；而较低的temperature值则会增强概率分布的峰值，使得高概率词更容易被选择，生成结果更加确定。 en_US: Used to control the degree of randomness and diversity. Specifically, the temperature value controls the degree to which the probability distribution of each candidate word is smoothed when generating text. A higher temperature value will reduce the peak value of the probability distribution, allowing more low-probability words to be selected, and the generated results will be more diverse; while a lower temperature value will enhance the peak value of the probability distribution, making it easier for high-probability words to be selected. , the generated results are more certain. + - name: max_tokens + use_template: max_tokens + type: int + default: 2000 + min: 1 + max: 2000 + help: + zh_Hans: 用于指定模型在生成内容时token的最大数量，它定义了生成的上限，但不保证每次都会生成到这个数量。 + en_US: It is used to specify the maximum number of tokens when the model generates content. It defines the upper limit of generation, but does not guarantee that this number will be generated every time. - name: top_p use_template: top_p + type: float default: 0.8 min: 0.1 max: 0.9 help: zh_Hans: 生成过程中核采样方法概率阈值，例如，取值为0.8时，仅保留概率加起来大于等于0.8的最可能token的最小集合作为候选集。取值范围为（0,1.0)，取值越大，生成的随机性越高；取值越低，生成的确定性越高。 en_US: The probability threshold of the kernel sampling method during the generation process. For example, when the value is 0.8, only the smallest set of the most likely tokens with a sum of probabilities greater than or equal to 0.8 is retained as the candidate set. The value range is (0,1.0). The larger the value, the higher the randomness generated; the lower the value, the higher the certainty generated. - - name: max_tokens - use_template: max_tokens - default: 2000 - min: 1 - max: 28000 - help: - zh_Hans: 用于限制模型生成token的数量，max_tokens设置的是生成上限，并不表示一定会生成这么多的token数量。 - en_US: It is used to limit the number of tokens generated by the model. max_tokens sets the upper limit of generation, which does not mean that so many tokens will be generated. - name: top_k + type: int + min: 0 + max: 99 label: zh_Hans: 取样数量 en_US: Top k - type: int help: - zh_Hans: 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。默认不传递该参数，取值为None或当top_k大于100时，表示不启用top_k策略，此时，仅有top_p策略生效。 - en_US: The size of the sample candidate set when generated. For example, when the value is 50, only the 50 highest-scoring tokens in a single generation form a randomly sampled candidate set. The larger the value, the higher the randomness generated; the smaller the value, the higher the certainty generated. This parameter is not passed by default. The value is None or when top_k is greater than 100, it means that the top_k policy is not enabled. At this time, only the top_p policy takes effect. - required: false + zh_Hans: 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。 + en_US: The size of the sample candidate set when generated. For example, when the value is 50, only the 50 highest-scoring tokens in a single generation form a randomly sampled candidate set. The larger the value, the higher the randomness generated; the smaller the value, the higher the certainty generated. - name: seed + required: false + type: int + default: 1234 label: zh_Hans: 随机种子 en_US: Random seed - type: int help: - zh_Hans: 生成时，随机数的种子，用于控制模型生成的随机性。如果使用相同的种子，每次运行生成的结果都将相同；当需要复现模型的生成结果时，可以使用相同的种子。seed参数支持无符号64位整数类型。 - en_US: When generating, the random number seed is used to control the randomness of model generation. If you use the same seed, the results generated by each run will be the same; when you need to reproduce the results of the model, you can use the same seed. The seed parameter supports unsigned 64-bit integer types. - required: false + zh_Hans: 生成时使用的随机数种子，用户控制模型生成内容的随机性。支持无符号64位整数，默认值为 1234。在使用seed时，模型将尽可能生成相同或相似的结果，但目前不保证每次生成的结果完全相同。 + en_US: The random number seed used when generating, the user controls the randomness of the content generated by the model. Supports unsigned 64-bit integers, default value is 1234. When using seed, the model will try its best to generate the same or similar results, but there is currently no guarantee that the results will be exactly the same every time. - name: repetition_penalty - label: - en_US: Repetition penalty + required: false type: float default: 1.1 + label: + en_US: Repetition penalty help: zh_Hans: 用于控制模型生成时的重复度。提高repetition_penalty时可以降低模型生成的重复度。1.0表示不做惩罚。 - en_US: Used to control the repetition of model generation. Increasing the repetition_penalty can reduce the repetition of model generation. 1.0 means no punishment. - required: false + en_US: Used to control the repeatability when generating models. Increasing repetition_penalty can reduce the duplication of model generation. 1.0 means no punishment. + - name: enable_search + type: boolean + default: false + help: + zh_Hans: 模型内置了互联网搜索服务，该参数控制模型在生成文本时是否参考使用互联网搜索结果。启用互联网搜索，模型会将搜索结果作为文本生成过程中的参考信息，但模型会基于其内部逻辑“自行判断”是否使用互联网搜索结果。 + en_US: The model has a built-in Internet search service. This parameter controls whether the model refers to Internet search results when generating text. When Internet search is enabled, the model will use the search results as reference information in the text generation process, but the model will "judge" whether to use Internet search results based on its internal logic. - name: response_format use_template: response_format pricing: diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max.yaml index 8260b5081d..5d6b69f21f 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-max.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-max.yaml @@ -8,55 +8,66 @@ model_properties: parameter_rules: - name: temperature use_template: temperature - default: 1.0 + type: float + default: 0.85 min: 0.0 max: 2.0 help: zh_Hans: 用于控制随机性和多样性的程度。具体来说，temperature值控制了生成文本时对每个候选词的概率分布进行平滑的程度。较高的temperature值会降低概率分布的峰值，使得更多的低概率词被选择，生成结果更加多样化；而较低的temperature值则会增强概率分布的峰值，使得高概率词更容易被选择，生成结果更加确定。 en_US: Used to control the degree of randomness and diversity. Specifically, the temperature value controls the degree to which the probability distribution of each candidate word is smoothed when generating text. A higher temperature value will reduce the peak value of the probability distribution, allowing more low-probability words to be selected, and the generated results will be more diverse; while a lower temperature value will enhance the peak value of the probability distribution, making it easier for high-probability words to be selected. , the generated results are more certain. + - name: max_tokens + use_template: max_tokens + type: int + default: 2000 + min: 1 + max: 2000 + help: + zh_Hans: 用于指定模型在生成内容时token的最大数量，它定义了生成的上限，但不保证每次都会生成到这个数量。 + en_US: It is used to specify the maximum number of tokens when the model generates content. It defines the upper limit of generation, but does not guarantee that this number will be generated every time. - name: top_p use_template: top_p + type: float default: 0.8 min: 0.1 max: 0.9 help: zh_Hans: 生成过程中核采样方法概率阈值，例如，取值为0.8时，仅保留概率加起来大于等于0.8的最可能token的最小集合作为候选集。取值范围为（0,1.0)，取值越大，生成的随机性越高；取值越低，生成的确定性越高。 en_US: The probability threshold of the kernel sampling method during the generation process. For example, when the value is 0.8, only the smallest set of the most likely tokens with a sum of probabilities greater than or equal to 0.8 is retained as the candidate set. The value range is (0,1.0). The larger the value, the higher the randomness generated; the lower the value, the higher the certainty generated. - - name: max_tokens - use_template: max_tokens - default: 1500 - min: 1 - max: 6000 - help: - zh_Hans: 用于限制模型生成token的数量，max_tokens设置的是生成上限，并不表示一定会生成这么多的token数量。 - en_US: It is used to limit the number of tokens generated by the model. max_tokens sets the upper limit of generation, which does not mean that so many tokens will be generated. - name: top_k + type: int + min: 0 + max: 99 label: zh_Hans: 取样数量 en_US: Top k - type: int help: - zh_Hans: 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。默认不传递该参数，取值为None或当top_k大于100时，表示不启用top_k策略，此时，仅有top_p策略生效。 - en_US: The size of the sample candidate set when generated. For example, when the value is 50, only the 50 highest-scoring tokens in a single generation form a randomly sampled candidate set. The larger the value, the higher the randomness generated; the smaller the value, the higher the certainty generated. This parameter is not passed by default. The value is None or when top_k is greater than 100, it means that the top_k policy is not enabled. At this time, only the top_p policy takes effect. - required: false + zh_Hans: 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。 + en_US: The size of the sample candidate set when generated. For example, when the value is 50, only the 50 highest-scoring tokens in a single generation form a randomly sampled candidate set. The larger the value, the higher the randomness generated; the smaller the value, the higher the certainty generated. - name: seed + required: false + type: int + default: 1234 label: zh_Hans: 随机种子 en_US: Random seed - type: int help: - zh_Hans: 生成时，随机数的种子，用于控制模型生成的随机性。如果使用相同的种子，每次运行生成的结果都将相同；当需要复现模型的生成结果时，可以使用相同的种子。seed参数支持无符号64位整数类型。 - en_US: When generating, the random number seed is used to control the randomness of model generation. If you use the same seed, the results generated by each run will be the same; when you need to reproduce the results of the model, you can use the same seed. The seed parameter supports unsigned 64-bit integer types. - required: false + zh_Hans: 生成时使用的随机数种子，用户控制模型生成内容的随机性。支持无符号64位整数，默认值为 1234。在使用seed时，模型将尽可能生成相同或相似的结果，但目前不保证每次生成的结果完全相同。 + en_US: The random number seed used when generating, the user controls the randomness of the content generated by the model. Supports unsigned 64-bit integers, default value is 1234. When using seed, the model will try its best to generate the same or similar results, but there is currently no guarantee that the results will be exactly the same every time. - name: repetition_penalty - label: - en_US: Repetition penalty + required: false type: float default: 1.1 + label: + en_US: Repetition penalty help: zh_Hans: 用于控制模型生成时的重复度。提高repetition_penalty时可以降低模型生成的重复度。1.0表示不做惩罚。 - en_US: Used to control the repetition of model generation. Increasing the repetition_penalty can reduce the repetition of model generation. 1.0 means no punishment. - required: false + en_US: Used to control the repeatability when generating models. Increasing repetition_penalty can reduce the duplication of model generation. 1.0 means no punishment. + - name: enable_search + type: boolean + default: false + help: + zh_Hans: 模型内置了互联网搜索服务，该参数控制模型在生成文本时是否参考使用互联网搜索结果。启用互联网搜索，模型会将搜索结果作为文本生成过程中的参考信息，但模型会基于其内部逻辑“自行判断”是否使用互联网搜索结果。 + en_US: The model has a built-in Internet search service. This parameter controls whether the model refers to Internet search results when generating text. When Internet search is enabled, the model will use the search results as reference information in the text generation process, but the model will "judge" whether to use Internet search results based on its internal logic. - name: response_format use_template: response_format pricing: diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus.yaml index 83640371f9..7c25e8802b 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus.yaml @@ -4,58 +4,70 @@ label: model_type: llm model_properties: mode: completion - context_size: 32000 + context_size: 32768 parameter_rules: - name: temperature use_template: temperature - default: 1.0 + type: float + default: 0.85 min: 0.0 max: 2.0 help: zh_Hans: 用于控制随机性和多样性的程度。具体来说，temperature值控制了生成文本时对每个候选词的概率分布进行平滑的程度。较高的temperature值会降低概率分布的峰值，使得更多的低概率词被选择，生成结果更加多样化；而较低的temperature值则会增强概率分布的峰值，使得高概率词更容易被选择，生成结果更加确定。 en_US: Used to control the degree of randomness and diversity. Specifically, the temperature value controls the degree to which the probability distribution of each candidate word is smoothed when generating text. A higher temperature value will reduce the peak value of the probability distribution, allowing more low-probability words to be selected, and the generated results will be more diverse; while a lower temperature value will enhance the peak value of the probability distribution, making it easier for high-probability words to be selected. , the generated results are more certain. + - name: max_tokens + use_template: max_tokens + type: int + default: 1500 + min: 1 + max: 1500 + help: + zh_Hans: 用于指定模型在生成内容时token的最大数量，它定义了生成的上限，但不保证每次都会生成到这个数量。 + en_US: It is used to specify the maximum number of tokens when the model generates content. It defines the upper limit of generation, but does not guarantee that this number will be generated every time. - name: top_p use_template: top_p + type: float default: 0.8 min: 0.1 max: 0.9 help: zh_Hans: 生成过程中核采样方法概率阈值，例如，取值为0.8时，仅保留概率加起来大于等于0.8的最可能token的最小集合作为候选集。取值范围为（0,1.0)，取值越大，生成的随机性越高；取值越低，生成的确定性越高。 en_US: The probability threshold of the kernel sampling method during the generation process. For example, when the value is 0.8, only the smallest set of the most likely tokens with a sum of probabilities greater than or equal to 0.8 is retained as the candidate set. The value range is (0,1.0). The larger the value, the higher the randomness generated; the lower the value, the higher the certainty generated. - - name: max_tokens - use_template: max_tokens - default: 2000 - min: 1 - max: 30000 - help: - zh_Hans: 用于限制模型生成token的数量，max_tokens设置的是生成上限，并不表示一定会生成这么多的token数量。 - en_US: It is used to limit the number of tokens generated by the model. max_tokens sets the upper limit of generation, which does not mean that so many tokens will be generated. - name: top_k + type: int + min: 0 + max: 99 label: zh_Hans: 取样数量 en_US: Top k - type: int help: - zh_Hans: 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。默认不传递该参数，取值为None或当top_k大于100时，表示不启用top_k策略，此时，仅有top_p策略生效。 - en_US: The size of the sample candidate set when generated. For example, when the value is 50, only the 50 highest-scoring tokens in a single generation form a randomly sampled candidate set. The larger the value, the higher the randomness generated; the smaller the value, the higher the certainty generated. This parameter is not passed by default. The value is None or when top_k is greater than 100, it means that the top_k policy is not enabled. At this time, only the top_p policy takes effect. - required: false + zh_Hans: 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。 + en_US: The size of the sample candidate set when generated. For example, when the value is 50, only the 50 highest-scoring tokens in a single generation form a randomly sampled candidate set. The larger the value, the higher the randomness generated; the smaller the value, the higher the certainty generated. - name: seed + required: false + type: int + default: 1234 label: zh_Hans: 随机种子 en_US: Random seed - type: int help: - zh_Hans: 生成时，随机数的种子，用于控制模型生成的随机性。如果使用相同的种子，每次运行生成的结果都将相同；当需要复现模型的生成结果时，可以使用相同的种子。seed参数支持无符号64位整数类型。 - en_US: When generating, the random number seed is used to control the randomness of model generation. If you use the same seed, the results generated by each run will be the same; when you need to reproduce the results of the model, you can use the same seed. The seed parameter supports unsigned 64-bit integer types. - required: false + zh_Hans: 生成时使用的随机数种子，用户控制模型生成内容的随机性。支持无符号64位整数，默认值为 1234。在使用seed时，模型将尽可能生成相同或相似的结果，但目前不保证每次生成的结果完全相同。 + en_US: The random number seed used when generating, the user controls the randomness of the content generated by the model. Supports unsigned 64-bit integers, default value is 1234. When using seed, the model will try its best to generate the same or similar results, but there is currently no guarantee that the results will be exactly the same every time. - name: repetition_penalty - label: - en_US: Repetition penalty + required: false type: float default: 1.1 + label: + en_US: Repetition penalty help: zh_Hans: 用于控制模型生成时的重复度。提高repetition_penalty时可以降低模型生成的重复度。1.0表示不做惩罚。 - en_US: Used to control the repetition of model generation. Increasing the repetition_penalty can reduce the repetition of model generation. 1.0 means no punishment. + en_US: Used to control the repeatability when generating models. Increasing repetition_penalty can reduce the duplication of model generation. 1.0 means no punishment. + - name: enable_search + type: boolean + default: false + help: + zh_Hans: 模型内置了互联网搜索服务，该参数控制模型在生成文本时是否参考使用互联网搜索结果。启用互联网搜索，模型会将搜索结果作为文本生成过程中的参考信息，但模型会基于其内部逻辑“自行判断”是否使用互联网搜索结果。 + en_US: The model has a built-in Internet search service. This parameter controls whether the model refers to Internet search results when generating text. When Internet search is enabled, the model will use the search results as reference information in the text generation process, but the model will "judge" whether to use Internet search results based on its internal logic. - name: response_format use_template: response_format pricing: diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-turbo.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-turbo.yaml index 5455555bbd..20b46de6f3 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-turbo.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-turbo.yaml @@ -8,55 +8,66 @@ model_properties: parameter_rules: - name: temperature use_template: temperature - default: 1.0 + type: float + default: 0.85 min: 0.0 max: 2.0 help: zh_Hans: 用于控制随机性和多样性的程度。具体来说，temperature值控制了生成文本时对每个候选词的概率分布进行平滑的程度。较高的temperature值会降低概率分布的峰值，使得更多的低概率词被选择，生成结果更加多样化；而较低的temperature值则会增强概率分布的峰值，使得高概率词更容易被选择，生成结果更加确定。 en_US: Used to control the degree of randomness and diversity. Specifically, the temperature value controls the degree to which the probability distribution of each candidate word is smoothed when generating text. A higher temperature value will reduce the peak value of the probability distribution, allowing more low-probability words to be selected, and the generated results will be more diverse; while a lower temperature value will enhance the peak value of the probability distribution, making it easier for high-probability words to be selected. , the generated results are more certain. + - name: max_tokens + use_template: max_tokens + type: int + default: 1500 + min: 1 + max: 1500 + help: + zh_Hans: 用于指定模型在生成内容时token的最大数量，它定义了生成的上限，但不保证每次都会生成到这个数量。 + en_US: It is used to specify the maximum number of tokens when the model generates content. It defines the upper limit of generation, but does not guarantee that this number will be generated every time. - name: top_p use_template: top_p + type: float default: 0.8 min: 0.1 max: 0.9 help: zh_Hans: 生成过程中核采样方法概率阈值，例如，取值为0.8时，仅保留概率加起来大于等于0.8的最可能token的最小集合作为候选集。取值范围为（0,1.0)，取值越大，生成的随机性越高；取值越低，生成的确定性越高。 en_US: The probability threshold of the kernel sampling method during the generation process. For example, when the value is 0.8, only the smallest set of the most likely tokens with a sum of probabilities greater than or equal to 0.8 is retained as the candidate set. The value range is (0,1.0). The larger the value, the higher the randomness generated; the lower the value, the higher the certainty generated. - - name: max_tokens - use_template: max_tokens - default: 1500 - min: 1 - max: 6000 - help: - zh_Hans: 用于限制模型生成token的数量，max_tokens设置的是生成上限，并不表示一定会生成这么多的token数量。 - en_US: It is used to limit the number of tokens generated by the model. max_tokens sets the upper limit of generation, which does not mean that so many tokens will be generated. - name: top_k + type: int + min: 0 + max: 99 label: zh_Hans: 取样数量 en_US: Top k - type: int help: - zh_Hans: 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。默认不传递该参数，取值为None或当top_k大于100时，表示不启用top_k策略，此时，仅有top_p策略生效。 - en_US: The size of the sample candidate set when generated. For example, when the value is 50, only the 50 highest-scoring tokens in a single generation form a randomly sampled candidate set. The larger the value, the higher the randomness generated; the smaller the value, the higher the certainty generated. This parameter is not passed by default. The value is None or when top_k is greater than 100, it means that the top_k policy is not enabled. At this time, only the top_p policy takes effect. - required: false + zh_Hans: 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。 + en_US: The size of the sample candidate set when generated. For example, when the value is 50, only the 50 highest-scoring tokens in a single generation form a randomly sampled candidate set. The larger the value, the higher the randomness generated; the smaller the value, the higher the certainty generated. - name: seed + required: false + type: int + default: 1234 label: zh_Hans: 随机种子 en_US: Random seed - type: int help: - zh_Hans: 生成时，随机数的种子，用于控制模型生成的随机性。如果使用相同的种子，每次运行生成的结果都将相同；当需要复现模型的生成结果时，可以使用相同的种子。seed参数支持无符号64位整数类型。 - en_US: When generating, the random number seed is used to control the randomness of model generation. If you use the same seed, the results generated by each run will be the same; when you need to reproduce the results of the model, you can use the same seed. The seed parameter supports unsigned 64-bit integer types. - required: false + zh_Hans: 生成时使用的随机数种子，用户控制模型生成内容的随机性。支持无符号64位整数，默认值为 1234。在使用seed时，模型将尽可能生成相同或相似的结果，但目前不保证每次生成的结果完全相同。 + en_US: The random number seed used when generating, the user controls the randomness of the content generated by the model. Supports unsigned 64-bit integers, default value is 1234. When using seed, the model will try its best to generate the same or similar results, but there is currently no guarantee that the results will be exactly the same every time. - name: repetition_penalty - label: - en_US: Repetition penalty + required: false type: float default: 1.1 + label: + en_US: Repetition penalty help: zh_Hans: 用于控制模型生成时的重复度。提高repetition_penalty时可以降低模型生成的重复度。1.0表示不做惩罚。 - en_US: Used to control the repetition of model generation. Increasing the repetition_penalty can reduce the repetition of model generation. 1.0 means no punishment. - required: false + en_US: Used to control the repeatability when generating models. Increasing repetition_penalty can reduce the duplication of model generation. 1.0 means no punishment. + - name: enable_search + type: boolean + default: false + help: + zh_Hans: 模型内置了互联网搜索服务，该参数控制模型在生成文本时是否参考使用互联网搜索结果。启用互联网搜索，模型会将搜索结果作为文本生成过程中的参考信息，但模型会基于其内部逻辑“自行判断”是否使用互联网搜索结果。 + en_US: The model has a built-in Internet search service. This parameter controls whether the model refers to Internet search results when generating text. When Internet search is enabled, the model will use the search results as reference information in the text generation process, but the model will "judge" whether to use Internet search results based on its internal logic. - name: response_format use_template: response_format pricing: From 85da94aac478e4ee32305eb6f0b6be08c20f6ef4 Mon Sep 17 00:00:00 2001 From: Lance Mao Date: Tue, 19 Mar 2024 18:17:12 +0800 Subject: [PATCH 17/21] =?UTF-8?q?fix=20incorrect=20exception=20raised=20by?= =?UTF-8?q?=20api=20tool=20which=20leads=20to=20incorrect=20L=E2=80=A6=20(?= =?UTF-8?q?#2886)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: OSS-MAOLONGDONG\kaihong --- api/core/tools/tool/api_tool.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/api/core/tools/tool/api_tool.py b/api/core/tools/tool/api_tool.py index fa7e7567dd..54e2f41019 100644 --- a/api/core/tools/tool/api_tool.py +++ b/api/core/tools/tool/api_tool.py @@ -9,7 +9,7 @@ import requests import core.helper.ssrf_proxy as ssrf_proxy from core.tools.entities.tool_bundle import ApiBasedToolBundle from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.errors import ToolInvokeError, ToolParameterValidationError, ToolProviderCredentialValidationError from core.tools.tool.tool import Tool API_TOOL_DEFAULT_TIMEOUT = (10, 60) @@ -81,7 +81,7 @@ class ApiTool(Tool): needed_parameters = [parameter for parameter in self.api_bundle.parameters if parameter.required] for parameter in needed_parameters: if parameter.required and parameter.name not in parameters: - raise ToolProviderCredentialValidationError(f"Missing required parameter {parameter.name}") + raise ToolParameterValidationError(f"Missing required parameter {parameter.name}") if parameter.default is not None and parameter.name not in parameters: parameters[parameter.name] = parameter.default @@ -94,7 +94,7 @@ class ApiTool(Tool): """ if isinstance(response, httpx.Response): if response.status_code >= 400: - raise ToolProviderCredentialValidationError(f"Request failed with status code {response.status_code}") + raise ToolInvokeError(f"Request failed with status code {response.status_code} and {response.text}") if not response.content: return 'Empty response from the tool, please check your parameters and try again.' try: @@ -107,7 +107,7 @@ class ApiTool(Tool): return response.text elif isinstance(response, requests.Response): if not response.ok: - raise ToolProviderCredentialValidationError(f"Request failed with status code {response.status_code}") + raise ToolInvokeError(f"Request failed with status code {response.status_code} and {response.text}") if not response.content: return 'Empty response from the tool, please check your parameters and try again.' try: @@ -139,7 +139,7 @@ class ApiTool(Tool): if parameter['name'] in parameters: value = parameters[parameter['name']] elif parameter['required']: - raise ToolProviderCredentialValidationError(f"Missing required parameter {parameter['name']}") + raise ToolParameterValidationError(f"Missing required parameter {parameter['name']}") else: value = (parameter.get('schema', {}) or {}).get('default', '') path_params[parameter['name']] = value @@ -149,7 +149,7 @@ class ApiTool(Tool): if parameter['name'] in parameters: value = parameters[parameter['name']] elif parameter['required']: - raise ToolProviderCredentialValidationError(f"Missing required parameter {parameter['name']}") + raise ToolParameterValidationError(f"Missing required parameter {parameter['name']}") else: value = (parameter.get('schema', {}) or {}).get('default', '') params[parameter['name']] = value @@ -159,7 +159,7 @@ class ApiTool(Tool): if parameter['name'] in parameters: value = parameters[parameter['name']] elif parameter['required']: - raise ToolProviderCredentialValidationError(f"Missing required parameter {parameter['name']}") + raise ToolParameterValidationError(f"Missing required parameter {parameter['name']}") else: value = (parameter.get('schema', {}) or {}).get('default', '') cookies[parameter['name']] = value @@ -169,7 +169,7 @@ class ApiTool(Tool): if parameter['name'] in parameters: value = parameters[parameter['name']] elif parameter['required']: - raise ToolProviderCredentialValidationError(f"Missing required parameter {parameter['name']}") + raise ToolParameterValidationError(f"Missing required parameter {parameter['name']}") else: value = (parameter.get('schema', {}) or {}).get('default', '') headers[parameter['name']] = value @@ -188,7 +188,7 @@ class ApiTool(Tool): # convert type body[name] = self._convert_body_property_type(property, parameters[name]) elif name in required: - raise ToolProviderCredentialValidationError( + raise ToolParameterValidationError( f"Missing required parameter {name} in operation {self.api_bundle.operation_id}" ) elif 'default' in property: From 7c7f3958ff11d03c9776bc7ca447272e06529662 Mon Sep 17 00:00:00 2001 From: takatost Date: Tue, 19 Mar 2024 18:34:23 +0800 Subject: [PATCH 18/21] feat: optimize ollama model default parameters (#2894) --- .../model_runtime/model_providers/ollama/llm/llm.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/api/core/model_runtime/model_providers/ollama/llm/llm.py b/api/core/model_runtime/model_providers/ollama/llm/llm.py index e4388699e3..3589ca77cc 100644 --- a/api/core/model_runtime/model_providers/ollama/llm/llm.py +++ b/api/core/model_runtime/model_providers/ollama/llm/llm.py @@ -449,7 +449,7 @@ class OllamaLargeLanguageModel(LargeLanguageModel): help=I18nObject(en_US="The temperature of the model. " "Increasing the temperature will make the model answer " "more creatively. (Default: 0.8)"), - default=0.8, + default=0.1, min=0, max=2 ), @@ -472,7 +472,6 @@ class OllamaLargeLanguageModel(LargeLanguageModel): help=I18nObject(en_US="Reduces the probability of generating nonsense. " "A higher value (e.g. 100) will give more diverse answers, " "while a lower value (e.g. 10) will be more conservative. (Default: 40)"), - default=40, min=1, max=100 ), @@ -483,7 +482,6 @@ class OllamaLargeLanguageModel(LargeLanguageModel): help=I18nObject(en_US="Sets how strongly to penalize repetitions. " "A higher value (e.g., 1.5) will penalize repetitions more strongly, " "while a lower value (e.g., 0.9) will be more lenient. (Default: 1.1)"), - default=1.1, min=-2, max=2 ), @@ -494,7 +492,7 @@ class OllamaLargeLanguageModel(LargeLanguageModel): type=ParameterType.INT, help=I18nObject(en_US="Maximum number of tokens to predict when generating text. " "(Default: 128, -1 = infinite generation, -2 = fill context)"), - default=128, + default=512 if int(credentials.get('max_tokens', 4096)) >= 768 else 128, min=-2, max=int(credentials.get('max_tokens', 4096)), ), @@ -504,7 +502,6 @@ class OllamaLargeLanguageModel(LargeLanguageModel): type=ParameterType.INT, help=I18nObject(en_US="Enable Mirostat sampling for controlling perplexity. " "(default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)"), - default=0, min=0, max=2 ), @@ -516,7 +513,6 @@ class OllamaLargeLanguageModel(LargeLanguageModel): "the generated text. A lower learning rate will result in slower adjustments, " "while a higher learning rate will make the algorithm more responsive. " "(Default: 0.1)"), - default=0.1, precision=1 ), ParameterRule( @@ -525,7 +521,6 @@ class OllamaLargeLanguageModel(LargeLanguageModel): type=ParameterType.FLOAT, help=I18nObject(en_US="Controls the balance between coherence and diversity of the output. " "A lower value will result in more focused and coherent text. (Default: 5.0)"), - default=5.0, precision=1 ), ParameterRule( @@ -543,7 +538,6 @@ class OllamaLargeLanguageModel(LargeLanguageModel): type=ParameterType.INT, help=I18nObject(en_US="The number of layers to send to the GPU(s). " "On macOS it defaults to 1 to enable metal support, 0 to disable."), - default=1, min=0, max=1 ), @@ -563,7 +557,6 @@ class OllamaLargeLanguageModel(LargeLanguageModel): type=ParameterType.INT, help=I18nObject(en_US="Sets how far back for the model to look back to prevent repetition. " "(Default: 64, 0 = disabled, -1 = num_ctx)"), - default=64, min=-1 ), ParameterRule( @@ -573,7 +566,6 @@ class OllamaLargeLanguageModel(LargeLanguageModel): help=I18nObject(en_US="Tail free sampling is used to reduce the impact of less probable tokens " "from the output. A higher value (e.g., 2.0) will reduce the impact more, " "while a value of 1.0 disables this setting. (default: 1)"), - default=1, precision=1 ), ParameterRule( @@ -583,7 +575,6 @@ class OllamaLargeLanguageModel(LargeLanguageModel): help=I18nObject(en_US="Sets the random number seed to use for generation. Setting this to " "a specific number will make the model generate the same text for " "the same prompt. (Default: 0)"), - default=0 ), ParameterRule( name='format', From bae1bc2e4b9fa62987f53d0d39093f6376c48b55 Mon Sep 17 00:00:00 2001 From: takatost Date: Tue, 19 Mar 2024 18:37:27 +0800 Subject: [PATCH 19/21] fix --- .../model_runtime/model_providers/anthropic/llm/llm.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/core/model_runtime/model_providers/anthropic/llm/llm.py b/api/core/model_runtime/model_providers/anthropic/llm/llm.py index 724a0401b7..bae96b427a 100644 --- a/api/core/model_runtime/model_providers/anthropic/llm/llm.py +++ b/api/core/model_runtime/model_providers/anthropic/llm/llm.py @@ -345,13 +345,13 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): first_loop = True for message in prompt_messages: if isinstance(message, SystemPromptMessage): - message.content=message.content.strip() + message.content = message.content.strip() if first_loop: - system=message.content - first_loop=False + system = message.content + first_loop = False else: - system+="\n" - system+=message.content + system += "\n" + system += message.content prompt_message_dicts = [] for message in prompt_messages: From a9e44b1fd27578ec10bb75788ce5ed43c62f151c Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Tue, 19 Mar 2024 18:37:37 +0800 Subject: [PATCH 20/21] fix: missing head --- api/core/workflow/nodes/http_request/entities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/workflow/nodes/http_request/entities.py b/api/core/workflow/nodes/http_request/entities.py index 0683008954..87529d8f58 100644 --- a/api/core/workflow/nodes/http_request/entities.py +++ b/api/core/workflow/nodes/http_request/entities.py @@ -37,7 +37,7 @@ class HttpRequestNodeData(BaseNodeData): data: Union[None, str] variables: list[VariableSelector] - method: Literal['get', 'post', 'put', 'patch', 'delete'] + method: Literal['get', 'post', 'put', 'patch', 'delete', 'head'] url: str authorization: Authorization headers: str From 3969ed6f69b3778389e7137f9f52bda0dae03ec5 Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Tue, 19 Mar 2024 19:01:09 +0800 Subject: [PATCH 21/21] enhance: check valid JSON --- .../nodes/http_request/http_executor.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index fbbd9a1b55..673c199196 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -1,3 +1,4 @@ +import json import re from copy import deepcopy from random import randint @@ -147,6 +148,19 @@ class HttpExecutor: # init template self._init_template(node_data, variables) + def _is_json_body(self, node_data: HttpRequestNodeData): + """ + check if body is json + """ + if node_data.body and node_data.body.type == 'json': + try: + json.loads(node_data.body.data) + return True + except: + return False + + return False + def _init_template(self, node_data: HttpRequestNodeData, variables: dict[str, Any]): """ init template @@ -217,14 +231,20 @@ class HttpExecutor: # extract all template in body if node_data.body: + # check if it's a valid JSON + is_valid_json = self._is_json_body(node_data) body_template = re.findall(r'{{(.*?)}}', node_data.body.data or '') or [] body_template = list(set(body_template)) original_body = node_data.body.data or '' for body in body_template: if not body: continue - - original_body = original_body.replace(f'{{{{{body}}}}}', str(variables.get(body, ''))) + + body_value = variables.get(body, '') + if is_valid_json: + body_value = body_value.replace('"', '\\"') + + original_body = original_body.replace(f'{{{{{body}}}}}', body_value) if node_data.body.type == 'json': self.headers['Content-Type'] = 'application/json'