From 0bb7569d4640095adf29bf6cf7bb701f36931f8c Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Thu, 12 Sep 2024 11:28:14 +0800 Subject: [PATCH 1/5] fix: markdown paragraph margin (#8289) --- web/app/components/base/markdown.tsx | 4 ++-- web/app/styles/markdown.scss | 12 +++--------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/web/app/components/base/markdown.tsx b/web/app/components/base/markdown.tsx index 11bcd84e18..d4e7dac4ae 100644 --- a/web/app/components/base/markdown.tsx +++ b/web/app/components/base/markdown.tsx @@ -198,11 +198,11 @@ const Paragraph = (paragraph: any) => { return ( <> -
{paragraph.children.slice(1)}
+

{paragraph.children.slice(1)}

) } - return
{paragraph.children}
+ return

{paragraph.children}

} const Img = ({ src }: any) => { diff --git a/web/app/styles/markdown.scss b/web/app/styles/markdown.scss index b0a1f60cd2..214d8d2782 100644 --- a/web/app/styles/markdown.scss +++ b/web/app/styles/markdown.scss @@ -321,18 +321,12 @@ .markdown-body h4, .markdown-body h5, .markdown-body h6 { - margin-top: 24px; - margin-bottom: 16px; + padding-top: 12px; + margin-bottom: 12px; font-weight: var(--base-text-weight-semibold, 600); line-height: 1.25; } - -.markdown-body p { - margin-top: 0; - margin-bottom: 10px; -} - .markdown-body blockquote { margin: 0; padding: 0 8px; @@ -449,7 +443,7 @@ .markdown-body pre, .markdown-body details { margin-top: 0; - margin-bottom: 16px; + margin-bottom: 12px; } .markdown-body blockquote> :first-child { From 0f1487325531bf4d8b38225218a91f06624dcc35 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Thu, 12 Sep 2024 12:55:45 +0800 Subject: [PATCH 2/5] chore: cleanup ruff flake8-simplify linter rules (#8286) Co-authored-by: -LAN- --- .../based_generate_task_pipeline.py | 2 +- .../baichuan/llm/baichuan_turbo.py | 4 +- .../model_providers/google/llm/llm.py | 4 +- .../model_providers/oci/llm/llm.py | 4 +- .../model_providers/tongyi/llm/llm.py | 4 +- .../model_providers/vertex_ai/llm/llm.py | 4 +- .../model_providers/xinference/llm/llm.py | 6 +-- .../model_providers/zhipuai/llm/llm.py | 11 ++--- api/core/moderation/keywords/keywords.py | 13 ++---- api/core/ops/ops_trace_manager.py | 2 +- .../rag/datasource/vdb/relyt/relyt_vector.py | 46 +++++++++---------- .../datasource/vdb/tencent/tencent_vector.py | 5 +- .../datasource/vdb/tidb_vector/tidb_vector.py | 32 ++++++------- api/core/rag/extractor/word_extractor.py | 3 +- api/core/rag/rerank/weight_rerank.py | 4 +- api/core/rag/retrieval/dataset_retrieval.py | 4 +- .../hap/tools/list_worksheet_records.py | 4 +- .../novitaai/tools/novitaai_modelquery.py | 2 +- .../builtin/qrcode/tools/qrcode_generator.py | 2 +- .../builtin/searchapi/tools/google.py | 20 ++++---- .../builtin/searchapi/tools/google_jobs.py | 6 +-- .../builtin/searchapi/tools/google_news.py | 10 ++-- .../searchapi/tools/youtube_transcripts.py | 4 +- .../builtin/stability/tools/text2image.py | 2 +- .../stablediffusion/tools/stable_diffusion.py | 7 ++- .../wikipedia/tools/wikipedia_search.py | 2 +- api/core/tools/utils/message_transformer.py | 4 +- .../workflow/graph_engine/entities/graph.py | 4 +- .../answer/answer_stream_generate_router.py | 2 +- .../nodes/end/end_stream_generate_router.py | 2 +- api/core/workflow/nodes/tool/entities.py | 2 +- api/pyproject.toml | 15 ++++-- api/services/file_service.py | 4 +- api/services/ops_service.py | 4 +- 34 files changed, 108 insertions(+), 136 deletions(-) diff --git a/api/core/app/task_pipeline/based_generate_task_pipeline.py b/api/core/app/task_pipeline/based_generate_task_pipeline.py index 49f58af12c..a43be5fdf2 100644 --- a/api/core/app/task_pipeline/based_generate_task_pipeline.py +++ b/api/core/app/task_pipeline/based_generate_task_pipeline.py @@ -65,7 +65,7 @@ class BasedGenerateTaskPipeline: if isinstance(e, InvokeAuthorizationError): err = InvokeAuthorizationError("Incorrect API key provided") - elif isinstance(e, InvokeError) or isinstance(e, ValueError): + elif isinstance(e, InvokeError | ValueError): err = e else: err = Exception(e.description if getattr(e, "description", None) is not None else str(e)) diff --git a/api/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo.py b/api/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo.py index 39f867118b..d5fda73009 100644 --- a/api/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo.py +++ b/api/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo.py @@ -45,7 +45,7 @@ class BaichuanModel: parameters: dict[str, Any], tools: Optional[list[PromptMessageTool]] = None, ) -> dict[str, Any]: - if model in self._model_mapping.keys(): + if model in self._model_mapping: # the LargeLanguageModel._code_block_mode_wrapper() method will remove the response_format of parameters. # we need to rename it to res_format to get its value if parameters.get("res_format") == "json_object": @@ -94,7 +94,7 @@ class BaichuanModel: timeout: int, tools: Optional[list[PromptMessageTool]] = None, ) -> Union[Iterator, dict]: - if model in self._model_mapping.keys(): + if model in self._model_mapping: api_base = "https://api.baichuan-ai.com/v1/chat/completions" else: raise BadRequestError(f"Unknown model: {model}") diff --git a/api/core/model_runtime/model_providers/google/llm/llm.py b/api/core/model_runtime/model_providers/google/llm/llm.py index 274ff02095..307c15e1fd 100644 --- a/api/core/model_runtime/model_providers/google/llm/llm.py +++ b/api/core/model_runtime/model_providers/google/llm/llm.py @@ -337,9 +337,7 @@ class GoogleLargeLanguageModel(LargeLanguageModel): message_text = f"{human_prompt} {content}" elif isinstance(message, AssistantPromptMessage): message_text = f"{ai_prompt} {content}" - elif isinstance(message, SystemPromptMessage): - message_text = f"{human_prompt} {content}" - elif isinstance(message, ToolPromptMessage): + elif isinstance(message, SystemPromptMessage | ToolPromptMessage): message_text = f"{human_prompt} {content}" else: raise ValueError(f"Got unknown type {message}") diff --git a/api/core/model_runtime/model_providers/oci/llm/llm.py b/api/core/model_runtime/model_providers/oci/llm/llm.py index ad5197a154..51b634c6cf 100644 --- a/api/core/model_runtime/model_providers/oci/llm/llm.py +++ b/api/core/model_runtime/model_providers/oci/llm/llm.py @@ -442,9 +442,7 @@ class OCILargeLanguageModel(LargeLanguageModel): message_text = f"{human_prompt} {content}" elif isinstance(message, AssistantPromptMessage): message_text = f"{ai_prompt} {content}" - elif isinstance(message, SystemPromptMessage): - message_text = f"{human_prompt} {content}" - elif isinstance(message, ToolPromptMessage): + elif isinstance(message, SystemPromptMessage | ToolPromptMessage): message_text = f"{human_prompt} {content}" else: raise ValueError(f"Got unknown type {message}") diff --git a/api/core/model_runtime/model_providers/tongyi/llm/llm.py b/api/core/model_runtime/model_providers/tongyi/llm/llm.py index 72c319d395..db0b2deaa5 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/llm.py +++ b/api/core/model_runtime/model_providers/tongyi/llm/llm.py @@ -350,9 +350,7 @@ class TongyiLargeLanguageModel(LargeLanguageModel): break elif isinstance(message, AssistantPromptMessage): message_text = f"{ai_prompt} {content}" - elif isinstance(message, SystemPromptMessage): - message_text = content - elif isinstance(message, ToolPromptMessage): + elif isinstance(message, SystemPromptMessage | ToolPromptMessage): message_text = content else: raise ValueError(f"Got unknown type {message}") diff --git a/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py b/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py index 09a7f53f28..110028a288 100644 --- a/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py +++ b/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py @@ -633,9 +633,7 @@ class VertexAiLargeLanguageModel(LargeLanguageModel): message_text = f"{human_prompt} {content}" elif isinstance(message, AssistantPromptMessage): message_text = f"{ai_prompt} {content}" - elif isinstance(message, SystemPromptMessage): - message_text = f"{human_prompt} {content}" - elif isinstance(message, ToolPromptMessage): + elif isinstance(message, SystemPromptMessage | ToolPromptMessage): message_text = f"{human_prompt} {content}" else: raise ValueError(f"Got unknown type {message}") diff --git a/api/core/model_runtime/model_providers/xinference/llm/llm.py b/api/core/model_runtime/model_providers/xinference/llm/llm.py index b2c837dee1..bc7531ee20 100644 --- a/api/core/model_runtime/model_providers/xinference/llm/llm.py +++ b/api/core/model_runtime/model_providers/xinference/llm/llm.py @@ -272,11 +272,7 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): """ text = "" for item in message: - if isinstance(item, UserPromptMessage): - text += item.content - elif isinstance(item, SystemPromptMessage): - text += item.content - elif isinstance(item, AssistantPromptMessage): + if isinstance(item, UserPromptMessage | SystemPromptMessage | AssistantPromptMessage): text += item.content else: raise NotImplementedError(f"PromptMessage type {type(item)} is not supported") diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py index 484ac088db..498962bd0f 100644 --- a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py +++ b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py @@ -209,9 +209,10 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): ): new_prompt_messages[-1].content += "\n\n" + copy_prompt_message.content else: - if copy_prompt_message.role == PromptMessageRole.USER: - new_prompt_messages.append(copy_prompt_message) - elif copy_prompt_message.role == PromptMessageRole.TOOL: + if ( + copy_prompt_message.role == PromptMessageRole.USER + or copy_prompt_message.role == PromptMessageRole.TOOL + ): new_prompt_messages.append(copy_prompt_message) elif copy_prompt_message.role == PromptMessageRole.SYSTEM: new_prompt_message = SystemPromptMessage(content=copy_prompt_message.content) @@ -461,9 +462,7 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): message_text = f"{human_prompt} {content}" elif isinstance(message, AssistantPromptMessage): message_text = f"{ai_prompt} {content}" - elif isinstance(message, SystemPromptMessage): - message_text = content - elif isinstance(message, ToolPromptMessage): + elif isinstance(message, SystemPromptMessage | ToolPromptMessage): message_text = content else: raise ValueError(f"Got unknown type {message}") diff --git a/api/core/moderation/keywords/keywords.py b/api/core/moderation/keywords/keywords.py index 17e48b8fbe..dc6a7ec564 100644 --- a/api/core/moderation/keywords/keywords.py +++ b/api/core/moderation/keywords/keywords.py @@ -56,14 +56,7 @@ class KeywordsModeration(Moderation): ) def _is_violated(self, inputs: dict, keywords_list: list) -> bool: - for value in inputs.values(): - if self._check_keywords_in_value(keywords_list, value): - return True + return any(self._check_keywords_in_value(keywords_list, value) for value in inputs.values()) - return False - - def _check_keywords_in_value(self, keywords_list, value): - for keyword in keywords_list: - if keyword.lower() in value.lower(): - return True - return False + def _check_keywords_in_value(self, keywords_list, value) -> bool: + return any(keyword.lower() in value.lower() for keyword in keywords_list) diff --git a/api/core/ops/ops_trace_manager.py b/api/core/ops/ops_trace_manager.py index d6156e479a..68fcdf32da 100644 --- a/api/core/ops/ops_trace_manager.py +++ b/api/core/ops/ops_trace_manager.py @@ -223,7 +223,7 @@ class OpsTraceManager: :return: """ # auth check - if tracing_provider not in provider_config_map.keys() and tracing_provider is not None: + if tracing_provider not in provider_config_map and tracing_provider is not None: raise ValueError(f"Invalid tracing provider: {tracing_provider}") app_config: App = db.session.query(App).filter(App.id == app_id).first() diff --git a/api/core/rag/datasource/vdb/relyt/relyt_vector.py b/api/core/rag/datasource/vdb/relyt/relyt_vector.py index 0c9d3b343d..54290eaa5d 100644 --- a/api/core/rag/datasource/vdb/relyt/relyt_vector.py +++ b/api/core/rag/datasource/vdb/relyt/relyt_vector.py @@ -127,27 +127,26 @@ class RelytVector(BaseVector): ) chunks_table_data = [] - with self.client.connect() as conn: - with conn.begin(): - for document, metadata, chunk_id, embedding in zip(texts, metadatas, ids, embeddings): - chunks_table_data.append( - { - "id": chunk_id, - "embedding": embedding, - "document": document, - "metadata": metadata, - } - ) + with self.client.connect() as conn, conn.begin(): + for document, metadata, chunk_id, embedding in zip(texts, metadatas, ids, embeddings): + chunks_table_data.append( + { + "id": chunk_id, + "embedding": embedding, + "document": document, + "metadata": metadata, + } + ) - # Execute the batch insert when the batch size is reached - if len(chunks_table_data) == 500: - conn.execute(insert(chunks_table).values(chunks_table_data)) - # Clear the chunks_table_data list for the next batch - chunks_table_data.clear() - - # Insert any remaining records that didn't make up a full batch - if chunks_table_data: + # Execute the batch insert when the batch size is reached + if len(chunks_table_data) == 500: conn.execute(insert(chunks_table).values(chunks_table_data)) + # Clear the chunks_table_data list for the next batch + chunks_table_data.clear() + + # Insert any remaining records that didn't make up a full batch + if chunks_table_data: + conn.execute(insert(chunks_table).values(chunks_table_data)) return ids @@ -186,11 +185,10 @@ class RelytVector(BaseVector): ) try: - with self.client.connect() as conn: - with conn.begin(): - delete_condition = chunks_table.c.id.in_(ids) - conn.execute(chunks_table.delete().where(delete_condition)) - return True + with self.client.connect() as conn, conn.begin(): + delete_condition = chunks_table.c.id.in_(ids) + conn.execute(chunks_table.delete().where(delete_condition)) + return True except Exception as e: print("Delete operation failed:", str(e)) return False diff --git a/api/core/rag/datasource/vdb/tencent/tencent_vector.py b/api/core/rag/datasource/vdb/tencent/tencent_vector.py index ada0c5cf46..dbedc1d4e9 100644 --- a/api/core/rag/datasource/vdb/tencent/tencent_vector.py +++ b/api/core/rag/datasource/vdb/tencent/tencent_vector.py @@ -63,10 +63,7 @@ class TencentVector(BaseVector): def _has_collection(self) -> bool: collections = self._db.list_collections() - for collection in collections: - if collection.collection_name == self._collection_name: - return True - return False + return any(collection.collection_name == self._collection_name for collection in collections) def _create_collection(self, dimension: int) -> None: lock_name = "vector_indexing_lock_{}".format(self._collection_name) diff --git a/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py b/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py index e1ac9d596c..7eaf189292 100644 --- a/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py +++ b/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py @@ -124,20 +124,19 @@ class TiDBVector(BaseVector): texts = [d.page_content for d in documents] chunks_table_data = [] - with self._engine.connect() as conn: - with conn.begin(): - for id, text, meta, embedding in zip(ids, texts, metas, embeddings): - chunks_table_data.append({"id": id, "vector": embedding, "text": text, "meta": meta}) + with self._engine.connect() as conn, conn.begin(): + for id, text, meta, embedding in zip(ids, texts, metas, embeddings): + chunks_table_data.append({"id": id, "vector": embedding, "text": text, "meta": meta}) - # Execute the batch insert when the batch size is reached - if len(chunks_table_data) == 500: - conn.execute(insert(table).values(chunks_table_data)) - # Clear the chunks_table_data list for the next batch - chunks_table_data.clear() - - # Insert any remaining records that didn't make up a full batch - if chunks_table_data: + # Execute the batch insert when the batch size is reached + if len(chunks_table_data) == 500: conn.execute(insert(table).values(chunks_table_data)) + # Clear the chunks_table_data list for the next batch + chunks_table_data.clear() + + # Insert any remaining records that didn't make up a full batch + if chunks_table_data: + conn.execute(insert(table).values(chunks_table_data)) return ids def text_exists(self, id: str) -> bool: @@ -160,11 +159,10 @@ class TiDBVector(BaseVector): raise ValueError("No ids provided to delete.") table = self._table(self._dimension) try: - with self._engine.connect() as conn: - with conn.begin(): - delete_condition = table.c.id.in_(ids) - conn.execute(table.delete().where(delete_condition)) - return True + with self._engine.connect() as conn, conn.begin(): + delete_condition = table.c.id.in_(ids) + conn.execute(table.delete().where(delete_condition)) + return True except Exception as e: print("Delete operation failed:", str(e)) return False diff --git a/api/core/rag/extractor/word_extractor.py b/api/core/rag/extractor/word_extractor.py index 2db00d161b..c6f15e55b6 100644 --- a/api/core/rag/extractor/word_extractor.py +++ b/api/core/rag/extractor/word_extractor.py @@ -48,7 +48,8 @@ class WordExtractor(BaseExtractor): raise ValueError(f"Check the url of your file; returned status code {r.status_code}") self.web_path = self.file_path - self.temp_file = tempfile.NamedTemporaryFile() + # TODO: use a better way to handle the file + self.temp_file = tempfile.NamedTemporaryFile() # noqa: SIM115 self.temp_file.write(r.content) self.file_path = self.temp_file.name elif not os.path.isfile(self.file_path): diff --git a/api/core/rag/rerank/weight_rerank.py b/api/core/rag/rerank/weight_rerank.py index 4375079ee5..16d6b879a4 100644 --- a/api/core/rag/rerank/weight_rerank.py +++ b/api/core/rag/rerank/weight_rerank.py @@ -120,8 +120,8 @@ class WeightRerankRunner: intersection = set(vec1.keys()) & set(vec2.keys()) numerator = sum(vec1[x] * vec2[x] for x in intersection) - sum1 = sum(vec1[x] ** 2 for x in vec1.keys()) - sum2 = sum(vec2[x] ** 2 for x in vec2.keys()) + sum1 = sum(vec1[x] ** 2 for x in vec1) + sum2 = sum(vec2[x] ** 2 for x in vec2) denominator = math.sqrt(sum1) * math.sqrt(sum2) if not denominator: diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index 4948ec6ba8..e4ad78ed2b 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -581,8 +581,8 @@ class DatasetRetrieval: intersection = set(vec1.keys()) & set(vec2.keys()) numerator = sum(vec1[x] * vec2[x] for x in intersection) - sum1 = sum(vec1[x] ** 2 for x in vec1.keys()) - sum2 = sum(vec2[x] ** 2 for x in vec2.keys()) + sum1 = sum(vec1[x] ** 2 for x in vec1) + sum2 = sum(vec2[x] ** 2 for x in vec2) denominator = math.sqrt(sum1) * math.sqrt(sum2) if not denominator: diff --git a/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py index 592fa230cf..71f8356ab8 100644 --- a/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py +++ b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py @@ -201,9 +201,7 @@ class ListWorksheetRecordsTool(BuiltinTool): elif value.startswith('[{"organizeId"'): value = json.loads(value) value = "、".join([item["organizeName"] for item in value]) - elif value.startswith('[{"file_id"'): - value = "" - elif value == "[]": + elif value.startswith('[{"file_id"') or value == "[]": value = "" elif hasattr(value, "accountId"): value = value["fullname"] diff --git a/api/core/tools/provider/builtin/novitaai/tools/novitaai_modelquery.py b/api/core/tools/provider/builtin/novitaai/tools/novitaai_modelquery.py index fe105f70a7..9ca14b327c 100644 --- a/api/core/tools/provider/builtin/novitaai/tools/novitaai_modelquery.py +++ b/api/core/tools/provider/builtin/novitaai/tools/novitaai_modelquery.py @@ -35,7 +35,7 @@ class NovitaAiModelQueryTool(BuiltinTool): models_data=[], headers=headers, params=params, - recursive=False if result_type == "first sd_name" or result_type == "first name sd_name pair" else True, + recursive=not (result_type == "first sd_name" or result_type == "first name sd_name pair"), ) result_str = "" diff --git a/api/core/tools/provider/builtin/qrcode/tools/qrcode_generator.py b/api/core/tools/provider/builtin/qrcode/tools/qrcode_generator.py index cac59f76d8..d8ca20bde6 100644 --- a/api/core/tools/provider/builtin/qrcode/tools/qrcode_generator.py +++ b/api/core/tools/provider/builtin/qrcode/tools/qrcode_generator.py @@ -39,7 +39,7 @@ class QRCodeGeneratorTool(BuiltinTool): # get error_correction error_correction = tool_parameters.get("error_correction", "") - if error_correction not in self.error_correction_levels.keys(): + if error_correction not in self.error_correction_levels: return self.create_text_message("Invalid parameter error_correction") try: diff --git a/api/core/tools/provider/builtin/searchapi/tools/google.py b/api/core/tools/provider/builtin/searchapi/tools/google.py index d632304a46..6d88d74635 100644 --- a/api/core/tools/provider/builtin/searchapi/tools/google.py +++ b/api/core/tools/provider/builtin/searchapi/tools/google.py @@ -44,36 +44,36 @@ class SearchAPI: @staticmethod def _process_response(res: dict, type: str) -> str: """Process response from SearchAPI.""" - if "error" in res.keys(): + if "error" in res: raise ValueError(f"Got error from SearchApi: {res['error']}") toret = "" if type == "text": - if "answer_box" in res.keys() and "answer" in res["answer_box"].keys(): + if "answer_box" in res and "answer" in res["answer_box"]: toret += res["answer_box"]["answer"] + "\n" - if "answer_box" in res.keys() and "snippet" in res["answer_box"].keys(): + if "answer_box" in res and "snippet" in res["answer_box"]: toret += res["answer_box"]["snippet"] + "\n" - if "knowledge_graph" in res.keys() and "description" in res["knowledge_graph"].keys(): + if "knowledge_graph" in res and "description" in res["knowledge_graph"]: toret += res["knowledge_graph"]["description"] + "\n" - if "organic_results" in res.keys() and "snippet" in res["organic_results"][0].keys(): + if "organic_results" in res and "snippet" in res["organic_results"][0]: for item in res["organic_results"]: toret += "content: " + item["snippet"] + "\n" + "link: " + item["link"] + "\n" if toret == "": toret = "No good search result found" elif type == "link": - if "answer_box" in res.keys() and "organic_result" in res["answer_box"].keys(): - if "title" in res["answer_box"]["organic_result"].keys(): + if "answer_box" in res and "organic_result" in res["answer_box"]: + if "title" in res["answer_box"]["organic_result"]: toret = f"[{res['answer_box']['organic_result']['title']}]({res['answer_box']['organic_result']['link']})\n" - elif "organic_results" in res.keys() and "link" in res["organic_results"][0].keys(): + elif "organic_results" in res and "link" in res["organic_results"][0]: toret = "" for item in res["organic_results"]: toret += f"[{item['title']}]({item['link']})\n" - elif "related_questions" in res.keys() and "link" in res["related_questions"][0].keys(): + elif "related_questions" in res and "link" in res["related_questions"][0]: toret = "" for item in res["related_questions"]: toret += f"[{item['title']}]({item['link']})\n" - elif "related_searches" in res.keys() and "link" in res["related_searches"][0].keys(): + elif "related_searches" in res and "link" in res["related_searches"][0]: toret = "" for item in res["related_searches"]: toret += f"[{item['title']}]({item['link']})\n" diff --git a/api/core/tools/provider/builtin/searchapi/tools/google_jobs.py b/api/core/tools/provider/builtin/searchapi/tools/google_jobs.py index 1544061c08..d29cb0ae3f 100644 --- a/api/core/tools/provider/builtin/searchapi/tools/google_jobs.py +++ b/api/core/tools/provider/builtin/searchapi/tools/google_jobs.py @@ -44,12 +44,12 @@ class SearchAPI: @staticmethod def _process_response(res: dict, type: str) -> str: """Process response from SearchAPI.""" - if "error" in res.keys(): + if "error" in res: raise ValueError(f"Got error from SearchApi: {res['error']}") toret = "" if type == "text": - if "jobs" in res.keys() and "title" in res["jobs"][0].keys(): + if "jobs" in res and "title" in res["jobs"][0]: for item in res["jobs"]: toret += ( "title: " @@ -65,7 +65,7 @@ class SearchAPI: toret = "No good search result found" elif type == "link": - if "jobs" in res.keys() and "apply_link" in res["jobs"][0].keys(): + if "jobs" in res and "apply_link" in res["jobs"][0]: for item in res["jobs"]: toret += f"[{item['title']} - {item['company_name']}]({item['apply_link']})\n" else: diff --git a/api/core/tools/provider/builtin/searchapi/tools/google_news.py b/api/core/tools/provider/builtin/searchapi/tools/google_news.py index 95a7aad736..8458c8c958 100644 --- a/api/core/tools/provider/builtin/searchapi/tools/google_news.py +++ b/api/core/tools/provider/builtin/searchapi/tools/google_news.py @@ -44,25 +44,25 @@ class SearchAPI: @staticmethod def _process_response(res: dict, type: str) -> str: """Process response from SearchAPI.""" - if "error" in res.keys(): + if "error" in res: raise ValueError(f"Got error from SearchApi: {res['error']}") toret = "" if type == "text": - if "organic_results" in res.keys() and "snippet" in res["organic_results"][0].keys(): + if "organic_results" in res and "snippet" in res["organic_results"][0]: for item in res["organic_results"]: toret += "content: " + item["snippet"] + "\n" + "link: " + item["link"] + "\n" - if "top_stories" in res.keys() and "title" in res["top_stories"][0].keys(): + if "top_stories" in res and "title" in res["top_stories"][0]: for item in res["top_stories"]: toret += "title: " + item["title"] + "\n" + "link: " + item["link"] + "\n" if toret == "": toret = "No good search result found" elif type == "link": - if "organic_results" in res.keys() and "title" in res["organic_results"][0].keys(): + if "organic_results" in res and "title" in res["organic_results"][0]: for item in res["organic_results"]: toret += f"[{item['title']}]({item['link']})\n" - elif "top_stories" in res.keys() and "title" in res["top_stories"][0].keys(): + elif "top_stories" in res and "title" in res["top_stories"][0]: for item in res["top_stories"]: toret += f"[{item['title']}]({item['link']})\n" else: diff --git a/api/core/tools/provider/builtin/searchapi/tools/youtube_transcripts.py b/api/core/tools/provider/builtin/searchapi/tools/youtube_transcripts.py index 88def504fc..d7bfb53bd7 100644 --- a/api/core/tools/provider/builtin/searchapi/tools/youtube_transcripts.py +++ b/api/core/tools/provider/builtin/searchapi/tools/youtube_transcripts.py @@ -44,11 +44,11 @@ class SearchAPI: @staticmethod def _process_response(res: dict) -> str: """Process response from SearchAPI.""" - if "error" in res.keys(): + if "error" in res: raise ValueError(f"Got error from SearchApi: {res['error']}") toret = "" - if "transcripts" in res.keys() and "text" in res["transcripts"][0].keys(): + if "transcripts" in res and "text" in res["transcripts"][0]: for item in res["transcripts"]: toret += item["text"] + " " if toret == "": diff --git a/api/core/tools/provider/builtin/stability/tools/text2image.py b/api/core/tools/provider/builtin/stability/tools/text2image.py index 12b6cc3352..9f415ceb55 100644 --- a/api/core/tools/provider/builtin/stability/tools/text2image.py +++ b/api/core/tools/provider/builtin/stability/tools/text2image.py @@ -35,7 +35,7 @@ class StableDiffusionTool(BuiltinTool, BaseStabilityAuthorization): if model in ["sd3", "sd3-turbo"]: payload["model"] = tool_parameters.get("model") - if not model == "sd3-turbo": + if model != "sd3-turbo": payload["negative_prompt"] = tool_parameters.get("negative_prompt", "") response = post( diff --git a/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py b/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py index 46137886bd..344f916494 100644 --- a/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py +++ b/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py @@ -206,10 +206,9 @@ class StableDiffusionTool(BuiltinTool): # Convert image to RGB and save as PNG try: - with Image.open(io.BytesIO(image_binary)) as image: - with io.BytesIO() as buffer: - image.convert("RGB").save(buffer, format="PNG") - image_binary = buffer.getvalue() + with Image.open(io.BytesIO(image_binary)) as image, io.BytesIO() as buffer: + image.convert("RGB").save(buffer, format="PNG") + image_binary = buffer.getvalue() except Exception as e: return self.create_text_message(f"Failed to process the image: {str(e)}") diff --git a/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py b/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py index 67efcf0954..cb88e9519a 100644 --- a/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py +++ b/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py @@ -27,7 +27,7 @@ class WikipediaAPIWrapper: self.doc_content_chars_max = doc_content_chars_max def run(self, query: str, lang: str = "") -> str: - if lang in wikipedia.languages().keys(): + if lang in wikipedia.languages(): self.lang = lang wikipedia.set_lang(self.lang) diff --git a/api/core/tools/utils/message_transformer.py b/api/core/tools/utils/message_transformer.py index c4983ebc65..1109ed7df2 100644 --- a/api/core/tools/utils/message_transformer.py +++ b/api/core/tools/utils/message_transformer.py @@ -19,9 +19,7 @@ class ToolFileMessageTransformer: result = [] for message in messages: - if message.type == ToolInvokeMessage.MessageType.TEXT: - result.append(message) - elif message.type == ToolInvokeMessage.MessageType.LINK: + if message.type == ToolInvokeMessage.MessageType.TEXT or message.type == ToolInvokeMessage.MessageType.LINK: result.append(message) elif message.type == ToolInvokeMessage.MessageType.IMAGE: # try to download image diff --git a/api/core/workflow/graph_engine/entities/graph.py b/api/core/workflow/graph_engine/entities/graph.py index f1f677b8c1..c156dd8c98 100644 --- a/api/core/workflow/graph_engine/entities/graph.py +++ b/api/core/workflow/graph_engine/entities/graph.py @@ -224,9 +224,7 @@ class Graph(BaseModel): """ leaf_node_ids = [] for node_id in self.node_ids: - if node_id not in self.edge_mapping: - leaf_node_ids.append(node_id) - elif ( + if node_id not in self.edge_mapping or ( len(self.edge_mapping[node_id]) == 1 and self.edge_mapping[node_id][0].target_node_id == self.root_node_id ): diff --git a/api/core/workflow/nodes/answer/answer_stream_generate_router.py b/api/core/workflow/nodes/answer/answer_stream_generate_router.py index 06050e1549..e31a1479a8 100644 --- a/api/core/workflow/nodes/answer/answer_stream_generate_router.py +++ b/api/core/workflow/nodes/answer/answer_stream_generate_router.py @@ -24,7 +24,7 @@ class AnswerStreamGeneratorRouter: # parse stream output node value selectors of answer nodes answer_generate_route: dict[str, list[GenerateRouteChunk]] = {} for answer_node_id, node_config in node_id_config_mapping.items(): - if not node_config.get("data", {}).get("type") == NodeType.ANSWER.value: + if node_config.get("data", {}).get("type") != NodeType.ANSWER.value: continue # get generate route for stream output diff --git a/api/core/workflow/nodes/end/end_stream_generate_router.py b/api/core/workflow/nodes/end/end_stream_generate_router.py index 30ce8fe018..a38d982393 100644 --- a/api/core/workflow/nodes/end/end_stream_generate_router.py +++ b/api/core/workflow/nodes/end/end_stream_generate_router.py @@ -17,7 +17,7 @@ class EndStreamGeneratorRouter: # parse stream output node value selector of end nodes end_stream_variable_selectors_mapping: dict[str, list[list[str]]] = {} for end_node_id, node_config in node_id_config_mapping.items(): - if not node_config.get("data", {}).get("type") == NodeType.END.value: + if node_config.get("data", {}).get("type") != NodeType.END.value: continue # skip end node in parallel diff --git a/api/core/workflow/nodes/tool/entities.py b/api/core/workflow/nodes/tool/entities.py index 28fbf789fd..9d222b10b9 100644 --- a/api/core/workflow/nodes/tool/entities.py +++ b/api/core/workflow/nodes/tool/entities.py @@ -20,7 +20,7 @@ class ToolEntity(BaseModel): if not isinstance(value, dict): raise ValueError("tool_configurations must be a dictionary") - for key in values.data.get("tool_configurations", {}).keys(): + for key in values.data.get("tool_configurations", {}): value = values.data.get("tool_configurations", {}).get(key) if not isinstance(value, str | int | float | bool): raise ValueError(f"{key} must be a string") diff --git a/api/pyproject.toml b/api/pyproject.toml index 23e2b5c549..3d100ebc58 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -17,14 +17,12 @@ select = [ "F", # pyflakes rules "I", # isort rules "N", # pep8-naming - "UP", # pyupgrade rules "RUF019", # unnecessary-key-check "RUF100", # unused-noqa "RUF101", # redirected-noqa "S506", # unsafe-yaml-load - "SIM116", # if-else-block-instead-of-dict-lookup - "SIM401", # if-else-block-instead-of-dict-get - "SIM910", # dict-get-with-none-default + "SIM", # flake8-simplify rules + "UP", # pyupgrade rules "W191", # tab-indentation "W605", # invalid-escape-sequence ] @@ -50,6 +48,15 @@ ignore = [ "B905", # zip-without-explicit-strict "N806", # non-lowercase-variable-in-function "N815", # mixed-case-variable-in-class-scope + "SIM102", # collapsible-if + "SIM103", # needless-bool + "SIM105", # suppressible-exception + "SIM107", # return-in-try-except-finally + "SIM108", # if-else-block-instead-of-if-exp + "SIM113", # eumerate-for-loop + "SIM117", # multiple-with-statements + "SIM210", # if-expr-with-true-false + "SIM300", # yoda-conditions ] [tool.ruff.lint.per-file-ignores] diff --git a/api/services/file_service.py b/api/services/file_service.py index 5780abb2be..bedec76334 100644 --- a/api/services/file_service.py +++ b/api/services/file_service.py @@ -56,9 +56,7 @@ class FileService: if etl_type == "Unstructured" else ALLOWED_EXTENSIONS + IMAGE_EXTENSIONS ) - if extension.lower() not in allowed_extensions: - raise UnsupportedFileTypeError() - elif only_image and extension.lower() not in IMAGE_EXTENSIONS: + if extension.lower() not in allowed_extensions or only_image and extension.lower() not in IMAGE_EXTENSIONS: raise UnsupportedFileTypeError() # read file content diff --git a/api/services/ops_service.py b/api/services/ops_service.py index 1e7935d299..d8e2b1689a 100644 --- a/api/services/ops_service.py +++ b/api/services/ops_service.py @@ -54,7 +54,7 @@ class OpsService: :param tracing_config: tracing config :return: """ - if tracing_provider not in provider_config_map.keys() and tracing_provider: + if tracing_provider not in provider_config_map and tracing_provider: return {"error": f"Invalid tracing provider: {tracing_provider}"} config_class, other_keys = ( @@ -113,7 +113,7 @@ class OpsService: :param tracing_config: tracing config :return: """ - if tracing_provider not in provider_config_map.keys(): + if tracing_provider not in provider_config_map: raise ValueError(f"Invalid tracing provider: {tracing_provider}") # check if trace config already exists From 56c90e212a1b3df7a17a6f13ce60a5940bebed16 Mon Sep 17 00:00:00 2001 From: takatost Date: Thu, 12 Sep 2024 13:59:48 +0800 Subject: [PATCH 3/5] fix(workflow): missing content in the answer node stream output during iterations (#8292) Co-authored-by: -LAN- --- api/core/rag/extractor/word_extractor.py | 4 ++-- api/core/workflow/nodes/iteration/iteration_node.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/api/core/rag/extractor/word_extractor.py b/api/core/rag/extractor/word_extractor.py index c6f15e55b6..947644a90b 100644 --- a/api/core/rag/extractor/word_extractor.py +++ b/api/core/rag/extractor/word_extractor.py @@ -7,8 +7,8 @@ import os import re import tempfile import uuid -import xml.etree.ElementTree as ET from urllib.parse import urlparse +from xml.etree import ElementTree import requests from docx import Document as DocxDocument @@ -218,7 +218,7 @@ class WordExtractor(BaseExtractor): hyperlinks_url = None if "HYPERLINK" in run.element.xml: try: - xml = ET.XML(run.element.xml) + xml = ElementTree.XML(run.element.xml) x_child = [c for c in xml.iter() if c is not None] for x in x_child: if x_child is None: diff --git a/api/core/workflow/nodes/iteration/iteration_node.py b/api/core/workflow/nodes/iteration/iteration_node.py index 77b14e36a1..4d944e93db 100644 --- a/api/core/workflow/nodes/iteration/iteration_node.py +++ b/api/core/workflow/nodes/iteration/iteration_node.py @@ -16,6 +16,7 @@ from core.workflow.graph_engine.entities.event import ( IterationRunNextEvent, IterationRunStartedEvent, IterationRunSucceededEvent, + NodeRunStreamChunkEvent, NodeRunSucceededEvent, ) from core.workflow.graph_engine.entities.graph import Graph @@ -154,7 +155,11 @@ class IterationNode(BaseNode): if isinstance(event, (BaseNodeEvent | BaseParallelBranchEvent)) and not event.in_iteration_id: event.in_iteration_id = self.node_id - if isinstance(event, BaseNodeEvent) and event.node_type == NodeType.ITERATION_START: + if ( + isinstance(event, BaseNodeEvent) + and event.node_type == NodeType.ITERATION_START + and not isinstance(event, NodeRunStreamChunkEvent) + ): continue if isinstance(event, NodeRunSucceededEvent): From c69f5b07ba241e6ef08c7028c12b1125c93c1d01 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Thu, 12 Sep 2024 14:00:36 +0800 Subject: [PATCH 4/5] chore: apply ruff E501 line-too-long linter rule (#8275) Co-authored-by: -LAN- --- api/commands.py | 3 +- api/controllers/console/app/statistic.py | 170 +++++++++++------- .../console/app/workflow_statistic.py | 102 ++++++----- api/controllers/console/wraps.py | 3 +- api/controllers/web/wraps.py | 3 +- api/core/agent/prompt/template.py | 6 +- .../app/apps/workflow_logging_callback.py | 19 +- api/core/file/message_file_parser.py | 3 +- .../helper/code_executor/code_executor.py | 3 +- api/core/helper/tool_parameter_cache.py | 5 +- api/core/llm_generator/prompts.py | 19 +- api/core/model_runtime/entities/defaults.py | 16 +- .../model_providers/__base/ai_model.py | 4 +- .../__base/large_language_model.py | 5 +- .../model_providers/anthropic/llm/llm.py | 2 +- .../model_providers/azure_openai/_constant.py | 69 +++---- .../baichuan/llm/baichuan_tokenizer.py | 3 +- .../model_providers/bedrock/llm/llm.py | 6 +- .../model_providers/google/llm/llm.py | 2 +- .../huggingface_tei/tei_helper.py | 3 +- .../model_providers/hunyuan/llm/llm.py | 3 +- .../model_providers/nvidia/llm/llm.py | 3 +- .../model_providers/oci/llm/llm.py | 3 +- .../oci/text_embedding/text_embedding.py | 3 +- .../model_providers/ollama/llm/llm.py | 7 +- .../model_providers/openai/llm/llm.py | 2 +- .../openai_api_compatible/llm/llm.py | 6 +- .../model_providers/spark/llm/_client.py | 5 +- .../model_providers/upstage/llm/llm.py | 2 +- .../model_providers/vertex_ai/llm/llm.py | 6 +- .../model_providers/wenxin/llm/llm.py | 2 +- .../model_providers/xinference/llm/llm.py | 9 +- .../xinference/speech2text/speech2text.py | 10 +- .../xinference/xinference_helper.py | 6 +- .../model_providers/zhipuai/llm/llm.py | 2 +- .../zhipuai/zhipuai_sdk/core/_base_type.py | 3 +- .../schema_validators/common_validator.py | 3 +- .../advanced_prompt_templates.py | 8 +- .../rag/datasource/vdb/oracle/oraclevector.py | 6 +- .../rag/datasource/vdb/pgvector/pgvector.py | 3 +- api/core/rag/extractor/extract_processor.py | 5 +- .../router/multi_dataset_react_route.py | 9 +- api/core/tools/entities/tool_bundle.py | 3 +- api/core/tools/entities/values.py | 32 ++-- .../provider/builtin/aippt/tools/aippt.py | 3 +- .../builtin/arxiv/tools/arxiv_search.py | 3 +- .../builtin/aws/tools/apply_guardrail.py | 3 +- .../builtin/devdocs/tools/searchDevDocs.py | 3 +- .../builtin/gitlab/tools/gitlab_files.py | 5 +- .../google_translate/tools/translate.py | 3 +- .../builtin/hap/tools/get_worksheet_fields.py | 3 +- .../hap/tools/list_worksheet_records.py | 5 +- .../builtin/searchapi/tools/google.py | 5 +- .../stablediffusion/tools/stable_diffusion.py | 21 ++- .../trello/tools/create_list_on_board.py | 3 +- .../trello/tools/create_new_card_on_board.py | 3 +- .../builtin/trello/tools/delete_board.py | 3 +- .../builtin/trello/tools/delete_card.py | 3 +- .../builtin/trello/tools/get_board_actions.py | 3 +- .../builtin/trello/tools/get_board_by_id.py | 3 +- .../builtin/trello/tools/get_board_cards.py | 3 +- .../trello/tools/get_filterd_board_cards.py | 3 +- .../trello/tools/get_lists_on_board.py | 3 +- .../builtin/trello/tools/update_board.py | 3 +- .../builtin/trello/tools/update_card.py | 3 +- .../builtin/twilio/tools/send_message.py | 3 +- .../builtin/vectorizer/tools/test_data.py | 2 +- api/core/tools/tool_engine.py | 5 +- api/core/tools/utils/feishu_api_utils.py | 2 +- api/core/tools/utils/message_transformer.py | 2 +- api/core/tools/utils/parser.py | 6 +- api/core/tools/utils/web_reader_tool.py | 3 +- api/core/workflow/nodes/code/code_node.py | 18 +- .../knowledge_retrieval_node.py | 7 +- .../nodes/parameter_extractor/prompts.py | 12 +- .../question_classifier/template_prompts.py | 8 +- api/libs/gmpy2_pkcs10aep_cipher.py | 3 +- api/models/provider.py | 5 +- api/models/tools.py | 3 +- api/models/workflow.py | 3 +- api/poetry.lock | 2 +- api/pyproject.toml | 14 +- .../tools/api_tools_manage_service.py | 3 +- .../model_runtime/__mock/openai_embeddings.py | 2 +- .../core/app/segments/test_segment.py | 6 +- 85 files changed, 459 insertions(+), 324 deletions(-) diff --git a/api/commands.py b/api/commands.py index 3bf8bc0ecc..db96fbae46 100644 --- a/api/commands.py +++ b/api/commands.py @@ -411,7 +411,8 @@ def migrate_knowledge_vector_database(): try: click.echo( click.style( - f"Start to created vector index with {len(documents)} documents of {segments_count} segments for dataset {dataset.id}.", + f"Start to created vector index with {len(documents)} documents of {segments_count}" + f" segments for dataset {dataset.id}.", fg="green", ) ) diff --git a/api/controllers/console/app/statistic.py b/api/controllers/console/app/statistic.py index 4806b02b55..3ef442812d 100644 --- a/api/controllers/console/app/statistic.py +++ b/api/controllers/console/app/statistic.py @@ -29,10 +29,13 @@ class DailyMessageStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """ - SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(*) AS message_count - FROM messages where app_id = :app_id - """ + sql_query = """SELECT + DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + COUNT(*) AS message_count +FROM + messages +WHERE + app_id = :app_id""" arg_dict = {"tz": account.timezone, "app_id": app_model.id} timezone = pytz.timezone(account.timezone) @@ -45,7 +48,7 @@ class DailyMessageStatistic(Resource): start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at >= :start" + sql_query += " AND created_at >= :start" arg_dict["start"] = start_datetime_utc if args["end"]: @@ -55,10 +58,10 @@ class DailyMessageStatistic(Resource): end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at < :end" + sql_query += " AND created_at < :end" arg_dict["end"] = end_datetime_utc - sql_query += " GROUP BY date order by date" + sql_query += " GROUP BY date ORDER BY date" response_data = [] @@ -83,10 +86,13 @@ class DailyConversationStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """ - SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(distinct messages.conversation_id) AS conversation_count - FROM messages where app_id = :app_id - """ + sql_query = """SELECT + DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + COUNT(DISTINCT messages.conversation_id) AS conversation_count +FROM + messages +WHERE + app_id = :app_id""" arg_dict = {"tz": account.timezone, "app_id": app_model.id} timezone = pytz.timezone(account.timezone) @@ -99,7 +105,7 @@ class DailyConversationStatistic(Resource): start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at >= :start" + sql_query += " AND created_at >= :start" arg_dict["start"] = start_datetime_utc if args["end"]: @@ -109,10 +115,10 @@ class DailyConversationStatistic(Resource): end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at < :end" + sql_query += " AND created_at < :end" arg_dict["end"] = end_datetime_utc - sql_query += " GROUP BY date order by date" + sql_query += " GROUP BY date ORDER BY date" response_data = [] @@ -137,10 +143,13 @@ class DailyTerminalsStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """ - SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(distinct messages.from_end_user_id) AS terminal_count - FROM messages where app_id = :app_id - """ + sql_query = """SELECT + DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + COUNT(DISTINCT messages.from_end_user_id) AS terminal_count +FROM + messages +WHERE + app_id = :app_id""" arg_dict = {"tz": account.timezone, "app_id": app_model.id} timezone = pytz.timezone(account.timezone) @@ -153,7 +162,7 @@ class DailyTerminalsStatistic(Resource): start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at >= :start" + sql_query += " AND created_at >= :start" arg_dict["start"] = start_datetime_utc if args["end"]: @@ -163,10 +172,10 @@ class DailyTerminalsStatistic(Resource): end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at < :end" + sql_query += " AND created_at < :end" arg_dict["end"] = end_datetime_utc - sql_query += " GROUP BY date order by date" + sql_query += " GROUP BY date ORDER BY date" response_data = [] @@ -191,12 +200,14 @@ class DailyTokenCostStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """ - SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, - (sum(messages.message_tokens) + sum(messages.answer_tokens)) as token_count, - sum(total_price) as total_price - FROM messages where app_id = :app_id - """ + sql_query = """SELECT + DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + (SUM(messages.message_tokens) + SUM(messages.answer_tokens)) AS token_count, + SUM(total_price) AS total_price +FROM + messages +WHERE + app_id = :app_id""" arg_dict = {"tz": account.timezone, "app_id": app_model.id} timezone = pytz.timezone(account.timezone) @@ -209,7 +220,7 @@ class DailyTokenCostStatistic(Resource): start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at >= :start" + sql_query += " AND created_at >= :start" arg_dict["start"] = start_datetime_utc if args["end"]: @@ -219,10 +230,10 @@ class DailyTokenCostStatistic(Resource): end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at < :end" + sql_query += " AND created_at < :end" arg_dict["end"] = end_datetime_utc - sql_query += " GROUP BY date order by date" + sql_query += " GROUP BY date ORDER BY date" response_data = [] @@ -249,12 +260,22 @@ class AverageSessionInteractionStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """SELECT date(DATE_TRUNC('day', c.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, -AVG(subquery.message_count) AS interactions -FROM (SELECT m.conversation_id, COUNT(m.id) AS message_count - FROM conversations c - JOIN messages m ON c.id = m.conversation_id - WHERE c.override_model_configs IS NULL AND c.app_id = :app_id""" + sql_query = """SELECT + DATE(DATE_TRUNC('day', c.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + AVG(subquery.message_count) AS interactions +FROM + ( + SELECT + m.conversation_id, + COUNT(m.id) AS message_count + FROM + conversations c + JOIN + messages m + ON c.id = m.conversation_id + WHERE + c.override_model_configs IS NULL + AND c.app_id = :app_id""" arg_dict = {"tz": account.timezone, "app_id": app_model.id} timezone = pytz.timezone(account.timezone) @@ -267,7 +288,7 @@ FROM (SELECT m.conversation_id, COUNT(m.id) AS message_count start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - sql_query += " and c.created_at >= :start" + sql_query += " AND c.created_at >= :start" arg_dict["start"] = start_datetime_utc if args["end"]: @@ -277,14 +298,19 @@ FROM (SELECT m.conversation_id, COUNT(m.id) AS message_count end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query += " and c.created_at < :end" + sql_query += " AND c.created_at < :end" arg_dict["end"] = end_datetime_utc sql_query += """ - GROUP BY m.conversation_id) subquery -LEFT JOIN conversations c on c.id=subquery.conversation_id -GROUP BY date -ORDER BY date""" + GROUP BY m.conversation_id + ) subquery +LEFT JOIN + conversations c + ON c.id = subquery.conversation_id +GROUP BY + date +ORDER BY + date""" response_data = [] @@ -311,13 +337,17 @@ class UserSatisfactionRateStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """ - SELECT date(DATE_TRUNC('day', m.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, - COUNT(m.id) as message_count, COUNT(mf.id) as feedback_count - FROM messages m - LEFT JOIN message_feedbacks mf on mf.message_id=m.id and mf.rating='like' - WHERE m.app_id = :app_id - """ + sql_query = """SELECT + DATE(DATE_TRUNC('day', m.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + COUNT(m.id) AS message_count, + COUNT(mf.id) AS feedback_count +FROM + messages m +LEFT JOIN + message_feedbacks mf + ON mf.message_id=m.id AND mf.rating='like' +WHERE + m.app_id = :app_id""" arg_dict = {"tz": account.timezone, "app_id": app_model.id} timezone = pytz.timezone(account.timezone) @@ -330,7 +360,7 @@ class UserSatisfactionRateStatistic(Resource): start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - sql_query += " and m.created_at >= :start" + sql_query += " AND m.created_at >= :start" arg_dict["start"] = start_datetime_utc if args["end"]: @@ -340,10 +370,10 @@ class UserSatisfactionRateStatistic(Resource): end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query += " and m.created_at < :end" + sql_query += " AND m.created_at < :end" arg_dict["end"] = end_datetime_utc - sql_query += " GROUP BY date order by date" + sql_query += " GROUP BY date ORDER BY date" response_data = [] @@ -373,12 +403,13 @@ class AverageResponseTimeStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """ - SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, - AVG(provider_response_latency) as latency - FROM messages - WHERE app_id = :app_id - """ + sql_query = """SELECT + DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + AVG(provider_response_latency) AS latency +FROM + messages +WHERE + app_id = :app_id""" arg_dict = {"tz": account.timezone, "app_id": app_model.id} timezone = pytz.timezone(account.timezone) @@ -391,7 +422,7 @@ class AverageResponseTimeStatistic(Resource): start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at >= :start" + sql_query += " AND created_at >= :start" arg_dict["start"] = start_datetime_utc if args["end"]: @@ -401,10 +432,10 @@ class AverageResponseTimeStatistic(Resource): end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at < :end" + sql_query += " AND created_at < :end" arg_dict["end"] = end_datetime_utc - sql_query += " GROUP BY date order by date" + sql_query += " GROUP BY date ORDER BY date" response_data = [] @@ -429,13 +460,16 @@ class TokensPerSecondStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, - CASE + sql_query = """SELECT + DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + CASE WHEN SUM(provider_response_latency) = 0 THEN 0 ELSE (SUM(answer_tokens) / SUM(provider_response_latency)) END as tokens_per_second -FROM messages -WHERE app_id = :app_id""" +FROM + messages +WHERE + app_id = :app_id""" arg_dict = {"tz": account.timezone, "app_id": app_model.id} timezone = pytz.timezone(account.timezone) @@ -448,7 +482,7 @@ WHERE app_id = :app_id""" start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at >= :start" + sql_query += " AND created_at >= :start" arg_dict["start"] = start_datetime_utc if args["end"]: @@ -458,10 +492,10 @@ WHERE app_id = :app_id""" end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at < :end" + sql_query += " AND created_at < :end" arg_dict["end"] = end_datetime_utc - sql_query += " GROUP BY date order by date" + sql_query += " GROUP BY date ORDER BY date" response_data = [] diff --git a/api/controllers/console/app/workflow_statistic.py b/api/controllers/console/app/workflow_statistic.py index 942271a634..c7e54f2be0 100644 --- a/api/controllers/console/app/workflow_statistic.py +++ b/api/controllers/console/app/workflow_statistic.py @@ -30,12 +30,14 @@ class WorkflowDailyRunsStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """ - SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(id) AS runs - FROM workflow_runs - WHERE app_id = :app_id - AND triggered_from = :triggered_from - """ + sql_query = """SELECT + DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + COUNT(id) AS runs +FROM + workflow_runs +WHERE + app_id = :app_id + AND triggered_from = :triggered_from""" arg_dict = { "tz": account.timezone, "app_id": app_model.id, @@ -52,7 +54,7 @@ class WorkflowDailyRunsStatistic(Resource): start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at >= :start" + sql_query += " AND created_at >= :start" arg_dict["start"] = start_datetime_utc if args["end"]: @@ -62,10 +64,10 @@ class WorkflowDailyRunsStatistic(Resource): end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at < :end" + sql_query += " AND created_at < :end" arg_dict["end"] = end_datetime_utc - sql_query += " GROUP BY date order by date" + sql_query += " GROUP BY date ORDER BY date" response_data = [] @@ -90,12 +92,14 @@ class WorkflowDailyTerminalsStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """ - SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(distinct workflow_runs.created_by) AS terminal_count - FROM workflow_runs - WHERE app_id = :app_id - AND triggered_from = :triggered_from - """ + sql_query = """SELECT + DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + COUNT(DISTINCT workflow_runs.created_by) AS terminal_count +FROM + workflow_runs +WHERE + app_id = :app_id + AND triggered_from = :triggered_from""" arg_dict = { "tz": account.timezone, "app_id": app_model.id, @@ -112,7 +116,7 @@ class WorkflowDailyTerminalsStatistic(Resource): start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at >= :start" + sql_query += " AND created_at >= :start" arg_dict["start"] = start_datetime_utc if args["end"]: @@ -122,10 +126,10 @@ class WorkflowDailyTerminalsStatistic(Resource): end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at < :end" + sql_query += " AND created_at < :end" arg_dict["end"] = end_datetime_utc - sql_query += " GROUP BY date order by date" + sql_query += " GROUP BY date ORDER BY date" response_data = [] @@ -150,14 +154,14 @@ class WorkflowDailyTokenCostStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """ - SELECT - date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, - SUM(workflow_runs.total_tokens) as token_count - FROM workflow_runs - WHERE app_id = :app_id - AND triggered_from = :triggered_from - """ + sql_query = """SELECT + DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + SUM(workflow_runs.total_tokens) AS token_count +FROM + workflow_runs +WHERE + app_id = :app_id + AND triggered_from = :triggered_from""" arg_dict = { "tz": account.timezone, "app_id": app_model.id, @@ -174,7 +178,7 @@ class WorkflowDailyTokenCostStatistic(Resource): start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at >= :start" + sql_query += " AND created_at >= :start" arg_dict["start"] = start_datetime_utc if args["end"]: @@ -184,10 +188,10 @@ class WorkflowDailyTokenCostStatistic(Resource): end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query += " and created_at < :end" + sql_query += " AND created_at < :end" arg_dict["end"] = end_datetime_utc - sql_query += " GROUP BY date order by date" + sql_query += " GROUP BY date ORDER BY date" response_data = [] @@ -217,23 +221,27 @@ class WorkflowAverageAppInteractionStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """ - SELECT - AVG(sub.interactions) as interactions, - sub.date - FROM - (SELECT - date(DATE_TRUNC('day', c.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, - c.created_by, - COUNT(c.id) AS interactions - FROM workflow_runs c - WHERE c.app_id = :app_id - AND c.triggered_from = :triggered_from - {{start}} - {{end}} - GROUP BY date, c.created_by) sub - GROUP BY sub.date - """ + sql_query = """SELECT + AVG(sub.interactions) AS interactions, + sub.date +FROM + ( + SELECT + DATE(DATE_TRUNC('day', c.created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, + c.created_by, + COUNT(c.id) AS interactions + FROM + workflow_runs c + WHERE + c.app_id = :app_id + AND c.triggered_from = :triggered_from + {{start}} + {{end}} + GROUP BY + date, c.created_by + ) sub +GROUP BY + sub.date""" arg_dict = { "tz": account.timezone, "app_id": app_model.id, @@ -262,7 +270,7 @@ class WorkflowAverageAppInteractionStatistic(Resource): end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) - sql_query = sql_query.replace("{{end}}", " and c.created_at < :end") + sql_query = sql_query.replace("{{end}}", " AND c.created_at < :end") arg_dict["end"] = end_datetime_utc else: sql_query = sql_query.replace("{{end}}", "") diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index 7667b30e34..46223d104f 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -64,7 +64,8 @@ def cloud_edition_billing_resource_check(resource: str): elif resource == "vector_space" and 0 < vector_space.limit <= vector_space.size: abort(403, "The capacity of the vector space has reached the limit of your subscription.") elif resource == "documents" and 0 < documents_upload_quota.limit <= documents_upload_quota.size: - # The api of file upload is used in the multiple places, so we need to check the source of the request from datasets + # The api of file upload is used in the multiple places, + # so we need to check the source of the request from datasets source = request.args.get("source") if source == "datasets": abort(403, "The number of documents has reached the limit of your subscription.") diff --git a/api/controllers/web/wraps.py b/api/controllers/web/wraps.py index 93dc691d62..c327c3df18 100644 --- a/api/controllers/web/wraps.py +++ b/api/controllers/web/wraps.py @@ -80,7 +80,8 @@ def _validate_web_sso_token(decoded, system_features, app_code): if not source or source != "sso": raise WebSSOAuthRequiredError() - # Check if SSO is not enforced for web, and if the token source is SSO, raise an error and redirect to normal passport login + # Check if SSO is not enforced for web, and if the token source is SSO, + # raise an error and redirect to normal passport login if not system_features.sso_enforced_for_web or not app_web_sso_enabled: source = decoded.get("token_source") if source and source == "sso": diff --git a/api/core/agent/prompt/template.py b/api/core/agent/prompt/template.py index cb98f5501d..ef64fd29fc 100644 --- a/api/core/agent/prompt/template.py +++ b/api/core/agent/prompt/template.py @@ -41,7 +41,8 @@ Begin! Reminder to ALWAYS respond with a valid json blob of a single action. Use {{historic_messages}} Question: {{query}} {{agent_scratchpad}} -Thought:""" +Thought:""" # noqa: E501 + ENGLISH_REACT_COMPLETION_AGENT_SCRATCHPAD_TEMPLATES = """Observation: {{observation}} Thought:""" @@ -86,7 +87,8 @@ Action: ``` Begin! Reminder to ALWAYS respond with a valid json blob of a single action. Use tools if necessary. Respond directly if appropriate. Format is Action:```$JSON_BLOB```then Observation:. -""" +""" # noqa: E501 + ENGLISH_REACT_CHAT_AGENT_SCRATCHPAD_TEMPLATES = "" diff --git a/api/core/app/apps/workflow_logging_callback.py b/api/core/app/apps/workflow_logging_callback.py index cdd21bf7c2..388cb83180 100644 --- a/api/core/app/apps/workflow_logging_callback.py +++ b/api/core/app/apps/workflow_logging_callback.py @@ -84,10 +84,12 @@ class WorkflowLoggingCallback(WorkflowCallback): if route_node_state.node_run_result: node_run_result = route_node_state.node_run_result self.print_text( - f"Inputs: {jsonable_encoder(node_run_result.inputs) if node_run_result.inputs else ''}", color="green" + f"Inputs: " f"{jsonable_encoder(node_run_result.inputs) if node_run_result.inputs else ''}", + color="green", ) self.print_text( - f"Process Data: {jsonable_encoder(node_run_result.process_data) if node_run_result.process_data else ''}", + f"Process Data: " + f"{jsonable_encoder(node_run_result.process_data) if node_run_result.process_data else ''}", color="green", ) self.print_text( @@ -114,14 +116,17 @@ class WorkflowLoggingCallback(WorkflowCallback): node_run_result = route_node_state.node_run_result self.print_text(f"Error: {node_run_result.error}", color="red") self.print_text( - f"Inputs: {jsonable_encoder(node_run_result.inputs) if node_run_result.inputs else ''}", color="red" - ) - self.print_text( - f"Process Data: {jsonable_encoder(node_run_result.process_data) if node_run_result.process_data else ''}", + f"Inputs: " f"" f"{jsonable_encoder(node_run_result.inputs) if node_run_result.inputs else ''}", color="red", ) self.print_text( - f"Outputs: {jsonable_encoder(node_run_result.outputs) if node_run_result.outputs else ''}", color="red" + f"Process Data: " + f"{jsonable_encoder(node_run_result.process_data) if node_run_result.process_data else ''}", + color="red", + ) + self.print_text( + f"Outputs: " f"{jsonable_encoder(node_run_result.outputs) if node_run_result.outputs else ''}", + color="red", ) def on_node_text_chunk(self, event: NodeRunStreamChunkEvent) -> None: diff --git a/api/core/file/message_file_parser.py b/api/core/file/message_file_parser.py index 8feaabedbb..83059b216e 100644 --- a/api/core/file/message_file_parser.py +++ b/api/core/file/message_file_parser.py @@ -188,7 +188,8 @@ class MessageFileParser: def _check_image_remote_url(self, url): try: headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)" + " Chrome/91.0.4472.124 Safari/537.36" } def is_s3_presigned_url(url): diff --git a/api/core/helper/code_executor/code_executor.py b/api/core/helper/code_executor/code_executor.py index 7ee6e63817..4932284540 100644 --- a/api/core/helper/code_executor/code_executor.py +++ b/api/core/helper/code_executor/code_executor.py @@ -89,7 +89,8 @@ class CodeExecutor: raise CodeExecutionError("Code execution service is unavailable") elif response.status_code != 200: raise Exception( - f"Failed to execute code, got status code {response.status_code}, please check if the sandbox service is running" + f"Failed to execute code, got status code {response.status_code}," + f" please check if the sandbox service is running" ) except CodeExecutionError as e: raise e diff --git a/api/core/helper/tool_parameter_cache.py b/api/core/helper/tool_parameter_cache.py index 4c3b736186..e848b46c56 100644 --- a/api/core/helper/tool_parameter_cache.py +++ b/api/core/helper/tool_parameter_cache.py @@ -14,7 +14,10 @@ class ToolParameterCache: def __init__( self, tenant_id: str, provider: str, tool_name: str, cache_type: ToolParameterCacheType, identity_id: str ): - self.cache_key = f"{cache_type.value}_secret:tenant_id:{tenant_id}:provider:{provider}:tool_name:{tool_name}:identity_id:{identity_id}" + self.cache_key = ( + f"{cache_type.value}_secret:tenant_id:{tenant_id}:provider:{provider}:tool_name:{tool_name}" + f":identity_id:{identity_id}" + ) def get(self) -> Optional[dict]: """ diff --git a/api/core/llm_generator/prompts.py b/api/core/llm_generator/prompts.py index 7ab257872f..c40b6d1808 100644 --- a/api/core/llm_generator/prompts.py +++ b/api/core/llm_generator/prompts.py @@ -59,24 +59,27 @@ User Input: yo, 你今天咋样？ } User Input: -""" +""" # noqa: E501 SUGGESTED_QUESTIONS_AFTER_ANSWER_INSTRUCTION_PROMPT = ( "Please help me predict the three most likely questions that human would ask, " "and keeping each question under 20 characters.\n" - "MAKE SURE your output is the SAME language as the Assistant's latest response(if the main response is written in Chinese, then the language of your output must be using Chinese.)!\n" + "MAKE SURE your output is the SAME language as the Assistant's latest response" + "(if the main response is written in Chinese, then the language of your output must be using Chinese.)!\n" "The output must be an array in JSON format following the specified schema:\n" '["question1","question2","question3"]\n' ) GENERATOR_QA_PROMPT = ( - " The user will send a long text. Generate a Question and Answer pairs only using the knowledge in the long text. Please think step by step." + " The user will send a long text. Generate a Question and Answer pairs only using the knowledge" + " in the long text. Please think step by step." "Step 1: Understand and summarize the main content of this text.\n" "Step 2: What key information or concepts are mentioned in this text?\n" "Step 3: Decompose or combine multiple pieces of information and concepts.\n" "Step 4: Generate questions and answers based on these key information and concepts.\n" " The questions should be clear and detailed, and the answers should be detailed and complete. " - "You must answer in {language}, in a style that is clear and detailed in {language}. No language other than {language} should be used. \n" + "You must answer in {language}, in a style that is clear and detailed in {language}." + " No language other than {language} should be used. \n" " Use the following format: Q1:\nA1:\nQ2:\nA2:...\n" "" ) @@ -94,7 +97,7 @@ Based on task description, please create a well-structured prompt template that - Use the same language as task description. - Output in ``` xml ``` and start with Please generate the full prompt template with at least 300 words and output only the prompt template. -""" +""" # noqa: E501 RULE_CONFIG_PROMPT_GENERATE_TEMPLATE = """ Here is a task description for which I would like you to create a high-quality prompt template for: @@ -109,7 +112,7 @@ Based on task description, please create a well-structured prompt template that - Use the same language as task description. - Output in ``` xml ``` and start with Please generate the full prompt template and output only the prompt template. -""" +""" # noqa: E501 RULE_CONFIG_PARAMETER_GENERATE_TEMPLATE = """ I need to extract the following information from the input text. The tag specifies the 'type', 'description' and 'required' of the information to be extracted. @@ -134,7 +137,7 @@ Inside XML tags, there is a text that I should extract parameters ### Answer I should always output a valid list. Output nothing other than the list of variable_name. Output an empty list if there is no variable name in input text. -""" +""" # noqa: E501 RULE_CONFIG_STATEMENT_GENERATE_TEMPLATE = """ @@ -150,4 +153,4 @@ Welcome! I'm here to assist you with any questions or issues you might have with Here is the task description: {{INPUT_TEXT}} You just need to generate the output -""" +""" # noqa: E501 diff --git a/api/core/model_runtime/entities/defaults.py b/api/core/model_runtime/entities/defaults.py index e94be6f918..4d0c9aa08f 100644 --- a/api/core/model_runtime/entities/defaults.py +++ b/api/core/model_runtime/entities/defaults.py @@ -8,8 +8,11 @@ PARAMETER_RULE_TEMPLATE: dict[DefaultParameterName, dict] = { }, "type": "float", "help": { - "en_US": "Controls randomness. Lower temperature results in less random completions. As the temperature approaches zero, the model will become deterministic and repetitive. Higher temperature results in more random completions.", - "zh_Hans": "温度控制随机性。较低的温度会导致较少的随机完成。随着温度接近零，模型将变得确定性和重复性。较高的温度会导致更多的随机完成。", + "en_US": "Controls randomness. Lower temperature results in less random completions." + " As the temperature approaches zero, the model will become deterministic and repetitive." + " Higher temperature results in more random completions.", + "zh_Hans": "温度控制随机性。较低的温度会导致较少的随机完成。随着温度接近零，模型将变得确定性和重复性。" + "较高的温度会导致更多的随机完成。", }, "required": False, "default": 0.0, @@ -24,7 +27,8 @@ PARAMETER_RULE_TEMPLATE: dict[DefaultParameterName, dict] = { }, "type": "float", "help": { - "en_US": "Controls diversity via nucleus sampling: 0.5 means half of all likelihood-weighted options are considered.", + "en_US": "Controls diversity via nucleus sampling: 0.5 means half of all likelihood-weighted options" + " are considered.", "zh_Hans": "通过核心采样控制多样性：0.5表示考虑了一半的所有可能性加权选项。", }, "required": False, @@ -88,7 +92,8 @@ PARAMETER_RULE_TEMPLATE: dict[DefaultParameterName, dict] = { }, "type": "int", "help": { - "en_US": "Specifies the upper limit on the length of generated results. If the generated results are truncated, you can increase this parameter.", + "en_US": "Specifies the upper limit on the length of generated results." + " If the generated results are truncated, you can increase this parameter.", "zh_Hans": "指定生成结果长度的上限。如果生成结果截断，可以调大该参数。", }, "required": False, @@ -104,7 +109,8 @@ PARAMETER_RULE_TEMPLATE: dict[DefaultParameterName, dict] = { }, "type": "string", "help": { - "en_US": "Set a response format, ensure the output from llm is a valid code block as possible, such as JSON, XML, etc.", + "en_US": "Set a response format, ensure the output from llm is a valid code block as possible," + " such as JSON, XML, etc.", "zh_Hans": "设置一个返回格式，确保llm的输出尽可能是有效的代码块，如JSON、XML等", }, "required": False, diff --git a/api/core/model_runtime/model_providers/__base/ai_model.py b/api/core/model_runtime/model_providers/__base/ai_model.py index 09d2d7e54d..e7e343f00d 100644 --- a/api/core/model_runtime/model_providers/__base/ai_model.py +++ b/api/core/model_runtime/model_providers/__base/ai_model.py @@ -72,7 +72,9 @@ class AIModel(ABC): if isinstance(error, tuple(model_errors)): if invoke_error == InvokeAuthorizationError: return invoke_error( - description=f"[{provider_name}] Incorrect model credentials provided, please check and try again. " + description=( + f"[{provider_name}] Incorrect model credentials provided, please check and try again." + ) ) return invoke_error(description=f"[{provider_name}] {invoke_error.description}, {str(error)}") diff --git a/api/core/model_runtime/model_providers/__base/large_language_model.py b/api/core/model_runtime/model_providers/__base/large_language_model.py index 5c39186e65..e8789ec7df 100644 --- a/api/core/model_runtime/model_providers/__base/large_language_model.py +++ b/api/core/model_runtime/model_providers/__base/large_language_model.py @@ -187,7 +187,7 @@ if you are not sure about the structure. {{instructions}} -""" +""" # noqa: E501 code_block = model_parameters.get("response_format", "") if not code_block: @@ -830,7 +830,8 @@ if you are not sure about the structure. else: if parameter_value != round(parameter_value, parameter_rule.precision): raise ValueError( - f"Model Parameter {parameter_name} should be round to {parameter_rule.precision} decimal places." + f"Model Parameter {parameter_name} should be round to {parameter_rule.precision}" + f" decimal places." ) # validate parameter value range diff --git a/api/core/model_runtime/model_providers/anthropic/llm/llm.py b/api/core/model_runtime/model_providers/anthropic/llm/llm.py index 30e9d2e9f2..0cb66842e7 100644 --- a/api/core/model_runtime/model_providers/anthropic/llm/llm.py +++ b/api/core/model_runtime/model_providers/anthropic/llm/llm.py @@ -51,7 +51,7 @@ if you are not sure about the structure. {{instructions}} -""" +""" # noqa: E501 class AnthropicLargeLanguageModel(LargeLanguageModel): diff --git a/api/core/model_runtime/model_providers/azure_openai/_constant.py b/api/core/model_runtime/model_providers/azure_openai/_constant.py index c2744691c3..0dada70cc5 100644 --- a/api/core/model_runtime/model_providers/azure_openai/_constant.py +++ b/api/core/model_runtime/model_providers/azure_openai/_constant.py @@ -16,6 +16,15 @@ from core.model_runtime.entities.model_entities import ( AZURE_OPENAI_API_VERSION = "2024-02-15-preview" +AZURE_DEFAULT_PARAM_SEED_HELP = I18nObject( + zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，" + "您应该参考 system_fingerprint 响应参数来监视变化。", + en_US="If specified, model will make a best effort to sample deterministically," + " such that repeated requests with the same seed and parameters should return the same result." + " Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter" + " to monitor changes in the backend.", +) + def _get_max_tokens(default: int, min_val: int, max_val: int) -> ParameterRule: rule = ParameterRule( @@ -229,10 +238,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, @@ -297,10 +303,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, @@ -365,10 +368,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, @@ -433,10 +433,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, @@ -502,10 +499,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, @@ -571,10 +565,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, @@ -650,10 +641,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, @@ -719,10 +707,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, @@ -788,10 +773,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, @@ -867,10 +849,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, @@ -936,10 +915,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, @@ -1000,10 +976,7 @@ LLM_BASE_MODELS = [ name="seed", label=I18nObject(zh_Hans="种子", en_US="Seed"), type="int", - help=I18nObject( - zh_Hans="如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。", - en_US="If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.", - ), + help=AZURE_DEFAULT_PARAM_SEED_HELP, required=False, precision=2, min=0, diff --git a/api/core/model_runtime/model_providers/baichuan/llm/baichuan_tokenizer.py b/api/core/model_runtime/model_providers/baichuan/llm/baichuan_tokenizer.py index bea6777f83..a7ca28d49d 100644 --- a/api/core/model_runtime/model_providers/baichuan/llm/baichuan_tokenizer.py +++ b/api/core/model_runtime/model_providers/baichuan/llm/baichuan_tokenizer.py @@ -15,6 +15,7 @@ class BaichuanTokenizer: @classmethod def _get_num_tokens(cls, text: str) -> int: - # tokens = number of Chinese characters + number of English words * 1.3 (for estimation only, subject to actual return) + # tokens = number of Chinese characters + number of English words * 1.3 + # (for estimation only, subject to actual return) # https://platform.baichuan-ai.com/docs/text-Embedding return int(cls.count_chinese_characters(text) + cls.count_english_vocabularies(text) * 1.3) diff --git a/api/core/model_runtime/model_providers/bedrock/llm/llm.py b/api/core/model_runtime/model_providers/bedrock/llm/llm.py index e07f2a419a..239ae52b4c 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/llm.py +++ b/api/core/model_runtime/model_providers/bedrock/llm/llm.py @@ -52,7 +52,7 @@ if you are not sure about the structure. {{instructions}} -""" +""" # noqa: E501 class BedrockLargeLanguageModel(LargeLanguageModel): @@ -541,7 +541,9 @@ class BedrockLargeLanguageModel(LargeLanguageModel): "max_tokens": 32, } elif "ai21" in model: - # ValidationException: Malformed input request: #/temperature: expected type: Number, found: Null#/maxTokens: expected type: Integer, found: Null#/topP: expected type: Number, found: Null, please reformat your input and try again. + # ValidationException: Malformed input request: #/temperature: expected type: Number, + # found: Null#/maxTokens: expected type: Integer, found: Null#/topP: expected type: Number, found: Null, + # please reformat your input and try again. required_params = { "temperature": 0.7, "topP": 0.9, diff --git a/api/core/model_runtime/model_providers/google/llm/llm.py b/api/core/model_runtime/model_providers/google/llm/llm.py index 307c15e1fd..b10d0edba3 100644 --- a/api/core/model_runtime/model_providers/google/llm/llm.py +++ b/api/core/model_runtime/model_providers/google/llm/llm.py @@ -45,7 +45,7 @@ if you are not sure about the structure. {{instructions}} -""" +""" # noqa: E501 class GoogleLargeLanguageModel(LargeLanguageModel): diff --git a/api/core/model_runtime/model_providers/huggingface_tei/tei_helper.py b/api/core/model_runtime/model_providers/huggingface_tei/tei_helper.py index 56c51e8888..288637495f 100644 --- a/api/core/model_runtime/model_providers/huggingface_tei/tei_helper.py +++ b/api/core/model_runtime/model_providers/huggingface_tei/tei_helper.py @@ -54,7 +54,8 @@ class TeiHelper: url = str(URL(server_url) / "info") - # this method is surrounded by a lock, and default requests may hang forever, so we just set a Adapter with max_retries=3 + # this method is surrounded by a lock, and default requests may hang forever, + # so we just set a Adapter with max_retries=3 session = Session() session.mount("http://", HTTPAdapter(max_retries=3)) session.mount("https://", HTTPAdapter(max_retries=3)) diff --git a/api/core/model_runtime/model_providers/hunyuan/llm/llm.py b/api/core/model_runtime/model_providers/hunyuan/llm/llm.py index c056ab7a08..b57e5e1c2b 100644 --- a/api/core/model_runtime/model_providers/hunyuan/llm/llm.py +++ b/api/core/model_runtime/model_providers/hunyuan/llm/llm.py @@ -131,7 +131,8 @@ class HunyuanLargeLanguageModel(LargeLanguageModel): { "Role": message.role.value, # fix set content = "" while tool_call request - # fix [hunyuan] None, [TencentCloudSDKException] code:InvalidParameter message:Messages Content and Contents not allowed empty at the same time. + # fix [hunyuan] None, [TencentCloudSDKException] code:InvalidParameter + # message:Messages Content and Contents not allowed empty at the same time. "Content": " ", # message.content if (message.content is not None) else "", "ToolCalls": dict_tool_calls, } diff --git a/api/core/model_runtime/model_providers/nvidia/llm/llm.py b/api/core/model_runtime/model_providers/nvidia/llm/llm.py index 4d3747dc84..1c98c6be6c 100644 --- a/api/core/model_runtime/model_providers/nvidia/llm/llm.py +++ b/api/core/model_runtime/model_providers/nvidia/llm/llm.py @@ -93,7 +93,8 @@ class NVIDIALargeLanguageModel(OAIAPICompatLargeLanguageModel): def _validate_credentials(self, model: str, credentials: dict) -> None: """ - Validate model credentials using requests to ensure compatibility with all providers following OpenAI's API standard. + Validate model credentials using requests to ensure compatibility with all providers following + OpenAI's API standard. :param model: model name :param credentials: model credentials diff --git a/api/core/model_runtime/model_providers/oci/llm/llm.py b/api/core/model_runtime/model_providers/oci/llm/llm.py index 51b634c6cf..1e1fc5b3ea 100644 --- a/api/core/model_runtime/model_providers/oci/llm/llm.py +++ b/api/core/model_runtime/model_providers/oci/llm/llm.py @@ -239,7 +239,8 @@ class OCILargeLanguageModel(LargeLanguageModel): config_items = oci_config_content.split("/") if len(config_items) != 5: raise CredentialsValidateFailedError( - "oci_config_content should be base64.b64encode('user_ocid/fingerprint/tenancy_ocid/region/compartment_ocid'.encode('utf-8'))" + "oci_config_content should be base64.b64encode(" + "'user_ocid/fingerprint/tenancy_ocid/region/compartment_ocid'.encode('utf-8'))" ) oci_config["user"] = config_items[0] oci_config["fingerprint"] = config_items[1] diff --git a/api/core/model_runtime/model_providers/oci/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/oci/text_embedding/text_embedding.py index df77db47d9..80ad2be9f5 100644 --- a/api/core/model_runtime/model_providers/oci/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/oci/text_embedding/text_embedding.py @@ -146,7 +146,8 @@ class OCITextEmbeddingModel(TextEmbeddingModel): config_items = oci_config_content.split("/") if len(config_items) != 5: raise CredentialsValidateFailedError( - "oci_config_content should be base64.b64encode('user_ocid/fingerprint/tenancy_ocid/region/compartment_ocid'.encode('utf-8'))" + "oci_config_content should be base64.b64encode(" + "'user_ocid/fingerprint/tenancy_ocid/region/compartment_ocid'.encode('utf-8'))" ) oci_config["user"] = config_items[0] oci_config["fingerprint"] = config_items[1] diff --git a/api/core/model_runtime/model_providers/ollama/llm/llm.py b/api/core/model_runtime/model_providers/ollama/llm/llm.py index 160eea0148..3f32f454e4 100644 --- a/api/core/model_runtime/model_providers/ollama/llm/llm.py +++ b/api/core/model_runtime/model_providers/ollama/llm/llm.py @@ -639,9 +639,10 @@ class OllamaLargeLanguageModel(LargeLanguageModel): type=ParameterType.STRING, help=I18nObject( en_US="Sets how long the model is kept in memory after generating a response. " - "This must be a duration string with a unit (e.g., '10m' for 10 minutes or '24h' for 24 hours). " - "A negative number keeps the model loaded indefinitely, and '0' unloads the model immediately after generating a response. " - "Valid time units are 's','m','h'. (Default: 5m)" + "This must be a duration string with a unit (e.g., '10m' for 10 minutes or '24h' for 24 hours)." + " A negative number keeps the model loaded indefinitely, and '0' unloads the model" + " immediately after generating a response." + " Valid time units are 's','m','h'. (Default: 5m)" ), ), ParameterRule( diff --git a/api/core/model_runtime/model_providers/openai/llm/llm.py b/api/core/model_runtime/model_providers/openai/llm/llm.py index 5950b77a96..578687b5d3 100644 --- a/api/core/model_runtime/model_providers/openai/llm/llm.py +++ b/api/core/model_runtime/model_providers/openai/llm/llm.py @@ -37,7 +37,7 @@ if you are not sure about the structure. {{instructions}} -""" +""" # noqa: E501 class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): diff --git a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py index 24317b488c..41ca163a92 100644 --- a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py +++ b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py @@ -103,7 +103,8 @@ class OAIAPICompatLargeLanguageModel(_CommonOaiApiCompat, LargeLanguageModel): def validate_credentials(self, model: str, credentials: dict) -> None: """ - Validate model credentials using requests to ensure compatibility with all providers following OpenAI's API standard. + Validate model credentials using requests to ensure compatibility with all providers following + OpenAI's API standard. :param model: model name :param credentials: model credentials @@ -262,7 +263,8 @@ class OAIAPICompatLargeLanguageModel(_CommonOaiApiCompat, LargeLanguageModel): return entity - # validate_credentials method has been rewritten to use the requests library for compatibility with all providers following OpenAI's API standard. + # validate_credentials method has been rewritten to use the requests library for compatibility with all providers + # following OpenAI's API standard. def _generate( self, model: str, diff --git a/api/core/model_runtime/model_providers/spark/llm/_client.py b/api/core/model_runtime/model_providers/spark/llm/_client.py index 25223e8340..b99a657e71 100644 --- a/api/core/model_runtime/model_providers/spark/llm/_client.py +++ b/api/core/model_runtime/model_providers/spark/llm/_client.py @@ -61,7 +61,10 @@ class SparkLLMClient: signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding="utf-8") - authorization_origin = f'api_key="{api_key}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha_base64}"' + authorization_origin = ( + f'api_key="{api_key}", algorithm="hmac-sha256", headers="host date request-line",' + f' signature="{signature_sha_base64}"' + ) authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(encoding="utf-8") diff --git a/api/core/model_runtime/model_providers/upstage/llm/llm.py b/api/core/model_runtime/model_providers/upstage/llm/llm.py index 1014b53f39..9646e209b2 100644 --- a/api/core/model_runtime/model_providers/upstage/llm/llm.py +++ b/api/core/model_runtime/model_providers/upstage/llm/llm.py @@ -34,7 +34,7 @@ if you are not sure about the structure. {{instructions}} -""" +""" # noqa: E501 class UpstageLargeLanguageModel(_CommonUpstage, LargeLanguageModel): diff --git a/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py b/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py index 110028a288..1b9931d2c3 100644 --- a/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py +++ b/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py @@ -114,7 +114,8 @@ class VertexAiLargeLanguageModel(LargeLanguageModel): credentials.refresh(request) token = credentials.token - # Vertex AI Anthropic Claude3 Opus model available in us-east5 region, Sonnet and Haiku available in us-central1 region + # Vertex AI Anthropic Claude3 Opus model available in us-east5 region, Sonnet and Haiku available + # in us-central1 region if "opus" in model or "claude-3-5-sonnet" in model: location = "us-east5" else: @@ -123,7 +124,8 @@ class VertexAiLargeLanguageModel(LargeLanguageModel): # use access token to authenticate if token: client = AnthropicVertex(region=location, project_id=project_id, access_token=token) - # When access token is empty, try to use the Google Cloud VM's built-in service account or the GOOGLE_APPLICATION_CREDENTIALS environment variable + # When access token is empty, try to use the Google Cloud VM's built-in service account + # or the GOOGLE_APPLICATION_CREDENTIALS environment variable else: client = AnthropicVertex( region=location, diff --git a/api/core/model_runtime/model_providers/wenxin/llm/llm.py b/api/core/model_runtime/model_providers/wenxin/llm/llm.py index 1ff0ac7ad2..8feedbfe55 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/llm.py +++ b/api/core/model_runtime/model_providers/wenxin/llm/llm.py @@ -28,7 +28,7 @@ if you are not sure about the structure. You should also complete the text started with ``` but not tell ``` directly. -""" +""" # noqa: E501 class ErnieBotLargeLanguageModel(LargeLanguageModel): diff --git a/api/core/model_runtime/model_providers/xinference/llm/llm.py b/api/core/model_runtime/model_providers/xinference/llm/llm.py index bc7531ee20..7ad236880b 100644 --- a/api/core/model_runtime/model_providers/xinference/llm/llm.py +++ b/api/core/model_runtime/model_providers/xinference/llm/llm.py @@ -130,7 +130,8 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): credentials["completion_type"] = "completion" else: raise ValueError( - f"xinference model ability {extra_param.model_ability} is not supported, check if you have the right model type" + f"xinference model ability {extra_param.model_ability} is not supported," + f" check if you have the right model type" ) if extra_param.support_function_call: @@ -358,7 +359,8 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): help=I18nObject( en_US="Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they " "appear in the text so far, increasing the model's likelihood to talk about new topics.", - zh_Hans="介于 -2.0 和 2.0 之间的数字。正值会根据新词是否已出现在文本中对其进行惩罚，从而增加模型谈论新话题的可能性。", + zh_Hans="介于 -2.0 和 2.0 之间的数字。正值会根据新词是否已出现在文本中对其进行惩罚，" + "从而增加模型谈论新话题的可能性。", ), default=0.0, min=-2.0, @@ -378,7 +380,8 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): en_US="Number between -2.0 and 2.0. Positive values penalize new tokens based on their " "existing frequency in the text so far, decreasing the model's likelihood to repeat the " "same line verbatim.", - zh_Hans="介于 -2.0 和 2.0 之间的数字。正值会根据新词在文本中的现有频率对其进行惩罚，从而降低模型逐字重复相同内容的可能性。", + zh_Hans="介于 -2.0 和 2.0 之间的数字。正值会根据新词在文本中的现有频率对其进行惩罚，" + "从而降低模型逐字重复相同内容的可能性。", ), default=0.0, min=-2.0, diff --git a/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py b/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py index 54c8b51654..18efde758c 100644 --- a/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py +++ b/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py @@ -101,12 +101,16 @@ class XinferenceSpeech2TextModel(Speech2TextModel): :param model: model name :param credentials: model credentials - :param file: The audio file object (not file name) to transcribe, in one of these formats: flac, mp3, mp4, mpe g,mpga, m4a, ogg, wav, or webm. + :param file: The audio file object (not file name) to transcribe, in one of these formats: flac, mp3, mp4, mpeg, + mpga, m4a, ogg, wav, or webm. :param language: The language of the input audio. Supplying the input language in ISO-639-1 :param prompt: An optional text to guide the model's style or continue a previous audio segment. The prompt should match the audio language. - :param response_format: The format of the transcript output, in one of these options: json, text, srt, verbose _json, or vtt. - :param temperature: The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output mor e random,while lower values like 0.2 will make it more focused and deterministic.If set to 0, the model wi ll use log probability to automatically increase the temperature until certain thresholds are hit. + :param response_format: The format of the transcript output, in one of these options: json, text, srt, + verbose_json, or vtt. + :param temperature: The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more + random,while lower values like 0.2 will make it more focused and deterministic.If set to 0, the model will use + log probability to automatically increase the temperature until certain thresholds are hit. :return: text for given audio file """ server_url = credentials["server_url"] diff --git a/api/core/model_runtime/model_providers/xinference/xinference_helper.py b/api/core/model_runtime/model_providers/xinference/xinference_helper.py index 6ad10e690d..1e05da9c56 100644 --- a/api/core/model_runtime/model_providers/xinference/xinference_helper.py +++ b/api/core/model_runtime/model_providers/xinference/xinference_helper.py @@ -76,7 +76,8 @@ class XinferenceHelper: url = str(URL(server_url) / "v1" / "models" / model_uid) - # this method is surrounded by a lock, and default requests may hang forever, so we just set a Adapter with max_retries=3 + # this method is surrounded by a lock, and default requests may hang forever, + # so we just set a Adapter with max_retries=3 session = Session() session.mount("http://", HTTPAdapter(max_retries=3)) session.mount("https://", HTTPAdapter(max_retries=3)) @@ -88,7 +89,8 @@ class XinferenceHelper: raise RuntimeError(f"get xinference model extra parameter failed, url: {url}, error: {e}") if response.status_code != 200: raise RuntimeError( - f"get xinference model extra parameter failed, status code: {response.status_code}, response: {response.text}" + f"get xinference model extra parameter failed, status code: {response.status_code}," + f" response: {response.text}" ) response_json = response.json() diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py index 498962bd0f..29b873fd06 100644 --- a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py +++ b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py @@ -31,7 +31,7 @@ And you should always end the block with a "```" to indicate the end of the JSON {{instructions}} -```JSON""" +```JSON""" # noqa: E501 class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py index b7cf6bb7fd..7a91f9b796 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py @@ -75,7 +75,8 @@ Headers = Mapping[str, Union[str, Omit]] ResponseT = TypeVar( "ResponseT", - bound="Union[str, None, BaseModel, list[Any], Dict[str, Any], Response, UnknownResponse, ModelBuilderProtocol, BinaryResponseContent]", + bound="Union[str, None, BaseModel, list[Any], Dict[str, Any], Response, UnknownResponse, ModelBuilderProtocol," + " BinaryResponseContent]", ) # for user input files diff --git a/api/core/model_runtime/schema_validators/common_validator.py b/api/core/model_runtime/schema_validators/common_validator.py index e4f3541475..c05edb72e3 100644 --- a/api/core/model_runtime/schema_validators/common_validator.py +++ b/api/core/model_runtime/schema_validators/common_validator.py @@ -67,7 +67,8 @@ class CommonValidator: if credential_form_schema.max_length: if len(value) > credential_form_schema.max_length: raise ValueError( - f"Variable {credential_form_schema.variable} length should not greater than {credential_form_schema.max_length}" + f"Variable {credential_form_schema.variable} length should not" + f" greater than {credential_form_schema.max_length}" ) # check the type of value diff --git a/api/core/prompt/prompt_templates/advanced_prompt_templates.py b/api/core/prompt/prompt_templates/advanced_prompt_templates.py index e4b3a61cb4..0ab7f526cc 100644 --- a/api/core/prompt/prompt_templates/advanced_prompt_templates.py +++ b/api/core/prompt/prompt_templates/advanced_prompt_templates.py @@ -1,11 +1,11 @@ -CONTEXT = "Use the following context as your learned knowledge, inside XML tags.\n\n\n{{#context#}}\n\n\nWhen answer to user:\n- If you don't know, just say that you don't know.\n- If you don't know when you are not sure, ask for clarification.\nAvoid mentioning that you obtained the information from the context.\nAnd answer according to the language of the user's question.\n" +CONTEXT = "Use the following context as your learned knowledge, inside XML tags.\n\n\n{{#context#}}\n\n\nWhen answer to user:\n- If you don't know, just say that you don't know.\n- If you don't know when you are not sure, ask for clarification.\nAvoid mentioning that you obtained the information from the context.\nAnd answer according to the language of the user's question.\n" # noqa: E501 -BAICHUAN_CONTEXT = "用户在与一个客观的助手对话。助手会尊重找到的材料，给出全面专业的解释，但不会过度演绎。同时回答中不会暴露引用的材料：\n\n```\n{{#context#}}\n```\n\n" +BAICHUAN_CONTEXT = "用户在与一个客观的助手对话。助手会尊重找到的材料，给出全面专业的解释，但不会过度演绎。同时回答中不会暴露引用的材料：\n\n```\n{{#context#}}\n```\n\n" # noqa: E501 CHAT_APP_COMPLETION_PROMPT_CONFIG = { "completion_prompt_config": { "prompt": { - "text": "{{#pre_prompt#}}\nHere is the chat histories between human and assistant, inside XML tags.\n\n\n{{#histories#}}\n\n\n\nHuman: {{#query#}}\n\nAssistant: " + "text": "{{#pre_prompt#}}\nHere is the chat histories between human and assistant, inside XML tags.\n\n\n{{#histories#}}\n\n\n\nHuman: {{#query#}}\n\nAssistant: " # noqa: E501 }, "conversation_histories_role": {"user_prefix": "Human", "assistant_prefix": "Assistant"}, }, @@ -24,7 +24,7 @@ COMPLETION_APP_COMPLETION_PROMPT_CONFIG = { BAICHUAN_CHAT_APP_COMPLETION_PROMPT_CONFIG = { "completion_prompt_config": { "prompt": { - "text": "{{#pre_prompt#}}\n\n用户和助手的历史对话内容如下：\n```\n{{#histories#}}\n```\n\n\n\n用户：{{#query#}}" + "text": "{{#pre_prompt#}}\n\n用户和助手的历史对话内容如下：\n```\n{{#histories#}}\n```\n\n\n\n用户：{{#query#}}" # noqa: E501 }, "conversation_histories_role": {"user_prefix": "用户", "assistant_prefix": "助手"}, }, diff --git a/api/core/rag/datasource/vdb/oracle/oraclevector.py b/api/core/rag/datasource/vdb/oracle/oraclevector.py index 06c20ceb5f..d223b0decf 100644 --- a/api/core/rag/datasource/vdb/oracle/oraclevector.py +++ b/api/core/rag/datasource/vdb/oracle/oraclevector.py @@ -195,7 +195,8 @@ class OracleVector(BaseVector): top_k = kwargs.get("top_k", 5) with self._get_cursor() as cur: cur.execute( - f"SELECT meta, text, vector_distance(embedding,:1) AS distance FROM {self.table_name} ORDER BY distance fetch first {top_k} rows only", + f"SELECT meta, text, vector_distance(embedding,:1) AS distance FROM {self.table_name}" + f" ORDER BY distance fetch first {top_k} rows only", [numpy.array(query_vector)], ) docs = [] @@ -254,7 +255,8 @@ class OracleVector(BaseVector): entities.append(token) with self._get_cursor() as cur: cur.execute( - f"select meta, text, embedding FROM {self.table_name} WHERE CONTAINS(text, :1, 1) > 0 order by score(1) desc fetch first {top_k} rows only", + f"select meta, text, embedding FROM {self.table_name}" + f" WHERE CONTAINS(text, :1, 1) > 0 order by score(1) desc fetch first {top_k} rows only", [" ACCUM ".join(entities)], ) docs = [] diff --git a/api/core/rag/datasource/vdb/pgvector/pgvector.py b/api/core/rag/datasource/vdb/pgvector/pgvector.py index 38dfd24b56..d2d9e5238b 100644 --- a/api/core/rag/datasource/vdb/pgvector/pgvector.py +++ b/api/core/rag/datasource/vdb/pgvector/pgvector.py @@ -139,7 +139,8 @@ class PGVector(BaseVector): with self._get_cursor() as cur: cur.execute( - f"SELECT meta, text, embedding <=> %s AS distance FROM {self.table_name} ORDER BY distance LIMIT {top_k}", + f"SELECT meta, text, embedding <=> %s AS distance FROM {self.table_name}" + f" ORDER BY distance LIMIT {top_k}", (json.dumps(query_vector),), ) docs = [] diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py index a00b3cba53..244ef9614a 100644 --- a/api/core/rag/extractor/extract_processor.py +++ b/api/core/rag/extractor/extract_processor.py @@ -30,7 +30,10 @@ from extensions.ext_storage import storage from models.model import UploadFile SUPPORT_URL_CONTENT_TYPES = ["application/pdf", "text/plain", "application/json"] -USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" +USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124" + " Safari/537.36" +) class ExtractProcessor: diff --git a/api/core/rag/retrieval/router/multi_dataset_react_route.py b/api/core/rag/retrieval/router/multi_dataset_react_route.py index 33841cac06..a0494adc60 100644 --- a/api/core/rag/retrieval/router/multi_dataset_react_route.py +++ b/api/core/rag/retrieval/router/multi_dataset_react_route.py @@ -14,7 +14,7 @@ from core.workflow.nodes.llm.llm_node import LLMNode PREFIX = """Respond to the human as helpfully and accurately as possible. You have access to the following tools:""" SUFFIX = """Begin! Reminder to ALWAYS respond with a valid json blob of a single action. Use tools if necessary. Respond directly if appropriate. Format is Action:```$JSON_BLOB```then Observation:. -Thought:""" +Thought:""" # noqa: E501 FORMAT_INSTRUCTIONS = """Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input). The nouns in the format of "Thought", "Action", "Action Input", "Final Answer" must be expressed in English. @@ -46,7 +46,7 @@ Action: "action": "Final Answer", "action_input": "Final response to human" }} -```""" +```""" # noqa: E501 class ReactMultiDatasetRouter: @@ -204,7 +204,8 @@ class ReactMultiDatasetRouter: tool_strings = [] for tool in tools: tool_strings.append( - f"{tool.name}: {tool.description}, args: {{'query': {{'title': 'Query', 'description': 'Query for the dataset to be used to retrieve the dataset.', 'type': 'string'}}}}" + f"{tool.name}: {tool.description}, args: {{'query': {{'title': 'Query'," + f" 'description': 'Query for the dataset to be used to retrieve the dataset.', 'type': 'string'}}}}" ) formatted_tools = "\n".join(tool_strings) unique_tool_names = {tool.name for tool in tools} @@ -236,7 +237,7 @@ class ReactMultiDatasetRouter: suffix = """Begin! Reminder to ALWAYS respond with a valid json blob of a single action. Use tools if necessary. Respond directly if appropriate. Format is Action:```$JSON_BLOB```then Observation:. Question: {input} Thought: {agent_scratchpad} -""" +""" # noqa: E501 tool_strings = "\n".join([f"{tool.name}: {tool.description}" for tool in tools]) tool_names = ", ".join([tool.name for tool in tools]) diff --git a/api/core/tools/entities/tool_bundle.py b/api/core/tools/entities/tool_bundle.py index da6201c5aa..0c15b2a371 100644 --- a/api/core/tools/entities/tool_bundle.py +++ b/api/core/tools/entities/tool_bundle.py @@ -7,7 +7,8 @@ from core.tools.entities.tool_entities import ToolParameter class ApiToolBundle(BaseModel): """ - This class is used to store the schema information of an api based tool. such as the url, the method, the parameters, etc. + This class is used to store the schema information of an api based tool. + such as the url, the method, the parameters, etc. """ # server_url diff --git a/api/core/tools/entities/values.py b/api/core/tools/entities/values.py index f9db190f91..f460df7e25 100644 --- a/api/core/tools/entities/values.py +++ b/api/core/tools/entities/values.py @@ -4,52 +4,52 @@ from core.tools.entities.tool_entities import ToolLabel, ToolLabelEnum ICONS = { ToolLabelEnum.SEARCH: """ -""", +""", # noqa: E501 ToolLabelEnum.IMAGE: """ -""", +""", # noqa: E501 ToolLabelEnum.VIDEOS: """ -""", +""", # noqa: E501 ToolLabelEnum.WEATHER: """ -""", +""", # noqa: E501 ToolLabelEnum.FINANCE: """ -""", +""", # noqa: E501 ToolLabelEnum.DESIGN: """ -""", +""", # noqa: E501 ToolLabelEnum.TRAVEL: """ -""", +""", # noqa: E501 ToolLabelEnum.SOCIAL: """ -""", +""", # noqa: E501 ToolLabelEnum.NEWS: """ -""", +""", # noqa: E501 ToolLabelEnum.MEDICAL: """ -""", +""", # noqa: E501 ToolLabelEnum.PRODUCTIVITY: """ -""", +""", # noqa: E501 ToolLabelEnum.EDUCATION: """ -""", +""", # noqa: E501 ToolLabelEnum.BUSINESS: """ -""", +""", # noqa: E501 ToolLabelEnum.ENTERTAINMENT: """ -""", +""", # noqa: E501 ToolLabelEnum.UTILITIES: """ -""", +""", # noqa: E501 ToolLabelEnum.OTHER: """ -""", +""", # noqa: E501 } default_tool_label_dict = { diff --git a/api/core/tools/provider/builtin/aippt/tools/aippt.py b/api/core/tools/provider/builtin/aippt/tools/aippt.py index 7cee8f9f79..a2d69fbcd1 100644 --- a/api/core/tools/provider/builtin/aippt/tools/aippt.py +++ b/api/core/tools/provider/builtin/aippt/tools/aippt.py @@ -46,7 +46,8 @@ class AIPPTGenerateTool(BuiltinTool): tool_parameters (dict[str, Any]): The parameters for the tool Returns: - ToolInvokeMessage | list[ToolInvokeMessage]: The result of the tool invocation, which can be a single message or a list of messages. + ToolInvokeMessage | list[ToolInvokeMessage]: The result of the tool invocation, + which can be a single message or a list of messages. """ title = tool_parameters.get("title", "") if not title: diff --git a/api/core/tools/provider/builtin/arxiv/tools/arxiv_search.py b/api/core/tools/provider/builtin/arxiv/tools/arxiv_search.py index 98d82c233e..2d65ba2d6f 100644 --- a/api/core/tools/provider/builtin/arxiv/tools/arxiv_search.py +++ b/api/core/tools/provider/builtin/arxiv/tools/arxiv_search.py @@ -104,7 +104,8 @@ class ArxivSearchTool(BuiltinTool): tool_parameters (dict[str, Any]): The parameters for the tool, including the 'query' parameter. Returns: - ToolInvokeMessage | list[ToolInvokeMessage]: The result of the tool invocation, which can be a single message or a list of messages. + ToolInvokeMessage | list[ToolInvokeMessage]: The result of the tool invocation, + which can be a single message or a list of messages. """ query = tool_parameters.get("query", "") diff --git a/api/core/tools/provider/builtin/aws/tools/apply_guardrail.py b/api/core/tools/provider/builtin/aws/tools/apply_guardrail.py index d6a65b1708..a04f5c0fe9 100644 --- a/api/core/tools/provider/builtin/aws/tools/apply_guardrail.py +++ b/api/core/tools/provider/builtin/aws/tools/apply_guardrail.py @@ -62,7 +62,8 @@ class ApplyGuardrailTool(BuiltinTool): if isinstance(policy_data, dict) and "topics" in policy_data: for topic in policy_data["topics"]: formatted_assessments.append( - f"Policy: {policy_type}, Topic: {topic['name']}, Type: {topic['type']}, Action: {topic['action']}" + f"Policy: {policy_type}, Topic: {topic['name']}, Type: {topic['type']}," + f" Action: {topic['action']}" ) else: formatted_assessments.append(f"Policy: {policy_type}, Data: {policy_data}") diff --git a/api/core/tools/provider/builtin/devdocs/tools/searchDevDocs.py b/api/core/tools/provider/builtin/devdocs/tools/searchDevDocs.py index e1effd066c..57cf6d7a30 100644 --- a/api/core/tools/provider/builtin/devdocs/tools/searchDevDocs.py +++ b/api/core/tools/provider/builtin/devdocs/tools/searchDevDocs.py @@ -24,7 +24,8 @@ class SearchDevDocsTool(BuiltinTool): tool_parameters (dict[str, Any]): The parameters for the tool, including 'doc' and 'topic'. Returns: - ToolInvokeMessage | list[ToolInvokeMessage]: The result of the tool invocation, which can be a single message or a list of messages. + ToolInvokeMessage | list[ToolInvokeMessage]: The result of the tool invocation, + which can be a single message or a list of messages. """ doc = tool_parameters.get("doc", "") topic = tool_parameters.get("topic", "") diff --git a/api/core/tools/provider/builtin/gitlab/tools/gitlab_files.py b/api/core/tools/provider/builtin/gitlab/tools/gitlab_files.py index 7606eee7af..1e77f3c6df 100644 --- a/api/core/tools/provider/builtin/gitlab/tools/gitlab_files.py +++ b/api/core/tools/provider/builtin/gitlab/tools/gitlab_files.py @@ -70,7 +70,10 @@ class GitlabFilesTool(BuiltinTool): ) else: # It's a file if is_repository: - file_url = f"{domain}/api/v4/projects/{encoded_identifier}/repository/files/{item_path}/raw?ref={branch}" + file_url = ( + f"{domain}/api/v4/projects/{encoded_identifier}/repository/files" + f"/{item_path}/raw?ref={branch}" + ) else: file_url = ( f"{domain}/api/v4/projects/{project_id}/repository/files/{item_path}/raw?ref={branch}" diff --git a/api/core/tools/provider/builtin/google_translate/tools/translate.py b/api/core/tools/provider/builtin/google_translate/tools/translate.py index 5d57b5fabf..ea3f2077d5 100644 --- a/api/core/tools/provider/builtin/google_translate/tools/translate.py +++ b/api/core/tools/provider/builtin/google_translate/tools/translate.py @@ -35,7 +35,8 @@ class GoogleTranslate(BuiltinTool): params = {"client": "gtx", "sl": "auto", "tl": dest, "dt": "t", "q": content} headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)" + " Chrome/91.0.4472.124 Safari/537.36" } response_json = requests.get(url, params=params, headers=headers).json() diff --git a/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py index 69cf8aa740..9e0918afa9 100644 --- a/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py +++ b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py @@ -114,7 +114,8 @@ class GetWorksheetFieldsTool(BuiltinTool): } fields.append(field) fields_list.append( - f"|{field['id']}|{field['name']}|{field['type']}|{field['typeId']}|{field['description']}|{field['options'] if field['options'] else ''}|" + f"|{field['id']}|{field['name']}|{field['type']}|{field['typeId']}|{field['description']}" + f"|{field['options'] if field['options'] else ''}|" ) fields.append( diff --git a/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py index 71f8356ab8..5888f7443f 100644 --- a/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py +++ b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py @@ -112,7 +112,10 @@ class ListWorksheetRecordsTool(BuiltinTool): else: result_text = f"Found {result['total']} rows in worksheet \"{worksheet_name}\"." if result["total"] > 0: - result_text += f" The following are {result['total'] if result['total'] < limit else limit} pieces of data presented in a table format:\n\n{table_header}" + result_text += ( + f" The following are {result['total'] if result['total'] < limit else limit}" + f" pieces of data presented in a table format:\n\n{table_header}" + ) for row in rows: result_values = [] for f in fields: diff --git a/api/core/tools/provider/builtin/searchapi/tools/google.py b/api/core/tools/provider/builtin/searchapi/tools/google.py index 6d88d74635..16ae14549d 100644 --- a/api/core/tools/provider/builtin/searchapi/tools/google.py +++ b/api/core/tools/provider/builtin/searchapi/tools/google.py @@ -64,7 +64,10 @@ class SearchAPI: elif type == "link": if "answer_box" in res and "organic_result" in res["answer_box"]: if "title" in res["answer_box"]["organic_result"]: - toret = f"[{res['answer_box']['organic_result']['title']}]({res['answer_box']['organic_result']['link']})\n" + toret = ( + f"[{res['answer_box']['organic_result']['title']}]" + f"({res['answer_box']['organic_result']['link']})\n" + ) elif "organic_results" in res and "link" in res["organic_results"][0]: toret = "" for item in res["organic_results"]: diff --git a/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py b/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py index 344f916494..64fdc961b4 100644 --- a/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py +++ b/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py @@ -310,7 +310,8 @@ class StableDiffusionTool(BuiltinTool): ), type=ToolParameter.ToolParameterType.STRING, form=ToolParameter.ToolParameterForm.LLM, - llm_description="Image prompt of Stable Diffusion, you should describe the image you want to generate as a list of words as possible as detailed, the prompt must be written in English.", + llm_description="Image prompt of Stable Diffusion, you should describe the image you want to generate" + " as a list of words as possible as detailed, the prompt must be written in English.", required=True, ), ] @@ -320,12 +321,14 @@ class StableDiffusionTool(BuiltinTool): name="image_id", label=I18nObject(en_US="image_id", zh_Hans="image_id"), human_description=I18nObject( - en_US="Image id of the image you want to generate based on, if you want to generate image based on the default image, you can leave this field empty.", + en_US="Image id of the image you want to generate based on, if you want to generate image based" + " on the default image, you can leave this field empty.", zh_Hans="您想要生成的图像的图像 ID，如果您想要基于默认图像生成图像，则可以将此字段留空。", ), type=ToolParameter.ToolParameterType.STRING, form=ToolParameter.ToolParameterForm.LLM, - llm_description="Image id of the original image, you can leave this field empty if you want to generate a new image.", + llm_description="Image id of the original image, you can leave this field empty if you want to" + " generate a new image.", required=True, options=[ ToolParameterOption(value=i.name, label=I18nObject(en_US=i.name, zh_Hans=i.name)) @@ -343,12 +346,14 @@ class StableDiffusionTool(BuiltinTool): name="model", label=I18nObject(en_US="Model", zh_Hans="Model"), human_description=I18nObject( - en_US="Model of Stable Diffusion, you can check the official documentation of Stable Diffusion", + en_US="Model of Stable Diffusion, you can check the official documentation" + " of Stable Diffusion", zh_Hans="Stable Diffusion 的模型，您可以查看 Stable Diffusion 的官方文档", ), type=ToolParameter.ToolParameterType.SELECT, form=ToolParameter.ToolParameterForm.FORM, - llm_description="Model of Stable Diffusion, you can check the official documentation of Stable Diffusion", + llm_description="Model of Stable Diffusion, you can check the official documentation" + " of Stable Diffusion", required=True, default=models[0], options=[ @@ -367,12 +372,14 @@ class StableDiffusionTool(BuiltinTool): name="sampler_name", label=I18nObject(en_US="Sampling method", zh_Hans="Sampling method"), human_description=I18nObject( - en_US="Sampling method of Stable Diffusion, you can check the official documentation of Stable Diffusion", + en_US="Sampling method of Stable Diffusion, you can check the official documentation" + " of Stable Diffusion", zh_Hans="Stable Diffusion 的Sampling method，您可以查看 Stable Diffusion 的官方文档", ), type=ToolParameter.ToolParameterType.SELECT, form=ToolParameter.ToolParameterForm.FORM, - llm_description="Sampling method of Stable Diffusion, you can check the official documentation of Stable Diffusion", + llm_description="Sampling method of Stable Diffusion, you can check the official documentation" + " of Stable Diffusion", required=True, default=sample_methods[0], options=[ diff --git a/api/core/tools/provider/builtin/trello/tools/create_list_on_board.py b/api/core/tools/provider/builtin/trello/tools/create_list_on_board.py index 26f12864c3..b32b0124dd 100644 --- a/api/core/tools/provider/builtin/trello/tools/create_list_on_board.py +++ b/api/core/tools/provider/builtin/trello/tools/create_list_on_board.py @@ -17,7 +17,8 @@ class CreateListOnBoardTool(BuiltinTool): Args: user_id (str): The ID of the user invoking the tool. - tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, including the board ID and list name. + tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, + including the board ID and list name. Returns: ToolInvokeMessage: The result of the tool invocation. diff --git a/api/core/tools/provider/builtin/trello/tools/create_new_card_on_board.py b/api/core/tools/provider/builtin/trello/tools/create_new_card_on_board.py index dfc013a6b8..e98efb81ca 100644 --- a/api/core/tools/provider/builtin/trello/tools/create_new_card_on_board.py +++ b/api/core/tools/provider/builtin/trello/tools/create_new_card_on_board.py @@ -17,7 +17,8 @@ class CreateNewCardOnBoardTool(BuiltinTool): Args: user_id (str): The ID of the user invoking the tool. - tool_parameters (dict[str, Union[str, int, bool, None]]): The parameters for the tool invocation, including details for the new card. + tool_parameters (dict[str, Union[str, int, bool, None]]): The parameters for the tool invocation, + including details for the new card. Returns: ToolInvokeMessage: The result of the tool invocation. diff --git a/api/core/tools/provider/builtin/trello/tools/delete_board.py b/api/core/tools/provider/builtin/trello/tools/delete_board.py index 9dbd8f78d5..7fc9d1f13c 100644 --- a/api/core/tools/provider/builtin/trello/tools/delete_board.py +++ b/api/core/tools/provider/builtin/trello/tools/delete_board.py @@ -17,7 +17,8 @@ class DeleteBoardTool(BuiltinTool): Args: user_id (str): The ID of the user invoking the tool. - tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, including the board ID. + tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, + including the board ID. Returns: ToolInvokeMessage: The result of the tool invocation. diff --git a/api/core/tools/provider/builtin/trello/tools/delete_card.py b/api/core/tools/provider/builtin/trello/tools/delete_card.py index 960c3055fe..1de98d639e 100644 --- a/api/core/tools/provider/builtin/trello/tools/delete_card.py +++ b/api/core/tools/provider/builtin/trello/tools/delete_card.py @@ -17,7 +17,8 @@ class DeleteCardByIdTool(BuiltinTool): Args: user_id (str): The ID of the user invoking the tool. - tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, including the card ID. + tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, + including the card ID. Returns: ToolInvokeMessage: The result of the tool invocation. diff --git a/api/core/tools/provider/builtin/trello/tools/get_board_actions.py b/api/core/tools/provider/builtin/trello/tools/get_board_actions.py index 03510f1964..cabc7ce093 100644 --- a/api/core/tools/provider/builtin/trello/tools/get_board_actions.py +++ b/api/core/tools/provider/builtin/trello/tools/get_board_actions.py @@ -17,7 +17,8 @@ class GetBoardActionsTool(BuiltinTool): Args: user_id (str): The ID of the user invoking the tool. - tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, including the board ID. + tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, + including the board ID. Returns: ToolInvokeMessage: The result of the tool invocation. diff --git a/api/core/tools/provider/builtin/trello/tools/get_board_by_id.py b/api/core/tools/provider/builtin/trello/tools/get_board_by_id.py index 5b41b128d0..fe42cd9c5c 100644 --- a/api/core/tools/provider/builtin/trello/tools/get_board_by_id.py +++ b/api/core/tools/provider/builtin/trello/tools/get_board_by_id.py @@ -17,7 +17,8 @@ class GetBoardByIdTool(BuiltinTool): Args: user_id (str): The ID of the user invoking the tool. - tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, including the board ID. + tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, + including the board ID. Returns: ToolInvokeMessage: The result of the tool invocation. diff --git a/api/core/tools/provider/builtin/trello/tools/get_board_cards.py b/api/core/tools/provider/builtin/trello/tools/get_board_cards.py index e3bed2e6e6..ff2b1221e7 100644 --- a/api/core/tools/provider/builtin/trello/tools/get_board_cards.py +++ b/api/core/tools/provider/builtin/trello/tools/get_board_cards.py @@ -17,7 +17,8 @@ class GetBoardCardsTool(BuiltinTool): Args: user_id (str): The ID of the user invoking the tool. - tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, including the board ID. + tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, + including the board ID. Returns: ToolInvokeMessage: The result of the tool invocation. diff --git a/api/core/tools/provider/builtin/trello/tools/get_filterd_board_cards.py b/api/core/tools/provider/builtin/trello/tools/get_filterd_board_cards.py index 4d8854747c..3d7f9f4ad1 100644 --- a/api/core/tools/provider/builtin/trello/tools/get_filterd_board_cards.py +++ b/api/core/tools/provider/builtin/trello/tools/get_filterd_board_cards.py @@ -17,7 +17,8 @@ class GetFilteredBoardCardsTool(BuiltinTool): Args: user_id (str): The ID of the user invoking the tool. - tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, including the board ID and filter. + tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, + including the board ID and filter. Returns: ToolInvokeMessage: The result of the tool invocation. diff --git a/api/core/tools/provider/builtin/trello/tools/get_lists_on_board.py b/api/core/tools/provider/builtin/trello/tools/get_lists_on_board.py index ca8aa9c2d5..ccf404068f 100644 --- a/api/core/tools/provider/builtin/trello/tools/get_lists_on_board.py +++ b/api/core/tools/provider/builtin/trello/tools/get_lists_on_board.py @@ -17,7 +17,8 @@ class GetListsFromBoardTool(BuiltinTool): Args: user_id (str): The ID of the user invoking the tool. - tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, including the board ID. + tool_parameters (dict[str, Union[str, int, bool]]): The parameters for the tool invocation, + including the board ID. Returns: ToolInvokeMessage: The result of the tool invocation. diff --git a/api/core/tools/provider/builtin/trello/tools/update_board.py b/api/core/tools/provider/builtin/trello/tools/update_board.py index 62681eea6b..1e358b00f4 100644 --- a/api/core/tools/provider/builtin/trello/tools/update_board.py +++ b/api/core/tools/provider/builtin/trello/tools/update_board.py @@ -17,7 +17,8 @@ class UpdateBoardByIdTool(BuiltinTool): Args: user_id (str): The ID of the user invoking the tool. - tool_parameters (dict[str, Union[str, int, bool, None]]): The parameters for the tool invocation, including board ID and updates. + tool_parameters (dict[str, Union[str, int, bool, None]]): The parameters for the tool invocation, + including board ID and updates. Returns: ToolInvokeMessage: The result of the tool invocation. diff --git a/api/core/tools/provider/builtin/trello/tools/update_card.py b/api/core/tools/provider/builtin/trello/tools/update_card.py index 26113f1229..d25fcbafaa 100644 --- a/api/core/tools/provider/builtin/trello/tools/update_card.py +++ b/api/core/tools/provider/builtin/trello/tools/update_card.py @@ -17,7 +17,8 @@ class UpdateCardByIdTool(BuiltinTool): Args: user_id (str): The ID of the user invoking the tool. - tool_parameters (dict[str, Union[str, int, bool, None]]): The parameters for the tool invocation, including the card ID and updates. + tool_parameters (dict[str, Union[str, int, bool, None]]): The parameters for the tool invocation, + including the card ID and updates. Returns: ToolInvokeMessage: The result of the tool invocation. diff --git a/api/core/tools/provider/builtin/twilio/tools/send_message.py b/api/core/tools/provider/builtin/twilio/tools/send_message.py index 822d0c0ebd..156249bc96 100644 --- a/api/core/tools/provider/builtin/twilio/tools/send_message.py +++ b/api/core/tools/provider/builtin/twilio/tools/send_message.py @@ -72,7 +72,8 @@ class SendMessageTool(BuiltinTool): tool_parameters (Dict[str, Any]): The parameters required for sending the message. Returns: - Union[ToolInvokeMessage, List[ToolInvokeMessage]]: The result of invoking the tool, which includes the status of the message sending operation. + Union[ToolInvokeMessage, List[ToolInvokeMessage]]: The result of invoking the tool, + which includes the status of the message sending operation. """ def _invoke( diff --git a/api/core/tools/provider/builtin/vectorizer/tools/test_data.py b/api/core/tools/provider/builtin/vectorizer/tools/test_data.py index 8e1b097776..8effa9818a 100644 --- a/api/core/tools/provider/builtin/vectorizer/tools/test_data.py +++ b/api/core/tools/provider/builtin/vectorizer/tools/test_data.py @@ -1 +1 @@ -VECTORIZER_ICON_PNG = "iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAYAAADimHc4AAAACXBIWXMAACxLAAAsSwGlPZapAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAboSURBVHgB7Z09bBxFFMffRoAvcQqbguBUxu4wCUikMCZ0TmQK4NLQJCJOlQIkokgEGhQ7NCFIKEhQuIqNnIaGMxRY2GVwmlggDHS+pIHELmIXMTEULPP3eeXz7e7szO7MvE1ufpKV03nuNn7/mfcxH7tEHo/H42lXgqwG1bGw65+/aTQM6K0gpJdCoi7ypCIMui5s9Qv9R1OVTqrVxoL1jPbpvH4hrIp/rnmj5+YOhTQ++1kwmdZgT9ovRi6EF4Xhv/XGL0Sv6OLXYMu0BokjYOSDcBQfJI8xhKFP/HAlqCW8v5vqubBr8yn6maCexxiIDR376LnWmBBzQZtPEvx+L3mMAleOZKb1/XgM2EOnyWMFZJKt78UEQKpJHisk2TYmgM967JFk2z3kYcULwIwXgBkvADNeAGa8AMw8Qcwc6N55/eAh0cYmGaOzQtR/kOhQX+M6+/c23r+3RlT/i2ipTrSyRqw4F+CwMMbgANHQwG7jRywLw/wqDDNzI79xYPjqa2L262jjtYzaT0QT3xEbsck4MXUakgWOvUx08liy0ZPYEKNhel4Y6AZpgR7/8Tvq1wEQ+sMJN6Nh9kqwy+bWYwAM8elZovNv6xmlU7iLs280RNO9ls51os/h/8eBVQEig8Dt5OXUsNrno2tluZw0cI3qUXKONQHy9sYkVHqnjntLA2LnFTAv1gSA+zBhfIDvkfVO/B4xRgWZn4fbe2WAnGJFAAxn03+I7PtUXdzE90Sjl4ne+6L4d5nCigAyYyHPn7tFdPN30uJwX/qI6jtISkQZFVLdhd9SrtNPTrFSB6QZBAaYntsptpAyfvk+KYOCamVR/XrNtLqepduiFnkh3g4iIw6YLAhlOJmKwB9zaarhApr/MPREjAZVisSU1s/KYsGzhmKXClYEWLm/8xpV7btXhcv5I7lt2vtJFA3q/T07r1HopdG5l5xhxQVdn28YFn8kBJCBOZmiPHio1m5QuJzlu9ntXApgZwSsNYJslvGjtjrfm8Sq4neceFUtz3dZCzwW09Gqo2hreuPN7HZRnNqa1BP1x8lhczVNK+zT0TqkjYAF4e7Okxoo2PZX5K4IrhNpb/P8FTK2S1+TcUq1HpBFmquJYo1qEYU6RVarJE0c2ooL7C5IRwBZ5nJ9joyRtk5hA3YBdHqWzG1gBKgE/bzMaK5LqMIugKrbUDHu59/YWVRBsWhrsYZdANV5HBUXYGNlC9dFBW8LdgH6FQVYUnQvkQgm3NH8YuO7bM4LsWZBfT3qRY9OxRyJgJRz+Ij+FDPEQ1C3GVMiWAVQ7f31u/ncytxi4wdZTbRGgdcHnpYLD/FcwSrAoOKizfKfVAiIF4kBMPK+Opfe1iWsMUB1BJh2BRgBabSNAOiFqkXYbcNFUF9P+u82FGdWTcEmgGrvh0FUppB1kC073muXEaDq/21kIjLxV9tFAC7/n5X6tkUM0PH/dcP+P0v41fvkFBYBVHs/MD0CDmVsOzEdb7JgEYDT/8uq4rpj44NSjwDTc/CyzV1gxbH7Ac4F0PH/S4ZHAOaFZLiY+2nFuQA6/t9kQMTCz1CG66tbWvWS4VwAVf9vugAbel6efqrsYbKBcwFeVNz8ajobyTppw2F84FQAnfl/kwER6wJZcWdBc7e2KZwKoOP/TVakWb0f7md+kVhwOwI0BDCFyq42rt4PSiuAiRGAEXdK4ZQlV+8HTgVwefwHvR7nhbOA0FwBGDgTIM/Z3SLXUj2hOW1wR10eSrs7Ou9eTB3jo/dzuh/gTABdn35c8dhpM3BxOmeTuXs/cDoCdDY4qe7l32pbaZxL1jF+GXo/cLotBcWVTiZU3T7RMn8rHiijW9FgauP4Ef1TLdhHWgacCgAj6tYCqGKjU/DNbqxIkMYZNs7MpxmnLuhmwYJna1dbdzHjY42hDL4/wqkA6HWuDkAngRH0iYVjRkVwnoZO/0gsuLwpkw7OBcAtwlwvfESHxctmfMBSiOG0oStj4HCF7T3+RWARwIU7QK/HbWlqls52mYJtezqMj3v34C5VOveFy8Ll4QoTsJ8Txp0RsW8/Os2im2LCtSC1RIqLw3RldTVplOKkPEYDhMAPqttnune2rzTv5Y+WKdEem2ixkWqZYSeDSUp3qwIYNOrR7cBjcbOORxkvADNeAGa8AMx4AZjxAjATf5Ab0Tp5rJBk2/iD3PAwYo8Vkmyb9CjDGfLYIaCp1rdiAnT8S5PeDVkgoDuVCsWeJxwToHZ163m3Z8hjloDGk54vn5gFbT/5eZw8phifvZz8XPlA9qmRj8JRCumi+OkljzbbrvxM0qPMm9rIqY6FXZubVBUinMbzcP3jbuXA6Mh2kMx07KPJJLfj8Xg8Hg/4H+KfFYb2WM4MAAAAAElFTkSuQmCC" +VECTORIZER_ICON_PNG = "iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAYAAADimHc4AAAACXBIWXMAACxLAAAsSwGlPZapAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAboSURBVHgB7Z09bBxFFMffRoAvcQqbguBUxu4wCUikMCZ0TmQK4NLQJCJOlQIkokgEGhQ7NCFIKEhQuIqNnIaGMxRY2GVwmlggDHS+pIHELmIXMTEULPP3eeXz7e7szO7MvE1ufpKV03nuNn7/mfcxH7tEHo/H42lXgqwG1bGw65+/aTQM6K0gpJdCoi7ypCIMui5s9Qv9R1OVTqrVxoL1jPbpvH4hrIp/rnmj5+YOhTQ++1kwmdZgT9ovRi6EF4Xhv/XGL0Sv6OLXYMu0BokjYOSDcBQfJI8xhKFP/HAlqCW8v5vqubBr8yn6maCexxiIDR376LnWmBBzQZtPEvx+L3mMAleOZKb1/XgM2EOnyWMFZJKt78UEQKpJHisk2TYmgM967JFk2z3kYcULwIwXgBkvADNeAGa8AMw8Qcwc6N55/eAh0cYmGaOzQtR/kOhQX+M6+/c23r+3RlT/i2ipTrSyRqw4F+CwMMbgANHQwG7jRywLw/wqDDNzI79xYPjqa2L262jjtYzaT0QT3xEbsck4MXUakgWOvUx08liy0ZPYEKNhel4Y6AZpgR7/8Tvq1wEQ+sMJN6Nh9kqwy+bWYwAM8elZovNv6xmlU7iLs280RNO9ls51os/h/8eBVQEig8Dt5OXUsNrno2tluZw0cI3qUXKONQHy9sYkVHqnjntLA2LnFTAv1gSA+zBhfIDvkfVO/B4xRgWZn4fbe2WAnGJFAAxn03+I7PtUXdzE90Sjl4ne+6L4d5nCigAyYyHPn7tFdPN30uJwX/qI6jtISkQZFVLdhd9SrtNPTrFSB6QZBAaYntsptpAyfvk+KYOCamVR/XrNtLqepduiFnkh3g4iIw6YLAhlOJmKwB9zaarhApr/MPREjAZVisSU1s/KYsGzhmKXClYEWLm/8xpV7btXhcv5I7lt2vtJFA3q/T07r1HopdG5l5xhxQVdn28YFn8kBJCBOZmiPHio1m5QuJzlu9ntXApgZwSsNYJslvGjtjrfm8Sq4neceFUtz3dZCzwW09Gqo2hreuPN7HZRnNqa1BP1x8lhczVNK+zT0TqkjYAF4e7Okxoo2PZX5K4IrhNpb/P8FTK2S1+TcUq1HpBFmquJYo1qEYU6RVarJE0c2ooL7C5IRwBZ5nJ9joyRtk5hA3YBdHqWzG1gBKgE/bzMaK5LqMIugKrbUDHu59/YWVRBsWhrsYZdANV5HBUXYGNlC9dFBW8LdgH6FQVYUnQvkQgm3NH8YuO7bM4LsWZBfT3qRY9OxRyJgJRz+Ij+FDPEQ1C3GVMiWAVQ7f31u/ncytxi4wdZTbRGgdcHnpYLD/FcwSrAoOKizfKfVAiIF4kBMPK+Opfe1iWsMUB1BJh2BRgBabSNAOiFqkXYbcNFUF9P+u82FGdWTcEmgGrvh0FUppB1kC073muXEaDq/21kIjLxV9tFAC7/n5X6tkUM0PH/dcP+P0v41fvkFBYBVHs/MD0CDmVsOzEdb7JgEYDT/8uq4rpj44NSjwDTc/CyzV1gxbH7Ac4F0PH/S4ZHAOaFZLiY+2nFuQA6/t9kQMTCz1CG66tbWvWS4VwAVf9vugAbel6efqrsYbKBcwFeVNz8ajobyTppw2F84FQAnfl/kwER6wJZcWdBc7e2KZwKoOP/TVakWb0f7md+kVhwOwI0BDCFyq42rt4PSiuAiRGAEXdK4ZQlV+8HTgVwefwHvR7nhbOA0FwBGDgTIM/Z3SLXUj2hOW1wR10eSrs7Ou9eTB3jo/dzuh/gTABdn35c8dhpM3BxOmeTuXs/cDoCdDY4qe7l32pbaZxL1jF+GXo/cLotBcWVTiZU3T7RMn8rHiijW9FgauP4Ef1TLdhHWgacCgAj6tYCqGKjU/DNbqxIkMYZNs7MpxmnLuhmwYJna1dbdzHjY42hDL4/wqkA6HWuDkAngRH0iYVjRkVwnoZO/0gsuLwpkw7OBcAtwlwvfESHxctmfMBSiOG0oStj4HCF7T3+RWARwIU7QK/HbWlqls52mYJtezqMj3v34C5VOveFy8Ll4QoTsJ8Txp0RsW8/Os2im2LCtSC1RIqLw3RldTVplOKkPEYDhMAPqttnune2rzTv5Y+WKdEem2ixkWqZYSeDSUp3qwIYNOrR7cBjcbOORxkvADNeAGa8AMx4AZjxAjATf5Ab0Tp5rJBk2/iD3PAwYo8Vkmyb9CjDGfLYIaCp1rdiAnT8S5PeDVkgoDuVCsWeJxwToHZ163m3Z8hjloDGk54vn5gFbT/5eZw8phifvZz8XPlA9qmRj8JRCumi+OkljzbbrvxM0qPMm9rIqY6FXZubVBUinMbzcP3jbuXA6Mh2kMx07KPJJLfj8Xg8Hg/4H+KfFYb2WM4MAAAAAElFTkSuQmCC" # noqa: E501 diff --git a/api/core/tools/tool_engine.py b/api/core/tools/tool_engine.py index 9a6a49d8f4..645f0861fa 100644 --- a/api/core/tools/tool_engine.py +++ b/api/core/tools/tool_engine.py @@ -193,7 +193,10 @@ class ToolEngine: response.type == ToolInvokeMessage.MessageType.IMAGE_LINK or response.type == ToolInvokeMessage.MessageType.IMAGE ): - result += "image has been created and sent to user already, you do not need to create it, just tell the user to check it now." + result += ( + "image has been created and sent to user already, you do not need to create it," + " just tell the user to check it now." + ) elif response.type == ToolInvokeMessage.MessageType.JSON: result += f"tool response: {json.dumps(response.message, ensure_ascii=False)}." else: diff --git a/api/core/tools/utils/feishu_api_utils.py b/api/core/tools/utils/feishu_api_utils.py index 7bb026a383..44803d7d65 100644 --- a/api/core/tools/utils/feishu_api_utils.py +++ b/api/core/tools/utils/feishu_api_utils.py @@ -89,7 +89,7 @@ class FeishuRequest: "content": "云文档\n多人实时协同，插入一切元素。不仅是在线文档，更是强大的创作和互动工具\n云文档：专为协作而生\n" } } - """ + """ # noqa: E501 params = { "document_id": document_id, } diff --git a/api/core/tools/utils/message_transformer.py b/api/core/tools/utils/message_transformer.py index 1109ed7df2..bf040d91d3 100644 --- a/api/core/tools/utils/message_transformer.py +++ b/api/core/tools/utils/message_transformer.py @@ -43,7 +43,7 @@ class ToolFileMessageTransformer: result.append( ToolInvokeMessage( type=ToolInvokeMessage.MessageType.TEXT, - message=f"Failed to download image: {message.message}, you can try to download it yourself.", + message=f"Failed to download image: {message.message}, please try to download it manually.", meta=message.meta.copy() if message.meta is not None else {}, save_as=message.save_as, ) diff --git a/api/core/tools/utils/parser.py b/api/core/tools/utils/parser.py index 654c9acaf9..210b84b29a 100644 --- a/api/core/tools/utils/parser.py +++ b/api/core/tools/utils/parser.py @@ -315,7 +315,8 @@ class ApiBasedToolSchemaParser: yaml_error = e if loaded_content is None: raise ToolApiSchemaError( - f"Invalid api schema, schema is neither json nor yaml. json error: {str(json_error)}, yaml error: {str(yaml_error)}" + f"Invalid api schema, schema is neither json nor yaml. json error: {str(json_error)}," + f" yaml error: {str(yaml_error)}" ) swagger_error = None @@ -355,5 +356,6 @@ class ApiBasedToolSchemaParser: openapi_plugin_error = e raise ToolApiSchemaError( - f"Invalid api schema, openapi error: {str(openapi_error)}, swagger error: {str(swagger_error)}, openapi plugin error: {str(openapi_plugin_error)}" + f"Invalid api schema, openapi error: {str(openapi_error)}, swagger error: {str(swagger_error)}," + f" openapi plugin error: {str(openapi_plugin_error)}" ) diff --git a/api/core/tools/utils/web_reader_tool.py b/api/core/tools/utils/web_reader_tool.py index 3639b5fff7..fc2f63a241 100644 --- a/api/core/tools/utils/web_reader_tool.py +++ b/api/core/tools/utils/web_reader_tool.py @@ -38,7 +38,8 @@ def page_result(text: str, cursor: int, max_length: int) -> str: def get_url(url: str, user_agent: str = None) -> str: """Fetch URL and return the contents as a string.""" headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)" + " Chrome/91.0.4472.124 Safari/537.36" } if user_agent: headers["User-Agent"] = user_agent diff --git a/api/core/workflow/nodes/code/code_node.py b/api/core/workflow/nodes/code/code_node.py index a07ba2f740..73164fff9a 100644 --- a/api/core/workflow/nodes/code/code_node.py +++ b/api/core/workflow/nodes/code/code_node.py @@ -179,7 +179,8 @@ class CodeNode(BaseNode): ) else: raise ValueError( - f"Output {prefix}.{output_name} is not a valid array. make sure all elements are of the same type." + f"Output {prefix}.{output_name} is not a valid array." + f" make sure all elements are of the same type." ) elif isinstance(output_value, type(None)): pass @@ -201,7 +202,8 @@ class CodeNode(BaseNode): transformed_result[output_name] = None else: raise ValueError( - f"Output {prefix}{dot}{output_name} is not an object, got {type(result.get(output_name))} instead." + f"Output {prefix}{dot}{output_name} is not an object," + f" got {type(result.get(output_name))} instead." ) else: transformed_result[output_name] = self._transform_result( @@ -228,7 +230,8 @@ class CodeNode(BaseNode): transformed_result[output_name] = None else: raise ValueError( - f"Output {prefix}{dot}{output_name} is not an array, got {type(result.get(output_name))} instead." + f"Output {prefix}{dot}{output_name} is not an array," + f" got {type(result.get(output_name))} instead." ) else: if len(result[output_name]) > dify_config.CODE_MAX_NUMBER_ARRAY_LENGTH: @@ -248,7 +251,8 @@ class CodeNode(BaseNode): transformed_result[output_name] = None else: raise ValueError( - f"Output {prefix}{dot}{output_name} is not an array, got {type(result.get(output_name))} instead." + f"Output {prefix}{dot}{output_name} is not an array," + f" got {type(result.get(output_name))} instead." ) else: if len(result[output_name]) > dify_config.CODE_MAX_STRING_ARRAY_LENGTH: @@ -268,7 +272,8 @@ class CodeNode(BaseNode): transformed_result[output_name] = None else: raise ValueError( - f"Output {prefix}{dot}{output_name} is not an array, got {type(result.get(output_name))} instead." + f"Output {prefix}{dot}{output_name} is not an array," + f" got {type(result.get(output_name))} instead." ) else: if len(result[output_name]) > dify_config.CODE_MAX_OBJECT_ARRAY_LENGTH: @@ -283,7 +288,8 @@ class CodeNode(BaseNode): pass else: raise ValueError( - f"Output {prefix}{dot}{output_name}[{i}] is not an object, got {type(value)} instead at index {i}." + f"Output {prefix}{dot}{output_name}[{i}] is not an object," + f" got {type(value)} instead at index {i}." ) transformed_result[output_name] = [ diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index 53e8be6415..af55688a52 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -128,11 +128,12 @@ class KnowledgeRetrievalNode(BaseNode): weights = None elif node_data.multiple_retrieval_config.reranking_mode == "weighted_score": reranking_model = None + vector_setting = node_data.multiple_retrieval_config.weights.vector_setting weights = { "vector_setting": { - "vector_weight": node_data.multiple_retrieval_config.weights.vector_setting.vector_weight, - "embedding_provider_name": node_data.multiple_retrieval_config.weights.vector_setting.embedding_provider_name, - "embedding_model_name": node_data.multiple_retrieval_config.weights.vector_setting.embedding_model_name, + "vector_weight": vector_setting.vector_weight, + "embedding_provider_name": vector_setting.embedding_provider_name, + "embedding_model_name": vector_setting.embedding_model_name, }, "keyword_setting": { "keyword_weight": node_data.multiple_retrieval_config.weights.keyword_setting.keyword_weight diff --git a/api/core/workflow/nodes/parameter_extractor/prompts.py b/api/core/workflow/nodes/parameter_extractor/prompts.py index c63fded4d0..58fcecc53b 100644 --- a/api/core/workflow/nodes/parameter_extractor/prompts.py +++ b/api/core/workflow/nodes/parameter_extractor/prompts.py @@ -23,7 +23,7 @@ Steps: To illustrate, if the task involves extracting a user's name and their request, your function call might look like this: Ensure your output follows a similar structure to examples. ### Final Output Produce well-formatted function calls in json without XML tags, as shown in the example. -""" +""" # noqa: E501 FUNCTION_CALLING_EXTRACTOR_USER_TEMPLATE = f"""extract structured information from context inside XML tags by calling the function {FUNCTION_CALLING_EXTRACTOR_NAME} with the correct parameters with structure inside XML tags. @@ -33,7 +33,7 @@ FUNCTION_CALLING_EXTRACTOR_USER_TEMPLATE = f"""extract structured information fr \x7bstructure\x7d -""" +""" # noqa: E501 FUNCTION_CALLING_EXTRACTOR_EXAMPLE = [ { @@ -55,7 +55,8 @@ FUNCTION_CALLING_EXTRACTOR_EXAMPLE = [ }, }, "assistant": { - "text": "I need always call the function with the correct parameters. in this case, I need to call the function with the location parameter.", + "text": "I need always call the function with the correct parameters." + " in this case, I need to call the function with the location parameter.", "function_call": {"name": FUNCTION_CALLING_EXTRACTOR_NAME, "parameters": {"location": "San Francisco"}}, }, }, @@ -72,7 +73,8 @@ FUNCTION_CALLING_EXTRACTOR_EXAMPLE = [ }, }, "assistant": { - "text": "I need always call the function with the correct parameters. in this case, I need to call the function with the food parameter.", + "text": "I need always call the function with the correct parameters." + " in this case, I need to call the function with the food parameter.", "function_call": {"name": FUNCTION_CALLING_EXTRACTOR_NAME, "parameters": {"food": "apple pie"}}, }, }, @@ -117,7 +119,7 @@ Inside XML tags, there is a text that I should extract parameters ### Answer I should always output a valid JSON object. Output nothing other than the JSON object. ```JSON -""" +""" # noqa: E501 CHAT_GENERATE_JSON_PROMPT = """You should always follow the instructions and output a valid JSON object. The structure of the JSON object you can found in the instructions. diff --git a/api/core/workflow/nodes/question_classifier/template_prompts.py b/api/core/workflow/nodes/question_classifier/template_prompts.py index 581f986922..ce32b01aa4 100644 --- a/api/core/workflow/nodes/question_classifier/template_prompts.py +++ b/api/core/workflow/nodes/question_classifier/template_prompts.py @@ -12,13 +12,13 @@ QUESTION_CLASSIFIER_SYSTEM_PROMPT = """ {histories} -""" +""" # noqa: E501 QUESTION_CLASSIFIER_USER_PROMPT_1 = """ { "input_text": ["I recently had a great experience with your company. The service was prompt and the staff was very friendly."], "categories": [{"category_id":"f5660049-284f-41a7-b301-fd24176a711c","category_name":"Customer Service"},{"category_id":"8d007d06-f2c9-4be5-8ff6-cd4381c13c60","category_name":"Satisfaction"},{"category_id":"5fbbbb18-9843-466d-9b8e-b9bfbb9482c8","category_name":"Sales"},{"category_id":"23623c75-7184-4a2e-8226-466c2e4631e4","category_name":"Product"}], "classification_instructions": ["classify the text based on the feedback provided by customer"]} -""" +""" # noqa: E501 QUESTION_CLASSIFIER_ASSISTANT_PROMPT_1 = """ ```json @@ -32,7 +32,7 @@ QUESTION_CLASSIFIER_USER_PROMPT_2 = """ {"input_text": ["bad service, slow to bring the food"], "categories": [{"category_id":"80fb86a0-4454-4bf5-924c-f253fdd83c02","category_name":"Food Quality"},{"category_id":"f6ff5bc3-aca0-4e4a-8627-e760d0aca78f","category_name":"Experience"},{"category_id":"cc771f63-74e7-4c61-882e-3eda9d8ba5d7","category_name":"Price"}], "classification_instructions": []} -""" +""" # noqa: E501 QUESTION_CLASSIFIER_ASSISTANT_PROMPT_2 = """ ```json @@ -73,4 +73,4 @@ Here is the chat histories between human and assistant, inside " + return ( + f"" + ) @property def token_is_set(self): diff --git a/api/models/tools.py b/api/models/tools.py index 6b69a219b1..861066a2d5 100644 --- a/api/models/tools.py +++ b/api/models/tools.py @@ -62,7 +62,8 @@ class PublishedAppTool(db.Model): description = db.Column(db.Text, nullable=False) # llm_description of the tool, for LLM llm_description = db.Column(db.Text, nullable=False) - # query description, query will be seem as a parameter of the tool, to describe this parameter to llm, we need this field + # query description, query will be seem as a parameter of the tool, + # to describe this parameter to llm, we need this field query_description = db.Column(db.Text, nullable=False) # query name, the name of the query parameter query_name = db.Column(db.String(40), nullable=False) diff --git a/api/models/workflow.py b/api/models/workflow.py index d52749f0ff..9c93ea4cea 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -246,7 +246,8 @@ class Workflow(db.Model): if any(var for var in value if not var.id): raise ValueError("environment variable require a unique id") - # Compare inputs and origin variables, if the value is HIDDEN_VALUE, use the origin variable value (only update `name`). + # Compare inputs and origin variables, + # if the value is HIDDEN_VALUE, use the origin variable value (only update `name`). origin_variables_dictionary = {var.id: var for var in self.environment_variables} for i, variable in enumerate(value): if variable.id in origin_variables_dictionary and variable.value == HIDDEN_VALUE: diff --git a/api/poetry.lock b/api/poetry.lock index 103423e5c7..6023f98e2a 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -10388,4 +10388,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "2dbff415c3c9ca95c8dcfb59fc088ce2c0d00037c44f386a34c87c98e1d8b942" +content-hash = "8179c7e3f91b5a00054e26297040b1969f59b37cb9a707fbaa9c2ea419954718" diff --git a/api/pyproject.toml b/api/pyproject.toml index 3d100ebc58..616794cf3a 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -27,7 +27,6 @@ select = [ "W605", # invalid-escape-sequence ] ignore = [ - "E501", # line-too-long "E402", # module-import-not-at-top-of-file "E711", # none-comparison "E712", # true-false-comparison @@ -68,16 +67,19 @@ ignore = [ "F401", # unused-import "F811", # redefined-while-unused ] -"tests/*" = [ - "F401", # unused-import - "F811", # redefined-while-unused -] "configs/*" = [ "N802", # invalid-function-name ] "libs/gmpy2_pkcs10aep_cipher.py" = [ "N803", # invalid-argument-name ] +"migrations/versions/*" = [ + "E501", # line-too-long +] +"tests/*" = [ + "F401", # unused-import + "F811", # redefined-while-unused +] [tool.ruff.format] exclude = [ @@ -270,4 +272,4 @@ optional = true [tool.poetry.group.lint.dependencies] dotenv-linter = "~0.5.0" -ruff = "~0.6.1" +ruff = "~0.6.4" diff --git a/api/services/tools/api_tools_manage_service.py b/api/services/tools/api_tools_manage_service.py index 3ded9c0989..6f6074f596 100644 --- a/api/services/tools/api_tools_manage_service.py +++ b/api/services/tools/api_tools_manage_service.py @@ -176,7 +176,8 @@ class ApiToolManageService: get api tool provider remote schema """ headers = { - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko)" + " Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0", "Accept": "*/*", } diff --git a/api/tests/integration_tests/model_runtime/__mock/openai_embeddings.py b/api/tests/integration_tests/model_runtime/__mock/openai_embeddings.py index 4138cdd40d..025913cb17 100644 --- a/api/tests/integration_tests/model_runtime/__mock/openai_embeddings.py +++ b/api/tests/integration_tests/model_runtime/__mock/openai_embeddings.py @@ -40,7 +40,7 @@ class MockEmbeddingsClass: usage=Usage(prompt_tokens=2, total_tokens=2), ) - embeddings = "VEfNvMLUnrwFleO8hcj9vEE/yrzyjOA84E1MvNfoCrxjrI+8sZUKvNgrBT17uY07gJ/IvNvhHLrUemc8KXXGumalIT3YKwU7ZsnbPMhATrwTt6u8JEwRPNMmCjxGREW7TRKvu6/MG7zAyDU8wXLkuuMDZDsXsL28zHzaOw0IArzOiMO8LtASvPKM4Dul5l+80V0bPGVDZ7wYNrI89ucsvJZdYztzRm+8P8ysOyGbc7zrdgK9sdiEPKQ8sbulKdq7KIgdvKIMDj25dNc8k0AXPBn/oLzrdgK8IXe5uz0Dvrt50V68tTjLO4ZOcjoG9x29oGfZufiwmzwMDXy8EL6ZPHvdx7nKjzE8+LCbPG22hTs3EZq7TM+0POrRzTxVZo084wPkO8Nak7z8cpw8pDwxvA2T8LvBC7C72fltvC8Atjp3fYE8JHDLvEYgC7xAdls8YiabPPkEeTzPUbK8gOLCPEBSIbyt5Oy8CpreusNakzywUhA824vLPHRlr7zAhTs7IZtzvHd9AT2xY/O6ok8IvOihqrql5l88K4EvuknWorvYKwW9iXkbvGMTRLw5qPG7onPCPLgNIzwAbK67ftbZPMxYILvAyDW9TLB0vIid1buzCKi7u+d0u8iDSLxNVam8PZyJPNxnETvVANw8Oi5mu9nVszzl65I7DIKNvLGVirxsMJE7tPXQu2PvCT1zRm87p1l9uyRMkbsdfqe8U52ePHRlr7wt9Mw8/C8ivTu02rwJFGq8tpoFPWnC7blWumq7sfy+vG1zCzy9Nlg8iv+PuvxT3DuLU228kVhoOkmTqDrv1kg8ocmTu1WpBzsKml48DzglvI8ECzxwTd27I+pWvIWkQ7xUR007GqlPPBFEDrzGECu865q8PI7BkDwNxYc8tgG6ullMSLsIajs84lk1PNLjD70mv648ZmInO2tnIjzvb5Q8o5KCPLo9xrwKMyq9QqGEvI8ECzxO2508ATUdPRAlTry5kxc8KVGMPJyBHjxIUC476KGqvIU9DzwX87c88PUIParrWrzdlzS/G3K+uzEw2TxB2BU86AhfPAMiRj2dK808a85WPPCft7xU4Bg95Q9NPDxZjzwrpek7yNkZvHa0EjyQ0nM6Nq9fuyjvUbsRq8I7CAMHO3VSWLyuauE7U1qkvPkEeTxs7ZY7B6FMO48Eizy75/S7ieBPvB07rTxmyVu8onPCO5rc6Tu7XIa7oEMfPYngT7u24vk7/+W5PE8eGDxJ1iI9t4cuvBGHiLyH1GY7jfghu+oUSDwa7Mk7iXmbuut2grrq8I2563v8uyofdTxRTrs44lm1vMeWnzukf6s7r4khvEKhhDyhyZO8G5Z4Oy56wTz4sBs81Zknuz3fg7wnJuO74n1vvASEADu98128gUl3vBtyvrtZCU47yep8u5FYaDx2G0e8a85WO5cmUjz3kds8qgqbPCUaerx50d67WKIZPI7BkDua3Om74vKAvL3zXbzXpRA9CI51vLo9xryKzXg7tXtFO9RWLTwnJuM854LqPEIs8zuO5cq8d8V1u9P0cjrQ++C8cGwdPDdUlLoOGeW8auEtu8Z337nlzFK8aRg/vFCkDD0nRSM879bIvKUFID1iStU8EL6ZvLufgLtKgNE7KVEMvJOnSzwahRU895HbvJiIjLvc8n88bmC0PPLP2rywM9C7jTscOoS3mjy/Znu7dhvHuu5Q1Dyq61o6CI71u09hkry0jhw8gb6IPI8EC7uoVAM8gs9rvGM3fjx2G8e81FYtu/ojubyYRRK72Riuu83elDtNNmk70/TyuzUFsbvgKZI7onNCvAehzLumr8679R6+urr6SztX2So8Bl5SOwSEgLv5NpA8LwC2PGPvibzJ6vw7H2tQvOtXwrzXpRC8j0z/uxwcbTy2vr+8VWYNu+t2ArwKmt68NKN2O3XrIzw9A747UU47vaavzjwU+qW8YBqyvE02aTyEt5o8cCmjOxtyPrxs7ZY775NOu+SJWLxMJQY8/bWWu6IMDrzSSsQ7GSPbPLlQnbpVzcE7Pka4PJ96sLycxJg8v/9GPO2HZTyeW3C8Vpawtx2iYTwWBg87/qI/OviwGzxyWcY7M9WNPIA4FD32C2e8tNGWPJ43trxCoYS8FGHavItTbbu7n4C80NemPLm30Ty1OMu7vG1pvG3aPztBP0o75Q/NPJhFEj2V9i683PL/O97+aLz6iu27cdPRum/mKLwvVgc89fqDu3LA+jvm2Ls8mVZ1PIuFBD3ZGK47Cpreut7+aLziWTU8XSEgPMvSKzzO73e5040+vBlmVTxS1K+8mQ4BPZZ8o7w8FpW6OR0DPSSPCz21Vwu99fqDOjMYiDy7XAY8oYaZO+aVwTyX49c84OaXOqdZfTunEQk7B8AMvMDs7zo/D6e8OP5CvN9gIzwNCII8FefOPE026TpzIjU8XsvOO+J9b7rkIiQ8is34O+e0AbxBpv67hcj9uiPq1jtCoQQ8JfY/u86nAz0Wkf28LnrBPJlW9Tt8P4K7BbSjO9grhbyAOJS8G3K+vJLe3LzXpZA7NQUxPJs+JDz6vAS8QHZbvYNVYDrj3yk88PWIPOJ97zuSIVc8ZUPnPMqPsbx2cZi7QfzPOxYGDz2hqtO6H2tQO543NjyFPY+7JRUAOt0wgDyJeZu8MpKTu6AApTtg1ze82JI5vKllZjvrV0I7HX6nu7vndDxg1ze8jwQLu1ZTNjuJvBU7BXGpvAP+C7xJk6g8j2u/vBABlLzlqBi8M9WNutRWLTx0zGM9sHbKPLoZDDtmyVu8tpqFOvPumjyuRqe87lBUvFU0drxs7Za8ejMZOzJPGbyC7qu863v8PDPVjTxJ1iI7Ca01PLuAQLuNHFy7At9LOwP+i7tYxlO80NemO9elkDx45LU8h9TmuzxZjzz/5bk8p84OurvndLwAkGi7XL9luCSzRTwMgg08vrxMPKIwyDwdomG8K6VpPGPvCTxkmTi7M/lHPGxUSzxwKSM8wQuwvOqtkzrLFSa8SbdivAMixjw2r9+7xWt2vAyCDT1NEi87B8CMvG1zi7xpwm27MrbNO9R6Z7xJt+K7jNnhu9ZiFrve/ug55CKkvCwHJLqsOr47+ortvPwvIr2v8NW8YmmVOE+FTLywUhA8MTBZvMiDyLtx8hG8OEE9vMDsbzroCF88DelBOobnPbx+b6U8sbnEOywr3ro93wO9dMzjup2xwbwnRaO7cRZMu8Z337vS44+7VpYwvFWphzxKgNE8L1aHPLPFLbunzo66zFggPN+jHbs7tFo8nW7HO9JKRLyoeD28Fm1DPGZip7u5dNe7KMsXvFnlkzxQpAw7MrZNPHpX0zwSyoK7ayQovPR0Dz3gClK8/juLPDjaCLvqrZO7a4vcO9HEzzvife88KKzXvDmocbwpMkw7t2huvaIMjjznguo7Gy/EOzxZjzoLuZ48qi5VvCjLFzuDmNo654LquyrXgDy7XAa8e7mNvJ7QAb0Rq8K7ojBIvBN0MTuOfha8GoUVveb89bxMsHS8jV9WPPKM4LyAOJS8me9AvZv7qbsbcr47tuL5uaXmXzweKNa7rkYnPINV4Lxcv+W8tVcLvI8oxbzvbxS7oYaZu9+jHT0cHO08c7uAPCSzRTywUhA85xu2u+wBcTuJvJU8PBYVusTghzsnAim8acJtPFQE0zzFIwI9C7meO1DIRry7XAY8MKpkPJZd47suN0e5JTm6u6BDn7zfx1e8AJDoOr9CQbwaQps7x/1TPLTRFryqLtU8JybjPIXI/Tz6I7k6mVb1PMWKNryd1fs8Ok0mPHt2kzy9Ep48TTZpvPS3ibwGOpi8Ns4fPBqFlbr3Kqc8+QR5vHLA+rt7uY289YXyPI6iULxL4gu8Tv/XuycCKbwCnFG8C7kevVG1b7zIXw68GoWVO4rNeDnrM4i8MxgIPUNLs7zSoJW86ScfO+rRzbs6Cqw8NxGautP0cjw0wjY8CGq7vAkU6rxKgNG5+uA+vJXXbrwKM6o86vCNOu+yjjoQAZS8xATCOQVxKbynzo68wxcZvMhATjzS4488ArsRvNEaobwRh4i7t4euvAvd2DwnAik8UtQvvBFEDrz4sJs79gtnvOknnzy+vEy8D3sfPLH8vjzmLo28KVGMvOtXwjvpapm8HBxtPH3K8Lu753Q8/l9FvLvn9DomoG48fET8u9zy/7wMpke8zmQJu3oU2TzlD828KteAPAwNfLu+mBI5ldduPNZDVjq+vEy8eEvqvDHJpLwUPaC6qi7VPABsLjwFcSm72sJcu+bYO7v41NW8RiALvYB7DjzL0is7qLs3us1FSbzaf2K8MnNTuxABFDzF8Wo838fXvOBNzDzre3w8afQEvQE1nbulBaC78zEVvG5B9LzH/VM82Riuuwu5nrwsByQ8Y6yPvHXro7yQ0nM8nStNPJkyOzwnJmM80m7+O1VmjTzqrZM8dhvHOyAQBbz3baG8KTJMPOlqmbxsVEs8Pq3suy56QbzUVq08X3CDvAE1nTwUHuA7hue9vF8tCbvwOAO6F7A9ugd9kryqLtW7auEtu9ONPryPa7+8o9r2O570OzyFpEO8ntCBPOqtk7sykhO7lC1AOw2TcLswhiq6vx4HvP5fRbwuesG7Mk8ZvA4Z5TlfcAM9DrIwPL//xrzMm5q8JEwRPHBsnbxL4gu8jyjFu99gozrkZZ483GeRPLuAwDuYiIw8iv8PvK5Gpzx+b6W87Yflu3NGbzyE+hQ8a4tcPItT7bsoy5e8L1YHvWQyBDwrga86kPEzvBQ9oDxtl0W8lwKYvGpIYrxQ5wY8AJDovOLyALyw3f489JjJvMdTpTkKMyo8V9mqvH3K8LpyNYy8JHDLOixu2LpQ54Y8Q0uzu8LUnrs0wrY84vIAveihqjwfihA8DIKNvLDd/jywM1C7FB7gOxsLirxAUqE7sulnvH3K8DkAkGg8jsGQvO+TzrynWf287CCxvK4Drbwg8UQ8JRr6vFEqAbskjwu76q2TPNP0cjopDhK8dVJYvFIXKrxLn5G8AK8oPAb3HbxbOXE8Bvedun5Q5ThHyjk8QdiVvBXDlLw0o/Y7aLGKupkOgTxKPdc81kNWPtUAXLxUR827X1FDPf47izxsEVE8akhiPIhaWzxYX5+7hT0PPSrXgLxQC0E8i4WEvKUp2jtCLHM8DcWHO768zLxnK5a89R6+vH9czrorpem73h0pvAnwr7yKzXi8gDgUPf47Czq9zyO8728UOf34EDy6PUY76OSkvKZIGr2ZDgE8gzEmPG3av7v77Ce7/oP/O3MiNTtas/w8x1OlO/D1CDvDfs27ll1jO2Ufrbv1hXK8WINZuxN0sbuxlYq8OYS3uia/rjyiTwi9O7TaO+/WyDyiDA49E7erO3fF9bj6I7k7qHi9O3SoKbyBSfc7drSSvGPvCT2pQay7t2huPGnC7byUCQY8CEaBu6rHoDhx8hE8/fgQvCjLl7zdeHS8x/3TO0Isc7tas3y8jwQLvUKhhDz+foU8fCDCPC+ZgTywD5Y7ZR8tOla66rtCCLm8gWg3vDoKrLxbWDE76SefPBkj2zrlqJi7pebfuv6Df7zWQ9a7lHA6PGDXtzzMv1Q8mtxpOwJ4lzxKGZ28mGnMPDw6z7yxY/O7m2Leu7juYjwvVge8zFigPGpIYjtWumo5xs2wOgyCjbxrZ6K8bbaFvKzTCbsks8W7C7mePIU9DzxQyEY8posUvAW0ozrHlh88CyBTPJRwursxySQ757SBuqcRCbwNCIK8EL6ZvIG+iLsIRgE8rF74vOJZtbuUcDq8r/DVPMpMt7sL3Vi8eWqquww/kzqj2vY5auGtu85kiTwMPxM66KGqvBIxNzuwUpA8v2b7u09C0rx7ms08NUirvFYQPLxKPdc68mimvP5fRTtoPPm7XuqOOgOJ+jxfLYm7u58AvXz8B72PR4W6ldfuuys+tbvYKwW7pkiaPLB2SjvKj7G875POvA6yML7qFEg9Eu68O6Up2rz77Kc84CmSPP6ivzz4sJu6/C+iOaUpWjwq14A84E3MOYB7Dr2d1Xu775NOvC6e+7spUYw8PzPhO5TGizt29ww9yNkZPY7lyrz020M7QRsQu3z8BzwkCZe79YXyO8jZmTzvGUM8HgQcO9kYrrzxBmy8hLeaPLYBOjz+oj88flBlO6GqUzuiMMi8fxlUvCr7ujz41NU8DA38PBeMAzx7uY28TTZpvFG1bzxtc4s89ucsPEereTwfipC82p4iPKtNFbzo5KQ7pcKlOW5gtDzO73c7B6FMOzRbgjxCXoo8v0JBOSl1RrwxDJ+7XWSaPD3Aw7sOsjA8tuJ5vKw6Pry5k5c8ZUNnvG/H6DyVTAA8Shkdvd7+aDvtpiW9qUGsPFTgmDwbcr68TTbpO1DnhryNX9a7mrivvIqpPjxsqhy81HrnOzv31Dvth+U6UtQvPBz4MrvtpqW84OYXvRz4sjxwkFe8zSGPuycCqbyFPY8818nKOw84JTy8bWk8USqBvBGHiLtosQo8BOs0u9skl7xQ54Y8uvrLPOknn7w705o8Jny0PAd9EjxhoKa8Iv2tu2M3/jtsVEs8DcUHPQSEADs3eE48GkKbupRR+rvdeHQ7Xy2JvO1jKz0xMFm8sWPzux07LbyrTZW7bdq/O6Pa9r0ahRW9CyDTOjSjdjyQ8bO8yaIIPfupLTz/CfQ7xndfvJs+JD0zPEK8KO/RvMpw8bwObzY7fm+lPJtiXrz5BHm8WmsIvKlBrLuDdKA7hWHJOgd9Ers0o/Y7nlvwu5NAl7u8BrW6utYRO2SZuDxyNYw8CppevAY6GDxVqQe9oGdZPFa6ary3RLS70NcmO2PQSb36ZrM86q2TPML42LwewaE8k2RRPDmocTsi/S29o/k2PHRlr7zjnC+8gHsOPUpcFzxtl8W6tuL5vHw/gry/2wy9yaIIvINV4Dx3fQG7ISFoPO7pnzwGXlK8HPiyPGAaMjzBC7A7MQyfu+eC6jyV1+67pDyxvBWkVLxrJKg754LqOScCKbwpUQy8KIgdOJDSc7zDfk08tLLWvNZDVjyh7c28ShmdvMnlgjs2NdS8ISHovP5+hbxGIIs8ayQouyKnXDzBcmS6zw44u86IQ7yl5l+7cngGvWvOVrsEhIC7yNkZPJODkbuAn0g8XN6lPOaVwbuTgxG8OR2DPAb3HTzlqJi8nUoNvCAVf73Mmxo9afSEu4FotzveHSk8c0ZvOMFOqjwP9Sq87iwavIEBg7xIUK68IbozuozZ4btg17c7vx4Hvarr2rtp9IQ8Rt0QO+1jqzyeNzY8kNLzO8sVpry98108OCL9uyisV7vhr4Y8FgaPvLFjczw42og8gWg3vPX6gzsNk/C83GeRPCUVgDy0jpw7yNkZu2VD5zvh93o81h+cuw3Fhzyl5t+86Y7TvHa0EjyzCCi7WmsIPIy1Jzy00Ra6NUiru50rTTx50d47/HKcO2wwETw0f7y8sFIQvNxnkbzS4w855pVBu9FdGzx9yvC6TM80vFQjkzy/Zvs7BhtYPLjKKLqPa787A/6LOyiInbzooSq8728UPIFJ97wq+7q8R6v5u1tYMbwdomG6iSPKPAb3HTx3oTu7fGO8POqtk7ze/ug84wNkPMnq/DsB8iK9ogwOu6lBrDznguo8NQUxvHKcwDo28tm7yNmZPN1UurxCoYS80m7+Oy+9OzzGzTC836MdvCDNCrtaawi7dVLYPEfKuTxzRm88cCmjOyXSBbwGOpi879ZIO8dTJbtqnrO8NMI2vR1+J7xwTV087umfPFG17zsC30s8oYaZPKllZrzZGK47zss9vP21FryZywa9bbYFPVNapDt2G0e7E3SxPMUjgry5dNc895Hbu0H8z7ueN7a7OccxPFhfH7vC1B48n3owvEhQLrzu6Z+8HTutvEBSITw6Taa5g1XgPCzEqbxfLYk9OYQ3vBlm1bvPUTI8wIU7PIy1pzyFyP07gzGmO3NGb7yS3ty7O5CguyEhaLyWoF28pmxUOaZImrz+g/87mnU1vFbsgTxvo668PFmPO2KNTzy09VC8LG5YPHhL6rsvJPC7kTQuvEGCxDlhB9s6u58AvfCAd7z0t4k7kVjoOCkOkrxMjDq8iPOmPL0SnrxsMJG7OEG9vCUa+rvx4rE7cpxAPDCGqjukf6u8TEnAvNn57TweBBw7JdKFvIy1p7vIg8i7" + embeddings = "VEfNvMLUnrwFleO8hcj9vEE/yrzyjOA84E1MvNfoCrxjrI+8sZUKvNgrBT17uY07gJ/IvNvhHLrUemc8KXXGumalIT3YKwU7ZsnbPMhATrwTt6u8JEwRPNMmCjxGREW7TRKvu6/MG7zAyDU8wXLkuuMDZDsXsL28zHzaOw0IArzOiMO8LtASvPKM4Dul5l+80V0bPGVDZ7wYNrI89ucsvJZdYztzRm+8P8ysOyGbc7zrdgK9sdiEPKQ8sbulKdq7KIgdvKIMDj25dNc8k0AXPBn/oLzrdgK8IXe5uz0Dvrt50V68tTjLO4ZOcjoG9x29oGfZufiwmzwMDXy8EL6ZPHvdx7nKjzE8+LCbPG22hTs3EZq7TM+0POrRzTxVZo084wPkO8Nak7z8cpw8pDwxvA2T8LvBC7C72fltvC8Atjp3fYE8JHDLvEYgC7xAdls8YiabPPkEeTzPUbK8gOLCPEBSIbyt5Oy8CpreusNakzywUhA824vLPHRlr7zAhTs7IZtzvHd9AT2xY/O6ok8IvOihqrql5l88K4EvuknWorvYKwW9iXkbvGMTRLw5qPG7onPCPLgNIzwAbK67ftbZPMxYILvAyDW9TLB0vIid1buzCKi7u+d0u8iDSLxNVam8PZyJPNxnETvVANw8Oi5mu9nVszzl65I7DIKNvLGVirxsMJE7tPXQu2PvCT1zRm87p1l9uyRMkbsdfqe8U52ePHRlr7wt9Mw8/C8ivTu02rwJFGq8tpoFPWnC7blWumq7sfy+vG1zCzy9Nlg8iv+PuvxT3DuLU228kVhoOkmTqDrv1kg8ocmTu1WpBzsKml48DzglvI8ECzxwTd27I+pWvIWkQ7xUR007GqlPPBFEDrzGECu865q8PI7BkDwNxYc8tgG6ullMSLsIajs84lk1PNLjD70mv648ZmInO2tnIjzvb5Q8o5KCPLo9xrwKMyq9QqGEvI8ECzxO2508ATUdPRAlTry5kxc8KVGMPJyBHjxIUC476KGqvIU9DzwX87c88PUIParrWrzdlzS/G3K+uzEw2TxB2BU86AhfPAMiRj2dK808a85WPPCft7xU4Bg95Q9NPDxZjzwrpek7yNkZvHa0EjyQ0nM6Nq9fuyjvUbsRq8I7CAMHO3VSWLyuauE7U1qkvPkEeTxs7ZY7B6FMO48Eizy75/S7ieBPvB07rTxmyVu8onPCO5rc6Tu7XIa7oEMfPYngT7u24vk7/+W5PE8eGDxJ1iI9t4cuvBGHiLyH1GY7jfghu+oUSDwa7Mk7iXmbuut2grrq8I2563v8uyofdTxRTrs44lm1vMeWnzukf6s7r4khvEKhhDyhyZO8G5Z4Oy56wTz4sBs81Zknuz3fg7wnJuO74n1vvASEADu98128gUl3vBtyvrtZCU47yep8u5FYaDx2G0e8a85WO5cmUjz3kds8qgqbPCUaerx50d67WKIZPI7BkDua3Om74vKAvL3zXbzXpRA9CI51vLo9xryKzXg7tXtFO9RWLTwnJuM854LqPEIs8zuO5cq8d8V1u9P0cjrQ++C8cGwdPDdUlLoOGeW8auEtu8Z337nlzFK8aRg/vFCkDD0nRSM879bIvKUFID1iStU8EL6ZvLufgLtKgNE7KVEMvJOnSzwahRU895HbvJiIjLvc8n88bmC0PPLP2rywM9C7jTscOoS3mjy/Znu7dhvHuu5Q1Dyq61o6CI71u09hkry0jhw8gb6IPI8EC7uoVAM8gs9rvGM3fjx2G8e81FYtu/ojubyYRRK72Riuu83elDtNNmk70/TyuzUFsbvgKZI7onNCvAehzLumr8679R6+urr6SztX2So8Bl5SOwSEgLv5NpA8LwC2PGPvibzJ6vw7H2tQvOtXwrzXpRC8j0z/uxwcbTy2vr+8VWYNu+t2ArwKmt68NKN2O3XrIzw9A747UU47vaavzjwU+qW8YBqyvE02aTyEt5o8cCmjOxtyPrxs7ZY775NOu+SJWLxMJQY8/bWWu6IMDrzSSsQ7GSPbPLlQnbpVzcE7Pka4PJ96sLycxJg8v/9GPO2HZTyeW3C8Vpawtx2iYTwWBg87/qI/OviwGzxyWcY7M9WNPIA4FD32C2e8tNGWPJ43trxCoYS8FGHavItTbbu7n4C80NemPLm30Ty1OMu7vG1pvG3aPztBP0o75Q/NPJhFEj2V9i683PL/O97+aLz6iu27cdPRum/mKLwvVgc89fqDu3LA+jvm2Ls8mVZ1PIuFBD3ZGK47Cpreut7+aLziWTU8XSEgPMvSKzzO73e5040+vBlmVTxS1K+8mQ4BPZZ8o7w8FpW6OR0DPSSPCz21Vwu99fqDOjMYiDy7XAY8oYaZO+aVwTyX49c84OaXOqdZfTunEQk7B8AMvMDs7zo/D6e8OP5CvN9gIzwNCII8FefOPE026TpzIjU8XsvOO+J9b7rkIiQ8is34O+e0AbxBpv67hcj9uiPq1jtCoQQ8JfY/u86nAz0Wkf28LnrBPJlW9Tt8P4K7BbSjO9grhbyAOJS8G3K+vJLe3LzXpZA7NQUxPJs+JDz6vAS8QHZbvYNVYDrj3yk88PWIPOJ97zuSIVc8ZUPnPMqPsbx2cZi7QfzPOxYGDz2hqtO6H2tQO543NjyFPY+7JRUAOt0wgDyJeZu8MpKTu6AApTtg1ze82JI5vKllZjvrV0I7HX6nu7vndDxg1ze8jwQLu1ZTNjuJvBU7BXGpvAP+C7xJk6g8j2u/vBABlLzlqBi8M9WNutRWLTx0zGM9sHbKPLoZDDtmyVu8tpqFOvPumjyuRqe87lBUvFU0drxs7Za8ejMZOzJPGbyC7qu863v8PDPVjTxJ1iI7Ca01PLuAQLuNHFy7At9LOwP+i7tYxlO80NemO9elkDx45LU8h9TmuzxZjzz/5bk8p84OurvndLwAkGi7XL9luCSzRTwMgg08vrxMPKIwyDwdomG8K6VpPGPvCTxkmTi7M/lHPGxUSzxwKSM8wQuwvOqtkzrLFSa8SbdivAMixjw2r9+7xWt2vAyCDT1NEi87B8CMvG1zi7xpwm27MrbNO9R6Z7xJt+K7jNnhu9ZiFrve/ug55CKkvCwHJLqsOr47+ortvPwvIr2v8NW8YmmVOE+FTLywUhA8MTBZvMiDyLtx8hG8OEE9vMDsbzroCF88DelBOobnPbx+b6U8sbnEOywr3ro93wO9dMzjup2xwbwnRaO7cRZMu8Z337vS44+7VpYwvFWphzxKgNE8L1aHPLPFLbunzo66zFggPN+jHbs7tFo8nW7HO9JKRLyoeD28Fm1DPGZip7u5dNe7KMsXvFnlkzxQpAw7MrZNPHpX0zwSyoK7ayQovPR0Dz3gClK8/juLPDjaCLvqrZO7a4vcO9HEzzvife88KKzXvDmocbwpMkw7t2huvaIMjjznguo7Gy/EOzxZjzoLuZ48qi5VvCjLFzuDmNo654LquyrXgDy7XAa8e7mNvJ7QAb0Rq8K7ojBIvBN0MTuOfha8GoUVveb89bxMsHS8jV9WPPKM4LyAOJS8me9AvZv7qbsbcr47tuL5uaXmXzweKNa7rkYnPINV4Lxcv+W8tVcLvI8oxbzvbxS7oYaZu9+jHT0cHO08c7uAPCSzRTywUhA85xu2u+wBcTuJvJU8PBYVusTghzsnAim8acJtPFQE0zzFIwI9C7meO1DIRry7XAY8MKpkPJZd47suN0e5JTm6u6BDn7zfx1e8AJDoOr9CQbwaQps7x/1TPLTRFryqLtU8JybjPIXI/Tz6I7k6mVb1PMWKNryd1fs8Ok0mPHt2kzy9Ep48TTZpvPS3ibwGOpi8Ns4fPBqFlbr3Kqc8+QR5vHLA+rt7uY289YXyPI6iULxL4gu8Tv/XuycCKbwCnFG8C7kevVG1b7zIXw68GoWVO4rNeDnrM4i8MxgIPUNLs7zSoJW86ScfO+rRzbs6Cqw8NxGautP0cjw0wjY8CGq7vAkU6rxKgNG5+uA+vJXXbrwKM6o86vCNOu+yjjoQAZS8xATCOQVxKbynzo68wxcZvMhATjzS4488ArsRvNEaobwRh4i7t4euvAvd2DwnAik8UtQvvBFEDrz4sJs79gtnvOknnzy+vEy8D3sfPLH8vjzmLo28KVGMvOtXwjvpapm8HBxtPH3K8Lu753Q8/l9FvLvn9DomoG48fET8u9zy/7wMpke8zmQJu3oU2TzlD828KteAPAwNfLu+mBI5ldduPNZDVjq+vEy8eEvqvDHJpLwUPaC6qi7VPABsLjwFcSm72sJcu+bYO7v41NW8RiALvYB7DjzL0is7qLs3us1FSbzaf2K8MnNTuxABFDzF8Wo838fXvOBNzDzre3w8afQEvQE1nbulBaC78zEVvG5B9LzH/VM82Riuuwu5nrwsByQ8Y6yPvHXro7yQ0nM8nStNPJkyOzwnJmM80m7+O1VmjTzqrZM8dhvHOyAQBbz3baG8KTJMPOlqmbxsVEs8Pq3suy56QbzUVq08X3CDvAE1nTwUHuA7hue9vF8tCbvwOAO6F7A9ugd9kryqLtW7auEtu9ONPryPa7+8o9r2O570OzyFpEO8ntCBPOqtk7sykhO7lC1AOw2TcLswhiq6vx4HvP5fRbwuesG7Mk8ZvA4Z5TlfcAM9DrIwPL//xrzMm5q8JEwRPHBsnbxL4gu8jyjFu99gozrkZZ483GeRPLuAwDuYiIw8iv8PvK5Gpzx+b6W87Yflu3NGbzyE+hQ8a4tcPItT7bsoy5e8L1YHvWQyBDwrga86kPEzvBQ9oDxtl0W8lwKYvGpIYrxQ5wY8AJDovOLyALyw3f489JjJvMdTpTkKMyo8V9mqvH3K8LpyNYy8JHDLOixu2LpQ54Y8Q0uzu8LUnrs0wrY84vIAveihqjwfihA8DIKNvLDd/jywM1C7FB7gOxsLirxAUqE7sulnvH3K8DkAkGg8jsGQvO+TzrynWf287CCxvK4Drbwg8UQ8JRr6vFEqAbskjwu76q2TPNP0cjopDhK8dVJYvFIXKrxLn5G8AK8oPAb3HbxbOXE8Bvedun5Q5ThHyjk8QdiVvBXDlLw0o/Y7aLGKupkOgTxKPdc81kNWPtUAXLxUR827X1FDPf47izxsEVE8akhiPIhaWzxYX5+7hT0PPSrXgLxQC0E8i4WEvKUp2jtCLHM8DcWHO768zLxnK5a89R6+vH9czrorpem73h0pvAnwr7yKzXi8gDgUPf47Czq9zyO8728UOf34EDy6PUY76OSkvKZIGr2ZDgE8gzEmPG3av7v77Ce7/oP/O3MiNTtas/w8x1OlO/D1CDvDfs27ll1jO2Ufrbv1hXK8WINZuxN0sbuxlYq8OYS3uia/rjyiTwi9O7TaO+/WyDyiDA49E7erO3fF9bj6I7k7qHi9O3SoKbyBSfc7drSSvGPvCT2pQay7t2huPGnC7byUCQY8CEaBu6rHoDhx8hE8/fgQvCjLl7zdeHS8x/3TO0Isc7tas3y8jwQLvUKhhDz+foU8fCDCPC+ZgTywD5Y7ZR8tOla66rtCCLm8gWg3vDoKrLxbWDE76SefPBkj2zrlqJi7pebfuv6Df7zWQ9a7lHA6PGDXtzzMv1Q8mtxpOwJ4lzxKGZ28mGnMPDw6z7yxY/O7m2Leu7juYjwvVge8zFigPGpIYjtWumo5xs2wOgyCjbxrZ6K8bbaFvKzTCbsks8W7C7mePIU9DzxQyEY8posUvAW0ozrHlh88CyBTPJRwursxySQ757SBuqcRCbwNCIK8EL6ZvIG+iLsIRgE8rF74vOJZtbuUcDq8r/DVPMpMt7sL3Vi8eWqquww/kzqj2vY5auGtu85kiTwMPxM66KGqvBIxNzuwUpA8v2b7u09C0rx7ms08NUirvFYQPLxKPdc68mimvP5fRTtoPPm7XuqOOgOJ+jxfLYm7u58AvXz8B72PR4W6ldfuuys+tbvYKwW7pkiaPLB2SjvKj7G875POvA6yML7qFEg9Eu68O6Up2rz77Kc84CmSPP6ivzz4sJu6/C+iOaUpWjwq14A84E3MOYB7Dr2d1Xu775NOvC6e+7spUYw8PzPhO5TGizt29ww9yNkZPY7lyrz020M7QRsQu3z8BzwkCZe79YXyO8jZmTzvGUM8HgQcO9kYrrzxBmy8hLeaPLYBOjz+oj88flBlO6GqUzuiMMi8fxlUvCr7ujz41NU8DA38PBeMAzx7uY28TTZpvFG1bzxtc4s89ucsPEereTwfipC82p4iPKtNFbzo5KQ7pcKlOW5gtDzO73c7B6FMOzRbgjxCXoo8v0JBOSl1RrwxDJ+7XWSaPD3Aw7sOsjA8tuJ5vKw6Pry5k5c8ZUNnvG/H6DyVTAA8Shkdvd7+aDvtpiW9qUGsPFTgmDwbcr68TTbpO1DnhryNX9a7mrivvIqpPjxsqhy81HrnOzv31Dvth+U6UtQvPBz4MrvtpqW84OYXvRz4sjxwkFe8zSGPuycCqbyFPY8818nKOw84JTy8bWk8USqBvBGHiLtosQo8BOs0u9skl7xQ54Y8uvrLPOknn7w705o8Jny0PAd9EjxhoKa8Iv2tu2M3/jtsVEs8DcUHPQSEADs3eE48GkKbupRR+rvdeHQ7Xy2JvO1jKz0xMFm8sWPzux07LbyrTZW7bdq/O6Pa9r0ahRW9CyDTOjSjdjyQ8bO8yaIIPfupLTz/CfQ7xndfvJs+JD0zPEK8KO/RvMpw8bwObzY7fm+lPJtiXrz5BHm8WmsIvKlBrLuDdKA7hWHJOgd9Ers0o/Y7nlvwu5NAl7u8BrW6utYRO2SZuDxyNYw8CppevAY6GDxVqQe9oGdZPFa6ary3RLS70NcmO2PQSb36ZrM86q2TPML42LwewaE8k2RRPDmocTsi/S29o/k2PHRlr7zjnC+8gHsOPUpcFzxtl8W6tuL5vHw/gry/2wy9yaIIvINV4Dx3fQG7ISFoPO7pnzwGXlK8HPiyPGAaMjzBC7A7MQyfu+eC6jyV1+67pDyxvBWkVLxrJKg754LqOScCKbwpUQy8KIgdOJDSc7zDfk08tLLWvNZDVjyh7c28ShmdvMnlgjs2NdS8ISHovP5+hbxGIIs8ayQouyKnXDzBcmS6zw44u86IQ7yl5l+7cngGvWvOVrsEhIC7yNkZPJODkbuAn0g8XN6lPOaVwbuTgxG8OR2DPAb3HTzlqJi8nUoNvCAVf73Mmxo9afSEu4FotzveHSk8c0ZvOMFOqjwP9Sq87iwavIEBg7xIUK68IbozuozZ4btg17c7vx4Hvarr2rtp9IQ8Rt0QO+1jqzyeNzY8kNLzO8sVpry98108OCL9uyisV7vhr4Y8FgaPvLFjczw42og8gWg3vPX6gzsNk/C83GeRPCUVgDy0jpw7yNkZu2VD5zvh93o81h+cuw3Fhzyl5t+86Y7TvHa0EjyzCCi7WmsIPIy1Jzy00Ra6NUiru50rTTx50d47/HKcO2wwETw0f7y8sFIQvNxnkbzS4w855pVBu9FdGzx9yvC6TM80vFQjkzy/Zvs7BhtYPLjKKLqPa787A/6LOyiInbzooSq8728UPIFJ97wq+7q8R6v5u1tYMbwdomG6iSPKPAb3HTx3oTu7fGO8POqtk7ze/ug84wNkPMnq/DsB8iK9ogwOu6lBrDznguo8NQUxvHKcwDo28tm7yNmZPN1UurxCoYS80m7+Oy+9OzzGzTC836MdvCDNCrtaawi7dVLYPEfKuTxzRm88cCmjOyXSBbwGOpi879ZIO8dTJbtqnrO8NMI2vR1+J7xwTV087umfPFG17zsC30s8oYaZPKllZrzZGK47zss9vP21FryZywa9bbYFPVNapDt2G0e7E3SxPMUjgry5dNc895Hbu0H8z7ueN7a7OccxPFhfH7vC1B48n3owvEhQLrzu6Z+8HTutvEBSITw6Taa5g1XgPCzEqbxfLYk9OYQ3vBlm1bvPUTI8wIU7PIy1pzyFyP07gzGmO3NGb7yS3ty7O5CguyEhaLyWoF28pmxUOaZImrz+g/87mnU1vFbsgTxvo668PFmPO2KNTzy09VC8LG5YPHhL6rsvJPC7kTQuvEGCxDlhB9s6u58AvfCAd7z0t4k7kVjoOCkOkrxMjDq8iPOmPL0SnrxsMJG7OEG9vCUa+rvx4rE7cpxAPDCGqjukf6u8TEnAvNn57TweBBw7JdKFvIy1p7vIg8i7" # noqa: E501 data = [] for i, text in enumerate(input): diff --git a/api/tests/unit_tests/core/app/segments/test_segment.py b/api/tests/unit_tests/core/app/segments/test_segment.py index 7cc339d212..73002623f0 100644 --- a/api/tests/unit_tests/core/app/segments/test_segment.py +++ b/api/tests/unit_tests/core/app/segments/test_segment.py @@ -21,9 +21,9 @@ def test_segment_group_to_text(): segments_group = parser.convert_template(template=template, variable_pool=variable_pool) assert segments_group.text == "Hello, fake-user-id! Your query is fake-user-query. And your key is fake-secret-key." - assert ( - segments_group.log - == f"Hello, fake-user-id! Your query is fake-user-query. And your key is {encrypter.obfuscated_token('fake-secret-key')}." + assert segments_group.log == ( + f"Hello, fake-user-id! Your query is fake-user-query." + f" And your key is {encrypter.obfuscated_token('fake-secret-key')}." ) From 40fb4d16ef0f5e8dd984c205b9941df68a445b0f Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Thu, 12 Sep 2024 15:50:49 +0800 Subject: [PATCH 5/5] chore: refurbish Python code by applying refurb linter rules (#8296) --- api/controllers/console/admin.py | 24 +++++++------------ api/controllers/console/app/audio.py | 8 ++----- api/controllers/console/auth/oauth.py | 2 +- api/controllers/console/datasets/datasets.py | 7 +----- api/controllers/console/explore/audio.py | 8 ++----- .../console/workspace/tool_providers.py | 2 +- api/controllers/service_api/app/audio.py | 8 ++----- api/controllers/web/audio.py | 8 ++----- api/core/agent/cot_agent_runner.py | 2 +- api/core/agent/fc_agent_runner.py | 2 +- api/core/app/apps/base_app_runner.py | 8 +++---- api/core/extension/extensible.py | 4 ++-- api/core/memory/token_buffer_memory.py | 2 +- .../__base/large_language_model.py | 2 +- .../model_providers/anthropic/llm/llm.py | 2 +- .../azure_ai_studio/llm/llm.py | 2 +- .../model_providers/azure_openai/llm/llm.py | 10 ++++---- .../model_providers/azure_openai/tts/tts.py | 2 +- .../model_providers/bedrock/llm/llm.py | 8 +++---- .../model_providers/chatglm/llm/llm.py | 4 ++-- .../model_providers/localai/llm/llm.py | 6 ++--- .../model_providers/minimax/llm/llm.py | 4 ++-- .../ollama/text_embedding/text_embedding.py | 2 +- .../model_providers/openai/llm/llm.py | 8 +++---- .../model_providers/openai/tts/tts.py | 2 +- .../openai_api_compatible/llm/llm.py | 4 ++-- .../model_providers/openllm/llm/llm.py | 4 ++-- .../openllm/llm/openllm_generate.py | 2 +- .../model_providers/replicate/llm/llm.py | 2 +- .../sagemaker/rerank/rerank.py | 3 ++- .../model_providers/sagemaker/tts/tts.py | 2 +- .../model_providers/spark/llm/llm.py | 2 +- .../tencent/speech2text/flash_recognizer.py | 3 ++- .../model_providers/tongyi/llm/llm.py | 4 ++-- .../model_providers/upstage/llm/llm.py | 6 ++--- .../model_providers/vertex_ai/llm/llm.py | 4 ++-- .../legacy/volc_sdk/base/auth.py | 3 ++- .../legacy/volc_sdk/base/util.py | 3 ++- .../volcengine_maas/llm/llm.py | 8 +++---- .../model_providers/wenxin/llm/llm.py | 6 ++--- .../wenxin/text_embedding/text_embedding.py | 2 +- .../model_providers/xinference/llm/llm.py | 6 ++--- .../model_providers/xinference/tts/tts.py | 2 +- .../model_providers/zhipuai/llm/llm.py | 4 ++-- .../zhipuai/zhipuai_sdk/core/_http_client.py | 4 +++- api/core/ops/langfuse_trace/langfuse_trace.py | 22 ++++++++--------- .../ops/langsmith_trace/langsmith_trace.py | 10 ++++---- api/core/ops/ops_trace_manager.py | 15 ++++++------ api/core/prompt/simple_prompt_transform.py | 6 ++--- .../rag/datasource/keyword/keyword_base.py | 2 +- .../vdb/analyticdb/analyticdb_vector.py | 4 ++-- .../datasource/vdb/chroma/chroma_vector.py | 2 +- .../vdb/elasticsearch/elasticsearch_vector.py | 6 ++--- .../datasource/vdb/milvus/milvus_vector.py | 2 +- .../datasource/vdb/myscale/myscale_vector.py | 2 +- .../vdb/opensearch/opensearch_vector.py | 2 +- .../rag/datasource/vdb/oracle/oraclevector.py | 4 ++-- .../datasource/vdb/pgvecto_rs/pgvecto_rs.py | 2 +- .../rag/datasource/vdb/pgvector/pgvector.py | 2 +- .../datasource/vdb/qdrant/qdrant_vector.py | 2 +- .../rag/datasource/vdb/relyt/relyt_vector.py | 2 +- .../datasource/vdb/tencent/tencent_vector.py | 2 +- .../datasource/vdb/tidb_vector/tidb_vector.py | 2 +- api/core/rag/datasource/vdb/vector_base.py | 2 +- api/core/rag/datasource/vdb/vector_factory.py | 2 +- .../vdb/weaviate/weaviate_vector.py | 2 +- api/core/rag/extractor/blob/blob.py | 8 +++---- api/core/rag/extractor/extract_processor.py | 7 +++--- api/core/rag/extractor/helpers.py | 4 ++-- api/core/rag/extractor/markdown_extractor.py | 7 +++--- api/core/rag/extractor/text_extractor.py | 7 +++--- api/core/rag/extractor/word_extractor.py | 2 +- api/core/rag/retrieval/dataset_retrieval.py | 10 ++++---- api/core/tools/provider/api_tool_provider.py | 2 +- .../aws/tools/sagemaker_text_rerank.py | 3 ++- .../builtin/hap/tools/get_worksheet_fields.py | 2 +- .../hap/tools/get_worksheet_pivot_data.py | 2 +- .../hap/tools/list_worksheet_records.py | 2 +- .../searchapi/tools/youtube_transcripts.py | 2 +- .../dataset_multi_retriever_tool.py | 6 ++--- .../dataset_retriever_tool.py | 6 ++--- api/core/tools/utils/web_reader_tool.py | 6 ++--- api/core/tools/utils/yaml_utils.py | 2 +- .../workflow/graph_engine/entities/graph.py | 5 ++-- api/core/workflow/nodes/code/code_node.py | 8 +++---- .../workflow/nodes/if_else/if_else_node.py | 2 +- api/core/workflow/nodes/llm/llm_node.py | 2 +- .../question_classifier_node.py | 2 +- ...aset_join_when_app_model_config_updated.py | 3 +-- ...oin_when_app_published_workflow_updated.py | 3 +-- api/extensions/storage/local_storage.py | 8 +++---- api/models/dataset.py | 4 ++-- api/models/model.py | 10 ++++---- api/pyproject.toml | 3 +++ api/services/account_service.py | 2 +- api/services/app_dsl_service.py | 12 ++++------ api/services/dataset_service.py | 8 ++----- api/services/hit_testing_service.py | 6 ++--- api/services/model_provider_service.py | 6 ++--- api/services/recommended_app_service.py | 8 +++---- api/services/tag_service.py | 2 +- .../tools/builtin_tools_manage_service.py | 4 ++-- api/services/website_service.py | 4 ++-- api/services/workflow/workflow_converter.py | 8 +++---- .../model_runtime/__mock/huggingface_tei.py | 2 +- 105 files changed, 220 insertions(+), 276 deletions(-) diff --git a/api/controllers/console/admin.py b/api/controllers/console/admin.py index a4ceec2662..f78ea9b288 100644 --- a/api/controllers/console/admin.py +++ b/api/controllers/console/admin.py @@ -60,23 +60,15 @@ class InsertExploreAppListApi(Resource): site = app.site if not site: - desc = args["desc"] if args["desc"] else "" - copy_right = args["copyright"] if args["copyright"] else "" - privacy_policy = args["privacy_policy"] if args["privacy_policy"] else "" - custom_disclaimer = args["custom_disclaimer"] if args["custom_disclaimer"] else "" + desc = args["desc"] or "" + copy_right = args["copyright"] or "" + privacy_policy = args["privacy_policy"] or "" + custom_disclaimer = args["custom_disclaimer"] or "" else: - desc = site.description if site.description else args["desc"] if args["desc"] else "" - copy_right = site.copyright if site.copyright else args["copyright"] if args["copyright"] else "" - privacy_policy = ( - site.privacy_policy if site.privacy_policy else args["privacy_policy"] if args["privacy_policy"] else "" - ) - custom_disclaimer = ( - site.custom_disclaimer - if site.custom_disclaimer - else args["custom_disclaimer"] - if args["custom_disclaimer"] - else "" - ) + desc = site.description or args["desc"] or "" + copy_right = site.copyright or args["copyright"] or "" + privacy_policy = site.privacy_policy or args["privacy_policy"] or "" + custom_disclaimer = site.custom_disclaimer or args["custom_disclaimer"] or "" recommended_app = RecommendedApp.query.filter(RecommendedApp.app_id == args["app_id"]).first() diff --git a/api/controllers/console/app/audio.py b/api/controllers/console/app/audio.py index 437a6a7b38..7332758e83 100644 --- a/api/controllers/console/app/audio.py +++ b/api/controllers/console/app/audio.py @@ -99,14 +99,10 @@ class ChatMessageTextApi(Resource): and app_model.workflow.features_dict ): text_to_speech = app_model.workflow.features_dict.get("text_to_speech") - voice = args.get("voice") if args.get("voice") else text_to_speech.get("voice") + voice = args.get("voice") or text_to_speech.get("voice") else: try: - voice = ( - args.get("voice") - if args.get("voice") - else app_model.app_model_config.text_to_speech_dict.get("voice") - ) + voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice") except Exception: voice = None response = AudioService.transcript_tts(app_model=app_model, text=text, message_id=message_id, voice=voice) diff --git a/api/controllers/console/auth/oauth.py b/api/controllers/console/auth/oauth.py index ae1b49f3ec..1df0f5de9d 100644 --- a/api/controllers/console/auth/oauth.py +++ b/api/controllers/console/auth/oauth.py @@ -101,7 +101,7 @@ def _generate_account(provider: str, user_info: OAuthUserInfo): if not account: # Create account - account_name = user_info.name if user_info.name else "Dify" + account_name = user_info.name or "Dify" account = RegisterService.register( email=user_info.email, name=account_name, password=None, open_id=user_info.id, provider=provider ) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 6ccacc78ee..08603bfc21 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -550,12 +550,7 @@ class DatasetApiBaseUrlApi(Resource): @login_required @account_initialization_required def get(self): - return { - "api_base_url": ( - dify_config.SERVICE_API_URL if dify_config.SERVICE_API_URL else request.host_url.rstrip("/") - ) - + "/v1" - } + return {"api_base_url": (dify_config.SERVICE_API_URL or request.host_url.rstrip("/")) + "/v1"} class DatasetRetrievalSettingApi(Resource): diff --git a/api/controllers/console/explore/audio.py b/api/controllers/console/explore/audio.py index 71cb060ecc..2eb7e04490 100644 --- a/api/controllers/console/explore/audio.py +++ b/api/controllers/console/explore/audio.py @@ -86,14 +86,10 @@ class ChatTextApi(InstalledAppResource): and app_model.workflow.features_dict ): text_to_speech = app_model.workflow.features_dict.get("text_to_speech") - voice = args.get("voice") if args.get("voice") else text_to_speech.get("voice") + voice = args.get("voice") or text_to_speech.get("voice") else: try: - voice = ( - args.get("voice") - if args.get("voice") - else app_model.app_model_config.text_to_speech_dict.get("voice") - ) + voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice") except Exception: voice = None response = AudioService.transcript_tts(app_model=app_model, message_id=message_id, voice=voice, text=text) diff --git a/api/controllers/console/workspace/tool_providers.py b/api/controllers/console/workspace/tool_providers.py index c41a898fdc..d2a17b133b 100644 --- a/api/controllers/console/workspace/tool_providers.py +++ b/api/controllers/console/workspace/tool_providers.py @@ -327,7 +327,7 @@ class ToolApiProviderPreviousTestApi(Resource): return ApiToolManageService.test_api_tool_preview( current_user.current_tenant_id, - args["provider_name"] if args["provider_name"] else "", + args["provider_name"] or "", args["tool_name"], args["credentials"], args["parameters"], diff --git a/api/controllers/service_api/app/audio.py b/api/controllers/service_api/app/audio.py index 85aab047a7..8d8ca8d78c 100644 --- a/api/controllers/service_api/app/audio.py +++ b/api/controllers/service_api/app/audio.py @@ -84,14 +84,10 @@ class TextApi(Resource): and app_model.workflow.features_dict ): text_to_speech = app_model.workflow.features_dict.get("text_to_speech") - voice = args.get("voice") if args.get("voice") else text_to_speech.get("voice") + voice = args.get("voice") or text_to_speech.get("voice") else: try: - voice = ( - args.get("voice") - if args.get("voice") - else app_model.app_model_config.text_to_speech_dict.get("voice") - ) + voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice") except Exception: voice = None response = AudioService.transcript_tts( diff --git a/api/controllers/web/audio.py b/api/controllers/web/audio.py index d062d2893b..49c467dbe1 100644 --- a/api/controllers/web/audio.py +++ b/api/controllers/web/audio.py @@ -83,14 +83,10 @@ class TextApi(WebApiResource): and app_model.workflow.features_dict ): text_to_speech = app_model.workflow.features_dict.get("text_to_speech") - voice = args.get("voice") if args.get("voice") else text_to_speech.get("voice") + voice = args.get("voice") or text_to_speech.get("voice") else: try: - voice = ( - args.get("voice") - if args.get("voice") - else app_model.app_model_config.text_to_speech_dict.get("voice") - ) + voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice") except Exception: voice = None diff --git a/api/core/agent/cot_agent_runner.py b/api/core/agent/cot_agent_runner.py index 29b428a7c3..ebe04bf260 100644 --- a/api/core/agent/cot_agent_runner.py +++ b/api/core/agent/cot_agent_runner.py @@ -256,7 +256,7 @@ class CotAgentRunner(BaseAgentRunner, ABC): model=model_instance.model, prompt_messages=prompt_messages, message=AssistantPromptMessage(content=final_answer), - usage=llm_usage["usage"] if llm_usage["usage"] else LLMUsage.empty_usage(), + usage=llm_usage["usage"] or LLMUsage.empty_usage(), system_fingerprint="", ) ), diff --git a/api/core/agent/fc_agent_runner.py b/api/core/agent/fc_agent_runner.py index 27cf561e3d..13164e0bfc 100644 --- a/api/core/agent/fc_agent_runner.py +++ b/api/core/agent/fc_agent_runner.py @@ -298,7 +298,7 @@ class FunctionCallAgentRunner(BaseAgentRunner): model=model_instance.model, prompt_messages=prompt_messages, message=AssistantPromptMessage(content=final_answer), - usage=llm_usage["usage"] if llm_usage["usage"] else LLMUsage.empty_usage(), + usage=llm_usage["usage"] or LLMUsage.empty_usage(), system_fingerprint="", ) ), diff --git a/api/core/app/apps/base_app_runner.py b/api/core/app/apps/base_app_runner.py index aadb43ad39..bd2be18bfd 100644 --- a/api/core/app/apps/base_app_runner.py +++ b/api/core/app/apps/base_app_runner.py @@ -161,7 +161,7 @@ class AppRunner: app_mode=AppMode.value_of(app_record.mode), prompt_template_entity=prompt_template_entity, inputs=inputs, - query=query if query else "", + query=query or "", files=files, context=context, memory=memory, @@ -189,7 +189,7 @@ class AppRunner: prompt_messages = prompt_transform.get_prompt( prompt_template=prompt_template, inputs=inputs, - query=query if query else "", + query=query or "", files=files, context=context, memory_config=memory_config, @@ -238,7 +238,7 @@ class AppRunner: model=app_generate_entity.model_conf.model, prompt_messages=prompt_messages, message=AssistantPromptMessage(content=text), - usage=usage if usage else LLMUsage.empty_usage(), + usage=usage or LLMUsage.empty_usage(), ), ), PublishFrom.APPLICATION_MANAGER, @@ -351,7 +351,7 @@ class AppRunner: tenant_id=tenant_id, app_config=app_generate_entity.app_config, inputs=inputs, - query=query if query else "", + query=query or "", message_id=message_id, trace_manager=app_generate_entity.trace_manager, ) diff --git a/api/core/extension/extensible.py b/api/core/extension/extensible.py index f1a49c4921..97dbaf2026 100644 --- a/api/core/extension/extensible.py +++ b/api/core/extension/extensible.py @@ -3,6 +3,7 @@ import importlib.util import json import logging import os +from pathlib import Path from typing import Any, Optional from pydantic import BaseModel @@ -63,8 +64,7 @@ class Extensible: builtin_file_path = os.path.join(subdir_path, "__builtin__") if os.path.exists(builtin_file_path): - with open(builtin_file_path, encoding="utf-8") as f: - position = int(f.read().strip()) + position = int(Path(builtin_file_path).read_text(encoding="utf-8").strip()) position_map[extension_name] = position if (extension_name + ".py") not in file_names: diff --git a/api/core/memory/token_buffer_memory.py b/api/core/memory/token_buffer_memory.py index 54b1d8212b..a14d237a12 100644 --- a/api/core/memory/token_buffer_memory.py +++ b/api/core/memory/token_buffer_memory.py @@ -39,7 +39,7 @@ class TokenBufferMemory: ) if message_limit and message_limit > 0: - message_limit = message_limit if message_limit <= 500 else 500 + message_limit = min(message_limit, 500) else: message_limit = 500 diff --git a/api/core/model_runtime/model_providers/__base/large_language_model.py b/api/core/model_runtime/model_providers/__base/large_language_model.py index e8789ec7df..ba88cc1f38 100644 --- a/api/core/model_runtime/model_providers/__base/large_language_model.py +++ b/api/core/model_runtime/model_providers/__base/large_language_model.py @@ -449,7 +449,7 @@ if you are not sure about the structure. model=real_model, prompt_messages=prompt_messages, message=prompt_message, - usage=usage if usage else LLMUsage.empty_usage(), + usage=usage or LLMUsage.empty_usage(), system_fingerprint=system_fingerprint, ), credentials=credentials, diff --git a/api/core/model_runtime/model_providers/anthropic/llm/llm.py b/api/core/model_runtime/model_providers/anthropic/llm/llm.py index 0cb66842e7..ff741e0240 100644 --- a/api/core/model_runtime/model_providers/anthropic/llm/llm.py +++ b/api/core/model_runtime/model_providers/anthropic/llm/llm.py @@ -409,7 +409,7 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): ), ) elif isinstance(chunk, ContentBlockDeltaEvent): - chunk_text = chunk.delta.text if chunk.delta.text else "" + chunk_text = chunk.delta.text or "" full_assistant_content += chunk_text # transform assistant message to prompt message diff --git a/api/core/model_runtime/model_providers/azure_ai_studio/llm/llm.py b/api/core/model_runtime/model_providers/azure_ai_studio/llm/llm.py index 42eae6c1e5..516ef8b295 100644 --- a/api/core/model_runtime/model_providers/azure_ai_studio/llm/llm.py +++ b/api/core/model_runtime/model_providers/azure_ai_studio/llm/llm.py @@ -213,7 +213,7 @@ class AzureAIStudioLargeLanguageModel(LargeLanguageModel): model=real_model, prompt_messages=prompt_messages, message=prompt_message, - usage=usage if usage else LLMUsage.empty_usage(), + usage=usage or LLMUsage.empty_usage(), system_fingerprint=system_fingerprint, ), credentials=credentials, diff --git a/api/core/model_runtime/model_providers/azure_openai/llm/llm.py b/api/core/model_runtime/model_providers/azure_openai/llm/llm.py index 3b9fb52e24..f0033ea051 100644 --- a/api/core/model_runtime/model_providers/azure_openai/llm/llm.py +++ b/api/core/model_runtime/model_providers/azure_openai/llm/llm.py @@ -225,7 +225,7 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): continue # transform assistant message to prompt message - text = delta.text if delta.text else "" + text = delta.text or "" assistant_prompt_message = AssistantPromptMessage(content=text) full_text += text @@ -400,15 +400,13 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): continue # transform assistant message to prompt message - assistant_prompt_message = AssistantPromptMessage( - content=delta.delta.content if delta.delta.content else "", tool_calls=tool_calls - ) + assistant_prompt_message = AssistantPromptMessage(content=delta.delta.content or "", tool_calls=tool_calls) - full_assistant_content += delta.delta.content if delta.delta.content else "" + full_assistant_content += delta.delta.content or "" real_model = chunk.model system_fingerprint = chunk.system_fingerprint - completion += delta.delta.content if delta.delta.content else "" + completion += delta.delta.content or "" yield LLMResultChunk( model=real_model, diff --git a/api/core/model_runtime/model_providers/azure_openai/tts/tts.py b/api/core/model_runtime/model_providers/azure_openai/tts/tts.py index bbad726467..8db044b24d 100644 --- a/api/core/model_runtime/model_providers/azure_openai/tts/tts.py +++ b/api/core/model_runtime/model_providers/azure_openai/tts/tts.py @@ -84,7 +84,7 @@ class AzureOpenAIText2SpeechModel(_CommonAzureOpenAI, TTSModel): ) for i in range(len(sentences)) ] - for index, future in enumerate(futures): + for future in futures: yield from future.result().__enter__().iter_bytes(1024) else: diff --git a/api/core/model_runtime/model_providers/bedrock/llm/llm.py b/api/core/model_runtime/model_providers/bedrock/llm/llm.py index 239ae52b4c..953ac0741f 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/llm.py +++ b/api/core/model_runtime/model_providers/bedrock/llm/llm.py @@ -331,10 +331,10 @@ class BedrockLargeLanguageModel(LargeLanguageModel): elif "contentBlockDelta" in chunk: delta = chunk["contentBlockDelta"]["delta"] if "text" in delta: - chunk_text = delta["text"] if delta["text"] else "" + chunk_text = delta["text"] or "" full_assistant_content += chunk_text assistant_prompt_message = AssistantPromptMessage( - content=chunk_text if chunk_text else "", + content=chunk_text or "", ) index = chunk["contentBlockDelta"]["contentBlockIndex"] yield LLMResultChunk( @@ -751,7 +751,7 @@ class BedrockLargeLanguageModel(LargeLanguageModel): elif model_prefix == "cohere": output = response_body.get("generations")[0].get("text") prompt_tokens = self.get_num_tokens(model, credentials, prompt_messages) - completion_tokens = self.get_num_tokens(model, credentials, output if output else "") + completion_tokens = self.get_num_tokens(model, credentials, output or "") else: raise ValueError(f"Got unknown model prefix {model_prefix} when handling block response") @@ -828,7 +828,7 @@ class BedrockLargeLanguageModel(LargeLanguageModel): # transform assistant message to prompt message assistant_prompt_message = AssistantPromptMessage( - content=content_delta if content_delta else "", + content=content_delta or "", ) index += 1 diff --git a/api/core/model_runtime/model_providers/chatglm/llm/llm.py b/api/core/model_runtime/model_providers/chatglm/llm/llm.py index 114acc1ec3..b3eeb48e22 100644 --- a/api/core/model_runtime/model_providers/chatglm/llm/llm.py +++ b/api/core/model_runtime/model_providers/chatglm/llm/llm.py @@ -302,11 +302,11 @@ class ChatGLMLargeLanguageModel(LargeLanguageModel): if delta.delta.function_call: function_calls = [delta.delta.function_call] - assistant_message_tool_calls = self._extract_response_tool_calls(function_calls if function_calls else []) + assistant_message_tool_calls = self._extract_response_tool_calls(function_calls or []) # transform assistant message to prompt message assistant_prompt_message = AssistantPromptMessage( - content=delta.delta.content if delta.delta.content else "", tool_calls=assistant_message_tool_calls + content=delta.delta.content or "", tool_calls=assistant_message_tool_calls ) if delta.finish_reason is not None: diff --git a/api/core/model_runtime/model_providers/localai/llm/llm.py b/api/core/model_runtime/model_providers/localai/llm/llm.py index 94c03efe7b..e7295355f6 100644 --- a/api/core/model_runtime/model_providers/localai/llm/llm.py +++ b/api/core/model_runtime/model_providers/localai/llm/llm.py @@ -511,7 +511,7 @@ class LocalAILanguageModel(LargeLanguageModel): delta = chunk.choices[0] # transform assistant message to prompt message - assistant_prompt_message = AssistantPromptMessage(content=delta.text if delta.text else "", tool_calls=[]) + assistant_prompt_message = AssistantPromptMessage(content=delta.text or "", tool_calls=[]) if delta.finish_reason is not None: # temp_assistant_prompt_message is used to calculate usage @@ -578,11 +578,11 @@ class LocalAILanguageModel(LargeLanguageModel): if delta.delta.function_call: function_calls = [delta.delta.function_call] - assistant_message_tool_calls = self._extract_response_tool_calls(function_calls if function_calls else []) + assistant_message_tool_calls = self._extract_response_tool_calls(function_calls or []) # transform assistant message to prompt message assistant_prompt_message = AssistantPromptMessage( - content=delta.delta.content if delta.delta.content else "", tool_calls=assistant_message_tool_calls + content=delta.delta.content or "", tool_calls=assistant_message_tool_calls ) if delta.finish_reason is not None: diff --git a/api/core/model_runtime/model_providers/minimax/llm/llm.py b/api/core/model_runtime/model_providers/minimax/llm/llm.py index 76ed704a75..4250c40cfb 100644 --- a/api/core/model_runtime/model_providers/minimax/llm/llm.py +++ b/api/core/model_runtime/model_providers/minimax/llm/llm.py @@ -211,7 +211,7 @@ class MinimaxLargeLanguageModel(LargeLanguageModel): index=0, message=AssistantPromptMessage(content=message.content, tool_calls=[]), usage=usage, - finish_reason=message.stop_reason if message.stop_reason else None, + finish_reason=message.stop_reason or None, ), ) elif message.function_call: @@ -244,7 +244,7 @@ class MinimaxLargeLanguageModel(LargeLanguageModel): delta=LLMResultChunkDelta( index=0, message=AssistantPromptMessage(content=message.content, tool_calls=[]), - finish_reason=message.stop_reason if message.stop_reason else None, + finish_reason=message.stop_reason or None, ), ) diff --git a/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py index 2cfb79b241..b4c61d8a6d 100644 --- a/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py @@ -65,7 +65,7 @@ class OllamaEmbeddingModel(TextEmbeddingModel): inputs = [] used_tokens = 0 - for i, text in enumerate(texts): + for text in texts: # Here token count is only an approximation based on the GPT2 tokenizer num_tokens = self._get_num_tokens_by_gpt2(text) diff --git a/api/core/model_runtime/model_providers/openai/llm/llm.py b/api/core/model_runtime/model_providers/openai/llm/llm.py index 578687b5d3..c4b381df41 100644 --- a/api/core/model_runtime/model_providers/openai/llm/llm.py +++ b/api/core/model_runtime/model_providers/openai/llm/llm.py @@ -508,7 +508,7 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): continue # transform assistant message to prompt message - text = delta.text if delta.text else "" + text = delta.text or "" assistant_prompt_message = AssistantPromptMessage(content=text) full_text += text @@ -760,11 +760,9 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): final_tool_calls.extend(tool_calls) # transform assistant message to prompt message - assistant_prompt_message = AssistantPromptMessage( - content=delta.delta.content if delta.delta.content else "", tool_calls=tool_calls - ) + assistant_prompt_message = AssistantPromptMessage(content=delta.delta.content or "", tool_calls=tool_calls) - full_assistant_content += delta.delta.content if delta.delta.content else "" + full_assistant_content += delta.delta.content or "" if has_finish_reason: final_chunk = LLMResultChunk( diff --git a/api/core/model_runtime/model_providers/openai/tts/tts.py b/api/core/model_runtime/model_providers/openai/tts/tts.py index bfb443698c..b50b43199f 100644 --- a/api/core/model_runtime/model_providers/openai/tts/tts.py +++ b/api/core/model_runtime/model_providers/openai/tts/tts.py @@ -88,7 +88,7 @@ class OpenAIText2SpeechModel(_CommonOpenAI, TTSModel): ) for i in range(len(sentences)) ] - for index, future in enumerate(futures): + for future in futures: yield from future.result().__enter__().iter_bytes(1024) else: diff --git a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py index 41ca163a92..5a8a754f72 100644 --- a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py +++ b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py @@ -179,9 +179,9 @@ class OAIAPICompatLargeLanguageModel(_CommonOaiApiCompat, LargeLanguageModel): features = [] function_calling_type = credentials.get("function_calling_type", "no_call") - if function_calling_type in ["function_call"]: + if function_calling_type == "function_call": features.append(ModelFeature.TOOL_CALL) - elif function_calling_type in ["tool_call"]: + elif function_calling_type == "tool_call": features.append(ModelFeature.MULTI_TOOL_CALL) stream_function_calling = credentials.get("stream_function_calling", "supported") diff --git a/api/core/model_runtime/model_providers/openllm/llm/llm.py b/api/core/model_runtime/model_providers/openllm/llm/llm.py index b560afca39..34b4de7962 100644 --- a/api/core/model_runtime/model_providers/openllm/llm/llm.py +++ b/api/core/model_runtime/model_providers/openllm/llm/llm.py @@ -179,7 +179,7 @@ class OpenLLMLargeLanguageModel(LargeLanguageModel): index=0, message=AssistantPromptMessage(content=message.content, tool_calls=[]), usage=usage, - finish_reason=message.stop_reason if message.stop_reason else None, + finish_reason=message.stop_reason or None, ), ) else: @@ -189,7 +189,7 @@ class OpenLLMLargeLanguageModel(LargeLanguageModel): delta=LLMResultChunkDelta( index=0, message=AssistantPromptMessage(content=message.content, tool_calls=[]), - finish_reason=message.stop_reason if message.stop_reason else None, + finish_reason=message.stop_reason or None, ), ) diff --git a/api/core/model_runtime/model_providers/openllm/llm/openllm_generate.py b/api/core/model_runtime/model_providers/openllm/llm/openllm_generate.py index e754479ec0..351dcced15 100644 --- a/api/core/model_runtime/model_providers/openllm/llm/openllm_generate.py +++ b/api/core/model_runtime/model_providers/openllm/llm/openllm_generate.py @@ -106,7 +106,7 @@ class OpenLLMGenerate: timeout = 120 data = { - "stop": stop if stop else [], + "stop": stop or [], "prompt": "\n".join([message.content for message in prompt_messages]), "llm_config": default_llm_config, } diff --git a/api/core/model_runtime/model_providers/replicate/llm/llm.py b/api/core/model_runtime/model_providers/replicate/llm/llm.py index 87c8bc4a91..e77372cae2 100644 --- a/api/core/model_runtime/model_providers/replicate/llm/llm.py +++ b/api/core/model_runtime/model_providers/replicate/llm/llm.py @@ -214,7 +214,7 @@ class ReplicateLargeLanguageModel(_CommonReplicate, LargeLanguageModel): index += 1 - assistant_prompt_message = AssistantPromptMessage(content=output if output else "") + assistant_prompt_message = AssistantPromptMessage(content=output or "") if index < prediction_output_length: yield LLMResultChunk( diff --git a/api/core/model_runtime/model_providers/sagemaker/rerank/rerank.py b/api/core/model_runtime/model_providers/sagemaker/rerank/rerank.py index 7e7614055c..959dff6a21 100644 --- a/api/core/model_runtime/model_providers/sagemaker/rerank/rerank.py +++ b/api/core/model_runtime/model_providers/sagemaker/rerank/rerank.py @@ -1,5 +1,6 @@ import json import logging +import operator from typing import Any, Optional import boto3 @@ -94,7 +95,7 @@ class SageMakerRerankModel(RerankModel): for idx in range(len(scores)): candidate_docs.append({"content": docs[idx], "score": scores[idx]}) - sorted(candidate_docs, key=lambda x: x["score"], reverse=True) + sorted(candidate_docs, key=operator.itemgetter("score"), reverse=True) line = 3 rerank_documents = [] diff --git a/api/core/model_runtime/model_providers/sagemaker/tts/tts.py b/api/core/model_runtime/model_providers/sagemaker/tts/tts.py index 3dd5f8f64c..a22bd6dd6e 100644 --- a/api/core/model_runtime/model_providers/sagemaker/tts/tts.py +++ b/api/core/model_runtime/model_providers/sagemaker/tts/tts.py @@ -260,7 +260,7 @@ class SageMakerText2SpeechModel(TTSModel): for payload in payloads ] - for index, future in enumerate(futures): + for future in futures: resp = future.result() audio_bytes = requests.get(resp.get("s3_presign_url")).content for i in range(0, len(audio_bytes), 1024): diff --git a/api/core/model_runtime/model_providers/spark/llm/llm.py b/api/core/model_runtime/model_providers/spark/llm/llm.py index 0c42acf5aa..57193dc031 100644 --- a/api/core/model_runtime/model_providers/spark/llm/llm.py +++ b/api/core/model_runtime/model_providers/spark/llm/llm.py @@ -220,7 +220,7 @@ class SparkLargeLanguageModel(LargeLanguageModel): delta = content assistant_prompt_message = AssistantPromptMessage( - content=delta if delta else "", + content=delta or "", ) prompt_tokens = self.get_num_tokens(model, credentials, prompt_messages) diff --git a/api/core/model_runtime/model_providers/tencent/speech2text/flash_recognizer.py b/api/core/model_runtime/model_providers/tencent/speech2text/flash_recognizer.py index 9fd4a45f45..c3c21793e8 100644 --- a/api/core/model_runtime/model_providers/tencent/speech2text/flash_recognizer.py +++ b/api/core/model_runtime/model_providers/tencent/speech2text/flash_recognizer.py @@ -1,6 +1,7 @@ import base64 import hashlib import hmac +import operator import time import requests @@ -127,7 +128,7 @@ class FlashRecognizer: return s def _build_req_with_signature(self, secret_key, params, header): - query = sorted(params.items(), key=lambda d: d[0]) + query = sorted(params.items(), key=operator.itemgetter(0)) signstr = self._format_sign_string(query) signature = self._sign(signstr, secret_key) header["Authorization"] = signature diff --git a/api/core/model_runtime/model_providers/tongyi/llm/llm.py b/api/core/model_runtime/model_providers/tongyi/llm/llm.py index db0b2deaa5..ca708d931c 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/llm.py +++ b/api/core/model_runtime/model_providers/tongyi/llm/llm.py @@ -4,6 +4,7 @@ import tempfile import uuid from collections.abc import Generator from http import HTTPStatus +from pathlib import Path from typing import Optional, Union, cast from dashscope import Generation, MultiModalConversation, get_tokenizer @@ -454,8 +455,7 @@ class TongyiLargeLanguageModel(LargeLanguageModel): file_path = os.path.join(temp_dir, f"{uuid.uuid4()}.{mime_type.split('/')[1]}") - with open(file_path, "wb") as image_file: - image_file.write(base64.b64decode(encoded_string)) + Path(file_path).write_bytes(base64.b64decode(encoded_string)) return f"file://{file_path}" diff --git a/api/core/model_runtime/model_providers/upstage/llm/llm.py b/api/core/model_runtime/model_providers/upstage/llm/llm.py index 9646e209b2..74524e81e2 100644 --- a/api/core/model_runtime/model_providers/upstage/llm/llm.py +++ b/api/core/model_runtime/model_providers/upstage/llm/llm.py @@ -368,11 +368,9 @@ class UpstageLargeLanguageModel(_CommonUpstage, LargeLanguageModel): final_tool_calls.extend(tool_calls) # transform assistant message to prompt message - assistant_prompt_message = AssistantPromptMessage( - content=delta.delta.content if delta.delta.content else "", tool_calls=tool_calls - ) + assistant_prompt_message = AssistantPromptMessage(content=delta.delta.content or "", tool_calls=tool_calls) - full_assistant_content += delta.delta.content if delta.delta.content else "" + full_assistant_content += delta.delta.content or "" if has_finish_reason: final_chunk = LLMResultChunk( diff --git a/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py b/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py index 1b9931d2c3..dad5002f35 100644 --- a/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py +++ b/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py @@ -231,10 +231,10 @@ class VertexAiLargeLanguageModel(LargeLanguageModel): ), ) elif isinstance(chunk, ContentBlockDeltaEvent): - chunk_text = chunk.delta.text if chunk.delta.text else "" + chunk_text = chunk.delta.text or "" full_assistant_content += chunk_text assistant_prompt_message = AssistantPromptMessage( - content=chunk_text if chunk_text else "", + content=chunk_text or "", ) index = chunk.index yield LLMResultChunk( diff --git a/api/core/model_runtime/model_providers/volcengine_maas/legacy/volc_sdk/base/auth.py b/api/core/model_runtime/model_providers/volcengine_maas/legacy/volc_sdk/base/auth.py index 7435720252..97c77de8d3 100644 --- a/api/core/model_runtime/model_providers/volcengine_maas/legacy/volc_sdk/base/auth.py +++ b/api/core/model_runtime/model_providers/volcengine_maas/legacy/volc_sdk/base/auth.py @@ -1,5 +1,6 @@ # coding : utf-8 import datetime +from itertools import starmap import pytz @@ -48,7 +49,7 @@ class SignResult: self.authorization = "" def __str__(self): - return "\n".join(["{}:{}".format(*item) for item in self.__dict__.items()]) + return "\n".join(list(starmap("{}:{}".format, self.__dict__.items()))) class Credentials: diff --git a/api/core/model_runtime/model_providers/volcengine_maas/legacy/volc_sdk/base/util.py b/api/core/model_runtime/model_providers/volcengine_maas/legacy/volc_sdk/base/util.py index 44f9959965..178d63714e 100644 --- a/api/core/model_runtime/model_providers/volcengine_maas/legacy/volc_sdk/base/util.py +++ b/api/core/model_runtime/model_providers/volcengine_maas/legacy/volc_sdk/base/util.py @@ -1,5 +1,6 @@ import hashlib import hmac +import operator from functools import reduce from urllib.parse import quote @@ -40,4 +41,4 @@ class Util: if len(hv) == 1: hv = "0" + hv lst.append(hv) - return reduce(lambda x, y: x + y, lst) + return reduce(operator.add, lst) diff --git a/api/core/model_runtime/model_providers/volcengine_maas/llm/llm.py b/api/core/model_runtime/model_providers/volcengine_maas/llm/llm.py index c25851fc45..f8bf8fb821 100644 --- a/api/core/model_runtime/model_providers/volcengine_maas/llm/llm.py +++ b/api/core/model_runtime/model_providers/volcengine_maas/llm/llm.py @@ -174,9 +174,7 @@ class VolcengineMaaSLargeLanguageModel(LargeLanguageModel): prompt_messages=prompt_messages, delta=LLMResultChunkDelta( index=index, - message=AssistantPromptMessage( - content=message["content"] if message["content"] else "", tool_calls=[] - ), + message=AssistantPromptMessage(content=message["content"] or "", tool_calls=[]), usage=usage, finish_reason=choice.get("finish_reason"), ), @@ -208,7 +206,7 @@ class VolcengineMaaSLargeLanguageModel(LargeLanguageModel): model=model, prompt_messages=prompt_messages, message=AssistantPromptMessage( - content=message["content"] if message["content"] else "", + content=message["content"] or "", tool_calls=tool_calls, ), usage=self._calc_response_usage( @@ -284,7 +282,7 @@ class VolcengineMaaSLargeLanguageModel(LargeLanguageModel): model=model, prompt_messages=prompt_messages, message=AssistantPromptMessage( - content=message.content if message.content else "", + content=message.content or "", tool_calls=tool_calls, ), usage=self._calc_response_usage( diff --git a/api/core/model_runtime/model_providers/wenxin/llm/llm.py b/api/core/model_runtime/model_providers/wenxin/llm/llm.py index 8feedbfe55..ec3556f7da 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/llm.py +++ b/api/core/model_runtime/model_providers/wenxin/llm/llm.py @@ -199,7 +199,7 @@ class ErnieBotLargeLanguageModel(LargeLanguageModel): secret_key=credentials["secret_key"], ) - user = user if user else "ErnieBotDefault" + user = user or "ErnieBotDefault" # convert prompt messages to baichuan messages messages = [ @@ -289,7 +289,7 @@ class ErnieBotLargeLanguageModel(LargeLanguageModel): index=0, message=AssistantPromptMessage(content=message.content, tool_calls=[]), usage=usage, - finish_reason=message.stop_reason if message.stop_reason else None, + finish_reason=message.stop_reason or None, ), ) else: @@ -299,7 +299,7 @@ class ErnieBotLargeLanguageModel(LargeLanguageModel): delta=LLMResultChunkDelta( index=0, message=AssistantPromptMessage(content=message.content, tool_calls=[]), - finish_reason=message.stop_reason if message.stop_reason else None, + finish_reason=message.stop_reason or None, ), ) diff --git a/api/core/model_runtime/model_providers/wenxin/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/wenxin/text_embedding/text_embedding.py index db323ae4c1..4d6f6dccd0 100644 --- a/api/core/model_runtime/model_providers/wenxin/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/wenxin/text_embedding/text_embedding.py @@ -85,7 +85,7 @@ class WenxinTextEmbeddingModel(TextEmbeddingModel): api_key = credentials["api_key"] secret_key = credentials["secret_key"] embedding: TextEmbedding = self._create_text_embedding(api_key, secret_key) - user = user if user else "ErnieBotDefault" + user = user or "ErnieBotDefault" context_size = self._get_context_size(model, credentials) max_chunks = self._get_max_chunks(model, credentials) diff --git a/api/core/model_runtime/model_providers/xinference/llm/llm.py b/api/core/model_runtime/model_providers/xinference/llm/llm.py index 7ad236880b..f8f6c6b12d 100644 --- a/api/core/model_runtime/model_providers/xinference/llm/llm.py +++ b/api/core/model_runtime/model_providers/xinference/llm/llm.py @@ -589,7 +589,7 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): # convert tool call to assistant message tool call tool_calls = assistant_message.tool_calls - assistant_prompt_message_tool_calls = self._extract_response_tool_calls(tool_calls if tool_calls else []) + assistant_prompt_message_tool_calls = self._extract_response_tool_calls(tool_calls or []) function_call = assistant_message.function_call if function_call: assistant_prompt_message_tool_calls += [self._extract_response_function_call(function_call)] @@ -652,7 +652,7 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): # transform assistant message to prompt message assistant_prompt_message = AssistantPromptMessage( - content=delta.delta.content if delta.delta.content else "", tool_calls=assistant_message_tool_calls + content=delta.delta.content or "", tool_calls=assistant_message_tool_calls ) if delta.finish_reason is not None: @@ -749,7 +749,7 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): delta = chunk.choices[0] # transform assistant message to prompt message - assistant_prompt_message = AssistantPromptMessage(content=delta.text if delta.text else "", tool_calls=[]) + assistant_prompt_message = AssistantPromptMessage(content=delta.text or "", tool_calls=[]) if delta.finish_reason is not None: # temp_assistant_prompt_message is used to calculate usage diff --git a/api/core/model_runtime/model_providers/xinference/tts/tts.py b/api/core/model_runtime/model_providers/xinference/tts/tts.py index 60db151302..d29e8ed8f9 100644 --- a/api/core/model_runtime/model_providers/xinference/tts/tts.py +++ b/api/core/model_runtime/model_providers/xinference/tts/tts.py @@ -215,7 +215,7 @@ class XinferenceText2SpeechModel(TTSModel): for i in range(len(sentences)) ] - for index, future in enumerate(futures): + for future in futures: response = future.result() for i in range(0, len(response), 1024): yield response[i : i + 1024] diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py index 29b873fd06..f76e51fee9 100644 --- a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py +++ b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py @@ -414,10 +414,10 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): # transform assistant message to prompt message assistant_prompt_message = AssistantPromptMessage( - content=delta.delta.content if delta.delta.content else "", tool_calls=assistant_tool_calls + content=delta.delta.content or "", tool_calls=assistant_tool_calls ) - full_assistant_content += delta.delta.content if delta.delta.content else "" + full_assistant_content += delta.delta.content or "" if delta.finish_reason is not None and chunk.usage is not None: completion_tokens = chunk.usage.completion_tokens diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py index 48eeb37c41..5f7f6d04f2 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py @@ -30,6 +30,8 @@ def _merge_map(map1: Mapping, map2: Mapping) -> Mapping: return {key: val for key, val in merged.items() if val is not None} +from itertools import starmap + from httpx._config import DEFAULT_TIMEOUT_CONFIG as HTTPX_DEFAULT_TIMEOUT ZHIPUAI_DEFAULT_TIMEOUT = httpx.Timeout(timeout=300.0, connect=8.0) @@ -159,7 +161,7 @@ class HttpClient: return [(key, str_data)] def _make_multipartform(self, data: Mapping[object, object]) -> dict[str, object]: - items = flatten([self._object_to_formdata(k, v) for k, v in data.items()]) + items = flatten(list(starmap(self._object_to_formdata, data.items()))) serialized: dict[str, object] = {} for key, value in items: diff --git a/api/core/ops/langfuse_trace/langfuse_trace.py b/api/core/ops/langfuse_trace/langfuse_trace.py index a0f3ac7f86..6aefbec9aa 100644 --- a/api/core/ops/langfuse_trace/langfuse_trace.py +++ b/api/core/ops/langfuse_trace/langfuse_trace.py @@ -65,7 +65,7 @@ class LangFuseDataTrace(BaseTraceInstance): self.generate_name_trace(trace_info) def workflow_trace(self, trace_info: WorkflowTraceInfo): - trace_id = trace_info.workflow_app_log_id if trace_info.workflow_app_log_id else trace_info.workflow_run_id + trace_id = trace_info.workflow_app_log_id or trace_info.workflow_run_id user_id = trace_info.metadata.get("user_id") if trace_info.message_id: trace_id = trace_info.message_id @@ -84,7 +84,7 @@ class LangFuseDataTrace(BaseTraceInstance): ) self.add_trace(langfuse_trace_data=trace_data) workflow_span_data = LangfuseSpan( - id=(trace_info.workflow_app_log_id if trace_info.workflow_app_log_id else trace_info.workflow_run_id), + id=(trace_info.workflow_app_log_id or trace_info.workflow_run_id), name=TraceTaskName.WORKFLOW_TRACE.value, input=trace_info.workflow_run_inputs, output=trace_info.workflow_run_outputs, @@ -93,7 +93,7 @@ class LangFuseDataTrace(BaseTraceInstance): end_time=trace_info.end_time, metadata=trace_info.metadata, level=LevelEnum.DEFAULT if trace_info.error == "" else LevelEnum.ERROR, - status_message=trace_info.error if trace_info.error else "", + status_message=trace_info.error or "", ) self.add_span(langfuse_span_data=workflow_span_data) else: @@ -143,7 +143,7 @@ class LangFuseDataTrace(BaseTraceInstance): else: inputs = json.loads(node_execution.inputs) if node_execution.inputs else {} outputs = json.loads(node_execution.outputs) if node_execution.outputs else {} - created_at = node_execution.created_at if node_execution.created_at else datetime.now() + created_at = node_execution.created_at or datetime.now() elapsed_time = node_execution.elapsed_time finished_at = created_at + timedelta(seconds=elapsed_time) @@ -172,10 +172,8 @@ class LangFuseDataTrace(BaseTraceInstance): end_time=finished_at, metadata=metadata, level=(LevelEnum.DEFAULT if status == "succeeded" else LevelEnum.ERROR), - status_message=trace_info.error if trace_info.error else "", - parent_observation_id=( - trace_info.workflow_app_log_id if trace_info.workflow_app_log_id else trace_info.workflow_run_id - ), + status_message=trace_info.error or "", + parent_observation_id=(trace_info.workflow_app_log_id or trace_info.workflow_run_id), ) else: span_data = LangfuseSpan( @@ -188,7 +186,7 @@ class LangFuseDataTrace(BaseTraceInstance): end_time=finished_at, metadata=metadata, level=(LevelEnum.DEFAULT if status == "succeeded" else LevelEnum.ERROR), - status_message=trace_info.error if trace_info.error else "", + status_message=trace_info.error or "", ) self.add_span(langfuse_span_data=span_data) @@ -212,7 +210,7 @@ class LangFuseDataTrace(BaseTraceInstance): output=outputs, metadata=metadata, level=(LevelEnum.DEFAULT if status == "succeeded" else LevelEnum.ERROR), - status_message=trace_info.error if trace_info.error else "", + status_message=trace_info.error or "", usage=generation_usage, ) @@ -277,7 +275,7 @@ class LangFuseDataTrace(BaseTraceInstance): output=message_data.answer, metadata=metadata, level=(LevelEnum.DEFAULT if message_data.status != "error" else LevelEnum.ERROR), - status_message=message_data.error if message_data.error else "", + status_message=message_data.error or "", usage=generation_usage, ) @@ -319,7 +317,7 @@ class LangFuseDataTrace(BaseTraceInstance): end_time=trace_info.end_time, metadata=trace_info.metadata, level=(LevelEnum.DEFAULT if message_data.status != "error" else LevelEnum.ERROR), - status_message=message_data.error if message_data.error else "", + status_message=message_data.error or "", usage=generation_usage, ) diff --git a/api/core/ops/langsmith_trace/langsmith_trace.py b/api/core/ops/langsmith_trace/langsmith_trace.py index eea7bb3535..37cbea13fd 100644 --- a/api/core/ops/langsmith_trace/langsmith_trace.py +++ b/api/core/ops/langsmith_trace/langsmith_trace.py @@ -82,7 +82,7 @@ class LangSmithDataTrace(BaseTraceInstance): langsmith_run = LangSmithRunModel( file_list=trace_info.file_list, total_tokens=trace_info.total_tokens, - id=trace_info.workflow_app_log_id if trace_info.workflow_app_log_id else trace_info.workflow_run_id, + id=trace_info.workflow_app_log_id or trace_info.workflow_run_id, name=TraceTaskName.WORKFLOW_TRACE.value, inputs=trace_info.workflow_run_inputs, run_type=LangSmithRunType.tool, @@ -94,7 +94,7 @@ class LangSmithDataTrace(BaseTraceInstance): }, error=trace_info.error, tags=["workflow"], - parent_run_id=trace_info.message_id if trace_info.message_id else None, + parent_run_id=trace_info.message_id or None, ) self.add_run(langsmith_run) @@ -133,7 +133,7 @@ class LangSmithDataTrace(BaseTraceInstance): else: inputs = json.loads(node_execution.inputs) if node_execution.inputs else {} outputs = json.loads(node_execution.outputs) if node_execution.outputs else {} - created_at = node_execution.created_at if node_execution.created_at else datetime.now() + created_at = node_execution.created_at or datetime.now() elapsed_time = node_execution.elapsed_time finished_at = created_at + timedelta(seconds=elapsed_time) @@ -180,9 +180,7 @@ class LangSmithDataTrace(BaseTraceInstance): extra={ "metadata": metadata, }, - parent_run_id=trace_info.workflow_app_log_id - if trace_info.workflow_app_log_id - else trace_info.workflow_run_id, + parent_run_id=trace_info.workflow_app_log_id or trace_info.workflow_run_id, tags=["node_execution"], ) diff --git a/api/core/ops/ops_trace_manager.py b/api/core/ops/ops_trace_manager.py index 68fcdf32da..6f17bade97 100644 --- a/api/core/ops/ops_trace_manager.py +++ b/api/core/ops/ops_trace_manager.py @@ -354,11 +354,11 @@ class TraceTask: workflow_run_inputs = json.loads(workflow_run.inputs) if workflow_run.inputs else {} workflow_run_outputs = json.loads(workflow_run.outputs) if workflow_run.outputs else {} workflow_run_version = workflow_run.version - error = workflow_run.error if workflow_run.error else "" + error = workflow_run.error or "" total_tokens = workflow_run.total_tokens - file_list = workflow_run_inputs.get("sys.file") if workflow_run_inputs.get("sys.file") else [] + file_list = workflow_run_inputs.get("sys.file") or [] query = workflow_run_inputs.get("query") or workflow_run_inputs.get("sys.query") or "" # get workflow_app_log_id @@ -452,7 +452,7 @@ class TraceTask: message_tokens=message_tokens, answer_tokens=message_data.answer_tokens, total_tokens=message_tokens + message_data.answer_tokens, - error=message_data.error if message_data.error else "", + error=message_data.error or "", inputs=inputs, outputs=message_data.answer, file_list=file_list, @@ -487,7 +487,7 @@ class TraceTask: workflow_app_log_id = str(workflow_app_log_data.id) if workflow_app_log_data else None moderation_trace_info = ModerationTraceInfo( - message_id=workflow_app_log_id if workflow_app_log_id else message_id, + message_id=workflow_app_log_id or message_id, inputs=inputs, message_data=message_data.to_dict(), flagged=moderation_result.flagged, @@ -527,7 +527,7 @@ class TraceTask: workflow_app_log_id = str(workflow_app_log_data.id) if workflow_app_log_data else None suggested_question_trace_info = SuggestedQuestionTraceInfo( - message_id=workflow_app_log_id if workflow_app_log_id else message_id, + message_id=workflow_app_log_id or message_id, message_data=message_data.to_dict(), inputs=message_data.message, outputs=message_data.answer, @@ -569,7 +569,7 @@ class TraceTask: dataset_retrieval_trace_info = DatasetRetrievalTraceInfo( message_id=message_id, - inputs=message_data.query if message_data.query else message_data.inputs, + inputs=message_data.query or message_data.inputs, documents=[doc.model_dump() for doc in documents], start_time=timer.get("start"), end_time=timer.get("end"), @@ -695,8 +695,7 @@ class TraceQueueManager: self.start_timer() def add_trace_task(self, trace_task: TraceTask): - global trace_manager_timer - global trace_manager_queue + global trace_manager_timer, trace_manager_queue try: if self.trace_instance: trace_task.app_id = self.app_id diff --git a/api/core/prompt/simple_prompt_transform.py b/api/core/prompt/simple_prompt_transform.py index 13e5c5253e..7479560520 100644 --- a/api/core/prompt/simple_prompt_transform.py +++ b/api/core/prompt/simple_prompt_transform.py @@ -112,11 +112,11 @@ class SimplePromptTransform(PromptTransform): for v in prompt_template_config["special_variable_keys"]: # support #context#, #query# and #histories# if v == "#context#": - variables["#context#"] = context if context else "" + variables["#context#"] = context or "" elif v == "#query#": - variables["#query#"] = query if query else "" + variables["#query#"] = query or "" elif v == "#histories#": - variables["#histories#"] = histories if histories else "" + variables["#histories#"] = histories or "" prompt_template = prompt_template_config["prompt_template"] prompt = prompt_template.format(variables) diff --git a/api/core/rag/datasource/keyword/keyword_base.py b/api/core/rag/datasource/keyword/keyword_base.py index 27e4f383ad..4b9ec460e6 100644 --- a/api/core/rag/datasource/keyword/keyword_base.py +++ b/api/core/rag/datasource/keyword/keyword_base.py @@ -34,7 +34,7 @@ class BaseKeyword(ABC): raise NotImplementedError def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: - for text in texts[:]: + for text in texts.copy(): doc_id = text.metadata["doc_id"] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: diff --git a/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py b/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py index a9c0eefb78..ef48d22963 100644 --- a/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py +++ b/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py @@ -239,7 +239,7 @@ class AnalyticdbVector(BaseVector): def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]: from alibabacloud_gpdb20160503 import models as gpdb_20160503_models - score_threshold = kwargs.get("score_threshold", 0.0) if kwargs.get("score_threshold", 0.0) else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) request = gpdb_20160503_models.QueryCollectionDataRequest( dbinstance_id=self.config.instance_id, region_id=self.config.region_id, @@ -267,7 +267,7 @@ class AnalyticdbVector(BaseVector): def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: from alibabacloud_gpdb20160503 import models as gpdb_20160503_models - score_threshold = kwargs.get("score_threshold", 0.0) if kwargs.get("score_threshold", 0.0) else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) request = gpdb_20160503_models.QueryCollectionDataRequest( dbinstance_id=self.config.instance_id, region_id=self.config.region_id, diff --git a/api/core/rag/datasource/vdb/chroma/chroma_vector.py b/api/core/rag/datasource/vdb/chroma/chroma_vector.py index cb38cf94a9..41f749279d 100644 --- a/api/core/rag/datasource/vdb/chroma/chroma_vector.py +++ b/api/core/rag/datasource/vdb/chroma/chroma_vector.py @@ -92,7 +92,7 @@ class ChromaVector(BaseVector): def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]: collection = self._client.get_or_create_collection(self._collection_name) results: QueryResult = collection.query(query_embeddings=query_vector, n_results=kwargs.get("top_k", 4)) - score_threshold = kwargs.get("score_threshold", 0.0) if kwargs.get("score_threshold", 0.0) else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) ids: list[str] = results["ids"][0] documents: list[str] = results["documents"][0] diff --git a/api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py b/api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py index f13723b51f..c52ca5f488 100644 --- a/api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py +++ b/api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py @@ -86,8 +86,8 @@ class ElasticSearchVector(BaseVector): id=uuids[i], document={ Field.CONTENT_KEY.value: documents[i].page_content, - Field.VECTOR.value: embeddings[i] if embeddings[i] else None, - Field.METADATA_KEY.value: documents[i].metadata if documents[i].metadata else {}, + Field.VECTOR.value: embeddings[i] or None, + Field.METADATA_KEY.value: documents[i].metadata or {}, }, ) self._client.indices.refresh(index=self._collection_name) @@ -131,7 +131,7 @@ class ElasticSearchVector(BaseVector): docs = [] for doc, score in docs_and_scores: - score_threshold = kwargs.get("score_threshold", 0.0) if kwargs.get("score_threshold", 0.0) else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) if score > score_threshold: doc.metadata["score"] = score docs.append(doc) diff --git a/api/core/rag/datasource/vdb/milvus/milvus_vector.py b/api/core/rag/datasource/vdb/milvus/milvus_vector.py index d6d7136282..7d78f54256 100644 --- a/api/core/rag/datasource/vdb/milvus/milvus_vector.py +++ b/api/core/rag/datasource/vdb/milvus/milvus_vector.py @@ -141,7 +141,7 @@ class MilvusVector(BaseVector): for result in results[0]: metadata = result["entity"].get(Field.METADATA_KEY.value) metadata["score"] = result["distance"] - score_threshold = kwargs.get("score_threshold") if kwargs.get("score_threshold") else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) if result["distance"] > score_threshold: doc = Document(page_content=result["entity"].get(Field.CONTENT_KEY.value), metadata=metadata) docs.append(doc) diff --git a/api/core/rag/datasource/vdb/myscale/myscale_vector.py b/api/core/rag/datasource/vdb/myscale/myscale_vector.py index 90464ac42a..fbd7864edd 100644 --- a/api/core/rag/datasource/vdb/myscale/myscale_vector.py +++ b/api/core/rag/datasource/vdb/myscale/myscale_vector.py @@ -122,7 +122,7 @@ class MyScaleVector(BaseVector): def _search(self, dist: str, order: SortOrder, **kwargs: Any) -> list[Document]: top_k = kwargs.get("top_k", 5) - score_threshold = kwargs.get("score_threshold") or 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) where_str = ( f"WHERE dist < {1 - score_threshold}" if self._metric.upper() == "COSINE" and order == SortOrder.ASC and score_threshold > 0.0 diff --git a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py index 7c0f620956..a6f70c8733 100644 --- a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py +++ b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py @@ -170,7 +170,7 @@ class OpenSearchVector(BaseVector): metadata = {} metadata["score"] = hit["_score"] - score_threshold = kwargs.get("score_threshold") if kwargs.get("score_threshold") else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) if hit["_score"] > score_threshold: doc = Document(page_content=hit["_source"].get(Field.CONTENT_KEY.value), metadata=metadata) docs.append(doc) diff --git a/api/core/rag/datasource/vdb/oracle/oraclevector.py b/api/core/rag/datasource/vdb/oracle/oraclevector.py index d223b0decf..1636582362 100644 --- a/api/core/rag/datasource/vdb/oracle/oraclevector.py +++ b/api/core/rag/datasource/vdb/oracle/oraclevector.py @@ -200,7 +200,7 @@ class OracleVector(BaseVector): [numpy.array(query_vector)], ) docs = [] - score_threshold = kwargs.get("score_threshold") if kwargs.get("score_threshold") else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) for record in cur: metadata, text, distance = record score = 1 - distance @@ -212,7 +212,7 @@ class OracleVector(BaseVector): def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: top_k = kwargs.get("top_k", 5) # just not implement fetch by score_threshold now, may be later - score_threshold = kwargs.get("score_threshold") if kwargs.get("score_threshold") else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) if len(query) > 0: # Check which language the query is in zh_pattern = re.compile("[\u4e00-\u9fa5]+") diff --git a/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py b/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py index 24b391d63a..765436006b 100644 --- a/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py +++ b/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py @@ -198,7 +198,7 @@ class PGVectoRS(BaseVector): metadata = record.meta score = 1 - dis metadata["score"] = score - score_threshold = kwargs.get("score_threshold") if kwargs.get("score_threshold") else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) if score > score_threshold: doc = Document(page_content=record.text, metadata=metadata) docs.append(doc) diff --git a/api/core/rag/datasource/vdb/pgvector/pgvector.py b/api/core/rag/datasource/vdb/pgvector/pgvector.py index d2d9e5238b..c35464b464 100644 --- a/api/core/rag/datasource/vdb/pgvector/pgvector.py +++ b/api/core/rag/datasource/vdb/pgvector/pgvector.py @@ -144,7 +144,7 @@ class PGVector(BaseVector): (json.dumps(query_vector),), ) docs = [] - score_threshold = kwargs.get("score_threshold") if kwargs.get("score_threshold") else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) for record in cur: metadata, text, distance = record score = 1 - distance diff --git a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py index 83d561819c..2bf417e993 100644 --- a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py +++ b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py @@ -339,7 +339,7 @@ class QdrantVector(BaseVector): for result in results: metadata = result.payload.get(Field.METADATA_KEY.value) or {} # duplicate check score threshold - score_threshold = kwargs.get("score_threshold", 0.0) if kwargs.get("score_threshold", 0.0) else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) if result.score > score_threshold: metadata["score"] = result.score doc = Document( diff --git a/api/core/rag/datasource/vdb/relyt/relyt_vector.py b/api/core/rag/datasource/vdb/relyt/relyt_vector.py index 54290eaa5d..76f214ffa4 100644 --- a/api/core/rag/datasource/vdb/relyt/relyt_vector.py +++ b/api/core/rag/datasource/vdb/relyt/relyt_vector.py @@ -230,7 +230,7 @@ class RelytVector(BaseVector): # Organize results. docs = [] for document, score in results: - score_threshold = kwargs.get("score_threshold") if kwargs.get("score_threshold") else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) if 1 - score > score_threshold: docs.append(document) return docs diff --git a/api/core/rag/datasource/vdb/tencent/tencent_vector.py b/api/core/rag/datasource/vdb/tencent/tencent_vector.py index dbedc1d4e9..b550743f39 100644 --- a/api/core/rag/datasource/vdb/tencent/tencent_vector.py +++ b/api/core/rag/datasource/vdb/tencent/tencent_vector.py @@ -153,7 +153,7 @@ class TencentVector(BaseVector): limit=kwargs.get("top_k", 4), timeout=self._client_config.timeout, ) - score_threshold = kwargs.get("score_threshold", 0.0) if kwargs.get("score_threshold", 0.0) else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) return self._get_search_res(res, score_threshold) def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: diff --git a/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py b/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py index 7eaf189292..a2eff6ff04 100644 --- a/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py +++ b/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py @@ -185,7 +185,7 @@ class TiDBVector(BaseVector): def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]: top_k = kwargs.get("top_k", 5) - score_threshold = kwargs.get("score_threshold") if kwargs.get("score_threshold") else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) filter = kwargs.get("filter") distance = 1 - score_threshold diff --git a/api/core/rag/datasource/vdb/vector_base.py b/api/core/rag/datasource/vdb/vector_base.py index fb80cdec87..1a0dc7f48b 100644 --- a/api/core/rag/datasource/vdb/vector_base.py +++ b/api/core/rag/datasource/vdb/vector_base.py @@ -49,7 +49,7 @@ class BaseVector(ABC): raise NotImplementedError def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: - for text in texts[:]: + for text in texts.copy(): doc_id = text.metadata["doc_id"] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index bb24143a41..ca90233b7f 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -153,7 +153,7 @@ class Vector: return CacheEmbedding(embedding_model) def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: - for text in texts[:]: + for text in texts.copy(): doc_id = text.metadata["doc_id"] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: diff --git a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py index ca1123c6a0..64e92c5b82 100644 --- a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py +++ b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py @@ -205,7 +205,7 @@ class WeaviateVector(BaseVector): docs = [] for doc, score in docs_and_scores: - score_threshold = kwargs.get("score_threshold", 0.0) if kwargs.get("score_threshold", 0.0) else 0.0 + score_threshold = kwargs.get("score_threshold", 0.0) # check score threshold if score > score_threshold: doc.metadata["score"] = score diff --git a/api/core/rag/extractor/blob/blob.py b/api/core/rag/extractor/blob/blob.py index f4c7b4b5f7..e46ab8b7fd 100644 --- a/api/core/rag/extractor/blob/blob.py +++ b/api/core/rag/extractor/blob/blob.py @@ -12,7 +12,7 @@ import mimetypes from abc import ABC, abstractmethod from collections.abc import Generator, Iterable, Mapping from io import BufferedReader, BytesIO -from pathlib import PurePath +from pathlib import Path, PurePath from typing import Any, Optional, Union from pydantic import BaseModel, ConfigDict, model_validator @@ -56,8 +56,7 @@ class Blob(BaseModel): def as_string(self) -> str: """Read data as a string.""" if self.data is None and self.path: - with open(str(self.path), encoding=self.encoding) as f: - return f.read() + return Path(str(self.path)).read_text(encoding=self.encoding) elif isinstance(self.data, bytes): return self.data.decode(self.encoding) elif isinstance(self.data, str): @@ -72,8 +71,7 @@ class Blob(BaseModel): elif isinstance(self.data, str): return self.data.encode(self.encoding) elif self.data is None and self.path: - with open(str(self.path), "rb") as f: - return f.read() + return Path(str(self.path)).read_bytes() else: raise ValueError(f"Unable to get bytes for blob {self}") diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py index 244ef9614a..3181656f59 100644 --- a/api/core/rag/extractor/extract_processor.py +++ b/api/core/rag/extractor/extract_processor.py @@ -68,8 +68,7 @@ class ExtractProcessor: suffix = "." + re.search(r"\.(\w+)$", filename).group(1) file_path = f"{temp_dir}/{next(tempfile._get_candidate_names())}{suffix}" - with open(file_path, "wb") as file: - file.write(response.content) + Path(file_path).write_bytes(response.content) extract_setting = ExtractSetting(datasource_type="upload_file", document_model="text_model") if return_text: delimiter = "\n" @@ -111,7 +110,7 @@ class ExtractProcessor: ) elif file_extension in [".htm", ".html"]: extractor = HtmlExtractor(file_path) - elif file_extension in [".docx"]: + elif file_extension == ".docx": extractor = WordExtractor(file_path, upload_file.tenant_id, upload_file.created_by) elif file_extension == ".csv": extractor = CSVExtractor(file_path, autodetect_encoding=True) @@ -143,7 +142,7 @@ class ExtractProcessor: extractor = MarkdownExtractor(file_path, autodetect_encoding=True) elif file_extension in [".htm", ".html"]: extractor = HtmlExtractor(file_path) - elif file_extension in [".docx"]: + elif file_extension == ".docx": extractor = WordExtractor(file_path, upload_file.tenant_id, upload_file.created_by) elif file_extension == ".csv": extractor = CSVExtractor(file_path, autodetect_encoding=True) diff --git a/api/core/rag/extractor/helpers.py b/api/core/rag/extractor/helpers.py index 9a21d4272a..69ca9d5d63 100644 --- a/api/core/rag/extractor/helpers.py +++ b/api/core/rag/extractor/helpers.py @@ -1,6 +1,7 @@ """Document loader helpers.""" import concurrent.futures +from pathlib import Path from typing import NamedTuple, Optional, cast @@ -28,8 +29,7 @@ def detect_file_encodings(file_path: str, timeout: int = 5) -> list[FileEncoding import chardet def read_and_detect(file_path: str) -> list[dict]: - with open(file_path, "rb") as f: - rawdata = f.read() + rawdata = Path(file_path).read_bytes() return cast(list[dict], chardet.detect_all(rawdata)) with concurrent.futures.ThreadPoolExecutor() as executor: diff --git a/api/core/rag/extractor/markdown_extractor.py b/api/core/rag/extractor/markdown_extractor.py index ca125ecf55..849852ac23 100644 --- a/api/core/rag/extractor/markdown_extractor.py +++ b/api/core/rag/extractor/markdown_extractor.py @@ -1,6 +1,7 @@ """Abstract interface for document loader implementations.""" import re +from pathlib import Path from typing import Optional, cast from core.rag.extractor.extractor_base import BaseExtractor @@ -102,15 +103,13 @@ class MarkdownExtractor(BaseExtractor): """Parse file into tuples.""" content = "" try: - with open(filepath, encoding=self._encoding) as f: - content = f.read() + content = Path(filepath).read_text(encoding=self._encoding) except UnicodeDecodeError as e: if self._autodetect_encoding: detected_encodings = detect_file_encodings(filepath) for encoding in detected_encodings: try: - with open(filepath, encoding=encoding.encoding) as f: - content = f.read() + content = Path(filepath).read_text(encoding=encoding.encoding) break except UnicodeDecodeError: continue diff --git a/api/core/rag/extractor/text_extractor.py b/api/core/rag/extractor/text_extractor.py index ed0ae41f51..b2b51d71d7 100644 --- a/api/core/rag/extractor/text_extractor.py +++ b/api/core/rag/extractor/text_extractor.py @@ -1,5 +1,6 @@ """Abstract interface for document loader implementations.""" +from pathlib import Path from typing import Optional from core.rag.extractor.extractor_base import BaseExtractor @@ -25,15 +26,13 @@ class TextExtractor(BaseExtractor): """Load from file path.""" text = "" try: - with open(self._file_path, encoding=self._encoding) as f: - text = f.read() + text = Path(self._file_path).read_text(encoding=self._encoding) except UnicodeDecodeError as e: if self._autodetect_encoding: detected_encodings = detect_file_encodings(self._file_path) for encoding in detected_encodings: try: - with open(self._file_path, encoding=encoding.encoding) as f: - text = f.read() + text = Path(self._file_path).read_text(encoding=encoding.encoding) break except UnicodeDecodeError: continue diff --git a/api/core/rag/extractor/word_extractor.py b/api/core/rag/extractor/word_extractor.py index 947644a90b..7352ef378b 100644 --- a/api/core/rag/extractor/word_extractor.py +++ b/api/core/rag/extractor/word_extractor.py @@ -153,7 +153,7 @@ class WordExtractor(BaseExtractor): if col_index >= total_cols: break cell_content = self._parse_cell(cell, image_map).strip() - cell_colspan = cell.grid_span if cell.grid_span else 1 + cell_colspan = cell.grid_span or 1 for i in range(cell_colspan): if col_index + i < total_cols: row_cells[col_index + i] = cell_content if i == 0 else "" diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index e4ad78ed2b..cdaca8387d 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -256,7 +256,7 @@ class DatasetRetrieval: # get retrieval model config dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first() if dataset: - retrieval_model_config = dataset.retrieval_model if dataset.retrieval_model else default_retrieval_model + retrieval_model_config = dataset.retrieval_model or default_retrieval_model # get top k top_k = retrieval_model_config["top_k"] @@ -410,7 +410,7 @@ class DatasetRetrieval: return [] # get retrieval model , if the model is not setting , using default - retrieval_model = dataset.retrieval_model if dataset.retrieval_model else default_retrieval_model + retrieval_model = dataset.retrieval_model or default_retrieval_model if dataset.indexing_technique == "economy": # use keyword table query @@ -433,9 +433,7 @@ class DatasetRetrieval: reranking_model=retrieval_model.get("reranking_model", None) if retrieval_model["reranking_enable"] else None, - reranking_mode=retrieval_model.get("reranking_mode") - if retrieval_model.get("reranking_mode") - else "reranking_model", + reranking_mode=retrieval_model.get("reranking_mode") or "reranking_model", weights=retrieval_model.get("weights", None), ) @@ -486,7 +484,7 @@ class DatasetRetrieval: } for dataset in available_datasets: - retrieval_model_config = dataset.retrieval_model if dataset.retrieval_model else default_retrieval_model + retrieval_model_config = dataset.retrieval_model or default_retrieval_model # get top k top_k = retrieval_model_config["top_k"] diff --git a/api/core/tools/provider/api_tool_provider.py b/api/core/tools/provider/api_tool_provider.py index 2e6018cffc..d99314e33a 100644 --- a/api/core/tools/provider/api_tool_provider.py +++ b/api/core/tools/provider/api_tool_provider.py @@ -106,7 +106,7 @@ class ApiToolProviderController(ToolProviderController): "human": {"en_US": tool_bundle.summary or "", "zh_Hans": tool_bundle.summary or ""}, "llm": tool_bundle.summary or "", }, - "parameters": tool_bundle.parameters if tool_bundle.parameters else [], + "parameters": tool_bundle.parameters or [], } ) diff --git a/api/core/tools/provider/builtin/aws/tools/sagemaker_text_rerank.py b/api/core/tools/provider/builtin/aws/tools/sagemaker_text_rerank.py index 3c35b65e66..bffcd058b5 100644 --- a/api/core/tools/provider/builtin/aws/tools/sagemaker_text_rerank.py +++ b/api/core/tools/provider/builtin/aws/tools/sagemaker_text_rerank.py @@ -1,4 +1,5 @@ import json +import operator from typing import Any, Union import boto3 @@ -71,7 +72,7 @@ class SageMakerReRankTool(BuiltinTool): candidate_docs[idx]["score"] = scores[idx] line = 8 - sorted_candidate_docs = sorted(candidate_docs, key=lambda x: x["score"], reverse=True) + sorted_candidate_docs = sorted(candidate_docs, key=operator.itemgetter("score"), reverse=True) line = 9 return [self.create_json_message(res) for res in sorted_candidate_docs[: self.topk]] diff --git a/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py index 9e0918afa9..40e1af043b 100644 --- a/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py +++ b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py @@ -115,7 +115,7 @@ class GetWorksheetFieldsTool(BuiltinTool): fields.append(field) fields_list.append( f"|{field['id']}|{field['name']}|{field['type']}|{field['typeId']}|{field['description']}" - f"|{field['options'] if field['options'] else ''}|" + f"|{field['options'] or ''}|" ) fields.append( diff --git a/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.py b/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.py index 6b831f3145..01c1af9b3e 100644 --- a/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.py +++ b/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.py @@ -130,7 +130,7 @@ class GetWorksheetPivotDataTool(BuiltinTool): # ] rows = [] for row in data["data"]: - row_data = row["rows"] if row["rows"] else {} + row_data = row["rows"] or {} row_data.update(row["columns"]) row_data.update(row["values"]) rows.append(row_data) diff --git a/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py index 5888f7443f..171895a306 100644 --- a/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py +++ b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py @@ -113,7 +113,7 @@ class ListWorksheetRecordsTool(BuiltinTool): result_text = f"Found {result['total']} rows in worksheet \"{worksheet_name}\"." if result["total"] > 0: result_text += ( - f" The following are {result['total'] if result['total'] < limit else limit}" + f" The following are {min(limit, result['total'])}" f" pieces of data presented in a table format:\n\n{table_header}" ) for row in rows: diff --git a/api/core/tools/provider/builtin/searchapi/tools/youtube_transcripts.py b/api/core/tools/provider/builtin/searchapi/tools/youtube_transcripts.py index d7bfb53bd7..09725cf8a2 100644 --- a/api/core/tools/provider/builtin/searchapi/tools/youtube_transcripts.py +++ b/api/core/tools/provider/builtin/searchapi/tools/youtube_transcripts.py @@ -37,7 +37,7 @@ class SearchAPI: return { "engine": "youtube_transcripts", "video_id": video_id, - "lang": language if language else "en", + "lang": language or "en", **{key: value for key, value in kwargs.items() if value not in [None, ""]}, } diff --git a/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py b/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py index e76af6fe70..067600c601 100644 --- a/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py +++ b/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py @@ -160,7 +160,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool): hit_callback.on_query(query, dataset.id) # get retrieval model , if the model is not setting , using default - retrieval_model = dataset.retrieval_model if dataset.retrieval_model else default_retrieval_model + retrieval_model = dataset.retrieval_model or default_retrieval_model if dataset.indexing_technique == "economy": # use keyword table query @@ -183,9 +183,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool): reranking_model=retrieval_model.get("reranking_model", None) if retrieval_model["reranking_enable"] else None, - reranking_mode=retrieval_model.get("reranking_mode") - if retrieval_model.get("reranking_mode") - else "reranking_model", + reranking_mode=retrieval_model.get("reranking_mode") or "reranking_model", weights=retrieval_model.get("weights", None), ) diff --git a/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py b/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py index f61458278e..ad533946a1 100644 --- a/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py +++ b/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py @@ -55,7 +55,7 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool): hit_callback.on_query(query, dataset.id) # get retrieval model , if the model is not setting , using default - retrieval_model = dataset.retrieval_model if dataset.retrieval_model else default_retrieval_model + retrieval_model = dataset.retrieval_model or default_retrieval_model if dataset.indexing_technique == "economy": # use keyword table query documents = RetrievalService.retrieve( @@ -76,9 +76,7 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool): reranking_model=retrieval_model.get("reranking_model", None) if retrieval_model["reranking_enable"] else None, - reranking_mode=retrieval_model.get("reranking_mode") - if retrieval_model.get("reranking_mode") - else "reranking_model", + reranking_mode=retrieval_model.get("reranking_mode") or "reranking_model", weights=retrieval_model.get("weights", None), ) else: diff --git a/api/core/tools/utils/web_reader_tool.py b/api/core/tools/utils/web_reader_tool.py index fc2f63a241..e57cae9f16 100644 --- a/api/core/tools/utils/web_reader_tool.py +++ b/api/core/tools/utils/web_reader_tool.py @@ -8,6 +8,7 @@ import subprocess import tempfile import unicodedata from contextlib import contextmanager +from pathlib import Path from urllib.parse import unquote import chardet @@ -98,7 +99,7 @@ def get_url(url: str, user_agent: str = None) -> str: authors=a["byline"], publish_date=a["date"], top_image="", - text=a["plain_text"] if a["plain_text"] else "", + text=a["plain_text"] or "", ) return res @@ -117,8 +118,7 @@ def extract_using_readabilipy(html): subprocess.check_call(["node", "ExtractArticle.js", "-i", html_path, "-o", article_json_path]) # Read output of call to Readability.parse() from JSON file and return as Python dictionary - with open(article_json_path, encoding="utf-8") as json_file: - input_json = json.loads(json_file.read()) + input_json = json.loads(Path(article_json_path).read_text(encoding="utf-8")) # Deleting files after processing os.unlink(article_json_path) diff --git a/api/core/tools/utils/yaml_utils.py b/api/core/tools/utils/yaml_utils.py index bcb061376d..99b9f80499 100644 --- a/api/core/tools/utils/yaml_utils.py +++ b/api/core/tools/utils/yaml_utils.py @@ -21,7 +21,7 @@ def load_yaml_file(file_path: str, ignore_error: bool = True, default_value: Any with open(file_path, encoding="utf-8") as yaml_file: try: yaml_content = yaml.safe_load(yaml_file) - return yaml_content if yaml_content else default_value + return yaml_content or default_value except Exception as e: raise YAMLError(f"Failed to load YAML file {file_path}: {e}") except Exception as e: diff --git a/api/core/workflow/graph_engine/entities/graph.py b/api/core/workflow/graph_engine/entities/graph.py index c156dd8c98..20efe91a59 100644 --- a/api/core/workflow/graph_engine/entities/graph.py +++ b/api/core/workflow/graph_engine/entities/graph.py @@ -268,7 +268,7 @@ class Graph(BaseModel): f"Node {graph_edge.source_node_id} is connected to the previous node, please check the graph." ) - new_route = route[:] + new_route = route.copy() new_route.append(graph_edge.target_node_id) cls._check_connected_to_previous_node( route=new_route, @@ -679,8 +679,7 @@ class Graph(BaseModel): all_routes_node_ids = set() parallel_start_node_ids: dict[str, list[str]] = {} for branch_node_id, node_ids in routes_node_ids.items(): - for node_id in node_ids: - all_routes_node_ids.add(node_id) + all_routes_node_ids.update(node_ids) if branch_node_id in reverse_edge_mapping: for graph_edge in reverse_edge_mapping[branch_node_id]: diff --git a/api/core/workflow/nodes/code/code_node.py b/api/core/workflow/nodes/code/code_node.py index 73164fff9a..9da7ad99f3 100644 --- a/api/core/workflow/nodes/code/code_node.py +++ b/api/core/workflow/nodes/code/code_node.py @@ -74,7 +74,7 @@ class CodeNode(BaseNode): :return: """ if not isinstance(value, str): - if isinstance(value, type(None)): + if value is None: return None else: raise ValueError(f"Output variable `{variable}` must be a string") @@ -95,7 +95,7 @@ class CodeNode(BaseNode): :return: """ if not isinstance(value, int | float): - if isinstance(value, type(None)): + if value is None: return None else: raise ValueError(f"Output variable `{variable}` must be a number") @@ -182,7 +182,7 @@ class CodeNode(BaseNode): f"Output {prefix}.{output_name} is not a valid array." f" make sure all elements are of the same type." ) - elif isinstance(output_value, type(None)): + elif output_value is None: pass else: raise ValueError(f"Output {prefix}.{output_name} is not a valid type.") @@ -284,7 +284,7 @@ class CodeNode(BaseNode): for i, value in enumerate(result[output_name]): if not isinstance(value, dict): - if isinstance(value, type(None)): + if value is None: pass else: raise ValueError( diff --git a/api/core/workflow/nodes/if_else/if_else_node.py b/api/core/workflow/nodes/if_else/if_else_node.py index 5b4737c6e5..37384202d8 100644 --- a/api/core/workflow/nodes/if_else/if_else_node.py +++ b/api/core/workflow/nodes/if_else/if_else_node.py @@ -79,7 +79,7 @@ class IfElseNode(BaseNode): status=WorkflowNodeExecutionStatus.SUCCEEDED, inputs=node_inputs, process_data=process_datas, - edge_source_handle=selected_case_id if selected_case_id else "false", # Use case ID or 'default' + edge_source_handle=selected_case_id or "false", # Use case ID or 'default' outputs=outputs, ) diff --git a/api/core/workflow/nodes/llm/llm_node.py b/api/core/workflow/nodes/llm/llm_node.py index 049c211488..3d336b0b0b 100644 --- a/api/core/workflow/nodes/llm/llm_node.py +++ b/api/core/workflow/nodes/llm/llm_node.py @@ -580,7 +580,7 @@ class LLMNode(BaseNode): prompt_messages = prompt_transform.get_prompt( prompt_template=node_data.prompt_template, inputs=inputs, - query=query if query else "", + query=query or "", files=files, context=context, memory_config=node_data.memory, diff --git a/api/core/workflow/nodes/question_classifier/question_classifier_node.py b/api/core/workflow/nodes/question_classifier/question_classifier_node.py index d860f848ec..2ae58bc5f7 100644 --- a/api/core/workflow/nodes/question_classifier/question_classifier_node.py +++ b/api/core/workflow/nodes/question_classifier/question_classifier_node.py @@ -250,7 +250,7 @@ class QuestionClassifierNode(LLMNode): for class_ in classes: category = {"category_id": class_.id, "category_name": class_.name} categories.append(category) - instruction = node_data.instruction if node_data.instruction else "" + instruction = node_data.instruction or "" input_text = query memory_str = "" if memory: diff --git a/api/events/event_handlers/update_app_dataset_join_when_app_model_config_updated.py b/api/events/event_handlers/update_app_dataset_join_when_app_model_config_updated.py index 59375b1a0b..de7c0f4dfe 100644 --- a/api/events/event_handlers/update_app_dataset_join_when_app_model_config_updated.py +++ b/api/events/event_handlers/update_app_dataset_join_when_app_model_config_updated.py @@ -18,8 +18,7 @@ def handle(sender, **kwargs): added_dataset_ids = dataset_ids else: old_dataset_ids = set() - for app_dataset_join in app_dataset_joins: - old_dataset_ids.add(app_dataset_join.dataset_id) + old_dataset_ids.update(app_dataset_join.dataset_id for app_dataset_join in app_dataset_joins) added_dataset_ids = dataset_ids - old_dataset_ids removed_dataset_ids = old_dataset_ids - dataset_ids diff --git a/api/events/event_handlers/update_app_dataset_join_when_app_published_workflow_updated.py b/api/events/event_handlers/update_app_dataset_join_when_app_published_workflow_updated.py index 333b85ecb2..c5e98e263f 100644 --- a/api/events/event_handlers/update_app_dataset_join_when_app_published_workflow_updated.py +++ b/api/events/event_handlers/update_app_dataset_join_when_app_published_workflow_updated.py @@ -22,8 +22,7 @@ def handle(sender, **kwargs): added_dataset_ids = dataset_ids else: old_dataset_ids = set() - for app_dataset_join in app_dataset_joins: - old_dataset_ids.add(app_dataset_join.dataset_id) + old_dataset_ids.update(app_dataset_join.dataset_id for app_dataset_join in app_dataset_joins) added_dataset_ids = dataset_ids - old_dataset_ids removed_dataset_ids = old_dataset_ids - dataset_ids diff --git a/api/extensions/storage/local_storage.py b/api/extensions/storage/local_storage.py index 46ee4bf80f..f833ae85dc 100644 --- a/api/extensions/storage/local_storage.py +++ b/api/extensions/storage/local_storage.py @@ -1,6 +1,7 @@ import os import shutil from collections.abc import Generator +from pathlib import Path from flask import Flask @@ -26,8 +27,7 @@ class LocalStorage(BaseStorage): folder = os.path.dirname(filename) os.makedirs(folder, exist_ok=True) - with open(os.path.join(os.getcwd(), filename), "wb") as f: - f.write(data) + Path(os.path.join(os.getcwd(), filename)).write_bytes(data) def load_once(self, filename: str) -> bytes: if not self.folder or self.folder.endswith("/"): @@ -38,9 +38,7 @@ class LocalStorage(BaseStorage): if not os.path.exists(filename): raise FileNotFoundError("File not found") - with open(filename, "rb") as f: - data = f.read() - + data = Path(filename).read_bytes() return data def load_stream(self, filename: str) -> Generator: diff --git a/api/models/dataset.py b/api/models/dataset.py index 55f6ed3180..0da35910cd 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -144,7 +144,7 @@ class Dataset(db.Model): "top_k": 2, "score_threshold_enabled": False, } - return self.retrieval_model if self.retrieval_model else default_retrieval_model + return self.retrieval_model or default_retrieval_model @property def tags(self): @@ -160,7 +160,7 @@ class Dataset(db.Model): .all() ) - return tags if tags else [] + return tags or [] @staticmethod def gen_collection_name_by_id(dataset_id: str) -> str: diff --git a/api/models/model.py b/api/models/model.py index 8ab3026522..a8b2e00ee4 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -118,7 +118,7 @@ class App(db.Model): @property def api_base_url(self): - return (dify_config.SERVICE_API_URL if dify_config.SERVICE_API_URL else request.host_url.rstrip("/")) + "/v1" + return (dify_config.SERVICE_API_URL or request.host_url.rstrip("/")) + "/v1" @property def tenant(self): @@ -207,7 +207,7 @@ class App(db.Model): .all() ) - return tags if tags else [] + return tags or [] class AppModelConfig(db.Model): @@ -908,7 +908,7 @@ class Message(db.Model): "id": message_file.id, "type": message_file.type, "url": url, - "belongs_to": message_file.belongs_to if message_file.belongs_to else "user", + "belongs_to": message_file.belongs_to or "user", } ) @@ -1212,7 +1212,7 @@ class Site(db.Model): @property def app_base_url(self): - return dify_config.APP_WEB_URL if dify_config.APP_WEB_URL else request.url_root.rstrip("/") + return dify_config.APP_WEB_URL or request.url_root.rstrip("/") class ApiToken(db.Model): @@ -1488,7 +1488,7 @@ class TraceAppConfig(db.Model): @property def tracing_config_dict(self): - return self.tracing_config if self.tracing_config else {} + return self.tracing_config or {} @property def tracing_config_str(self): diff --git a/api/pyproject.toml b/api/pyproject.toml index 616794cf3a..7d64d07678 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -15,6 +15,7 @@ select = [ "C4", # flake8-comprehensions "E", # pycodestyle E rules "F", # pyflakes rules + "FURB", # refurb rules "I", # isort rules "N", # pep8-naming "RUF019", # unnecessary-key-check @@ -37,6 +38,8 @@ ignore = [ "F405", # undefined-local-with-import-star-usage "F821", # undefined-name "F841", # unused-variable + "FURB113", # repeated-append + "FURB152", # math-constant "UP007", # non-pep604-annotation "UP032", # f-string "B005", # strip-with-multi-characters diff --git a/api/services/account_service.py b/api/services/account_service.py index 7fb42f9e81..e839ae54ba 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -544,7 +544,7 @@ class RegisterService: """Register account""" try: account = AccountService.create_account( - email=email, name=name, interface_language=language if language else languages[0], password=password + email=email, name=name, interface_language=language or languages[0], password=password ) account.status = AccountStatus.ACTIVE.value if not status else status.value account.initialized_at = datetime.now(timezone.utc).replace(tzinfo=None) diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index 73c446b83b..8dcefbadc0 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -81,13 +81,11 @@ class AppDslService: raise ValueError("Missing app in data argument") # get app basic info - name = args.get("name") if args.get("name") else app_data.get("name") - description = args.get("description") if args.get("description") else app_data.get("description", "") - icon_type = args.get("icon_type") if args.get("icon_type") else app_data.get("icon_type") - icon = args.get("icon") if args.get("icon") else app_data.get("icon") - icon_background = ( - args.get("icon_background") if args.get("icon_background") else app_data.get("icon_background") - ) + name = args.get("name") or app_data.get("name") + description = args.get("description") or app_data.get("description", "") + icon_type = args.get("icon_type") or app_data.get("icon_type") + icon = args.get("icon") or app_data.get("icon") + icon_background = args.get("icon_background") or app_data.get("icon_background") use_icon_as_answer_icon = app_data.get("use_icon_as_answer_icon", False) # import dsl and create app diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index cce0874cf4..a52b5b7e7d 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -155,7 +155,7 @@ class DatasetService: dataset.tenant_id = tenant_id dataset.embedding_model_provider = embedding_model.provider if embedding_model else None dataset.embedding_model = embedding_model.model if embedding_model else None - dataset.permission = permission if permission else DatasetPermissionEnum.ONLY_ME + dataset.permission = permission or DatasetPermissionEnum.ONLY_ME db.session.add(dataset) db.session.commit() return dataset @@ -681,11 +681,7 @@ class DocumentService: "score_threshold_enabled": False, } - dataset.retrieval_model = ( - document_data.get("retrieval_model") - if document_data.get("retrieval_model") - else default_retrieval_model - ) + dataset.retrieval_model = document_data.get("retrieval_model") or default_retrieval_model documents = [] batch = time.strftime("%Y%m%d%H%M%S") + str(random.randint(100000, 999999)) diff --git a/api/services/hit_testing_service.py b/api/services/hit_testing_service.py index 2f911f5036..a9f963dbac 100644 --- a/api/services/hit_testing_service.py +++ b/api/services/hit_testing_service.py @@ -33,7 +33,7 @@ class HitTestingService: # get retrieval model , if the model is not setting , using default if not retrieval_model: - retrieval_model = dataset.retrieval_model if dataset.retrieval_model else default_retrieval_model + retrieval_model = dataset.retrieval_model or default_retrieval_model all_documents = RetrievalService.retrieve( retrieval_method=retrieval_model.get("search_method", "semantic_search"), @@ -46,9 +46,7 @@ class HitTestingService: reranking_model=retrieval_model.get("reranking_model", None) if retrieval_model["reranking_enable"] else None, - reranking_mode=retrieval_model.get("reranking_mode") - if retrieval_model.get("reranking_mode") - else "reranking_model", + reranking_mode=retrieval_model.get("reranking_mode") or "reranking_model", weights=retrieval_model.get("weights", None), ) diff --git a/api/services/model_provider_service.py b/api/services/model_provider_service.py index c0f3c40762..384a072b37 100644 --- a/api/services/model_provider_service.py +++ b/api/services/model_provider_service.py @@ -1,6 +1,7 @@ import logging import mimetypes import os +from pathlib import Path from typing import Optional, cast import requests @@ -453,9 +454,8 @@ class ModelProviderService: mimetype = mimetype or "application/octet-stream" # read binary from file - with open(file_path, "rb") as f: - byte_data = f.read() - return byte_data, mimetype + byte_data = Path(file_path).read_bytes() + return byte_data, mimetype def switch_preferred_provider(self, tenant_id: str, provider: str, preferred_provider_type: str) -> None: """ diff --git a/api/services/recommended_app_service.py b/api/services/recommended_app_service.py index 10abf0a764..daec8393d0 100644 --- a/api/services/recommended_app_service.py +++ b/api/services/recommended_app_service.py @@ -1,6 +1,7 @@ import json import logging from os import path +from pathlib import Path from typing import Optional import requests @@ -218,10 +219,9 @@ class RecommendedAppService: return cls.builtin_data root_path = current_app.root_path - with open(path.join(root_path, "constants", "recommended_apps.json"), encoding="utf-8") as f: - json_data = f.read() - data = json.loads(json_data) - cls.builtin_data = data + cls.builtin_data = json.loads( + Path(path.join(root_path, "constants", "recommended_apps.json")).read_text(encoding="utf-8") + ) return cls.builtin_data diff --git a/api/services/tag_service.py b/api/services/tag_service.py index 0c17485a9f..5e2851cd8f 100644 --- a/api/services/tag_service.py +++ b/api/services/tag_service.py @@ -57,7 +57,7 @@ class TagService: .all() ) - return tags if tags else [] + return tags or [] @staticmethod def save_tags(args: dict) -> Tag: diff --git a/api/services/tools/builtin_tools_manage_service.py b/api/services/tools/builtin_tools_manage_service.py index dc8cebb587..e2e49d017e 100644 --- a/api/services/tools/builtin_tools_manage_service.py +++ b/api/services/tools/builtin_tools_manage_service.py @@ -1,5 +1,6 @@ import json import logging +from pathlib import Path from configs import dify_config from core.helper.position_helper import is_filtered @@ -183,8 +184,7 @@ class BuiltinToolManageService: get tool provider icon and it's mimetype """ icon_path, mime_type = ToolManager.get_builtin_provider_icon(provider) - with open(icon_path, "rb") as f: - icon_bytes = f.read() + icon_bytes = Path(icon_path).read_bytes() return icon_bytes, mime_type diff --git a/api/services/website_service.py b/api/services/website_service.py index 6dff35d63f..fea605cf30 100644 --- a/api/services/website_service.py +++ b/api/services/website_service.py @@ -50,8 +50,8 @@ class WebsiteService: excludes = options.get("excludes").split(",") if options.get("excludes") else [] params = { "crawlerOptions": { - "includes": includes if includes else [], - "excludes": excludes if excludes else [], + "includes": includes or [], + "excludes": excludes or [], "generateImgAltText": True, "limit": options.get("limit", 1), "returnOnlyUrls": False, diff --git a/api/services/workflow/workflow_converter.py b/api/services/workflow/workflow_converter.py index 4b845be2f4..db1a036e68 100644 --- a/api/services/workflow/workflow_converter.py +++ b/api/services/workflow/workflow_converter.py @@ -63,11 +63,11 @@ class WorkflowConverter: # create new app new_app = App() new_app.tenant_id = app_model.tenant_id - new_app.name = name if name else app_model.name + "(workflow)" + new_app.name = name or app_model.name + "(workflow)" new_app.mode = AppMode.ADVANCED_CHAT.value if app_model.mode == AppMode.CHAT.value else AppMode.WORKFLOW.value - new_app.icon_type = icon_type if icon_type else app_model.icon_type - new_app.icon = icon if icon else app_model.icon - new_app.icon_background = icon_background if icon_background else app_model.icon_background + new_app.icon_type = icon_type or app_model.icon_type + new_app.icon = icon or app_model.icon + new_app.icon_background = icon_background or app_model.icon_background new_app.enable_site = app_model.enable_site new_app.enable_api = app_model.enable_api new_app.api_rpm = app_model.api_rpm diff --git a/api/tests/integration_tests/model_runtime/__mock/huggingface_tei.py b/api/tests/integration_tests/model_runtime/__mock/huggingface_tei.py index 83317e59de..b9a721c803 100644 --- a/api/tests/integration_tests/model_runtime/__mock/huggingface_tei.py +++ b/api/tests/integration_tests/model_runtime/__mock/huggingface_tei.py @@ -51,7 +51,7 @@ class MockTEIClass: # } # } embeddings = [] - for idx, text in enumerate(texts): + for idx in range(len(texts)): embedding = [0.1] * 768 embeddings.append( {