From 193aaaeab6db8f90b85683c52b4dc9bd6e6bcb6f Mon Sep 17 00:00:00 2001 From: -LAN- Date: Mon, 24 Mar 2025 17:25:58 +0800 Subject: [PATCH] feat: disable token counting in GPT2Tokenizer by returning 0. Signed-off-by: -LAN- --- .../model_providers/__base/tokenizers/gpt2_tokenzier.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py b/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py index 2f6f4fbbef..5aeb25c6f2 100644 --- a/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py +++ b/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py @@ -25,7 +25,8 @@ class GPT2Tokenizer: # future = _executor.submit(GPT2Tokenizer._get_num_tokens_by_gpt2, text) # result = future.result() # return cast(int, result) - return GPT2Tokenizer._get_num_tokens_by_gpt2(text) + # return GPT2Tokenizer._get_num_tokens_by_gpt2(text) + return 0 @staticmethod def get_encoder() -> Any: