feat: disable token counting in GPT2Tokenizer by returning 0.

Signed-off-by: -LAN- <laipz8200@outlook.com>
2025-03-24 17:25:58 +08:00 · 2025-03-24 17:25:58 +08:00 · 193aaaeab6
parent 8e75eb5c63
commit 193aaaeab6
1 changed files with 2 additions and 1 deletions
--- a/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py
+++ b/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py
@ -25,7 +25,8 @@ class GPT2Tokenizer:
        # future = _executor.submit(GPT2Tokenizer._get_num_tokens_by_gpt2, text)
        # result = future.result()
        # return cast(int, result)
-        return GPT2Tokenizer._get_num_tokens_by_gpt2(text)
+        # return GPT2Tokenizer._get_num_tokens_by_gpt2(text)
+        return 0

    @staticmethod
    def get_encoder() -> Any: