From 5ea4a24ad43d991af8343ec612ab1d77625bd0d7 Mon Sep 17 00:00:00 2001 From: CyanideByte Date: Fri, 1 Dec 2023 06:50:40 -0800 Subject: [PATCH 1/2] Fix crash in %tokens where count_tokens fails to find tokenizer because of openai/ in model name --- interpreter/terminal_interface/utils/count_tokens.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/interpreter/terminal_interface/utils/count_tokens.py b/interpreter/terminal_interface/utils/count_tokens.py index 19e38d41e8..d06156592b 100644 --- a/interpreter/terminal_interface/utils/count_tokens.py +++ b/interpreter/terminal_interface/utils/count_tokens.py @@ -7,7 +7,16 @@ def count_tokens(text="", model="gpt-4"): Count the number of tokens in a string """ - encoder = tiktoken.encoding_for_model(model) + # Fix bug where models starting with openai/ for example can'git t find tokenizer + if '/' in model: + model = model.split('/')[-1] + + # At least give an estimate if we can't find the tokenizer + try: + encoder = tiktoken.encoding_for_model(model) + except KeyError: + print(f"Could not find tokenizer for {model}. Defaulting to gpt-4 tokenizer.") + encoder = tiktoken.encoding_for_model("gpt-4") return len(encoder.encode(text)) From 5c77ab8c2825f1b912b3ac81c5553a5714d50f65 Mon Sep 17 00:00:00 2001 From: CyanideByte Date: Fri, 1 Dec 2023 07:15:05 -0800 Subject: [PATCH 2/2] typo in comment --- interpreter/terminal_interface/utils/count_tokens.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interpreter/terminal_interface/utils/count_tokens.py b/interpreter/terminal_interface/utils/count_tokens.py index d06156592b..2c2bd387b9 100644 --- a/interpreter/terminal_interface/utils/count_tokens.py +++ b/interpreter/terminal_interface/utils/count_tokens.py @@ -7,7 +7,7 @@ def count_tokens(text="", model="gpt-4"): Count the number of tokens in a string """ - # Fix bug where models starting with openai/ for example can'git t find tokenizer + # Fix bug where models starting with openai/ for example can't find tokenizer if '/' in model: model = model.split('/')[-1]