OpenAIで消費されるトークン数を計算する方法

pip install tiktoken

import tiktoken

chat = "こんにちは。私の名前はMETAG太郎です。METAGに所属している架空の社員です。"

encoding_2 = tiktoken.encoding_for_model("gpt2")
encoding_35 = tiktoken.encoding_for_model("gpt-3.5-turbo-0301")
encoding_4 = tiktoken.encoding_for_model("gpt-4-0314")
encoding_d2 = tiktoken.encoding_for_model("text-davinci-002")

def calc_token_tiktoken(chat, encoding_name):
    encoding = tiktoken.get_encoding(encoding_name)
    num_tokens = len(encoding.encode(chat))
    return num_tokens

print("tiktokenで計測したGPT2のトークン数　：", calc_token_tiktoken(chat, encoding_2.name))
print("tiktokenで計測したGPT3.5のトークン数：", calc_token_tiktoken(chat, encoding_35.name))
print("tiktokenで計測したGPT4のトークン数　：", calc_token_tiktoken(chat, encoding_4.name))
print("tiktokenで計測したtext-davinchi002のトークン数", calc_token_tiktoken(chat, encoding_d2.name))

curl https://api.openai.com/v1/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $OPENAI_API_KEY" \
  -d '{
    "model": "gpt-3.5-turbo-instruct",
    "prompt": "Say this is a test",
    "max_tokens": 7,
    "temperature": 0
  }'

{
  "id": "cmpl-uqkvlQyYK7bGYrRHQ0eXlWi7",
  "object": "text_completion",
  "created": 1589478378,
  "model": "gpt-3.5-turbo-instruct",
  "system_fingerprint": "fp_44709d6fcb",
  "choices": [
    {
      "text": "\n\nThis is indeed a test",
      "index": 0,
      "logprobs": null,
      "finish_reason": "length"
    }
  ],
  "usage": {
    "prompt_tokens": 5,
    "completion_tokens": 7,
    "total_tokens": 12
  }
}