【AI Shift Advent Calendar 2023】ユーザー体験を向上させるOpenAI(Compatible) Streaming APIのレスポンス速度計測

{"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0613", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}

{"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0613", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}

{"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0613", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}]}

....

{"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0613", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"content":" today"},"finish_reason":null}]}

{"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0613", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"content":"?"},"finish_reason":null}]}

{"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0613", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}

#!/bin/bash

event_count=0
start_time=$(perl -MTime::HiRes -le 'print int(Time::HiRes::time() * 1000)')
first_event_time=
end_time=

# Handle each line (event) from the server
while IFS= read -r line; do
  # Update event count
  ((event_count++))

  # Capture the time of the first event
  if [ -z "$first_event_time" ]; then
    first_event_time=$(perl -MTime::HiRes -le 'print int(Time::HiRes::time() * 1000)')
    time_to_first_event=$(echo "scale=3; $((first_event_time - start_time)) / 1000" | bc)
    echo "Time To First Token: $time_to_first_event sec"
  fi

  # Continuously capture the latest event time as potential end time
  end_time=$(perl -MTime::HiRes -le 'print int(Time::HiRes::time() * 1000)')

done < <(curl -s -X POST https://api.openai.com/v1/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer <YOUR OPENAI KEY>" \
  -d @chat_stream_input.json)

time_from_first_to_last_event=$(echo "scale=3; $((end_time - first_event_time)) / 1000" | bc)
sec_per_events=$(echo "scale=3; 1000 * $time_from_first_to_last_event / $event_count" | bc -l)
latency=$(echo "scale=3; $((end_time - start_time)) / 1000" | bc -l)

echo "Elapsed time: $time_from_first_to_last_event sec" # 1つ目のトークンが出力されてから最後のトークンが出力されるまでの時間
echo "Latency: $latency sec" # リクエストを送信してから最後のトークンが出力されるまでの時間
echo "Time Per Output Tokens: $sec_per_events millisec/token"

{
  "model": "gpt-3.5-turbo",
  "max_tokens": 100,
  "stream": true,
  "messages": [
      {
          "role": "system",
          "content": "あなたは株式会社AI Shiftのカスタマーサポートです。"
      },
      {
          "role": "user",
          "content": "株式会社AI Shiftについて教えてください。"
      }
  ]
}

Time to First Token: 2.964 sec
Elapsed time: 4.534 sec
Latency: 7.498 sec
Time Per Output Tokens: 27.646 millisec/token

Time to First Token: 6.090 sec
Elapsed time: 5.193 sec
Latency: 11.283 sec
Time Per Output Tokens: 28.532 millisec/token

Time to First Token: 4.931 sec
Elapsed time: 5.196 sec
Latency: 10.127 sec
Time Per Output Tokens: 27.935 millisec/token

Time to First Token: 0.412 sec
Elapsed time: 1.692 sec
Latency: 2.104 sec
Time Per Output Tokens: 30.712 millisec/token