KI-Server-Benchmarking: Leistung messen und vergleichen

import time
from openai import OpenAI

client = OpenAI(base_url="http://localhost:8000/v1", api_key="sk-fake")

prompt = "Schreiben Sie einen 500-Wörter-Aufsatz über KI im Mittelstand."

# Benchmark: Tokens/Second
start = time.time()
response = client.chat.completions.create(
    model="llama3.3:70b",
    messages=[{"role": "user", "content": prompt}],
    max_tokens=500,
    temperature=0,
    stream=True
)

total_tokens = 0
ttft = None
for chunk in response:
    if ttft is None:
        ttft = time.time() - start
    total_tokens += 1

total_time = time.time() - start
tokens_per_second = total_tokens / total_time

print(f"TTFT: {ttft*1000:.0f}ms")
print(f"Tokens/Second: {tokens_per_second:.1f}")
print(f"Total Tokens: {total_tokens}")

# Modell-Benchmark mit LM Evaluation Harness
pip install lm-eval

lm_eval --model hf --model_args "pretrained=meta-llama/Llama-3.3-70B,inference" \
  --tasks mmlu,humaneval,ifeval \
  --batch_size 1