{
  "anthropic-claude-mythos-5": {
    "model_name": "Claude Mythos 5",
    "provider": "Anthropic",
    "category": "flagship",
    "avg_benchmark": 84.7,
    "blended_cost": 15.0,
    "perf_per_dollar": 5.6,
    "gpqa_diamond": 94.1,
    "aime_2025": null,
    "swe_bench": 95.5,
    "hles": 64.5,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": null,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "anthropic-claude-fable-5": {
    "model_name": "Claude Fable 5",
    "provider": "Anthropic",
    "category": "flagship",
    "avg_benchmark": 94.5,
    "blended_cost": 30.0,
    "perf_per_dollar": 3.1,
    "gpqa_diamond": 94.1,
    "aime_2025": null,
    "swe_bench": 95.0,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": null,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "anthropic-claude-opus-48": {
    "model_name": "Claude Opus 4.8",
    "provider": "Anthropic",
    "category": "flagship",
    "avg_benchmark": 80.0,
    "blended_cost": 15.0,
    "perf_per_dollar": 5.3,
    "gpqa_diamond": 93.6,
    "aime_2025": null,
    "swe_bench": 88.6,
    "hles": 57.9,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": null,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "anthropic-claude-opus-47": {
    "model_name": "Claude Opus 4.7",
    "provider": "Anthropic",
    "category": "flagship",
    "avg_benchmark": 90.9,
    "blended_cost": 15.0,
    "perf_per_dollar": 6.1,
    "gpqa_diamond": 94.2,
    "aime_2025": null,
    "swe_bench": 87.6,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": null,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "anthropic-claude-opus-46": {
    "model_name": "Claude Opus 4.6",
    "provider": "Anthropic",
    "category": "flagship",
    "avg_benchmark": 86.6,
    "blended_cost": 15.0,
    "perf_per_dollar": 5.8,
    "gpqa_diamond": null,
    "aime_2025": 99.8,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": 68.8,
    "mmmlu": 91.1,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 67,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "anthropic-claude-opus-45": {
    "model_name": "Claude Opus 4.5",
    "provider": "Anthropic",
    "category": "flagship",
    "avg_benchmark": 64.2,
    "blended_cost": 15.0,
    "perf_per_dollar": 4.3,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": 37.6,
    "mmmlu": 90.8,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": null,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "anthropic-claude-opus-41": {
    "model_name": "Claude Opus 4.1",
    "provider": "Anthropic",
    "category": "flagship",
    "avg_benchmark": 91.1,
    "blended_cost": 45.0,
    "perf_per_dollar": 2.0,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": 89.5,
    "humaneval": 92.7,
    "math_500": null,
    "bfcl": null,
    "speed_tps": null,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "anthropic-claude-sonnet-46": {
    "model_name": "Claude Sonnet 4.6",
    "provider": "Anthropic",
    "category": "mid-tier",
    "avg_benchmark": 73.8,
    "blended_cost": 9.0,
    "perf_per_dollar": 8.2,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": 58.3,
    "mmmlu": 89.3,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 55,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "anthropic-claude-sonnet-45": {
    "model_name": "Claude Sonnet 4.5",
    "provider": "Anthropic",
    "category": "mid-tier",
    "avg_benchmark": 82.0,
    "blended_cost": 9.0,
    "perf_per_dollar": 9.1,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": 82.0,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 69,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "anthropic-claude-haiku-45": {
    "model_name": "Claude Haiku 4.5",
    "provider": "Anthropic",
    "category": "budget",
    "avg_benchmark": 82.9,
    "blended_cost": 3.0,
    "perf_per_dollar": 27.6,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": 87.8,
    "math_500": 78.0,
    "bfcl": null,
    "speed_tps": null,
    "source": "Anthropic model card",
    "last_updated": "2026-02-15"
  },
  "anthropic-claude-haiku-35": {
    "model_name": "Claude Haiku 3.5",
    "provider": "Anthropic",
    "category": "budget",
    "avg_benchmark": 78.8,
    "blended_cost": 2.4,
    "perf_per_dollar": 32.8,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": 81.0,
    "humaneval": 84.1,
    "math_500": 71.2,
    "bfcl": null,
    "speed_tps": null,
    "source": "Kaggle LLM Benchmark Wars 2025-2026",
    "last_updated": "2026-03-02"
  },
  "openai-gpt-55": {
    "model_name": "GPT-5.5",
    "provider": "OpenAI",
    "category": "flagship",
    "avg_benchmark": 64.0,
    "blended_cost": 17.5,
    "perf_per_dollar": 3.7,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": null,
    "hles": 43.1,
    "arc_agi_2": 85.0,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": null,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "openai-gpt-54": {
    "model_name": "GPT-5.4",
    "provider": "OpenAI",
    "category": "flagship",
    "avg_benchmark": 78.7,
    "blended_cost": 8.75,
    "perf_per_dollar": 9.0,
    "gpqa_diamond": 88.0,
    "aime_2025": 95.5,
    "swe_bench": 75.0,
    "hles": 40.2,
    "arc_agi_2": 50.0,
    "mmmlu": 88.0,
    "humaneval": 94.5,
    "math_500": 92.0,
    "bfcl": 85.0,
    "speed_tps": 120,
    "source": "OpenAI model card, Artificial Analysis",
    "last_updated": "2026-05-01"
  },
  "openai-gpt-54-mini": {
    "model_name": "GPT-5.4 mini",
    "provider": "OpenAI",
    "category": "mid-tier",
    "avg_benchmark": 66.7,
    "blended_cost": 2.62,
    "perf_per_dollar": 25.4,
    "gpqa_diamond": 75.0,
    "aime_2025": 82.0,
    "swe_bench": 55.0,
    "hles": 25.0,
    "arc_agi_2": 30.0,
    "mmmlu": 80.0,
    "humaneval": 90.0,
    "math_500": 85.0,
    "bfcl": 78.0,
    "speed_tps": 180,
    "source": "OpenAI model card",
    "last_updated": "2026-05-01"
  },
  "openai-gpt-41": {
    "model_name": "GPT-4.1",
    "provider": "OpenAI",
    "category": "mid-tier",
    "avg_benchmark": 61.5,
    "blended_cost": 5.0,
    "perf_per_dollar": 12.3,
    "gpqa_diamond": 70.0,
    "aime_2025": 60.0,
    "swe_bench": 49.0,
    "hles": 20.0,
    "arc_agi_2": 24.0,
    "mmmlu": 85.0,
    "humaneval": 90.2,
    "math_500": 80.0,
    "bfcl": 75.0,
    "speed_tps": 150,
    "source": "Kaggle LLM Benchmark Wars 2025-2026",
    "last_updated": "2026-03-02"
  },
  "openai-gpt-41-mini": {
    "model_name": "GPT-4.1 mini",
    "provider": "OpenAI",
    "category": "budget",
    "avg_benchmark": 50.6,
    "blended_cost": 1.0,
    "perf_per_dollar": 50.6,
    "gpqa_diamond": 55.0,
    "aime_2025": 40.0,
    "swe_bench": 30.0,
    "hles": 12.0,
    "arc_agi_2": 15.0,
    "mmmlu": 78.0,
    "humaneval": 85.0,
    "math_500": 70.0,
    "bfcl": 70.0,
    "speed_tps": 200,
    "source": "OpenAI model card",
    "last_updated": "2026-04-15"
  },
  "openai-gpt-41-nano": {
    "model_name": "GPT-4.1 nano",
    "provider": "OpenAI",
    "category": "budget",
    "avg_benchmark": 39.6,
    "blended_cost": 0.25,
    "perf_per_dollar": 158.4,
    "gpqa_diamond": 40.0,
    "aime_2025": 25.0,
    "swe_bench": 15.0,
    "hles": 8.0,
    "arc_agi_2": 8.0,
    "mmmlu": 70.0,
    "humaneval": 75.0,
    "math_500": 55.0,
    "bfcl": 60.0,
    "speed_tps": 250,
    "source": "OpenAI model card",
    "last_updated": "2026-04-15"
  },
  "openai-o4-mini": {
    "model_name": "o4-mini",
    "provider": "OpenAI",
    "category": "reasoning",
    "avg_benchmark": 73.7,
    "blended_cost": 2.75,
    "perf_per_dollar": 26.8,
    "gpqa_diamond": 82.0,
    "aime_2025": 88.0,
    "swe_bench": 68.0,
    "hles": 35.0,
    "arc_agi_2": 40.0,
    "mmmlu": 86.0,
    "humaneval": 92.0,
    "math_500": 90.0,
    "bfcl": 82.0,
    "speed_tps": 100,
    "source": "OpenAI model card, Artificial Analysis",
    "last_updated": "2026-03-01"
  },
  "openai-o3-mini": {
    "model_name": "o3-mini",
    "provider": "OpenAI",
    "category": "reasoning",
    "avg_benchmark": 69.8,
    "blended_cost": 2.75,
    "perf_per_dollar": 25.4,
    "gpqa_diamond": 78.0,
    "aime_2025": 85.0,
    "swe_bench": 60.0,
    "hles": 30.0,
    "arc_agi_2": 35.0,
    "mmmlu": 84.0,
    "humaneval": 90.0,
    "math_500": 88.0,
    "bfcl": 78.0,
    "speed_tps": 214,
    "source": "OpenAI model card",
    "last_updated": "2026-01-15"
  },
  "google-gemini-31-pro": {
    "model_name": "Gemini 3.1 Pro",
    "provider": "Google",
    "category": "flagship",
    "avg_benchmark": 79.3,
    "blended_cost": 7.0,
    "perf_per_dollar": 11.3,
    "gpqa_diamond": 88.0,
    "aime_2025": 100.0,
    "swe_bench": 75.0,
    "hles": 45.8,
    "arc_agi_2": 45.0,
    "mmmlu": 91.8,
    "humaneval": 93.0,
    "math_500": 95.0,
    "bfcl": 80.0,
    "speed_tps": 128,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "google-gemini-35-flash": {
    "model_name": "Gemini 3.5 Flash",
    "provider": "Google",
    "category": "flagship",
    "avg_benchmark": 66.4,
    "blended_cost": 5.25,
    "perf_per_dollar": 12.6,
    "gpqa_diamond": 72.0,
    "aime_2025": 78.0,
    "swe_bench": 55.0,
    "hles": 28.0,
    "arc_agi_2": 35.0,
    "mmmlu": 85.0,
    "humaneval": 88.0,
    "math_500": 82.0,
    "bfcl": 75.0,
    "speed_tps": 400,
    "source": "Google model card",
    "last_updated": "2026-05-01"
  },
  "google-gemini-31-flash": {
    "model_name": "Gemini 3.1 Flash",
    "provider": "Google",
    "category": "mid-tier",
    "avg_benchmark": 60.3,
    "blended_cost": 1.4,
    "perf_per_dollar": 43.1,
    "gpqa_diamond": 65.0,
    "aime_2025": 70.0,
    "swe_bench": 45.0,
    "hles": 22.0,
    "arc_agi_2": 28.0,
    "mmmlu": 80.0,
    "humaneval": 85.0,
    "math_500": 78.0,
    "bfcl": 70.0,
    "speed_tps": 500,
    "source": "Google model card",
    "last_updated": "2026-04-15"
  },
  "google-gemini-31-flash-lite": {
    "model_name": "Gemini 3.1 Flash-Lite",
    "provider": "Google",
    "category": "budget",
    "avg_benchmark": 48.7,
    "blended_cost": 0.88,
    "perf_per_dollar": 55.7,
    "gpqa_diamond": 50.0,
    "aime_2025": 50.0,
    "swe_bench": 30.0,
    "hles": 15.0,
    "arc_agi_2": 18.0,
    "mmmlu": 72.0,
    "humaneval": 78.0,
    "math_500": 65.0,
    "bfcl": 60.0,
    "speed_tps": 600,
    "source": "Google model card",
    "last_updated": "2026-04-15"
  },
  "google-gemini-25-pro": {
    "model_name": "Gemini 2.5 Pro",
    "provider": "Google",
    "category": "mid-tier",
    "avg_benchmark": 62.8,
    "blended_cost": 5.62,
    "perf_per_dollar": 11.2,
    "gpqa_diamond": 68.0,
    "aime_2025": 75.0,
    "swe_bench": 50.0,
    "hles": 25.0,
    "arc_agi_2": 25.0,
    "mmmlu": 83.0,
    "humaneval": 87.0,
    "math_500": 80.0,
    "bfcl": 72.0,
    "speed_tps": 191,
    "source": "Kaggle LLM Benchmark Wars 2025-2026",
    "last_updated": "2026-03-02"
  },
  "google-gemini-25-flash": {
    "model_name": "Gemini 2.5 Flash",
    "provider": "Google",
    "category": "budget",
    "avg_benchmark": 53.7,
    "blended_cost": 0.19,
    "perf_per_dollar": 286.4,
    "gpqa_diamond": 55.0,
    "aime_2025": 60.0,
    "swe_bench": 38.0,
    "hles": 18.0,
    "arc_agi_2": 15.0,
    "mmmlu": 78.0,
    "humaneval": 82.0,
    "math_500": 72.0,
    "bfcl": 65.0,
    "speed_tps": 350,
    "source": "Kaggle LLM Benchmark Wars 2025-2026",
    "last_updated": "2026-03-02"
  },
  "google-gemini-25-flash-lite": {
    "model_name": "Gemini 2.5 Flash-Lite",
    "provider": "Google",
    "category": "budget",
    "avg_benchmark": 42.4,
    "blended_cost": 0.25,
    "perf_per_dollar": 169.6,
    "gpqa_diamond": 40.0,
    "aime_2025": 42.0,
    "swe_bench": 22.0,
    "hles": 10.0,
    "arc_agi_2": 10.0,
    "mmmlu": 70.0,
    "humaneval": 75.0,
    "math_500": 58.0,
    "bfcl": 55.0,
    "speed_tps": 450,
    "source": "Google model card",
    "last_updated": "2026-02-01"
  },
  "xai-grok-43": {
    "model_name": "Grok 4.3",
    "provider": "xAI",
    "category": "flagship",
    "avg_benchmark": 72.4,
    "blended_cost": 1.88,
    "perf_per_dollar": 38.6,
    "gpqa_diamond": 80.0,
    "aime_2025": 85.0,
    "swe_bench": 65.0,
    "hles": 38.0,
    "arc_agi_2": 42.0,
    "mmmlu": 86.0,
    "humaneval": 90.0,
    "math_500": 88.0,
    "bfcl": 78.0,
    "speed_tps": 130,
    "source": "xAI model card, Artificial Analysis",
    "last_updated": "2026-05-15"
  },
  "xai-grok-4": {
    "model_name": "Grok 4",
    "provider": "xAI",
    "category": "flagship",
    "avg_benchmark": 66.2,
    "blended_cost": 9.0,
    "perf_per_dollar": 7.4,
    "gpqa_diamond": 75.0,
    "aime_2025": 78.0,
    "swe_bench": 55.0,
    "hles": 30.0,
    "arc_agi_2": 35.0,
    "mmmlu": 82.0,
    "humaneval": 87.0,
    "math_500": 82.0,
    "bfcl": 72.0,
    "speed_tps": 110,
    "source": "Kaggle LLM Benchmark Wars 2025-2026",
    "last_updated": "2026-03-02"
  },
  "xai-grok-41-fast": {
    "model_name": "Grok 4.1 Fast",
    "provider": "xAI",
    "category": "budget",
    "avg_benchmark": 57.0,
    "blended_cost": 0.35,
    "perf_per_dollar": 162.9,
    "gpqa_diamond": 62.0,
    "aime_2025": 65.0,
    "swe_bench": 40.0,
    "hles": 20.0,
    "arc_agi_2": 25.0,
    "mmmlu": 78.0,
    "humaneval": 83.0,
    "math_500": 72.0,
    "bfcl": 68.0,
    "speed_tps": 300,
    "source": "xAI model card",
    "last_updated": "2026-04-01"
  },
  "xai-grok-420": {
    "model_name": "Grok 4.20",
    "provider": "xAI",
    "category": "mid-tier",
    "avg_benchmark": 68.7,
    "blended_cost": 4.0,
    "perf_per_dollar": 17.2,
    "gpqa_diamond": 78.0,
    "aime_2025": 80.0,
    "swe_bench": 58.0,
    "hles": 32.0,
    "arc_agi_2": 38.0,
    "mmmlu": 84.0,
    "humaneval": 88.0,
    "math_500": 85.0,
    "bfcl": 75.0,
    "speed_tps": 120,
    "source": "xAI model card",
    "last_updated": "2026-05-20"
  },
  "deepseek-v4-pro": {
    "model_name": "DeepSeek V4 Pro",
    "provider": "DeepSeek",
    "category": "mid-tier",
    "avg_benchmark": 64.1,
    "blended_cost": 0.65,
    "perf_per_dollar": 98.2,
    "gpqa_diamond": 72.0,
    "aime_2025": 75.0,
    "swe_bench": 50.0,
    "hles": 28.0,
    "arc_agi_2": 30.0,
    "mmmlu": 82.0,
    "humaneval": 88.0,
    "math_500": 80.0,
    "bfcl": 72.0,
    "speed_tps": 80,
    "source": "DeepSeek model card, Artificial Analysis",
    "last_updated": "2026-04-01"
  },
  "deepseek-v4-flash": {
    "model_name": "DeepSeek V4 Flash",
    "provider": "DeepSeek",
    "category": "budget",
    "avg_benchmark": 51.4,
    "blended_cost": 0.21,
    "perf_per_dollar": 244.8,
    "gpqa_diamond": 55.0,
    "aime_2025": 58.0,
    "swe_bench": 32.0,
    "hles": 15.0,
    "arc_agi_2": 18.0,
    "mmmlu": 75.0,
    "humaneval": 80.0,
    "math_500": 68.0,
    "bfcl": 62.0,
    "speed_tps": 200,
    "source": "DeepSeek model card",
    "last_updated": "2026-04-01"
  },
  "alibaba-qwen3-max": {
    "model_name": "Qwen3-Max",
    "provider": "Alibaba",
    "category": "flagship",
    "avg_benchmark": 61.9,
    "blended_cost": 3.0,
    "perf_per_dollar": 20.6,
    "gpqa_diamond": 70.0,
    "aime_2025": 72.0,
    "swe_bench": 48.0,
    "hles": 25.0,
    "arc_agi_2": 28.0,
    "mmmlu": 80.0,
    "humaneval": 86.0,
    "math_500": 78.0,
    "bfcl": 70.0,
    "speed_tps": 100,
    "source": "Alibaba model card",
    "last_updated": "2026-03-15"
  },
  "alibaba-qwq-plus": {
    "model_name": "QwQ-Plus",
    "provider": "Alibaba",
    "category": "reasoning",
    "avg_benchmark": 57.3,
    "blended_cost": 1.6,
    "perf_per_dollar": 35.8,
    "gpqa_diamond": 65.0,
    "aime_2025": 68.0,
    "swe_bench": 40.0,
    "hles": 20.0,
    "arc_agi_2": 22.0,
    "mmmlu": 78.0,
    "humaneval": 83.0,
    "math_500": 75.0,
    "bfcl": 65.0,
    "speed_tps": 90,
    "source": "Alibaba model card",
    "last_updated": "2026-02-01"
  },
  "meta-llama-4-scout": {
    "model_name": "Llama 4 Scout",
    "provider": "Meta",
    "category": "open-weights",
    "avg_benchmark": 47.6,
    "blended_cost": 0.23,
    "perf_per_dollar": 211.6,
    "gpqa_diamond": 50.0,
    "aime_2025": 45.0,
    "swe_bench": 28.0,
    "hles": 12.0,
    "arc_agi_2": 15.0,
    "mmmlu": 75.0,
    "humaneval": 80.0,
    "math_500": 65.0,
    "bfcl": 58.0,
    "speed_tps": 2600,
    "source": "Vellum LLM Leaderboard (Jun 2026), Meta model card",
    "last_updated": "2026-06-09"
  },
  "meta-llama-33-70b": {
    "model_name": "Llama 3.3 70B",
    "provider": "Meta",
    "category": "open-weights",
    "avg_benchmark": 43.6,
    "blended_cost": 0.69,
    "perf_per_dollar": 63.2,
    "gpqa_diamond": 45.0,
    "aime_2025": 38.0,
    "swe_bench": 22.0,
    "hles": 10.0,
    "arc_agi_2": 12.0,
    "mmmlu": 72.0,
    "humaneval": 78.0,
    "math_500": 60.0,
    "bfcl": 55.0,
    "speed_tps": 2500,
    "source": "Vellum LLM Leaderboard (Jun 2026), Kaggle dataset",
    "last_updated": "2026-06-09"
  },
  "meta-llama-31-8b": {
    "model_name": "Llama 3.1 8B",
    "provider": "Meta",
    "category": "open-weights",
    "avg_benchmark": 34.0,
    "blended_cost": 0.07,
    "perf_per_dollar": 523.1,
    "gpqa_diamond": 30.0,
    "aime_2025": 20.0,
    "swe_bench": 12.0,
    "hles": 5.0,
    "arc_agi_2": 6.0,
    "mmmlu": 65.0,
    "humaneval": 72.0,
    "math_500": 48.0,
    "bfcl": 48.0,
    "speed_tps": 1800,
    "source": "Vellum LLM Leaderboard (Jun 2026), Kaggle dataset",
    "last_updated": "2026-06-09"
  },
  "mistral-large-3": {
    "model_name": "Mistral Large 3",
    "provider": "Mistral",
    "category": "mid-tier",
    "avg_benchmark": 53.6,
    "blended_cost": 1.0,
    "perf_per_dollar": 53.6,
    "gpqa_diamond": 58.0,
    "aime_2025": 52.0,
    "swe_bench": 35.0,
    "hles": 18.0,
    "arc_agi_2": 20.0,
    "mmmlu": 78.0,
    "humaneval": 85.0,
    "math_500": 68.0,
    "bfcl": 68.0,
    "speed_tps": 120,
    "source": "Mistral model card, Artificial Analysis",
    "last_updated": "2026-03-01"
  },
  "mistral-codestral-2508": {
    "model_name": "Codestral 2508",
    "provider": "Mistral",
    "category": "coding",
    "avg_benchmark": 65.0,
    "blended_cost": 0.6,
    "perf_per_dollar": 108.3,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": 42.0,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": 88.0,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 200,
    "source": "Mistral model card",
    "last_updated": "2026-08-01"
  },
  "mistral-codestral": {
    "model_name": "Codestral",
    "provider": "Mistral",
    "category": "coding",
    "avg_benchmark": 61.5,
    "blended_cost": 0.6,
    "perf_per_dollar": 102.5,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": 38.0,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": 85.0,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 220,
    "source": "Mistral model card",
    "last_updated": "2026-01-01"
  },
  "zhipu-glm-52": {
    "model_name": "GLM-5.2",
    "provider": "Z.AI",
    "category": "flagship",
    "avg_benchmark": 61.2,
    "blended_cost": 2.9,
    "perf_per_dollar": 21.1,
    "gpqa_diamond": 68.0,
    "aime_2025": 70.0,
    "swe_bench": 45.0,
    "hles": 28.0,
    "arc_agi_2": 30.0,
    "mmmlu": 80.0,
    "humaneval": 84.0,
    "math_500": 76.0,
    "bfcl": 70.0,
    "speed_tps": 150,
    "source": "Z.AI model card",
    "last_updated": "2026-04-01"
  },
  "zhipu-glm-51": {
    "model_name": "GLM-5.1",
    "provider": "Z.AI",
    "category": "mid-tier",
    "avg_benchmark": 58.9,
    "blended_cost": 2.9,
    "perf_per_dollar": 20.3,
    "gpqa_diamond": 65.0,
    "aime_2025": 68.0,
    "swe_bench": 42.0,
    "hles": 25.0,
    "arc_agi_2": 28.0,
    "mmmlu": 78.0,
    "humaneval": 82.0,
    "math_500": 74.0,
    "bfcl": 68.0,
    "speed_tps": 160,
    "source": "Z.AI model card",
    "last_updated": "2026-03-01"
  },
  "zhipu-glm-5": {
    "model_name": "GLM-5",
    "provider": "Z.AI",
    "category": "mid-tier",
    "avg_benchmark": 56.1,
    "blended_cost": 2.1,
    "perf_per_dollar": 26.7,
    "gpqa_diamond": 62.0,
    "aime_2025": 65.0,
    "swe_bench": 38.0,
    "hles": 22.0,
    "arc_agi_2": 25.0,
    "mmmlu": 76.0,
    "humaneval": 80.0,
    "math_500": 72.0,
    "bfcl": 65.0,
    "speed_tps": 170,
    "source": "Z.AI model card",
    "last_updated": "2026-02-01"
  },
  "minimax-m3": {
    "model_name": "MiniMax-M3",
    "provider": "MiniMax",
    "category": "budget",
    "avg_benchmark": 55.0,
    "blended_cost": 0.75,
    "perf_per_dollar": 73.3,
    "gpqa_diamond": 60.0,
    "aime_2025": 55.0,
    "swe_bench": 45.0,
    "hles": 22.0,
    "arc_agi_2": 20.0,
    "mmmlu": 76.0,
    "humaneval": 82.0,
    "math_500": 70.0,
    "bfcl": 65.0,
    "speed_tps": 100,
    "source": "Vellum LLM Leaderboard (Jun 2026), MiniMax model card",
    "last_updated": "2026-06-09"
  },
  "moonshot-kimi-k27-code": {
    "model_name": "Kimi K2.7 Code",
    "provider": "Moonshot",
    "category": "coding",
    "avg_benchmark": 69.5,
    "blended_cost": 2.48,
    "perf_per_dollar": 28.1,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": 50.0,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": 89.0,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 80,
    "source": "Moonshot model card",
    "last_updated": "2026-05-01"
  },
  "moonshot-kimi-k26": {
    "model_name": "Kimi K2.6",
    "provider": "Moonshot",
    "category": "mid-tier",
    "avg_benchmark": 55.4,
    "blended_cost": 2.48,
    "perf_per_dollar": 22.4,
    "gpqa_diamond": 60.0,
    "aime_2025": 62.0,
    "swe_bench": 40.0,
    "hles": 20.0,
    "arc_agi_2": 22.0,
    "mmmlu": 76.0,
    "humaneval": 82.0,
    "math_500": 72.0,
    "bfcl": 65.0,
    "speed_tps": 90,
    "source": "Moonshot model card",
    "last_updated": "2026-04-01"
  },
  "cohere-command-a": {
    "model_name": "Command A",
    "provider": "Cohere",
    "category": "flagship",
    "avg_benchmark": 46.8,
    "blended_cost": 1.5,
    "perf_per_dollar": 31.2,
    "gpqa_diamond": 50.0,
    "aime_2025": 45.0,
    "swe_bench": 25.0,
    "hles": 15.0,
    "arc_agi_2": 12.0,
    "mmmlu": 74.0,
    "humaneval": 78.0,
    "math_500": 62.0,
    "bfcl": 60.0,
    "speed_tps": 150,
    "source": "Cohere model card",
    "last_updated": "2026-03-01"
  },
  "amazon-nova-premier": {
    "model_name": "Nova Premier",
    "provider": "Amazon",
    "category": "flagship",
    "avg_benchmark": 50.4,
    "blended_cost": 7.5,
    "perf_per_dollar": 6.7,
    "gpqa_diamond": 55.0,
    "aime_2025": 50.0,
    "swe_bench": 30.0,
    "hles": 18.0,
    "arc_agi_2": 15.0,
    "mmmlu": 76.0,
    "humaneval": 80.0,
    "math_500": 68.0,
    "bfcl": 62.0,
    "speed_tps": 80,
    "source": "AWS model card",
    "last_updated": "2026-02-01"
  },
  "amazon-nova-pro": {
    "model_name": "Nova Pro",
    "provider": "Amazon",
    "category": "mid-tier",
    "avg_benchmark": 43.3,
    "blended_cost": 2.0,
    "perf_per_dollar": 21.6,
    "gpqa_diamond": 45.0,
    "aime_2025": 40.0,
    "swe_bench": 22.0,
    "hles": 12.0,
    "arc_agi_2": 10.0,
    "mmmlu": 72.0,
    "humaneval": 76.0,
    "math_500": 58.0,
    "bfcl": 55.0,
    "speed_tps": 120,
    "source": "AWS model card",
    "last_updated": "2026-02-01"
  },
  "amazon-nova-lite": {
    "model_name": "Nova Lite",
    "provider": "Amazon",
    "category": "budget",
    "avg_benchmark": 36.9,
    "blended_cost": 0.15,
    "perf_per_dollar": 246.0,
    "gpqa_diamond": 35.0,
    "aime_2025": 30.0,
    "swe_bench": 15.0,
    "hles": 8.0,
    "arc_agi_2": 6.0,
    "mmmlu": 68.0,
    "humaneval": 72.0,
    "math_500": 50.0,
    "bfcl": 48.0,
    "speed_tps": 200,
    "source": "AWS model card",
    "last_updated": "2026-02-01"
  },
  "amazon-nova-micro": {
    "model_name": "Nova Micro",
    "provider": "Amazon",
    "category": "budget",
    "avg_benchmark": 30.9,
    "blended_cost": 0.09,
    "perf_per_dollar": 353.1,
    "gpqa_diamond": 25.0,
    "aime_2025": 20.0,
    "swe_bench": 10.0,
    "hles": 5.0,
    "arc_agi_2": 4.0,
    "mmmlu": 62.0,
    "humaneval": 68.0,
    "math_500": 42.0,
    "bfcl": 42.0,
    "speed_tps": 300,
    "source": "AWS model card",
    "last_updated": "2026-02-01"
  },
  "nvidia-llama-nemotron-ultra-253b": {
    "model_name": "Llama Nemotron Ultra 253B",
    "provider": "NVIDIA",
    "category": "open-weights",
    "avg_benchmark": 48.2,
    "blended_cost": 2.1,
    "perf_per_dollar": 23.0,
    "gpqa_diamond": 52.0,
    "aime_2025": 45.0,
    "swe_bench": 30.0,
    "hles": 15.0,
    "arc_agi_2": 14.0,
    "mmmlu": 75.0,
    "humaneval": 80.0,
    "math_500": 65.0,
    "bfcl": 58.0,
    "speed_tps": 60,
    "source": "NVIDIA model card",
    "last_updated": "2026-03-01"
  },
  "perplexity-sonar": {
    "model_name": "Sonar",
    "provider": "Perplexity",
    "category": "mid-tier",
    "avg_benchmark": null,
    "blended_cost": 1.0,
    "perf_per_dollar": null,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 150,
    "source": "Perplexity model card",
    "last_updated": "2026-03-01"
  },
  "groq-llama-31-8b-instant": {
    "model_name": "Llama 3.1 8B Instant",
    "provider": "Groq",
    "category": "fast",
    "avg_benchmark": null,
    "blended_cost": 0.53,
    "perf_per_dollar": null,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 1800,
    "source": "Groq model card",
    "last_updated": "2026-06-01"
  },
  "groq-llama-33-70b-versatile": {
    "model_name": "Llama 3.3 70B Versatile",
    "provider": "Groq",
    "category": "fast",
    "avg_benchmark": null,
    "blended_cost": 0.79,
    "perf_per_dollar": null,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 2500,
    "source": "Groq model card",
    "last_updated": "2026-06-01"
  },
  "groq-llama-4-scout": {
    "model_name": "Llama 4 Scout",
    "provider": "Groq",
    "category": "fast",
    "avg_benchmark": null,
    "blended_cost": 0.56,
    "perf_per_dollar": null,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 2600,
    "source": "Groq model card",
    "last_updated": "2026-06-01"
  },
  "ibm-granite-4-h-large": {
    "model_name": "Granite 4 H Large",
    "provider": "IBM",
    "category": "mid-tier",
    "avg_benchmark": 40.8,
    "blended_cost": 0.75,
    "perf_per_dollar": 54.4,
    "gpqa_diamond": 42.0,
    "aime_2025": 35.0,
    "swe_bench": 20.0,
    "hles": 10.0,
    "arc_agi_2": 8.0,
    "mmmlu": 70.0,
    "humaneval": 75.0,
    "math_500": 55.0,
    "bfcl": 52.0,
    "speed_tps": 120,
    "source": "IBM model card",
    "last_updated": "2026-03-01"
  },
  "ibm-granite-4-h-medium": {
    "model_name": "Granite 4 H Medium",
    "provider": "IBM",
    "category": "mid-tier",
    "avg_benchmark": 36.4,
    "blended_cost": 0.38,
    "perf_per_dollar": 97.1,
    "gpqa_diamond": 35.0,
    "aime_2025": 28.0,
    "swe_bench": 15.0,
    "hles": 7.0,
    "arc_agi_2": 5.0,
    "mmmlu": 68.0,
    "humaneval": 72.0,
    "math_500": 50.0,
    "bfcl": 48.0,
    "speed_tps": 200,
    "source": "IBM model card",
    "last_updated": "2026-03-01"
  },
  "together-qwen37-max": {
    "model_name": "Qwen3.7-Max",
    "provider": "Together",
    "category": "mid-tier",
    "avg_benchmark": 60.4,
    "blended_cost": 2.5,
    "perf_per_dollar": 24.2,
    "gpqa_diamond": 68.0,
    "aime_2025": 70.0,
    "swe_bench": 45.0,
    "hles": 25.0,
    "arc_agi_2": 28.0,
    "mmmlu": 80.0,
    "humaneval": 84.0,
    "math_500": 76.0,
    "bfcl": 68.0,
    "speed_tps": 100,
    "source": "Together AI, Qwen model card",
    "last_updated": "2026-04-01"
  },
  "fireworks-gpt-oss-20b": {
    "model_name": "GPT OSS 20B",
    "provider": "Fireworks",
    "category": "budget",
    "avg_benchmark": 98.7,
    "blended_cost": 0.18,
    "perf_per_dollar": 533.5,
    "gpqa_diamond": null,
    "aime_2025": 98.7,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 564,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  },
  "fireworks-gpt-oss-120b": {
    "model_name": "GPT OSS 120B",
    "provider": "Fireworks",
    "category": "budget",
    "avg_benchmark": null,
    "blended_cost": 0.38,
    "perf_per_dollar": null,
    "gpqa_diamond": null,
    "aime_2025": null,
    "swe_bench": null,
    "hles": null,
    "arc_agi_2": null,
    "mmmlu": null,
    "humaneval": null,
    "math_500": null,
    "bfcl": null,
    "speed_tps": 260,
    "source": "Vellum LLM Leaderboard",
    "last_updated": "2026-06-25"
  }
}