{
  "arena_text": {
    "name": "Arena Text",
    "description": "LMArena Text Arena — human preference for general chat, writing, creativity. ELO-based, continuously updated, stable across model generations.",
    "maxScore": null,
    "type": "elo",
    "higherIsBetter": true,
    "source": "https://lmarena.ai/leaderboard",
    "recommended": true,
    "stable": true
  },
  "arena_code": {
    "name": "Arena Code (WebDev)",
    "description": "LMArena WebDev Arena — human preference for coding and web development. ELO-based, stable across generations.",
    "maxScore": null,
    "type": "elo",
    "higherIsBetter": true,
    "source": "https://lmarena.ai/leaderboard/code",
    "recommended": true,
    "stable": true
  },
  "aa_intelligence": {
    "name": "AA Intelligence Index",
    "description": "Artificial Analysis composite intelligence score. Aggregates multiple benchmarks into one normalized 0–100 score. Maintained continuously with consistent methodology.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://artificialanalysis.ai/",
    "recommended": true,
    "stable": true
  },
  "aider_polyglot": {
    "name": "Aider Polyglot",
    "description": "Real-world coding benchmark across 6 languages. Tests actual code generation quality.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://aider.chat/docs/leaderboards/",
    "recommended": true,
    "stable": false
  },
  "swe_bench": {
    "name": "SWE-bench Verified",
    "description": "Real GitHub issues — tests ability to fix actual bugs in production codebases.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://www.swebench.com/",
    "recommended": true,
    "stable": false
  },
  "livebench": {
    "name": "LiveBench",
    "description": "Contamination-resistant benchmark updated monthly with new questions.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://livebench.ai/",
    "recommended": true,
    "stable": false
  },
  "bigcodebench": {
    "name": "BigCodeBench",
    "description": "Challenging code benchmark with 36% gap between human (97%) and best models (61%).",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://bigcode-bench.github.io/",
    "recommended": true,
    "stable": false
  },
  "gpqa_diamond": {
    "name": "GPQA Diamond",
    "description": "Graduate-level science QA. Still challenging but approaching saturation at 90%.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://github.com/idavidrein/gpqa",
    "recommended": true,
    "stable": false
  },
  "mmlu": {
    "name": "MMLU",
    "description": "Massive Multitask Language Understanding. SATURATED — top models cluster within 2–4%.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://huggingface.co/spaces/open-llm-leaderboard",
    "recommended": false,
    "stable": false,
    "warning": "Saturated benchmark — no longer differentiates frontier models"
  },
  "humaneval": {
    "name": "HumanEval",
    "description": "Code generation benchmark. SATURATED — SOTA at 99.4%.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://github.com/openai/human-eval",
    "recommended": false,
    "stable": false,
    "warning": "Saturated benchmark"
  },
  "aa_coding_index": {
    "name": "AA Coding Index",
    "description": "Artificial Analysis composite coding score. Aggregates coding-focused evaluations (LiveCodeBench, SciCode, Terminal-Bench, SWE-Bench-family) into one score.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://artificialanalysis.ai/",
    "recommended": true,
    "stable": true
  },
  "aa_math_index": {
    "name": "AA Math Index",
    "description": "Artificial Analysis composite math score. Aggregates math evaluations including AIME and MATH-500.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://artificialanalysis.ai/",
    "recommended": true,
    "stable": true
  },
  "hle": {
    "name": "Humanity's Last Exam",
    "description": "2,500 expert-authored questions spanning dozens of fields, designed to resist guessing and memorization. Frontier models still score below 50%.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://lastexam.ai/",
    "recommended": true,
    "stable": true
  },
  "scicode": {
    "name": "SciCode",
    "description": "Scientific coding benchmark. Tests ability to implement scientific algorithms and problems across physics, math, chemistry, biology.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://scicode-bench.github.io/",
    "recommended": true,
    "stable": true
  },
  "aa_lcr": {
    "name": "AA-LCR (Long-Context Reasoning)",
    "description": "Artificial Analysis long-context reasoning benchmark. Tests multi-step reasoning over long inputs — closer to agentic real-world tasks than single-turn QA.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://artificialanalysis.ai/",
    "recommended": true,
    "stable": true
  },
  "tau2_bench": {
    "name": "τ²-Bench (Telecom)",
    "description": "Dual-control conversational benchmark simulating telecom support. Both agent and simulated user must coordinate actions to resolve issues — tests agentic behavior.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://artificialanalysis.ai/evaluations/tau2-bench-telecom",
    "recommended": true,
    "stable": true
  },
  "terminalbench_hard": {
    "name": "Terminal-Bench Hard",
    "description": "Agentic terminal benchmark. Tests multi-step command-line task completion — software engineering, sysadmin, data processing. Frontier models score <65%.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://www.tbench.ai/",
    "recommended": true,
    "stable": true
  },
  "ifbench": {
    "name": "IFBench (Instruction Following)",
    "description": "Measures how precisely models follow complex multi-constraint instructions. Direct proxy for reliability in real workflows.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://artificialanalysis.ai/",
    "recommended": true,
    "stable": true
  },
  "mmlu_pro": {
    "name": "MMLU-Pro",
    "description": "Harder, contamination-resistant successor to MMLU. 10 answer options, reasoning-heavy questions. Not yet saturated.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro",
    "recommended": true,
    "stable": true
  },
  "livecodebench": {
    "name": "LiveCodeBench",
    "description": "Contamination-resistant coding benchmark drawn from ongoing programming contests. Problems added monthly after model training cutoffs.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://livecodebench.github.io/",
    "recommended": true,
    "stable": true
  },
  "aime_25": {
    "name": "AIME 2025",
    "description": "American Invitational Mathematics Examination 2025 problems. Fresh problems chosen to avoid contamination with pretraining data.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://artificialanalysis.ai/",
    "recommended": true,
    "stable": true
  },
  "gdpval_aa": {
    "name": "GDPval-AA",
    "description": "Artificial Analysis' ELO-based evaluation of OpenAI's GDPval benchmark: 1,320 real-world occupational tasks across 44 professions, graded by domain experts. The highest-weight benchmark in AA Intelligence Index v4.0 (16.7%).",
    "maxScore": null,
    "type": "elo",
    "higherIsBetter": true,
    "source": "https://artificialanalysis.ai/evaluations/gdpval-aa",
    "recommended": true,
    "stable": true
  },
  "aa_omniscience": {
    "name": "AA-Omniscience",
    "description": "Knowledge + hallucination benchmark. 6,000 questions across 42 topics; rewards correct answers, penalizes hallucinations, no penalty for abstaining. Signed index from -100 to +100.",
    "maxScore": 100,
    "type": "index",
    "higherIsBetter": true,
    "source": "https://artificialanalysis.ai/evaluations/aa-omniscience",
    "recommended": true,
    "stable": true
  },
  "critpt": {
    "name": "CritPt",
    "description": "Critical-reasoning benchmark introduced in AA Intelligence Index v4.0. Frontier-discriminating — top models still score well under 20%.",
    "maxScore": 100,
    "type": "percentage",
    "higherIsBetter": true,
    "source": "https://artificialanalysis.ai/",
    "recommended": true,
    "stable": true
  }
}