{
  "name": "Frontier AI Scientific and Reasoning Benchmarks Comparison",
  "description": "Comparative specifications, validation cohorts, question counts, and human baseline accuracies across prominent frontier AI scientific reasoning and capability benchmarks, compiled from official technical reports, repositories, and peer-reviewed benchmark papers.",
  "page": "https://casrai.org/guides/gpqa-benchmark",
  "publisher": "CASRAI",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "dateModified": "2026-11-19T12:51:27Z",
  "temporalCoverage": "2021-2026",
  "spatialCoverage": "Global",
  "columns": [
    {
      "name": "Benchmark Name",
      "unit": "",
      "description": "Name and official subset of the evaluation benchmark"
    },
    {
      "name": "Domain Scope",
      "unit": "",
      "description": "Scientific disciplines and topic coverage included in the benchmark"
    },
    {
      "name": "Item Count",
      "unit": "questions",
      "description": "Total number of validated questions in the primary evaluation split"
    },
    {
      "name": "Expert Baseline Acc",
      "unit": "%",
      "description": "Validation accuracy achieved by credentialed subject-matter PhD experts"
    },
    {
      "name": "Non-Expert Web Acc",
      "unit": "%",
      "description": "Validation accuracy scored by skilled non-experts with unrestricted internet search"
    },
    {
      "name": "Random Guess Baseline",
      "unit": "%",
      "description": "Mathematical chance accuracy from random guessing"
    },
    {
      "name": "Release Year",
      "unit": "",
      "description": "Calendar year of initial benchmark release"
    }
  ],
  "rows": [
    {
      "Benchmark Name": "GPQA Diamond",
      "Domain Scope": "Biology, Chemistry, Physics",
      "Item Count": 198,
      "Expert Baseline Acc": 81.3,
      "Non-Expert Web Acc": 22.1,
      "Random Guess Baseline": 25,
      "Release Year": "2023"
    },
    {
      "Benchmark Name": "GPQA Main",
      "Domain Scope": "Biology, Chemistry, Physics",
      "Item Count": 448,
      "Expert Baseline Acc": 71.9,
      "Non-Expert Web Acc": 30.4,
      "Random Guess Baseline": 25,
      "Release Year": "2023"
    },
    {
      "Benchmark Name": "GPQA Extended",
      "Domain Scope": "Biology, Chemistry, Physics",
      "Item Count": 546,
      "Expert Baseline Acc": 64.8,
      "Non-Expert Web Acc": 34.1,
      "Random Guess Baseline": 25,
      "Release Year": "2023"
    },
    {
      "Benchmark Name": "FrontierMath",
      "Domain Scope": "Research-Level Mathematics",
      "Item Count": 350,
      "Expert Baseline Acc": 90,
      "Non-Expert Web Acc": 2,
      "Random Guess Baseline": 0,
      "Release Year": "2024"
    },
    {
      "Benchmark Name": "Humanity's Last Exam",
      "Domain Scope": "Multidisciplinary STEM and Humanities",
      "Item Count": 3000,
      "Expert Baseline Acc": 85,
      "Non-Expert Web Acc": 10,
      "Random Guess Baseline": 25,
      "Release Year": "2025"
    },
    {
      "Benchmark Name": "MMLU",
      "Domain Scope": "Undergraduate Multitask Knowledge (57 subjects)",
      "Item Count": 15908,
      "Expert Baseline Acc": 89.8,
      "Non-Expert Web Acc": 34,
      "Random Guess Baseline": 25,
      "Release Year": "2021"
    }
  ],
  "sources": [
    {
      "title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark",
      "url": "https://arxiv.org/abs/2311.12022",
      "accessed": "2026-11-19"
    },
    {
      "title": "OpenAI simple-evals Benchmark Results",
      "url": "https://github.com/openai/simple-evals",
      "accessed": "2026-11-19"
    },
    {
      "title": "Idavidrein/gpqa Dataset Card",
      "url": "https://huggingface.co/datasets/Idavidrein/gpqa",
      "accessed": "2026-11-19"
    }
  ]
}