{
  "name": "Frontier AI Autonomous Capability Benchmarks and Evaluation Parameters",
  "description": "Comparison of technical specifications, task volumes, maximum duration baselines, human calibration methods, execution harnesses, and target capability metrics across frontier AI evaluation suites compiled from primary technical reports and papers.",
  "page": "https://casrai.org/guides/metr-time-horizon",
  "publisher": "CASRAI",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "dateModified": "2026-11-16T00:46:57Z",
  "temporalCoverage": "2023/2026",
  "spatialCoverage": "Global",
  "columns": [
    {
      "name": "Benchmark Suite",
      "unit": "",
      "description": "Name and formal version of the agent capability benchmark"
    },
    {
      "name": "Active Tasks",
      "unit": "tasks",
      "description": "Number of validated evaluation tasks included in the active test suite"
    },
    {
      "name": "Max Human Duration",
      "unit": "hours",
      "description": "Upper bound of calibrated human expert completion time across benchmark tasks"
    },
    {
      "name": "Calibration Method",
      "unit": "",
      "description": "Methodology utilized to establish human baseline task completion durations"
    },
    {
      "name": "Evaluation Harness",
      "unit": "",
      "description": "Execution framework or container management harness used to run agents"
    },
    {
      "name": "Primary Metric",
      "unit": "",
      "description": "Core quantitative capability metric extracted from evaluation results"
    }
  ],
  "rows": [
    {
      "Benchmark Suite": "METR Time Horizon 1.1",
      "Active Tasks": 228,
      "Max Human Duration": 8,
      "Calibration Method": "Geometric mean of professional attempts and QA estimates",
      "Evaluation Harness": "Inspect",
      "Primary Metric": "50% task-completion time horizon"
    },
    {
      "Benchmark Suite": "METR Time Horizon 1.0",
      "Active Tasks": 170,
      "Max Human Duration": 8,
      "Calibration Method": "Geometric mean of contractor completion times",
      "Evaluation Harness": "Vivaria",
      "Primary Metric": "50% task-completion time horizon"
    },
    {
      "Benchmark Suite": "METR RE-Bench v1",
      "Active Tasks": 7,
      "Max Human Duration": 8,
      "Calibration Method": "Baseline trials across 61 human ML experts",
      "Evaluation Harness": "Vivaria and custom scaffolds",
      "Primary Metric": "R&D relative capability score"
    },
    {
      "Benchmark Suite": "SWE-bench Verified",
      "Active Tasks": 500,
      "Max Human Duration": 4,
      "Calibration Method": "Human validation and estimated resolution times",
      "Evaluation Harness": "SWE-bench Docker environment",
      "Primary Metric": "Resolved issue percentage"
    },
    {
      "Benchmark Suite": "GAIA Benchmark",
      "Active Tasks": 466,
      "Max Human Duration": 0.5,
      "Calibration Method": "Human annotator timing and validation pass rate",
      "Evaluation Harness": "Standard tool and browser scaffolding",
      "Primary Metric": "Multi-modal task completion accuracy"
    }
  ],
  "sources": [
    {
      "title": "METR: Time Horizon 1.1 Release",
      "url": "https://metr.org/blog/2026-01-29-time-horizon-1-1/",
      "accessed": "2026-11-15"
    },
    {
      "title": "Measuring AI Ability to Complete Long Software Tasks (arXiv:2503.14499)",
      "url": "https://arxiv.org/abs/2503.14499",
      "accessed": "2026-11-15"
    },
    {
      "title": "RE-Bench: Evaluating Frontier AI R&D Capabilities (arXiv:2411.15114)",
      "url": "https://arxiv.org/abs/2411.15114",
      "accessed": "2026-11-15"
    },
    {
      "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues? (arXiv:2310.06770)",
      "url": "https://arxiv.org/abs/2310.06770",
      "accessed": "2026-11-15"
    },
    {
      "title": "GAIA: A Benchmark for General AI Assistants (arXiv:2311.12983)",
      "url": "https://arxiv.org/abs/2311.12983",
      "accessed": "2026-11-15"
    }
  ]
}