{
  "name": "Reported versus AutoAttack robust accuracy of selected CIFAR-10 Linf defenses (eps = 8/255)",
  "description": "Selected rows from the AutoAttack repository's CIFAR-10 Linf results table (eps = 8/255): clean accuracy, the robust accuracy reported by each defense's authors, and the robust accuracy under the standard AutoAttack evaluation on the full test set. The last column is the difference, computed as reported minus AutoAttack; it shows how far claimed robustness was overestimated.",
  "page": "https://casrai.org/guides/autoattack",
  "publisher": "CASRAI",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "dateModified": "2026-11-17T15:33:04Z",
  "temporalCoverage": "2018/2021",
  "spatialCoverage": null,
  "columns": [
    {
      "name": "Defense (paper)",
      "unit": "",
      "description": ""
    },
    {
      "name": "Architecture",
      "unit": "",
      "description": ""
    },
    {
      "name": "Clean accuracy",
      "unit": "%",
      "description": ""
    },
    {
      "name": "Reported robust accuracy",
      "unit": "%",
      "description": ""
    },
    {
      "name": "AutoAttack robust accuracy",
      "unit": "%",
      "description": ""
    },
    {
      "name": "Overestimate (reported minus AutoAttack)",
      "unit": "percentage points",
      "description": ""
    }
  ],
  "rows": [
    {
      "Defense (paper)": "Gowal et al. 2020 (extra data)",
      "Architecture": "WRN-70-16",
      "Clean accuracy": 91.1,
      "Reported robust accuracy": 65.87,
      "AutoAttack robust accuracy": 65.88,
      "Overestimate (reported minus AutoAttack)": -0.01
    },
    {
      "Defense (paper)": "Carmon et al. 2019 (extra data)",
      "Architecture": "WRN-28-10",
      "Clean accuracy": 89.69,
      "Reported robust accuracy": 62.5,
      "AutoAttack robust accuracy": 59.53,
      "Overestimate (reported minus AutoAttack)": 2.97
    },
    {
      "Defense (paper)": "Wang et al. 2020 (extra data)",
      "Architecture": "WRN-28-10",
      "Clean accuracy": 87.5,
      "Reported robust accuracy": 65.04,
      "AutoAttack robust accuracy": 56.29,
      "Overestimate (reported minus AutoAttack)": 8.75
    },
    {
      "Defense (paper)": "Hendrycks et al. 2019 (extra data)",
      "Architecture": "WRN-28-10",
      "Clean accuracy": 87.11,
      "Reported robust accuracy": 57.4,
      "AutoAttack robust accuracy": 54.92,
      "Overestimate (reported minus AutoAttack)": 2.48
    },
    {
      "Defense (paper)": "Rice et al. 2020",
      "Architecture": "WRN-34-20",
      "Clean accuracy": 85.34,
      "Reported robust accuracy": 58,
      "AutoAttack robust accuracy": 53.42,
      "Overestimate (reported minus AutoAttack)": 4.58
    },
    {
      "Defense (paper)": "Engstrom et al. 2019",
      "Architecture": "RN-50",
      "Clean accuracy": 87.03,
      "Reported robust accuracy": 53.29,
      "AutoAttack robust accuracy": 49.25,
      "Overestimate (reported minus AutoAttack)": 4.04
    },
    {
      "Defense (paper)": "Madry et al. 2018",
      "Architecture": "WRN-34-10",
      "Clean accuracy": 87.14,
      "Reported robust accuracy": 47.04,
      "AutoAttack robust accuracy": 44.04,
      "Overestimate (reported minus AutoAttack)": 3
    },
    {
      "Defense (paper)": "Pang et al. 2020a",
      "Architecture": "RN-32",
      "Clean accuracy": 80.89,
      "Reported robust accuracy": 55,
      "AutoAttack robust accuracy": 43.48,
      "Overestimate (reported minus AutoAttack)": 11.52
    },
    {
      "Defense (paper)": "Zhang and Wang 2019",
      "Architecture": "WRN-28-10",
      "Clean accuracy": 89.98,
      "Reported robust accuracy": 60.6,
      "AutoAttack robust accuracy": 36.64,
      "Overestimate (reported minus AutoAttack)": 23.96
    },
    {
      "Defense (paper)": "Zhang and Xu 2020",
      "Architecture": "WRN-28-10",
      "Clean accuracy": 90.25,
      "Reported robust accuracy": 68.7,
      "AutoAttack robust accuracy": 36.45,
      "Overestimate (reported minus AutoAttack)": 32.25
    },
    {
      "Defense (paper)": "Wang and Zhang 2019",
      "Architecture": "WRN-28-10",
      "Clean accuracy": 92.8,
      "Reported robust accuracy": 58.6,
      "AutoAttack robust accuracy": 29.35,
      "Overestimate (reported minus AutoAttack)": 29.25
    },
    {
      "Defense (paper)": "Jin and Rinard 2020",
      "Architecture": "RN-18",
      "Clean accuracy": 90.84,
      "Reported robust accuracy": 71.22,
      "AutoAttack robust accuracy": 1.35,
      "Overestimate (reported minus AutoAttack)": 69.87
    },
    {
      "Defense (paper)": "Chan et al. 2020",
      "Architecture": "WRN-34-10",
      "Clean accuracy": 93.79,
      "Reported robust accuracy": 15.5,
      "AutoAttack robust accuracy": 0.26,
      "Overestimate (reported minus AutoAttack)": 15.24
    }
  ],
  "sources": [
    {
      "title": "fra31/auto-attack README: Adversarial Defenses Evaluation, CIFAR-10 Linf",
      "url": "https://github.com/fra31/auto-attack",
      "accessed": "2026-11-17"
    },
    {
      "title": "Croce and Hein (2020), arXiv 2003.01690",
      "url": "https://arxiv.org/abs/2003.01690",
      "accessed": "2026-11-17"
    }
  ]
}