{
  "name": "Published benchmarks used in interview-coding practice: saturation counts and kappa agreement bands",
  "description": "Compiled from four published sources read for this page: McHugh 2012 (Cohen's kappa interpretation bands), Hennink et al. 2017 and Guest et al. 2006 (interview counts in saturation studies) and McDonald et al. 2019 (share of qualitative papers reporting inter-rater reliability). Each row is one figure with its source; ranges are given as text.",
  "page": "https://casrai.org/guides/interview-coding-examples",
  "publisher": "CASRAI",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "dateModified": "2026-11-17T07:01:00Z",
  "temporalCoverage": "2006/2019",
  "spatialCoverage": null,
  "columns": [
    {
      "name": "Measure",
      "unit": "",
      "description": ""
    },
    {
      "name": "Source",
      "unit": "",
      "description": ""
    },
    {
      "name": "Value",
      "unit": "",
      "description": "number or range as published"
    },
    {
      "name": "Unit",
      "unit": "",
      "description": ""
    },
    {
      "name": "Year",
      "unit": "",
      "description": ""
    }
  ],
  "rows": [
    {
      "Measure": "Kappa band: none",
      "Source": "McHugh 2012",
      "Value": "0 to 0.20",
      "Unit": "Cohen's kappa",
      "Year": 2012
    },
    {
      "Measure": "Kappa band: minimal",
      "Source": "McHugh 2012",
      "Value": "0.21 to 0.39",
      "Unit": "Cohen's kappa",
      "Year": 2012
    },
    {
      "Measure": "Kappa band: weak",
      "Source": "McHugh 2012",
      "Value": "0.40 to 0.59",
      "Unit": "Cohen's kappa",
      "Year": 2012
    },
    {
      "Measure": "Kappa band: moderate",
      "Source": "McHugh 2012",
      "Value": "0.60 to 0.79",
      "Unit": "Cohen's kappa",
      "Year": 2012
    },
    {
      "Measure": "Kappa band: strong",
      "Source": "McHugh 2012",
      "Value": "0.80 to 0.90",
      "Unit": "Cohen's kappa",
      "Year": 2012
    },
    {
      "Measure": "Kappa band: almost perfect",
      "Source": "McHugh 2012",
      "Value": "above 0.90",
      "Unit": "Cohen's kappa",
      "Year": 2012
    },
    {
      "Measure": "Interviews in study (code vs meaning saturation)",
      "Source": "Hennink et al. 2017",
      "Value": 25,
      "Unit": "interviews",
      "Year": 2017
    },
    {
      "Measure": "Interviews to code saturation",
      "Source": "Hennink et al. 2017",
      "Value": 9,
      "Unit": "interviews",
      "Year": 2017
    },
    {
      "Measure": "Interviews to meaning saturation",
      "Source": "Hennink et al. 2017",
      "Value": "16 to 24",
      "Unit": "interviews",
      "Year": 2017
    },
    {
      "Measure": "Interviews in saturation experiment",
      "Source": "Guest et al. 2006",
      "Value": 60,
      "Unit": "interviews",
      "Year": 2006
    },
    {
      "Measure": "Interviews per analysis set",
      "Source": "Guest et al. 2006",
      "Value": 6,
      "Unit": "interviews",
      "Year": 2006
    },
    {
      "Measure": "Qualitative papers reporting IRR (CSCW/HCI 2016-2018)",
      "Source": "McDonald et al. 2019",
      "Value": "about 1 in 9",
      "Unit": "share of papers",
      "Year": 2019
    }
  ],
  "sources": [
    {
      "title": "McHugh, Interrater reliability: the kappa statistic (Europe PMC full text)",
      "url": "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC3900052/fullTextXML",
      "accessed": "2026-11-17"
    },
    {
      "title": "Hennink, Kaiser, Marconi, Code saturation versus meaning saturation",
      "url": "https://doi.org/10.1177/1049732316665344",
      "accessed": "2026-11-17"
    },
    {
      "title": "Guest, Bunce, Johnson, How many interviews are enough?",
      "url": "https://doi.org/10.1177/1525822X05279903",
      "accessed": "2026-11-17"
    },
    {
      "title": "McDonald, Schoenebeck, Forte, Reliability and inter-rater reliability in qualitative research",
      "url": "https://doi.org/10.1145/3359174",
      "accessed": "2026-11-17"
    }
  ]
}