{
 "bundle_of": "C17",
 "schema_version": "0.43",
 "paper": {
  "id": "entrylib:2026.000008",
  "version": 2,
  "title": "Evaluating forecasts and classifiers: which measures, scoring rules and tests to use",
  "licence": "CC-BY-4.0",
  "status": "corrected",
  "date": "2026-11-14",
  "status_notice": {
   "date": "2026-11-14",
   "reason_code": "correction",
   "statement": "Version 1 labelled the review's findings and recommendations extension, with an extends relation to the very works cited as their evidence, which said they extend those works; they state what those works establish, so version 2 labels them synthesis (claimset v0.43) and drops those relations, keeping extension only for C16, the one rule that goes beyond its sources, which extends Gneiting and Raftery (2007); and, at the guarantor's request, it rewords C3 (RMSSE as an example of the scaling), C4 (MAE and MASE as the main measure only when the mean and median forecasts coincide), C8 (absolute-error measures avoided on intermittent series), C13 (sharpness compared only among calibrated distributions and intervals) and C24 (choose a measure for what the methods predict, even if precedent used another).",
   "issued_by": "Ivan Svetunkov (ORCID 0000-0001-7826-0281)"
  }
 },
 "extensions": {
  "review": "0.1"
 },
 "claims": [
  {
   "id": "C17",
   "statement": "[[D22:AUC]] should not be the measure of forecast class probabilities.",
   "about": {
    "subject": "D22",
    "relation": "should not be the measure of",
    "object": "class probabilities",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "argument",
     "warrant": "W17"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H76",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "depends_on": [
    "C42"
   ],
   "scope": {
    "data_type": "forecast class probabilities, ranked by a classifier's scores"
   },
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L1",
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "moderate",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. L1: the finding it rests on was checked only in the abstract of its work; a full text could qualify it."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C42",
   "statement": "The [[D22:AUC]] is equivalent to averaging the misclassification loss over a distribution of misclassification costs that depends on the classifier.",
   "about": {
    "subject": "D22",
    "relation": "averages the misclassification loss over",
    "object": "a cost distribution that depends on the classifier",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B14"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H49",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "classifiers that score cases"
   },
   "ext": {
    "review": {
     "kind": "summary"
    }
   },
   "limitations": [
    "L1",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  }
 ],
 "definitions": [
  {
   "id": "D22",
   "term": "AUC",
   "kind": "variable",
   "definition": "The area under the ROC curve: the probability that a randomly chosen case of the positive class receives a higher score than a randomly chosen case of the negative class.",
   "source": "B14"
  }
 ],
 "warrants": [
  {
   "id": "W17",
   "reasoning": "deductive",
   "premises": [
    "C42"
   ],
   "text": "The AUC averages the misclassification loss over a cost distribution that depends on the classifier, so different classifiers are compared with different weightings of the costs of errors; and it depends only on how the scores rank cases, not on whether the probabilities are right."
  }
 ],
 "limitations": [
  {
   "id": "L1",
   "category": "data_quality",
   "statement": "These claims rest only on works checked by their abstracts and bibliographic records (Brier 1950, Hand 2009: lab notes NT7 and NT8); a full text could qualify what the abstract states.",
   "affects": [
    "C42",
    "C17"
   ],
   "impact": "could_reverse",
   "mitigation": "none"
  },
  {
   "id": "L5",
   "category": "researcher_position",
   "statement": "The claims restate EntryLib's study-planning guide on prediction, drafted by an agent of the same system that checked them here, and were stated before the works were read; the works were read to confirm what the guide states, and no search was run for work that disagrees with it.",
   "affects": [
    "C1",
    "C2",
    "C3",
    "C4",
    "C5",
    "C6",
    "C7",
    "C8",
    "C9",
    "C10",
    "C11",
    "C12",
    "C13",
    "C14",
    "C15",
    "C16",
    "C17",
    "C18",
    "C19",
    "C20",
    "C21",
    "C22",
    "C23",
    "C24",
    "C25",
    "C26",
    "C27",
    "C28",
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C40",
    "C41",
    "C42",
    "C43",
    "C44",
    "C45",
    "C46",
    "C47",
    "C48",
    "C49",
    "C50",
    "C51",
    "C52",
    "C53"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  }
 ],
 "references": [
  {
   "id": "B14",
   "target": "10.1007/s10994-009-5119-5",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "abstract",
     "version": "published",
     "checked_on": "2026-11-13",
     "lab_event": "SC16"
    }
   }
  }
 ]
}
