{
 "bundle_of": "C16",
 "schema_version": "0.43",
 "paper": {
  "id": "entrylib:2026.000008",
  "version": 2,
  "title": "Evaluating forecasts and classifiers: which measures, scoring rules and tests to use",
  "licence": "CC-BY-4.0",
  "status": "corrected",
  "date": "2026-11-14",
  "status_notice": {
   "date": "2026-11-14",
   "reason_code": "correction",
   "statement": "Version 1 labelled the review's findings and recommendations extension, with an extends relation to the very works cited as their evidence, which said they extend those works; they state what those works establish, so version 2 labels them synthesis (claimset v0.43) and drops those relations, keeping extension only for C16, the one rule that goes beyond its sources, which extends Gneiting and Raftery (2007); and, at the guarantor's request, it rewords C3 (RMSSE as an example of the scaling), C4 (MAE and MASE as the main measure only when the mean and median forecasts coincide), C8 (absolute-error measures avoided on intermittent series), C13 (sharpness compared only among calibrated distributions and intervals) and C24 (choose a measure for what the methods predict, even if precedent used another).",
   "issued_by": "Ivan Svetunkov (ORCID 0000-0001-7826-0281)"
  }
 },
 "extensions": {
  "review": "0.1"
 },
 "claims": [
  {
   "id": "C16",
   "statement": "[[D21:Accuracy]] should not be the measure of forecast class probabilities.",
   "about": {
    "subject": "D21",
    "relation": "should not be the measure of",
    "object": "class probabilities",
    "comparator": null,
    "conditions": [
     "predictions are probabilities, or the costs of errors differ"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W16"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H75",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "depends_on": [
    "C38",
    "C37"
   ],
   "scope": {
    "data_type": "forecast class probabilities, thresholded to labels"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B3",
     "note": "Gneiting and Raftery (2007) show that the Brier and logarithmic scores are proper for forecast probabilities of classes."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L4",
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "moderate",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. L4: no cited work states it; it rests on an argument from the definition of accuracy."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C37",
   "statement": "The [[D16:log score]] is a strictly [[D2:proper scoring rule]].",
   "about": {
    "subject": "D16",
    "relation": "is strictly proper for",
    "object": "predictive distributions",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B3",
     "locator": "Section 4.1"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H44",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B3",
       "locator": "Section 4.1",
       "lab_event": "XT11",
       "exact": "The logarithmic score is strictly proper",
       "prefix": "rithmic score (Ta-       2002). ",
       "suffix": " relative to the\nble 1), the asy",
       "start": 53590,
       "end": 53630,
       "artefact": "A2"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "predictive distributions"
   },
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C38",
   "statement": "The [[D17:Brier score]] is a [[D2:proper scoring rule]] for forecast probabilities of classes.",
   "about": {
    "subject": "D17",
    "relation": "is proper for",
    "object": "forecast probabilities of classes",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B3",
     "locator": "Section 3, Example 1 (Quadratic or Brier score)"
    },
    {
     "kind": "cited",
     "reference": "B13"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H45",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B3",
       "locator": "Section 3, Example 1 (Quadratic or Brier score)",
       "lab_event": "XT12",
       "exact": "Example 1 (Quadratic or Brier score)",
       "prefix": "                           \u0005\n   ",
       "suffix": ". If G(p) = m    j=1 pj −\n      ",
       "start": 33270,
       "end": 33306,
       "artefact": "A2"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "forecast probabilities of classes"
   },
   "limitations": [
    "L2",
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  }
 ],
 "definitions": [
  {
   "id": "D2",
   "term": "proper scoring rule",
   "kind": "concept",
   "definition": "A scoring rule for probabilistic forecasts is proper when its expected value is best when the forecast distribution equals the distribution of the outcome, and strictly proper when it is best only then.",
   "source": "B3"
  },
  {
   "id": "D16",
   "term": "log score",
   "kind": "variable",
   "definition": "The logarithmic score: the negative logarithm of the forecast density (or probability) at the outcome.",
   "source": "B3"
  },
  {
   "id": "D17",
   "term": "Brier score",
   "kind": "variable",
   "definition": "The mean squared difference between forecast probabilities of a class and the outcome coded 1 if the class occurred and 0 otherwise.",
   "source": "B13"
  },
  {
   "id": "D21",
   "term": "accuracy",
   "kind": "variable",
   "definition": "The share of cases whose predicted class, after thresholding the predicted probabilities, equals the observed class."
  }
 ],
 "warrants": [
  {
   "id": "W16",
   "reasoning": "deductive",
   "premises": [
    "C38",
    "C37"
   ],
   "text": "Accuracy uses a forecast probability only through which side of a threshold it falls, and counts every misclassification as the same loss; two forecasts with different probabilities but the same thresholded labels score the same, and unequal costs of errors are not reflected. The Brier and log scores, which are proper, reward the probabilities themselves."
  }
 ],
 "limitations": [
  {
   "id": "L2",
   "category": "data_quality",
   "statement": "Part of the support of these claims comes from a work checked only by its abstract (Brier 1950, lab note NT7); the rest comes from works read in full.",
   "affects": [
    "C38"
   ],
   "impact": "reduces_precision",
   "mitigation": "none"
  },
  {
   "id": "L3",
   "category": "measurement",
   "statement": "Section and theorem numbers for Gneiting 2011, Gneiting and Raftery 2007, Gneiting, Balabdaoui and Raftery 2007 and Hyndman and Koehler 2006 were read in their preprints or authors' copies (arXiv:0912.0902 and the authors' web pages), whose numbering may differ from the published versions.",
   "affects": [
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C44",
    "C46"
   ],
   "impact": "reduces_precision",
   "mitigation": "none"
  },
  {
   "id": "L4",
   "category": "other",
   "statement": "No cited work states this recommendation or its reason; it rests on an argument from the definition of accuracy (its warrant), with the properness of the Brier and log scores as premises.",
   "affects": [
    "C16"
   ],
   "impact": "could_reverse",
   "mitigation": "none"
  },
  {
   "id": "L5",
   "category": "researcher_position",
   "statement": "The claims restate EntryLib's study-planning guide on prediction, drafted by an agent of the same system that checked them here, and were stated before the works were read; the works were read to confirm what the guide states, and no search was run for work that disagrees with it.",
   "affects": [
    "C1",
    "C2",
    "C3",
    "C4",
    "C5",
    "C6",
    "C7",
    "C8",
    "C9",
    "C10",
    "C11",
    "C12",
    "C13",
    "C14",
    "C15",
    "C16",
    "C17",
    "C18",
    "C19",
    "C20",
    "C21",
    "C22",
    "C23",
    "C24",
    "C25",
    "C26",
    "C27",
    "C28",
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C40",
    "C41",
    "C42",
    "C43",
    "C44",
    "C45",
    "C46",
    "C47",
    "C48",
    "C49",
    "C50",
    "C51",
    "C52",
    "C53"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  }
 ],
 "artefacts": [
  {
   "id": "A2",
   "kind": "data",
   "version": "sha256:d31a0c5f0ae8fec1a0a6544db5d056645b2d7296d71b44a2e8efb293c7d87ba2",
   "description": "Strictly proper scoring rules, prediction, and estimation (Gneiting and Raftery 2007, JASA 102(477), 359-378, doi:10.1198/016214506000001437), author's copy (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR2 on 2026-11-13. venue: Journal of the American Statistical Association. Written by others.",
   "uri": "https://sites.stat.washington.edu/raftery/Research/PDF/Gneiting2007jasa.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:d31a0c5f0ae8fec1a0a6544db5d056645b2d7296d71b44a2e8efb293c7d87ba2"
   },
   "reference": "B3"
  }
 ],
 "references": [
  {
   "id": "B3",
   "target": "10.1198/016214506000001437",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "author_copy",
     "checked_on": "2026-11-13",
     "artefact": "A2",
     "lab_event": "SC2"
    }
   }
  },
  {
   "id": "B13",
   "target": "10.1175/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "abstract",
     "version": "published",
     "checked_on": "2026-11-13",
     "lab_event": "SC15"
    }
   }
  }
 ]
}
