{
 "bundle_of": "C14",
 "schema_version": "0.43",
 "paper": {
  "id": "entrylib:2026.000008",
  "version": 2,
  "title": "Evaluating forecasts and classifiers: which measures, scoring rules and tests to use",
  "licence": "CC-BY-4.0",
  "status": "corrected",
  "date": "2026-11-14",
  "status_notice": {
   "date": "2026-11-14",
   "reason_code": "correction",
   "statement": "Version 1 labelled the review's findings and recommendations extension, with an extends relation to the very works cited as their evidence, which said they extend those works; they state what those works establish, so version 2 labels them synthesis (claimset v0.43) and drops those relations, keeping extension only for C16, the one rule that goes beyond its sources, which extends Gneiting and Raftery (2007); and, at the guarantor's request, it rewords C3 (RMSSE as an example of the scaling), C4 (MAE and MASE as the main measure only when the mean and median forecasts coincide), C8 (absolute-error measures avoided on intermittent series), C13 (sharpness compared only among calibrated distributions and intervals) and C24 (choose a measure for what the methods predict, even if precedent used another).",
   "issued_by": "Ivan Svetunkov (ORCID 0000-0001-7826-0281)"
  }
 },
 "extensions": {
  "review": "0.1"
 },
 "claims": [
  {
   "id": "C14",
   "statement": "Predictive distributions of counts, such as forecasts of intermittent sales, should be evaluated with proper scoring rules and a randomised [[D18:probability integral transform]].",
   "about": {
    "subject": "proper scoring rules and the randomised PIT",
    "relation": "should be used to evaluate",
    "object": "predictive distributions of counts",
    "comparator": null,
    "conditions": [
     "count data"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "argument",
     "warrant": "W14"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H73",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "depends_on": [
    "C40",
    "C36",
    "C37"
   ],
   "scope": {
    "data_type": "predictive distributions of counts"
   },
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C36",
   "statement": "The [[D15:CRPS]] is a [[D2:proper scoring rule]] for predictive distributions of continuous outcomes.",
   "about": {
    "subject": "D15",
    "relation": "is proper for",
    "object": "predictive distributions",
    "comparator": null,
    "conditions": [
     "continuous outcomes"
    ]
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B3",
     "locator": "Section 4.2 (Continuous ranked probability score)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H43",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B3",
       "locator": "Section 4.2 (Continuous ranked probability score)",
       "lab_event": "XT10",
       "exact": "The CRPS is proper relative to the class P",
       "prefix": "003) implies the equality of\n   ",
       "suffix": " and strictly proper\n           ",
       "start": 64068,
       "end": 64110,
       "artefact": "A2"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "predictive distributions of continuous outcomes"
   },
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C37",
   "statement": "The [[D16:log score]] is a strictly [[D2:proper scoring rule]].",
   "about": {
    "subject": "D16",
    "relation": "is strictly proper for",
    "object": "predictive distributions",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B3",
     "locator": "Section 4.1"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H44",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B3",
       "locator": "Section 4.1",
       "lab_event": "XT11",
       "exact": "The logarithmic score is strictly proper",
       "prefix": "rithmic score (Ta-       2002). ",
       "suffix": " relative to the\nble 1), the asy",
       "start": 53590,
       "end": 53630,
       "artefact": "A2"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "predictive distributions"
   },
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C40",
   "statement": "The calibration of predictive distributions of counts can be assessed with a randomised [[D18:probability integral transform]].",
   "about": {
    "subject": "D18",
    "relation": "assesses",
    "object": "the calibration of predictive distributions of counts",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B12"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H47",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "predictive distributions of counts"
   },
   "ext": {
    "review": {
     "kind": "summary"
    }
   },
   "limitations": [
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  }
 ],
 "definitions": [
  {
   "id": "D2",
   "term": "proper scoring rule",
   "kind": "concept",
   "definition": "A scoring rule for probabilistic forecasts is proper when its expected value is best when the forecast distribution equals the distribution of the outcome, and strictly proper when it is best only then.",
   "source": "B3"
  },
  {
   "id": "D15",
   "term": "CRPS",
   "kind": "variable",
   "definition": "Continuous ranked probability score: the integral over thresholds of the squared difference between the forecast's cumulative distribution function and the step function at the outcome.",
   "source": "B3"
  },
  {
   "id": "D16",
   "term": "log score",
   "kind": "variable",
   "definition": "The logarithmic score: the negative logarithm of the forecast density (or probability) at the outcome.",
   "source": "B3"
  },
  {
   "id": "D18",
   "term": "probability integral transform",
   "kind": "concept",
   "definition": "The value of the forecast's cumulative distribution function at the outcome; for a calibrated forecast its values are uniformly distributed.",
   "source": "B4"
  }
 ],
 "warrants": [
  {
   "id": "W14",
   "reasoning": "deductive",
   "premises": [
    "C40",
    "C36",
    "C37"
   ],
   "text": "The probability integral transform of a discrete distribution takes few values and is not uniform even when the forecast is calibrated, so the calibration of count forecasts is checked with the randomised transform; proper scoring rules apply to count distributions as to any other."
  }
 ],
 "limitations": [
  {
   "id": "L3",
   "category": "measurement",
   "statement": "Section and theorem numbers for Gneiting 2011, Gneiting and Raftery 2007, Gneiting, Balabdaoui and Raftery 2007 and Hyndman and Koehler 2006 were read in their preprints or authors' copies (arXiv:0912.0902 and the authors' web pages), whose numbering may differ from the published versions.",
   "affects": [
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C44",
    "C46"
   ],
   "impact": "reduces_precision",
   "mitigation": "none"
  },
  {
   "id": "L5",
   "category": "researcher_position",
   "statement": "The claims restate EntryLib's study-planning guide on prediction, drafted by an agent of the same system that checked them here, and were stated before the works were read; the works were read to confirm what the guide states, and no search was run for work that disagrees with it.",
   "affects": [
    "C1",
    "C2",
    "C3",
    "C4",
    "C5",
    "C6",
    "C7",
    "C8",
    "C9",
    "C10",
    "C11",
    "C12",
    "C13",
    "C14",
    "C15",
    "C16",
    "C17",
    "C18",
    "C19",
    "C20",
    "C21",
    "C22",
    "C23",
    "C24",
    "C25",
    "C26",
    "C27",
    "C28",
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C40",
    "C41",
    "C42",
    "C43",
    "C44",
    "C45",
    "C46",
    "C47",
    "C48",
    "C49",
    "C50",
    "C51",
    "C52",
    "C53"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  },
  {
   "id": "L6",
   "category": "data_quality",
   "statement": "These claims rest in part on works read in full by the guarantor, not by the agent that wrote the claimset (lab note NT14); no passage of those works is quoted, so a reader checks these claims against the works themselves.",
   "affects": [
    "C30",
    "C33",
    "C40",
    "C41",
    "C45",
    "C47",
    "C48",
    "C49",
    "C52",
    "C53"
   ],
   "impact": "reduces_precision",
   "mitigation": "none"
  }
 ],
 "artefacts": [
  {
   "id": "A2",
   "kind": "data",
   "version": "sha256:d31a0c5f0ae8fec1a0a6544db5d056645b2d7296d71b44a2e8efb293c7d87ba2",
   "description": "Strictly proper scoring rules, prediction, and estimation (Gneiting and Raftery 2007, JASA 102(477), 359-378, doi:10.1198/016214506000001437), author's copy (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR2 on 2026-11-13. venue: Journal of the American Statistical Association. Written by others.",
   "uri": "https://sites.stat.washington.edu/raftery/Research/PDF/Gneiting2007jasa.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:d31a0c5f0ae8fec1a0a6544db5d056645b2d7296d71b44a2e8efb293c7d87ba2"
   },
   "reference": "B3"
  },
  {
   "id": "A3",
   "kind": "data",
   "version": "sha256:5ca1dc51db5fc619ed757bf1b79f3601408d579373c43d6dc06d601d51b271ea",
   "description": "Probabilistic forecasts, calibration and sharpness (Gneiting, Balabdaoui and Raftery 2007, JRSS B 69(2), 243-268, doi:10.1111/j.1467-9868.2007.00587.x), author's copy (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR3 on 2026-11-13. venue: Journal of the Royal Statistical Society B. Written by others.",
   "uri": "https://sites.stat.washington.edu/raftery/Research/PDF/Gneiting2007jrssb.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:5ca1dc51db5fc619ed757bf1b79f3601408d579373c43d6dc06d601d51b271ea"
   },
   "reference": "B4"
  }
 ],
 "references": [
  {
   "id": "B3",
   "target": "10.1198/016214506000001437",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "author_copy",
     "checked_on": "2026-11-13",
     "artefact": "A2",
     "lab_event": "SC2"
    }
   }
  },
  {
   "id": "B4",
   "target": "10.1111/j.1467-9868.2007.00587.x",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "author_copy",
     "checked_on": "2026-11-13",
     "artefact": "A3",
     "lab_event": "SC3"
    }
   }
  },
  {
   "id": "B12",
   "target": "10.1016/j.ijforecast.2015.12.004",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC14"
    }
   }
  }
 ]
}
