{
 "bundle_of": "C9",
 "schema_version": "0.43",
 "paper": {
  "id": "entrylib:2026.000008",
  "version": 2,
  "title": "Evaluating forecasts and classifiers: which measures, scoring rules and tests to use",
  "licence": "CC-BY-4.0",
  "status": "corrected",
  "date": "2026-11-14",
  "status_notice": {
   "date": "2026-11-14",
   "reason_code": "correction",
   "statement": "Version 1 labelled the review's findings and recommendations extension, with an extends relation to the very works cited as their evidence, which said they extend those works; they state what those works establish, so version 2 labels them synthesis (claimset v0.43) and drops those relations, keeping extension only for C16, the one rule that goes beyond its sources, which extends Gneiting and Raftery (2007); and, at the guarantor's request, it rewords C3 (RMSSE as an example of the scaling), C4 (MAE and MASE as the main measure only when the mean and median forecasts coincide), C8 (absolute-error measures avoided on intermittent series), C13 (sharpness compared only among calibrated distributions and intervals) and C24 (choose a measure for what the methods predict, even if precedent used another).",
   "issued_by": "Ivan Svetunkov (ORCID 0000-0001-7826-0281)"
  }
 },
 "extensions": {
  "review": "0.1"
 },
 "claims": [
  {
   "id": "C9",
   "statement": "Quantile forecasts should be evaluated with the [[D12:pinball loss]] at the quantile's level, averaged over the levels when there are several.",
   "about": {
    "subject": "D12",
    "relation": "should be used to evaluate",
    "object": "quantile forecasts",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "argument",
     "warrant": "W9"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H83",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "depends_on": [
    "C34"
   ],
   "scope": {
    "data_type": "quantile forecasts"
   },
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C34",
   "statement": "The [[D12:pinball loss]] at a level is a [[D1:consistent scoring function]] for the quantile at that level.",
   "about": {
    "subject": "D12",
    "relation": "is consistent for",
    "object": "the quantile at its level",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B2",
     "locator": "Abstract; Theorem 3.3 (arXiv:0912.0902 numbering)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H41",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B2",
       "locator": "Abstract; Theorem 3.3 (arXiv:0912.0902 numbering)",
       "lab_event": "XT7",
       "exact": "It is consistent for a quantile if and only if it is generalized piecewise linear",
       "prefix": "                      function. ",
       "suffix": ".\n                              ",
       "start": 3351,
       "end": 3432,
       "artefact": "A1"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "quantile forecasts"
   },
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  }
 ],
 "definitions": [
  {
   "id": "D1",
   "term": "consistent scoring function",
   "kind": "concept",
   "definition": "A scoring function (an error measure for point forecasts) is consistent for a functional of the predictive distribution, such as the mean, the median or a quantile, when its expected value is lowest when the point forecast equals that functional.",
   "source": "B2"
  },
  {
   "id": "D12",
   "term": "pinball loss",
   "kind": "variable",
   "definition": "The quantile (asymmetric piecewise linear) loss at level tau: tau times the error when the outcome is above the forecast, and (1 - tau) times the absolute error when it is below.",
   "source": "B2"
  }
 ],
 "warrants": [
  {
   "id": "W9",
   "reasoning": "deductive",
   "premises": [
    "C34"
   ],
   "text": "The pinball loss at a level is consistent for the quantile at that level, so its expected value is lowest for the correct quantile; averaging over the levels gives each quantile the same weight."
  }
 ],
 "limitations": [
  {
   "id": "L3",
   "category": "measurement",
   "statement": "Section and theorem numbers for Gneiting 2011, Gneiting and Raftery 2007, Gneiting, Balabdaoui and Raftery 2007 and Hyndman and Koehler 2006 were read in their preprints or authors' copies (arXiv:0912.0902 and the authors' web pages), whose numbering may differ from the published versions.",
   "affects": [
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C44",
    "C46"
   ],
   "impact": "reduces_precision",
   "mitigation": "none"
  },
  {
   "id": "L5",
   "category": "researcher_position",
   "statement": "The claims restate EntryLib's study-planning guide on prediction, drafted by an agent of the same system that checked them here, and were stated before the works were read; the works were read to confirm what the guide states, and no search was run for work that disagrees with it.",
   "affects": [
    "C1",
    "C2",
    "C3",
    "C4",
    "C5",
    "C6",
    "C7",
    "C8",
    "C9",
    "C10",
    "C11",
    "C12",
    "C13",
    "C14",
    "C15",
    "C16",
    "C17",
    "C18",
    "C19",
    "C20",
    "C21",
    "C22",
    "C23",
    "C24",
    "C25",
    "C26",
    "C27",
    "C28",
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C40",
    "C41",
    "C42",
    "C43",
    "C44",
    "C45",
    "C46",
    "C47",
    "C48",
    "C49",
    "C50",
    "C51",
    "C52",
    "C53"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  }
 ],
 "artefacts": [
  {
   "id": "A1",
   "kind": "data",
   "version": "sha256:6b6cc2f0fd3daa1da42506f42f05fe88425880b2806f864ceaab85fc606ab5c9",
   "description": "Making and evaluating point forecasts (arXiv:0912.0902, the preprint of Gneiting 2011, JASA 106(494), 746-762, doi:10.1198/jasa.2011.r10138) (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR1 on 2026-11-13. venue: arXiv. Written by others.",
   "uri": "https://arxiv.org/abs/0912.0902",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:6b6cc2f0fd3daa1da42506f42f05fe88425880b2806f864ceaab85fc606ab5c9"
   },
   "reference": "B2"
  }
 ],
 "references": [
  {
   "id": "B2",
   "target": "10.1198/jasa.2011.r10138",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "preprint",
     "checked_on": "2026-11-13",
     "artefact": "A1",
     "lab_event": "SC1"
    }
   }
  }
 ]
}
