{
 "bundle_of": "C25",
 "schema_version": "0.43",
 "paper": {
  "id": "entrylib:2026.000008",
  "version": 2,
  "title": "Evaluating forecasts and classifiers: which measures, scoring rules and tests to use",
  "licence": "CC-BY-4.0",
  "status": "corrected",
  "date": "2026-11-14",
  "status_notice": {
   "date": "2026-11-14",
   "reason_code": "correction",
   "statement": "Version 1 labelled the review's findings and recommendations extension, with an extends relation to the very works cited as their evidence, which said they extend those works; they state what those works establish, so version 2 labels them synthesis (claimset v0.43) and drops those relations, keeping extension only for C16, the one rule that goes beyond its sources, which extends Gneiting and Raftery (2007); and, at the guarantor's request, it rewords C3 (RMSSE as an example of the scaling), C4 (MAE and MASE as the main measure only when the mean and median forecasts coincide), C8 (absolute-error measures avoided on intermittent series), C13 (sharpness compared only among calibrated distributions and intervals) and C24 (choose a measure for what the methods predict, even if precedent used another).",
   "issued_by": "Ivan Svetunkov (ORCID 0000-0001-7826-0281)"
  }
 },
 "extensions": {
  "review": "0.1"
 },
 "claims": [
  {
   "id": "C25",
   "statement": "Scale-dependent error measures, such as [[D5:RMSE]] and [[D7:MAE]], should not be summed or averaged across series on different scales.",
   "about": {
    "subject": "scale-dependent error measures",
    "relation": "should not be aggregated across",
    "object": "series on different scales",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "argument",
     "warrant": "W25"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H81",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "depends_on": [
    "C44",
    "C45"
   ],
   "scope": {
    "data_type": "point forecasts over series on different scales"
   },
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C44",
   "statement": "Scale-dependent error measures, such as [[D5:RMSE]] and [[D7:MAE]], cannot be compared across series on different scales.",
   "about": {
    "subject": "scale-dependent error measures",
    "relation": "cannot be compared across",
    "object": "series on different scales",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B6",
     "locator": "Section 2.1 (Scale-dependent measures)"
    },
    {
     "kind": "cited",
     "reference": "B6",
     "locator": "Section 2.4 (Relative measures)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H51",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B6",
       "locator": "Section 2.1 (Scale-dependent measures)",
       "lab_event": "XT17",
       "exact": "should\nnot be used, for example, when comparing across data sets that have different scales",
       "prefix": "ds on the same set of data, but ",
       "suffix": ". Never-\ntheless, the MSE was us",
       "start": 13183,
       "end": 13274,
       "artefact": "A7"
      },
      {
       "reference": "B6",
       "locator": "Section 2.4 (Relative measures)",
       "lab_event": "XT18",
       "exact": "It makes no sense to compute the MAE across series (due to their different scales)",
       "prefix": "horizon across multiple\nseries. ",
       "suffix": ".\n\nA related approach is to use ",
       "start": 21483,
       "end": 21565,
       "artefact": "A7"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "error measures over series on different scales"
   },
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C45",
   "statement": "When the errors of series on different scales are summed or averaged without scaling, the series with the largest values dominate the result.",
   "about": {
    "subject": "unscaled errors summed over series",
    "relation": "are dominated by",
    "object": "the series with the largest values",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B17"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H52",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "error measures over series on different scales"
   },
   "ext": {
    "review": {
     "kind": "summary"
    }
   },
   "limitations": [
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  }
 ],
 "definitions": [
  {
   "id": "D5",
   "term": "RMSE",
   "kind": "variable",
   "definition": "Root mean squared error: the square root of the mean of the squared differences between outcomes and point forecasts.",
   "source": "B6"
  },
  {
   "id": "D7",
   "term": "MAE",
   "kind": "variable",
   "definition": "Mean absolute error: the mean of the absolute differences between outcomes and point forecasts.",
   "source": "B6"
  }
 ],
 "warrants": [
  {
   "id": "W25",
   "reasoning": "deductive",
   "premises": [
    "C44",
    "C45"
   ],
   "text": "Scale-dependent measures cannot be compared across series on different scales, and summed or averaged without scaling they are dominated by the series with the largest values; the result then says little about accuracy on the other series."
  }
 ],
 "limitations": [
  {
   "id": "L3",
   "category": "measurement",
   "statement": "Section and theorem numbers for Gneiting 2011, Gneiting and Raftery 2007, Gneiting, Balabdaoui and Raftery 2007 and Hyndman and Koehler 2006 were read in their preprints or authors' copies (arXiv:0912.0902 and the authors' web pages), whose numbering may differ from the published versions.",
   "affects": [
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C44",
    "C46"
   ],
   "impact": "reduces_precision",
   "mitigation": "none"
  },
  {
   "id": "L5",
   "category": "researcher_position",
   "statement": "The claims restate EntryLib's study-planning guide on prediction, drafted by an agent of the same system that checked them here, and were stated before the works were read; the works were read to confirm what the guide states, and no search was run for work that disagrees with it.",
   "affects": [
    "C1",
    "C2",
    "C3",
    "C4",
    "C5",
    "C6",
    "C7",
    "C8",
    "C9",
    "C10",
    "C11",
    "C12",
    "C13",
    "C14",
    "C15",
    "C16",
    "C17",
    "C18",
    "C19",
    "C20",
    "C21",
    "C22",
    "C23",
    "C24",
    "C25",
    "C26",
    "C27",
    "C28",
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C40",
    "C41",
    "C42",
    "C43",
    "C44",
    "C45",
    "C46",
    "C47",
    "C48",
    "C49",
    "C50",
    "C51",
    "C52",
    "C53"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  },
  {
   "id": "L6",
   "category": "data_quality",
   "statement": "These claims rest in part on works read in full by the guarantor, not by the agent that wrote the claimset (lab note NT14); no passage of those works is quoted, so a reader checks these claims against the works themselves.",
   "affects": [
    "C30",
    "C33",
    "C40",
    "C41",
    "C45",
    "C47",
    "C48",
    "C49",
    "C52",
    "C53"
   ],
   "impact": "reduces_precision",
   "mitigation": "none"
  }
 ],
 "artefacts": [
  {
   "id": "A7",
   "kind": "data",
   "version": "sha256:b86b2d78457b62195faa87e2071beb3cacea2d6cb413c296dab746b00bb90a80",
   "description": "Another look at measures of forecast accuracy (preprint of Hyndman and Koehler 2006, IJF 22(4), 679-688, doi:10.1016/j.ijforecast.2006.03.001) (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR7 on 2026-11-13. venue: International Journal of Forecasting. Written by others.",
   "uri": "https://robjhyndman.com/papers/mase.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:b86b2d78457b62195faa87e2071beb3cacea2d6cb413c296dab746b00bb90a80"
   },
   "reference": "B6"
  }
 ],
 "references": [
  {
   "id": "B6",
   "target": "10.1016/j.ijforecast.2006.03.001",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "author_copy",
     "checked_on": "2026-11-13",
     "artefact": "A7",
     "lab_event": "SC7"
    }
   }
  },
  {
   "id": "B17",
   "target": "10.1016/j.ijforecast.2022.08.003",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC19"
    }
   }
  }
 ]
}
