{
 "bundle_of": "C20",
 "schema_version": "0.43",
 "paper": {
  "id": "entrylib:2026.000008",
  "version": 2,
  "title": "Evaluating forecasts and classifiers: which measures, scoring rules and tests to use",
  "licence": "CC-BY-4.0",
  "status": "corrected",
  "date": "2026-11-14",
  "status_notice": {
   "date": "2026-11-14",
   "reason_code": "correction",
   "statement": "Version 1 labelled the review's findings and recommendations extension, with an extends relation to the very works cited as their evidence, which said they extend those works; they state what those works establish, so version 2 labels them synthesis (claimset v0.43) and drops those relations, keeping extension only for C16, the one rule that goes beyond its sources, which extends Gneiting and Raftery (2007); and, at the guarantor's request, it rewords C3 (RMSSE as an example of the scaling), C4 (MAE and MASE as the main measure only when the mean and median forecasts coincide), C8 (absolute-error measures avoided on intermittent series), C13 (sharpness compared only among calibrated distributions and intervals) and C24 (choose a measure for what the methods predict, even if precedent used another).",
   "issued_by": "Ivan Svetunkov (ORCID 0000-0001-7826-0281)"
  }
 },
 "extensions": {
  "review": "0.1"
 },
 "claims": [
  {
   "id": "C20",
   "statement": "When methods are compared over many series, their accuracy should be summarised by the mean, the median and the quartiles of a scaled measure across the series, such as [[D6:RMSSE]] or [[D8:MASE]].",
   "about": {
    "subject": "aggregates of a scaled measure across series",
    "relation": "should summarise",
    "object": "accuracy over many series",
    "comparator": null,
    "conditions": [
     "many series"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "argument",
     "warrant": "W20"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H84",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "depends_on": [
    "C46",
    "C50"
   ],
   "scope": {
    "data_type": "comparisons of methods over many series"
   },
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C46",
   "statement": "Scaled errors, as in [[D8:MASE]], are independent of the scale of the data and can be averaged over series on different scales.",
   "about": {
    "subject": "D8",
    "relation": "can be averaged over",
    "object": "series on different scales",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B6",
     "locator": "Abstract; Section 3 (Scaled errors)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H53",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B6",
       "locator": "Abstract; Section 3 (Scaled errors)",
       "lab_event": "XT19",
       "exact": "we propose that the mean absolute scaled error become the standard measure for\ncomparing forecast accuracy across multiple time series",
       "prefix": "ccurring situ-\nations. Instead, ",
       "suffix": ".\n\n\n\nKeywords: forecast accuracy",
       "start": 840,
       "end": 974,
       "artefact": "A7"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "error measures over series on different scales"
   },
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C50",
   "statement": "Averages of a performance measure over many data sets are susceptible to outliers.",
   "about": {
    "subject": "averages over data sets",
    "relation": "are susceptible to",
    "object": "outliers",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B5",
     "locator": "Section 3.1 (Comparisons of two classifiers)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H57",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B5",
       "locator": "Section 3.1 (Comparisons of two classifiers)",
       "lab_event": "XT23",
       "exact": "Averages are also susceptible to outliers",
       "prefix": "ms\nwith similar properties.\n    ",
       "suffix": ". They allow classifier’s excell",
       "start": 18817,
       "end": 18858,
       "artefact": "A4"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "comparisons of methods over many data sets"
   },
   "limitations": [
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  }
 ],
 "definitions": [
  {
   "id": "D6",
   "term": "RMSSE",
   "kind": "variable",
   "definition": "Root mean squared scaled error: the RMSE of a series' forecasts divided by the in-sample RMSE of the one-step naive forecast of that series, so that series on different scales can be averaged.",
   "source": "B9"
  },
  {
   "id": "D8",
   "term": "MASE",
   "kind": "variable",
   "definition": "Mean absolute scaled error: the MAE of a series' forecasts divided by the in-sample MAE of the one-step naive forecast of that series.",
   "source": "B6"
  }
 ],
 "warrants": [
  {
   "id": "W20",
   "reasoning": "deductive",
   "premises": [
    "C46",
    "C50"
   ],
   "text": "A scaled measure can be averaged over series, but averages over many data sets are susceptible to outliers; the median and the quartiles show how accuracy is spread over the series, which the mean alone hides."
  }
 ],
 "limitations": [
  {
   "id": "L3",
   "category": "measurement",
   "statement": "Section and theorem numbers for Gneiting 2011, Gneiting and Raftery 2007, Gneiting, Balabdaoui and Raftery 2007 and Hyndman and Koehler 2006 were read in their preprints or authors' copies (arXiv:0912.0902 and the authors' web pages), whose numbering may differ from the published versions.",
   "affects": [
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C44",
    "C46"
   ],
   "impact": "reduces_precision",
   "mitigation": "none"
  },
  {
   "id": "L5",
   "category": "researcher_position",
   "statement": "The claims restate EntryLib's study-planning guide on prediction, drafted by an agent of the same system that checked them here, and were stated before the works were read; the works were read to confirm what the guide states, and no search was run for work that disagrees with it.",
   "affects": [
    "C1",
    "C2",
    "C3",
    "C4",
    "C5",
    "C6",
    "C7",
    "C8",
    "C9",
    "C10",
    "C11",
    "C12",
    "C13",
    "C14",
    "C15",
    "C16",
    "C17",
    "C18",
    "C19",
    "C20",
    "C21",
    "C22",
    "C23",
    "C24",
    "C25",
    "C26",
    "C27",
    "C28",
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C40",
    "C41",
    "C42",
    "C43",
    "C44",
    "C45",
    "C46",
    "C47",
    "C48",
    "C49",
    "C50",
    "C51",
    "C52",
    "C53"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  }
 ],
 "artefacts": [
  {
   "id": "A4",
   "kind": "data",
   "version": "sha256:aa5fa1c71338d0d380e7a97e3503ce486d4e5798069647a8d39f57a1f99a7c8c",
   "description": "Statistical comparisons of classifiers over multiple data sets (Demsar 2006, JMLR 7, 1-30) (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR4 on 2026-11-13. venue: Journal of Machine Learning Research. Written by others.",
   "uri": "https://jmlr.org/papers/volume7/demsar06a/demsar06a.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:aa5fa1c71338d0d380e7a97e3503ce486d4e5798069647a8d39f57a1f99a7c8c"
   },
   "reference": "B5"
  },
  {
   "id": "A7",
   "kind": "data",
   "version": "sha256:b86b2d78457b62195faa87e2071beb3cacea2d6cb413c296dab746b00bb90a80",
   "description": "Another look at measures of forecast accuracy (preprint of Hyndman and Koehler 2006, IJF 22(4), 679-688, doi:10.1016/j.ijforecast.2006.03.001) (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR7 on 2026-11-13. venue: International Journal of Forecasting. Written by others.",
   "uri": "https://robjhyndman.com/papers/mase.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:b86b2d78457b62195faa87e2071beb3cacea2d6cb413c296dab746b00bb90a80"
   },
   "reference": "B6"
  }
 ],
 "references": [
  {
   "id": "B5",
   "target": "https://jmlr.org/papers/v7/demsar06a.html",
   "id_type": "url",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "published",
     "checked_on": "2026-11-13",
     "artefact": "A4",
     "lab_event": "SC4"
    }
   }
  },
  {
   "id": "B6",
   "target": "10.1016/j.ijforecast.2006.03.001",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "author_copy",
     "checked_on": "2026-11-13",
     "artefact": "A7",
     "lab_event": "SC7"
    }
   }
  },
  {
   "id": "B9",
   "target": "10.1016/j.ijforecast.2021.11.013",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC11"
    }
   }
  }
 ]
}
