{
 "bundle_of": "C21",
 "schema_version": "0.43",
 "paper": {
  "id": "entrylib:2026.000008",
  "version": 2,
  "title": "Evaluating forecasts and classifiers: which measures, scoring rules and tests to use",
  "licence": "CC-BY-4.0",
  "status": "corrected",
  "date": "2026-11-14",
  "status_notice": {
   "date": "2026-11-14",
   "reason_code": "correction",
   "statement": "Version 1 labelled the review's findings and recommendations extension, with an extends relation to the very works cited as their evidence, which said they extend those works; they state what those works establish, so version 2 labels them synthesis (claimset v0.43) and drops those relations, keeping extension only for C16, the one rule that goes beyond its sources, which extends Gneiting and Raftery (2007); and, at the guarantor's request, it rewords C3 (RMSSE as an example of the scaling), C4 (MAE and MASE as the main measure only when the mean and median forecasts coincide), C8 (absolute-error measures avoided on intermittent series), C13 (sharpness compared only among calibrated distributions and intervals) and C24 (choose a measure for what the methods predict, even if precedent used another).",
   "issued_by": "Ivan Svetunkov (ORCID 0000-0001-7826-0281)"
  }
 },
 "extensions": {
  "review": "0.1"
 },
 "claims": [
  {
   "id": "C21",
   "statement": "To tell whether several methods whose aggregate accuracy is similar differ statistically significantly over many series or data sets, their ranks on each series should be tested with the [[D25:Friedman test]] followed by [[D26:Nemenyi test]] post-hoc comparisons.",
   "about": {
    "subject": "D25",
    "relation": "should be used to test",
    "object": "differences between methods of similar accuracy",
    "comparator": null,
    "conditions": [
     "many series or data sets"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "argument",
     "warrant": "W21"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H91",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "depends_on": [
    "C51"
   ],
   "scope": {
    "data_type": "comparisons of several methods over many series or data sets"
   },
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C51",
   "statement": "The [[D25:Friedman test]], followed by the [[D26:Nemenyi test]], compares several methods over many data sets by their ranks on each data set, assuming only limited commensurability of the measure across data sets.",
   "about": {
    "subject": "D25",
    "relation": "compares methods by",
    "object": "their ranks on each data set",
    "comparator": null,
    "conditions": [
     "many data sets"
    ]
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "synthesis",
   "support": [
    {
     "kind": "cited",
     "reference": "B5",
     "locator": "Section 3.2.2 (Friedman test)"
    },
    {
     "kind": "cited",
     "reference": "B5",
     "locator": "Section 5 (Conclusion)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H58",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B5",
       "locator": "Section 3.2.2 (Friedman test)",
       "lab_event": "XT24",
       "exact": "It ranks the algorithms for each data set separately",
       "prefix": "of the repeated-measures\nANOVA. ",
       "suffix": ", the best performing algorithm ",
       "start": 38209,
       "end": 38261,
       "artefact": "A4"
      },
      {
       "reference": "B5",
       "locator": "Section 3.2.2 (Friedman test)",
       "lab_event": "XT25",
       "exact": "The Nemenyi test",
       "prefix": "n proceed with a post-hoc test. ",
       "suffix": " (Ne-\nmenyi, 1963) is similar to",
       "start": 40625,
       "end": 40641,
       "artefact": "A4"
      },
      {
       "reference": "B5",
       "locator": "Section 5 (Conclusion)",
       "lab_event": "XT26",
       "exact": "They are appropriate since they assume some, but limited commensurability",
       "prefix": " are suitable for our\nproblems. ",
       "suffix": ". They\nare safer than parametric",
       "start": 432529,
       "end": 432602,
       "artefact": "A4"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "comparisons of several methods over many data sets"
   },
   "limitations": [
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  }
 ],
 "definitions": [
  {
   "id": "D25",
   "term": "Friedman test",
   "kind": "method",
   "definition": "A non-parametric test of whether several methods differ, applied to their ranks on each of several data sets.",
   "source": "B5"
  },
  {
   "id": "D26",
   "term": "Nemenyi test",
   "kind": "method",
   "definition": "A post-hoc test after the Friedman test that compares every pair of methods by their mean ranks against a critical difference.",
   "source": "B5"
  }
 ],
 "warrants": [
  {
   "id": "W21",
   "reasoning": "deductive",
   "premises": [
    "C51"
   ],
   "text": "The Friedman test, followed by the Nemenyi test, compares methods by their ranks on each data set and assumes only limited commensurability of the measure across data sets; it tells whether differences between methods whose aggregate accuracy is close are larger than chance."
  }
 ],
 "limitations": [
  {
   "id": "L5",
   "category": "researcher_position",
   "statement": "The claims restate EntryLib's study-planning guide on prediction, drafted by an agent of the same system that checked them here, and were stated before the works were read; the works were read to confirm what the guide states, and no search was run for work that disagrees with it.",
   "affects": [
    "C1",
    "C2",
    "C3",
    "C4",
    "C5",
    "C6",
    "C7",
    "C8",
    "C9",
    "C10",
    "C11",
    "C12",
    "C13",
    "C14",
    "C15",
    "C16",
    "C17",
    "C18",
    "C19",
    "C20",
    "C21",
    "C22",
    "C23",
    "C24",
    "C25",
    "C26",
    "C27",
    "C28",
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C40",
    "C41",
    "C42",
    "C43",
    "C44",
    "C45",
    "C46",
    "C47",
    "C48",
    "C49",
    "C50",
    "C51",
    "C52",
    "C53"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  }
 ],
 "artefacts": [
  {
   "id": "A4",
   "kind": "data",
   "version": "sha256:aa5fa1c71338d0d380e7a97e3503ce486d4e5798069647a8d39f57a1f99a7c8c",
   "description": "Statistical comparisons of classifiers over multiple data sets (Demsar 2006, JMLR 7, 1-30) (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR4 on 2026-11-13. venue: Journal of Machine Learning Research. Written by others.",
   "uri": "https://jmlr.org/papers/volume7/demsar06a/demsar06a.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:aa5fa1c71338d0d380e7a97e3503ce486d4e5798069647a8d39f57a1f99a7c8c"
   },
   "reference": "B5"
  }
 ],
 "references": [
  {
   "id": "B5",
   "target": "https://jmlr.org/papers/v7/demsar06a.html",
   "id_type": "url",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "published",
     "checked_on": "2026-11-13",
     "artefact": "A4",
     "lab_event": "SC4"
    }
   }
  }
 ]
}
