{
 "schema_version": "0.42",
 "paper": {
  "title": "Evaluating forecasts and classifiers: which measures, scoring rules and tests to use",
  "contributions": [
   "C1",
   "C2",
   "C3",
   "C4",
   "C5",
   "C6",
   "C7",
   "C8",
   "C9",
   "C10",
   "C11",
   "C12",
   "C13",
   "C14",
   "C15",
   "C16",
   "C17",
   "C18",
   "C19",
   "C20",
   "C21",
   "C22",
   "C23",
   "C24",
   "C25",
   "C26",
   "C27",
   "C28"
  ],
  "autonomy": "human_supervised",
  "contribution_record": "complete",
  "origin": "native",
  "exploration": {
   "lab_record": "A10",
   "questions": 1,
   "hypotheses": {
    "total": 55,
    "supported": 53,
    "not_supported": 0,
    "inconclusive": 0,
    "revised": 0,
    "abandoned": 2,
    "open": 0,
    "from_exploration": 0
   },
   "tests": {
    "total": 0,
    "supported": 0,
    "not_supported": 0,
    "inconclusive": 0
   },
   "runs": {
    "total": 0,
    "confirmatory": 0,
    "exploratory": 0,
    "failed": 0
   }
  },
  "study": "prediction-good-practice",
  "ext": {
   "review": {
    "flow": {
     "assessed": 20,
     "excluded_full_text": {
      "out of scope: it was cited only for implementation facts of ETS models and back-transformed forecasts, which decision DC1 left out of this study": 2
     },
     "included": 18,
     "other_sources": 18
    }
   }
  },
  "licence": "CC-BY-4.0",
  "summary": "How should a study measure and compare predictions? This review states what the methodological literature establishes and the rules it draws from it. A measure has to match what a prediction targets: squared error (RMSE) for forecasts of the mean, absolute error (MAE) for the median, the pinball loss for a quantile, the interval score with coverage for intervals, CRPS or the log score for distributions, and log loss or the Brier score for class probabilities. MAPE rewards forecasts that are too low, sMAPE prefers forecasts that are too high, and AUC and accuracy do not judge probabilities. Errors of series on different scales should be scaled per series before they are averaged, never summed as they are. The mean, median and quartiles of a scaled measure summarise accuracy over many series; rank-based tests (Friedman with Nemenyi, MCB) and the Diebold-Mariano test tell whether similar methods differ statistically.",
  "classification": {
   "keywords": [
    "forecast evaluation",
    "scoring rules",
    "error measures",
    "classifier evaluation",
    "literature review"
   ],
   "subfield": "openalex:subfields/1803",
   "topics": [
    "openalex:T11918",
    "openalex:T13487",
    "openalex:T13309"
   ]
  },
  "declarations": {
   "funding": [],
   "competing_interests": "Ivan Svetunkov runs EntryLib, which publishes the study-planning guides that this claimset summarises."
  },
  "highlights": [
   "C1",
   "C26",
   "C5",
   "C12",
   "C15"
  ],
  "id": "entrylib:2026.000008",
  "version": 1,
  "date": "2026-11-14",
  "status": "active"
 },
 "contributors": [
  {
   "id": "P1",
   "type": "agent",
   "name": "Claude Code",
   "positions": [
    "author"
   ],
   "models": [
    {
     "model": "M1",
     "role": "sole"
    }
   ],
   "run_period": {
    "start": "2026-11-13",
    "end": "2026-11-13"
   },
   "system": {
    "name": "Claude Code"
   },
   "label": "literature summary",
   "record_source": "self_reported",
   "credit_roles": [
    "investigation",
    "writing_original_draft"
   ],
   "tools": [
    "web_search",
    "code_execution",
    "file_access"
   ],
   "ai_tasks": [
    {
     "task": "literature_search",
     "extent": "full"
    },
    {
     "task": "writing",
     "extent": "full"
    },
    {
     "task": "json_preparation",
     "extent": "full"
    }
   ]
  },
  {
   "id": "P2",
   "type": "human",
   "name": "Ivan Svetunkov",
   "positions": [
    "supervisor",
    "guarantor"
   ],
   "orcid": "0000-0001-7826-0281",
   "credit_roles": [
    "conceptualization",
    "supervision",
    "validation"
   ]
  }
 ],
 "models": [
  {
   "id": "M1",
   "provider": "Anthropic",
   "name": "Claude Opus",
   "version": "5.5",
   "identifier": "claude-opus-5-5",
   "accessed": "2026-11-13",
   "weights": "api_only"
  }
 ],
 "definitions": [
  {
   "id": "D1",
   "term": "consistent scoring function",
   "kind": "concept",
   "definition": "A scoring function (an error measure for point forecasts) is consistent for a functional of the predictive distribution, such as the mean, the median or a quantile, when its expected value is lowest when the point forecast equals that functional.",
   "source": "B2"
  },
  {
   "id": "D2",
   "term": "proper scoring rule",
   "kind": "concept",
   "definition": "A scoring rule for probabilistic forecasts is proper when its expected value is best when the forecast distribution equals the distribution of the outcome, and strictly proper when it is best only then.",
   "source": "B3"
  },
  {
   "id": "D3",
   "term": "point forecast of the mean",
   "kind": "concept",
   "definition": "A single-valued forecast meant to equal the mean (expected value) of the outcome's distribution given the information available.",
   "source": "B2"
  },
  {
   "id": "D4",
   "term": "point forecast of the median",
   "kind": "concept",
   "definition": "A single-valued forecast meant to equal the median of the outcome's distribution given the information available.",
   "source": "B2"
  },
  {
   "id": "D5",
   "term": "RMSE",
   "kind": "variable",
   "definition": "Root mean squared error: the square root of the mean of the squared differences between outcomes and point forecasts.",
   "source": "B6"
  },
  {
   "id": "D6",
   "term": "RMSSE",
   "kind": "variable",
   "definition": "Root mean squared scaled error: the RMSE of a series' forecasts divided by the in-sample RMSE of the one-step naive forecast of that series, so that series on different scales can be averaged.",
   "source": "B9"
  },
  {
   "id": "D7",
   "term": "MAE",
   "kind": "variable",
   "definition": "Mean absolute error: the mean of the absolute differences between outcomes and point forecasts.",
   "source": "B6"
  },
  {
   "id": "D8",
   "term": "MASE",
   "kind": "variable",
   "definition": "Mean absolute scaled error: the MAE of a series' forecasts divided by the in-sample MAE of the one-step naive forecast of that series.",
   "source": "B6"
  },
  {
   "id": "D9",
   "term": "MAPE",
   "kind": "variable",
   "definition": "Mean absolute percentage error: the mean of the absolute differences between outcomes and point forecasts, each divided by the outcome.",
   "source": "B6"
  },
  {
   "id": "D10",
   "term": "sMAPE",
   "kind": "variable",
   "definition": "Symmetric mean absolute percentage error: the mean of the absolute differences between outcomes and point forecasts, each divided by the mean of the absolute outcome and the absolute forecast.",
   "source": "B11"
  },
  {
   "id": "D11",
   "term": "median of order -1",
   "kind": "concept",
   "definition": "The beta-median with beta = -1: the point forecast that minimises the expected absolute percentage error; for positive outcomes it lies below the median.",
   "source": "B2"
  },
  {
   "id": "D12",
   "term": "pinball loss",
   "kind": "variable",
   "definition": "The quantile (asymmetric piecewise linear) loss at level tau: tau times the error when the outcome is above the forecast, and (1 - tau) times the absolute error when it is below.",
   "source": "B2"
  },
  {
   "id": "D13",
   "term": "interval score",
   "kind": "variable",
   "definition": "The score of a central (1 - alpha) prediction interval: its width plus 2/alpha times the distance by which the outcome falls outside it.",
   "source": "B3"
  },
  {
   "id": "D14",
   "term": "coverage",
   "kind": "variable",
   "definition": "The share of outcomes that fall inside the prediction intervals, compared with their nominal level.",
   "source": "B3"
  },
  {
   "id": "D15",
   "term": "CRPS",
   "kind": "variable",
   "definition": "Continuous ranked probability score: the integral over thresholds of the squared difference between the forecast's cumulative distribution function and the step function at the outcome.",
   "source": "B3"
  },
  {
   "id": "D16",
   "term": "log score",
   "kind": "variable",
   "definition": "The logarithmic score: the negative logarithm of the forecast density (or probability) at the outcome.",
   "source": "B3"
  },
  {
   "id": "D17",
   "term": "Brier score",
   "kind": "variable",
   "definition": "The mean squared difference between forecast probabilities of a class and the outcome coded 1 if the class occurred and 0 otherwise.",
   "source": "B13"
  },
  {
   "id": "D18",
   "term": "probability integral transform",
   "kind": "concept",
   "definition": "The value of the forecast's cumulative distribution function at the outcome; for a calibrated forecast its values are uniformly distributed.",
   "source": "B4"
  },
  {
   "id": "D19",
   "term": "calibration",
   "kind": "concept",
   "definition": "Statistical consistency between forecast distributions and outcomes: events forecast with a given probability occur with that frequency.",
   "source": "B4"
  },
  {
   "id": "D20",
   "term": "sharpness",
   "kind": "concept",
   "definition": "The concentration of the forecast distributions, a property of the forecasts alone; among calibrated forecasts, sharper ones are better.",
   "source": "B4"
  },
  {
   "id": "D21",
   "term": "accuracy",
   "kind": "variable",
   "definition": "The share of cases whose predicted class, after thresholding the predicted probabilities, equals the observed class."
  },
  {
   "id": "D22",
   "term": "AUC",
   "kind": "variable",
   "definition": "The area under the ROC curve: the probability that a randomly chosen case of the positive class receives a higher score than a randomly chosen case of the negative class.",
   "source": "B14"
  },
  {
   "id": "D23",
   "term": "precision-recall curve",
   "kind": "concept",
   "definition": "The curve of precision (the share of predicted positives that are positive) against recall (the share of positives predicted as positive) over the classification thresholds.",
   "source": "B7"
  },
  {
   "id": "D24",
   "term": "intermittent series",
   "kind": "data",
   "definition": "A time series with many zero values, such as the sales of a slow-moving item.",
   "source": "B12"
  },
  {
   "id": "D25",
   "term": "Friedman test",
   "kind": "method",
   "definition": "A non-parametric test of whether several methods differ, applied to their ranks on each of several data sets.",
   "source": "B5"
  },
  {
   "id": "D26",
   "term": "Nemenyi test",
   "kind": "method",
   "definition": "A post-hoc test after the Friedman test that compares every pair of methods by their mean ranks against a critical difference.",
   "source": "B5"
  },
  {
   "id": "D27",
   "term": "MCB",
   "kind": "method",
   "definition": "Multiple comparisons with the best: a test, on mean ranks over series, of which methods differ significantly from the best-ranked one.",
   "source": "B15"
  },
  {
   "id": "D28",
   "term": "Diebold-Mariano test",
   "kind": "method",
   "definition": "A test of equal expected loss of two competing forecasts of the same series, based on the mean loss differential and its long-run variance.",
   "source": "B16"
  }
 ],
 "claims": [
  {
   "id": "C1",
   "statement": "Point forecasts should be evaluated with a [[D1:consistent scoring function]] for the functional they target, such as the mean, the median or a quantile.",
   "about": {
    "subject": "D1",
    "relation": "should be used to evaluate",
    "object": "point forecasts",
    "comparator": null,
    "conditions": [
     "the functional the forecasts target is known"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W1"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H90",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "point forecasts"
   },
   "depends_on": [
    "C30"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B2",
     "note": "Gneiting (2011) shows that a scoring function must be consistent for the functional a point forecast targets for the evaluation to reward that forecast."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C2",
   "statement": "[[D3:Point forecasts of the mean]] should be evaluated with squared error, as in [[D5:RMSE]].",
   "about": {
    "subject": "D5",
    "relation": "should be used to evaluate",
    "object": "D3",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W2"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H63",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "point forecasts of the mean"
   },
   "depends_on": [
    "C29",
    "C30"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B2",
     "note": "Gneiting (2011) shows that squared error is consistent for the mean."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C3",
   "statement": "When [[D3:point forecasts of the mean]] are compared over series on different scales, each series' [[D5:RMSE]] should be scaled by the in-sample RMSE of its one-step naive forecast, as in [[D6:RMSSE]].",
   "about": {
    "subject": "D6",
    "relation": "should be used to evaluate",
    "object": "D3",
    "comparator": null,
    "conditions": [
     "series on different scales"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W3"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H64",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "point forecasts of the mean, over series on different scales"
   },
   "depends_on": [
    "C47",
    "C29",
    "C46"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B1",
     "note": "The M5 competition scored point forecasts by the RMSSE, chosen because squared errors are optimised for the mean and the scaling makes series comparable."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C4",
   "statement": "[[D7:MAE]] and [[D8:MASE]] should not be the main measure for [[D3:point forecasts of the mean]].",
   "about": {
    "subject": "D7",
    "relation": "should not be the main measure for",
    "object": "D3",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W4"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H65",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "point forecasts of the mean"
   },
   "depends_on": [
    "C31",
    "C29"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B2",
     "note": "Gneiting (2011) shows that absolute error is consistent for the median."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C5",
   "statement": "[[D9:MAPE]] should not be the measure for point forecasts of the mean or the median.",
   "about": {
    "subject": "D9",
    "relation": "should not be the measure for",
    "object": "point forecasts of the mean or the median",
    "comparator": null,
    "conditions": [
     "positive outcomes"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W5"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H66",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "point forecasts of positive outcomes"
   },
   "depends_on": [
    "C32"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B2",
     "note": "Gneiting (2011) shows that the absolute percentage error is consistent for the median of order -1, which supports underforecasts."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C6",
   "statement": "[[D10:sMAPE]] should not be the measure for point forecasts.",
   "about": {
    "subject": "D10",
    "relation": "should not be the measure for",
    "object": "point forecasts",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W6"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H61",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "point forecasts"
   },
   "depends_on": [
    "C33"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B11",
     "note": "Goodwin and Lawton (1999) show that the symmetric MAPE is not symmetric in its treatment of positive and negative errors of the same size."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C7",
   "statement": "[[D4:Point forecasts of the median]] should be evaluated with absolute error, as in [[D7:MAE]] or, across series on different scales, [[D8:MASE]].",
   "about": {
    "subject": "D7",
    "relation": "should be used to evaluate",
    "object": "D4",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W7"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H67",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "point forecasts of the median"
   },
   "depends_on": [
    "C31",
    "C46"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B6",
     "note": "Hyndman and Koehler (2006) propose the MASE as the standard measure across series."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C8",
   "statement": "On [[D24:intermittent series]], absolute-error measures should not be used alone.",
   "about": {
    "subject": "absolute-error measures",
    "relation": "should not be used alone on",
    "object": "D24",
    "comparator": null,
    "conditions": [
     "many zero outcomes"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W8"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H68",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "point forecasts of intermittent series"
   },
   "depends_on": [
    "C41",
    "C31"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B1",
     "note": "The M5 guide states that on intermittent series absolute errors favour forecasts close to zero."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C9",
   "statement": "Quantile forecasts should be evaluated with the [[D12:pinball loss]] at the quantile's level, averaged over the levels when there are several.",
   "about": {
    "subject": "D12",
    "relation": "should be used to evaluate",
    "object": "quantile forecasts",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W9"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H83",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "quantile forecasts"
   },
   "depends_on": [
    "C34"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B2",
     "note": "Gneiting (2011) shows that a scoring function is consistent for a quantile if and only if it is generalized piecewise linear."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C10",
   "statement": "Prediction intervals should be evaluated with the [[D13:interval score]], reported together with their [[D14:coverage]] against the nominal level.",
   "about": {
    "subject": "D13",
    "relation": "should be used to evaluate",
    "object": "prediction intervals",
    "comparator": null,
    "conditions": [
     "central intervals"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W10"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H69",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "central prediction intervals"
   },
   "depends_on": [
    "C35"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B3",
     "note": "Gneiting and Raftery (2007) show that the interval score is proper for central prediction intervals."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C11",
   "statement": "[[D14:Coverage]] alone should not be the measure of prediction intervals.",
   "about": {
    "subject": "D14",
    "relation": "should not be the only measure of",
    "object": "prediction intervals",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W11"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H70",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "prediction intervals"
   },
   "depends_on": [
    "C35"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B3",
     "note": "Gneiting and Raftery (2007) propose the interval score because it addresses width as well as coverage."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C12",
   "statement": "Predictive distributions should be evaluated with a [[D2:proper scoring rule]], such as the [[D15:CRPS]] or the [[D16:log score]].",
   "about": {
    "subject": "D2",
    "relation": "should be used to evaluate",
    "object": "predictive distributions",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W12"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H71",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "predictive distributions"
   },
   "depends_on": [
    "C36",
    "C37"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B3",
     "note": "Gneiting and Raftery (2007) show that the CRPS and the logarithmic score are proper."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C13",
   "statement": "The evaluation of predictive distributions should check their [[D19:calibration]] and report their [[D20:sharpness]].",
   "about": {
    "subject": "calibration and sharpness checks",
    "relation": "should be part of evaluating",
    "object": "predictive distributions",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W13"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H72",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "predictive distributions"
   },
   "depends_on": [
    "C39"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B4",
     "note": "Gneiting, Balabdaoui and Raftery (2007) evaluate predictive distributions by maximising sharpness subject to calibration."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C14",
   "statement": "Predictive distributions of counts, such as forecasts of intermittent sales, should be evaluated with proper scoring rules and a randomised [[D18:probability integral transform]].",
   "about": {
    "subject": "proper scoring rules and the randomised PIT",
    "relation": "should be used to evaluate",
    "object": "predictive distributions of counts",
    "comparator": null,
    "conditions": [
     "count data"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W14"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H73",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "predictive distributions of counts"
   },
   "depends_on": [
    "C40",
    "C36",
    "C37"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B12",
     "note": "Kolassa (2016) evaluates predictive count distributions with proper scoring rules and a randomised probability integral transform."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C15",
   "statement": "Forecast probabilities of classes should be evaluated with a [[D2:proper scoring rule]], such as the [[D16:log score]] or the [[D17:Brier score]].",
   "about": {
    "subject": "D2",
    "relation": "should be used to evaluate",
    "object": "class probabilities",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W15"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H74",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "forecast probabilities of classes"
   },
   "depends_on": [
    "C38",
    "C37"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B3",
     "note": "Gneiting and Raftery (2007) show that the Brier and logarithmic scores are strictly proper."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C16",
   "statement": "[[D21:Accuracy]] should not be the measure of forecast class probabilities.",
   "about": {
    "subject": "D21",
    "relation": "should not be the measure of",
    "object": "class probabilities",
    "comparator": null,
    "conditions": [
     "predictions are probabilities, or the costs of errors differ"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "no_prior_found",
   "support": [
    {
     "kind": "argument",
     "warrant": "W16"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H75",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "forecast class probabilities, thresholded to labels"
   },
   "depends_on": [
    "C38",
    "C37"
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L4",
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "moderate",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. L4: no cited work states it; it rests on an argument from the definition of accuracy."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C17",
   "statement": "[[D22:AUC]] should not be the measure of forecast class probabilities.",
   "about": {
    "subject": "D22",
    "relation": "should not be the measure of",
    "object": "class probabilities",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W17"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H76",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "forecast class probabilities, ranked by a classifier's scores"
   },
   "depends_on": [
    "C42"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B14",
     "note": "Hand (2009) shows that the AUC measures different classifiers with different misclassification cost distributions."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L1",
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "moderate",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. L1: the finding it rests on was checked only in the abstract of its work; a full text could qualify it."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C18",
   "statement": "With rare positive cases, the [[D23:precision-recall curve]] should be reported rather than the ROC curve alone.",
   "about": {
    "subject": "D23",
    "relation": "is more informative than the ROC curve for",
    "object": "classifiers",
    "comparator": null,
    "conditions": [
     "imbalanced classes"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W18"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H77",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "binary classifiers",
    "conditions": [
     "imbalanced classes"
    ]
   },
   "depends_on": [
    "C43"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B7",
     "note": "Saito and Rehmsmeier (2015) show that the precision-recall plot is more informative than the ROC plot on imbalanced data."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C19",
   "statement": "When the compared methods target different functionals, one the mean and another the median, each should be evaluated with a measure consistent for its own target, or the methods made to target the same functional.",
   "about": {
    "subject": "the comparison of methods with different targets",
    "relation": "should use",
    "object": "measures consistent for each target",
    "comparator": null,
    "conditions": [
     "methods target different functionals"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W19"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H78",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "point forecasts of methods that target different functionals"
   },
   "depends_on": [
    "C30"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B8",
     "note": "Kolassa (2020) shows that different error measures are minimised by different summaries of one predictive density."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C20",
   "statement": "When methods are compared over many series, their accuracy should be summarised by the mean, the median and the quartiles of a scaled measure across the series, such as [[D6:RMSSE]] or [[D8:MASE]].",
   "about": {
    "subject": "aggregates of a scaled measure across series",
    "relation": "should summarise",
    "object": "accuracy over many series",
    "comparator": null,
    "conditions": [
     "many series"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W20"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H84",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "comparisons of methods over many series"
   },
   "depends_on": [
    "C46",
    "C50"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B5",
     "note": "Demsar (2006) notes that averages over data sets are susceptible to outliers."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C21",
   "statement": "To tell whether several methods whose aggregate accuracy is similar differ statistically significantly over many series or data sets, their ranks on each series should be tested with the [[D25:Friedman test]] followed by [[D26:Nemenyi test]] post-hoc comparisons.",
   "about": {
    "subject": "D25",
    "relation": "should be used to test",
    "object": "differences between methods of similar accuracy",
    "comparator": null,
    "conditions": [
     "many series or data sets"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W21"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H91",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "comparisons of several methods over many series or data sets"
   },
   "depends_on": [
    "C51"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B5",
     "note": "Demsar (2006) recommends the Friedman test with post-hoc tests to test whether several classifiers differ over many data sets."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C22",
   "statement": "To tell which methods differ statistically significantly from the best one over many series, multiple comparisons with the best ([[D27:MCB]]) on ranks should be used.",
   "about": {
    "subject": "D27",
    "relation": "should be used to test",
    "object": "differences from the best method",
    "comparator": null,
    "conditions": [
     "many series"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W22"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H92",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "comparisons of methods over many series"
   },
   "depends_on": [
    "C52"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B15",
     "note": "Koning et al. (2005) test the M3 results with multiple comparisons with the best on ranks."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C23",
   "statement": "The accuracy of two forecasts of the same series should be compared with the [[D28:Diebold-Mariano test]].",
   "about": {
    "subject": "D28",
    "relation": "should be used to compare",
    "object": "the accuracy of two forecasts",
    "comparator": null,
    "conditions": [
     "one series"
    ]
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W23"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H79",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "two forecasts of one series"
   },
   "depends_on": [
    "C53"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B16",
     "note": "Diebold and Mariano (1995) propose a test of equal accuracy of two forecasts."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C24",
   "statement": "That a competition or a well-known study used an error measure should not be the reason for choosing it.",
   "about": {
    "subject": "the choice of an error measure by precedent",
    "relation": "should not decide",
    "object": "the measure of an evaluation",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W24"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H80",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "evaluations of point forecasts"
   },
   "depends_on": [
    "C30"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B8",
     "note": "Kolassa (2020), on the M4 measures, argues that the best point forecast depends on the error measure."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C25",
   "statement": "Scale-dependent error measures, such as [[D5:RMSE]] and [[D7:MAE]], should not be summed or averaged across series on different scales.",
   "about": {
    "subject": "scale-dependent error measures",
    "relation": "should not be aggregated across",
    "object": "series on different scales",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W25"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H81",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "point forecasts over series on different scales"
   },
   "depends_on": [
    "C44",
    "C45"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B6",
     "note": "Hyndman and Koehler (2006) state that scale-dependent measures should not be used to compare across data sets on different scales."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C26",
   "statement": "To aggregate the accuracy of point forecasts over series on different scales, each series' errors should first be scaled by a quantity of that series that does not depend on the compared forecasts, such as the in-sample error of its one-step naive forecast in [[D8:MASE]] or [[D6:RMSSE]], before the scaled errors are averaged.",
   "about": {
    "subject": "scaling per series before aggregation",
    "relation": "should be used to aggregate",
    "object": "accuracy over series on different scales",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W26"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H82",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "point forecasts over series on different scales"
   },
   "depends_on": [
    "C44",
    "C46",
    "C47"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B6",
     "note": "Hyndman and Koehler (2006) propose scaled errors as the standard approach for comparing accuracy across series on different scales."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C27",
   "statement": "When quantile forecasts are compared over series on different scales, each series' [[D12:pinball loss]] should be scaled by a quantity of that series.",
   "about": {
    "subject": "D12",
    "relation": "should be scaled per series over",
    "object": "series on different scales",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W27"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H87",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "quantile forecasts over series on different scales"
   },
   "depends_on": [
    "C48",
    "C34",
    "C44"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B1",
     "note": "The M5 competition scored quantile forecasts with the pinball loss scaled per series."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C28",
   "statement": "When prediction intervals are compared over series on different scales, each series' [[D13:interval score]] should be scaled by a quantity of that series.",
   "about": {
    "subject": "D13",
    "relation": "should be scaled per series over",
    "object": "series on different scales",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "principal",
   "novelty": "extension",
   "support": [
    {
     "kind": "argument",
     "warrant": "W28"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H88",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "prediction intervals over series on different scales"
   },
   "depends_on": [
    "C49",
    "C35",
    "C44"
   ],
   "relations": [
    {
     "type": "extends",
     "reference": "B18",
     "note": "The M4 competition scored prediction intervals with the mean scaled interval score."
    }
   ],
   "ext": {
    "review": {
     "kind": "recommendation"
    }
   },
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory in the lab's terms: a literature claim, stated from the guide before its papers were read, then confirmed against them (the status of its hypothesis in the lab record); no test was planned or run. It follows by the argument of its warrant from findings that its works state, quoted from full texts or read in full by the guarantor."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C29",
   "statement": "Squared error is a [[D1:consistent scoring function]] for the mean.",
   "about": {
    "subject": "squared error",
    "relation": "is consistent for",
    "object": "the mean",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B2",
     "locator": "Section 5 (Discussion); Theorem 3.1 (arXiv:0912.0902 numbering)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H36",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B2",
       "locator": "Section 5 (Discussion); Theorem 3.1 (arXiv:0912.0902 numbering)",
       "lab_event": "XT1",
       "exact": "the squared error scoring function is consistent for the mean",
       "prefix": "businesses and organi-\nzations, ",
       "suffix": ", and the absolute error\nscoring",
       "start": 74674,
       "end": 74735,
       "artefact": "A1"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "point forecasts"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B2",
     "note": "Gneiting (2011) shows that squared error is consistent for the mean."
    }
   ],
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C30",
   "statement": "Which point forecast has the lowest expected score depends on the scoring function: a [[D1:consistent scoring function]] for a functional, such as the mean or a quantile, is lowest in expectation for the forecast that equals that functional of the outcome's distribution.",
   "about": {
    "subject": "D1",
    "relation": "is lowest in expectation for",
    "object": "the forecast equal to its functional",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B2",
     "locator": "Abstract (arXiv:0912.0902 numbering)"
    },
    {
     "kind": "cited",
     "reference": "B8"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H37",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B2",
       "locator": "Abstract (arXiv:0912.0902 numbering)",
       "lab_event": "XT2",
       "exact": "it is critical that the\n                                         scoring function be consistent for it",
       "prefix": "ve in the form of a functional, ",
       "suffix": ", in the sense that the expected",
       "start": 2433,
       "end": 2535,
       "artefact": "A1"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "point forecasts"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B2",
     "note": "Gneiting (2011) shows that the scoring function must be consistent for the functional a point forecast targets for the evaluation to reward that forecast."
    }
   ],
   "limitations": [
    "L3",
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C31",
   "statement": "Absolute error is a [[D1:consistent scoring function]] for the median.",
   "about": {
    "subject": "absolute error",
    "relation": "is consistent for",
    "object": "the median",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B2",
     "locator": "Section 5 (Discussion) (arXiv:0912.0902 numbering)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H38",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B2",
       "locator": "Section 5 (Discussion) (arXiv:0912.0902 numbering)",
       "lab_event": "XT3",
       "exact": "the absolute error\nscoring function for the median",
       "prefix": "is consistent for the mean, and ",
       "suffix": ". The absolute percentage error ",
       "start": 74741,
       "end": 74791,
       "artefact": "A1"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "point forecasts"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B2",
     "note": "Gneiting (2011) shows that absolute error is consistent for the median."
    }
   ],
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C32",
   "statement": "The absolute percentage error averaged by [[D9:MAPE]] is consistent for the [[D11:median of order -1]], a functional that supports severe underforecasts compared with the mean or the median.",
   "about": {
    "subject": "D9",
    "relation": "is consistent for",
    "object": "D11",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B2",
     "locator": "Section 5 (Discussion); the beta-median in Section 1.3 (arXiv:0912.0902 numbering)"
    },
    {
     "kind": "cited",
     "reference": "B2",
     "locator": "Section 5 (Discussion) (arXiv:0912.0902 numbering)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H39",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B2",
       "locator": "Section 5 (Discussion); the beta-median in Section 1.3 (arXiv:0912.0902 numbering)",
       "lab_event": "XT4",
       "exact": "the median of order −1",
       "prefix": "on-standard functional, namely, ",
       "suffix": ", med(−1) , which tends to sup-\n",
       "start": 74980,
       "end": 75002,
       "artefact": "A1"
      },
      {
       "reference": "B2",
       "locator": "Section 5 (Discussion) (arXiv:0912.0902 numbering)",
       "lab_event": "XT5",
       "exact": "severe underforecasts, as compared to the mean or median",
       "prefix": "(−1) , which tends to sup-\nport ",
       "suffix": ". It thus seems prudent that\nbus",
       "start": 75039,
       "end": 75095,
       "artefact": "A1"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "point forecasts of positive outcomes"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B2",
     "note": "Gneiting (2011) shows that the absolute percentage error is consistent for the median of order -1, which supports underforecasts."
    }
   ],
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C33",
   "statement": "For the same outcome, [[D10:sMAPE]] penalises a forecast below the outcome more heavily than a forecast above it.",
   "about": {
    "subject": "D10",
    "relation": "penalises more heavily",
    "object": "forecasts below the outcome",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B6",
     "locator": "Section 2.2 (Measures based on percentage errors)"
    },
    {
     "kind": "cited",
     "reference": "B11"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H40",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B6",
       "locator": "Section 2.2 (Measures based on percentage errors)",
       "lab_event": "XT6",
       "exact": "has a\nheavier penalty when forecasts are low compared to when forecasts are high",
       "prefix": "value of 2|Yt − Ft |/(Yt + Ft ) ",
       "suffix": ". See Goodwin\nand Lawton (1999) ",
       "start": 17140,
       "end": 17220,
       "artefact": "A7"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "point forecasts of positive outcomes"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B11",
     "note": "Goodwin and Lawton (1999) show that the symmetric MAPE treats positive and negative errors of the same size differently."
    }
   ],
   "limitations": [
    "L3",
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C34",
   "statement": "The [[D12:pinball loss]] at a level is a [[D1:consistent scoring function]] for the quantile at that level.",
   "about": {
    "subject": "D12",
    "relation": "is consistent for",
    "object": "the quantile at its level",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B2",
     "locator": "Abstract; Theorem 3.3 (arXiv:0912.0902 numbering)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H41",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B2",
       "locator": "Abstract; Theorem 3.3 (arXiv:0912.0902 numbering)",
       "lab_event": "XT7",
       "exact": "It is consistent for a quantile if and only if it is generalized piecewise linear",
       "prefix": "                      function. ",
       "suffix": ".\n                              ",
       "start": 3351,
       "end": 3432,
       "artefact": "A1"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "quantile forecasts"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B2",
     "note": "Gneiting (2011) shows that a scoring function is consistent for a quantile if and only if it is generalized piecewise linear."
    }
   ],
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C35",
   "statement": "The [[D13:interval score]] is a [[D2:proper scoring rule]] for central prediction intervals that addresses both their width and their coverage.",
   "about": {
    "subject": "D13",
    "relation": "is proper for",
    "object": "central prediction intervals",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B3",
     "locator": "Section 1 (Introduction)"
    },
    {
     "kind": "cited",
     "reference": "B3",
     "locator": "Abstract"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H89",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B3",
       "locator": "Section 1 (Introduction)",
       "lab_event": "XT8",
       "exact": "a scoring rule for prediction intervals that is proper",
       "prefix": " the general idea dates back at\n",
       "suffix": " and has in-     least to Brier ",
       "start": 15726,
       "end": 15780,
       "artefact": "A2"
      },
      {
       "reference": "B3",
       "locator": "Abstract",
       "lab_event": "XT9",
       "exact": "propose the intuitively appealing interval score as a utility function in interval estimation that addresses width as well as\n           coverage",
       "prefix": "tile\n           estimation, and ",
       "suffix": ".\n           KEY WORDS: Bayes fa",
       "start": 2499,
       "end": 2644,
       "artefact": "A2"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "central prediction intervals"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B3",
     "note": "Gneiting and Raftery (2007) propose the interval score, proper for central intervals, which addresses width as well as coverage."
    }
   ],
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C36",
   "statement": "The [[D15:CRPS]] is a [[D2:proper scoring rule]] for predictive distributions of continuous outcomes.",
   "about": {
    "subject": "D15",
    "relation": "is proper for",
    "object": "predictive distributions",
    "comparator": null,
    "conditions": [
     "continuous outcomes"
    ]
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B3",
     "locator": "Section 4.2 (Continuous ranked probability score)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H43",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B3",
       "locator": "Section 4.2 (Continuous ranked probability score)",
       "lab_event": "XT10",
       "exact": "The CRPS is proper relative to the class P",
       "prefix": "003) implies the equality of\n   ",
       "suffix": " and strictly proper\n           ",
       "start": 64068,
       "end": 64110,
       "artefact": "A2"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "predictive distributions of continuous outcomes"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B3",
     "note": "Gneiting and Raftery (2007) show that the CRPS is proper."
    }
   ],
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C37",
   "statement": "The [[D16:log score]] is a strictly [[D2:proper scoring rule]].",
   "about": {
    "subject": "D16",
    "relation": "is strictly proper for",
    "object": "predictive distributions",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B3",
     "locator": "Section 4.1"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H44",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B3",
       "locator": "Section 4.1",
       "lab_event": "XT11",
       "exact": "The logarithmic score is strictly proper",
       "prefix": "rithmic score (Ta-       2002). ",
       "suffix": " relative to the\nble 1), the asy",
       "start": 53590,
       "end": 53630,
       "artefact": "A2"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "predictive distributions"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B3",
     "note": "Gneiting and Raftery (2007) show that the logarithmic score is strictly proper."
    }
   ],
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C38",
   "statement": "The [[D17:Brier score]] is a [[D2:proper scoring rule]] for forecast probabilities of classes.",
   "about": {
    "subject": "D17",
    "relation": "is proper for",
    "object": "forecast probabilities of classes",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B3",
     "locator": "Section 3, Example 1 (Quadratic or Brier score)"
    },
    {
     "kind": "cited",
     "reference": "B13"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H45",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B3",
       "locator": "Section 3, Example 1 (Quadratic or Brier score)",
       "lab_event": "XT12",
       "exact": "Example 1 (Quadratic or Brier score)",
       "prefix": "                           \u0005\n   ",
       "suffix": ". If G(p) = m    j=1 pj −\n      ",
       "start": 33270,
       "end": 33306,
       "artefact": "A2"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "forecast probabilities of classes"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B3",
     "note": "Gneiting and Raftery (2007) show that the Brier score is proper."
    }
   ],
   "limitations": [
    "L2",
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C39",
   "statement": "Predictive distributions are assessed by maximising their [[D20:sharpness]] subject to their [[D19:calibration]].",
   "about": {
    "subject": "predictive distributions",
    "relation": "are assessed by",
    "object": "D20",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B4",
     "locator": "Abstract"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H46",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B4",
       "locator": "Abstract",
       "lab_event": "XT13",
       "exact": "is based on the paradigm of maximizing the sharpness of\n      the predictive distributions subject to calibration",
       "prefix": " of predictive performance that ",
       "suffix": ". Calibration refers to the stat",
       "start": 635,
       "end": 748,
       "artefact": "A3"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "predictive distributions"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B4",
     "note": "Gneiting, Balabdaoui and Raftery (2007) evaluate predictive distributions by maximising sharpness subject to calibration."
    }
   ],
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C40",
   "statement": "The calibration of predictive distributions of counts can be assessed with a randomised [[D18:probability integral transform]].",
   "about": {
    "subject": "D18",
    "relation": "assesses",
    "object": "the calibration of predictive distributions of counts",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B12"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H47",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "predictive distributions of counts"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B12",
     "note": "Kolassa (2016) assesses predictive count distributions with a randomised probability integral transform."
    }
   ],
   "ext": {
    "review": {
     "kind": "summary"
    }
   },
   "limitations": [
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C41",
   "statement": "On [[D24:intermittent series]], absolute-error measures give their best scores to forecasts close to zero.",
   "about": {
    "subject": "absolute-error measures",
    "relation": "give their best scores to",
    "object": "forecasts close to zero",
    "comparator": null,
    "conditions": [
     "D24"
    ]
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B1",
     "locator": "Point forecasts"
    },
    {
     "kind": "cited",
     "reference": "B12"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H48",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B1",
       "locator": "Point forecasts",
       "lab_event": "XT14",
       "exact": "absolute errors, which are optimized for the median, would assign lower scores\n        (better performance) to forecasting methods that derive forecasts close to zero",
       "prefix": " zeros.\n        This means that ",
       "suffix": ". However, the\n        objective",
       "start": 15824,
       "end": 15990,
       "artefact": "A9"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "point forecasts of intermittent series"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B1",
     "note": "The M5 guide states that on intermittent series absolute errors favour forecasts close to zero."
    }
   ],
   "limitations": [
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C42",
   "statement": "The [[D22:AUC]] is equivalent to averaging the misclassification loss over a distribution of misclassification costs that depends on the classifier.",
   "about": {
    "subject": "D22",
    "relation": "averages the misclassification loss over",
    "object": "a cost distribution that depends on the classifier",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B14"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H49",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "classifiers that score cases"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B14",
     "note": "Hand (2009) shows that the AUC measures different classifiers with different misclassification cost distributions."
    }
   ],
   "ext": {
    "review": {
     "kind": "summary"
    }
   },
   "limitations": [
    "L1",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C43",
   "statement": "On imbalanced data, the [[D23:precision-recall curve]] is more informative than the ROC curve for evaluating binary classifiers.",
   "about": {
    "subject": "D23",
    "relation": "is more informative than",
    "object": "the ROC curve",
    "comparator": null,
    "conditions": [
     "imbalanced data"
    ]
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B7",
     "locator": "Title"
    },
    {
     "kind": "cited",
     "reference": "B7",
     "locator": "Abstract"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H50",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B7",
       "locator": "Title",
       "lab_event": "XT15",
       "exact": "The Precision-Recall Plot Is More Informative\n                                                          than the ROC Plot When Evaluating Binary\n                                                          Classifiers on Imbalanced Datasets",
       "prefix": "                                ",
       "suffix": "\n                               ",
       "start": 135,
       "end": 372,
       "artefact": "A8"
      },
      {
       "reference": "B7",
       "locator": "Abstract",
       "lab_event": "XT16",
       "exact": "can be deceptive",
       "prefix": " context of imbalanced datasets ",
       "suffix": " with respect to conclu-\nROC Plo",
       "start": 2381,
       "end": 2397,
       "artefact": "A8"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "binary classifiers on imbalanced data"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B7",
     "note": "Saito and Rehmsmeier (2015) show that the precision-recall plot is more informative than the ROC plot on imbalanced data."
    }
   ],
   "limitations": [
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C44",
   "statement": "Scale-dependent error measures, such as [[D5:RMSE]] and [[D7:MAE]], cannot be compared across series on different scales.",
   "about": {
    "subject": "scale-dependent error measures",
    "relation": "cannot be compared across",
    "object": "series on different scales",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B6",
     "locator": "Section 2.1 (Scale-dependent measures)"
    },
    {
     "kind": "cited",
     "reference": "B6",
     "locator": "Section 2.4 (Relative measures)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H51",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B6",
       "locator": "Section 2.1 (Scale-dependent measures)",
       "lab_event": "XT17",
       "exact": "should\nnot be used, for example, when comparing across data sets that have different scales",
       "prefix": "ds on the same set of data, but ",
       "suffix": ". Never-\ntheless, the MSE was us",
       "start": 13183,
       "end": 13274,
       "artefact": "A7"
      },
      {
       "reference": "B6",
       "locator": "Section 2.4 (Relative measures)",
       "lab_event": "XT18",
       "exact": "It makes no sense to compute the MAE across series (due to their different scales)",
       "prefix": "horizon across multiple\nseries. ",
       "suffix": ".\n\nA related approach is to use ",
       "start": 21483,
       "end": 21565,
       "artefact": "A7"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "error measures over series on different scales"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B6",
     "note": "Hyndman and Koehler (2006) state that scale-dependent measures should not be used to compare across data sets on different scales."
    }
   ],
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C45",
   "statement": "When the errors of series on different scales are summed or averaged without scaling, the series with the largest values dominate the result.",
   "about": {
    "subject": "unscaled errors summed over series",
    "relation": "are dominated by",
    "object": "the series with the largest values",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B17"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H52",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "error measures over series on different scales"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B17",
     "note": "Athanasopoulos and Kourentzes (2023) show how unscaled errors summed over series are dominated by the series with the largest values."
    }
   ],
   "ext": {
    "review": {
     "kind": "summary"
    }
   },
   "limitations": [
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C46",
   "statement": "Scaled errors, as in [[D8:MASE]], are independent of the scale of the data and can be averaged over series on different scales.",
   "about": {
    "subject": "D8",
    "relation": "can be averaged over",
    "object": "series on different scales",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B6",
     "locator": "Abstract; Section 3 (Scaled errors)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H53",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B6",
       "locator": "Abstract; Section 3 (Scaled errors)",
       "lab_event": "XT19",
       "exact": "we propose that the mean absolute scaled error become the standard measure for\ncomparing forecast accuracy across multiple time series",
       "prefix": "ccurring situ-\nations. Instead, ",
       "suffix": ".\n\n\n\nKeywords: forecast accuracy",
       "start": 840,
       "end": 974,
       "artefact": "A7"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "error measures over series on different scales"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B6",
     "note": "Hyndman and Koehler (2006) propose scaled errors as the standard approach for comparing accuracy across series on different scales."
    }
   ],
   "limitations": [
    "L3",
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C47",
   "statement": "The M5 accuracy competition scored point forecasts with the [[D6:RMSSE]]: each series' squared errors divided by the in-sample mean squared error of its one-step naive forecast.",
   "about": {
    "subject": "D6",
    "relation": "scored point forecasts in",
    "object": "the M5 accuracy competition",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B1",
     "locator": "Point forecasts"
    },
    {
     "kind": "cited",
     "reference": "B9"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H54",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B1",
       "locator": "Point forecasts",
       "lab_event": "XT20",
       "exact": "The accuracy of the point forecasts will be evaluated using the Root Mean Squared Scaled Error (RMSSE)",
       "prefix": "ce-\nprogram\n\n5\n\fPoint forecasts\n",
       "suffix": ",\nwhich is a variant of the well",
       "start": 14467,
       "end": 14569,
       "artefact": "A9"
      },
      {
       "reference": "B1",
       "locator": "Point forecasts",
       "lab_event": "XT21",
       "exact": "The measure is scale independent, meaning that it can be effectively used to compare forecasts\n        across series with different scales",
       "prefix": "optimized for the mean.\n       ",
       "suffix": ".\n       In contrast to other m",
       "start": 16197,
       "end": 16335,
       "artefact": "A9"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "point forecasts in the M5 accuracy competition"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B1",
     "note": "The M5 guide defines the RMSSE and scores point forecasts with it."
    }
   ],
   "limitations": [
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C48",
   "statement": "The M5 uncertainty competition scored quantile forecasts with the scaled pinball loss: each series' [[D12:pinball loss]] divided by the in-sample mean absolute error of its one-step naive forecast.",
   "about": {
    "subject": "the scaled pinball loss",
    "relation": "scored quantile forecasts in",
    "object": "the M5 uncertainty competition",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B1",
     "locator": "Probabilistic forecasts"
    },
    {
     "kind": "cited",
     "reference": "B10"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H55",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B1",
       "locator": "Probabilistic forecasts",
       "lab_event": "XT22",
       "exact": "The precision of the probabilistic forecasts will be evaluated using the Scaled Pinball Loss (SPL) function",
       "prefix": "ition.\n\nProbabilistic forecasts\n",
       "suffix": ".\nThe measure is calculated for ",
       "start": 17832,
       "end": 17939,
       "artefact": "A9"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "quantile forecasts in the M5 uncertainty competition"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B1",
     "note": "The M5 guide defines the scaled pinball loss and scores quantile forecasts with it."
    }
   ],
   "limitations": [
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C49",
   "statement": "The M4 competition scored prediction intervals with the mean scaled interval score: the [[D13:interval score]] of each series scaled by an in-sample error of that series.",
   "about": {
    "subject": "the mean scaled interval score",
    "relation": "scored prediction intervals in",
    "object": "the M4 competition",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B18"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H56",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "prediction intervals in the M4 competition"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B18",
     "note": "The M4 competition scored prediction intervals with the mean scaled interval score."
    }
   ],
   "ext": {
    "review": {
     "kind": "summary"
    }
   },
   "limitations": [
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C50",
   "statement": "Averages of a performance measure over many data sets are susceptible to outliers.",
   "about": {
    "subject": "averages over data sets",
    "relation": "are susceptible to",
    "object": "outliers",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B5",
     "locator": "Section 3.1 (Comparisons of two classifiers)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H57",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B5",
       "locator": "Section 3.1 (Comparisons of two classifiers)",
       "lab_event": "XT23",
       "exact": "Averages are also susceptible to outliers",
       "prefix": "ms\nwith similar properties.\n    ",
       "suffix": ". They allow classifier’s excell",
       "start": 18817,
       "end": 18858,
       "artefact": "A4"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "comparisons of methods over many data sets"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B5",
     "note": "Demsar (2006) notes that averages over data sets are susceptible to outliers."
    }
   ],
   "limitations": [
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C51",
   "statement": "The [[D25:Friedman test]], followed by the [[D26:Nemenyi test]], compares several methods over many data sets by their ranks on each data set, assuming only limited commensurability of the measure across data sets.",
   "about": {
    "subject": "D25",
    "relation": "compares methods by",
    "object": "their ranks on each data set",
    "comparator": null,
    "conditions": [
     "many data sets"
    ]
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B5",
     "locator": "Section 3.2.2 (Friedman test)"
    },
    {
     "kind": "cited",
     "reference": "B5",
     "locator": "Section 5 (Conclusion)"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H58",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "ext": {
    "review": {
     "extracts": [
      {
       "reference": "B5",
       "locator": "Section 3.2.2 (Friedman test)",
       "lab_event": "XT24",
       "exact": "It ranks the algorithms for each data set separately",
       "prefix": "of the repeated-measures\nANOVA. ",
       "suffix": ", the best performing algorithm ",
       "start": 38209,
       "end": 38261,
       "artefact": "A4"
      },
      {
       "reference": "B5",
       "locator": "Section 3.2.2 (Friedman test)",
       "lab_event": "XT25",
       "exact": "The Nemenyi test",
       "prefix": "n proceed with a post-hoc test. ",
       "suffix": " (Ne-\nmenyi, 1963) is similar to",
       "start": 40625,
       "end": 40641,
       "artefact": "A4"
      },
      {
       "reference": "B5",
       "locator": "Section 5 (Conclusion)",
       "lab_event": "XT26",
       "exact": "They are appropriate since they assume some, but limited commensurability",
       "prefix": " are suitable for our\nproblems. ",
       "suffix": ". They\nare safer than parametric",
       "start": 432529,
       "end": 432602,
       "artefact": "A4"
      }
     ],
     "kind": "summary"
    }
   },
   "scope": {
    "data_type": "comparisons of several methods over many data sets"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B5",
     "note": "Demsar (2006) recommends the Friedman test with post-hoc tests for comparing several classifiers over many data sets."
    }
   ],
   "limitations": [
    "L5"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C52",
   "statement": "[[D27:MCB]] on ranks was used to test the differences between the methods of the M3 competition.",
   "about": {
    "subject": "D27",
    "relation": "was used to test",
    "object": "the methods of the M3 competition",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B15"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H59",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "comparisons of the methods of the M3 competition"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B15",
     "note": "Koning et al. (2005) test the M3 results with multiple comparisons with the best on ranks."
    }
   ],
   "ext": {
    "review": {
     "kind": "summary"
    }
   },
   "limitations": [
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C53",
   "statement": "The [[D28:Diebold-Mariano test]] of equal accuracy of two forecasts allows forecast errors that are serially correlated and not normally distributed.",
   "about": {
    "subject": "D28",
    "relation": "allows",
    "object": "serially correlated, non-normal forecast errors",
    "comparator": null,
    "conditions": []
   },
   "type": "methodological",
   "role": "supporting",
   "novelty": "extension",
   "support": [
    {
     "kind": "cited",
     "reference": "B16"
    }
   ],
   "provenance": "author_stated",
   "hypothesis": {
    "lab_event": "H60",
    "outcome": "supported"
   },
   "planning": "exploratory",
   "scope": {
    "data_type": "two forecasts of one series"
   },
   "relations": [
    {
     "type": "extends",
     "reference": "B16",
     "note": "Diebold and Mariano (1995) propose a test of equal accuracy of two forecasts."
    }
   ],
   "ext": {
    "review": {
     "kind": "summary"
    }
   },
   "limitations": [
    "L5",
    "L6"
   ],
   "verifiability": "open",
   "verifiability_flags": []
  }
 ],
 "evidence": [],
 "warrants": [
  {
   "id": "W1",
   "reasoning": "deductive",
   "premises": [
    "C30"
   ],
   "text": "A forecast is built to target one functional, and only a scoring function consistent for that functional is lowest in expectation for it; a scoring function consistent for another functional ranks forecasts by how close they come to that other functional."
  },
  {
   "id": "W2",
   "reasoning": "deductive",
   "premises": [
    "C29",
    "C30"
   ],
   "text": "Squared error is consistent for the mean, so its expected value is lowest for the forecast that equals the mean; the RMSE, the square root of the mean squared error, ranks forecasts in the same order."
  },
  {
   "id": "W3",
   "reasoning": "deductive",
   "premises": [
    "C47",
    "C29",
    "C46"
   ],
   "text": "Dividing a series' squared errors by a fixed positive number, the in-sample mean squared error of its one-step naive forecast, does not change which forecast minimises their expected value, so the RMSSE keeps the target of the RMSE, the mean, and the scaled values of different series can be averaged, as in the M5 accuracy competition."
  },
  {
   "id": "W4",
   "reasoning": "deductive",
   "premises": [
    "C31",
    "C29"
   ],
   "text": "Absolute error, which MAE averages and MASE scales, is consistent for the median, not the mean; on skewed outcomes the two differ, and these measures then rank a median forecast above a forecast that is correct for the mean."
  },
  {
   "id": "W5",
   "reasoning": "deductive",
   "premises": [
    "C32"
   ],
   "text": "MAPE is consistent for the median of order -1, which supports severe underforecasts compared with the mean or the median; it therefore rewards forecasts that are too low for either target."
  },
  {
   "id": "W6",
   "reasoning": "deductive",
   "premises": [
    "C33"
   ],
   "text": "sMAPE penalises forecasts below the outcome more heavily than forecasts above it, so it rewards forecasts that are too high; and it divides by the mean of the absolute outcome and the absolute forecast, which is zero when both are zero, so it is then undefined."
  },
  {
   "id": "W7",
   "reasoning": "deductive",
   "premises": [
    "C31",
    "C46"
   ],
   "text": "Absolute error is consistent for the median, so its expected value is lowest for the median forecast; MASE divides each series' absolute errors by a fixed in-sample quantity, which keeps that target and lets series on different scales be averaged."
  },
  {
   "id": "W8",
   "reasoning": "deductive",
   "premises": [
    "C41",
    "C31"
   ],
   "text": "On intermittent series the median of the outcome can be zero, and absolute error, consistent for the median, then gives its best score to a forecast of zero in every period; a measure of the mean or of the whole distribution shows what such a forecast misses."
  },
  {
   "id": "W9",
   "reasoning": "deductive",
   "premises": [
    "C34"
   ],
   "text": "The pinball loss at a level is consistent for the quantile at that level, so its expected value is lowest for the correct quantile; averaging over the levels gives each quantile the same weight."
  },
  {
   "id": "W10",
   "reasoning": "deductive",
   "premises": [
    "C35"
   ],
   "text": "The interval score is proper for central intervals and addresses width as well as coverage; coverage against the nominal level shows how far the intervals' calibration departs from what they claim."
  },
  {
   "id": "W11",
   "reasoning": "deductive",
   "premises": [
    "C35"
   ],
   "text": "Coverage counts only whether each outcome falls inside its interval; widening an interval never lowers its coverage, so coverage alone cannot penalise needless width, which the interval score does."
  },
  {
   "id": "W12",
   "reasoning": "deductive",
   "premises": [
    "C36",
    "C37"
   ],
   "text": "The CRPS and the log score are proper: their expected values are best for the true distribution, so they never reward a forecaster for reporting a distribution other than the one believed."
  },
  {
   "id": "W13",
   "reasoning": "deductive",
   "premises": [
    "C39"
   ],
   "text": "Predictive distributions are assessed by maximising their sharpness subject to calibration; checking calibration, with the probability integral transform or coverage at several levels, and reporting sharpness show how a forecast fails, which a single score does not."
  },
  {
   "id": "W14",
   "reasoning": "deductive",
   "premises": [
    "C40",
    "C36",
    "C37"
   ],
   "text": "The probability integral transform of a discrete distribution takes few values and is not uniform even when the forecast is calibrated, so the calibration of count forecasts is checked with the randomised transform; proper scoring rules apply to count distributions as to any other."
  },
  {
   "id": "W15",
   "reasoning": "deductive",
   "premises": [
    "C38",
    "C37"
   ],
   "text": "The Brier score is proper and the log score strictly proper, so both are best in expectation for the true probabilities of the classes."
  },
  {
   "id": "W16",
   "reasoning": "deductive",
   "premises": [
    "C38",
    "C37"
   ],
   "text": "Accuracy uses a forecast probability only through which side of a threshold it falls, and counts every misclassification as the same loss; two forecasts with different probabilities but the same thresholded labels score the same, and unequal costs of errors are not reflected. The Brier and log scores, which are proper, reward the probabilities themselves."
  },
  {
   "id": "W17",
   "reasoning": "deductive",
   "premises": [
    "C42"
   ],
   "text": "The AUC averages the misclassification loss over a cost distribution that depends on the classifier, so different classifiers are compared with different weightings of the costs of errors; and it depends only on how the scores rank cases, not on whether the probabilities are right."
  },
  {
   "id": "W18",
   "reasoning": "deductive",
   "premises": [
    "C43"
   ],
   "text": "On imbalanced data the ROC curve can be deceptive about how reliable a classifier is, and the precision-recall curve is the more informative of the two; reporting it shows what the ROC curve can hide."
  },
  {
   "id": "W19",
   "reasoning": "deductive",
   "premises": [
    "C30"
   ],
   "text": "Each scoring function is lowest in expectation for the functional it is consistent for, so one measure favours the methods that target that functional; comparing methods that target different functionals with one measure compares their targets, not their skill."
  },
  {
   "id": "W20",
   "reasoning": "deductive",
   "premises": [
    "C46",
    "C50"
   ],
   "text": "A scaled measure can be averaged over series, but averages over many data sets are susceptible to outliers; the median and the quartiles show how accuracy is spread over the series, which the mean alone hides."
  },
  {
   "id": "W21",
   "reasoning": "deductive",
   "premises": [
    "C51"
   ],
   "text": "The Friedman test, followed by the Nemenyi test, compares methods by their ranks on each data set and assumes only limited commensurability of the measure across data sets; it tells whether differences between methods whose aggregate accuracy is close are larger than chance."
  },
  {
   "id": "W22",
   "reasoning": "deductive",
   "premises": [
    "C52"
   ],
   "text": "Multiple comparisons with the best on ranks test which methods differ from the best one, and were used for this on the methods of the M3 competition."
  },
  {
   "id": "W23",
   "reasoning": "deductive",
   "premises": [
    "C53"
   ],
   "text": "The Diebold-Mariano test allows forecast errors that are serially correlated and not normally distributed, as the errors of multi-step forecasts of one series usually are; tests that assume independent, normal errors do not hold for them."
  },
  {
   "id": "W24",
   "reasoning": "deductive",
   "premises": [
    "C30"
   ],
   "text": "A measure fits a study when it is consistent for what the compared methods predict; that a competition used a measure says nothing about whether it is consistent for the forecasts of this study."
  },
  {
   "id": "W25",
   "reasoning": "deductive",
   "premises": [
    "C44",
    "C45"
   ],
   "text": "Scale-dependent measures cannot be compared across series on different scales, and summed or averaged without scaling they are dominated by the series with the largest values; the result then says little about accuracy on the other series."
  },
  {
   "id": "W26",
   "reasoning": "deductive",
   "premises": [
    "C44",
    "C46",
    "C47"
   ],
   "text": "Scaled errors are independent of the scale of the data and can be averaged over series; dividing by a quantity of the series that does not depend on the compared forecasts keeps the target that the unscaled measure is consistent for."
  },
  {
   "id": "W27",
   "reasoning": "deductive",
   "premises": [
    "C48",
    "C34",
    "C44"
   ],
   "text": "Pinball losses of series on different scales cannot be compared unscaled; dividing each series' pinball loss by a quantity of that series that does not depend on the forecasts keeps the quantile it is consistent for, as the scaled pinball loss of the M5 uncertainty competition does."
  },
  {
   "id": "W28",
   "reasoning": "deductive",
   "premises": [
    "C49",
    "C35",
    "C44"
   ],
   "text": "Interval scores of series on different scales cannot be compared unscaled; dividing each series' interval score by a positive quantity of that series that does not depend on the forecasts keeps it proper, as the mean scaled interval score of the M4 competition does."
  }
 ],
 "limitations": [
  {
   "id": "L1",
   "category": "data_quality",
   "statement": "These claims rest only on works checked by their abstracts and bibliographic records (Brier 1950, Hand 2009: lab notes NT7 and NT8); a full text could qualify what the abstract states.",
   "affects": [
    "C42",
    "C17"
   ],
   "impact": "could_reverse",
   "mitigation": "none"
  },
  {
   "id": "L2",
   "category": "data_quality",
   "statement": "Part of the support of these claims comes from a work checked only by its abstract (Brier 1950, lab note NT7); the rest comes from works read in full.",
   "affects": [
    "C38"
   ],
   "impact": "reduces_precision",
   "mitigation": "none"
  },
  {
   "id": "L3",
   "category": "measurement",
   "statement": "Section and theorem numbers for Gneiting 2011, Gneiting and Raftery 2007, Gneiting, Balabdaoui and Raftery 2007 and Hyndman and Koehler 2006 were read in their preprints or authors' copies (arXiv:0912.0902 and the authors' web pages), whose numbering may differ from the published versions.",
   "affects": [
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C44",
    "C46"
   ],
   "impact": "reduces_precision",
   "mitigation": "none"
  },
  {
   "id": "L4",
   "category": "other",
   "statement": "No cited work states this recommendation or its reason; it rests on an argument from the definition of accuracy (its warrant), with the properness of the Brier and log scores as premises.",
   "affects": [
    "C16"
   ],
   "impact": "could_reverse",
   "mitigation": "none"
  },
  {
   "id": "L5",
   "category": "researcher_position",
   "statement": "The claims restate EntryLib's study-planning guide on prediction, drafted by an agent of the same system that checked them here, and were stated before the works were read; the works were read to confirm what the guide states, and no search was run for work that disagrees with it.",
   "affects": [
    "C1",
    "C2",
    "C3",
    "C4",
    "C5",
    "C6",
    "C7",
    "C8",
    "C9",
    "C10",
    "C11",
    "C12",
    "C13",
    "C14",
    "C15",
    "C16",
    "C17",
    "C18",
    "C19",
    "C20",
    "C21",
    "C22",
    "C23",
    "C24",
    "C25",
    "C26",
    "C27",
    "C28",
    "C29",
    "C30",
    "C31",
    "C32",
    "C33",
    "C34",
    "C35",
    "C36",
    "C37",
    "C38",
    "C39",
    "C40",
    "C41",
    "C42",
    "C43",
    "C44",
    "C45",
    "C46",
    "C47",
    "C48",
    "C49",
    "C50",
    "C51",
    "C52",
    "C53"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  },
  {
   "id": "L6",
   "category": "data_quality",
   "statement": "These claims rest in part on works read in full by the guarantor, not by the agent that wrote the claimset (lab note NT14); no passage of those works is quoted, so a reader checks these claims against the works themselves.",
   "affects": [
    "C30",
    "C33",
    "C40",
    "C41",
    "C45",
    "C47",
    "C48",
    "C49",
    "C52",
    "C53"
   ],
   "impact": "reduces_precision",
   "mitigation": "none"
  }
 ],
 "artefacts": [
  {
   "id": "A1",
   "kind": "data",
   "version": "sha256:6b6cc2f0fd3daa1da42506f42f05fe88425880b2806f864ceaab85fc606ab5c9",
   "description": "Making and evaluating point forecasts (arXiv:0912.0902, the preprint of Gneiting 2011, JASA 106(494), 746-762, doi:10.1198/jasa.2011.r10138) (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR1 on 2026-11-13. venue: arXiv. Written by others.",
   "uri": "https://arxiv.org/abs/0912.0902",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:6b6cc2f0fd3daa1da42506f42f05fe88425880b2806f864ceaab85fc606ab5c9"
   },
   "reference": "B2"
  },
  {
   "id": "A2",
   "kind": "data",
   "version": "sha256:d31a0c5f0ae8fec1a0a6544db5d056645b2d7296d71b44a2e8efb293c7d87ba2",
   "description": "Strictly proper scoring rules, prediction, and estimation (Gneiting and Raftery 2007, JASA 102(477), 359-378, doi:10.1198/016214506000001437), author's copy (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR2 on 2026-11-13. venue: Journal of the American Statistical Association. Written by others.",
   "uri": "https://sites.stat.washington.edu/raftery/Research/PDF/Gneiting2007jasa.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:d31a0c5f0ae8fec1a0a6544db5d056645b2d7296d71b44a2e8efb293c7d87ba2"
   },
   "reference": "B3"
  },
  {
   "id": "A3",
   "kind": "data",
   "version": "sha256:5ca1dc51db5fc619ed757bf1b79f3601408d579373c43d6dc06d601d51b271ea",
   "description": "Probabilistic forecasts, calibration and sharpness (Gneiting, Balabdaoui and Raftery 2007, JRSS B 69(2), 243-268, doi:10.1111/j.1467-9868.2007.00587.x), author's copy (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR3 on 2026-11-13. venue: Journal of the Royal Statistical Society B. Written by others.",
   "uri": "https://sites.stat.washington.edu/raftery/Research/PDF/Gneiting2007jrssb.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:5ca1dc51db5fc619ed757bf1b79f3601408d579373c43d6dc06d601d51b271ea"
   },
   "reference": "B4"
  },
  {
   "id": "A4",
   "kind": "data",
   "version": "sha256:aa5fa1c71338d0d380e7a97e3503ce486d4e5798069647a8d39f57a1f99a7c8c",
   "description": "Statistical comparisons of classifiers over multiple data sets (Demsar 2006, JMLR 7, 1-30) (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR4 on 2026-11-13. venue: Journal of Machine Learning Research. Written by others.",
   "uri": "https://jmlr.org/papers/volume7/demsar06a/demsar06a.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:aa5fa1c71338d0d380e7a97e3503ce486d4e5798069647a8d39f57a1f99a7c8c"
   },
   "reference": "B5"
  },
  {
   "id": "A7",
   "kind": "data",
   "version": "sha256:b86b2d78457b62195faa87e2071beb3cacea2d6cb413c296dab746b00bb90a80",
   "description": "Another look at measures of forecast accuracy (preprint of Hyndman and Koehler 2006, IJF 22(4), 679-688, doi:10.1016/j.ijforecast.2006.03.001) (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR7 on 2026-11-13. venue: International Journal of Forecasting. Written by others.",
   "uri": "https://robjhyndman.com/papers/mase.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:b86b2d78457b62195faa87e2071beb3cacea2d6cb413c296dab746b00bb90a80"
   },
   "reference": "B6"
  },
  {
   "id": "A8",
   "kind": "data",
   "version": "sha256:8ae56333b46ffae705ec463b6a3dd1d81bc4fe5934e5e13a635e4bbf401d1a97",
   "description": "The precision-recall plot is more informative than the ROC plot when evaluating binary classifiers on imbalanced datasets (Saito and Rehmsmeier 2015, PLoS ONE 10(3), e0118432, doi:10.1371/journal.pone.0118432) (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR8 on 2026-11-13. venue: PLoS ONE; published: 2015-03-04. Written by others.",
   "uri": "https://entrylib.org/p/2026.000008/v1/files/lab/sources/SR8/saito-rehmsmeier2015.pdf",
   "availability": {
    "level": "public"
   },
   "reference": "B7",
   "licence": "CC-BY-4.0"
  },
  {
   "id": "A9",
   "kind": "data",
   "version": "sha256:f2650a2dbfb14d2c7d4482bda66c129ce5eec9341a76b76a833219559bf17109",
   "description": "The M5 Competition: Competitors' Guide (the organisers' definition of the RMSSE, the scaled pinball loss and the reasons for them) (application/pdf), saved by hand on 2026-11-13; recorded in the lab record as SR9 on 2026-11-13. venue: M Open Forecasting Center, GitHub repository Mcompetitions/M5-methods. Written by others.",
   "uri": "https://github.com/Mcompetitions/M5-methods/blob/master/M5-Competitors-Guide.pdf",
   "availability": {
    "level": "author_only",
    "reason": "legal",
    "content_hash": "sha256:f2650a2dbfb14d2c7d4482bda66c129ce5eec9341a76b76a833219559bf17109"
   },
   "reference": "B1"
  },
  {
   "id": "A10",
   "kind": "lab_record",
   "uri": "https://github.com/entrylib/study-planning/blob/832f05acb63d49f2ae374c15920814e5d050b326/lab/lab.jsonl",
   "version": "sha256:80799913b6759e1ce1201351e05780390a7ef6b831e17b97dcaa13b575d9a695",
   "availability": {
    "level": "public"
   },
   "description": "Lab record of the study: questions, hypotheses, plans, runs, results and decisions, as an append-only, hash-chained event log (Claimset lab record specification v0.22, begun 2026-11-13; exported with claimset 0.42.7).",
   "licence": "CC-BY-4.0"
  }
 ],
 "references": [
  {
   "id": "B1",
   "target": "https://github.com/Mcompetitions/M5-methods/blob/master/M5-Competitors-Guide.pdf",
   "id_type": "url",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "other",
     "checked_on": "2026-11-13",
     "artefact": "A9",
     "lab_event": "SC9"
    }
   }
  },
  {
   "id": "B2",
   "target": "10.1198/jasa.2011.r10138",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "preprint",
     "checked_on": "2026-11-13",
     "artefact": "A1",
     "lab_event": "SC1"
    }
   }
  },
  {
   "id": "B3",
   "target": "10.1198/016214506000001437",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "author_copy",
     "checked_on": "2026-11-13",
     "artefact": "A2",
     "lab_event": "SC2"
    }
   }
  },
  {
   "id": "B4",
   "target": "10.1111/j.1467-9868.2007.00587.x",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "author_copy",
     "checked_on": "2026-11-13",
     "artefact": "A3",
     "lab_event": "SC3"
    }
   }
  },
  {
   "id": "B5",
   "target": "https://jmlr.org/papers/v7/demsar06a.html",
   "id_type": "url",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "published",
     "checked_on": "2026-11-13",
     "artefact": "A4",
     "lab_event": "SC4"
    }
   }
  },
  {
   "id": "B6",
   "target": "10.1016/j.ijforecast.2006.03.001",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "author_copy",
     "checked_on": "2026-11-13",
     "artefact": "A7",
     "lab_event": "SC7"
    }
   }
  },
  {
   "id": "B7",
   "target": "10.1371/journal.pone.0118432",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "version": "published",
     "checked_on": "2026-11-13",
     "artefact": "A8",
     "lab_event": "SC8"
    }
   }
  },
  {
   "id": "B8",
   "target": "10.1016/j.ijforecast.2019.02.017",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC10"
    }
   }
  },
  {
   "id": "B9",
   "target": "10.1016/j.ijforecast.2021.11.013",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC11"
    }
   }
  },
  {
   "id": "B10",
   "target": "10.1016/j.ijforecast.2021.10.009",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC12"
    }
   }
  },
  {
   "id": "B11",
   "target": "10.1016/S0169-2070(99)00007-2",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC13"
    }
   }
  },
  {
   "id": "B12",
   "target": "10.1016/j.ijforecast.2015.12.004",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC14"
    }
   }
  },
  {
   "id": "B13",
   "target": "10.1175/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "abstract",
     "version": "published",
     "checked_on": "2026-11-13",
     "lab_event": "SC15"
    }
   }
  },
  {
   "id": "B14",
   "target": "10.1007/s10994-009-5119-5",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "abstract",
     "version": "published",
     "checked_on": "2026-11-13",
     "lab_event": "SC16"
    }
   }
  },
  {
   "id": "B15",
   "target": "10.1016/j.ijforecast.2004.10.003",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC17"
    }
   }
  },
  {
   "id": "B16",
   "target": "10.1080/07350015.1995.10524599",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC18"
    }
   }
  },
  {
   "id": "B17",
   "target": "10.1016/j.ijforecast.2022.08.003",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC19"
    }
   }
  },
  {
   "id": "B18",
   "target": "10.1016/j.ijforecast.2019.04.014",
   "id_type": "doi",
   "ext": {
    "review": {
     "read": "full_text",
     "checked_on": "2026-11-13",
     "lab_event": "SC20"
    }
   }
  }
 ],
 "extensions": {
  "review": "0.1"
 }
}
