{
 "bundle_of": "C7",
 "schema_version": "0.44",
 "paper": {
  "id": "entrylib:2026.000010",
  "version": 1,
  "title": "Online Detection of Forecast Model Inadequacies Using Forecast Errors",
  "licence": "CC-BY-4.0",
  "status": "active",
  "date": "2026-11-16"
 },
 "claims": [
  {
   "id": "C7",
   "statement": "On [[D20:the GRP admissions series]], both detectors of the [[D1:forecast error monitoring framework]] have a shorter [[D22:detection delay]] than [[D21:model-based sequential changepoint detection]], whose minimum possible detection delay on data with yearly seasonality is 24 months.",
   "type": "descriptive",
   "role": "principal",
   "about": {
    "subject": "D1",
    "relation": "has a shorter detection delay than",
    "object": "D22",
    "comparator": "D21",
    "conditions": [
     "D20"
    ]
   },
   "novelty": "no_prior_found",
   "planning": "exploratory",
   "scope": {
    "setting": "A&E admissions in participating hospitals",
    "geography": "England",
    "data_type": "monthly proportions with yearly seasonality"
   },
   "support": [
    {
     "kind": "argument",
     "warrant": "W2"
    }
   ],
   "depends_on": [
    "C13",
    "C14"
   ],
   "warrant": "W2",
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan was recorded beforehand. The model-based delay is a bound argued from the model's seasonality, not a run of a model-based detector on the series."
    }
   ],
   "source_spans": [
    "p3.2",
    "p11.5",
    "p11.7",
    "p11.9"
   ],
   "provenance": "author_stated",
   "relations": [
    {
     "type": "addresses",
     "reference": "B4",
     "self": true,
     "note": "Taib et al. (2021) found the changepoint retrospectively, with all the data from 2010 to 2020; the claim is about detecting it sequentially."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C13",
   "statement": "On [[D20:the GRP admissions series]], the [[D22:detection delay]] of the [[D2:mean detector]] was 4 data points, counted from the changepoint that the retrospective analysis of Taib et al. (2021) identified.",
   "type": "descriptive",
   "role": "supporting",
   "about": {
    "subject": "D2",
    "relation": "signals the changepoint after",
    "object": "D22",
    "comparator": null,
    "conditions": [
     "D20"
    ]
   },
   "novelty": "not_applicable",
   "planning": "exploratory",
   "scope": {
    "setting": "A&E admissions in participating hospitals",
    "geography": "England",
    "period": "2010 to 2020",
    "data_type": "monthly proportions"
   },
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E10",
     "artefacts": [
      "A6"
     ],
     "results": [
      "E10.R1"
     ]
    }
   ],
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "One series with 36 training observations (L6), against a changepoint taken from an earlier retrospective analysis."
    }
   ],
   "source_spans": [
    "p11.3",
    "p11.6",
    "p11.7"
   ],
   "provenance": "author_stated",
   "verifiability": "unverifiable",
   "verifiability_flags": []
  },
  {
   "id": "C14",
   "statement": "On [[D20:the GRP admissions series]], the [[D22:detection delay]] of the [[D3:squared-error detector]] was 2 time points.",
   "type": "descriptive",
   "role": "supporting",
   "about": {
    "subject": "D3",
    "relation": "signals the changepoint after",
    "object": "D22",
    "comparator": null,
    "conditions": [
     "D20"
    ]
   },
   "novelty": "not_applicable",
   "planning": "exploratory",
   "scope": {
    "setting": "A&E admissions in participating hospitals",
    "geography": "England",
    "period": "2010 to 2020",
    "data_type": "monthly proportions"
   },
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E10",
     "artefacts": [
      "A6"
     ],
     "results": [
      "E10.R2"
     ]
    }
   ],
   "limitations": [
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "One series with 36 training observations (L6)."
    }
   ],
   "source_spans": [
    "p11.3",
    "p11.6",
    "p11.7"
   ],
   "provenance": "author_stated",
   "verifiability": "unverifiable",
   "verifiability_flags": []
  }
 ],
 "definitions": [
  {
   "id": "D1",
   "term": "forecast error monitoring framework",
   "kind": "method",
   "definition": "A framework that monitors a forecasting model online by applying sequential changepoint detection to its one-step-ahead forecast errors. For a univariate series $Y_t$ and a forecasting model $\\mathcal{F}$ with one-step-ahead point forecast $\\hat{y}_t(1) = \\mathbb{E}_{\\mathcal{F}}[Y_t | y_{t-1}, \\ldots, y_1]$, the forecast errors are $$e_t = Y_t - \\hat{y}_t(1).$$ The first $m$ forecast errors are the Phase I (training) sample; monitoring starts at $t = m + 1$ and stops when a detector, [[D2:the mean detector]] or [[D3:the squared-error detector]], crosses its threshold. Any forecasting approach can supply the errors: model-based, machine learning or expert judgement.",
   "source_spans": [
    "p2.4",
    "p3.3",
    "p3.4",
    "p3.5"
   ]
  },
  {
   "id": "D2",
   "term": "mean detector $D_\\mu(m,k)$",
   "kind": "method",
   "definition": "[[D24:Page's CUSUM detector]] applied to the forecast errors of the [[D1:forecast error monitoring framework]]: $$\\begin{aligned} Q_\\mu(m,k) &= \\sum_{t=m+1}^{m+k} e_t - \\frac{k}{m} \\sum_{t=1}^{m} e_t \\\\ D_\\mu(m,k) &= \\max_{0 \\le i \\le k} \\left| Q_\\mu(m,k) - Q_\\mu(m,i) \\right| \\end{aligned}$$ with stopping time $$\\tau_{\\mu,m} = \\min\\{k \\ge 1 : D_\\mu(m,k) \\ge \\hat{\\sigma}_{\\mu,m} c_{\\mu,\\alpha} g(m,k,\\gamma)\\},$$ where $\\hat{\\sigma}_{\\mu,m}$ is the standard deviation of the forecast errors in the Phase I sample. The weight function $g(m,k,\\gamma)$ is that of [[D24:Page's CUSUM detector]], and the critical constant $c_{\\mu,\\alpha}$ comes from the limit distribution of the detector under the null hypothesis.",
   "part_of": [
    "D1"
   ],
   "source_spans": [
    "p4.5",
    "p4.7",
    "p4.9",
    "s2.6"
   ],
   "source": "B11"
  },
  {
   "id": "D3",
   "term": "squared-error detector $D_\\xi(m,k)$",
   "kind": "method",
   "definition": "[[D24:Page's CUSUM detector]] applied to the centred squared forecast errors of the [[D1:forecast error monitoring framework]]: $$\\begin{aligned} Q_\\xi(m,k) &= \\sum_{t=m+1}^{m+k} (e_t - \\hat{b}_{\\mu,m})^2 - \\frac{k}{m} \\sum_{t=1}^{m} (e_t - \\hat{b}_{\\mu,m})^2 \\\\ D_\\xi(m,k) &= \\max_{0 \\le i \\le k} \\left| Q_\\xi(m,k) - Q_\\xi(m,i) \\right| \\end{aligned}$$ with stopping time $$\\tau_{\\xi,m} = \\min\\{k \\ge 1 : D_\\xi(m,k) \\ge \\hat{\\sigma}_{\\xi,m} c_{\\xi,\\alpha} g(m,k,\\gamma)\\},$$ where $\\hat{b}_{\\mu,m}$ is the mean of the forecast errors in the $m$ Phase I samples and $\\hat{\\sigma}_{\\xi,m}$ the standard deviation of the centred squared forecast errors in that sample. $g(m,k,\\gamma)$ is that of [[D24:Page's CUSUM detector]] and $c_{\\xi,\\alpha}$ comes from the limit distribution of the detector under the null hypothesis. The paper prints the second sum of $Q_\\xi(m,k)$ without the square, a typographical error that the guarantor, an author of the paper, has confirmed.",
   "part_of": [
    "D1"
   ],
   "source_spans": [
    "p5.3",
    "p5.5",
    "p5.6"
   ]
  },
  {
   "id": "D20",
   "term": "GRP admissions series",
   "kind": "data",
   "definition": "The monthly proportion of A&E admissions related to gallstone-related pathology (GRP) in participating hospitals across England, 2010 to 2020, based on NHS Hospital Episode Statistics. A SARMA(1,0,0)(1,0,0)$_{12}$ model fitted to the data up to 2013 gives one-step-ahead forecasts for the rest; 2013 to 2016 is the Phase I sample of size $m = 36$ and monitoring starts in 2016.",
   "source": "B4",
   "source_spans": [
    "p9.14",
    "p11.5",
    "p11.6"
   ]
  },
  {
   "id": "D21",
   "term": "model-based sequential changepoint detection",
   "kind": "method",
   "definition": "Sequential changepoint detection that fits the model to the data before and after a potential changepoint and decides by a likelihood-ratio style statistic.",
   "source_spans": [
    "p3.2"
   ]
  },
  {
   "id": "D22",
   "term": "detection delay",
   "kind": "variable",
   "definition": "The number of observations between a change and the time a detector signals it.",
   "source_spans": [
    "p8.1"
   ]
  },
  {
   "id": "D24",
   "term": "Page's CUSUM detector $D(m,k)$",
   "kind": "method",
   "definition": "The sequential detector of a mean change in a series $Y_t$ that follows the change in mean model $$Y_t = \\begin{cases} \\lambda + \\epsilon_t, & t = 1, \\ldots, m + k^* \\\\ \\lambda + \\epsilon_t + \\Delta_m, & t = m + k^* + 1, m + k^* + 2, \\ldots \\end{cases}$$ with zero-mean variables $\\epsilon_t$, a training period of length $m$ without changepoints and a change of size $\\Delta_m$ at time $k^*$ after it. With $$\\begin{aligned} Q(m,k) &= \\sum_{t=m+1}^{m+k} Y_t - \\frac{k}{m} \\sum_{t=1}^{m} Y_t \\\\ D(m,k) &= \\max_{0 \\le i \\le k} \\left| Q(m,k) - Q(m,i) \\right| \\end{aligned}$$ the null hypothesis $H_0: \\Delta_m = 0$ is rejected in favour of $H_A: \\Delta_m \\ne 0$ at the stopping time $$\\tau_m = \\min\\{k \\ge 1 : D(m,k) \\ge \\hat{\\sigma}_m c_\\alpha g(m,k,\\gamma)\\},$$ where $\\hat{\\sigma}_m$ is a weakly consistent estimator, from the $m$ training samples, of the constant $\\sigma$ of the weak invariance principle, the weight function is $$g(m,k,\\gamma) = \\sqrt{m}\\left(1 + \\frac{k}{m}\\right)\\left(\\frac{k}{m+k}\\right)^{\\gamma}, \\quad \\gamma \\in [0, 1/2).$$ and the critical value $c_\\alpha$ is taken from the limit distribution under $H_0$, so that $\\lim_{m\\to\\infty} P(\\tau_m < \\infty) = \\alpha$ under $H_0$ for the chosen false alarm rate $\\alpha$ and $\\lim_{m\\to\\infty} P(\\tau_m < \\infty) = 1$ under $H_A$. All limits are for $m \\to \\infty$; the model is trained once and not re-trained during monitoring. The paper takes $\\gamma = 0$.",
   "source": "B10",
   "source_spans": [
    "s1.5",
    "s1.6-s1.7",
    "s2.3-s2.4",
    "s2.5",
    "s2.6",
    "s2.7-s2.10",
    "s3.1"
   ]
  }
 ],
 "evidence": [
  {
   "id": "E10",
   "profile": "quant",
   "description": "Both detectors applied sequentially to the one-step-ahead forecast errors of a SARMA(1,0,0)(1,0,0)$_{12}$ model of the monthly proportion of GRP A&E admissions, fitted on the data up to 2013, with 2013 to 2016 as the Phase I sample of size 36 and monitoring from 2016 (Section 6 and Figure 6 of the paper).",
   "setup": {
    "studied": [
     "D2",
     "D3"
    ],
    "cases": [
     "D20"
    ],
    "procedure": [
     "D1"
    ],
    "lens": [
     "D22"
    ]
   },
   "design": "other",
   "n": 36,
   "data_origin": "secondary",
   "preregistered": false,
   "artefacts": [
    "A6"
   ],
   "results": [
    {
     "id": "R1",
     "variable": "D22",
     "value": 4,
     "unit": "observations",
     "conditions": {
      "detector": "D2",
      "reference": "the changepoint identified by Taib et al. (2021)"
     }
    },
    {
     "id": "R2",
     "variable": "D22",
     "value": 2,
     "unit": "observations",
     "conditions": {
      "detector": "D3"
     }
    }
   ],
   "source_spans": [
    "p11.3",
    "p11.5",
    "p11.6",
    "p11.7"
   ],
   "availability": {
    "level": "unavailable",
    "reason": "other"
   }
  }
 ],
 "warrants": [
  {
   "id": "W2",
   "reasoning": "deductive",
   "premises": [
    "C13",
    "C14"
   ],
   "text": "A model-based sequential detector must fit the post-change model, here a SARMA(1,0,0)(1,0,0) model with a 12-month season, which the paper states needs at least two years of data after the change; the detectors on the forecast errors signalled four and two observations after it."
  }
 ],
 "limitations": [
  {
   "id": "L5",
   "category": "sample_size",
   "statement": "The application uses one monthly series with a Phase I sample of 36 observations.",
   "affects": [
    "C13",
    "C14",
    "C7"
   ],
   "impact": "reduces_transferability",
   "mitigation": "none",
   "source_spans": [
    "p11.6"
   ]
  }
 ],
 "artefacts": [
  {
   "id": "A6",
   "kind": "figure",
   "function": "data_plot",
   "reference": "B7",
   "locator": "Figure 6",
   "description": "The proportion of GRP A&E admissions, the forecast errors of a SARIMA(1,0,0)(1,0,0)12 model, and the two detectors with their thresholds.",
   "availability": {
    "level": "public"
   },
   "source_spans": [
    "p11.3"
   ],
   "shows": [
    "E10.R1",
    "E10.R2"
   ]
  }
 ],
 "references": [
  {
   "id": "B4",
   "target": "10.1016/j.hpb.2021.04.009",
   "id_type": "doi"
  },
  {
   "id": "B7",
   "target": "10.1111/jtsa.12843",
   "id_type": "doi"
  },
  {
   "id": "B10",
   "target": "10.2307/2171955",
   "id_type": "doi"
  },
  {
   "id": "B11",
   "target": "10.1093/biomet/41.1-2.100",
   "id_type": "doi"
  }
 ]
}
