{ "bundle_of": "C1", "schema_version": "0.40", "paper": { "id": "entrylib:2026.000001", "version": 3, "title": "Calibration of the prediction intervals of AutoMSARIMA in smooth on M1, M3, Tourism and M4", "licence": "CC-BY-4.0", "status": "corrected", "date": "2026-11-10", "status_notice": { "date": "2026-11-10", "reason_code": "correction", "statement": "The test quantities of the claims bound result entries that version 2 had turned into table cells; they now bind the cells, which keep the full precision the verdicts were computed at, and each table names the CSV of the lab results (A10) as its source. The defining equations of D1, D4, D11 and D13 are set as displayed equations.", "issued_by": "Ivan Svetunkov (ORCID 0000-0001-7826-0281)" } }, "claims": [ { "id": "C1", "statement": "With the true orders and true parameters, the [[D1:MSARIMA analytic interval]] is calibrated over the 99 quantiles in the [[D2:known-model simulation]] at $T$ of 100 and 300: [[D13:MCE]] lies between 0.0007 and 0.0024 and the ratio of the interval width to the theoretical width between 0.9965 and 1.0019, with backcasting and with optimal initial states.", "about": { "subject": "D1", "relation": "is calibrated, given the true orders and parameters, on", "object": "D13", "comparator": null, "conditions": [ "D2" ] }, "type": "descriptive", "role": "principal", "novelty": "no_prior_found", "support": [ { "kind": "simulation", "evidence": "E1", "results": [ "E1.T1.r3", "E1.T1.r2", "E1.T1.r1" ], "artefacts": [ "A3", "A8" ] } ], "provenance": "author_stated", "hypothesis": { "lab_event": "H1", "outcome": "supported" }, "planning": "confirmatory", "planned_at": "2026-11-05T11:37:01Z", "planned_at_anchor": "git_remote", "test": { "framework": "deterministic", "parameters": { "sample": "2000 replicates per cell, h=1..18, fixed seed 20261105, no interim analysis" }, "quantities": { "m_b100": { "variable": "D13", "conditions": { "T": 100, "D3": "T", "initial states": "backcasting" } }, "x_b100": { "variable": "largest MCE over the 11 processes", "conditions": { "T": 100, "D3": "T", "initial states": "backcasting" } }, "w_b100": { "variable": "ratio of the 80% interval width to the theoretical width", "conditions": { "T": 100, "D3": "T", "initial states": "backcasting" } }, "m_o100": { "variable": "D13", "conditions": { "T": 100, "D3": "T", "initial states": "optimal" } }, "x_o100": { "variable": "largest MCE over the 11 processes", "conditions": { "T": 100, "D3": "T", "initial states": "optimal" } }, "w_o100": { "variable": "ratio of the 80% interval width to the theoretical width", "conditions": { "T": 100, "D3": "T", "initial states": "optimal" } }, "m_b300": { "variable": "D13", "conditions": { "T": 300, "D3": "T", "initial states": "backcasting" } }, "x_b300": { "variable": "largest MCE over the 11 processes", "conditions": { "T": 300, "D3": "T", "initial states": "backcasting" } }, "w_b300": { "variable": "ratio of the 80% interval width to the theoretical width", "conditions": { "T": 300, "D3": "T", "initial states": "backcasting" } }, "m_o300": { "variable": "D13", "conditions": { "T": 300, "D3": "T", "initial states": "optimal" } }, "x_o300": { "variable": "largest MCE over the 11 processes", "conditions": { "T": 300, "D3": "T", "initial states": "optimal" } }, "w_o300": { "variable": "ratio of the 80% interval width to the theoretical width", "conditions": { "T": 300, "D3": "T", "initial states": "optimal" } } }, "supported": "m_b100<0.01 and x_b100<0.02 and abs(w_b100-1)<0.02 and m_o100<0.01 and x_o100<0.02 and abs(w_o100-1)<0.02 and m_b300<0.01 and x_b300<0.02 and abs(w_b300-1)<0.02 and m_o300<0.01 and x_o300<0.02 and abs(w_o300-1)<0.02", "not_supported": "m_b100>0.02 or x_b100>0.04 or abs(w_b100-1)>0.04 or m_o100>0.02 or x_o100>0.04 or abs(w_o100-1)>0.04 or m_b300>0.02 or x_b300>0.04 or abs(w_b300-1)>0.04 or m_o300>0.02 or x_o300>0.04 or abs(w_o300-1)>0.04", "verdict": "computed", "runs": 1, "lab_plan": "PL4" }, "depends_on": [ "C7", "C12" ], "scope": { "data_type": "simulated Gaussian ARIMA and SARIMA series", "conditions": [ "holdout of 18", "smooth 1.3.0 (Python)", "case T of the cases T, E, S, W", "T of 100 and 300" ] }, "assessments": [ { "by": "author", "strength": "high", "rationale": "2,000 replicates of each of 11 processes; the tolerance fixed in plan PL4 was met by every quantity." } ], "assumptions": [ "C24" ], "relations": [ { "type": "addresses", "reference": "B6", "note": "The book derives the interval; this checks the implementation of smooth 1.3.0 against it." } ], "verifiability": "open", "verifiability_flags": [] }, { "id": "C7", "statement": "With estimated parameters in the [[D2:known-model simulation]], the [[D1:MSARIMA analytic interval]] and the simulated interval of the same fit have the same width and [[D13:MCE]] at $T$ of 50, 100 and 300 (widths 0.957 and 0.957, 0.981 and 0.980, 0.997 and 0.996; MCE 0.0196 and 0.0197, 0.0100 and 0.0100, 0.0026 and 0.0027).", "about": { "subject": "D1", "relation": "has the same width and calibration as the simulated interval in", "object": "D13", "comparator": "simulated interval", "conditions": [ "D2" ] }, "type": "descriptive", "role": "supporting", "novelty": "no_prior_found", "support": [ { "kind": "simulation", "evidence": "E3", "results": [ "E3.T1.r1", "E3.T1.r2", "E3.T1.r3", "E3.T1.r4", "E3.T1.r5", "E3.T1.r6" ], "artefacts": [ "A8" ] } ], "provenance": "author_stated", "hypothesis": { "lab_event": "H3", "outcome": "supported" }, "planning": "confirmatory", "planned_at": "2026-11-05T11:37:02Z", "planned_at_anchor": "git_remote", "test": { "framework": "deterministic", "parameters": { "sample": "2000 replicates per cell, h=1..18, fixed seed 20261105, no interim analysis" }, "quantities": { "wa50": { "variable": "ratio of the 80% interval width to the theoretical width", "conditions": { "T": 50, "D3": "E", "initial states": "backcasting", "interval": "analytic" } }, "ws50": { "variable": "ratio of the 80% interval width to the theoretical width", "conditions": { "T": 50, "D3": "E", "initial states": "backcasting", "interval": "simulated interval" } }, "ma50": { "variable": "D13", "conditions": { "T": 50, "D3": "E", "initial states": "backcasting", "interval": "analytic" } }, "ms50": { "variable": "D13", "conditions": { "T": 50, "D3": "E", "initial states": "backcasting", "interval": "simulated interval" } }, "wa100": { "variable": "ratio of the 80% interval width to the theoretical width", "conditions": { "T": 100, "D3": "E", "initial states": "backcasting", "interval": "analytic" } }, "ws100": { "variable": "ratio of the 80% interval width to the theoretical width", "conditions": { "T": 100, "D3": "E", "initial states": "backcasting", "interval": "simulated interval" } }, "ma100": { "variable": "D13", "conditions": { "T": 100, "D3": "E", "initial states": "backcasting", "interval": "analytic" } }, "ms100": { "variable": "D13", "conditions": { "T": 100, "D3": "E", "initial states": "backcasting", "interval": "simulated interval" } }, "wa300": { "variable": "ratio of the 80% interval width to the theoretical width", "conditions": { "T": 300, "D3": "E", "initial states": "backcasting", "interval": "analytic" } }, "ws300": { "variable": "ratio of the 80% interval width to the theoretical width", "conditions": { "T": 300, "D3": "E", "initial states": "backcasting", "interval": "simulated interval" } }, "ma300": { "variable": "D13", "conditions": { "T": 300, "D3": "E", "initial states": "backcasting", "interval": "analytic" } }, "ms300": { "variable": "D13", "conditions": { "T": 300, "D3": "E", "initial states": "backcasting", "interval": "simulated interval" } } }, "supported": "abs(ws50/wa50-1)<0.01 and abs(ms50-ma50)<0.003 and abs(ws100/wa100-1)<0.01 and abs(ms100-ma100)<0.003 and abs(ws300/wa300-1)<0.01 and abs(ms300-ma300)<0.003", "not_supported": "abs(ws50/wa50-1)>0.03 or abs(ms50-ma50)>0.01 or abs(ws100/wa100-1)>0.03 or abs(ms100-ma100)>0.01 or abs(ws300/wa300-1)>0.03 or abs(ms300-ma300)>0.01", "verdict": "computed", "runs": 1, "lab_plan": "PL6" }, "scope": { "data_type": "simulated Gaussian ARIMA and SARIMA series", "conditions": [ "holdout of 18", "smooth 1.3.0 (Python)", "case E", "backcasting" ] }, "assessments": [ { "by": "author", "strength": "high", "rationale": "Paired on the same fits, 2,000 replicates per process; the rule of plan PL6 was met at each length." } ], "relations": [ { "type": "addresses", "reference": "B6", "note": "Checks that the analytic variance of the implementation matches simulation from the same model." } ], "verifiability": "open", "verifiability_flags": [] }, { "id": "C12", "type": "descriptive", "role": "supporting", "statement": "R and Python give the same [[D10:AutoMSARIMA]] interval on 500 [[D5:benchmark]] series: the median ratio of the 80% widths is 1.000, the same model is selected on every series and both have an 80% [[D4:coverage]] of 0.677.", "about": { "subject": "D10", "relation": "gives the same interval in R and Python on", "object": "D4", "comparator": null, "conditions": [ "D5" ] }, "support": [ { "kind": "empirical_quant", "evidence": "E18", "results": [ "E18.T1.r1" ] } ], "scope": { "data_type": "500 M1, M3 and Tourism series", "conditions": [ "backcasting" ] }, "novelty": "not_applicable", "assessments": [ { "by": "author", "strength": "moderate", "rationale": "Exploratory, a random sample of 500 series." } ], "planning": "exploratory", "provenance": "author_stated", "verifiability": "open", "verifiability_flags": [] }, { "id": "C24", "type": "descriptive", "role": "supporting", "statement": "The coverage of a set of series is well summarised by first averaging the holdout observations within each series and then averaging over series, all horizons together.", "support": [ { "kind": "assumption" } ], "assumption_category": "methodological", "check": "unchecked", "provenance": "author_stated", "verifiability": "not_applicable", "verifiability_flags": [] } ], "definitions": [ { "id": "D1", "term": "MSARIMA analytic interval", "kind": "method", "source": "B6", "definition": "The quantiles of `predict(interval=\"prediction\")` of a pure ARIMA in smooth 1.3.0, which dispatches to the approximate interval. The ARIMA is the state space model\n$$\\begin{aligned} y_t &= w^\\top v_{t-l} + \\epsilon_t, \\\\ v_t &= F v_{t-l} + g \\epsilon_t, \\\\ \\epsilon_t &\\sim N(0, \\sigma^2). \\end{aligned}$$\nThe quantile at level $\\tau$ and horizon $h$ is\n$$q_{\\tau,h} = \\hat y_{T+h|T} + \\Phi^{-1}(\\tau) \\sqrt{V_h}, \\qquad V_h = \\hat\\sigma^2 \\frac{T}{T-k} c_h,$$\nwith\n$$\\hat\\sigma^2 = \\frac{1}{T} \\sum_{t=1}^{T} e_t^2, \\qquad c_h = 1 + \\sum_{j=1}^{h-1} \\left(w^\\top F^{j-1} g\\right)^2,$$\nwhere $k$ is the number of estimated parameters and $c_h$ the $h$-th diagonal element of the analytic covariance of the state space model, written here for a model with lag 1." }, { "id": "D2", "term": "known-model simulation", "kind": "data", "definition": "Gaussian ARIMA series with $\\sigma = 10$ generated from 11 data generating processes (white noise with a constant, random walk with and without drift, AR(1), MA(1), ARIMA(0,1,1), ARIMA(1,1,0), ARIMA(1,1,1), ARIMA(0,2,2), seasonal SARIMA(0,1,1)(0,1,1)$_{12}$ and SARIMA(1,0,0)(1,0,0)$_{12}$) plus an IMA(1,1) with Student-t errors on 4 degrees of freedom; in-sample lengths $T$ of 15, 25, 50, 100 and 300, holdout of 18; 2,000 replicates per process and length with seed 20261105 unless stated." }, { "id": "D3", "term": "cases T, E, S, W", "kind": "design", "definition": "What the fitted AutoMSARIMA or MSARIMA knows about the data generating process of the [[D2:known-model simulation]]: T, true orders and true AR, MA and constant parameters with initial states and scale estimated; E, true orders with estimated parameters; S, orders selected by AutoMSARIMA; W, a fixed wrong model for each process. Each with backcasting or optimal initial states." }, { "id": "D4", "term": "coverage", "kind": "variable", "definition": "For the quantile $q_{\\tau,j}$ at level $\\tau$ and horizon $j$, the share of holdout observations at or below it,\n$$c_\\tau = \\frac{1}{h} \\sum_{j=1}^{h} 1\\left(y_{T+j} \\le q_{\\tau,j}\\right),$$\naveraged first within a series and then over series. cov80 is $c_{0.9} - c_{0.1}$, the coverage of the central 80% interval; q50 is $c_{0.5}$." }, { "id": "D5", "term": "benchmark", "kind": "data", "definition": "The 5,315 series of M1 (1,001), M3 (3,003) and Tourism (1,311) from fcompdata 0.1.4, with their own holdouts and seasonal periods; lags $(1, m)$ for seasonal series.", "members": [ "D18", "D19", "D20" ] }, { "id": "D6", "term": "M4 sample", "kind": "data", "source": "B4", "definition": "1,500 series each of the yearly, quarterly and monthly M4 series, drawn with seed 20261105 without replacement, with the M4 holdouts." }, { "id": "D7", "term": "level terciles", "kind": "design", "definition": "Series split into terciles of the [[D17:level ratio]] across the series of a data set; cov80 reported for each tercile, and the spread as the largest minus the smallest of the three." }, { "id": "D8", "term": "auto.adam error selection", "kind": "method", "source": "B6", "definition": "R `auto.adam(y, model=\"NNN\", lags, distribution=c(\"dnorm\",\"dlnorm\"), initial=\"backcasting\")` of smooth 4.6: the ARIMA orders and constant are selected by AICc for each distribution with the maxima of auto.msarima (AR (3,3), I (2,1), MA (3,3)), then the distribution with the lower AICc is kept; with dnorm the model is the [[D10:AutoMSARIMA]] model, with dlnorm it is the [[D11:log-normal ARIMA]]. Quantiles from `forecast(interval=\"prediction\")`." }, { "id": "D9", "term": "fresh M4 sample", "kind": "data", "source": "B4", "definition": "1,500 series each of the yearly, quarterly and monthly M4 series, drawn with seed 20261107 from the series not in the [[D6:M4 sample]], so disjoint from it." }, { "id": "D10", "term": "AutoMSARIMA", "kind": "method", "source": "B6", "definition": "`AutoMSARIMA(lags, initial)` in Python and `auto.msarima()` in R of smooth 1.3.0 / 4.6: ARIMA orders up to AR (3,3), I (2,1), MA (3,3) and a constant selected by AICc, normal errors, the [[D1:MSARIMA analytic interval]] for the quantiles. Backcasting initial states unless stated." }, { "id": "D11", "term": "log-normal ARIMA", "kind": "method", "source": "B6", "definition": "ADAM with `model=\"NNN\"`, an ARIMA and `distribution=\"dlnorm\"`: the ARIMA applies to the logarithms,\n$$\\begin{aligned} \\log y_t &= w^\\top v_{t-l} + \\log(1+\\epsilon_t), \\\\ v_t &= F v_{t-l} + g \\log(1+\\epsilon_t), \\\\ 1+\\epsilon_t &\\sim \\mathrm{logN}(-\\sigma^2/2, \\sigma^2), \\end{aligned}$$\nso the scale of the error is proportional to the level. In the comparisons with fixed orders, the orders and constant are those [[D10:AutoMSARIMA]] selects on the same series." }, { "id": "D13", "term": "MCE", "kind": "variable", "definition": "Mean calibration error over the 99 quantile levels $\\tau = 0.01, \\dots, 0.99$:\n$$\\mathrm{MCE} = \\frac{1}{99} \\sum_{\\tau} \\left|\\bar c_\\tau - \\tau\\right|,$$\nwith $\\bar c_\\tau$ the [[D4:coverage]] of $q_\\tau$." }, { "id": "D15", "term": "ETS forced and selected", "kind": "method", "source": "B6", "definition": "ETS of smooth with backcasting: AutoADAM-style selection among ETS models (`ES(model=\"ZXZ\")`, or ZXN for non-seasonal series), or ETS(ANN) or ETS(MNN) fitted to every series." }, { "id": "D17", "term": "level ratio", "kind": "variable", "definition": "Mean of the last third of the in-sample data of a series divided by the mean of all its in-sample data." }, { "id": "D18", "term": "M1", "kind": "data", "source": "B1", "definition": "The 1,001 series of the M1 competition with their holdouts." }, { "id": "D19", "term": "M3", "kind": "data", "source": "B2", "definition": "The 3,003 series of the M3 competition with their holdouts." }, { "id": "D20", "term": "Tourism", "kind": "data", "source": "B3", "definition": "The 1,311 series of the tourism forecasting competition with their holdouts." }, { "id": "D21", "term": "smooth 1.0.8 and 1.3.0", "kind": "setting", "definition": "Python smooth at tag v1.0.8-py (a separate worktree with its own environment) and at 1.3.0 (commit 634657d8 of openforecast-org/smooth)." } ], "evidence": [ { "id": "E1", "profile": "simulation", "description": "Simulation of the [[D2:known-model simulation]]: in each replicate the model is fitted to the first T observations and its 99 quantiles (0.01 to 0.99) are scored on the 18 holdout observations; results are pooled over the 11 Gaussian processes (mean over the processes of the mean over replicates). Case T of [[D3:cases T, E, S, W]] with backcasting and optimal initial states (lab run RN6, plan PL4). Width is the ratio of the 80% interval width to the theoretical $h$-step width; maxcellmce is the largest MCE over the processes.", "availability": { "level": "public" }, "lab_runs": [ "RN6" ], "setup": { "studied": [ "D1" ], "cases": [ "D2" ], "procedure": [ "D3" ], "lens": [ "D4", "D13" ] }, "execution": { "code_version": "126077c66b2a87a60c6ac02415c343421b11ac94", "software": { "python": "3.14.4", "platform": "Linux-7.0.0-42-generic-x86_64-with-glibc2.43" } }, "model": "The 11 Gaussian ARIMA and SARIMA processes and the IMA(1,1) with t(4) errors of the known-model simulation", "parameters": "A8", "runs": 2000, "seeds": [ 20261105 ], "tables": [ { "id": "T1", "description": "Case T: the true orders with the true parameters, pooled over the 11 Gaussian processes", "row_labels": [ "T", "D3", "initial states" ], "columns": [ { "variable": "D4", "conditions": { "nominal level": 0.8 } }, "ratio of the 80% interval width to the theoretical width", "D13", "largest MCE over the 11 processes" ], "rows": [ { "id": "r1", "label": [ 100, "T", "backcasting" ], "values": [ 0.7997, 1.0018975430750083, 0.002347566574839311, 0.014508136924803598 ] }, { "id": "r2", "label": [ 100, "T", "optimal" ], "values": [ 0.7972, 0.9965079650994704, 0.002378711355984074, 0.014505611672278346 ] }, { "id": "r3", "label": [ 300, "T", "backcasting" ], "values": [ null, 1.000745813821255, 0.0006805683093562026, 0.0058672839506172785 ] }, { "id": "r4", "label": [ 300, "T", "optimal" ], "values": [ null, 0.999189350813739, 0.0008571064177124785, 0.0058672839506172785 ] } ], "artefact": "A10" } ], "artefacts": [ "A10" ] }, { "id": "E3", "profile": "simulation", "description": "Simulation of the [[D2:known-model simulation]]: in each replicate the model is fitted to the first T observations and its 99 quantiles (0.01 to 0.99) are scored on the 18 holdout observations; results are pooled over the 11 Gaussian processes (mean over the processes of the mean over replicates). Case E with the [[D1:MSARIMA analytic interval]] against the simulated interval of the same fit (`interval=\"simulated\"`, 10,000 paths) (run RN7, plan PL6).", "availability": { "level": "public" }, "lab_runs": [ "RN7" ], "setup": { "studied": [ "D1" ], "cases": [ "D2" ], "procedure": [ "D3" ], "lens": [ "D4", "D13" ] }, "execution": { "code_version": "126077c66b2a87a60c6ac02415c343421b11ac94", "software": { "python": "3.14.4", "platform": "Linux-7.0.0-42-generic-x86_64-with-glibc2.43" } }, "model": "The 11 Gaussian ARIMA and SARIMA processes and the IMA(1,1) with t(4) errors of the known-model simulation", "parameters": "A8", "runs": 2000, "seeds": [ 20261105 ], "tables": [ { "id": "T1", "description": "Case E: the analytic interval against the simulated interval of the same fit, pooled over the 11 Gaussian processes", "row_labels": [ "T", "D3", "initial states", "interval" ], "columns": [ "ratio of the 80% interval width to the theoretical width", "D13" ], "rows": [ { "id": "r1", "label": [ 50, "E", "backcasting", "analytic" ], "values": [ 0.9571865764457477, 0.019618406285072953 ] }, { "id": "r2", "label": [ 50, "E", "backcasting", "simulated interval" ], "values": [ 0.9570246020280071, 0.0196577134986226 ] }, { "id": "r3", "label": [ 100, "E", "backcasting", "analytic" ], "values": [ 0.98068173371577, 0.00999872461993675 ] }, { "id": "r4", "label": [ 100, "E", "backcasting", "simulated interval" ], "values": [ 0.9804924240470849, 0.010021553923069082 ] }, { "id": "r5", "label": [ 300, "E", "backcasting", "analytic" ], "values": [ 0.9966012447878664, 0.0026437863483317944 ] }, { "id": "r6", "label": [ 300, "E", "backcasting", "simulated interval" ], "values": [ 0.9964067752797617, 0.002688526680950923 ] } ], "artefact": "A10" } ], "artefacts": [ "A10" ] }, { "id": "E18", "profile": "quant", "design": "benchmark", "description": "500 [[D5:benchmark]] series drawn with seed 41: R msarima with the orders Python selected, and R auto.msarima against Python [[D10:AutoMSARIMA]], backcasting; ratio of the 80% interval widths and coverage (exploratory run RN37).", "availability": { "level": "public" }, "lab_runs": [ "RN37" ], "setup": { "studied": [ "D10" ], "cases": [ "D5" ], "lens": [ "D4" ] }, "execution": { "code_version": "90d726121ac928d8cfc9ad77a0b60d99fa28334b", "software": { "python": "3.14.4", "platform": "Linux-7.0.0-42-generic-x86_64-with-glibc2.43" } }, "tables": [ { "id": "T1", "description": "R against Python AutoMSARIMA on 500 benchmark series drawn with seed 41, backcasting", "row_labels": [], "columns": [ "median ratio of the R to the Python 80% interval width", "share of series where R and Python select the same model", { "variable": "D4", "conditions": { "nominal level": 0.8, "implementation": "R" } }, { "variable": "D4", "conditions": { "nominal level": 0.8, "implementation": "Python" } } ], "rows": [ { "id": "r1", "label": [], "values": [ 1.0, 1.0, 0.6766, 0.6766 ] } ], "artefact": "A10" } ], "artefacts": [ "A10" ] }, { "id": "E20", "profile": "quant", "design": "benchmark", "description": "MCE of each calibration curve of the 99 quantiles drawn from recorded outputs: the [[D2:known-model simulation]] (runs RN8, RN11, RN39), [[D21:smooth 1.0.8 and 1.3.0]] (RN16, RN19, RN43, RN44), the [[D5:benchmark]] (RN45 and the saved ETS benchmark) and the [[D9:fresh M4 sample]] (RN48), overall and by [[D7:level terciles]] (exploratory run RN49).", "availability": { "level": "public" }, "lab_runs": [ "RN49" ], "setup": { "studied": [ "D10", "D8", "D15" ], "cases": [ "D2", "D5", "D9" ], "procedure": [ "D7" ], "lens": [ "D13" ] }, "execution": { "code_version": "5ca96a0fb806fcabad975937952e4a1330c3759d", "software": { "python": "3.14.4", "platform": "Linux-7.0.0-42-generic-x86_64-with-glibc2.43" } }, "tables": [ { "id": "T1", "description": "MCE of each curve of figure A3", "row_labels": [ "panel", "line" ], "columns": [ "D13" ], "rows": [ { "id": "r1", "label": [ "T=15", "true parameters" ], "values": [ 0.01439 ] }, { "id": "r2", "label": [ "T=15", "estimated parameters" ], "values": [ 0.06533 ] }, { "id": "r3", "label": [ "T=15", "selected orders" ], "values": [ 0.05793 ] }, { "id": "r4", "label": [ "T=25", "true parameters" ], "values": [ 0.006361 ] }, { "id": "r5", "label": [ "T=25", "estimated parameters" ], "values": [ 0.03414 ] }, { "id": "r6", "label": [ "T=25", "selected orders" ], "values": [ 0.03659 ] }, { "id": "r7", "label": [ "T=50", "true parameters" ], "values": [ 0.001008 ] }, { "id": "r8", "label": [ "T=50", "estimated parameters" ], "values": [ 0.01962 ] }, { "id": "r9", "label": [ "T=50", "selected orders" ], "values": [ 0.02353 ] }, { "id": "r10", "label": [ "T=100", "true parameters" ], "values": [ 0.002348 ] }, { "id": "r11", "label": [ "T=100", "estimated parameters" ], "values": [ 0.009999 ] }, { "id": "r12", "label": [ "T=100", "selected orders" ], "values": [ 0.01388 ] }, { "id": "r13", "label": [ "T=300", "true parameters" ], "values": [ 0.0006806 ] }, { "id": "r14", "label": [ "T=300", "estimated parameters" ], "values": [ 0.002644 ] }, { "id": "r15", "label": [ "T=300", "selected orders" ], "values": [ 0.004572 ] } ], "artefact": "A10" }, { "id": "T2", "description": "MCE of each curve of figure A4", "row_labels": [ "panel", "line" ], "columns": [ "D13" ], "rows": [ { "id": "r1", "label": [ "benchmark, backcasting", "1.0.8" ], "values": [ 0.03258 ] }, { "id": "r2", "label": [ "benchmark, backcasting", "1.3.0" ], "values": [ 0.04024 ] }, { "id": "r3", "label": [ "benchmark, optimal", "1.0.8" ], "values": [ 0.03312 ] }, { "id": "r4", "label": [ "benchmark, optimal", "1.3.0" ], "values": [ 0.03899 ] }, { "id": "r5", "label": [ "simulation T=100, true parameters", "1.0.8" ], "values": [ 0.01229 ] }, { "id": "r6", "label": [ "simulation T=100, true parameters", "1.3.0" ], "values": [ 0.002348 ] }, { "id": "r7", "label": [ "simulation T=100, estimated", "1.0.8" ], "values": [ 0.01202 ] }, { "id": "r8", "label": [ "simulation T=100, estimated", "1.3.0" ], "values": [ 0.009999 ] }, { "id": "r9", "label": [ "simulation T=100, selected", "1.0.8" ], "values": [ 0.01724 ] }, { "id": "r10", "label": [ "simulation T=100, selected", "1.3.0" ], "values": [ 0.01388 ] } ], "artefact": "A10" }, { "id": "T3", "description": "MCE of each curve of figure A5", "row_labels": [ "panel", "line" ], "columns": [ "D13" ], "rows": [ { "id": "r1", "label": [ "all series", "D10" ], "values": [ 0.04024 ] }, { "id": "r2", "label": [ "all series", "D8" ], "values": [ 0.01866 ] }, { "id": "r3", "label": [ "all series", "ETS (AutoADAM ZXZ, saved)" ], "values": [ 0.04424 ] }, { "id": "r4", "label": [ "low level tercile", "D10" ], "values": [ 0.0281 ] }, { "id": "r5", "label": [ "low level tercile", "D8" ], "values": [ 0.0334 ] }, { "id": "r6", "label": [ "mid level tercile", "D10" ], "values": [ 0.05052 ] }, { "id": "r7", "label": [ "mid level tercile", "D8" ], "values": [ 0.03523 ] }, { "id": "r8", "label": [ "high level tercile", "D10" ], "values": [ 0.0623 ] }, { "id": "r9", "label": [ "high level tercile", "D8" ], "values": [ 0.06547 ] }, { "id": "r10", "label": [ "auto.adam by selected distribution", "chose dnorm" ], "values": [ 0.04425 ] }, { "id": "r11", "label": [ "auto.adam by selected distribution", "chose dlnorm" ], "values": [ 0.02725 ] } ], "artefact": "A10" }, { "id": "T4", "description": "MCE of each curve of figure A6", "row_labels": [ "panel", "line" ], "columns": [ "D13" ], "rows": [ { "id": "r1", "label": [ "all series", "D10" ], "values": [ 0.0352 ] }, { "id": "r2", "label": [ "all series", "D8" ], "values": [ 0.02236 ] }, { "id": "r3", "label": [ "low level tercile", "D10" ], "values": [ 0.03481 ] }, { "id": "r4", "label": [ "low level tercile", "D8" ], "values": [ 0.04103 ] }, { "id": "r5", "label": [ "mid level tercile", "D10" ], "values": [ 0.04118 ] }, { "id": "r6", "label": [ "mid level tercile", "D8" ], "values": [ 0.02004 ] }, { "id": "r7", "label": [ "high level tercile", "D10" ], "values": [ 0.07568 ] }, { "id": "r8", "label": [ "high level tercile", "D8" ], "values": [ 0.1052 ] }, { "id": "r9", "label": [ "auto.adam by selected distribution", "chose dnorm" ], "values": [ 0.02093 ] }, { "id": "r10", "label": [ "auto.adam by selected distribution", "chose dlnorm" ], "values": [ 0.03506 ] } ], "artefact": "A10" } ], "artefacts": [ "A10" ] } ], "artefacts": [ { "id": "A3", "kind": "figure", "function": "data_plot", "uri": "https://entrylib.org/files/dd72a6d8d5df9b4d37007ce3543a8615bf419ca2f4e0f164b189da0d50b2754c", "version": "sha256:dd72a6d8d5df9b4d37007ce3543a8615bf419ca2f4e0f164b189da0d50b2754c", "licence": "CC-BY-4.0", "availability": { "level": "public" }, "lab_runs": [ "RN49" ], "shows": [ "E20.T1.r1", "E20.T1.r2", "E20.T1.r3", "E20.T1.r4", "E20.T1.r5", "E20.T1.r6", "E20.T1.r7", "E20.T1.r8", "E20.T1.r9", "E20.T1.r10", "E20.T1.r11", "E20.T1.r12", "E20.T1.r13", "E20.T1.r14", "E20.T1.r15" ], "encoding": { "x": "quantile level tau", "y": "D4", "colour": "method or case", "panel": "subset" }, "description": "Calibration curves of the 99 quantiles in the known-model simulation, cases T, E and S, at T = 15, 25, 50, 100, 300: coverage of each quantile (top) and coverage minus level (bottom)." }, { "id": "A4", "kind": "figure", "function": "data_plot", "uri": "https://entrylib.org/files/3eaf8ce7e065846578307cb0613c5aa18d9e4221afce5668046a829596051cc0", "version": "sha256:3eaf8ce7e065846578307cb0613c5aa18d9e4221afce5668046a829596051cc0", "licence": "CC-BY-4.0", "availability": { "level": "public" }, "lab_runs": [ "RN49" ], "shows": [ "E20.T2.r1", "E20.T2.r2", "E20.T2.r3", "E20.T2.r4", "E20.T2.r5", "E20.T2.r6", "E20.T2.r7", "E20.T2.r8", "E20.T2.r9", "E20.T2.r10" ], "encoding": { "x": "quantile level tau", "y": "D4", "colour": "method or case", "panel": "subset" }, "description": "Calibration curves of the 99 quantiles of AutoMSARIMA 1.0.8 and 1.3.0 on the benchmark (backcasting, optimal) and in the simulation at T = 100." }, { "id": "A5", "kind": "figure", "function": "data_plot", "uri": "https://entrylib.org/files/c401a330502b3600edab7be2fd2c2ddb67ef81e9b8e5a67ae98bad17af12b698", "version": "sha256:c401a330502b3600edab7be2fd2c2ddb67ef81e9b8e5a67ae98bad17af12b698", "licence": "CC-BY-4.0", "availability": { "level": "public" }, "lab_runs": [ "RN49" ], "shows": [ "E20.T3.r1", "E20.T3.r2", "E20.T3.r3", "E20.T3.r4", "E20.T3.r5", "E20.T3.r6", "E20.T3.r7", "E20.T3.r8", "E20.T3.r9", "E20.T3.r10", "E20.T3.r11" ], "encoding": { "x": "quantile level tau", "y": "D4", "colour": "method or case", "panel": "subset" }, "description": "Calibration curves of the 99 quantiles on the 5,315 benchmark series: auto.msarima, auto.adam with error selection and the selected ETS, overall, by level tercile and by the distribution auto.adam chose." }, { "id": "A6", "kind": "figure", "function": "data_plot", "uri": "https://entrylib.org/files/b45b8fcf0b27a2e0f13303cd57a7ba226c9ab84fa8f4fa08604e7ee048a24f2c", "version": "sha256:b45b8fcf0b27a2e0f13303cd57a7ba226c9ab84fa8f4fa08604e7ee048a24f2c", "licence": "CC-BY-4.0", "availability": { "level": "public" }, "lab_runs": [ "RN49" ], "shows": [ "E20.T4.r1", "E20.T4.r2", "E20.T4.r3", "E20.T4.r4", "E20.T4.r5", "E20.T4.r6", "E20.T4.r7", "E20.T4.r8", "E20.T4.r9", "E20.T4.r10" ], "encoding": { "x": "quantile level tau", "y": "D4", "colour": "method or case", "panel": "subset" }, "description": "Calibration curves of the 99 quantiles on the fresh M4 sample: auto.msarima and auto.adam with error selection, overall, by level tercile and by the distribution auto.adam chose." }, { "id": "A7", "kind": "lab_record", "uri": "https://github.com/openforecast-org/smooth/blob/619d23403fa35bfe2f51a4f16effb5d0d3833102/lab/lab.jsonl", "version": "sha256:9ec8a2276507e5d7662d670e016119112d54c7ef508f4a71736ecbbf5191a14c", "licence": "CC-BY-4.0", "availability": { "level": "public" }, "description": "Lab record of the study (branch claimset/msarima-interval-calibration): questions, hypotheses, plans, approvals, runs, results and notes as an append-only, hash-chained event log (lab record 0.20, claimset 0.39.0)." }, { "id": "A8", "kind": "code", "uri": "https://github.com/openforecast-org/smooth/tree/5ca96a0fb806fcabad975937952e4a1330c3759d/claimset-code/msarima-interval-calibration", "version": "5ca96a0fb806fcabad975937952e4a1330c3759d", "licence": "CC-BY-4.0", "availability": { "level": "public" }, "description": "Scripts of the study: simulation (q1_*), version decomposition (q2_*), benchmark analyses (q3_*), M4 (q4_m4.py), auto.adam in R and the calibration curves (q5_*)." }, { "id": "A10", "kind": "data", "uri": "https://entrylib.org/p/2026.000001/v2/files/files/lab-results.csv", "licence": "CC-BY-4.0", "availability": { "level": "public" }, "description": "Every result recorded in the lab record (A7), one row per result: the lab result ID, the run that produced it, the quantity as the analysis script names it, its conditions as JSON and its full-precision value. The tables of the evidence objects hold these values rounded to four significant digits and at least four decimals.", "version": "sha256:0e97428c7dc3643e413d3651fe832c8e4696d6ce12177d84bd2a062219f8539b" } ], "references": [ { "id": "B1", "target": "10.1002/for.3980010202", "id_type": "doi" }, { "id": "B2", "target": "10.1016/S0169-2070(00)00057-1", "id_type": "doi" }, { "id": "B3", "target": "10.1016/j.ijforecast.2010.04.009", "id_type": "doi" }, { "id": "B4", "target": "10.1016/j.ijforecast.2019.04.014", "id_type": "doi" }, { "id": "B6", "target": "10.1201/9781003452652", "id_type": "doi", "self": true } ] }