{
 "bundle_of": "C15",
 "schema_version": "0.44",
 "paper": {
  "id": "entrylib:2026.000013",
  "version": 3,
  "title": "What does sMAPE reward? An analytical and simulation study",
  "licence": "CC-BY-4.0",
  "status": "corrected",
  "date": "2026-11-18",
  "status_notice": {
   "date": "2026-11-18",
   "reason_code": "correction",
   "statement": "Version 3 answers a reader's comments. Corrected: C3 no longer says sMAPE is minimised by the median for small errors, only near it, with a bound; C4 keeps the weighted median, and C12 now holds what was said about the ordinary median; the measure called msMAPE in version 2 is the Suilin loss. Added: how well a test set tells two forecasts apart as the actuals spread out (C18); the large-sample distribution of the sMAPE minimiser (C19); Smyl's msMAPE (C20) and what it rewards on intermittent demand (C21). Also added: sections, the definition of sMAPE first, and a limitation on C6.",
   "issued_by": "Ivan Svetunkov (ORCID 0000-0001-7826-0281)"
  }
 },
 "claims": [
  {
   "id": "C15",
   "name": "sMAPE finds the median less precisely than the sample median",
   "summary": "Even where sMAPE and MAE both reward the median, the forecast that minimises sMAPE on a test set varies more than the sample median. The loss of precision grows as the data spread out.",
   "statement": "Where sMAPE and MAE both target the median ([[D26:log-symmetric outcomes]]), the [[D8:sMAPE minimiser of a test set]] varies more than the sample median for every spread and test-set size studied: its [[D9:relative efficiency]] is below 1 and falls as the actuals spread out, at $n = 10000$ from 0.97 to 0.03 on lognormal actuals as $\\sigma$ grows from 0.25 to 6, and from 1.00 to 0.34 on log-Laplace actuals as $b$ grows from 0.25 to 6, close to the large-sample values.",
   "type": "descriptive",
   "role": "principal",
   "novelty": "no_prior_found",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E15",
     "results": [
      "E15.T1.r1",
      "E15.T1.r8",
      "E15.T1.r9",
      "E15.T1.r16"
     ]
    }
   ],
   "provenance": "author_stated",
   "planning": "planned_estimate",
   "planned_at": "2026-11-16T13:46:20Z",
   "planned_at_anchor": "git_remote",
   "estimate": {
    "estimands": {
     "re": {
      "variable": "D9",
      "table": true
     },
     "asymptotic_re": {
      "variable": "asymptotic relative efficiency",
      "table": true
     },
     "var_median": {
      "variable": "variance of ln(sample median)",
      "table": true
     },
     "var_smape": {
      "variable": "variance of ln(sMAPE minimiser)",
      "table": true
     },
     "mse_ratio": {
      "variable": "ratio of mean squared errors about the true median",
      "table": true
     },
     "bias": {
      "variable": "mean of ln(sMAPE minimiser) minus ln(median)",
      "table": true
     }
    },
    "runs": 1,
    "lab_plan": "PL2",
    "flags": [
     "code_changed_after_plan"
    ]
   },
   "limitations": [
    "L2"
   ],
   "about": {
    "subject": "D8",
    "relation": "is less efficient than",
    "object": "D9",
    "comparator": "the sample median",
    "conditions": [
     "D26"
    ]
   },
   "scope": {
    "data_type": "test sets of independent draws from known distributions",
    "conditions": [
     "lognormal and log-Laplace outcomes, spread 0.25 to 6",
     "test sets of 6 to 10000 outcomes"
    ]
   },
   "relations": [
    {
     "type": "supports",
     "note": "C11 found sMAPE's choices in short test sets spreading more than MAE's; C15 measures the loss of precision for test sets of 6 to 10000 outcomes.",
     "reference": "B7"
    },
    {
     "type": "supports",
     "note": "C6 recommends against sMAPE for point forecasts; C15 shows that even where it targets the median it estimates it less precisely than the sample median.",
     "reference": "B6"
    },
    {
     "type": "supports",
     "reference": "B13",
     "note": "Kolassa (2026) concludes that sMAPE should not be used; C15 shows that even where it targets the median it estimates it less precisely than the sample median."
    },
    {
     "type": "qualifies",
     "target": "C12",
     "id_type": "local",
     "note": "Where C12 makes the median the target, the forecast that minimises sMAPE on a test set finds it less precisely than the sample median.",
     "results": [
      "E15.T1"
     ]
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  }
 ],
 "definitions": [
  {
   "id": "D6",
   "term": "sMAPE target",
   "kind": "concept",
   "definition": "The value or values of $f$ that minimise expected sMAPE for a given distribution of the actual $y$, $\\arg\\min_f \\mathrm E[s(y,f)]$: the functional sMAPE rewards. It may be a single value or several."
  },
  {
   "id": "D11",
   "term": "MAE",
   "kind": "variable",
   "definition": "Mean absolute error of a point forecast $f$ over a test set of actuals $y_1, \\dots, y_n$: $\\frac1n\\sum_t |y_t-f|$."
  },
  {
   "id": "D8",
   "term": "sMAPE minimiser of a test set",
   "kind": "variable",
   "definition": "The forecast that minimises the sMAPE of one test set $y_1, \\dots, y_n$, $\\hat f_n = \\arg\\min_f \\frac1n\\sum_t s(y_t, f)$, found numerically (the sample counterpart of the [[D6:sMAPE target]]).",
   "symbol": "$\\hat f_n$"
  },
  {
   "id": "D9",
   "term": "relative efficiency",
   "kind": "variable",
   "definition": "For a test-set size $n$ and an outcome distribution where sMAPE and MAE both target the median: $\\mathrm{RE}(n) = \\operatorname{Var}(\\ln \\hat m_n) / \\operatorname{Var}(\\ln \\hat f_n)$ over test sets, where $\\hat m_n$ is the sample median (the MAE minimiser) and $\\hat f_n$ the sMAPE minimiser of the same test set. Above 1, sMAPE's minimiser varies less; below 1, MAE's does.",
   "symbol": "$\\mathrm{RE}(n)$"
  },
  {
   "id": "D15",
   "term": "derived value",
   "kind": "concept",
   "definition": "A value computed from the outcome distribution itself by numerical integration of its density (Gauss-Legendre quadrature and a convolution on a grid of 524288 points of $\\ln y$), never from a sample: the expected scores, the local minima of expected sMAPE, the formulas of the hypotheses."
  },
  {
   "id": "D17",
   "term": "efficiency design",
   "kind": "method",
   "definition": "For each outcome distribution and test-set size $n$: draw test sets of $n$ independent outcomes; in each, compute the sample median and the exact sMAPE minimiser (an outcome of the test set, since the test set's sMAPE is concave in $\\ln f$ between consecutive outcomes); over test sets, compare the variances of their logarithms."
  },
  {
   "id": "D26",
   "term": "log-symmetric outcomes",
   "kind": "population",
   "definition": "Lognormal outcomes with $\\sigma$, and log-Laplace outcomes with scale $b$, each 0.25, 0.5, 1, 1.5, 2, 3, 4 or 6, median 100: $\\ln y$ symmetric and strictly unimodal, so that sMAPE and MAE both target the median (C5)."
  }
 ],
 "evidence": [
  {
   "id": "E15",
   "profile": "quant",
   "design": "other",
   "description": "Lab runs RN7 under plan PL2. Design: Efficiency design (D17) on 16 log-symmetric distributions (D26): lognormal sigma and log-Laplace b, each 0.25, 0.5, 1, 1.5, 2, 3, 4, 6. Test sets of n = 6, 8, 12, 13, 14, 18, 24, 36, 48, 100, 250, 500, 1000 and 10000 outcomes, 10000 test sets per size, seed 4 with the setting's index and n. In each test set: the sample median and the exact sMAPE minimiser (D8). From each distribution, derived (D15): the large-sample n Var of ln of each, from H11, and their ratio. python3 claimset-code/smape-target/efficiency.py --seed 4 --reps 10000 (numpy 2.5.3, matplotlib 3.11.2). Analysis: Per distribution and n, over test sets: the variances of ln(sample median) and ln(sMAPE minimiser); the relative efficiency (D9) with a 99% bootstrap percentile interval (2000 resamples of test sets); the ratio of their mean squared errors about the true median; the mean of ln(sMAPE minimiser) minus ln(median). Per distribution: the asymptotic relative efficiency. Figure: relative efficiency over n, a panel per distribution, with its band, the asymptotic value dashed and 1 dotted. Compared with H11 and H14 (iii); no test is made and no threshold is set.",
   "availability": {
    "level": "public"
   },
   "lab_runs": [
    "RN7"
   ],
   "results": [],
   "tables": [
    {
     "id": "T1",
     "description": "Relative efficiency of the sMAPE minimiser of a test set against the sample median, at test-set sizes 6, 12, 100, 1000 and 10000, beside its large-sample value",
     "row_labels": [
      "setting"
     ],
     "columns": [
      {
       "variable": "D9",
       "conditions": {
        "test-set size": 6
       }
      },
      {
       "variable": "D9",
       "conditions": {
        "test-set size": 12
       }
      },
      {
       "variable": "D9",
       "conditions": {
        "test-set size": 100
       }
      },
      {
       "variable": "D9",
       "conditions": {
        "test-set size": 1000
       }
      },
      {
       "variable": "D9",
       "conditions": {
        "test-set size": 10000
       }
      },
      "asymptotic relative efficiency"
     ],
     "rows": [
      {
       "id": "r1",
       "label": [
        "lognormal sigma=0.25"
       ],
       "values": [
        0.6059,
        0.7026,
        0.8996,
        0.9662,
        0.9701,
        0.9696
       ]
      },
      {
       "id": "r2",
       "label": [
        "lognormal sigma=0.5"
       ],
       "values": [
        0.611,
        0.7083,
        0.8802,
        0.885,
        0.8906,
        0.8881
       ]
      },
      {
       "id": "r3",
       "label": [
        "lognormal sigma=1"
       ],
       "values": [
        0.6067,
        0.6555,
        0.6524,
        0.6659,
        0.6565,
        0.6601
       ]
      },
      {
       "id": "r4",
       "label": [
        "lognormal sigma=1.5"
       ],
       "values": [
        0.5484,
        0.5123,
        0.4595,
        0.4633,
        0.45,
        0.454
       ]
      },
      {
       "id": "r5",
       "label": [
        "lognormal sigma=2"
       ],
       "values": [
        0.501,
        0.4392,
        0.323,
        0.3077,
        0.3043,
        0.3073
       ]
      },
      {
       "id": "r6",
       "label": [
        "lognormal sigma=3"
       ],
       "values": [
        0.4381,
        0.3646,
        0.2058,
        0.1557,
        0.1513,
        0.1474
       ]
      },
      {
       "id": "r7",
       "label": [
        "lognormal sigma=4"
       ],
       "values": [
        0.4148,
        0.3272,
        0.1561,
        0.08898,
        0.07892,
        0.07778
       ]
      },
      {
       "id": "r8",
       "label": [
        "lognormal sigma=6"
       ],
       "values": [
        0.3933,
        0.3027,
        0.1229,
        0.05076,
        0.03174,
        0.0279
       ]
      },
      {
       "id": "r9",
       "label": [
        "log-Laplace b=0.25"
       ],
       "values": [
        0.7049,
        0.8268,
        0.9769,
        0.9918,
        0.9969,
        0.9969
       ]
      },
      {
       "id": "r10",
       "label": [
        "log-Laplace b=0.5"
       ],
       "values": [
        0.6989,
        0.819,
        0.9448,
        0.9779,
        0.9753,
        0.9765
       ]
      },
      {
       "id": "r11",
       "label": [
        "log-Laplace b=1"
       ],
       "values": [
        0.667,
        0.6887,
        0.8479,
        0.8763,
        0.8853,
        0.8953
       ]
      },
      {
       "id": "r12",
       "label": [
        "log-Laplace b=1.5"
       ],
       "values": [
        0.5766,
        0.5947,
        0.6989,
        0.7789,
        0.7804,
        0.8035
       ]
      },
      {
       "id": "r13",
       "label": [
        "log-Laplace b=2"
       ],
       "values": [
        0.5179,
        0.5036,
        0.612,
        0.683,
        0.7079,
        0.7206
       ]
      },
      {
       "id": "r14",
       "label": [
        "log-Laplace b=3"
       ],
       "values": [
        0.4672,
        0.4365,
        0.4586,
        0.5257,
        0.5628,
        0.5902
       ]
      },
      {
       "id": "r15",
       "label": [
        "log-Laplace b=4"
       ],
       "values": [
        0.4353,
        0.3764,
        0.3496,
        0.4267,
        0.4803,
        0.4968
       ]
      },
      {
       "id": "r16",
       "label": [
        "log-Laplace b=6"
       ],
       "values": [
        0.4167,
        0.3492,
        0.2667,
        0.2896,
        0.3441,
        0.3753
       ]
      }
     ],
     "artefact": "A49"
    },
    {
     "id": "T2",
     "description": "Bias and variance at test-set sizes 12 and 1000, for spreads 0.25, 1 and 6: the mean of ln(sMAPE minimiser) minus ln(median), and the variances of ln(sample median) and ln(sMAPE minimiser); every setting and size is in the run's CSV",
     "row_labels": [
      "setting",
      "test-set size"
     ],
     "columns": [
      "mean of ln(sMAPE minimiser) minus ln(median)",
      "variance of ln(sample median)",
      "variance of ln(sMAPE minimiser)",
      "ratio of mean squared errors about the true median"
     ],
     "rows": [
      {
       "id": "r1",
       "label": [
        "lognormal sigma=0.25",
        12
       ],
       "values": [
        0.0008431,
        0.007228,
        0.01029,
        0.7026
       ]
      },
      {
       "id": "r2",
       "label": [
        "lognormal sigma=0.25",
        1000
       ],
       "values": [
        3.788e-05,
        9.802e-05,
        0.0001015,
        0.9662
       ]
      },
      {
       "id": "r3",
       "label": [
        "lognormal sigma=1",
        12
       ],
       "values": [
        0.0007662,
        0.1159,
        0.1768,
        0.6555
       ]
      },
      {
       "id": "r4",
       "label": [
        "lognormal sigma=1",
        1000
       ],
       "values": [
        9.852e-05,
        0.001587,
        0.002383,
        0.6659
       ]
      },
      {
       "id": "r5",
       "label": [
        "lognormal sigma=6",
        12
       ],
       "values": [
        0.002108,
        4.287,
        14.17,
        0.3027
       ]
      },
      {
       "id": "r6",
       "label": [
        "lognormal sigma=6",
        1000
       ],
       "values": [
        -0.02091,
        0.05714,
        1.126,
        0.05074
       ]
      },
      {
       "id": "r7",
       "label": [
        "log-Laplace b=0.25",
        12
       ],
       "values": [
        -0.0004367,
        0.007304,
        0.008834,
        0.8268
       ]
      },
      {
       "id": "r8",
       "label": [
        "log-Laplace b=0.25",
        1000
       ],
       "values": [
        0.0001574,
        6.678e-05,
        6.733e-05,
        0.9918
       ]
      },
      {
       "id": "r9",
       "label": [
        "log-Laplace b=1",
        12
       ],
       "values": [
        0.001393,
        0.1191,
        0.173,
        0.6887
       ]
      },
      {
       "id": "r10",
       "label": [
        "log-Laplace b=1",
        1000
       ],
       "values": [
        0.0003075,
        0.001055,
        0.001204,
        0.8764
       ]
      },
      {
       "id": "r11",
       "label": [
        "log-Laplace b=6",
        12
       ],
       "values": [
        0.02118,
        4.267,
        12.22,
        0.3492
       ]
      },
      {
       "id": "r12",
       "label": [
        "log-Laplace b=6",
        1000
       ],
       "values": [
        0.002973,
        0.03773,
        0.1303,
        0.2896
       ]
      }
     ],
     "artefact": "A49"
    }
   ],
   "artefacts": [
    "A6",
    "A49",
    "A50",
    "A51"
   ],
   "setup": {
    "studied": [
     "D8",
     "D11"
    ],
    "cases": [
     "D26"
    ],
    "procedure": [
     "D17",
     "D15"
    ],
    "lens": [
     "D9"
    ]
   },
   "execution": {
    "code_version": "fa0ce626f18bd1ca22076d62452d2b68c484a297",
    "software": {
     "python": "3.13.18",
     "platform": "Linux-6.18.44-fc-v470-x86_64-with-glibc2.39"
    }
   }
  }
 ],
 "limitations": [
  {
   "id": "L2",
   "category": "analytic_flexibility",
   "statement": "Lab run RN7 deviated from plan PL2: RN7 ran efficiency.py at commit fa0ce62, not the plan's 92cd9a5: the script runs its 16 settings in parallel processes (one per core) instead of one after another; theory.py also gained a function (balance_exact) that efficiency.py does not use. Each (setting, test-set size) draws from its own seed as before, and on a subset the parallel code and the plan's code gave byte-identical results.json. Reason: Three earlier attempts on one core (over an hour) were killed by container restarts before the lab recorded them (notes NT2.R2, NT5, NT6); in parallel the run finished in about 20 minutes. The analysis, the settings, the seeds and the results are unchanged.",
   "affects": [
    "C15"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  }
 ],
 "artefacts": [
  {
   "id": "A6",
   "kind": "data",
   "uri": "https://entrylib.org/p/2026.000013/v3/files/files/smape-target.results.csv",
   "version": "sha256:dbcda16de8d13b8449906fa1142665fc1efc4b472ebe082233f3e457c2eb65eb",
   "availability": {
    "level": "public"
   },
   "description": "Every result of the lab record (17182 results of 17 runs) at full precision: the lab result's ID, its run, the quantity as the run's script named it, its conditions and its value (both as JSON). The evidence's tables give the values to four significant digits.",
   "columns": [
    {
     "name": "lab_result",
     "dimension": "the result event of the lab record"
    },
    {
     "name": "run",
     "dimension": "the lab run that produced it"
    },
    {
     "name": "quantity",
     "dimension": "the quantity, as the run's script named it"
    },
    {
     "name": "conditions",
     "dimension": "the result's conditions, as JSON"
    },
    {
     "name": "value",
     "variable": "the value of the quantity, as JSON, at full precision"
    }
   ],
   "licence": "CC-BY-4.0"
  },
  {
   "kind": "data",
   "uri": "https://entrylib.org/p/2026.000013/v3/files/files/smape-target.RN7.results.csv",
   "version": "sha256:e8dfaaa211d9abab024d15860ada00b42ea3ebf2d5d68d55979d1070085a5bd5",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "columns": [
    {
     "name": "lab_result",
     "dimension": "the result event of the lab record"
    },
    {
     "name": "run",
     "dimension": "the lab run that produced it"
    },
    {
     "name": "quantity",
     "dimension": "the quantity, as the run's script named it"
    },
    {
     "name": "conditions",
     "dimension": "the result's conditions, as JSON"
    },
    {
     "name": "value",
     "variable": "the value of the quantity, as JSON, at full precision"
    }
   ],
   "description": "Every result of lab run RN7 (PL2: efficiency of the sMAPE minimiser against the sample median) at full precision, one row per result, sorted by setting: the lab result's ID, its run, the quantity, its conditions and its value (both as JSON). The evidence's tables give selected values to four significant digits.",
   "id": "A49"
  },
  {
   "kind": "figure",
   "function": "data_plot",
   "uri": "https://entrylib.org/p/2026.000013/v3/files/files/figures/RN7-efficiency_lognormal.png",
   "version": "sha256:51087c8b21b4feaa3c513d5c02b08e2e9005f4ce3bf60c84cfcb04ff5569002a",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "shows": [
    "A49",
    "E15.T1"
   ],
   "description": "lognormal outcomes: relative efficiency of the sMAPE minimiser of a test set against the sample median, against test-set size, one panel per spread, with its 99% band and the large-sample value as a dashed line; below 1 the sMAPE minimiser varies more than the sample median. Drawn by lab run RN7 from its own results (lab/runs/RN7/figures/efficiency_lognormal.png).",
   "id": "A50"
  },
  {
   "kind": "figure",
   "function": "data_plot",
   "uri": "https://entrylib.org/p/2026.000013/v3/files/files/figures/RN7-efficiency_log-Laplace.png",
   "version": "sha256:2fbd7b03699010e6a3d8ef724559f363df2897ba07e2321bd713f69b78f71c44",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "shows": [
    "A49",
    "E15.T1"
   ],
   "description": "log-Laplace outcomes: relative efficiency of the sMAPE minimiser of a test set against the sample median, against test-set size, one panel per spread, with its 99% band and the large-sample value as a dashed line; below 1 the sMAPE minimiser varies more than the sample median. Drawn by lab run RN7 from its own results (lab/runs/RN7/figures/efficiency_log-Laplace.png).",
   "id": "A51"
  }
 ]
}
