{
 "schema_version": "0.41",
 "paper": {
  "id": "entrylib:2026.000005",
  "correction_reason": "The platform classified the paper under the OpenAlex subfield Management Science and Operations Research, which is wrong: the paper studies AI agents doing research. This version proposes the subfield Artificial Intelligence and replaces the keywords agent evaluation and scoring rubric with metascience and research methodology; claims, evidence and every other field are unchanged.",
  "title": "Claude Sonnet and Claude Haiku agents keeping Claimset lab records and completing claimsets: a pilot on one simulation study",
  "licence": "CC-BY-4.0",
  "classification": {
   "keywords": [
    "AI research agents",
    "lab records",
    "pre-registration",
    "claimsets",
    "metascience",
    "research methodology",
    "research integrity"
   ],
   "subfield": "openalex:subfields/1702",
   "topics": [
    "openalex:T11918",
    "openalex:T11700",
    "openalex:T13309"
   ]
  },
  "declarations": {
   "funding": [],
   "competing_interests": "The guarantor owns EntryLib, which develops the claimset tools, skills and rubric the study evaluates; the study was done to improve them."
  },
  "summary": "Can smaller AI models run a research project with a Claimset lab record and write it up as a claimset? Two Claude Sonnet and two Claude Haiku agents each did one simulation study, comparing how well two Python packages estimate exponential smoothing parameters; four more agents completed a claimset from one of the resulting records. All four lab records passed the tool's own verification and all four claimsets were valid. The Sonnet agents planned their tests before seeing data; the Haiku agents did not. No claim was wrong, but no claimset stated the study's main findings: the lab format turns only pass-or-fail hypotheses into claims, so the agents tested thresholds they chose themselves instead of reporting estimates. EntryLib's scoring rubric ranked a claimset without planned tests, and one without limitations or scope, above more careful ones. These are exploratory results from two runs per model on one task.",
  "contributions": [
   "C1",
   "C2",
   "C3",
   "C4",
   "C5",
   "C6",
   "C7",
   "C8",
   "C9",
   "C10",
   "C11"
  ],
  "highlights": [
   "C8",
   "C10",
   "C11",
   "C3",
   "C7"
  ],
  "autonomy": "human_supervised",
  "contribution_record": "partial",
  "origin": "native",
  "version": 2,
  "date": "2026-11-12",
  "status": "corrected",
  "previous_version": "entrylib:2026.000005@v1",
  "status_notice": {
   "date": "2026-11-12",
   "reason_code": "correction",
   "statement": "The platform classified the paper under the OpenAlex subfield Management Science and Operations Research, which is wrong: the paper studies AI agents doing research. This version proposes the subfield Artificial Intelligence and replaces the keywords agent evaluation and scoring rubric with metascience and research methodology; claims, evidence and every other field are unchanged.",
   "issued_by": "Ivan Svetunkov (ORCID 0000-0001-7826-0281)"
  },
  "changes": [
   {
    "op": "replace",
    "path": "/paper/classification/keywords",
    "value": [
     "AI research agents",
     "lab records",
     "pre-registration",
     "claimsets",
     "metascience",
     "research methodology",
     "research integrity"
    ],
    "reason": "The platform classified the paper under the OpenAlex subfield Management Science and Operations Research, which is wrong: the paper studies AI agents doing research. This version proposes the subfield Artificial Intelligence and replaces the keywords agent evaluation and scoring rubric with metascience and research methodology; claims, evidence and every other field are unchanged."
   }
  ]
 },
 "contributors": [
  {
   "id": "P1",
   "type": "agent",
   "name": "Claude Code",
   "label": "orchestration and analysis",
   "system": {
    "name": "Claude Code",
    "version": "2.1.322"
   },
   "positions": [
    "author"
   ],
   "models": [
    {
     "model": "M1",
     "role": "orchestrator"
    }
   ],
   "credit_roles": [
    "methodology",
    "investigation",
    "formal_analysis",
    "software",
    "data_curation",
    "writing_original_draft"
   ],
   "ai_tasks": [
    {
     "task": "analysis",
     "extent": "full"
    },
    {
     "task": "code",
     "extent": "full"
    },
    {
     "task": "writing",
     "extent": "full"
    },
    {
     "task": "json_preparation",
     "extent": "full"
    },
    {
     "task": "literature_search",
     "extent": "assisted"
    }
   ],
   "tools": [
    "code_execution",
    "file_access",
    "web_search"
   ],
   "record_source": "harness"
  },
  {
   "id": "P2",
   "type": "human",
   "name": "Ivan Svetunkov",
   "orcid": "0000-0001-7826-0281",
   "positions": [
    "guarantor",
    "supervisor"
   ],
   "credit_roles": [
    "conceptualization",
    "supervision"
   ]
  }
 ],
 "models": [
  {
   "id": "M1",
   "provider": "Anthropic",
   "name": "Claude Opus",
   "version": "5.5",
   "identifier": "claude-opus-5-5",
   "weights": "api_only",
   "accessed": "2026-11-11"
  },
  {
   "id": "M2",
   "provider": "Anthropic",
   "name": "Claude Sonnet",
   "version": "5.5",
   "identifier": "claude-sonnet-5-5",
   "weights": "api_only",
   "accessed": "2026-11-11"
  },
  {
   "id": "M3",
   "provider": "Anthropic",
   "name": "Claude Haiku",
   "version": "5.5",
   "identifier": "claude-haiku-5-5",
   "weights": "api_only",
   "accessed": "2026-11-11"
  }
 ],
 "definitions": [
  {
   "id": "D1",
   "term": "claimset-lab",
   "kind": "method",
   "definition": "The claimset-lab skill (`.claude/skills/claimset-lab/SKILL.md`, installed by `claimset init --agent claude`) and the `claimset lab` commands of claimset 0.41.6, which keep a lab record (lab record specification v0.21): an append-only, hash-chained log of questions, hypotheses, plans, approvals, runs, results and verdicts, kept on a study branch and pushed after every command."
  },
  {
   "id": "D2",
   "term": "claimset-submit",
   "kind": "method",
   "definition": "The claimset-submit skill of claimset 0.41.6 (claimset specification v0.41), together with `claimset lab export`, `claimset validate` and `claimset render`: it turns a lab record into a claimset and completes it for submission."
  },
  {
   "id": "D3",
   "term": "Sonnet agent",
   "kind": "method",
   "definition": "A Claude Code 2.1.322 subagent started with the Agent tool (type general-purpose) on the model claude-sonnet-5-5, working in a folder of its own that holds the study brief, `AGENTS.md` and the claimset skills as files; it asks the human by ending its turn with its questions.",
   "part_of": [
    "D5"
   ]
  },
  {
   "id": "D4",
   "term": "Haiku agent",
   "kind": "method",
   "definition": "The same as the [[D3:Sonnet agent]], on the model claude-haiku-5-5.",
   "part_of": [
    "D5"
   ]
  },
  {
   "id": "D5",
   "term": "agents under test",
   "kind": "method",
   "definition": "The two agent configurations compared: the [[D3:Sonnet agent]] and the [[D4:Haiku agent]].",
   "members": [
    "D3",
    "D4"
   ]
  },
  {
   "id": "D6",
   "term": "ETS estimation brief",
   "kind": "setting",
   "definition": "The research task given to every agent (`study/brief.md`): compare how well smooth 1.2.0 (`ES`) and statsforecast 2.1.1 (`AutoETS` with a fixed model form) estimate the smoothing parameters of the ETS model that generated simulated data, by bias, RMSE and MAE and by whether the RMSE falls as the series length grows; five processes (ETS(A,N,N) with α 0.1 and 0.5, ETS(A,A,N) with (α, β) (0.3, 0.05) and (0.5, 0.2), ETS(A,N,A) with m = 12 and (α, γ) (0.2, 0.1)), series lengths 50, 100, 250 and 500, 300 replicates per cell, smooth's default backcasting as a separate arm, failures counted."
  },
  {
   "id": "D7",
   "term": "lab phase",
   "kind": "design",
   "definition": "Phase 1: four runs, two of the [[D3:Sonnet agent]] and two of the [[D4:Haiku agent]], each in its own git repository with a local remote, given the same task (\"Do the study in brief.md with claimset-lab, following its skill, from the first question to the finished record. Ask the human anything you need through me.\"). The orchestrating agent relays questions and answers them only from [[D9:the fixed answers]], gives no other help, and never shows an agent another run or [[D10:the answer key]]. Runs in parallel on one 4-core machine."
  },
  {
   "id": "D8",
   "term": "claimset phase",
   "kind": "design",
   "definition": "Phase 2: the lab record of run lab-sonnet-1 (the only one that verifies, planned before its data and fixed statsforecast's model form) copied to four folders, with the draft its agent had exported removed; two [[D3:Sonnet agent]] and two [[D4:Haiku agent]] runs, each told \"Export this lab record with `claimset lab export` and complete the claimset for submission, following the claimset-submit skill. Do not submit it.\" Questions answered as in [[D7:the lab phase]]."
  },
  {
   "id": "D9",
   "term": "fixed answers",
   "kind": "design",
   "definition": "The answers the orchestrating agent gives in place of the human, the same to every run (`study/human.md`): name and ORCID iD; licence CC-BY-4.0; funding none; the competing interest of the brief; \"Approved.\" for a plan that names its quantities and decision rule before the data are run, otherwise \"Please state the quantities and the decision rule first.\"; for highlights \"Choose up to three principal claims you consider most important.\"; for anything else \"Use your judgement and say what you decided.\"",
   "part_of": [
    "D7",
    "D8"
   ]
  },
  {
   "id": "D10",
   "term": "answer key",
   "kind": "data",
   "definition": "The findings a correct paper on [[D6:the brief]] would state, with their numbers (`reference/answer_key.md`), from a reference simulation of the full design written by the orchestrating agent before Phase 1 without claimset-lab (`reference/simulate.py`, `optcheck.py`, `damped_check.py`). Amended once during Phase 1 (statsforecast's β cap)."
  },
  {
   "id": "D11",
   "term": "record verifies",
   "kind": "variable",
   "definition": "Whether `claimset lab verify` reports the lab record OK at the end of a run (hash chain, rules, run files)."
  },
  {
   "id": "D12",
   "term": "tool refusals",
   "kind": "variable",
   "definition": "Number of commands that `claimset` refused in a run with a lab rule (L1–L30) or an argument or overwrite error; validator errors on a claimset are counted separately."
  },
  {
   "id": "D13",
   "term": "plans before data",
   "kind": "variable",
   "definition": "Whether each confirmatory run of a lab record ran under a plan with its decision rule recorded, and approved by the human, before the run, and whether its hypotheses were written before any run informed them: `all`, `all, hypotheses after pilots` or `none` (no plan)."
  },
  {
   "id": "D14",
   "term": "model form fixed",
   "kind": "variable",
   "definition": "Whether a run's simulation script fixed statsforecast's model form as the brief asks (`damped=False`); without it, `AutoETS(model=\"AAN\")` and `ets_f(..., model=\"AAN\")` choose between ETS(A,A,N) and ETS(A,Ad,N)."
  },
  {
   "id": "D15",
   "term": "convergence checked",
   "kind": "variable",
   "definition": "Whether a run's script read any optimiser status (for statsforecast, the iteration count `model_[\"fit\"].nit`) and counted fits that did not converge."
  },
  {
   "id": "D16",
   "term": "output tokens",
   "kind": "variable",
   "definition": "Model output tokens of an agent over its whole run, summed from the usage of every model call in its transcript.",
   "unit": "1"
  },
  {
   "id": "D17",
   "term": "active time",
   "kind": "variable",
   "definition": "Time an agent spent working in a run, excluding waits for answers, as reported by the harness.",
   "unit": "min"
  },
  {
   "id": "D18",
   "term": "record anchored",
   "kind": "variable",
   "definition": "Whether the agent anchored the finished record (`claimset lab anchor`)."
  },
  {
   "id": "D19",
   "term": "runs written as the human",
   "kind": "variable",
   "definition": "Number of run events an agent recorded with `--by` the human's actor ID."
  },
  {
   "id": "D20",
   "term": "claimset validates",
   "kind": "variable",
   "definition": "Whether `claimset validate` reports the final claimset valid (no errors)."
  },
  {
   "id": "D21",
   "term": "validator outcomes until valid",
   "kind": "variable",
   "definition": "The position, among the validator outcomes a claimset run saw (the export's own summary included), of the first with no errors.",
   "unit": "1"
  },
  {
   "id": "D22",
   "term": "warnings left",
   "kind": "variable",
   "definition": "Validator warnings on the final claimset.",
   "unit": "1"
  },
  {
   "id": "D23",
   "term": "EntryLib rubric 0.2",
   "kind": "method",
   "definition": "The scoring rubric EntryLib applies to published claimsets (entrylib-rubric 0.2.0), run on this machine with `claimset score` and without `--novelty`: each principal claim starts at 100 and loses fixed costs for its problems (support, plan, checkability, context, open limitations and assumptions); the paper's score is that of its lowest principal claim."
  },
  {
   "id": "D24",
   "term": "reliability score",
   "kind": "variable",
   "definition": "The paper score of a claimset under [[D23:EntryLib rubric 0.2]], 0–100.",
   "unit": "1"
  },
  {
   "id": "D25",
   "term": "claim correctness",
   "kind": "variable",
   "definition": "Each claim of a claimset checked against [[D10:the answer key]]: `correct` (direction and magnitude agree within Monte Carlo error), `partly correct` (direction agrees; magnitude, scope or a qualifier does not), `wrong`, or `not supported by the record`."
  },
  {
   "id": "D26",
   "term": "answer-key findings stated",
   "kind": "variable",
   "definition": "Number of the answer key's findings on bias (K2–K5), statsforecast's non-convergence (K1.2) and statsforecast's β cap (K8) that a claimset states as claims.",
   "unit": "1"
  },
  {
   "id": "D27",
   "term": "principal claims with scope",
   "kind": "variable",
   "definition": "Number of a claimset's principal claims that have a scope.",
   "unit": "1"
  },
  {
   "id": "D28",
   "term": "limitations",
   "kind": "variable",
   "definition": "Number of limitations a claimset records.",
   "unit": "1"
  },
  {
   "id": "D29",
   "term": "research agent's model kept",
   "kind": "variable",
   "definition": "Whether a completed claimset keeps the model of the agent that did the research, as the lab record names it."
  },
  {
   "id": "D30",
   "term": "interventions",
   "kind": "variable",
   "definition": "Number of messages the orchestrating agent sent a run that were not answers from [[D9:the fixed answers]].",
   "unit": "1"
  },
  {
   "id": "D31",
   "term": "reference simulation",
   "kind": "design",
   "definition": "The reference analysis behind [[D10:the answer key]]: 300 series per cell of [[D6:the brief]], each fitted by smooth with optimised initial states, smooth with backcasting and statsforecast `AutoETS(model, season_length, damped=False)` with its defaults (likelihood, bounds `both`), seeds `numpy.random.default_rng([20261111, process, T, replicate])`."
  },
  {
   "id": "D32",
   "term": "statsforecast β cap",
   "kind": "concept",
   "definition": "In statsforecast 2.1.1 (`statsforecast.ets.initparam`), when α is estimated its starting value is $$\\alpha_0 = 0.0001 + 0.2 \\cdot \\frac{0.9999 - 0.0001}{m}$$ (0.20006 for $m = 1$), and the upper bound of β is set to $\\min(0.9999, \\alpha_0)$ for the whole optimisation, so β cannot exceed $\\alpha_0$."
  },
  {
   "id": "D33",
   "term": "decision thresholds",
   "kind": "variable",
   "definition": "The thresholds a lab record's confirmatory plans fix in their decision rules, as the plans state them; `none` when a record has no plan."
  },
  {
   "id": "D34",
   "term": "hypotheses about an estimate",
   "kind": "variable",
   "definition": "Number of a lab record's hypotheses that state the direction or size of an estimate (a bias towards zero, an estimate on a bound, a bias of about −0.04) rather than a pass-or-fail threshold.",
   "unit": "1"
  },
  {
   "id": "D35",
   "term": "rubric cost",
   "kind": "variable",
   "definition": "Points that [[D23:EntryLib rubric 0.2]] deducts from a claim's score for one problem, as its file rubric.json states them.",
   "unit": "1"
  },
  {
   "id": "D36",
   "term": "reading of the rubric's code",
   "kind": "design",
   "definition": "The orchestrating agent read the costs in src/rubric/rubric.json and the rule that applies the plan check in src/rubric/score.py (lines 28 and 111), at the commit artefact A10 names."
  },
  {
   "id": "D37",
   "term": "statsforecast iteration cap",
   "kind": "variable",
   "definition": "Number of statsforecast fits whose optimiser stopped at its iteration cap (`nit` = 1001) in [[D31:the reference simulation]].",
   "unit": "1"
  }
 ],
 "claims": [
  {
   "id": "C1",
   "statement": "Every lab record that the [[D5:agents under test]] kept with [[D1:claimset-lab]] on [[D6:the ETS estimation brief]] passes `claimset lab verify` ([[D11:record verifies]]).",
   "type": "descriptive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E1",
     "results": [
      "E1.T1"
     ],
     "artefacts": [
      "A1"
     ]
    }
   ],
   "about": {
    "subject": "D5",
    "relation": "keep lab records that pass",
    "object": "D11",
    "conditions": [
     "D1",
     "D6"
    ],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "records as left at the end of each lab run"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "assumptions": [
    "C21"
   ],
   "limitations": [
    "L2",
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "moderate",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; four of four records verify, each checked by the tool itself; four runs on one task. One web search found no earlier study of agents keeping pre-planned lab records or completing claimsets."
    }
   ],
   "relations": [
    {
     "type": "addresses",
     "reference": "B6",
     "note": "Agents that run research end to end, judged by an automated reviewer; here agents keep a verifiable lab record of one study, checked by the record's own verification."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C2",
   "statement": "In both of its lab runs, the [[D3:Sonnet agent]] ran every confirmatory test under a plan with a computed decision rule, written with its hypotheses before any run and approved by the human before the test ([[D13:plans before data]]).",
   "type": "descriptive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E1",
     "results": [
      "E1.T1.r1",
      "E1.T1.r2"
     ],
     "artefacts": [
      "A1",
      "A2"
     ]
    }
   ],
   "about": {
    "subject": "D3",
    "relation": "plans before the data in every run",
    "object": "D13",
    "conditions": [],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "the two Sonnet lab runs"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "assumptions": [
    "C22"
   ],
   "limitations": [
    "L1",
    "L2",
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; two runs; the record's own order of events shows it. One web search found no earlier study of agents keeping pre-planned lab records or completing claimsets."
    }
   ],
   "relations": [
    {
     "type": "addresses",
     "reference": "B1",
     "note": "Argues that predictions must be fixed before the data are seen; here the question is whether AI agents do so when a tool asks them to."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C3",
   "statement": "Neither lab run of the [[D4:Haiku agent]] planned its tests before seeing the data ([[D13:plans before data]]): one run made no plan at all, and the other set its hypotheses and thresholds after three pilot runs.",
   "type": "descriptive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E1",
     "results": [
      "E1.T1.r3",
      "E1.T1.r4"
     ],
     "artefacts": [
      "A1",
      "A2"
     ]
    }
   ],
   "about": {
    "subject": "D4",
    "relation": "does not plan before the data",
    "object": "D13",
    "conditions": [],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "the two Haiku lab runs"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "assumptions": [
    "C22"
   ],
   "limitations": [
    "L1",
    "L2",
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; two runs; in one, the human had left the choice of confirmatory tests to the agent's judgement. One web search found no earlier study of agents keeping pre-planned lab records or completing claimsets."
    }
   ],
   "relations": [
    {
     "type": "addresses",
     "reference": "B1",
     "note": "Argues that predictions must be fixed before the data are seen; here AI agents with the same tool did not, in either of two runs."
    },
    {
     "type": "addresses",
     "reference": "B2",
     "note": "Mandatory registration of trials was followed by more null results; here an agent that planned nothing reported more descriptive findings, on one task."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C4",
   "statement": "Three of the four lab runs of the [[D5:agents under test]] left statsforecast's damped-trend option unset, so that their statsforecast arm selected between a damped and an undamped trend, against the fixed model forms of [[D6:the brief]] ([[D14:model form fixed]]).",
   "type": "descriptive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E1",
     "results": [
      "E1.T1"
     ],
     "artefacts": [
      "A1"
     ]
    }
   ],
   "about": {
    "subject": "D5",
    "relation": "do not fix the model form",
    "object": "D14",
    "conditions": [
     "D6"
    ],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "the statsforecast arm of each run's script"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "limitations": [
    "L1",
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; read from each run's script; the consequence (a damped trend chosen in 7 to 61 of 300 fits per cell) is from the reference's damped_check. One web search found no earlier study of agents keeping pre-planned lab records or completing claimsets."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C5",
   "statement": "Every claimset that the [[D5:agents under test]] completed with [[D2:claimset-submit]] from one lab record validates with no errors ([[D20:claimset validates]]), at the fourth to sixth validator outcome ([[D21:validator outcomes until valid]]).",
   "type": "descriptive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E2",
     "results": [
      "E2.T1"
     ],
     "artefacts": [
      "A1",
      "A2"
     ]
    }
   ],
   "about": {
    "subject": "D5",
    "relation": "complete claimsets that validate",
    "object": [
     "D20",
     "D21"
    ],
    "conditions": [
     "D2"
    ],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "one lab record (lab-sonnet-1) completed four times",
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "the final file of each claimset run"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "assumptions": [
    "C22"
   ],
   "limitations": [
    "L2",
    "L5",
    "L6"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "moderate",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; four of four; the validator is the tool's own check. One web search found no earlier study of agents keeping pre-planned lab records or completing claimsets."
    }
   ],
   "relations": [
    {
     "type": "addresses",
     "reference": "B6",
     "note": "Agents write papers judged by an automated reviewer; here agents complete a claimset judged by a validator and against an answer key."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C6",
   "statement": "No claim of the four claimsets that the [[D5:agents under test]] completed with [[D2:claimset-submit]] is wrong against [[D10:the answer key]]: each is correct or partly correct ([[D25:claim correctness]]).",
   "type": "descriptive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E2",
     "results": [
      "E2.T1"
     ],
     "artefacts": [
      "A6",
      "A8"
     ]
    }
   ],
   "about": {
    "subject": "D5",
    "relation": "write no wrong claim",
    "object": "D25",
    "comparator": "D10",
    "conditions": [
     "D2"
    ]
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "one lab record (lab-sonnet-1) completed four times",
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "the 20 claims of the four claimsets"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "assumptions": [
    "C21"
   ],
   "limitations": [
    "L3",
    "L4",
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; 20 claims scored by one non-blind scorer against a key it wrote; the claims come from one record. One web search found no earlier study of agents keeping pre-planned lab records or completing claimsets."
    }
   ],
   "relations": [
    {
     "type": "addresses",
     "reference": "B6",
     "note": "Agent-written papers judged by an automated reviewer; here each claim is checked against an answer key from a reference simulation."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C7",
   "statement": "None of the four claimsets that the [[D5:agents under test]] completed with [[D2:claimset-submit]] states as a claim any of the main findings of [[D10:the answer key]]: how large the estimation bias is, that statsforecast's optimiser fails to converge for the seasonal model, or that statsforecast caps β ([[D26:answer-key findings stated]]).",
   "type": "descriptive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E2",
     "results": [
      "E2.T1"
     ],
     "artefacts": [
      "A6",
      "A8"
     ]
    }
   ],
   "about": {
    "subject": "D5",
    "relation": "do not state",
    "object": "D26",
    "comparator": "D10",
    "conditions": [
     "D2"
    ]
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "one lab record (lab-sonnet-1) completed four times",
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "claims only, not limitations or open questions"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "assumptions": [
    "C21"
   ],
   "limitations": [
    "L3",
    "L4",
    "L5"
   ],
   "depends_on": [
    "C12"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; the claimsets could only restate the one record they were given (C10); two of them put the seasonal γ result into a limitation or an open question. One web search found no earlier study of agents keeping pre-planned lab records or completing claimsets."
    }
   ],
   "relations": [
    {
     "type": "addresses",
     "reference": "B6",
     "note": "Agent-written papers judged by an automated reviewer; here the check is which findings of a reference analysis the claims state."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C8",
   "statement": "The [[D5:agents under test]] answered [[D6:the brief]]'s questions about how large the bias and RMSE are with pass-or-fail tests against thresholds they chose themselves, so the estimates never became claims: [[D1:claimset-lab]] turns only hypotheses decided by a rule into claims.",
   "type": "interpretive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E1",
     "results": [
      "E1.T2"
     ],
     "artefacts": [
      "A1"
     ]
    },
    {
     "kind": "argument",
     "warrant": "W1"
    }
   ],
   "about": {
    "subject": "D1",
    "relation": "leads agents to replace estimates by threshold tests",
    "object": null,
    "conditions": [
     "D6"
    ],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "the three lab runs with a plan and the one without"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "limitations": [
    "L1",
    "L5"
   ],
   "depends_on": [
    "C7",
    "C12"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; an interpretation of four runs: three planned runs invented thresholds (10 %, 0.02, 0.75, 12 violations); the unplanned run reported the estimates."
    }
   ],
   "relations": [
    {
     "type": "addresses",
     "reference": "B4",
     "note": "Recommends framing research questions as estimation rather than significance tests; here a tool's pass-or-fail plans led agents away from estimation."
    },
    {
     "type": "addresses",
     "reference": "B3",
     "note": "Registered Reports report far fewer positive results than standard articles; here the registered format changed what agents reported, from estimates to threshold tests."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C9",
   "statement": "The two claimsets that the [[D4:Haiku agent]] completed give a scope to none of their three principal claims, against all three in each of the two that the [[D3:Sonnet agent]] completed ([[D27:principal claims with scope]]).",
   "type": "descriptive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E2",
     "results": [
      "E2.T1"
     ],
     "artefacts": [
      "A1"
     ]
    }
   ],
   "about": {
    "subject": "D4",
    "relation": "gives no scope to principal claims",
    "object": "D27",
    "comparator": "D3",
    "conditions": []
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "one lab record (lab-sonnet-1) completed four times",
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "principal claims only"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "limitations": [
    "L1",
    "L2",
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; two runs per agent. One web search found no earlier study of agents keeping pre-planned lab records or completing claimsets."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C10",
   "statement": "Under [[D23:EntryLib rubric 0.2]], the one claimset that tested nothing under a plan gets a higher [[D24:reliability score]] than each of the six claimsets whose principal claims were planned tests.",
   "type": "descriptive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E2",
     "results": [
      "E2.T1",
      "E2.T2"
     ],
     "artefacts": [
      "A8",
      "A10"
     ]
    }
   ],
   "about": {
    "subject": "D23",
    "relation": "scores unplanned claims above planned ones",
    "object": "D24",
    "conditions": [
     "six claimsets of this study"
    ],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "conditions": [
     "entrylib-rubric 0.2.0, without --novelty",
     "the six claimsets of this study",
     "claimsets written in either phase"
    ],
    "excludes": [
     "other rubric versions"
    ]
   },
   "limitations": [
    "L5"
   ],
   "depends_on": [
    "C14"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "moderate",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; the scores are computed; the cause is in the rubric's code (C17). One web search found no earlier study of agents keeping pre-planned lab records or completing claimsets."
    }
   ],
   "relations": [
    {
     "type": "addresses",
     "reference": "B5",
     "note": "Incentives that reward output select for weaker methods; here a scoring rubric ranks a claimset without planned tests above planned ones."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C11",
   "statement": "Under [[D23:EntryLib rubric 0.2]], a claimset that states no limitations and gives its claims no scope (claimset-haiku-2) gets a higher [[D24:reliability score]] than one written from the same lab record with scoped claims and three specific limitations (claimset-sonnet-1).",
   "type": "descriptive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E2",
     "results": [
      "E2.T1.r1",
      "E2.T1.r4"
     ],
     "artefacts": [
      "A8",
      "A10"
     ]
    }
   ],
   "about": {
    "subject": "D23",
    "relation": "scores a claimset without limitations or scope above one with both",
    "object": "D24",
    "conditions": [
     "claimsets from one lab record"
    ],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "conditions": [
     "entrylib-rubric 0.2.0, without --novelty",
     "the six claimsets of this study",
     "claimsets completed from one lab record"
    ],
    "excludes": [
     "other rubric versions"
    ]
   },
   "limitations": [
    "L5"
   ],
   "depends_on": [
    "C13"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "moderate",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; two scores, computed; the cause is in the rubric's costs (C17). One web search found no earlier study of agents keeping pre-planned lab records or completing claimsets."
    }
   ],
   "relations": [
    {
     "type": "addresses",
     "reference": "B5",
     "note": "Incentives that reward output select for weaker methods; here a scoring rubric ranks a claimset without limitations or scope above one with both."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C12",
   "statement": "The lab record that [[D8:the claimset phase]] started from holds three hypotheses, each a threshold test on the RMSE (a band for the ratio of the two packages, backcasting better in most cells at the shortest length, the longest length against the shortest), and no hypothesis about the size of the bias or the RMSE.",
   "type": "descriptive",
   "role": "supporting",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E2",
     "artefacts": [
      "A1"
     ]
    }
   ],
   "about": {
    "subject": "the lab record of lab-sonnet-1",
    "relation": "holds only threshold tests",
    "object": null,
    "conditions": [
     "D8"
    ],
    "comparator": null
   },
   "novelty": "not_applicable",
   "scope": {
    "conditions": [
     "the lab record of run lab-sonnet-1, 1,130 events"
    ]
   },
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; read from the record itself."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C13",
   "statement": "In [[D23:EntryLib rubric 0.2]], an unmitigated limitation that could bias a claim's magnitude costs the claim 12 points, more than a missing scope (10 points).",
   "type": "descriptive",
   "role": "supporting",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E4",
     "results": [
      "E4.T1.r1",
      "E4.T1.r2"
     ],
     "artefacts": [
      "A10"
     ]
    }
   ],
   "about": {
    "subject": "D23",
    "relation": "charges",
    "object": null,
    "comparator": null,
    "conditions": []
   },
   "novelty": "not_applicable",
   "scope": {
    "conditions": [
     "entrylib-rubric 0.2.0, without --novelty",
     "the six claimsets of this study",
     "costs per claim"
    ],
    "excludes": [
     "other rubric versions"
    ]
   },
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Read from the rubric's code at the commit artefact A10 names (rubric.json, checks limitation and no_scope)."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C14",
   "statement": "In [[D23:EntryLib rubric 0.2]], a descriptive claim made without a plan costs nothing in the plan group, and a planned claim whose plan is anchored only by a git push costs 3 points.",
   "type": "descriptive",
   "role": "supporting",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E4",
     "results": [
      "E4.T1.r3",
      "E4.T1.r4"
     ],
     "artefacts": [
      "A10"
     ]
    }
   ],
   "about": {
    "subject": "D23",
    "relation": "charges planned claims more than unplanned descriptive ones",
    "object": null,
    "comparator": null,
    "conditions": []
   },
   "novelty": "not_applicable",
   "scope": {
    "conditions": [
     "entrylib-rubric 0.2.0, without --novelty",
     "the six claimsets of this study",
     "claims of type descriptive"
    ],
    "excludes": [
     "other rubric versions"
    ]
   },
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Read from the rubric's code at the commit artefact A10 names (score.py, lines 28 and 111: only associational, causal, mechanistic and predictive claims are charged without a plan; rubric.json, plan_anchor)."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C15",
   "statement": "None of the four lab runs of the [[D5:agents under test]] checked whether statsforecast's optimiser converged ([[D15:convergence checked]]).",
   "type": "descriptive",
   "role": "secondary",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E1",
     "results": [
      "E1.T1"
     ],
     "artefacts": [
      "A1"
     ]
    }
   ],
   "about": {
    "subject": "D5",
    "relation": "do not check",
    "object": "D15",
    "conditions": [],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "the statsforecast arm of each run's script"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "limitations": [
    "L5"
   ],
   "depends_on": [
    "C16"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; read from each run's script; it bears on the ETS(A,N,A) results (C6)."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C16",
   "statement": "In [[D31:the reference simulation]], statsforecast's optimiser stopped at its iteration cap in nearly every ETS(A,N,A) fit, at every series length ([[D37:statsforecast iteration cap]]).",
   "type": "descriptive",
   "role": "secondary",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "simulation",
     "evidence": "E3",
     "results": [
      "E3.T2"
     ],
     "artefacts": [
      "A6",
      "A7"
     ]
    }
   ],
   "about": {
    "subject": "statsforecast 2.1.1",
    "relation": "stops at its iteration cap for",
    "object": "D37",
    "conditions": [
     "D31"
    ],
    "comparator": null
   },
   "novelty": "not_applicable",
   "scope": {
    "conditions": [
     "statsforecast 2.1.1 AutoETS defaults",
     "ETS(A,N,A), m = 12, α = 0.2, γ = 0.1"
    ]
   },
   "assessments": [
    {
     "by": "author",
     "strength": "moderate",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; 300 fits per length, read from statsforecast's own iteration count."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C17",
   "statement": "In one of its two lab runs, the [[D4:Haiku agent]] recorded a run with the human's actor ID as its author, and `claimset lab` accepted it ([[D19:runs written as the human]]).",
   "type": "descriptive",
   "role": "secondary",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E1",
     "results": [
      "E1.T1.r3"
     ],
     "artefacts": [
      "A1",
      "A2"
     ]
    }
   ],
   "about": {
    "subject": "D4",
    "relation": "records a run as the human",
    "object": "D19",
    "conditions": [],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "the two Haiku lab runs"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "limitations": [
    "L1"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; one event in one record (RN4 by AC3); the agent disclosed it and noted the correction."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C18",
   "statement": "In one of its two claimset runs, the [[D4:Haiku agent]] replaced the model of the agent that had done the research, as the lab record names it, with its own ([[D29:research agent's model kept]]).",
   "type": "descriptive",
   "role": "secondary",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E2",
     "results": [
      "E2.T1.r3"
     ],
     "artefacts": [
      "A1"
     ]
    }
   ],
   "about": {
    "subject": "D4",
    "relation": "replaces the research agent's model",
    "object": "D29",
    "conditions": [],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "one lab record (lab-sonnet-1) completed four times",
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "the two Haiku claimset runs"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "limitations": [
    "L1"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; one claimset; the other Haiku run kept the research agent and added itself."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C19",
   "statement": "When statsforecast 2.1.1 fits ETS(A,A,N) with its defaults, its estimate of β never exceeds its starting value of α ([[D32:statsforecast β cap]]), so that no estimate can lie above a true β of 0.2.",
   "type": "descriptive",
   "role": "secondary",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "simulation",
     "evidence": "E3",
     "results": [
      "E3.T1"
     ],
     "artefacts": [
      "A6",
      "A7"
     ]
    }
   ],
   "about": {
    "subject": "statsforecast 2.1.1",
    "relation": "caps β at its starting α",
    "object": "D32",
    "conditions": [],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "conditions": [
     "statsforecast 2.1.1 AutoETS defaults, model AAN, m = 1",
     "ETS(A,A,N) with (α, β) = (0.5, 0.2)",
     "series lengths 50 to 500, 300 series each"
    ]
   },
   "assessments": [
    {
     "by": "author",
     "strength": "high",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; the cap is in the package's code (initparam) and every one of 1,200 estimates respects it. It makes statsforecast's lower RMSE for β = 0.2 at long series an artefact of the cap."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": []
  },
  {
   "id": "C20",
   "statement": "The [[D32:statsforecast β cap]] was reported by a [[D4:Haiku agent]] lab run (lab-haiku-1, in an exploratory run) and was missing from [[D10:the answer key]] as written before the lab phase.",
   "type": "descriptive",
   "role": "secondary",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E1",
     "results": [
      "E1.T1.r3"
     ],
     "artefacts": [
      "A2",
      "A6"
     ]
    }
   ],
   "about": {
    "subject": "D4",
    "relation": "finds what the reference missed",
    "object": "D32",
    "comparator": "D10",
    "conditions": []
   },
   "novelty": "not_applicable",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "the lab phase"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "limitations": [
    "L4"
   ],
   "depends_on": [
    "C19"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; one finding of one run; the reference's bound check compared β with the estimated α, not the starting one."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  },
  {
   "id": "C21",
   "statement": "[[D10:The answer key]] states the findings that a correct paper on [[D6:the brief]] would state, with correct numbers.",
   "type": "descriptive",
   "role": "supporting",
   "support": [
    {
     "kind": "assumption"
    }
   ],
   "provenance": "author_stated",
   "assumption_category": "methodological",
   "check": "unchecked",
   "verifiability": "not_applicable",
   "verifiability_flags": []
  },
  {
   "id": "C22",
   "statement": "The [[D5:agents under test]], run as subagents of an orchestrating session, act as they would in sessions of their own started in their run folders.",
   "type": "descriptive",
   "role": "supporting",
   "support": [
    {
     "kind": "assumption"
    }
   ],
   "provenance": "author_stated",
   "assumption_category": "methodological",
   "check": "unchecked",
   "verifiability": "not_applicable",
   "verifiability_flags": []
  }
 ],
 "evidence": [
  {
   "id": "E1",
   "profile": "quant",
   "design": "experiment",
   "description": "[[D7:The lab phase]]: two runs each of the [[D3:Sonnet agent]] and the [[D4:Haiku agent]] did [[D6:the ETS estimation brief]] with [[D1:claimset-lab]]. For each run the orchestrating agent read the final lab record (`tools/lab_summary.py`: plans, approvals and runs in record order, the actor of every event, anchors), the run's script, and its transcript (`tools/transcript.py`: every tool call, every refusal, model-call usage), and recorded the variables of table T1. Every question and answer is in `log/questions.md`.",
   "setup": {
    "studied": [
     "D3",
     "D4"
    ],
    "cases": [
     "D6"
    ],
    "procedure": [
     "D7",
     "D9",
     "D1"
    ],
    "lens": [
     "D10",
     "D11",
     "D12",
     "D13",
     "D14",
     "D15",
     "D16",
     "D17",
     "D18",
     "D19",
     "D33",
     "D34"
    ]
   },
   "estimand": "per-run outcomes of the lab phase",
   "n": 4,
   "data_origin": "primary",
   "preregistered": false,
   "models": [
    {
     "model": "M2",
     "role": "subject",
     "settings": {
      "harness": "Claude Code 2.1.322, Agent tool, subagent type general-purpose",
      "effort": null,
      "temperature": null
     },
     "record_source": "harness",
     "prompts": "A3"
    },
    {
     "model": "M3",
     "role": "subject",
     "settings": {
      "harness": "Claude Code 2.1.322, Agent tool, subagent type general-purpose",
      "effort": null,
      "temperature": null
     },
     "record_source": "harness",
     "prompts": "A3"
    }
   ],
   "artefacts": [
    "A1",
    "A2",
    "A3",
    "A4",
    "A5",
    "A8",
    "A9"
   ],
   "availability": {
    "level": "public"
   },
   "tables": [
    {
     "id": "T1",
     "description": "Outcomes of the four lab runs of the lab phase on the ETS estimation brief",
     "row_labels": [
      "run",
      "agent"
     ],
     "columns": [
      "D11",
      "D12",
      "D13",
      "D14",
      "D15",
      "D18",
      "D19",
      "D16",
      "D17"
     ],
     "rows": [
      {
       "id": "r1",
       "label": [
        "lab-sonnet-1",
        "D3"
       ],
       "values": [
        true,
        0,
        "all",
        true,
        false,
        true,
        0,
        52609,
        11.2
       ]
      },
      {
       "id": "r2",
       "label": [
        "lab-sonnet-2",
        "D3"
       ],
       "values": [
        true,
        0,
        "all",
        false,
        false,
        true,
        0,
        80054,
        13.5
       ]
      },
      {
       "id": "r3",
       "label": [
        "lab-haiku-1",
        "D4"
       ],
       "values": [
        true,
        3,
        "none",
        false,
        false,
        true,
        1,
        265066,
        16.7
       ]
      },
      {
       "id": "r4",
       "label": [
        "lab-haiku-2",
        "D4"
       ],
       "values": [
        true,
        1,
        "all, hypotheses after pilots",
        false,
        false,
        false,
        0,
        122979,
        11.8
       ]
      }
     ],
     "artefact": "A8"
    },
    {
     "id": "T2",
     "description": "The decision thresholds each lab record's plans fixed, and the record's hypotheses about the direction or size of an estimate",
     "row_labels": [
      "run",
      "agent"
     ],
     "columns": [
      "D33",
      "D34"
     ],
     "rows": [
      {
       "id": "r1",
       "label": [
        "lab-sonnet-1",
        "D3"
       ],
       "values": [
        "RMSE ratio of the two optimised arms between 0.9 and 1.1 in all 32 cells; backcasting with the lower RMSE in at least 6 of 8 cells at length 50; RMSE at length 500 below length 50 in all 24 comparisons",
        0
       ]
      },
      {
       "id": "r2",
       "label": [
        "lab-sonnet-2",
        "D3"
       ],
       "values": [
        "RMSE at length 500 below length 50 in all 24 series; smooth with the lower RMSE than statsforecast in at least 75 % of 32 comparisons; backcasting below optimised initial states in at least 75 %; |bias| + 1.645 SE below 0.02 at length 500 for every parameter",
        0
       ]
      },
      {
       "id": "r3",
       "label": [
        "lab-haiku-1",
        "D4"
       ],
       "values": [
        "none",
        3
       ]
      },
      {
       "id": "r4",
       "label": [
        "lab-haiku-2",
        "D4"
       ],
       "values": [
        "no violations of: RMSE falling with length; statsforecast's RMSE within 10 % of smooth's; |bias| at most 0.02 at length 500 (not supported only at 12, 16 and 12 violations)",
        0
       ]
      }
     ],
     "artefact": "A1"
    }
   ]
  },
  {
   "id": "E2",
   "profile": "quant",
   "design": "experiment",
   "description": "[[D8:The claimset phase]]: two runs each of the [[D3:Sonnet agent]] and the [[D4:Haiku agent]] exported one lab record and completed the claimset with [[D2:claimset-submit]]. For each final claimset the orchestrating agent ran `claimset validate` and `claimset score` ([[D23:EntryLib rubric 0.2]]), counted the validator outcomes in the transcript (`tools/validate_runs.py`), checked that no record-derived field (hypothesis, planning, planned_at, test, exploration counts) had changed, and scored every claim against [[D10:the answer key]] (table T1). Table T2 scores, under the same rubric, the two claimsets that lab-phase agents wrote on their own records after finishing them (beyond their task).",
   "setup": {
    "studied": [
     "D3",
     "D4"
    ],
    "cases": [
     "D6"
    ],
    "procedure": [
     "D8",
     "D9",
     "D2"
    ],
    "lens": [
     "D10",
     "D23",
     "D20",
     "D21",
     "D22",
     "D24",
     "D25",
     "D26",
     "D27",
     "D28",
     "D29",
     "D30",
     "D16"
    ]
   },
   "estimand": "per-run outcomes of the claimset phase",
   "n": 4,
   "data_origin": "primary",
   "preregistered": false,
   "models": [
    {
     "model": "M2",
     "role": "subject",
     "settings": {
      "harness": "Claude Code 2.1.322, Agent tool, subagent type general-purpose",
      "effort": null,
      "temperature": null
     },
     "record_source": "harness",
     "prompts": "A3"
    },
    {
     "model": "M3",
     "role": "subject",
     "settings": {
      "harness": "Claude Code 2.1.322, Agent tool, subagent type general-purpose",
      "effort": null,
      "temperature": null
     },
     "record_source": "harness",
     "prompts": "A3"
    }
   ],
   "artefacts": [
    "A1",
    "A2",
    "A3",
    "A6",
    "A8",
    "A9",
    "A10"
   ],
   "availability": {
    "level": "public"
   },
   "tables": [
    {
     "id": "T1",
     "description": "Outcomes of the four claimset runs of the claimset phase, all from the lab record of lab-sonnet-1",
     "row_labels": [
      "run",
      "agent"
     ],
     "columns": [
      "D20",
      "D21",
      "D22",
      "D24",
      {
       "variable": "D25",
       "conditions": {
        "outcome": "correct"
       }
      },
      {
       "variable": "D25",
       "conditions": {
        "outcome": "partly correct"
       }
      },
      {
       "variable": "D25",
       "conditions": {
        "outcome": "wrong"
       }
      },
      "D26",
      "D27",
      "D28",
      "D29",
      "D30",
      "D16"
     ],
     "rows": [
      {
       "id": "r1",
       "label": [
        "claimset-sonnet-1",
        "D3"
       ],
       "values": [
        true,
        4,
        2,
        73,
        4,
        1,
        0,
        0,
        3,
        3,
        true,
        0,
        50493
       ]
      },
      {
       "id": "r2",
       "label": [
        "claimset-sonnet-2",
        "D3"
       ],
       "values": [
        true,
        5,
        3,
        72,
        5,
        0,
        0,
        0,
        3,
        3,
        true,
        0,
        44792
       ]
      },
      {
       "id": "r3",
       "label": [
        "claimset-haiku-1",
        "D4"
       ],
       "values": [
        true,
        6,
        1,
        79,
        4,
        1,
        0,
        0,
        0,
        3,
        false,
        0,
        69641
       ]
      },
      {
       "id": "r4",
       "label": [
        "claimset-haiku-2",
        "D4"
       ],
       "values": [
        true,
        4,
        11,
        78,
        3,
        2,
        0,
        0,
        0,
        0,
        true,
        1,
        55371
       ]
      }
     ],
     "artefact": "A8"
    },
    {
     "id": "T2",
     "description": "EntryLib rubric 0.2 scores of the two claimsets written by lab-phase agents on their own records",
     "row_labels": [
      "run",
      "agent",
      "planning of the principal claims"
     ],
     "columns": [
      "D24",
      "D28",
      "D27"
     ],
     "rows": [
      {
       "id": "r1",
       "label": [
        "lab-sonnet-2",
        "D3",
        "confirmatory"
       ],
       "values": [
        73,
        4,
        3
       ]
      },
      {
       "id": "r2",
       "label": [
        "lab-haiku-1",
        "D4",
        "exploratory"
       ],
       "values": [
        92,
        0,
        3
       ]
      }
     ],
     "artefact": "A8"
    }
   ]
  },
  {
   "id": "E3",
   "profile": "simulation",
   "description": "[[D31:The reference simulation]] of [[D6:the brief]] (R = 300 per cell, 20 cells, three arms, 110 s on 4 cores), with `optcheck.py` and `damped_check.py`. Table T1: statsforecast's β estimates for ETS(A,A,N) with (α, β) = (0.5, 0.2), against smooth's; table T2: statsforecast fits of ETS(A,N,A) that stopped at the optimiser's iteration cap. The β cap was first reported by run lab-haiku-1 (its exploratory run RN1); it was verified here against the statsforecast 2.1.1 source and these estimates, and added to the answer key.",
   "setup": {
    "studied": [
     "D31"
    ],
    "cases": [
     "D6"
    ],
    "procedure": [
     "D31"
    ],
    "lens": [
     "D32",
     "D37"
    ]
   },
   "model": "ETS in innovations form with Gaussian errors, σ = 1, l0 = 100, b0 = 1, s_j = 10 sin(2πj/12)",
   "parameters": "A6",
   "runs": 300,
   "seeds": [
    "20261111"
   ],
   "artefacts": [
    "A6",
    "A7"
   ],
   "availability": {
    "level": "public"
   },
   "data_origin": "primary",
   "tables": [
    {
     "id": "T1",
     "description": "statsforecast and smooth (optimised initial states) estimates of β for ETS(A,A,N) with α = 0.5, β = 0.2, 300 series per length",
     "row_labels": [
      "series length"
     ],
     "columns": [
      {
       "variable": "largest statsforecast β estimate"
      },
      {
       "variable": "statsforecast fits within 0.001 of 0.20006"
      },
      {
       "variable": "smooth fits with β above 0.20006"
      }
     ],
     "rows": [
      {
       "id": "r1",
       "label": [
        50
       ],
       "values": [
        0.20006,
        100,
        131
       ]
      },
      {
       "id": "r2",
       "label": [
        100
       ],
       "values": [
        0.20006,
        135,
        146
       ]
      },
      {
       "id": "r3",
       "label": [
        250
       ],
       "values": [
        0.20006,
        132,
        132
       ]
      },
      {
       "id": "r4",
       "label": [
        500
       ],
       "values": [
        0.20006,
        144,
        137
       ]
      }
     ],
     "artefact": "A6"
    },
    {
     "id": "T2",
     "description": "statsforecast fits of ETS(A,N,A) (α = 0.2, γ = 0.1, m = 12) at the optimiser's iteration cap, of 300 per length",
     "row_labels": [
      "series length"
     ],
     "columns": [
      "D37"
     ],
     "rows": [
      {
       "id": "r1",
       "label": [
        50
       ],
       "values": [
        288
       ]
      },
      {
       "id": "r2",
       "label": [
        100
       ],
       "values": [
        296
       ]
      },
      {
       "id": "r3",
       "label": [
        250
       ],
       "values": [
        300
       ]
      },
      {
       "id": "r4",
       "label": [
        500
       ],
       "values": [
        299
       ]
      }
     ],
     "artefact": "A6"
    }
   ]
  },
  {
   "id": "E4",
   "profile": "quant",
   "design": "other",
   "description": "[[D36:A reading of the rubric's code]]: the costs [[D23:EntryLib rubric 0.2]] deducts for the problems that decide the scores compared in this paper.",
   "setup": {
    "studied": [
     "D23"
    ],
    "cases": [
     "D23"
    ],
    "procedure": [
     "D36"
    ],
    "lens": [
     "D35"
    ]
   },
   "data_origin": "secondary",
   "preregistered": false,
   "artefacts": [
    "A10"
   ],
   "availability": {
    "level": "public"
   },
   "tables": [
    {
     "id": "T1",
     "description": "Costs of four problems in EntryLib rubric 0.2, per claim",
     "row_labels": [
      "problem"
     ],
     "columns": [
      "D35"
     ],
     "rows": [
      {
       "id": "r1",
       "label": [
        "a limitation that could bias magnitude, not mitigated"
       ],
       "values": [
        12
       ]
      },
      {
       "id": "r2",
       "label": [
        "no scope"
       ],
       "values": [
        10
       ]
      },
      {
       "id": "r3",
       "label": [
        "a plan anchored only by a git push"
       ],
       "values": [
        3
       ]
      },
      {
       "id": "r4",
       "label": [
        "a descriptive claim without a plan"
       ],
       "values": [
        0
       ]
      }
     ],
     "artefact": "A10"
    }
   ]
  }
 ],
 "warrants": [
  {
   "id": "W1",
   "reasoning": "inductive",
   "premises": [
    "C12"
   ],
   "text": "The claimset-lab skill offers one route from a run to a claim: a hypothesis decided by a rule, with computed rules preferred whenever the outcome can be computed from numbers (SKILL.md, lines 83–100), and export makes claims only from hypotheses with a verdict or a final status (lab record v0.21 §9). Every planned run accordingly chose thresholds the brief did not give (an RMSE ratio within 10 %, |bias| at most 0.02, shares of at least 0.75, at least 12 violations), and its claims are the verdicts on them; the one run without a plan wrote notes and claims about the estimates themselves (the γ pile-up, statsforecast's seasonal α bias, the β cap)."
  }
 ],
 "limitations": [
  {
   "id": "L1",
   "category": "sample_size",
   "statement": "Two runs per agent and phase: differences between the Sonnet and Haiku agents rest on two runs each and could reverse with more runs.",
   "affects": [
    "C2",
    "C3",
    "C4",
    "C8",
    "C9",
    "C17",
    "C18"
   ],
   "impact": "could_reverse",
   "mitigation": "none"
  },
  {
   "id": "L2",
   "category": "confounding",
   "statement": "The agents ran as subagents of the orchestrating session: they read the skills as files, the claimset guard hook of their folders was not active, and they received the orchestrating session's own project instructions (which led one Haiku run to ask its questions one at a time); four runs shared one 4-core machine.",
   "affects": [
    "C1",
    "C2",
    "C3",
    "C5",
    "C9"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  },
  {
   "id": "L3",
   "category": "measurement",
   "statement": "One scorer, the orchestrating agent, wrote the answer key and the protocol and scored every claim, knowing which agent wrote it.",
   "affects": [
    "C6",
    "C7"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "partial",
   "mitigation_evidence": "E3"
  },
  {
   "id": "L4",
   "category": "deviation",
   "statement": "The answer key was amended once during the lab phase, after a Haiku run reported statsforecast's β cap; the claimsets were scored against the amended key.",
   "affects": [
    "C6",
    "C7",
    "C20"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  },
  {
   "id": "L5",
   "category": "sampling",
   "statement": "One study task (an ETS simulation comparison in Python), one version of claimset (0.41.6) and of its skills, one harness (Claude Code 2.1.322).",
   "affects": [
    "C1",
    "C2",
    "C3",
    "C4",
    "C15",
    "C5",
    "C6",
    "C7",
    "C8",
    "C9",
    "C10",
    "C11"
   ],
   "impact": "reduces_transferability",
   "mitigation": "none"
  },
  {
   "id": "L6",
   "category": "deviation",
   "statement": "One claimset run (claimset-haiku-2) stopped with its own stated work undone and was sent one message with no content (\"Continue.\") before it finished.",
   "affects": [
    "C5"
   ],
   "impact": "could_bias_magnitude",
   "mitigation": "none"
  }
 ],
 "artefacts": [
  {
   "id": "A1",
   "kind": "data",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/runs/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "The eight runs as the agents left them: each folder's files without its git directory, and its full git history (study branch included) in repo.bundle."
  },
  {
   "id": "A2",
   "kind": "trace",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/log/transcripts/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "The eight agents' transcripts (Claude Code subagent JSONL), with every tool call and result and the usage of every model call."
  },
  {
   "id": "A3",
   "kind": "prompt",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/log/prompts.md",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "The task prompts given to the agents in each phase."
  },
  {
   "id": "A4",
   "kind": "protocol",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/study/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "The study brief and the fixed answers given in place of the human."
  },
  {
   "id": "A5",
   "kind": "data",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/log/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "Setup notes, every question and answer, agent IDs and usage per turn."
  },
  {
   "id": "A6",
   "kind": "data",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/reference/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "The reference analysis: estimates.csv, summary.csv, consistency.csv, optcheck.csv, damped_check.csv and answer_key.md."
  },
  {
   "id": "A7",
   "kind": "code",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/reference/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "MIT",
   "availability": {
    "level": "public"
   },
   "description": "The reference analysis scripts simulate.py, optcheck.py and damped_check.py."
  },
  {
   "id": "A8",
   "kind": "data",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/results/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "Scoring per run, rubric scores as JSON, summary.md and report.md."
  },
  {
   "id": "A9",
   "kind": "code",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/tools/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "MIT",
   "availability": {
    "level": "public"
   },
   "description": "The scoring scripts transcript.py, lab_summary.py and validate_runs.py."
  },
  {
   "id": "A10",
   "kind": "software",
   "uri": "https://github.com/entrylib/rubric/tree/8a6f9f948a83c0a6f8c60c1e101ed5c4f5399caf",
   "version": "8a6f9f948a83c0a6f8c60c1e101ed5c4f5399caf",
   "licence": "Apache-2.0",
   "availability": {
    "level": "public"
   },
   "description": "EntryLib's scoring rubric, entrylib-rubric 0.2.0 (src/rubric/rubric.json holds the costs; src/rubric/score.py applies them)."
  }
 ],
 "references": [
  {
   "id": "B1",
   "target": "10.1073/pnas.1708274114",
   "id_type": "doi",
   "self": false
  },
  {
   "id": "B2",
   "target": "10.1371/journal.pone.0132382",
   "id_type": "doi",
   "self": false
  },
  {
   "id": "B3",
   "target": "10.1177/25152459211007467",
   "id_type": "doi",
   "self": false
  },
  {
   "id": "B4",
   "target": "10.1177/0956797613504966",
   "id_type": "doi",
   "self": false
  },
  {
   "id": "B5",
   "target": "10.1098/rsos.160384",
   "id_type": "doi",
   "self": false
  },
  {
   "id": "B6",
   "target": "2408.06292",
   "id_type": "arxiv",
   "self": false
  }
 ],
 "open_questions": [
  {
   "id": "O1",
   "question": "Whether the differences between the Sonnet and Haiku agents hold over more runs, other study tasks and agents in sessions of their own is not established.",
   "claims": [
    "C2",
    "C3",
    "C9",
    "C20"
   ]
  },
  {
   "id": "O2",
   "question": "Whether refusing writes by a human actor in agent harnesses, and keeping the research agent's model through export, would prevent the misattributions of a run and of a study's research is not tested.",
   "claims": [
    "C17",
    "C18"
   ]
  },
  {
   "id": "O3",
   "question": "Whether a skill instruction to report each fit's model form and optimiser status as results would close the gaps in fixing the model form and checking convergence is not tested.",
   "claims": [
    "C4",
    "C15"
   ]
  },
  {
   "id": "O4",
   "question": "Whether a plan that fixes estimates rather than a decision rule would lead agents to state the brief's findings as claims is not tested.",
   "claims": [
    "C7",
    "C8"
   ]
  }
 ]
}