{
 "bundle_of": "C4",
 "schema_version": "0.41",
 "paper": {
  "id": "entrylib:2026.000005",
  "version": 2,
  "title": "Claude Sonnet and Claude Haiku agents keeping Claimset lab records and completing claimsets: a pilot on one simulation study",
  "licence": "CC-BY-4.0",
  "status": "corrected",
  "date": "2026-11-12",
  "status_notice": {
   "date": "2026-11-12",
   "reason_code": "correction",
   "statement": "The platform classified the paper under the OpenAlex subfield Management Science and Operations Research, which is wrong: the paper studies AI agents doing research. This version proposes the subfield Artificial Intelligence and replaces the keywords agent evaluation and scoring rubric with metascience and research methodology; claims, evidence and every other field are unchanged.",
   "issued_by": "Ivan Svetunkov (ORCID 0000-0001-7826-0281)"
  }
 },
 "claims": [
  {
   "id": "C4",
   "statement": "Three of the four lab runs of the [[D5:agents under test]] left statsforecast's damped-trend option unset, so that their statsforecast arm selected between a damped and an undamped trend, against the fixed model forms of [[D6:the brief]] ([[D14:model form fixed]]).",
   "type": "descriptive",
   "role": "principal",
   "planning": "exploratory",
   "provenance": "author_stated",
   "support": [
    {
     "kind": "empirical_quant",
     "evidence": "E1",
     "results": [
      "E1.T1"
     ],
     "artefacts": [
      "A1"
     ]
    }
   ],
   "about": {
    "subject": "D5",
    "relation": "do not fix the model form",
    "object": "D14",
    "conditions": [
     "D6"
    ],
    "comparator": null
   },
   "novelty": "no_prior_found",
   "scope": {
    "data_type": "agent runs on one simulation study task",
    "conditions": [
     "claimset 0.41.6 with its skills read as files",
     "Claude Code 2.1.322 Agent-tool subagents",
     "two runs per agent",
     "fixed answers in place of the human",
     "the statsforecast arm of each run's script"
    ],
    "excludes": [
     "agents in sessions of their own with the guard hook active",
     "other study tasks and fields"
    ]
   },
   "limitations": [
    "L1",
    "L5"
   ],
   "assessments": [
    {
     "by": "author",
     "strength": "low",
     "rationale": "Exploratory: no plan or decision rule was registered before the runs; read from each run's script; the consequence (a damped trend chosen in 7 to 61 of 300 fits per cell) is from the reference's damped_check. One web search found no earlier study of agents keeping pre-planned lab records or completing claimsets."
    }
   ],
   "verifiability": "open",
   "verifiability_flags": [
    "api_model_dependency"
   ]
  }
 ],
 "definitions": [
  {
   "id": "D1",
   "term": "claimset-lab",
   "kind": "method",
   "definition": "The claimset-lab skill (`.claude/skills/claimset-lab/SKILL.md`, installed by `claimset init --agent claude`) and the `claimset lab` commands of claimset 0.41.6, which keep a lab record (lab record specification v0.21): an append-only, hash-chained log of questions, hypotheses, plans, approvals, runs, results and verdicts, kept on a study branch and pushed after every command."
  },
  {
   "id": "D3",
   "term": "Sonnet agent",
   "kind": "method",
   "definition": "A Claude Code 2.1.322 subagent started with the Agent tool (type general-purpose) on the model claude-sonnet-5-5, working in a folder of its own that holds the study brief, `AGENTS.md` and the claimset skills as files; it asks the human by ending its turn with its questions.",
   "part_of": [
    "D5"
   ]
  },
  {
   "id": "D4",
   "term": "Haiku agent",
   "kind": "method",
   "definition": "The same as the [[D3:Sonnet agent]], on the model claude-haiku-5-5.",
   "part_of": [
    "D5"
   ]
  },
  {
   "id": "D5",
   "term": "agents under test",
   "kind": "method",
   "definition": "The two agent configurations compared: the [[D3:Sonnet agent]] and the [[D4:Haiku agent]].",
   "members": [
    "D3",
    "D4"
   ]
  },
  {
   "id": "D6",
   "term": "ETS estimation brief",
   "kind": "setting",
   "definition": "The research task given to every agent (`study/brief.md`): compare how well smooth 1.2.0 (`ES`) and statsforecast 2.1.1 (`AutoETS` with a fixed model form) estimate the smoothing parameters of the ETS model that generated simulated data, by bias, RMSE and MAE and by whether the RMSE falls as the series length grows; five processes (ETS(A,N,N) with α 0.1 and 0.5, ETS(A,A,N) with (α, β) (0.3, 0.05) and (0.5, 0.2), ETS(A,N,A) with m = 12 and (α, γ) (0.2, 0.1)), series lengths 50, 100, 250 and 500, 300 replicates per cell, smooth's default backcasting as a separate arm, failures counted."
  },
  {
   "id": "D7",
   "term": "lab phase",
   "kind": "design",
   "definition": "Phase 1: four runs, two of the [[D3:Sonnet agent]] and two of the [[D4:Haiku agent]], each in its own git repository with a local remote, given the same task (\"Do the study in brief.md with claimset-lab, following its skill, from the first question to the finished record. Ask the human anything you need through me.\"). The orchestrating agent relays questions and answers them only from [[D9:the fixed answers]], gives no other help, and never shows an agent another run or [[D10:the answer key]]. Runs in parallel on one 4-core machine."
  },
  {
   "id": "D8",
   "term": "claimset phase",
   "kind": "design",
   "definition": "Phase 2: the lab record of run lab-sonnet-1 (the only one that verifies, planned before its data and fixed statsforecast's model form) copied to four folders, with the draft its agent had exported removed; two [[D3:Sonnet agent]] and two [[D4:Haiku agent]] runs, each told \"Export this lab record with `claimset lab export` and complete the claimset for submission, following the claimset-submit skill. Do not submit it.\" Questions answered as in [[D7:the lab phase]]."
  },
  {
   "id": "D9",
   "term": "fixed answers",
   "kind": "design",
   "definition": "The answers the orchestrating agent gives in place of the human, the same to every run (`study/human.md`): name and ORCID iD; licence CC-BY-4.0; funding none; the competing interest of the brief; \"Approved.\" for a plan that names its quantities and decision rule before the data are run, otherwise \"Please state the quantities and the decision rule first.\"; for highlights \"Choose up to three principal claims you consider most important.\"; for anything else \"Use your judgement and say what you decided.\"",
   "part_of": [
    "D7",
    "D8"
   ]
  },
  {
   "id": "D10",
   "term": "answer key",
   "kind": "data",
   "definition": "The findings a correct paper on [[D6:the brief]] would state, with their numbers (`reference/answer_key.md`), from a reference simulation of the full design written by the orchestrating agent before Phase 1 without claimset-lab (`reference/simulate.py`, `optcheck.py`, `damped_check.py`). Amended once during Phase 1 (statsforecast's β cap)."
  },
  {
   "id": "D11",
   "term": "record verifies",
   "kind": "variable",
   "definition": "Whether `claimset lab verify` reports the lab record OK at the end of a run (hash chain, rules, run files)."
  },
  {
   "id": "D12",
   "term": "tool refusals",
   "kind": "variable",
   "definition": "Number of commands that `claimset` refused in a run with a lab rule (L1–L30) or an argument or overwrite error; validator errors on a claimset are counted separately."
  },
  {
   "id": "D13",
   "term": "plans before data",
   "kind": "variable",
   "definition": "Whether each confirmatory run of a lab record ran under a plan with its decision rule recorded, and approved by the human, before the run, and whether its hypotheses were written before any run informed them: `all`, `all, hypotheses after pilots` or `none` (no plan)."
  },
  {
   "id": "D14",
   "term": "model form fixed",
   "kind": "variable",
   "definition": "Whether a run's simulation script fixed statsforecast's model form as the brief asks (`damped=False`); without it, `AutoETS(model=\"AAN\")` and `ets_f(..., model=\"AAN\")` choose between ETS(A,A,N) and ETS(A,Ad,N)."
  },
  {
   "id": "D15",
   "term": "convergence checked",
   "kind": "variable",
   "definition": "Whether a run's script read any optimiser status (for statsforecast, the iteration count `model_[\"fit\"].nit`) and counted fits that did not converge."
  },
  {
   "id": "D16",
   "term": "output tokens",
   "kind": "variable",
   "definition": "Model output tokens of an agent over its whole run, summed from the usage of every model call in its transcript.",
   "unit": "1"
  },
  {
   "id": "D17",
   "term": "active time",
   "kind": "variable",
   "definition": "Time an agent spent working in a run, excluding waits for answers, as reported by the harness.",
   "unit": "min"
  },
  {
   "id": "D18",
   "term": "record anchored",
   "kind": "variable",
   "definition": "Whether the agent anchored the finished record (`claimset lab anchor`)."
  },
  {
   "id": "D19",
   "term": "runs written as the human",
   "kind": "variable",
   "definition": "Number of run events an agent recorded with `--by` the human's actor ID."
  },
  {
   "id": "D33",
   "term": "decision thresholds",
   "kind": "variable",
   "definition": "The thresholds a lab record's confirmatory plans fix in their decision rules, as the plans state them; `none` when a record has no plan."
  },
  {
   "id": "D34",
   "term": "hypotheses about an estimate",
   "kind": "variable",
   "definition": "Number of a lab record's hypotheses that state the direction or size of an estimate (a bias towards zero, an estimate on a bound, a bias of about −0.04) rather than a pass-or-fail threshold.",
   "unit": "1"
  }
 ],
 "evidence": [
  {
   "id": "E1",
   "profile": "quant",
   "design": "experiment",
   "description": "[[D7:The lab phase]]: two runs each of the [[D3:Sonnet agent]] and the [[D4:Haiku agent]] did [[D6:the ETS estimation brief]] with [[D1:claimset-lab]]. For each run the orchestrating agent read the final lab record (`tools/lab_summary.py`: plans, approvals and runs in record order, the actor of every event, anchors), the run's script, and its transcript (`tools/transcript.py`: every tool call, every refusal, model-call usage), and recorded the variables of table T1. Every question and answer is in `log/questions.md`.",
   "setup": {
    "studied": [
     "D3",
     "D4"
    ],
    "cases": [
     "D6"
    ],
    "procedure": [
     "D7",
     "D9",
     "D1"
    ],
    "lens": [
     "D10",
     "D11",
     "D12",
     "D13",
     "D14",
     "D15",
     "D16",
     "D17",
     "D18",
     "D19",
     "D33",
     "D34"
    ]
   },
   "estimand": "per-run outcomes of the lab phase",
   "n": 4,
   "data_origin": "primary",
   "preregistered": false,
   "models": [
    {
     "model": "M2",
     "role": "subject",
     "settings": {
      "harness": "Claude Code 2.1.322, Agent tool, subagent type general-purpose",
      "effort": null,
      "temperature": null
     },
     "record_source": "harness",
     "prompts": "A3"
    },
    {
     "model": "M3",
     "role": "subject",
     "settings": {
      "harness": "Claude Code 2.1.322, Agent tool, subagent type general-purpose",
      "effort": null,
      "temperature": null
     },
     "record_source": "harness",
     "prompts": "A3"
    }
   ],
   "artefacts": [
    "A1",
    "A2",
    "A3",
    "A4",
    "A5",
    "A8",
    "A9"
   ],
   "availability": {
    "level": "public"
   },
   "tables": [
    {
     "id": "T1",
     "description": "Outcomes of the four lab runs of the lab phase on the ETS estimation brief",
     "row_labels": [
      "run",
      "agent"
     ],
     "columns": [
      "D11",
      "D12",
      "D13",
      "D14",
      "D15",
      "D18",
      "D19",
      "D16",
      "D17"
     ],
     "rows": [
      {
       "id": "r1",
       "label": [
        "lab-sonnet-1",
        "D3"
       ],
       "values": [
        true,
        0,
        "all",
        true,
        false,
        true,
        0,
        52609,
        11.2
       ]
      },
      {
       "id": "r2",
       "label": [
        "lab-sonnet-2",
        "D3"
       ],
       "values": [
        true,
        0,
        "all",
        false,
        false,
        true,
        0,
        80054,
        13.5
       ]
      },
      {
       "id": "r3",
       "label": [
        "lab-haiku-1",
        "D4"
       ],
       "values": [
        true,
        3,
        "none",
        false,
        false,
        true,
        1,
        265066,
        16.7
       ]
      },
      {
       "id": "r4",
       "label": [
        "lab-haiku-2",
        "D4"
       ],
       "values": [
        true,
        1,
        "all, hypotheses after pilots",
        false,
        false,
        false,
        0,
        122979,
        11.8
       ]
      }
     ],
     "artefact": "A8"
    },
    {
     "id": "T2",
     "description": "The decision thresholds each lab record's plans fixed, and the record's hypotheses about the direction or size of an estimate",
     "row_labels": [
      "run",
      "agent"
     ],
     "columns": [
      "D33",
      "D34"
     ],
     "rows": [
      {
       "id": "r1",
       "label": [
        "lab-sonnet-1",
        "D3"
       ],
       "values": [
        "RMSE ratio of the two optimised arms between 0.9 and 1.1 in all 32 cells; backcasting with the lower RMSE in at least 6 of 8 cells at length 50; RMSE at length 500 below length 50 in all 24 comparisons",
        0
       ]
      },
      {
       "id": "r2",
       "label": [
        "lab-sonnet-2",
        "D3"
       ],
       "values": [
        "RMSE at length 500 below length 50 in all 24 series; smooth with the lower RMSE than statsforecast in at least 75 % of 32 comparisons; backcasting below optimised initial states in at least 75 %; |bias| + 1.645 SE below 0.02 at length 500 for every parameter",
        0
       ]
      },
      {
       "id": "r3",
       "label": [
        "lab-haiku-1",
        "D4"
       ],
       "values": [
        "none",
        3
       ]
      },
      {
       "id": "r4",
       "label": [
        "lab-haiku-2",
        "D4"
       ],
       "values": [
        "no violations of: RMSE falling with length; statsforecast's RMSE within 10 % of smooth's; |bias| at most 0.02 at length 500 (not supported only at 12, 16 and 12 violations)",
        0
       ]
      }
     ],
     "artefact": "A1"
    }
   ]
  }
 ],
 "limitations": [
  {
   "id": "L1",
   "category": "sample_size",
   "statement": "Two runs per agent and phase: differences between the Sonnet and Haiku agents rest on two runs each and could reverse with more runs.",
   "affects": [
    "C2",
    "C3",
    "C4",
    "C8",
    "C9",
    "C17",
    "C18"
   ],
   "impact": "could_reverse",
   "mitigation": "none"
  },
  {
   "id": "L5",
   "category": "sampling",
   "statement": "One study task (an ETS simulation comparison in Python), one version of claimset (0.41.6) and of its skills, one harness (Claude Code 2.1.322).",
   "affects": [
    "C1",
    "C2",
    "C3",
    "C4",
    "C15",
    "C5",
    "C6",
    "C7",
    "C8",
    "C9",
    "C10",
    "C11"
   ],
   "impact": "reduces_transferability",
   "mitigation": "none"
  }
 ],
 "artefacts": [
  {
   "id": "A1",
   "kind": "data",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/runs/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "The eight runs as the agents left them: each folder's files without its git directory, and its full git history (study branch included) in repo.bundle."
  },
  {
   "id": "A2",
   "kind": "trace",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/log/transcripts/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "The eight agents' transcripts (Claude Code subagent JSONL), with every tool call and result and the usage of every model call."
  },
  {
   "id": "A3",
   "kind": "prompt",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/log/prompts.md",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "The task prompts given to the agents in each phase."
  },
  {
   "id": "A4",
   "kind": "protocol",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/study/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "The study brief and the fixed answers given in place of the human."
  },
  {
   "id": "A5",
   "kind": "data",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/log/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "Setup notes, every question and answer, agent IDs and usage per turn."
  },
  {
   "id": "A8",
   "kind": "data",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/results/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "CC-BY-4.0",
   "availability": {
    "level": "public"
   },
   "description": "Scoring per run, rubric scores as JSON, summary.md and report.md."
  },
  {
   "id": "A9",
   "kind": "code",
   "uri": "https://github.com/entrylib/claimset-eval/tree/d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87/experiments/lab-weak-models/tools/",
   "version": "d3eb0e00a61ab50d7d2a62dfeb89c681d1c9eb87",
   "licence": "MIT",
   "availability": {
    "level": "public"
   },
   "description": "The scoring scripts transcript.py, lab_summary.py and validate_runs.py."
  }
 ]
}