{
  "schema": "ssr-author-study-summary-v1",
  "reviewed": "2026-10-07",
  "title": "Project rules and the limits of prepared context",
  "kind": "Author summary derived from retained private result records, not raw logs or a runnable release",
  "lead": "Jonathan Simone, with AI assistance",
  "experiment_harness": "Claude-based local research; models recorded as sonnet",
  "inspection": "Read-only source and aggregate inspection. No experiments rerun for this publication.",
  "sources": [
    {
      "id": "S1",
      "role": "Synthetic project A corrected rerun",
      "sha256": "3f15db5fd3a7b7d47143784a8348627ad550ccc047905281724f4a9e7b9cd86e",
      "availability": "Retained privately; not included in this summary"
    },
    {
      "id": "S2",
      "role": "Synthetic project A guide repetitions",
      "sha256": "71d5429d194de9890ac41bbec81f61fccb4d2bb998b79ab10ecbd3a67da89acd",
      "availability": "Retained privately; not included in this summary"
    },
    {
      "id": "S3",
      "role": "Synthetic project B comparison",
      "sha256": "b821dd64550a66cb8059d215c5a3b965a0e2a85704191da83f1ea0f9d63abe5c",
      "availability": "Retained privately; not included in this summary"
    },
    {
      "id": "S4",
      "role": "Corrected real-repository task comparison",
      "sha256": "bcb87f2dbf9326bdf3144714aa5580c42e413d4a210c011cb8588369b1e1e1f3",
      "availability": "Retained privately; not included in this summary"
    },
    {
      "id": "S5",
      "role": "Synthetic study protocol and correction history",
      "sha256": "f5fecda20843ad230936150ff7b33be446d0360ba7104abee28bad8251962162",
      "availability": "Retained privately; not included in this summary"
    },
    {
      "id": "S6",
      "role": "Study execution and method record",
      "sha256": "5c6e3abceda67554721ca7b109b351bdb56721a018cf73c1e43914d8399311a9",
      "availability": "Retained privately; not included in this summary"
    },
    {
      "id": "S7",
      "role": "Real-task interpretation and next research question",
      "sha256": "e97f917cbdba1f2c8502bb1f8d95825c9dd6f0898c00d53e46007c6283680252",
      "availability": "Retained privately; not included in this summary"
    },
    {
      "id": "S8",
      "role": "Author-reported prototype status",
      "sha256": "a87293fc33ad1f187e6755da075b9d686b43cc6b9a35d05fb7f56dccef73e7cc",
      "availability": "Retained privately; not included in this summary"
    }
  ],
  "synthetic_project_a": {
    "source_ids": [
      "S1",
      "S2"
    ],
    "method": "Same synthetic coding task and 104 hidden checks per build, including 38 project-specific rule checks. Task-only and library arms each have 5 builds.",
    "task_only": {
      "builds": 5,
      "checks_per_build": [
        104
      ],
      "passed_checks_by_build": [
        66,
        66,
        66,
        66,
        66
      ],
      "mean_check_pass_fraction": 0.6346153846153846,
      "no_build_attempts": 0
    },
    "library_tools": {
      "builds": 5,
      "checks_per_build": [
        104
      ],
      "passed_checks_by_build": [
        104,
        104,
        104,
        104,
        95
      ],
      "mean_check_pass_fraction": 0.9826923076923076,
      "no_build_attempts": 0
    },
    "house_rule_checks_per_build": 38,
    "task_only_house_rule_passed": [
      0,
      0,
      0,
      0,
      0
    ],
    "library_house_rule_passed": [
      38,
      38,
      38,
      38,
      34
    ],
    "compiled_guide": {
      "builds": 11,
      "checks_per_build": [
        104
      ],
      "passed_checks_by_build": [
        104,
        104,
        104,
        104,
        104,
        104,
        104,
        104,
        104,
        104,
        104
      ],
      "mean_check_pass_fraction": 1.0,
      "no_build_attempts": 0,
      "distinct_compiles": 4
    },
    "early_selector": {
      "builds": 5,
      "checks_per_build": [
        0,
        104
      ],
      "passed_checks_by_build": [
        81,
        81,
        81,
        81,
        0
      ],
      "mean_check_pass_fraction": 0.6230769230769231,
      "no_build_attempts": 1
    },
    "early_selector_scoring": "The no-build attempt is retained with a zero score, not dropped."
  },
  "synthetic_project_b": {
    "source_ids": [
      "S3"
    ],
    "compiled_guide": {
      "builds": 11,
      "checks_per_build": [
        86
      ],
      "passed_checks_by_build": [
        86,
        86,
        86,
        86,
        86,
        86,
        86,
        86,
        86,
        86,
        86
      ],
      "mean_check_pass_fraction": 1.0,
      "no_build_attempts": 0,
      "distinct_compiles": 3
    },
    "whole_corpus_in_prompt": {
      "builds": 5,
      "checks_per_build": [
        86
      ],
      "passed_checks_by_build": [
        86,
        86,
        86,
        86,
        86
      ],
      "mean_check_pass_fraction": 1.0,
      "no_build_attempts": 0
    },
    "interpretation": "Whole-corpus input matched compiled guides on this small corpus; no unique teaching advantage established."
  },
  "real_repository_tasks": {
    "source_ids": [
      "S4"
    ],
    "total_builds": 222,
    "metric": "Fraction of originally failing checks that pass on each attempt; average two builds per task, then average equally across 37 tasks. Not the fraction of tasks solved.",
    "arms": {
      "plain_agent": {
        "tasks": 37,
        "builds_per_task": 2,
        "builds": 74,
        "mean_fail_to_pass_check_fraction": 0.5381310810810811,
        "fully_resolved_builds": 1
      },
      "compiled_guide": {
        "tasks": 37,
        "builds_per_task": 2,
        "builds": 74,
        "mean_fail_to_pass_check_fraction": 0.48805,
        "fully_resolved_builds": 0
      },
      "library_tools": {
        "tasks": 37,
        "builds_per_task": 2,
        "builds": 74,
        "mean_fail_to_pass_check_fraction": 0.48952837837837837,
        "fully_resolved_builds": 0
      }
    },
    "regression_failures": 4,
    "interpretation": "The compiled guide did not establish a coding benefit over the ordinary agent in this comparison. Not a universal verdict on context preparation."
  },
  "history": [
    "An initial synthetic run had unintended access to harvested material and was set aside. Reported scores use the later rerun after access-rule repairs. Recorded leak checks are not a guarantee of isolation.",
    "An early document selector performed worse than the task-only arm. Later compilation is a different method, not validation of that selector.",
    "The first real-task instrument had interface defects. Its initial floor result is not used as efficacy evidence; the figures here come from the repaired comparison."
  ],
  "limitations": [
    "Synthetic projects, documents and checks were AI-authored. Same-model-family compilation and building are not independent verification.",
    "Repeated builds on the same synthetic task are not independent tasks.",
    "Private code, corpora, task identities, sealed evaluation items and raw records are not distributed. Hashes identify retained bytes but do not enable reproduction or independent verification.",
    "No complete preparation, engineering, execution and review cost comparison. No general accuracy, efficiency, production-readiness or automatic-teaching claim.",
    "The public correction-and-restore fixture and GPT-assisted simulation work are separate implementations and evidence."
  ],
  "next_question": "Can a concise current-state briefing help a new session recover relevant facts, decisions and next steps? The briefing prototype is built; its benefit evaluation remains unrun.",
  "narrative_source_ids": [
    "S5",
    "S6",
    "S7",
    "S8"
  ],
  "next_question_status": "Prototype and unit-test status are author-reported in S8, inspected for this note, not rerun or independently reproduced by the website review."
}