{
  "schemaVersion": 1,
  "status": "published",
  "benchmarkId": "codebase-memory",
  "label": "Codebase Memory retrieval cost",
  "evidenceType": "derived-redacted-local-batch",
  "publicationEligible": true,
  "batch": {
    "batchId": "20260904-124556-6ffe",
    "status": "completed",
    "startedAt": "2026-09-04T12:45:56.897Z",
    "completedAt": "2026-09-04T15:32:46.890Z",
    "argv": [
      "run",
      "--config",
      "harness-evals.yaml",
      "--benchmark",
      "codebase-memory",
      "--concurrency",
      "1"
    ],
    "agents": [
      "felan-cbm-off",
      "felan-cbm-on"
    ],
    "declaredCaseCount": 3,
    "declaredRunCount": 18,
    "availableRunCount": 18,
    "availableCaseCount": 3,
    "sourceSha256": "2beacb30dfaaf8d10ad13c1dacb873d41919a48ce4059a7853a85455b2538c6e",
    "missingRunIds": [],
    "sourceManifestRelativePath": "sources/provenance/batches/20260904-124556-6ffe.json"
  },
  "definition": {
    "benchmarkRevision": 3,
    "benchmarkDigest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6",
    "sourceConfig": "missing from supplied bundle",
    "cases": [
      "cbm-extension-architecture",
      "cbm-extension-config-scope",
      "cbm-project-instructions"
    ],
    "arms": {
      "baseline": "felan-cbm-off",
      "candidate": "felan-cbm-on"
    },
    "trials": 3,
    "trialReducer": "median",
    "recommendedCaseEstimator": "ratioOfReducedSums",
    "activeRepositoryDefinitionAtPublication": {
      "revision": 4,
      "digest": "06b431978ea0ecdd3bcebc97715fb073aa7c727a39b3026c0c9bf9859be8a3d6",
      "relationship": "Publication aggregation reference only; the exact source configuration was not supplied."
    }
  },
  "commonRuntime": {
    "provider": "openai-codex",
    "model": "gpt-5.6-sol",
    "thinking": "medium",
    "packageVersion": "0.21.11",
    "repository": "https://github.com/felan-ai/felan.git"
  },
  "cases": [
    {
      "caseId": "cbm-extension-architecture",
      "sourceCommits": [
        "e5867637569bd1c7ad08420b79ec4031a5733f57"
      ],
      "arms": {
        "felan-cbm-off": {
          "attemptCount": 3,
          "attempts": [
            {
              "runId": "cbm-extension-architecture-felan-cbm-off-2026-09-04T12-45-56-901Z-0",
              "caseId": "cbm-extension-architecture",
              "arm": "felan-cbm-off",
              "attemptNumber": 1,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "e5867637569bd1c7ad08420b79ec4031a5733f57"
              },
              "metrics": {
                "cost.total": 1.1814840000000002,
                "duration.ms": 361912,
                "duration.stepsMs": 357640,
                "usage.promptTokens": 751242,
                "usage.outputTokens": 8275,
                "usage.totalTokens": 759517
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "0eba166203e6f20cf38e6beb24d49b82e3551dc0d016b5a4b04a95c3a37b03b2",
                  "bytes": 1764
                },
                "summary.json": {
                  "sha256": "5c54ff5628010c3d3ab4184daccb2a3664e9538f05863e553e96efc479ae1111",
                  "bytes": 16828
                },
                "cost-summary.json": {
                  "sha256": "3c3367bc21361936a684aceae227e3f7e8acc3ca3a849ddc0bb613c053c38c28",
                  "bytes": 6200
                },
                "workspace-source.json": {
                  "sha256": "9a2eeefcc703297374921ba63e3f78eedab5b02fa03310e954f1e53fb7e991c1",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            },
            {
              "runId": "cbm-extension-architecture-felan-cbm-off-2026-09-04T12-51-58-947Z-1",
              "caseId": "cbm-extension-architecture",
              "arm": "felan-cbm-off",
              "attemptNumber": 2,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "e5867637569bd1c7ad08420b79ec4031a5733f57"
              },
              "metrics": {
                "cost.total": 1.316029,
                "duration.ms": 369278,
                "duration.stepsMs": 365290,
                "usage.promptTokens": 914495,
                "usage.outputTokens": 8591,
                "usage.totalTokens": 923086
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "337bdc72719e867512142b985d7e9f744946a8f67404069d22d2b464b8c49c86",
                  "bytes": 1764
                },
                "summary.json": {
                  "sha256": "268f76202f9d55863439b18353cae830bac746419d78b48cfa9cb1d928b31273",
                  "bytes": 16638
                },
                "cost-summary.json": {
                  "sha256": "4ee2d76ea1e740b4817bb6f420f124432bde53ced4340f2eef7b9fd3fbda5712",
                  "bytes": 6080
                },
                "workspace-source.json": {
                  "sha256": "9a2eeefcc703297374921ba63e3f78eedab5b02fa03310e954f1e53fb7e991c1",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            },
            {
              "runId": "cbm-extension-architecture-felan-cbm-off-2026-09-04T12-58-08-372Z-2",
              "caseId": "cbm-extension-architecture",
              "arm": "felan-cbm-off",
              "attemptNumber": 3,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "e5867637569bd1c7ad08420b79ec4031a5733f57"
              },
              "metrics": {
                "cost.total": 1.201322,
                "duration.ms": 379429,
                "duration.stepsMs": 375861,
                "usage.promptTokens": 828316,
                "usage.outputTokens": 8801,
                "usage.totalTokens": 837117
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "449826d321ac271fbba2ded1bbf9442ceee622691444cdb7452dd5abf87c1d39",
                  "bytes": 1764
                },
                "summary.json": {
                  "sha256": "b162793f7be5a3260b42d4621f739e3493296841f220f40afc6cf69dd0f7dae0",
                  "bytes": 16638
                },
                "cost-summary.json": {
                  "sha256": "64f58c934865028519eb93c3eacebef0dd9f9ea729c0cf13343f7bb8dd620e93",
                  "bytes": 6080
                },
                "workspace-source.json": {
                  "sha256": "9a2eeefcc703297374921ba63e3f78eedab5b02fa03310e954f1e53fb7e991c1",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            }
          ],
          "reducedByMedian": {
            "cost.total": 1.201322,
            "duration.stepsMs": 365290,
            "usage.promptTokens": 828316,
            "usage.outputTokens": 8591,
            "usage.totalTokens": 837117
          },
          "passRate": 1.0,
          "meanVerifierReward": 1.0
        },
        "felan-cbm-on": {
          "attemptCount": 3,
          "attempts": [
            {
              "runId": "cbm-extension-architecture-felan-cbm-on-2026-09-04T13-04-27-913Z-3",
              "caseId": "cbm-extension-architecture",
              "arm": "felan-cbm-on",
              "attemptNumber": 1,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "e5867637569bd1c7ad08420b79ec4031a5733f57"
              },
              "metrics": {
                "cost.total": 1.0992439999999999,
                "duration.ms": 376871,
                "duration.stepsMs": 372954,
                "usage.promptTokens": 802324,
                "usage.outputTokens": 8348,
                "usage.totalTokens": 810672
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "0b3c23bf9fd34094a5ab21426efb23ad10ea6f8a6961587415936c988420fabe",
                  "bytes": 1759
                },
                "summary.json": {
                  "sha256": "6f75c3bd3ee822790104a5891f89c0961687049384e79200ff26e26ce17d9207",
                  "bytes": 16781
                },
                "cost-summary.json": {
                  "sha256": "f7a7ca374b6b32471396a0e60346a00590b099e0ff32c064e3dca0064731e54e",
                  "bytes": 6174
                },
                "workspace-source.json": {
                  "sha256": "9a2eeefcc703297374921ba63e3f78eedab5b02fa03310e954f1e53fb7e991c1",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            },
            {
              "runId": "cbm-extension-architecture-felan-cbm-on-2026-09-04T13-10-44-911Z-4",
              "caseId": "cbm-extension-architecture",
              "arm": "felan-cbm-on",
              "attemptNumber": 2,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "e5867637569bd1c7ad08420b79ec4031a5733f57"
              },
              "metrics": {
                "cost.total": 1.4177250000000001,
                "duration.ms": 365036,
                "duration.stepsMs": 361023,
                "usage.promptTokens": 903285,
                "usage.outputTokens": 7878,
                "usage.totalTokens": 911163
              },
              "verifierReward": 0.8,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "8be35ef655fcc8cded66d56841388720789cfc2241370618baa7b699febe8406",
                  "bytes": 1759
                },
                "summary.json": {
                  "sha256": "abd07e7cc80ab8ba4f70c317fa3332e4ae31af00393eb4ee5f089ea1db90cfe4",
                  "bytes": 16863
                },
                "cost-summary.json": {
                  "sha256": "6935d7e210a315ed7164b58e3fb67cd5d9ed6df1b6a8d07400aa700d4ef6c9aa",
                  "bytes": 6194
                },
                "workspace-source.json": {
                  "sha256": "9a2eeefcc703297374921ba63e3f78eedab5b02fa03310e954f1e53fb7e991c1",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "30ad512775f715ce404756fb831e3b8d95c20a86785f5c702c2b9987b6134fd5",
                  "bytes": 136
                }
              }
            },
            {
              "runId": "cbm-extension-architecture-felan-cbm-on-2026-09-04T13-16-50-044Z-5",
              "caseId": "cbm-extension-architecture",
              "arm": "felan-cbm-on",
              "attemptNumber": 3,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "e5867637569bd1c7ad08420b79ec4031a5733f57"
              },
              "metrics": {
                "cost.total": 1.9314859999999998,
                "duration.ms": 195576,
                "duration.stepsMs": 191573,
                "usage.promptTokens": 1094170,
                "usage.outputTokens": 8818,
                "usage.totalTokens": 1102988
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "6723cf1362a7c91b9cab4e3c8b327ffa8a4dab45669848c7cb6fd6e4fed6b683",
                  "bytes": 1759
                },
                "summary.json": {
                  "sha256": "944ea5a9bc04c7fe83e573753b275dfb04a79a8c7b53a39a5865935847535491",
                  "bytes": 16843
                },
                "cost-summary.json": {
                  "sha256": "15d96e4f1be2d4e8bed36c752ba5cb75f722126ce093ff55bbd55a36e4fe9bce",
                  "bytes": 6215
                },
                "workspace-source.json": {
                  "sha256": "9a2eeefcc703297374921ba63e3f78eedab5b02fa03310e954f1e53fb7e991c1",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            }
          ],
          "reducedByMedian": {
            "cost.total": 1.4177250000000001,
            "duration.stepsMs": 361023,
            "usage.promptTokens": 903285,
            "usage.outputTokens": 8348,
            "usage.totalTokens": 911163
          },
          "passRate": 1.0,
          "meanVerifierReward": 0.9333333333333332
        }
      }
    },
    {
      "caseId": "cbm-extension-config-scope",
      "sourceCommits": [
        "51a18d8f0c853a06867ddbd48046ad4a84307058"
      ],
      "arms": {
        "felan-cbm-off": {
          "attemptCount": 3,
          "attempts": [
            {
              "runId": "cbm-extension-config-scope-felan-cbm-off-2026-09-04T13-20-05-757Z-6",
              "caseId": "cbm-extension-config-scope",
              "arm": "felan-cbm-off",
              "attemptNumber": 1,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "51a18d8f0c853a06867ddbd48046ad4a84307058"
              },
              "metrics": {
                "cost.total": 3.106707000000001,
                "duration.ms": 1026870,
                "duration.stepsMs": 951847,
                "usage.promptTokens": 4188849,
                "usage.outputTokens": 16513,
                "usage.totalTokens": 4205362
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "4d3a794a1e527ec96439ab82f3a641c1194b903b6b60482577789d162b73f40e",
                  "bytes": 1786
                },
                "summary.json": {
                  "sha256": "e93cc6fae6701a8e5aa1c770a092238f914c7334b2b2e36ef47e5b31b75e8f2f",
                  "bytes": 21481
                },
                "cost-summary.json": {
                  "sha256": "3a08646d4c6e792ceb60bb3b5b79efbd909c0b357385def3a4ffa9329787a374",
                  "bytes": 6239
                },
                "workspace-source.json": {
                  "sha256": "1714e85bcbfcadc9209b43f611a1393d3791f61017684dfc42deb37bcbd5f401",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            },
            {
              "runId": "cbm-extension-config-scope-felan-cbm-off-2026-09-04T13-37-12-857Z-7",
              "caseId": "cbm-extension-config-scope",
              "arm": "felan-cbm-off",
              "attemptNumber": 2,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "51a18d8f0c853a06867ddbd48046ad4a84307058"
              },
              "metrics": {
                "cost.total": 3.236997999999999,
                "duration.ms": 947237,
                "duration.stepsMs": 875685,
                "usage.promptTokens": 4420046,
                "usage.outputTokens": 15520,
                "usage.totalTokens": 4435566
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "c44eb5158fffc0cc3b50b794c12db48bb4e2c6a946f04fae68a861f7f215f72e",
                  "bytes": 1786
                },
                "summary.json": {
                  "sha256": "1fcf12771af9610eefcc12686831418b653caab2d233eab8ad78f35162443df0",
                  "bytes": 21479
                },
                "cost-summary.json": {
                  "sha256": "384db637b7f5d12f710332efdbec4351928dea4a4641eb9cb7f865bc6fd3e9b2",
                  "bytes": 6239
                },
                "workspace-source.json": {
                  "sha256": "1714e85bcbfcadc9209b43f611a1393d3791f61017684dfc42deb37bcbd5f401",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            },
            {
              "runId": "cbm-extension-config-scope-felan-cbm-off-2026-09-04T13-53-00-314Z-8",
              "caseId": "cbm-extension-config-scope",
              "arm": "felan-cbm-off",
              "attemptNumber": 3,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "51a18d8f0c853a06867ddbd48046ad4a84307058"
              },
              "metrics": {
                "cost.total": 2.6328580000000006,
                "duration.ms": 868692,
                "duration.stepsMs": 793706,
                "usage.promptTokens": 3228752,
                "usage.outputTokens": 13451,
                "usage.totalTokens": 3242203
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "26d92f7a31496432aa3cf8c56eaf7580c1bcc0f41c785a77ba65e20b300fa401",
                  "bytes": 1786
                },
                "summary.json": {
                  "sha256": "55ab9742e7c5a7b43460906e9de152bf0610d09116d4f90937fa22da24e2714b",
                  "bytes": 21498
                },
                "cost-summary.json": {
                  "sha256": "39a5aa5060225747cbf7c3472aee05fa791f5632491cd94f877b065ee4d9b41f",
                  "bytes": 6251
                },
                "workspace-source.json": {
                  "sha256": "1714e85bcbfcadc9209b43f611a1393d3791f61017684dfc42deb37bcbd5f401",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            }
          ],
          "reducedByMedian": {
            "cost.total": 3.106707000000001,
            "duration.stepsMs": 875685,
            "usage.promptTokens": 4188849,
            "usage.outputTokens": 15520,
            "usage.totalTokens": 4205362
          },
          "passRate": 1.0,
          "meanVerifierReward": 1.0
        },
        "felan-cbm-on": {
          "attemptCount": 3,
          "attempts": [
            {
              "runId": "cbm-extension-config-scope-felan-cbm-on-2026-09-04T14-07-29-236Z-9",
              "caseId": "cbm-extension-config-scope",
              "arm": "felan-cbm-on",
              "attemptNumber": 1,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "51a18d8f0c853a06867ddbd48046ad4a84307058"
              },
              "metrics": {
                "cost.total": 2.639272000000001,
                "duration.ms": 822494,
                "duration.stepsMs": 747542,
                "usage.promptTokens": 3149912,
                "usage.outputTokens": 15112,
                "usage.totalTokens": 3165024
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "1ea50d167fa15efcd0344d67a36247cfdb4c4f1f0550df17e20b3f44ba215c98",
                  "bytes": 1781
                },
                "summary.json": {
                  "sha256": "cb2ab8f880964646bf7a1d4eaeee2732f588f61d7a66caa5ea2f588693b30800",
                  "bytes": 21463
                },
                "cost-summary.json": {
                  "sha256": "c189a97457a7d68b368d213c3b30fd78a548b1463d991177b80d047970e5883f",
                  "bytes": 6233
                },
                "workspace-source.json": {
                  "sha256": "1714e85bcbfcadc9209b43f611a1393d3791f61017684dfc42deb37bcbd5f401",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            },
            {
              "runId": "cbm-extension-config-scope-felan-cbm-on-2026-09-04T14-21-11-934Z-10",
              "caseId": "cbm-extension-config-scope",
              "arm": "felan-cbm-on",
              "attemptNumber": 2,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "51a18d8f0c853a06867ddbd48046ad4a84307058"
              },
              "metrics": {
                "cost.total": 2.6679589999999993,
                "duration.ms": 986838,
                "duration.stepsMs": 916796,
                "usage.promptTokens": 3637297,
                "usage.outputTokens": 14671,
                "usage.totalTokens": 3651968
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "deef7dabbf1440aa1d7f6b098e8e7270263d32f3f3d9bfff9c95a1e45795f865",
                  "bytes": 1783
                },
                "summary.json": {
                  "sha256": "ba3cd0a4842c58a8ecb22717f52ddc9a6a7ef07c5eb9918b61b180ce6585ca19",
                  "bytes": 21464
                },
                "cost-summary.json": {
                  "sha256": "4ac4e41438f35a66eecd6279fc4f1f6b8203d12f81fc52909a27e5226896dc2e",
                  "bytes": 6228
                },
                "workspace-source.json": {
                  "sha256": "1714e85bcbfcadc9209b43f611a1393d3791f61017684dfc42deb37bcbd5f401",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            },
            {
              "runId": "cbm-extension-config-scope-felan-cbm-on-2026-09-04T14-37-38-933Z-11",
              "caseId": "cbm-extension-config-scope",
              "arm": "felan-cbm-on",
              "attemptNumber": 3,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "51a18d8f0c853a06867ddbd48046ad4a84307058"
              },
              "metrics": {
                "cost.total": 2.5433260000000004,
                "duration.ms": 874193,
                "duration.stepsMs": 802482,
                "usage.promptTokens": 3150464,
                "usage.outputTokens": 13089,
                "usage.totalTokens": 3163553
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "b7ef23abb4b40017fdcbf13d6fa4983270661ff829342811a51e0d157d5dc3a1",
                  "bytes": 1783
                },
                "summary.json": {
                  "sha256": "05c6df3e790e9ce10c36eb6dac0dbc6da1106b335b5f98537f9cc1a0a35fa75b",
                  "bytes": 21492
                },
                "cost-summary.json": {
                  "sha256": "4abb1f76799aa99468d1adbbcad1a551908e6deda6eee80480f07b3ad83a56b1",
                  "bytes": 6248
                },
                "workspace-source.json": {
                  "sha256": "1714e85bcbfcadc9209b43f611a1393d3791f61017684dfc42deb37bcbd5f401",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            }
          ],
          "reducedByMedian": {
            "cost.total": 2.639272000000001,
            "duration.stepsMs": 802482,
            "usage.promptTokens": 3150464,
            "usage.outputTokens": 14671,
            "usage.totalTokens": 3165024
          },
          "passRate": 1.0,
          "meanVerifierReward": 1.0
        }
      }
    },
    {
      "caseId": "cbm-project-instructions",
      "sourceCommits": [
        "104faa5559029c8be9e8a1eb504d87974a5864e9"
      ],
      "arms": {
        "felan-cbm-off": {
          "attemptCount": 3,
          "attempts": [
            {
              "runId": "cbm-project-instructions-felan-cbm-off-2026-09-04T14-52-13-277Z-12",
              "caseId": "cbm-project-instructions",
              "arm": "felan-cbm-off",
              "attemptNumber": 1,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "104faa5559029c8be9e8a1eb504d87974a5864e9"
              },
              "metrics": {
                "cost.total": 0.9669319999999999,
                "duration.ms": 388133,
                "duration.stepsMs": 353422,
                "usage.promptTokens": 884884,
                "usage.outputTokens": 7400,
                "usage.totalTokens": 892284
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "a2d39d56179dd042b5e5940b5164aaa76fe3262b78c4bdefd1e6cc9aba8b97a7",
                  "bytes": 1736
                },
                "summary.json": {
                  "sha256": "56bc07f0f4773ce7093ccec13b38b2ed6627d82c04f4a635a6052aa6fb4fabbc",
                  "bytes": 18041
                },
                "cost-summary.json": {
                  "sha256": "a6f51790b4c6dea3b9db99f8fb9351dedf1edef7801ea0afc592375749bc2630",
                  "bytes": 6165
                },
                "workspace-source.json": {
                  "sha256": "aedfb9e29e32db9079c05eb7bb5524b91dcfde2842054a3acac0bf081c337c23",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            },
            {
              "runId": "cbm-project-instructions-felan-cbm-off-2026-09-04T14-58-41-496Z-13",
              "caseId": "cbm-project-instructions",
              "arm": "felan-cbm-off",
              "attemptNumber": 2,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "104faa5559029c8be9e8a1eb504d87974a5864e9"
              },
              "metrics": {
                "cost.total": 0.821042,
                "duration.ms": 319991,
                "duration.stepsMs": 285927,
                "usage.promptTokens": 555640,
                "usage.outputTokens": 6339,
                "usage.totalTokens": 561979
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "e696fe31dffaf446b02fd99e659db11207f566a84add8b32bbea20e2e0c55040",
                  "bytes": 1736
                },
                "summary.json": {
                  "sha256": "3cc22af6fd2aa3a319600e265f0d4629db2e654a11f5d239ce149414f0520b5d",
                  "bytes": 17851
                },
                "cost-summary.json": {
                  "sha256": "d366aab4c3bdfb70cd7962dfc5cd47d98bb218f66810a4764867e091140e78e7",
                  "bytes": 6045
                },
                "workspace-source.json": {
                  "sha256": "aedfb9e29e32db9079c05eb7bb5524b91dcfde2842054a3acac0bf081c337c23",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            },
            {
              "runId": "cbm-project-instructions-felan-cbm-off-2026-09-04T15-04-01-556Z-14",
              "caseId": "cbm-project-instructions",
              "arm": "felan-cbm-off",
              "attemptNumber": 3,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "104faa5559029c8be9e8a1eb504d87974a5864e9"
              },
              "metrics": {
                "cost.total": 0.9696010000000002,
                "duration.ms": 446984,
                "duration.stepsMs": 386601,
                "usage.promptTokens": 954887,
                "usage.outputTokens": 7745,
                "usage.totalTokens": 962632
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "7ecb809405a6ef8055791bcdcecb67b6aabd3118263c7fe66a662582a626d91f",
                  "bytes": 1736
                },
                "summary.json": {
                  "sha256": "53ee124c79db8febd709d9858b2b81afb1048f7bd921188110886530a9f95e6a",
                  "bytes": 18041
                },
                "cost-summary.json": {
                  "sha256": "2d29cbb94736235150b474441df648238ba771125327167d393e255ff3be41f2",
                  "bytes": 6165
                },
                "workspace-source.json": {
                  "sha256": "aedfb9e29e32db9079c05eb7bb5524b91dcfde2842054a3acac0bf081c337c23",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            }
          ],
          "reducedByMedian": {
            "cost.total": 0.9669319999999999,
            "duration.stepsMs": 353422,
            "usage.promptTokens": 884884,
            "usage.outputTokens": 7400,
            "usage.totalTokens": 892284
          },
          "passRate": 1.0,
          "meanVerifierReward": 1
        },
        "felan-cbm-on": {
          "attemptCount": 3,
          "attempts": [
            {
              "runId": "cbm-project-instructions-felan-cbm-on-2026-09-04T15-11-28-612Z-15",
              "caseId": "cbm-project-instructions",
              "arm": "felan-cbm-on",
              "attemptNumber": 1,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "104faa5559029c8be9e8a1eb504d87974a5864e9"
              },
              "metrics": {
                "cost.total": 1.4798630000000002,
                "duration.ms": 447035,
                "duration.stepsMs": 410782,
                "usage.promptTokens": 1095703,
                "usage.outputTokens": 8830,
                "usage.totalTokens": 1104533
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "d43bd5cba878d5ec2179ff2eaf9e2c5b07827a0e633463612010c6e477dc8c44",
                  "bytes": 1731
                },
                "summary.json": {
                  "sha256": "d1cac10e8e40e53d5f4c0e796933789f93b118d545d7b1f4782c1d2381d1f5e2",
                  "bytes": 18055
                },
                "cost-summary.json": {
                  "sha256": "ab8bde41fb545c56058abfc99509f00c9647bc93c8c0283e5e0f125cd9bedd5b",
                  "bytes": 6200
                },
                "workspace-source.json": {
                  "sha256": "aedfb9e29e32db9079c05eb7bb5524b91dcfde2842054a3acac0bf081c337c23",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            },
            {
              "runId": "cbm-project-instructions-felan-cbm-on-2026-09-04T15-18-55-726Z-16",
              "caseId": "cbm-project-instructions",
              "arm": "felan-cbm-on",
              "attemptNumber": 2,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "104faa5559029c8be9e8a1eb504d87974a5864e9"
              },
              "metrics": {
                "cost.total": 0.9439329999999998,
                "duration.ms": 422376,
                "duration.stepsMs": 382708,
                "usage.promptTokens": 925379,
                "usage.outputTokens": 7161,
                "usage.totalTokens": 932540
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "10ed2fa03522fb99e44ab7b30863f081c673d4f626c2d4144bb91fdfd3719e3e",
                  "bytes": 1731
                },
                "summary.json": {
                  "sha256": "37e13aa3c8c0da2ec245e310570a7eaffdbd8e0affe42827fc027e96b291df68",
                  "bytes": 18025
                },
                "cost-summary.json": {
                  "sha256": "4f67c11e72b5508488956359a53ee7caea81b834becd010090c3e8956efd7ab8",
                  "bytes": 6159
                },
                "workspace-source.json": {
                  "sha256": "aedfb9e29e32db9079c05eb7bb5524b91dcfde2842054a3acac0bf081c337c23",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            },
            {
              "runId": "cbm-project-instructions-felan-cbm-on-2026-09-04T15-25-58-176Z-17",
              "caseId": "cbm-project-instructions",
              "arm": "felan-cbm-on",
              "attemptNumber": 3,
              "status": "passed",
              "pass": true,
              "provider": "openai-codex",
              "model": "gpt-5.6-sol",
              "thinking": "medium",
              "packageVersion": "0.21.11",
              "benchmark": {
                "id": "codebase-memory",
                "revision": 3,
                "digest": "b2df3463b963f3d36003bc48391eef139fd32e9dfa6a1a9ab51f45441398d0d6"
              },
              "source": {
                "repository": "https://github.com/felan-ai/felan.git",
                "commit": "104faa5559029c8be9e8a1eb504d87974a5864e9"
              },
              "metrics": {
                "cost.total": 0.908612,
                "duration.ms": 408587,
                "duration.stepsMs": 360728,
                "usage.promptTokens": 804178,
                "usage.outputTokens": 7579,
                "usage.totalTokens": 811757
              },
              "verifierReward": 1,
              "evidenceFiles": {
                "run-started.json": {
                  "sha256": "b04e3f1baafb99849a9c469915c8cb17fc9904cc62c410756179ab51504d3c2e",
                  "bytes": 1731
                },
                "summary.json": {
                  "sha256": "dae2f31f3e12c7b03296d9ed5e5658a0fa6a402cf517fd77ca7eb6c780a0b7bd",
                  "bytes": 17837
                },
                "cost-summary.json": {
                  "sha256": "cc044fdcdca7cec54d6bbc75ed4c208e0596fd9b75e5e9201c45ee8892b84d44",
                  "bytes": 6039
                },
                "workspace-source.json": {
                  "sha256": "aedfb9e29e32db9079c05eb7bb5524b91dcfde2842054a3acac0bf081c337c23",
                  "bytes": 133
                },
                "verifier/reward.json": {
                  "sha256": "d2c27300a81c0056768a90237e8eec2f646fef44db730d2eaf75562394024156",
                  "bytes": 131
                }
              }
            }
          ],
          "reducedByMedian": {
            "cost.total": 0.9439329999999998,
            "duration.stepsMs": 382708,
            "usage.promptTokens": 925379,
            "usage.outputTokens": 7579,
            "usage.totalTokens": 932540
          },
          "passRate": 1.0,
          "meanVerifierReward": 1
        }
      }
    }
  ],
  "qualityDisclosure": {
    "frameworkPassRate": "All 18 published attempts reported status passed and pass=true.",
    "architectureCandidateVerifierRewards": [
      1.0,
      0.8,
      1.0
    ],
    "architectureCandidateVerifierNote": "The 0.8 reward reported coverage 4/5 and missed tool-to-model-path.",
    "configScopeVerifierRewards": {
      "felan-cbm-off": [
        1.0,
        1.0,
        1.0
      ],
      "felan-cbm-on": [
        1.0,
        1.0,
        1.0
      ]
    },
    "projectInstructionsVerifierRewards": {
      "felan-cbm-off": [
        1.0,
        1.0,
        1.0
      ],
      "felan-cbm-on": [
        1.0,
        1.0,
        1.0
      ]
    },
    "nonRequiredAssertionNote": "All six architecture attempts recorded a non-required clean-tool-execution failure, primarily grep ENOENT; these failures did not affect framework pass status."
  },
  "excludedFromPublication": [
    "credentials",
    "auth/model stores",
    "raw workspaces",
    "full transcripts",
    "session JSONL",
    "logs",
    "verifier command metadata",
    "hidden verifier assets",
    "local absolute paths",
    ".DS_Store"
  ],
  "publicationScope": {
    "status": "final",
    "caseCount": 3,
    "runCount": 18,
    "trialsPerArmPerCase": 3,
    "caseIds": [
      "cbm-extension-architecture",
      "cbm-extension-config-scope",
      "cbm-project-instructions"
    ]
  },
  "comparisons": {
    "cost.total": {
      "baselineReducedCaseSum": 5.274961000000001,
      "candidateReducedCaseSum": 5.000930000000001,
      "ratioOfReducedSumsGainPercent": 5.194938882012585,
      "macroMeanCaseGainPercent": -0.1963961754850403,
      "caseGainsPercent": [
        -18.013738198418086,
        15.045995647481398,
        2.378554024481567
      ],
      "rawAttemptTotalGainPercent": -1.2858637153061812
    },
    "duration.stepsMs": {
      "baselineReducedCaseSum": 1594397,
      "candidateReducedCaseSum": 1546213,
      "ratioOfReducedSumsGainPercent": 3.0220829567541836,
      "macroMeanCaseGainPercent": 0.413738232914393,
      "caseGainsPercent": [
        1.1681130061047387,
        8.3595128385207,
        -8.28641114588226
      ],
      "rawAttemptTotalGainPercent": 4.201261741781832
    },
    "usage.promptTokens": {
      "baselineReducedCaseSum": 5902049,
      "candidateReducedCaseSum": 4979128,
      "ratioOfReducedSumsGainPercent": 15.637298165433734,
      "macroMeanCaseGainPercent": 3.7207296489539305,
      "caseGainsPercent": [
        -9.050772893436804,
        24.789267887192878,
        -4.576306046894282
      ],
      "rawAttemptTotalGainPercent": 6.961148281971704
    },
    "usage.outputTokens": {
      "baselineReducedCaseSum": 31511,
      "candidateReducedCaseSum": 30598,
      "ratioOfReducedSumsGainPercent": 2.8974009076195615,
      "macroMeanCaseGainPercent": 1.9599944675795753,
      "caseGainsPercent": [
        2.828541496915377,
        5.470360824742268,
        -2.418918918918919
      ],
      "rawAttemptTotalGainPercent": 1.2403519188211798
    },
    "usage.totalTokens": {
      "baselineReducedCaseSum": 5934763,
      "candidateReducedCaseSum": 5008727,
      "ratioOfReducedSumsGainPercent": 15.603588551050818,
      "macroMeanCaseGainPercent": 3.793814376349093,
      "caseGainsPercent": [
        -8.84535853411172,
        24.73836972893178,
        -4.511568065772781
      ],
      "rawAttemptTotalGainPercent": 6.9296409113431325
    }
  },
  "sourceScopeNotes": [
    "The published result covers all 18 attempts across the three cases declared by the source batch manifest.",
    "The six cbm-project-instructions run directories were supplied separately and complete the original batch matrix.",
    "The exact source harness-evals.yaml for the batch was not supplied; the source benchmark revision and digest are retained from run metadata."
  ]
}