{
  "schemaVersion": 1,
  "id": "codebase-memory",
  "benchmarkId": "codebase-memory",
  "label": "Codebase Memory retrieval cost",
  "status": "complete",
  "publicationEligible": true,
  "coverage": {
    "cases": 3,
    "runs": 18,
    "trialsPerArmPerCase": 3,
    "caseIds": [
      "cbm-extension-architecture",
      "cbm-extension-config-scope",
      "cbm-project-instructions"
    ]
  },
  "arms": {
    "baseline": "felan-cbm-off",
    "candidate": "felan-cbm-on"
  },
  "trialReducer": "median",
  "caseEstimator": "ratioOfReducedSums",
  "comparisons": {
    "cost.total": {
      "baselineReducedCaseSum": 5.274961000000001,
      "candidateReducedCaseSum": 5.000930000000001,
      "ratioOfReducedSumsGainPercent": 5.194938882012585,
      "macroMeanCaseGainPercent": -0.1963961754850403,
      "caseGainsPercent": [
        -18.013738198418086,
        15.045995647481398,
        2.378554024481567
      ],
      "rawAttemptTotalGainPercent": -1.2858637153061812
    },
    "duration.stepsMs": {
      "baselineReducedCaseSum": 1594397,
      "candidateReducedCaseSum": 1546213,
      "ratioOfReducedSumsGainPercent": 3.0220829567541836,
      "macroMeanCaseGainPercent": 0.413738232914393,
      "caseGainsPercent": [
        1.1681130061047387,
        8.3595128385207,
        -8.28641114588226
      ],
      "rawAttemptTotalGainPercent": 4.201261741781832
    },
    "usage.promptTokens": {
      "baselineReducedCaseSum": 5902049,
      "candidateReducedCaseSum": 4979128,
      "ratioOfReducedSumsGainPercent": 15.637298165433734,
      "macroMeanCaseGainPercent": 3.7207296489539305,
      "caseGainsPercent": [
        -9.050772893436804,
        24.789267887192878,
        -4.576306046894282
      ],
      "rawAttemptTotalGainPercent": 6.961148281971704
    },
    "usage.outputTokens": {
      "baselineReducedCaseSum": 31511,
      "candidateReducedCaseSum": 30598,
      "ratioOfReducedSumsGainPercent": 2.8974009076195615,
      "macroMeanCaseGainPercent": 1.9599944675795753,
      "caseGainsPercent": [
        2.828541496915377,
        5.470360824742268,
        -2.418918918918919
      ],
      "rawAttemptTotalGainPercent": 1.2403519188211798
    },
    "usage.totalTokens": {
      "baselineReducedCaseSum": 5934763,
      "candidateReducedCaseSum": 5008727,
      "ratioOfReducedSumsGainPercent": 15.603588551050818,
      "macroMeanCaseGainPercent": 3.793814376349093,
      "caseGainsPercent": [
        -8.84535853411172,
        24.73836972893178,
        -4.511568065772781
      ],
      "rawAttemptTotalGainPercent": 6.9296409113431325
    }
  },
  "cases": [
    {
      "caseId": "cbm-extension-architecture",
      "sourceCommits": [
        "e5867637569bd1c7ad08420b79ec4031a5733f57"
      ],
      "baseline": {
        "reducedByMedian": {
          "cost.total": 1.201322,
          "duration.stepsMs": 365290,
          "usage.promptTokens": 828316,
          "usage.outputTokens": 8591,
          "usage.totalTokens": 837117
        },
        "passRate": 1.0,
        "meanVerifierReward": 1.0
      },
      "candidate": {
        "reducedByMedian": {
          "cost.total": 1.4177250000000001,
          "duration.stepsMs": 361023,
          "usage.promptTokens": 903285,
          "usage.outputTokens": 8348,
          "usage.totalTokens": 911163
        },
        "passRate": 1.0,
        "meanVerifierReward": 0.9333333333333332
      }
    },
    {
      "caseId": "cbm-extension-config-scope",
      "sourceCommits": [
        "51a18d8f0c853a06867ddbd48046ad4a84307058"
      ],
      "baseline": {
        "reducedByMedian": {
          "cost.total": 3.106707000000001,
          "duration.stepsMs": 875685,
          "usage.promptTokens": 4188849,
          "usage.outputTokens": 15520,
          "usage.totalTokens": 4205362
        },
        "passRate": 1.0,
        "meanVerifierReward": 1.0
      },
      "candidate": {
        "reducedByMedian": {
          "cost.total": 2.639272000000001,
          "duration.stepsMs": 802482,
          "usage.promptTokens": 3150464,
          "usage.outputTokens": 14671,
          "usage.totalTokens": 3165024
        },
        "passRate": 1.0,
        "meanVerifierReward": 1.0
      }
    },
    {
      "caseId": "cbm-project-instructions",
      "sourceCommits": [
        "104faa5559029c8be9e8a1eb504d87974a5864e9"
      ],
      "baseline": {
        "reducedByMedian": {
          "cost.total": 0.9669319999999999,
          "duration.stepsMs": 353422,
          "usage.promptTokens": 884884,
          "usage.outputTokens": 7400,
          "usage.totalTokens": 892284
        },
        "passRate": 1.0,
        "meanVerifierReward": 1
      },
      "candidate": {
        "reducedByMedian": {
          "cost.total": 0.9439329999999998,
          "duration.stepsMs": 382708,
          "usage.promptTokens": 925379,
          "usage.outputTokens": 7579,
          "usage.totalTokens": 932540
        },
        "passRate": 1.0,
        "meanVerifierReward": 1
      }
    }
  ],
  "qualityDisclosure": {
    "frameworkPassRate": "All 18 published attempts reported status passed and pass=true.",
    "architectureCandidateVerifierRewards": [
      1.0,
      0.8,
      1.0
    ],
    "architectureCandidateVerifierNote": "The 0.8 reward reported coverage 4/5 and missed tool-to-model-path.",
    "configScopeVerifierRewards": {
      "felan-cbm-off": [
        1.0,
        1.0,
        1.0
      ],
      "felan-cbm-on": [
        1.0,
        1.0,
        1.0
      ]
    },
    "projectInstructionsVerifierRewards": {
      "felan-cbm-off": [
        1.0,
        1.0,
        1.0
      ],
      "felan-cbm-on": [
        1.0,
        1.0,
        1.0
      ]
    },
    "nonRequiredAssertionNote": "All six architecture attempts recorded a non-required clean-tool-execution failure, primarily grep ENOENT; these failures did not affect framework pass status."
  },
  "evidence": "../../sources/provenance/codebase-memory.json",
  "sourceBatchCoverage": {
    "declaredCases": 3,
    "declaredRuns": 18,
    "retainedCases": 3,
    "retainedRuns": 18,
    "complete": true
  }
}