{
 "runId": "luna6-compaction-v2",
 "version": 2,
 "model": "gpt-6-luna",
 "dataset": "Terminal-Bench 2.1",
 "threshold": 50000,
 "repeats": 4,
 "armLabels": {
  "TextualFixed50kPi": "Pi textual",
  "Native50kPi": "OpenAI encrypted"
 },
 "updatedAt": "2026-09-26T21:55:16.182Z",
 "runState": "COMPLETE",
 "phase": "",
 "startedAt": "2026-09-25T13:44:01.550Z",
 "completedTrials": 192,
 "plannedTrials": 192,
 "totalCostUsd": 9.835129535000025,
 "preflightCostUsd": 0.08403918500000002,
 "arms": {
  "TextualFixed50kPi": {
   "passes": 57,
   "fails": 34,
   "errors": 5,
   "running": 0,
   "complete": 91,
   "costUsd": 5.656394679999998,
   "qualifiedCompactions": 151,
   "compactingTrials": 44
  },
  "Native50kPi": {
   "passes": 56,
   "fails": 37,
   "errors": 3,
   "running": 0,
   "complete": 93,
   "costUsd": 4.094695669999998,
   "qualifiedCompactions": 100,
   "compactingTrials": 49
  }
 },
 "sets": {
  "primary": {
   "taskCount": 12,
   "tasks": [
    "make-mips-interpreter",
    "fix-ocaml-gc",
    "make-doom-for-mips",
    "caffe-cifar-10",
    "path-tracing-reverse",
    "compile-compcert",
    "regex-chess",
    "path-tracing",
    "build-pov-ray",
    "mcmc-sampling-stan",
    "mteb-leaderboard",
    "protein-assembly"
   ],
   "arms": {
    "TextualFixed50kPi": {
     "passes": 25,
     "scored": 43,
     "meanPassRate": 0.569
    },
    "Native50kPi": {
     "passes": 24,
     "scored": 46,
     "meanPassRate": 0.535
    }
   },
   "effect": {
    "meanPp": -3.5,
    "lowPp": -15.3,
    "highPp": 6.9,
    "tasks": 12
   },
   "pairs": {
    "bothPass": 18,
    "bothFail": 12,
    "encryptedOnly": 5,
    "textualOnly": 7,
    "signTestP": 0.774
   }
  },
  "all": {
   "taskCount": 24,
   "tasks": [
    "make-mips-interpreter",
    "fix-ocaml-gc",
    "make-doom-for-mips",
    "caffe-cifar-10",
    "path-tracing-reverse",
    "compile-compcert",
    "regex-chess",
    "gpt2-codegolf",
    "path-tracing",
    "build-pov-ray",
    "mcmc-sampling-stan",
    "rstan-to-pystan",
    "distribution-search",
    "mteb-leaderboard",
    "portfolio-optimization",
    "reshard-c4-data",
    "sam-cell-seg",
    "train-fasttext",
    "winning-avg-corewars",
    "dna-assembly",
    "dna-insert",
    "llm-inference-batching-scheduler",
    "mteb-retrieve",
    "protein-assembly"
   ],
   "arms": {
    "TextualFixed50kPi": {
     "passes": 57,
     "scored": 91,
     "meanPassRate": 0.618
    },
    "Native50kPi": {
     "passes": 56,
     "scored": 93,
     "meanPassRate": 0.601
    }
   },
   "effect": {
    "meanPp": -1.7,
    "lowPp": -8.3,
    "highPp": 4.2,
    "tasks": 24
   },
   "pairs": {
    "bothPass": 49,
    "bothFail": 27,
    "encryptedOnly": 6,
    "textualOnly": 7,
    "signTestP": 1
   }
  },
  "placebo": {
   "taskCount": 6,
   "tasks": [
    "rstan-to-pystan",
    "distribution-search",
    "portfolio-optimization",
    "sam-cell-seg",
    "dna-assembly",
    "mteb-retrieve"
   ],
   "arms": {
    "TextualFixed50kPi": {
     "passes": 19,
     "scored": 24,
     "meanPassRate": 0.792
    },
    "Native50kPi": {
     "passes": 20,
     "scored": 24,
     "meanPassRate": 0.833
    }
   },
   "effect": {
    "meanPp": 4.2,
    "lowPp": 0,
    "highPp": 12.5,
    "tasks": 6
   },
   "pairs": {
    "bothPass": 19,
    "bothFail": 4,
    "encryptedOnly": 1,
    "textualOnly": 0,
    "signTestP": 1
   }
  }
 },
 "mechanics": {
  "TextualFixed50kPi": {
   "trials": 91,
   "meanCompactions": 1.52,
   "compactingTrials": 42,
   "medianTokensAtCompaction": 51478,
   "medianResidualTokens": 34427,
   "meanRequests": 37.2,
   "meanCostUsd": 0.0375,
   "meanTotalTokens": 1176378,
   "medianDurationSec": 512,
   "timedOut": 13,
   "readsBefore": 0.28,
   "readsAfter": 0.32
  },
  "Native50kPi": {
   "trials": 93,
   "meanCompactions": 1.03,
   "compactingTrials": 47,
   "medianTokensAtCompaction": 52476,
   "medianResidualTokens": 4407,
   "meanRequests": 37,
   "meanCostUsd": 0.0337,
   "meanTotalTokens": 914676,
   "medianDurationSec": 439,
   "timedOut": 13,
   "readsBefore": 0.15,
   "readsAfter": 0.51
  }
 },
 "tasks": [
  {
   "name": "make-mips-interpreter",
   "set": "A",
   "timeoutSec": 1800,
   "category": "software-engineering",
   "task": "make-mips-interpreter",
   "compactionRate": 1,
   "group": "primary",
   "outcome": "Tied",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 2,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "make-mips-interpreter__5mx3Myg",
       "job": "r0-00-make-mips-interpreter",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 10,
       "qualifiedCompactions": 10,
       "compactions": [
        {
         "at": "2026-09-25T13:46:41.670Z",
         "tokensAt": 51717,
         "residualTokens": 45207,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T13:47:50.112Z",
         "tokensAt": 53496,
         "residualTokens": 45891,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T13:48:40.904Z",
         "tokensAt": 50515,
         "residualTokens": 35262,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T13:49:46.421Z",
         "tokensAt": 50382,
         "residualTokens": 43282,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T13:50:26.270Z",
         "tokensAt": 51782,
         "residualTokens": 47885,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T13:51:21.777Z",
         "tokensAt": 50712,
         "residualTokens": 38054,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T13:52:30.878Z",
         "tokensAt": 53202,
         "residualTokens": 35083,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T13:54:17.608Z",
         "tokensAt": 51854,
         "residualTokens": 34448,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T13:55:22.384Z",
         "tokensAt": 53145,
         "residualTokens": 39518,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T13:56:45.106Z",
         "tokensAt": 50280,
         "residualTokens": 37145,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        }
       ],
       "providerRequests": 75,
       "agentRequests": 61,
       "costUsd": 0.12929897499999998,
       "inputTokens": 2528011,
       "outputTokens": 69102,
       "cacheTokens": 1923505,
       "totalTokens": 2597113,
       "durationSec": 1913.837,
       "startedAt": "2026-09-25T13:44:02.084886Z",
       "finishedAt": "2026-09-25T14:15:55.921614Z",
       "taskChecksum": "17bee6fa30f7b36123f86b6b0594c89eb80060b423af1134adb4bf143fe29040",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-mips-interpreter__X4nnivS",
       "job": "r1-00-make-mips-interpreter",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 5,
       "qualifiedCompactions": 5,
       "compactions": [
        {
         "at": "2026-09-25T18:06:39.602Z",
         "tokensAt": 54635,
         "residualTokens": 32349,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T18:09:33.749Z",
         "tokensAt": 50828,
         "residualTokens": 45477,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T18:10:21.787Z",
         "tokensAt": 52060,
         "residualTokens": 34405,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T18:12:01.409Z",
         "tokensAt": 54083,
         "residualTokens": 42309,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 1
        },
        {
         "at": "2026-09-25T18:12:33.314Z",
         "tokensAt": 50666,
         "residualTokens": 32642,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 1
        }
       ],
       "providerRequests": 53,
       "agentRequests": 46,
       "costUsd": 0.07259757000000003,
       "inputTokens": 1729705,
       "outputTokens": 36178,
       "cacheTokens": 1406092,
       "totalTokens": 1765883,
       "durationSec": 511.909,
       "startedAt": "2026-09-25T18:05:21.303708Z",
       "finishedAt": "2026-09-25T18:13:53.212207Z",
       "taskChecksum": "17bee6fa30f7b36123f86b6b0594c89eb80060b423af1134adb4bf143fe29040",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-mips-interpreter__3KgMoWF",
       "job": "r2-00-make-mips-interpreter",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 4,
       "qualifiedCompactions": 4,
       "compactions": [
        {
         "at": "2026-09-25T18:17:20.664Z",
         "tokensAt": 50312,
         "residualTokens": 26893,
         "qualified": true,
         "readsBefore": 0.8,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T18:17:54.297Z",
         "tokensAt": 52810,
         "residualTokens": 30073,
         "qualified": true,
         "readsBefore": 0.8,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T18:19:46.564Z",
         "tokensAt": 50142,
         "residualTokens": 43724,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T18:20:30.990Z",
         "tokensAt": 50609,
         "residualTokens": 40034,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0
        }
       ],
       "providerRequests": 41,
       "agentRequests": 36,
       "costUsd": 0.056715304999999994,
       "inputTokens": 1411792,
       "outputTokens": 28068,
       "cacheTokens": 1163388,
       "totalTokens": 1439860,
       "durationSec": 366.952,
       "startedAt": "2026-09-25T18:16:46.066631Z",
       "finishedAt": "2026-09-25T18:22:53.018844Z",
       "taskChecksum": "17bee6fa30f7b36123f86b6b0594c89eb80060b423af1134adb4bf143fe29040",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-mips-interpreter__rzThgmE",
       "job": "r3-00-make-mips-interpreter",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T22:52:39.990Z",
         "tokensAt": 55034,
         "residualTokens": 25408,
         "qualified": true,
         "readsBefore": 2,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T22:55:04.698Z",
         "tokensAt": 51597,
         "residualTokens": 34289,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 32,
       "agentRequests": 30,
       "costUsd": 0.04380957000000001,
       "inputTokens": 1074405,
       "outputTokens": 28529,
       "cacheTokens": 910897,
       "totalTokens": 1102934,
       "durationSec": 297.269,
       "startedAt": "2026-09-25T22:52:08.798814Z",
       "finishedAt": "2026-09-25T22:57:06.067489Z",
       "taskChecksum": "17bee6fa30f7b36123f86b6b0594c89eb80060b423af1134adb4bf143fe29040",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.5,
     "costUsd": 0.30242142,
     "qualifiedCompactions": 21
    },
    "Native50kPi": {
     "passes": 2,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "make-mips-interpreter__dK3sYCS",
       "job": "r0-00-make-mips-interpreter",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T13:46:25.533Z",
         "tokensAt": 55433,
         "residualTokens": 8316,
         "qualified": true,
         "readsBefore": 1.4,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T13:49:10.384Z",
         "tokensAt": 52017,
         "residualTokens": 4822,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.6
        }
       ],
       "providerRequests": 33,
       "agentRequests": 33,
       "costUsd": 0.044947130000000016,
       "inputTokens": 872919,
       "outputTokens": 27031,
       "cacheTokens": 648918,
       "totalTokens": 899950,
       "durationSec": 368.642,
       "startedAt": "2026-09-25T13:44:02.113616Z",
       "finishedAt": "2026-09-25T13:50:10.755313Z",
       "taskChecksum": "17bee6fa30f7b36123f86b6b0594c89eb80060b423af1134adb4bf143fe29040",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-mips-interpreter__zUGZJ33",
       "job": "r1-00-make-mips-interpreter",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 3,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T18:10:37.082Z",
         "tokensAt": 51687,
         "residualTokens": 5491,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T18:12:18.169Z",
         "tokensAt": 56910,
         "residualTokens": 6450,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T18:15:56.062Z",
         "tokensAt": 48955,
         "residualTokens": 9723,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 89,
       "agentRequests": 89,
       "costUsd": 0.07937089,
       "inputTokens": 2528778,
       "outputTokens": 38614,
       "cacheTokens": 2181354,
       "totalTokens": 2567392,
       "durationSec": 682.111,
       "startedAt": "2026-09-25T18:05:21.275872Z",
       "finishedAt": "2026-09-25T18:16:43.386202Z",
       "taskChecksum": "17bee6fa30f7b36123f86b6b0594c89eb80060b423af1134adb4bf143fe29040",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-mips-interpreter__3R2wUJP",
       "job": "r2-00-make-mips-interpreter",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T18:18:09.192Z",
         "tokensAt": 53535,
         "residualTokens": 5962,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T18:21:41.583Z",
         "tokensAt": 68618,
         "residualTokens": 5219,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 52,
       "agentRequests": 52,
       "costUsd": 0.05767956500000001,
       "inputTokens": 1440117,
       "outputTokens": 33182,
       "cacheTokens": 1175304,
       "totalTokens": 1473299,
       "durationSec": 482.426,
       "startedAt": "2026-09-25T18:16:46.098946Z",
       "finishedAt": "2026-09-25T18:24:48.524751Z",
       "taskChecksum": "17bee6fa30f7b36123f86b6b0594c89eb80060b423af1134adb4bf143fe29040",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-mips-interpreter__dPfDDFc",
       "job": "r3-00-make-mips-interpreter",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T22:53:54.527Z",
         "tokensAt": 52417,
         "residualTokens": 3863,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T22:54:41.803Z",
         "tokensAt": 50505,
         "residualTokens": 11414,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 48,
       "agentRequests": 48,
       "costUsd": 0.052920974999999995,
       "inputTokens": 1476278,
       "outputTokens": 30642,
       "cacheTokens": 1259880,
       "totalTokens": 1506920,
       "durationSec": 389.428,
       "startedAt": "2026-09-25T22:52:08.771608Z",
       "finishedAt": "2026-09-25T22:58:38.199407Z",
       "taskChecksum": "17bee6fa30f7b36123f86b6b0594c89eb80060b423af1134adb4bf143fe29040",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.5,
     "costUsd": 0.23491856,
     "qualifiedCompactions": 9
    }
   },
   "diff": 0
  },
  {
   "name": "fix-ocaml-gc",
   "set": "A",
   "timeoutSec": 3600,
   "category": "software-engineering",
   "task": "fix-ocaml-gc",
   "compactionRate": 0.875,
   "group": "primary",
   "outcome": "Tied",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 3,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "fix-ocaml-gc__Agu5qeo",
       "job": "r0-01-fix-ocaml-gc",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T13:45:03.284Z",
         "tokensAt": 50706,
         "residualTokens": 22025,
         "qualified": true,
         "readsBefore": 1,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 17,
       "agentRequests": 16,
       "costUsd": 0.01693225,
       "inputTokens": 410830,
       "outputTokens": 4556,
       "cacheTokens": 319115,
       "totalTokens": 415386,
       "durationSec": 3788.632,
       "startedAt": "2026-09-25T13:44:02.082267Z",
       "finishedAt": "2026-09-25T14:47:10.714594Z",
       "taskChecksum": "b6d57afd1632658546bfd5eab8b6c90568850a4c0c4a153dfaf3e1c0aed0ae9b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "fix-ocaml-gc__k3Snyo5",
       "job": "r1-01-fix-ocaml-gc",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T18:03:02.314Z",
         "tokensAt": 58303,
         "residualTokens": 28935,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.3333333333333333
        }
       ],
       "providerRequests": 18,
       "agentRequests": 17,
       "costUsd": 0.01633983,
       "inputTokens": 461351,
       "outputTokens": 4720,
       "cacheTokens": 379893,
       "totalTokens": 466071,
       "durationSec": 1279.11,
       "startedAt": "2026-09-25T17:53:45.941544Z",
       "finishedAt": "2026-09-25T18:15:05.051496Z",
       "taskChecksum": "b6d57afd1632658546bfd5eab8b6c90568850a4c0c4a153dfaf3e1c0aed0ae9b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "fix-ocaml-gc__5GmQCsN",
       "job": "r2-01-fix-ocaml-gc",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T18:36:59.551Z",
         "tokensAt": 56083,
         "residualTokens": 28511,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.3333333333333333
        }
       ],
       "providerRequests": 33,
       "agentRequests": 32,
       "costUsd": 0.022770465,
       "inputTokens": 943092,
       "outputTokens": 7721,
       "cacheTokens": 860644,
       "totalTokens": 950813,
       "durationSec": 1482.111,
       "startedAt": "2026-09-25T18:24:51.248065Z",
       "finishedAt": "2026-09-25T18:49:33.359196Z",
       "taskChecksum": "b6d57afd1632658546bfd5eab8b6c90568850a4c0c4a153dfaf3e1c0aed0ae9b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "fix-ocaml-gc__LAQgEA9",
       "job": "r3-01-fix-ocaml-gc",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T22:51:28.782Z",
         "tokensAt": 55458,
         "residualTokens": 29538,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.3333333333333333
        }
       ],
       "providerRequests": 42,
       "agentRequests": 41,
       "costUsd": 0.025237684999999996,
       "inputTokens": 1139143,
       "outputTokens": 8644,
       "cacheTokens": 1056296,
       "totalTokens": 1147787,
       "durationSec": 1482.45,
       "startedAt": "2026-09-25T22:39:21.448418Z",
       "finishedAt": "2026-09-25T23:04:03.898916Z",
       "taskChecksum": "b6d57afd1632658546bfd5eab8b6c90568850a4c0c4a153dfaf3e1c0aed0ae9b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.75,
     "costUsd": 0.08128023,
     "qualifiedCompactions": 4
    },
    "Native50kPi": {
     "passes": 3,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "fix-ocaml-gc__bQwcPQX",
       "job": "r0-01-fix-ocaml-gc",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T13:47:23.664Z",
         "tokensAt": 51695,
         "residualTokens": 3853,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.6
        }
       ],
       "providerRequests": 37,
       "agentRequests": 37,
       "costUsd": 0.029848150000000007,
       "inputTokens": 1004339,
       "outputTokens": 8616,
       "cacheTokens": 852850,
       "totalTokens": 1012955,
       "durationSec": 1462.435,
       "startedAt": "2026-09-25T13:44:02.056406Z",
       "finishedAt": "2026-09-25T14:08:24.491855Z",
       "taskChecksum": "b6d57afd1632658546bfd5eab8b6c90568850a4c0c4a153dfaf3e1c0aed0ae9b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "fix-ocaml-gc__Wrpi4Bn",
       "job": "r1-01-fix-ocaml-gc",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 34,
       "agentRequests": 34,
       "costUsd": 0.019335770000000002,
       "inputTokens": 849487,
       "outputTokens": 8809,
       "cacheTokens": 788772,
       "totalTokens": 858296,
       "durationSec": 3766.331,
       "startedAt": "2026-09-25T17:53:45.976433Z",
       "finishedAt": "2026-09-25T18:56:32.307560Z",
       "taskChecksum": "b6d57afd1632658546bfd5eab8b6c90568850a4c0c4a153dfaf3e1c0aed0ae9b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "fix-ocaml-gc__ASnhXvP",
       "job": "r2-01-fix-ocaml-gc",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T18:35:54.769Z",
         "tokensAt": 65285,
         "residualTokens": 1903,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 18,
       "agentRequests": 18,
       "costUsd": 0.021025125000000002,
       "inputTokens": 433542,
       "outputTokens": 5050,
       "cacheTokens": 294390,
       "totalTokens": 438592,
       "durationSec": 1414.877,
       "startedAt": "2026-09-25T18:24:51.222173Z",
       "finishedAt": "2026-09-25T18:48:26.099629Z",
       "taskChecksum": "b6d57afd1632658546bfd5eab8b6c90568850a4c0c4a153dfaf3e1c0aed0ae9b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "fix-ocaml-gc__exmQiTY",
       "job": "r3-01-fix-ocaml-gc",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T22:51:00.448Z",
         "tokensAt": 67196,
         "residualTokens": 1781,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 29,
       "agentRequests": 29,
       "costUsd": 0.025744145,
       "inputTokens": 744596,
       "outputTokens": 7104,
       "cacheTokens": 595202,
       "totalTokens": 751700,
       "durationSec": 1453.917,
       "startedAt": "2026-09-25T22:39:21.474480Z",
       "finishedAt": "2026-09-25T23:03:35.391066Z",
       "taskChecksum": "b6d57afd1632658546bfd5eab8b6c90568850a4c0c4a153dfaf3e1c0aed0ae9b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.75,
     "costUsd": 0.09595319000000001,
     "qualifiedCompactions": 3
    }
   },
   "diff": 0
  },
  {
   "name": "make-doom-for-mips",
   "set": "A",
   "timeoutSec": 900,
   "category": "software-engineering",
   "task": "make-doom-for-mips",
   "compactionRate": 1,
   "group": "primary",
   "outcome": "Both always fail",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 0,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "make-doom-for-mips__zc5kLrv",
       "job": "r0-02-make-doom-for-mips",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 10,
       "qualifiedCompactions": 10,
       "compactions": [
        {
         "at": "2026-09-25T13:47:08.520Z",
         "tokensAt": 50706,
         "residualTokens": 31550,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T13:49:02.192Z",
         "tokensAt": 51478,
         "residualTokens": 34348,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T13:51:05.673Z",
         "tokensAt": 50669,
         "residualTokens": 39468,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T13:51:46.883Z",
         "tokensAt": 51254,
         "residualTokens": 36351,
         "qualified": true,
         "readsBefore": 0.8,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T13:53:35.915Z",
         "tokensAt": 50648,
         "residualTokens": 38149,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T13:54:58.048Z",
         "tokensAt": 50229,
         "residualTokens": 36483,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T13:56:30.040Z",
         "tokensAt": 51637,
         "residualTokens": 39742,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T13:57:38.201Z",
         "tokensAt": 50068,
         "residualTokens": 39985,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T13:59:05.416Z",
         "tokensAt": 50406,
         "residualTokens": 42498,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T14:00:16.155Z",
         "tokensAt": 50010,
         "residualTokens": 40939,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0.25
        }
       ],
       "providerRequests": 182,
       "agentRequests": 166,
       "costUsd": 0.182961205,
       "inputTokens": 6996618,
       "outputTokens": 81756,
       "cacheTokens": 6369218,
       "totalTokens": 7078374,
       "durationSec": 988.865,
       "startedAt": "2026-09-25T13:44:02.073342Z",
       "finishedAt": "2026-09-25T14:00:30.938034Z",
       "taskChecksum": "d21f1e251c43a2271fb9604613da1a6f5768be18b2c5a7f05bac5b620ac271f5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-doom-for-mips__XzBnpx4",
       "job": "r1-02-make-doom-for-mips",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 8,
       "qualifiedCompactions": 8,
       "compactions": [
        {
         "at": "2026-09-25T17:51:22.440Z",
         "tokensAt": 51256,
         "residualTokens": 24297,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T17:52:45.326Z",
         "tokensAt": 50872,
         "residualTokens": 31850,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T17:55:40.430Z",
         "tokensAt": 51543,
         "residualTokens": 34003,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T17:57:47.278Z",
         "tokensAt": 50097,
         "residualTokens": 44151,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T17:58:27.094Z",
         "tokensAt": 50425,
         "residualTokens": 42479,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T17:59:31.209Z",
         "tokensAt": 50995,
         "residualTokens": 34857,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T18:02:04.904Z",
         "tokensAt": 52250,
         "residualTokens": 31916,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T18:03:33.101Z",
         "tokensAt": 51096,
         "residualTokens": 33778,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 1
        }
       ],
       "providerRequests": 163,
       "agentRequests": 148,
       "costUsd": 0.15964992500000003,
       "inputTokens": 6264015,
       "outputTokens": 73140,
       "cacheTokens": 5738165,
       "totalTokens": 6337155,
       "durationSec": 987.831,
       "startedAt": "2026-09-25T17:48:50.736416Z",
       "finishedAt": "2026-09-25T18:05:18.567947Z",
       "taskChecksum": "d21f1e251c43a2271fb9604613da1a6f5768be18b2c5a7f05bac5b620ac271f5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-doom-for-mips__7pUEGaS",
       "job": "r2-02-make-doom-for-mips",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 4,
       "qualifiedCompactions": 4,
       "compactions": [
        {
         "at": "2026-09-25T18:49:35.423Z",
         "tokensAt": 50912,
         "residualTokens": 29490,
         "qualified": true,
         "readsBefore": 1.2,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T18:52:05.548Z",
         "tokensAt": 58756,
         "residualTokens": 31470,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T18:53:08.779Z",
         "tokensAt": 50926,
         "residualTokens": 23603,
         "qualified": true,
         "readsBefore": 0.8,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T19:00:54.615Z",
         "tokensAt": 50055,
         "residualTokens": 34147,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.3333333333333333
        }
       ],
       "providerRequests": 77,
       "agentRequests": 71,
       "costUsd": 0.07674824000000001,
       "inputTokens": 2638043,
       "outputTokens": 36996,
       "cacheTokens": 2360864,
       "totalTokens": 2675039,
       "durationSec": 964.633,
       "startedAt": "2026-09-25T18:48:03.092390Z",
       "finishedAt": "2026-09-25T19:04:07.725842Z",
       "taskChecksum": "d21f1e251c43a2271fb9604613da1a6f5768be18b2c5a7f05bac5b620ac271f5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-doom-for-mips__WZj2z2s",
       "job": "r3-02-make-doom-for-mips",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 7,
       "qualifiedCompactions": 7,
       "compactions": [
        {
         "at": "2026-09-25T22:37:21.026Z",
         "tokensAt": 54567,
         "residualTokens": 28002,
         "qualified": true,
         "readsBefore": 0.8,
         "readsAfter": 1
        },
        {
         "at": "2026-09-25T22:39:16.941Z",
         "tokensAt": 57056,
         "residualTokens": 24984,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T22:42:13.280Z",
         "tokensAt": 50251,
         "residualTokens": 38459,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 1
        },
        {
         "at": "2026-09-25T22:43:34.816Z",
         "tokensAt": 51736,
         "residualTokens": 37882,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T22:44:41.122Z",
         "tokensAt": 50150,
         "residualTokens": 29965,
         "qualified": true,
         "readsBefore": 1,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T22:46:38.194Z",
         "tokensAt": 50071,
         "residualTokens": 30998,
         "qualified": true,
         "readsBefore": 0.8,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T22:48:29.473Z",
         "tokensAt": 50701,
         "residualTokens": 31537,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 163,
       "agentRequests": 149,
       "costUsd": 0.15422196500000002,
       "inputTokens": 5838703,
       "outputTokens": 79663,
       "cacheTokens": 5351444,
       "totalTokens": 5918366,
       "durationSec": 966.721,
       "startedAt": "2026-09-25T22:35:32.758928Z",
       "finishedAt": "2026-09-25T22:51:39.479659Z",
       "taskChecksum": "d21f1e251c43a2271fb9604613da1a6f5768be18b2c5a7f05bac5b620ac271f5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0,
     "costUsd": 0.5735813350000001,
     "qualifiedCompactions": 29
    },
    "Native50kPi": {
     "passes": 0,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "make-doom-for-mips__wN6chsE",
       "job": "r0-02-make-doom-for-mips",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 4,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T13:45:48.161Z",
         "tokensAt": 54561,
         "residualTokens": 22145,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 1
        },
        {
         "at": "2026-09-25T13:48:47.848Z",
         "tokensAt": 53060,
         "residualTokens": 4078,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 1.2
        },
        {
         "at": "2026-09-25T13:50:17.154Z",
         "tokensAt": 52677,
         "residualTokens": 14184,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T13:52:40.130Z",
         "tokensAt": 49865,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0.2,
         "readsAfter": 0
        }
       ],
       "providerRequests": 63,
       "agentRequests": 63,
       "costUsd": 0.07650582999999998,
       "inputTokens": 2383376,
       "outputTokens": 30467,
       "cacheTokens": 2017118,
       "totalTokens": 2413843,
       "durationSec": 554.683,
       "startedAt": "2026-09-25T13:44:02.100652Z",
       "finishedAt": "2026-09-25T13:53:16.783406Z",
       "taskChecksum": "d21f1e251c43a2271fb9604613da1a6f5768be18b2c5a7f05bac5b620ac271f5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-doom-for-mips__YUkyC9c",
       "job": "r1-02-make-doom-for-mips",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 3,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T17:50:47.441Z",
         "tokensAt": 53947,
         "residualTokens": 7951,
         "qualified": true,
         "readsBefore": 1.2,
         "readsAfter": 1.4
        },
        {
         "at": "2026-09-25T17:54:33.646Z",
         "tokensAt": 52504,
         "residualTokens": 6529,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 1
        },
        {
         "at": "2026-09-25T17:57:09.487Z",
         "tokensAt": 52488,
         "residualTokens": 11315,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 1
        }
       ],
       "providerRequests": 94,
       "agentRequests": 94,
       "costUsd": 0.09078293499999998,
       "inputTokens": 2933901,
       "outputTokens": 39311,
       "cacheTokens": 2517556,
       "totalTokens": 2973212,
       "durationSec": 964.462,
       "startedAt": "2026-09-25T17:48:50.706356Z",
       "finishedAt": "2026-09-25T18:04:55.168602Z",
       "taskChecksum": "d21f1e251c43a2271fb9604613da1a6f5768be18b2c5a7f05bac5b620ac271f5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-doom-for-mips__Hs9vWWm",
       "job": "r2-02-make-doom-for-mips",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 5,
       "qualifiedCompactions": 5,
       "compactions": [
        {
         "at": "2026-09-25T18:50:44.779Z",
         "tokensAt": 52722,
         "residualTokens": 8312,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 1
        },
        {
         "at": "2026-09-25T18:53:01.180Z",
         "tokensAt": 50355,
         "residualTokens": 5825,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 1.4
        },
        {
         "at": "2026-09-25T18:54:37.760Z",
         "tokensAt": 50588,
         "residualTokens": 4326,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T18:56:50.135Z",
         "tokensAt": 49019,
         "residualTokens": 3495,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 1.4
        },
        {
         "at": "2026-09-25T19:01:16.019Z",
         "tokensAt": 52174,
         "residualTokens": 9123,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 1.2
        }
       ],
       "providerRequests": 175,
       "agentRequests": 175,
       "costUsd": 0.13811471500000005,
       "inputTokens": 5429544,
       "outputTokens": 64136,
       "cacheTokens": 4929484,
       "totalTokens": 5493680,
       "durationSec": 984.937,
       "startedAt": "2026-09-25T18:48:03.122322Z",
       "finishedAt": "2026-09-25T19:04:28.059524Z",
       "taskChecksum": "d21f1e251c43a2271fb9604613da1a6f5768be18b2c5a7f05bac5b620ac271f5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "make-doom-for-mips__FwDUZxb",
       "job": "r3-02-make-doom-for-mips",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 6,
       "qualifiedCompactions": 6,
       "compactions": [
        {
         "at": "2026-09-25T22:38:08.436Z",
         "tokensAt": 54299,
         "residualTokens": 20676,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 1
        },
        {
         "at": "2026-09-25T22:41:05.440Z",
         "tokensAt": 54464,
         "residualTokens": 12149,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 2.4
        },
        {
         "at": "2026-09-25T22:42:42.125Z",
         "tokensAt": 52267,
         "residualTokens": 7397,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 1.4
        },
        {
         "at": "2026-09-25T22:44:44.227Z",
         "tokensAt": 51256,
         "residualTokens": 21461,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T22:46:52.425Z",
         "tokensAt": 52153,
         "residualTokens": 21902,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 1.6
        },
        {
         "at": "2026-09-25T22:50:05.444Z",
         "tokensAt": 53091,
         "residualTokens": 2917,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0.8
        }
       ],
       "providerRequests": 179,
       "agentRequests": 179,
       "costUsd": 0.16989794500000005,
       "inputTokens": 6180298,
       "outputTokens": 74113,
       "cacheTokens": 5479962,
       "totalTokens": 6254411,
       "durationSec": 993.283,
       "startedAt": "2026-09-25T22:35:32.727366Z",
       "finishedAt": "2026-09-25T22:52:06.010540Z",
       "taskChecksum": "d21f1e251c43a2271fb9604613da1a6f5768be18b2c5a7f05bac5b620ac271f5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0,
     "costUsd": 0.47530142500000006,
     "qualifiedCompactions": 17
    }
   },
   "diff": 0
  },
  {
   "name": "caffe-cifar-10",
   "set": "A",
   "timeoutSec": 3600,
   "category": "machine-learning",
   "task": "caffe-cifar-10",
   "compactionRate": 1,
   "group": "primary",
   "outcome": "Textual ahead",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 3,
     "scored": 3,
     "errors": 1,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "caffe-cifar-10__8wvXjVg",
       "job": "r0-03-caffe-cifar-10",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 3,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T14:12:39.824Z",
         "tokensAt": 59990,
         "residualTokens": 39703,
         "qualified": true,
         "readsBefore": 1.4,
         "readsAfter": 1
        },
        {
         "at": "2026-09-25T14:14:27.947Z",
         "tokensAt": 51263,
         "residualTokens": 15879,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 1
        },
        {
         "at": "2026-09-25T14:30:28.569Z",
         "tokensAt": 52281,
         "residualTokens": 36413,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 31,
       "agentRequests": 27,
       "costUsd": 0.04342613999999999,
       "inputTokens": 749300,
       "outputTokens": 16328,
       "cacheTokens": 507809,
       "totalTokens": 765628,
       "durationSec": 1981.71,
       "startedAt": "2026-09-25T14:00:33.507017Z",
       "finishedAt": "2026-09-25T14:33:35.217125Z",
       "taskChecksum": "5462402281e0bcc11b81ade62488f1013f970c7c0679c27df3f87185f4de40c3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "caffe-cifar-10__e4VwX6u",
       "job": "r1-03-caffe-cifar-10",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T18:01:20.020Z",
         "tokensAt": 51454,
         "residualTokens": 38129,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T18:16:19.463Z",
         "tokensAt": 52405,
         "residualTokens": 22668,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0
        }
       ],
       "providerRequests": 32,
       "agentRequests": 30,
       "costUsd": 0.02833671,
       "inputTokens": 938295,
       "outputTokens": 9511,
       "cacheTokens": 814811,
       "totalTokens": 947806,
       "durationSec": 2110.191,
       "startedAt": "2026-09-25T17:47:08.837070Z",
       "finishedAt": "2026-09-25T18:22:19.028840Z",
       "taskChecksum": "5462402281e0bcc11b81ade62488f1013f970c7c0679c27df3f87185f4de40c3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "caffe-cifar-10__CpA2w5p",
       "job": "r2-03-caffe-cifar-10",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T19:00:00.908Z",
         "tokensAt": 53349,
         "residualTokens": 28665,
         "qualified": true,
         "readsBefore": 1.4,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T19:08:18.929Z",
         "tokensAt": 56266,
         "residualTokens": 26852,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 42,
       "agentRequests": 39,
       "costUsd": 0.03585765000000001,
       "inputTokens": 1261170,
       "outputTokens": 13116,
       "cacheTokens": 1116030,
       "totalTokens": 1274286,
       "durationSec": 1214.883,
       "startedAt": "2026-09-25T18:49:36.007733Z",
       "finishedAt": "2026-09-25T19:09:50.890885Z",
       "taskChecksum": "5462402281e0bcc11b81ade62488f1013f970c7c0679c27df3f87185f4de40c3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "caffe-cifar-10__zbduktH",
       "job": "r3-03-caffe-cifar-10",
       "round": 3,
       "state": "ERROR",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": "Error: 10 minute request deadline",
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 10,
       "agentRequests": 10,
       "costUsd": 0.40576746,
       "inputTokens": 95049,
       "outputTokens": 2155,
       "cacheTokens": 62526,
       "totalTokens": 97204,
       "durationSec": 696.707,
       "startedAt": "2026-09-25T22:23:40.767210Z",
       "finishedAt": "2026-09-25T22:35:17.474540Z",
       "taskChecksum": "5462402281e0bcc11b81ade62488f1013f970c7c0679c27df3f87185f4de40c3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": true
      }
     ],
     "passRate": 1,
     "costUsd": 0.51338796,
     "qualifiedCompactions": 7
    },
    "Native50kPi": {
     "passes": 3,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "caffe-cifar-10__DAFyhQK",
       "job": "r0-03-caffe-cifar-10",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T14:11:24.505Z",
         "tokensAt": 64243,
         "residualTokens": 4384,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T14:23:57.187Z",
         "tokensAt": 54987,
         "residualTokens": 3307,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 43,
       "agentRequests": 43,
       "costUsd": 0.045498945000000006,
       "inputTokens": 1255615,
       "outputTokens": 16279,
       "cacheTokens": 1008532,
       "totalTokens": 1271894,
       "durationSec": 1526.444,
       "startedAt": "2026-09-25T14:00:33.481180Z",
       "finishedAt": "2026-09-25T14:25:59.925051Z",
       "taskChecksum": "5462402281e0bcc11b81ade62488f1013f970c7c0679c27df3f87185f4de40c3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "caffe-cifar-10__8CDdFUS",
       "job": "r1-03-caffe-cifar-10",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 3,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T17:50:07.218Z",
         "tokensAt": 54357,
         "residualTokens": 3190,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T18:21:57.143Z",
         "tokensAt": 49589,
         "residualTokens": 4398,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T18:42:28.932Z",
         "tokensAt": 58181,
         "residualTokens": 4360,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 47,
       "agentRequests": 47,
       "costUsd": 0.050207425000000014,
       "inputTokens": 1117247,
       "outputTokens": 18154,
       "cacheTokens": 822320,
       "totalTokens": 1135401,
       "durationSec": 3651.47,
       "startedAt": "2026-09-25T17:47:08.868175Z",
       "finishedAt": "2026-09-25T18:48:00.338204Z",
       "taskChecksum": "5462402281e0bcc11b81ade62488f1013f970c7c0679c27df3f87185f4de40c3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "caffe-cifar-10__yX9eHDg",
       "job": "r2-03-caffe-cifar-10",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T19:00:33.759Z",
         "tokensAt": 64526,
         "residualTokens": 3557,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.8
        }
       ],
       "providerRequests": 25,
       "agentRequests": 25,
       "costUsd": 0.024857890000000004,
       "inputTokens": 606426,
       "outputTokens": 6409,
       "cacheTokens": 453734,
       "totalTokens": 612835,
       "durationSec": 1811.052,
       "startedAt": "2026-09-25T18:49:35.983208Z",
       "finishedAt": "2026-09-25T19:19:47.035930Z",
       "taskChecksum": "5462402281e0bcc11b81ade62488f1013f970c7c0679c27df3f87185f4de40c3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "caffe-cifar-10__xMdGdQA",
       "job": "r3-03-caffe-cifar-10",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T22:29:11.670Z",
         "tokensAt": 54730,
         "residualTokens": 2560,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T22:51:35.099Z",
         "tokensAt": 52615,
         "residualTokens": 3408,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.8
        }
       ],
       "providerRequests": 51,
       "agentRequests": 51,
       "costUsd": 0.046354255,
       "inputTokens": 1320142,
       "outputTokens": 12247,
       "cacheTokens": 1052008,
       "totalTokens": 1332389,
       "durationSec": 3078.988,
       "startedAt": "2026-09-25T22:23:40.792291Z",
       "finishedAt": "2026-09-25T23:14:59.780155Z",
       "taskChecksum": "5462402281e0bcc11b81ade62488f1013f970c7c0679c27df3f87185f4de40c3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.75,
     "costUsd": 0.16691851500000002,
     "qualifiedCompactions": 8
    }
   },
   "diff": -0.25
  },
  {
   "name": "path-tracing-reverse",
   "set": "A",
   "timeoutSec": 1800,
   "category": "software-engineering",
   "task": "path-tracing-reverse",
   "compactionRate": 1,
   "group": "primary",
   "outcome": "Encrypted ahead",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 3,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "path-tracing-reverse__wKeujsZ",
       "job": "r0-04-path-tracing-reverse",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 6,
       "qualifiedCompactions": 5,
       "compactions": [
        {
         "at": "2026-09-25T14:16:45.314Z",
         "tokensAt": 55770,
         "residualTokens": 17100,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T14:17:48.843Z",
         "tokensAt": 59179,
         "residualTokens": 35203,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T14:18:14.007Z",
         "tokensAt": 57645,
         "residualTokens": 32336,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T14:20:24.634Z",
         "tokensAt": 57645,
         "residualTokens": 51741,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T14:21:06.287Z",
         "tokensAt": 55723,
         "residualTokens": 41199,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T14:24:33.852Z",
         "tokensAt": 84404,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0.2,
         "readsAfter": null
        }
       ],
       "providerRequests": 62,
       "agentRequests": 55,
       "costUsd": 0.09672712500000001,
       "inputTokens": 2712326,
       "outputTokens": 41338,
       "cacheTokens": 2286765,
       "totalTokens": 2753664,
       "durationSec": 540.72,
       "startedAt": "2026-09-25T14:15:58.694993Z",
       "finishedAt": "2026-09-25T14:24:59.414872Z",
       "taskChecksum": "2c4c5c4e76b6706f7a2e7f45efb8262fb3774ca92e27fbbcbb940e68ae5cd641",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "path-tracing-reverse__wvLZZ9D",
       "job": "r1-04-path-tracing-reverse",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 6,
       "qualifiedCompactions": 5,
       "compactions": [
        {
         "at": "2026-09-25T17:35:49.077Z",
         "tokensAt": 56756,
         "residualTokens": 26306,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T17:36:19.300Z",
         "tokensAt": 52500,
         "residualTokens": 28492,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T17:37:28.279Z",
         "tokensAt": 50239,
         "residualTokens": 25545,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T17:38:51.183Z",
         "tokensAt": 56362,
         "residualTokens": 38996,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T17:39:41.825Z",
         "tokensAt": 58609,
         "residualTokens": 27757,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T17:46:39.938Z",
         "tokensAt": 71134,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0,
         "readsAfter": null
        }
       ],
       "providerRequests": 72,
       "agentRequests": 62,
       "costUsd": 0.09897309499999997,
       "inputTokens": 3013542,
       "outputTokens": 54517,
       "cacheTokens": 2651937,
       "totalTokens": 3068059,
       "durationSec": 706.544,
       "startedAt": "2026-09-25T17:35:18.674083Z",
       "finishedAt": "2026-09-25T17:47:05.218058Z",
       "taskChecksum": "2c4c5c4e76b6706f7a2e7f45efb8262fb3774ca92e27fbbcbb940e68ae5cd641",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "path-tracing-reverse__Bn6sFz6",
       "job": "r2-04-path-tracing-reverse",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 18,
       "qualifiedCompactions": 17,
       "compactions": [
        {
         "at": "2026-09-25T18:57:06.696Z",
         "tokensAt": 51434,
         "residualTokens": 41664,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T18:58:00.784Z",
         "tokensAt": 53485,
         "residualTokens": 39491,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T18:58:43.304Z",
         "tokensAt": 57493,
         "residualTokens": 33159,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T18:59:39.644Z",
         "tokensAt": 52019,
         "residualTokens": 49199,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T18:59:54.277Z",
         "tokensAt": 53120,
         "residualTokens": 35405,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T19:00:40.599Z",
         "tokensAt": 51181,
         "residualTokens": 47881,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:01:37.859Z",
         "tokensAt": 53229,
         "residualTokens": 41557,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:02:30.921Z",
         "tokensAt": 50381,
         "residualTokens": 45701,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:03:25.802Z",
         "tokensAt": 51636,
         "residualTokens": 43656,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T19:03:53.145Z",
         "tokensAt": 57779,
         "residualTokens": 42813,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 1.2
        },
        {
         "at": "2026-09-25T19:04:18.723Z",
         "tokensAt": 58489,
         "residualTokens": 21213,
         "qualified": true,
         "readsBefore": 0.8,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T19:05:48.887Z",
         "tokensAt": 52228,
         "residualTokens": 41487,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T19:06:38.033Z",
         "tokensAt": 50122,
         "residualTokens": 44957,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T19:07:14.931Z",
         "tokensAt": 54152,
         "residualTokens": 43575,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:08:10.826Z",
         "tokensAt": 54382,
         "residualTokens": 49172,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:08:28.578Z",
         "tokensAt": 50453,
         "residualTokens": 48043,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:09:03.441Z",
         "tokensAt": 51134,
         "residualTokens": 44710,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:10:16.876Z",
         "tokensAt": 51011,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0,
         "readsAfter": null
        }
       ],
       "providerRequests": 98,
       "agentRequests": 74,
       "costUsd": 0.18464888499999993,
       "inputTokens": 3184792,
       "outputTokens": 88088,
       "cacheTokens": 2238741,
       "totalTokens": 3272880,
       "durationSec": 845.673,
       "startedAt": "2026-09-25T18:56:35.170219Z",
       "finishedAt": "2026-09-25T19:10:40.843624Z",
       "taskChecksum": "2c4c5c4e76b6706f7a2e7f45efb8262fb3774ca92e27fbbcbb940e68ae5cd641",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "path-tracing-reverse__HCxgU7p",
       "job": "r3-04-path-tracing-reverse",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 4,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T22:19:03.092Z",
         "tokensAt": 50944,
         "residualTokens": 40141,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T22:21:03.998Z",
         "tokensAt": 56687,
         "residualTokens": 23091,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T22:21:43.679Z",
         "tokensAt": 54892,
         "residualTokens": 47287,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T22:27:44.500Z",
         "tokensAt": 90373,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0,
         "readsAfter": null
        }
       ],
       "providerRequests": 60,
       "agentRequests": 54,
       "costUsd": 0.08888303,
       "inputTokens": 3194193,
       "outputTokens": 53911,
       "cacheTokens": 2933203,
       "totalTokens": 3248104,
       "durationSec": 589.53,
       "startedAt": "2026-09-25T22:18:24.774712Z",
       "finishedAt": "2026-09-25T22:28:14.304820Z",
       "taskChecksum": "2c4c5c4e76b6706f7a2e7f45efb8262fb3774ca92e27fbbcbb940e68ae5cd641",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.75,
     "costUsd": 0.46923213499999994,
     "qualifiedCompactions": 30
    },
    "Native50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "path-tracing-reverse__872kWt8",
       "job": "r0-04-path-tracing-reverse",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 4,
       "qualifiedCompactions": 4,
       "compactions": [
        {
         "at": "2026-09-25T14:17:09.447Z",
         "tokensAt": 52859,
         "residualTokens": 13681,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T14:18:51.292Z",
         "tokensAt": 52053,
         "residualTokens": 32155,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T14:20:11.856Z",
         "tokensAt": 52905,
         "residualTokens": 2585,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T14:23:36.234Z",
         "tokensAt": 51545,
         "residualTokens": 4305,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 66,
       "agentRequests": 66,
       "costUsd": 0.07597212499999999,
       "inputTokens": 1553431,
       "outputTokens": 36551,
       "cacheTokens": 1140140,
       "totalTokens": 1589982,
       "durationSec": 546.433,
       "startedAt": "2026-09-25T14:15:58.726360Z",
       "finishedAt": "2026-09-25T14:25:05.159069Z",
       "taskChecksum": "2c4c5c4e76b6706f7a2e7f45efb8262fb3774ca92e27fbbcbb940e68ae5cd641",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "path-tracing-reverse__SApDjC2",
       "job": "r1-04-path-tracing-reverse",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 3,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T17:36:05.032Z",
         "tokensAt": 57050,
         "residualTokens": 38544,
         "qualified": true,
         "readsBefore": 0.25,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T17:37:57.534Z",
         "tokensAt": 46901,
         "residualTokens": 24814,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T17:39:50.897Z",
         "tokensAt": 51487,
         "residualTokens": 2745,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 56,
       "agentRequests": 56,
       "costUsd": 0.05454816999999999,
       "inputTokens": 1060220,
       "outputTokens": 30938,
       "cacheTokens": 779502,
       "totalTokens": 1091158,
       "durationSec": 487.097,
       "startedAt": "2026-09-25T17:35:18.581407Z",
       "finishedAt": "2026-09-25T17:43:25.678810Z",
       "taskChecksum": "2c4c5c4e76b6706f7a2e7f45efb8262fb3774ca92e27fbbcbb940e68ae5cd641",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "path-tracing-reverse__XK6fNry",
       "job": "r2-04-path-tracing-reverse",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 7,
       "qualifiedCompactions": 7,
       "compactions": [
        {
         "at": "2026-09-25T18:57:25.534Z",
         "tokensAt": 59990,
         "residualTokens": 43060,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T18:57:48.501Z",
         "tokensAt": 54061,
         "residualTokens": 18927,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T18:59:05.004Z",
         "tokensAt": 49989,
         "residualTokens": 3145,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T19:00:19.553Z",
         "tokensAt": 55968,
         "residualTokens": 5157,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T19:01:33.059Z",
         "tokensAt": 49590,
         "residualTokens": 3869,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T19:03:19.419Z",
         "tokensAt": 51974,
         "residualTokens": 8412,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T19:07:20.540Z",
         "tokensAt": 54273,
         "residualTokens": 5038,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 86,
       "agentRequests": 86,
       "costUsd": 0.11673800000000001,
       "inputTokens": 2349186,
       "outputTokens": 59047,
       "cacheTokens": 1731680,
       "totalTokens": 2408233,
       "durationSec": 1830.342,
       "startedAt": "2026-09-25T18:56:35.197178Z",
       "finishedAt": "2026-09-25T19:27:05.539784Z",
       "taskChecksum": "2c4c5c4e76b6706f7a2e7f45efb8262fb3774ca92e27fbbcbb940e68ae5cd641",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "path-tracing-reverse__TLjHmX9",
       "job": "r3-04-path-tracing-reverse",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 7,
       "qualifiedCompactions": 7,
       "compactions": [
        {
         "at": "2026-09-25T22:19:15.090Z",
         "tokensAt": 59321,
         "residualTokens": 30905,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T22:20:14.982Z",
         "tokensAt": 46351,
         "residualTokens": 21055,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T22:22:29.660Z",
         "tokensAt": 52556,
         "residualTokens": 4109,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T22:24:17.046Z",
         "tokensAt": 54037,
         "residualTokens": 4416,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T22:28:01.569Z",
         "tokensAt": 50735,
         "residualTokens": 3558,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T22:29:56.992Z",
         "tokensAt": 50879,
         "residualTokens": 4223,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T22:34:55.972Z",
         "tokensAt": 51190,
         "residualTokens": 2887,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.25
        }
       ],
       "providerRequests": 105,
       "agentRequests": 105,
       "costUsd": 0.131400765,
       "inputTokens": 3095087,
       "outputTokens": 78487,
       "cacheTokens": 2501674,
       "totalTokens": 3173574,
       "durationSec": 1024.92,
       "startedAt": "2026-09-25T22:18:24.743692Z",
       "finishedAt": "2026-09-25T22:35:29.663533Z",
       "taskChecksum": "2c4c5c4e76b6706f7a2e7f45efb8262fb3774ca92e27fbbcbb940e68ae5cd641",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.37865906,
     "qualifiedCompactions": 21
    }
   },
   "diff": 0.25
  },
  {
   "name": "compile-compcert",
   "set": "A",
   "timeoutSec": 2400,
   "category": "system-administration",
   "task": "compile-compcert",
   "compactionRate": 1,
   "group": "primary",
   "outcome": "Both always pass",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "compile-compcert__auGgc5h",
       "job": "r0-05-compile-compcert",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T14:40:50.694Z",
         "tokensAt": 52466,
         "residualTokens": 21687,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 25,
       "agentRequests": 24,
       "costUsd": 0.01733096,
       "inputTokens": 587592,
       "outputTokens": 5084,
       "cacheTokens": 510071,
       "totalTokens": 592676,
       "durationSec": 988.818,
       "startedAt": "2026-09-25T14:25:08.286755Z",
       "finishedAt": "2026-09-25T14:41:37.104364Z",
       "taskChecksum": "f7285aa10ae9a8d0c80a0a35bfcdb994a058f0f817eeaeef080c0dc817efb842",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "compile-compcert__u7npDjF",
       "job": "r1-05-compile-compcert",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T17:48:10.909Z",
         "tokensAt": 50429,
         "residualTokens": 24861,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0
        }
       ],
       "providerRequests": 31,
       "agentRequests": 30,
       "costUsd": 0.019641154999999997,
       "inputTokens": 798561,
       "outputTokens": 5597,
       "cacheTokens": 721523,
       "totalTokens": 804158,
       "durationSec": 1212.817,
       "startedAt": "2026-09-25T17:28:32.588580Z",
       "finishedAt": "2026-09-25T17:48:45.405654Z",
       "taskChecksum": "f7285aa10ae9a8d0c80a0a35bfcdb994a058f0f817eeaeef080c0dc817efb842",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "compile-compcert__tHNXtoC",
       "job": "r2-05-compile-compcert",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T19:17:35.787Z",
         "tokensAt": 50686,
         "residualTokens": 25872,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 40,
       "agentRequests": 39,
       "costUsd": 0.028877340000000005,
       "inputTokens": 1205031,
       "outputTokens": 11207,
       "cacheTokens": 1107409,
       "totalTokens": 1216238,
       "durationSec": 1124.105,
       "startedAt": "2026-09-25T19:04:30.698802Z",
       "finishedAt": "2026-09-25T19:23:14.803393Z",
       "taskChecksum": "f7285aa10ae9a8d0c80a0a35bfcdb994a058f0f817eeaeef080c0dc817efb842",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "compile-compcert__upzb6k3",
       "job": "r3-05-compile-compcert",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T22:17:52.833Z",
         "tokensAt": 50052,
         "residualTokens": 26345,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 24,
       "agentRequests": 23,
       "costUsd": 0.017662449999999996,
       "inputTokens": 529629,
       "outputTokens": 4702,
       "cacheTokens": 442525,
       "totalTokens": 534331,
       "durationSec": 1035.967,
       "startedAt": "2026-09-25T22:14:08.984739Z",
       "finishedAt": "2026-09-25T22:31:24.951074Z",
       "taskChecksum": "f7285aa10ae9a8d0c80a0a35bfcdb994a058f0f817eeaeef080c0dc817efb842",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.083511905,
     "qualifiedCompactions": 4
    },
    "Native50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "compile-compcert__Tvfi8hv",
       "job": "r0-05-compile-compcert",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T14:42:34.741Z",
         "tokensAt": 50131,
         "residualTokens": 2195,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 43,
       "agentRequests": 43,
       "costUsd": 0.022471160000000004,
       "inputTokens": 1052529,
       "outputTokens": 6131,
       "cacheTokens": 968596,
       "totalTokens": 1058660,
       "durationSec": 1099.592,
       "startedAt": "2026-09-25T14:25:08.256319Z",
       "finishedAt": "2026-09-25T14:43:27.848539Z",
       "taskChecksum": "f7285aa10ae9a8d0c80a0a35bfcdb994a058f0f817eeaeef080c0dc817efb842",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "compile-compcert__p7unmQU",
       "job": "r1-05-compile-compcert",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T17:48:50.077Z",
         "tokensAt": 62496,
         "residualTokens": 3298,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.8
        }
       ],
       "providerRequests": 32,
       "agentRequests": 32,
       "costUsd": 0.027638464999999997,
       "inputTokens": 719073,
       "outputTokens": 8977,
       "cacheTokens": 559674,
       "totalTokens": 728050,
       "durationSec": 1510.693,
       "startedAt": "2026-09-25T17:28:32.613434Z",
       "finishedAt": "2026-09-25T17:53:43.306449Z",
       "taskChecksum": "f7285aa10ae9a8d0c80a0a35bfcdb994a058f0f817eeaeef080c0dc817efb842",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "compile-compcert__b7csQDB",
       "job": "r2-05-compile-compcert",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T19:20:53.626Z",
         "tokensAt": 54234,
         "residualTokens": 3098,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 32,
       "agentRequests": 32,
       "costUsd": 0.026409475000000005,
       "inputTokens": 586528,
       "outputTokens": 7716,
       "cacheTokens": 415290,
       "totalTokens": 594244,
       "durationSec": 1048.47,
       "startedAt": "2026-09-25T19:04:30.674272Z",
       "finishedAt": "2026-09-25T19:21:59.144020Z",
       "taskChecksum": "f7285aa10ae9a8d0c80a0a35bfcdb994a058f0f817eeaeef080c0dc817efb842",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "compile-compcert__mw4DefJ",
       "job": "r3-05-compile-compcert",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T22:34:20.681Z",
         "tokensAt": 53231,
         "residualTokens": 2307,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 52,
       "agentRequests": 52,
       "costUsd": 0.0335835,
       "inputTokens": 1214121,
       "outputTokens": 10741,
       "cacheTokens": 1056850,
       "totalTokens": 1224862,
       "durationSec": 1509.689,
       "startedAt": "2026-09-25T22:14:09.015006Z",
       "finishedAt": "2026-09-25T22:39:18.704462Z",
       "taskChecksum": "f7285aa10ae9a8d0c80a0a35bfcdb994a058f0f817eeaeef080c0dc817efb842",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.11010260000000001,
     "qualifiedCompactions": 4
    }
   },
   "diff": 0
  },
  {
   "name": "regex-chess",
   "set": "A",
   "timeoutSec": 3600,
   "category": "software-engineering",
   "task": "regex-chess",
   "compactionRate": 0.7142857142857143,
   "group": "primary",
   "outcome": "Textual ahead",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 2,
     "scored": 3,
     "errors": 1,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "regex-chess__xLDUm6c",
       "job": "r0-06-regex-chess",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T14:41:23.684Z",
         "tokensAt": 50437,
         "residualTokens": 50110,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T14:45:23.828Z",
         "tokensAt": 79280,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0,
         "readsAfter": null
        }
       ],
       "providerRequests": 60,
       "agentRequests": 58,
       "costUsd": 0.071769345,
       "inputTokens": 2872156,
       "outputTokens": 53593,
       "cacheTokens": 2730652,
       "totalTokens": 2925749,
       "durationSec": 842.154,
       "startedAt": "2026-09-25T14:33:38.373522Z",
       "finishedAt": "2026-09-25T14:47:40.527419Z",
       "taskChecksum": "9125ddfbd11e611d4c9553432db49cc06380e0a37267d32f67cf57fe6f66c87b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "regex-chess__4vSGQiq",
       "job": "r1-06-regex-chess",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 28,
       "agentRequests": 28,
       "costUsd": 0.03041461,
       "inputTokens": 884283,
       "outputTokens": 32703,
       "cacheTokens": 838871,
       "totalTokens": 916986,
       "durationSec": 741.929,
       "startedAt": "2026-09-25T17:21:36.042106Z",
       "finishedAt": "2026-09-25T17:33:57.971201Z",
       "taskChecksum": "9125ddfbd11e611d4c9553432db49cc06380e0a37267d32f67cf57fe6f66c87b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "regex-chess__rHFuUz5",
       "job": "r2-06-regex-chess",
       "round": 2,
       "state": "ERROR",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": "Error: Stream ended without terminal usage",
       "compactionsObserved": 5,
       "qualifiedCompactions": 5,
       "compactions": [
        {
         "at": "2026-09-25T19:29:05.152Z",
         "tokensAt": 58982,
         "residualTokens": 56491,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T19:29:21.730Z",
         "tokensAt": 59534,
         "residualTokens": 56145,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:29:39.089Z",
         "tokensAt": 56966,
         "residualTokens": 53177,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:29:48.430Z",
         "tokensAt": 53994,
         "residualTokens": 53685,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:30:12.246Z",
         "tokensAt": 54515,
         "residualTokens": 13566,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        }
       ],
       "providerRequests": 58,
       "agentRequests": 53,
       "costUsd": 0.47937665,
       "inputTokens": 1623049,
       "outputTokens": 50155,
       "cacheTokens": 1291980,
       "totalTokens": 1673204,
       "durationSec": 1075.853,
       "startedAt": "2026-09-25T19:19:50.542122Z",
       "finishedAt": "2026-09-25T19:37:46.395420Z",
       "taskChecksum": "9125ddfbd11e611d4c9553432db49cc06380e0a37267d32f67cf57fe6f66c87b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": true
      },
      {
       "trial": "regex-chess__ysvQSEZ",
       "job": "r3-06-regex-chess",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 0,
       "compactions": [
        {
         "at": "2026-09-25T22:09:57.859Z",
         "tokensAt": 70203,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0,
         "readsAfter": null
        }
       ],
       "providerRequests": 58,
       "agentRequests": 57,
       "costUsd": 0.058549870000000004,
       "inputTokens": 2589109,
       "outputTokens": 47257,
       "cacheTokens": 2510547,
       "totalTokens": 2636366,
       "durationSec": 676.003,
       "startedAt": "2026-09-25T22:00:45.349682Z",
       "finishedAt": "2026-09-25T22:12:01.352582Z",
       "taskChecksum": "9125ddfbd11e611d4c9553432db49cc06380e0a37267d32f67cf57fe6f66c87b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.6666666666666666,
     "costUsd": 0.640110475,
     "qualifiedCompactions": 6
    },
    "Native50kPi": {
     "passes": 2,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "regex-chess__acRL5Lt",
       "job": "r0-06-regex-chess",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T14:42:51.172Z",
         "tokensAt": 50098,
         "residualTokens": 3527,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.8
        }
       ],
       "providerRequests": 46,
       "agentRequests": 46,
       "costUsd": 0.045442505,
       "inputTokens": 1273123,
       "outputTokens": 42025,
       "cacheTokens": 1163468,
       "totalTokens": 1315148,
       "durationSec": 925.346,
       "startedAt": "2026-09-25T14:33:38.410069Z",
       "finishedAt": "2026-09-25T14:49:03.756273Z",
       "taskChecksum": "9125ddfbd11e611d4c9553432db49cc06380e0a37267d32f67cf57fe6f66c87b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "regex-chess__F7xP9GK",
       "job": "r1-06-regex-chess",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T17:27:41.140Z",
         "tokensAt": 53649,
         "residualTokens": 7160,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.6
        }
       ],
       "providerRequests": 46,
       "agentRequests": 46,
       "costUsd": 0.04639473499999999,
       "inputTokens": 1223467,
       "outputTokens": 40030,
       "cacheTokens": 1084186,
       "totalTokens": 1263497,
       "durationSec": 819.399,
       "startedAt": "2026-09-25T17:21:36.016669Z",
       "finishedAt": "2026-09-25T17:35:15.415631Z",
       "taskChecksum": "9125ddfbd11e611d4c9553432db49cc06380e0a37267d32f67cf57fe6f66c87b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "regex-chess__mYBXZLu",
       "job": "r2-06-regex-chess",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T19:26:36.434Z",
         "tokensAt": 51241,
         "residualTokens": 7600,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 57,
       "agentRequests": 57,
       "costUsd": 0.06226388500000001,
       "inputTokens": 1836136,
       "outputTokens": 51345,
       "cacheTokens": 1658406,
       "totalTokens": 1887481,
       "durationSec": 699.628,
       "startedAt": "2026-09-25T19:19:50.572373Z",
       "finishedAt": "2026-09-25T19:31:30.200184Z",
       "taskChecksum": "9125ddfbd11e611d4c9553432db49cc06380e0a37267d32f67cf57fe6f66c87b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "regex-chess__KCzhqgD",
       "job": "r3-06-regex-chess",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T22:07:02.768Z",
         "tokensAt": 52613,
         "residualTokens": 9656,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 52,
       "agentRequests": 52,
       "costUsd": 0.052535995,
       "inputTokens": 1299059,
       "outputTokens": 45347,
       "cacheTokens": 1132192,
       "totalTokens": 1344406,
       "durationSec": 800.97,
       "startedAt": "2026-09-25T22:00:45.323416Z",
       "finishedAt": "2026-09-25T22:14:06.293510Z",
       "taskChecksum": "9125ddfbd11e611d4c9553432db49cc06380e0a37267d32f67cf57fe6f66c87b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.5,
     "costUsd": 0.20663712,
     "qualifiedCompactions": 4
    }
   },
   "diff": -0.16666666666666663
  },
  {
   "name": "gpt2-codegolf",
   "set": "A",
   "timeoutSec": 900,
   "category": "software-engineering",
   "task": "gpt2-codegolf",
   "compactionRate": 0.14285714285714285,
   "group": "other",
   "outcome": "Textual ahead",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 1,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "gpt2-codegolf__jJsZpU2",
       "job": "r0-07-gpt2-codegolf",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 29,
       "agentRequests": 29,
       "costUsd": 0.020241139999999998,
       "inputTokens": 445210,
       "outputTokens": 24765,
       "cacheTokens": 415569,
       "totalTokens": 469975,
       "durationSec": 341.11,
       "startedAt": "2026-09-25T14:43:30.369233Z",
       "finishedAt": "2026-09-25T14:49:11.479117Z",
       "taskChecksum": "384c32098bf0ee970715f65516726f9b17b8a797261f9a8dce44c99c371dd8f3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "gpt2-codegolf__93WE7PA",
       "job": "r1-07-gpt2-codegolf",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 8,
       "agentRequests": 8,
       "costUsd": 0.007910400000000001,
       "inputTokens": 54725,
       "outputTokens": 6965,
       "cacheTokens": 20975,
       "totalTokens": 61690,
       "durationSec": 111.099,
       "startedAt": "2026-09-25T17:19:01.020167Z",
       "finishedAt": "2026-09-25T17:20:52.119260Z",
       "taskChecksum": "384c32098bf0ee970715f65516726f9b17b8a797261f9a8dce44c99c371dd8f3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "gpt2-codegolf__ifnTdGz",
       "job": "r2-07-gpt2-codegolf",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 35,
       "agentRequests": 35,
       "costUsd": 0.02293139,
       "inputTokens": 689994,
       "outputTokens": 24224,
       "cacheTokens": 655889,
       "totalTokens": 714218,
       "durationSec": 285.703,
       "startedAt": "2026-09-25T19:23:17.444960Z",
       "finishedAt": "2026-09-25T19:28:03.147366Z",
       "taskChecksum": "384c32098bf0ee970715f65516726f9b17b8a797261f9a8dce44c99c371dd8f3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "gpt2-codegolf__pUpQBFV",
       "job": "r3-07-gpt2-codegolf",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 38,
       "agentRequests": 38,
       "costUsd": 0.03306851,
       "inputTokens": 841347,
       "outputTokens": 38808,
       "cacheTokens": 795661,
       "totalTokens": 880155,
       "durationSec": 454.549,
       "startedAt": "2026-09-25T21:53:07.793018Z",
       "finishedAt": "2026-09-25T22:00:42.342646Z",
       "taskChecksum": "384c32098bf0ee970715f65516726f9b17b8a797261f9a8dce44c99c371dd8f3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.25,
     "costUsd": 0.08415144,
     "qualifiedCompactions": 0
    },
    "Native50kPi": {
     "passes": 0,
     "scored": 3,
     "errors": 1,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "gpt2-codegolf__2hG7jce",
       "job": "r0-07-gpt2-codegolf",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 6,
       "agentRequests": 6,
       "costUsd": 0.005195815,
       "inputTokens": 19439,
       "outputTokens": 7553,
       "cacheTokens": 7324,
       "totalTokens": 26992,
       "durationSec": 134.137,
       "startedAt": "2026-09-25T14:43:30.344392Z",
       "finishedAt": "2026-09-25T14:45:44.481494Z",
       "taskChecksum": "384c32098bf0ee970715f65516726f9b17b8a797261f9a8dce44c99c371dd8f3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "gpt2-codegolf__Uv3WMUy",
       "job": "r1-07-gpt2-codegolf",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T17:26:48.117Z",
         "tokensAt": 48737,
         "residualTokens": 4769,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 38,
       "agentRequests": 38,
       "costUsd": 0.03818359999999999,
       "inputTokens": 822286,
       "outputTokens": 43877,
       "cacheTokens": 747690,
       "totalTokens": 866163,
       "durationSec": 568.949,
       "startedAt": "2026-09-25T17:19:01.047627Z",
       "finishedAt": "2026-09-25T17:28:29.996653Z",
       "taskChecksum": "384c32098bf0ee970715f65516726f9b17b8a797261f9a8dce44c99c371dd8f3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "gpt2-codegolf__3Reuvjq",
       "job": "r2-07-gpt2-codegolf",
       "round": 2,
       "state": "ERROR",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": "AgentTimeoutError",
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T19:29:46.832Z",
         "tokensAt": 49266,
         "residualTokens": 4975,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 47,
       "agentRequests": 47,
       "costUsd": 0.105484075,
       "inputTokens": 1310633,
       "outputTokens": 167350,
       "cacheTokens": 1228570,
       "totalTokens": 1477983,
       "durationSec": 940.903,
       "startedAt": "2026-09-25T19:23:17.419981Z",
       "finishedAt": "2026-09-25T19:38:58.322018Z",
       "taskChecksum": "384c32098bf0ee970715f65516726f9b17b8a797261f9a8dce44c99c371dd8f3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "gpt2-codegolf__56JeyoM",
       "job": "r3-07-gpt2-codegolf",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 8,
       "agentRequests": 8,
       "costUsd": 0.0036138650000000004,
       "inputTokens": 19545,
       "outputTokens": 4721,
       "cacheTokens": 8504,
       "totalTokens": 24266,
       "durationSec": 84.685,
       "startedAt": "2026-09-25T21:53:07.825042Z",
       "finishedAt": "2026-09-25T21:54:32.510957Z",
       "taskChecksum": "384c32098bf0ee970715f65516726f9b17b8a797261f9a8dce44c99c371dd8f3",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0,
     "costUsd": 0.15247735499999998,
     "qualifiedCompactions": 2
    }
   },
   "diff": -0.25
  },
  {
   "name": "path-tracing",
   "set": "A",
   "timeoutSec": 1800,
   "category": "software-engineering",
   "task": "path-tracing",
   "compactionRate": 0.7142857142857143,
   "group": "primary",
   "outcome": "Encrypted ahead",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 3,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "path-tracing__cffCeux",
       "job": "r0-08-path-tracing",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T14:56:52.754Z",
         "tokensAt": 50119,
         "residualTokens": 38584,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T15:11:02.281Z",
         "tokensAt": 104047,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0.4,
         "readsAfter": null
        }
       ],
       "providerRequests": 81,
       "agentRequests": 78,
       "costUsd": 0.10389811000000003,
       "inputTokens": 5042652,
       "outputTokens": 66275,
       "cacheTokens": 4865781,
       "totalTokens": 5108927,
       "durationSec": 1475.793,
       "startedAt": "2026-09-25T14:47:13.272907Z",
       "finishedAt": "2026-09-25T15:11:49.065823Z",
       "taskChecksum": "c0059cc8b4f539405befee4386b355bff397f9dabe2610bdff279fcb0453e808",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "path-tracing__Jf3D5cb",
       "job": "r1-08-path-tracing",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 0,
       "compactions": [
        {
         "at": "2026-09-25T17:31:45.630Z",
         "tokensAt": 92836,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0.2,
         "readsAfter": null
        }
       ],
       "providerRequests": 49,
       "agentRequests": 48,
       "costUsd": 0.06526005,
       "inputTokens": 2667035,
       "outputTokens": 54526,
       "cacheTokens": 2568510,
       "totalTokens": 2721561,
       "durationSec": 845.675,
       "startedAt": "2026-09-25T17:18:10.603003Z",
       "finishedAt": "2026-09-25T17:32:16.278089Z",
       "taskChecksum": "c0059cc8b4f539405befee4386b355bff397f9dabe2610bdff279fcb0453e808",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "path-tracing__vDe2MJo",
       "job": "r2-08-path-tracing",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 37,
       "agentRequests": 37,
       "costUsd": 0.034350885000000005,
       "inputTokens": 1268429,
       "outputTokens": 30013,
       "cacheTokens": 1210491,
       "totalTokens": 1298442,
       "durationSec": 1836.865,
       "startedAt": "2026-09-25T19:27:08.183114Z",
       "finishedAt": "2026-09-25T19:57:45.048315Z",
       "taskChecksum": "c0059cc8b4f539405befee4386b355bff397f9dabe2610bdff279fcb0453e808",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "path-tracing__tcFjuUg",
       "job": "r3-08-path-tracing",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 4,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T21:53:04.506Z",
         "tokensAt": 50590,
         "residualTokens": 38750,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T22:01:57.188Z",
         "tokensAt": 51366,
         "residualTokens": 50870,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T22:03:39.484Z",
         "tokensAt": 51503,
         "residualTokens": 51494,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T22:13:05.328Z",
         "tokensAt": 102473,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0.2,
         "readsAfter": null
        }
       ],
       "providerRequests": 77,
       "agentRequests": 72,
       "costUsd": 0.11182179999999999,
       "inputTokens": 4204235,
       "outputTokens": 75029,
       "cacheTokens": 3923620,
       "totalTokens": 4279264,
       "durationSec": 1546.833,
       "startedAt": "2026-09-25T21:47:45.538749Z",
       "finishedAt": "2026-09-25T22:13:32.371018Z",
       "taskChecksum": "c0059cc8b4f539405befee4386b355bff397f9dabe2610bdff279fcb0453e808",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.75,
     "costUsd": 0.315330845,
     "qualifiedCompactions": 4
    },
    "Native50kPi": {
     "passes": 3,
     "scored": 3,
     "errors": 1,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "path-tracing__k8UeHuH",
       "job": "r0-08-path-tracing",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T14:56:26.788Z",
         "tokensAt": 51801,
         "residualTokens": 3301,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T15:13:03.514Z",
         "tokensAt": 52464,
         "residualTokens": 3290,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        }
       ],
       "providerRequests": 57,
       "agentRequests": 57,
       "costUsd": 0.06368038500000002,
       "inputTokens": 1207185,
       "outputTokens": 54276,
       "cacheTokens": 964936,
       "totalTokens": 1261461,
       "durationSec": 1874.229,
       "startedAt": "2026-09-25T14:47:13.297915Z",
       "finishedAt": "2026-09-25T15:18:27.526936Z",
       "taskChecksum": "c0059cc8b4f539405befee4386b355bff397f9dabe2610bdff279fcb0453e808",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "path-tracing__taYZtDR",
       "job": "r1-08-path-tracing",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T17:22:34.211Z",
         "tokensAt": 49804,
         "residualTokens": 6517,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T17:43:57.391Z",
         "tokensAt": 50692,
         "residualTokens": 2850,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 49,
       "agentRequests": 49,
       "costUsd": 0.05582569,
       "inputTokens": 984783,
       "outputTokens": 60273,
       "cacheTokens": 839904,
       "totalTokens": 1045056,
       "durationSec": 1837.51,
       "startedAt": "2026-09-25T17:18:10.576315Z",
       "finishedAt": "2026-09-25T17:48:48.086607Z",
       "taskChecksum": "c0059cc8b4f539405befee4386b355bff397f9dabe2610bdff279fcb0453e808",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "path-tracing__QatasRC",
       "job": "r2-08-path-tracing",
       "round": 2,
       "state": "ERROR",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": "Error: Stream ended without terminal usage",
       "compactionsObserved": 3,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T19:33:39.976Z",
         "tokensAt": 52502,
         "residualTokens": 3093,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T19:35:08.884Z",
         "tokensAt": 50168,
         "residualTokens": 13164,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:36:53.881Z",
         "tokensAt": 54424,
         "residualTokens": null,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.5
        }
       ],
       "providerRequests": 35,
       "agentRequests": 35,
       "costUsd": 0.45648009500000003,
       "inputTokens": 1062492,
       "outputTokens": 36346,
       "cacheTokens": 794127,
       "totalTokens": 1098838,
       "durationSec": 609.471,
       "startedAt": "2026-09-25T19:27:08.209016Z",
       "finishedAt": "2026-09-25T19:37:17.680578Z",
       "taskChecksum": "c0059cc8b4f539405befee4386b355bff397f9dabe2610bdff279fcb0453e808",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": true
      },
      {
       "trial": "path-tracing__qVaq4Yq",
       "job": "r3-08-path-tracing",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 3,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T21:52:42.337Z",
         "tokensAt": 51185,
         "residualTokens": 3435,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T22:08:33.757Z",
         "tokensAt": 53997,
         "residualTokens": 4184,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T22:18:14.296Z",
         "tokensAt": 52223,
         "residualTokens": 4212,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.3333333333333333
        }
       ],
       "providerRequests": 110,
       "agentRequests": 110,
       "costUsd": 0.11042094,
       "inputTokens": 3119086,
       "outputTokens": 95082,
       "cacheTokens": 2807824,
       "totalTokens": 3214168,
       "durationSec": 1835.757,
       "startedAt": "2026-09-25T21:47:45.509184Z",
       "finishedAt": "2026-09-25T22:18:21.266408Z",
       "taskChecksum": "c0059cc8b4f539405befee4386b355bff397f9dabe2610bdff279fcb0453e808",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.6864071100000001,
     "qualifiedCompactions": 10
    }
   },
   "diff": 0.25
  },
  {
   "name": "build-pov-ray",
   "set": "A",
   "timeoutSec": 12000,
   "category": "software-engineering",
   "task": "build-pov-ray",
   "compactionRate": 1,
   "group": "primary",
   "outcome": "Tied",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 2,
     "scored": 3,
     "errors": 1,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "build-pov-ray__4p2x2pg",
       "job": "r0-09-build-pov-ray",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T14:52:21.581Z",
         "tokensAt": 51297,
         "residualTokens": 29363,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.8
        }
       ],
       "providerRequests": 44,
       "agentRequests": 43,
       "costUsd": 0.026380624999999998,
       "inputTokens": 1053613,
       "outputTokens": 8159,
       "cacheTokens": 951280,
       "totalTokens": 1061772,
       "durationSec": 263.599,
       "startedAt": "2026-09-25T14:49:06.252859Z",
       "finishedAt": "2026-09-25T14:53:29.851090Z",
       "taskChecksum": "62e9feaef83fb9febfe3fc2a55ab7ca2fd505231114155b2e55ed880bb2c58b4",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "build-pov-ray__Qix9Qyx",
       "job": "r1-09-build-pov-ray",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T17:15:18.342Z",
         "tokensAt": 50098,
         "residualTokens": 24002,
         "qualified": true,
         "readsBefore": 1.8,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T17:17:10.382Z",
         "tokensAt": 52500,
         "residualTokens": 30731,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.3333333333333333
        }
       ],
       "providerRequests": 44,
       "agentRequests": 42,
       "costUsd": 0.039588124999999995,
       "inputTokens": 1381949,
       "outputTokens": 15303,
       "cacheTokens": 1224380,
       "totalTokens": 1397252,
       "durationSec": 349.51,
       "startedAt": "2026-09-25T17:12:18.331052Z",
       "finishedAt": "2026-09-25T17:18:07.841721Z",
       "taskChecksum": "62e9feaef83fb9febfe3fc2a55ab7ca2fd505231114155b2e55ed880bb2c58b4",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "build-pov-ray__EyEagA7",
       "job": "r2-09-build-pov-ray",
       "round": 2,
       "state": "ERROR",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": "Error: Stream ended without terminal usage",
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 4,
       "agentRequests": 4,
       "costUsd": 0.400676705,
       "inputTokens": 5256,
       "outputTokens": 642,
       "cacheTokens": 2618,
       "totalTokens": 5898,
       "durationSec": 41.482,
       "startedAt": "2026-09-25T19:37:49.108957Z",
       "finishedAt": "2026-09-25T19:38:30.590634Z",
       "taskChecksum": "62e9feaef83fb9febfe3fc2a55ab7ca2fd505231114155b2e55ed880bb2c58b4",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": true
      },
      {
       "trial": "build-pov-ray__cANEwVP",
       "job": "r3-09-build-pov-ray",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T21:43:05.762Z",
         "tokensAt": 53446,
         "residualTokens": 26896,
         "qualified": true,
         "readsBefore": 1.4,
         "readsAfter": 0.8
        }
       ],
       "providerRequests": 30,
       "agentRequests": 29,
       "costUsd": 0.02531887,
       "inputTokens": 780238,
       "outputTokens": 10180,
       "cacheTokens": 672162,
       "totalTokens": 790418,
       "durationSec": 233.506,
       "startedAt": "2026-09-25T21:40:29.580510Z",
       "finishedAt": "2026-09-25T21:44:23.086903Z",
       "taskChecksum": "62e9feaef83fb9febfe3fc2a55ab7ca2fd505231114155b2e55ed880bb2c58b4",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.6666666666666666,
     "costUsd": 0.491964325,
     "qualifiedCompactions": 4
    },
    "Native50kPi": {
     "passes": 2,
     "scored": 3,
     "errors": 1,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "build-pov-ray__yNcETiT",
       "job": "r0-09-build-pov-ray",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T14:51:03.918Z",
         "tokensAt": 50198,
         "residualTokens": 5262,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T14:52:27.406Z",
         "tokensAt": 58227,
         "residualTokens": 2380,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 44,
       "agentRequests": 44,
       "costUsd": 0.0369858,
       "inputTokens": 988059,
       "outputTokens": 9983,
       "cacheTokens": 774470,
       "totalTokens": 998042,
       "durationSec": 296.231,
       "startedAt": "2026-09-25T14:49:06.227855Z",
       "finishedAt": "2026-09-25T14:54:02.458484Z",
       "taskChecksum": "62e9feaef83fb9febfe3fc2a55ab7ca2fd505231114155b2e55ed880bb2c58b4",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "build-pov-ray__vfbH7aR",
       "job": "r1-09-build-pov-ray",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T17:15:12.418Z",
         "tokensAt": 51245,
         "residualTokens": 2133,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 54,
       "agentRequests": 54,
       "costUsd": 0.027963660000000005,
       "inputTokens": 843996,
       "outputTokens": 8762,
       "cacheTokens": 692306,
       "totalTokens": 852758,
       "durationSec": 253.148,
       "startedAt": "2026-09-25T17:12:18.359989Z",
       "finishedAt": "2026-09-25T17:16:31.507629Z",
       "taskChecksum": "62e9feaef83fb9febfe3fc2a55ab7ca2fd505231114155b2e55ed880bb2c58b4",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "build-pov-ray__vbqcjNo",
       "job": "r2-09-build-pov-ray",
       "round": 2,
       "state": "ERROR",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": "Error: Stream ended without terminal usage",
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 1,
       "agentRequests": 1,
       "costUsd": 0.4,
       "inputTokens": 0,
       "outputTokens": 0,
       "cacheTokens": 0,
       "totalTokens": 0,
       "durationSec": 32.658,
       "startedAt": "2026-09-25T19:37:49.081713Z",
       "finishedAt": "2026-09-25T19:38:21.739313Z",
       "taskChecksum": "62e9feaef83fb9febfe3fc2a55ab7ca2fd505231114155b2e55ed880bb2c58b4",
       "actualModels": [],
       "uncertainCost": true
      },
      {
       "trial": "build-pov-ray__MQyBdUW",
       "job": "r3-09-build-pov-ray",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T21:43:33.401Z",
         "tokensAt": 53093,
         "residualTokens": 6204,
         "qualified": true,
         "readsBefore": 1.8,
         "readsAfter": 1.2
        },
        {
         "at": "2026-09-25T21:47:08.961Z",
         "tokensAt": 49746,
         "residualTokens": 2331,
         "qualified": true,
         "readsBefore": 0.6,
         "readsAfter": 0
        }
       ],
       "providerRequests": 43,
       "agentRequests": 43,
       "costUsd": 0.034644264999999994,
       "inputTokens": 1047059,
       "outputTokens": 10856,
       "cacheTokens": 862534,
       "totalTokens": 1057915,
       "durationSec": 432.762,
       "startedAt": "2026-09-25T21:40:29.620445Z",
       "finishedAt": "2026-09-25T21:47:42.382070Z",
       "taskChecksum": "62e9feaef83fb9febfe3fc2a55ab7ca2fd505231114155b2e55ed880bb2c58b4",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.6666666666666666,
     "costUsd": 0.49959372500000004,
     "qualifiedCompactions": 5
    }
   },
   "diff": 0
  },
  {
   "name": "mcmc-sampling-stan",
   "set": "A",
   "timeoutSec": 1800,
   "category": "data-science",
   "task": "mcmc-sampling-stan",
   "compactionRate": 0.875,
   "group": "primary",
   "outcome": "Tied",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 1,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "mcmc-sampling-stan__hYb9ZF7",
       "job": "r0-10-mcmc-sampling-stan",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T15:18:01.903Z",
         "tokensAt": 55012,
         "residualTokens": 13072,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.8
        }
       ],
       "providerRequests": 22,
       "agentRequests": 21,
       "costUsd": 0.018851260000000005,
       "inputTokens": 483298,
       "outputTokens": 10633,
       "cacheTokens": 407616,
       "totalTokens": 493931,
       "durationSec": 2652.276,
       "startedAt": "2026-09-25T14:49:14.066580Z",
       "finishedAt": "2026-09-25T15:33:26.342336Z",
       "taskChecksum": "97dd19504c429a02e1dbec93ee9c0cc64d46fd886be8d88c59b84feb26144bef",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mcmc-sampling-stan__vCoswi5",
       "job": "r1-10-mcmc-sampling-stan",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T17:13:52.311Z",
         "tokensAt": 55260,
         "residualTokens": 20693,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T17:18:18.036Z",
         "tokensAt": 51559,
         "residualTokens": 29419,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 26,
       "agentRequests": 23,
       "costUsd": 0.028563180000000007,
       "inputTokens": 583728,
       "outputTokens": 12016,
       "cacheTokens": 438338,
       "totalTokens": 595744,
       "durationSec": 1642.831,
       "startedAt": "2026-09-25T16:53:51.239693Z",
       "finishedAt": "2026-09-25T17:21:14.070614Z",
       "taskChecksum": "97dd19504c429a02e1dbec93ee9c0cc64d46fd886be8d88c59b84feb26144bef",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mcmc-sampling-stan__ixoMY7g",
       "job": "r2-10-mcmc-sampling-stan",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T19:54:34.165Z",
         "tokensAt": 50183,
         "residualTokens": 27553,
         "qualified": true,
         "readsBefore": 1,
         "readsAfter": 1
        }
       ],
       "providerRequests": 22,
       "agentRequests": 21,
       "costUsd": 0.01792603,
       "inputTokens": 510139,
       "outputTokens": 6253,
       "cacheTokens": 425793,
       "totalTokens": 516392,
       "durationSec": 1117.863,
       "startedAt": "2026-09-25T19:38:33.307111Z",
       "finishedAt": "2026-09-25T19:57:11.170976Z",
       "taskChecksum": "97dd19504c429a02e1dbec93ee9c0cc64d46fd886be8d88c59b84feb26144bef",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mcmc-sampling-stan__E8md2TF",
       "job": "r3-10-mcmc-sampling-stan",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 14,
       "agentRequests": 14,
       "costUsd": 0.010717945000000001,
       "inputTokens": 309148,
       "outputTokens": 6409,
       "cacheTokens": 270687,
       "totalTokens": 315557,
       "durationSec": 1010.61,
       "startedAt": "2026-09-25T21:32:22.504427Z",
       "finishedAt": "2026-09-25T21:49:13.114609Z",
       "taskChecksum": "97dd19504c429a02e1dbec93ee9c0cc64d46fd886be8d88c59b84feb26144bef",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.25,
     "costUsd": 0.07605841500000002,
     "qualifiedCompactions": 4
    },
    "Native50kPi": {
     "passes": 1,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "mcmc-sampling-stan__CbJJGLs",
       "job": "r0-10-mcmc-sampling-stan",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T15:04:29.302Z",
         "tokensAt": 58877,
         "residualTokens": 2686,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.8
        }
       ],
       "providerRequests": 16,
       "agentRequests": 16,
       "costUsd": 0.017161500000000003,
       "inputTokens": 206804,
       "outputTokens": 5567,
       "cacheTokens": 83290,
       "totalTokens": 212371,
       "durationSec": 1997.02,
       "startedAt": "2026-09-25T14:49:14.091890Z",
       "finishedAt": "2026-09-25T15:22:31.111778Z",
       "taskChecksum": "97dd19504c429a02e1dbec93ee9c0cc64d46fd886be8d88c59b84feb26144bef",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mcmc-sampling-stan__iEyDuBp",
       "job": "r1-10-mcmc-sampling-stan",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T17:17:51.020Z",
         "tokensAt": 60041,
         "residualTokens": 3813,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.5
        }
       ],
       "providerRequests": 18,
       "agentRequests": 18,
       "costUsd": 0.01975186,
       "inputTokens": 325995,
       "outputTokens": 6720,
       "cacheTokens": 196086,
       "totalTokens": 332715,
       "durationSec": 1662.202,
       "startedAt": "2026-09-25T16:53:51.213397Z",
       "finishedAt": "2026-09-25T17:21:33.415426Z",
       "taskChecksum": "97dd19504c429a02e1dbec93ee9c0cc64d46fd886be8d88c59b84feb26144bef",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mcmc-sampling-stan__GLHXERT",
       "job": "r2-10-mcmc-sampling-stan",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T19:54:52.462Z",
         "tokensAt": 51887,
         "residualTokens": 2913,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 1
        }
       ],
       "providerRequests": 13,
       "agentRequests": 13,
       "costUsd": 0.014801725000000002,
       "inputTokens": 190057,
       "outputTokens": 5165,
       "cacheTokens": 87230,
       "totalTokens": 195222,
       "durationSec": 1164.245,
       "startedAt": "2026-09-25T19:38:33.334486Z",
       "finishedAt": "2026-09-25T19:57:57.579185Z",
       "taskChecksum": "97dd19504c429a02e1dbec93ee9c0cc64d46fd886be8d88c59b84feb26144bef",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mcmc-sampling-stan__NJG65RA",
       "job": "r3-10-mcmc-sampling-stan",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T21:45:42.321Z",
         "tokensAt": 58742,
         "residualTokens": 2680,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 18,
       "agentRequests": 18,
       "costUsd": 0.019578190000000002,
       "inputTokens": 352842,
       "outputTokens": 4794,
       "cacheTokens": 219794,
       "totalTokens": 357636,
       "durationSec": 1242.596,
       "startedAt": "2026-09-25T21:32:22.376973Z",
       "finishedAt": "2026-09-25T21:53:04.972109Z",
       "taskChecksum": "97dd19504c429a02e1dbec93ee9c0cc64d46fd886be8d88c59b84feb26144bef",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.25,
     "costUsd": 0.07129327500000002,
     "qualifiedCompactions": 4
    }
   },
   "diff": 0
  },
  {
   "name": "rstan-to-pystan",
   "set": "A",
   "timeoutSec": 1800,
   "category": "data-science",
   "task": "rstan-to-pystan",
   "compactionRate": 0,
   "group": "placebo",
   "outcome": "Both always pass",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "rstan-to-pystan__WEKjof6",
       "job": "r0-11-rstan-to-pystan",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 23,
       "agentRequests": 23,
       "costUsd": 0.012799900000000001,
       "inputTokens": 435482,
       "outputTokens": 7876,
       "cacheTokens": 396275,
       "totalTokens": 443358,
       "durationSec": 1381.924,
       "startedAt": "2026-09-25T14:54:05.070414Z",
       "finishedAt": "2026-09-25T15:17:06.994534Z",
       "taskChecksum": "1abf8912a1a885c218b97f347559dcba74804c675e5afe653aba1e24be5de6a8",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "rstan-to-pystan__6iNqBxu",
       "job": "r1-11-rstan-to-pystan",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 14,
       "agentRequests": 14,
       "costUsd": 0.010049984999999999,
       "inputTokens": 260501,
       "outputTokens": 7823,
       "cacheTokens": 229766,
       "totalTokens": 268324,
       "durationSec": 786.499,
       "startedAt": "2026-09-25T16:52:02.365801Z",
       "finishedAt": "2026-09-25T17:05:08.864058Z",
       "taskChecksum": "1abf8912a1a885c218b97f347559dcba74804c675e5afe653aba1e24be5de6a8",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "rstan-to-pystan__Bhj48qz",
       "job": "r2-11-rstan-to-pystan",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 21,
       "agentRequests": 21,
       "costUsd": 0.011015065000000001,
       "inputTokens": 337307,
       "outputTokens": 6152,
       "cacheTokens": 297589,
       "totalTokens": 343459,
       "durationSec": 817.419,
       "startedAt": "2026-09-25T19:39:00.980428Z",
       "finishedAt": "2026-09-25T19:52:38.399569Z",
       "taskChecksum": "1abf8912a1a885c218b97f347559dcba74804c675e5afe653aba1e24be5de6a8",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "rstan-to-pystan__owh5QWS",
       "job": "r3-11-rstan-to-pystan",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 18,
       "agentRequests": 18,
       "costUsd": 0.010458500000000003,
       "inputTokens": 284314,
       "outputTokens": 5998,
       "cacheTokens": 244160,
       "totalTokens": 290312,
       "durationSec": 893.776,
       "startedAt": "2026-09-25T21:25:25.134623Z",
       "finishedAt": "2026-09-25T21:40:18.910326Z",
       "taskChecksum": "1abf8912a1a885c218b97f347559dcba74804c675e5afe653aba1e24be5de6a8",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.04432345000000001,
     "qualifiedCompactions": 0
    },
    "Native50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "rstan-to-pystan__KDxYiQM",
       "job": "r0-11-rstan-to-pystan",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 22,
       "agentRequests": 22,
       "costUsd": 0.015463264999999999,
       "inputTokens": 564069,
       "outputTokens": 7589,
       "cacheTokens": 509464,
       "totalTokens": 571658,
       "durationSec": 1390.834,
       "startedAt": "2026-09-25T14:54:05.044509Z",
       "finishedAt": "2026-09-25T15:17:15.878250Z",
       "taskChecksum": "1abf8912a1a885c218b97f347559dcba74804c675e5afe653aba1e24be5de6a8",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "rstan-to-pystan__PCw7vuJ",
       "job": "r1-11-rstan-to-pystan",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 27,
       "agentRequests": 27,
       "costUsd": 0.016485899999999998,
       "inputTokens": 604592,
       "outputTokens": 7320,
       "cacheTokens": 541520,
       "totalTokens": 611912,
       "durationSec": 1212.916,
       "startedAt": "2026-09-25T16:52:02.391869Z",
       "finishedAt": "2026-09-25T17:12:15.307770Z",
       "taskChecksum": "1abf8912a1a885c218b97f347559dcba74804c675e5afe653aba1e24be5de6a8",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "rstan-to-pystan__2V5PtK9",
       "job": "r2-11-rstan-to-pystan",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 21,
       "agentRequests": 21,
       "costUsd": 0.013666574999999999,
       "inputTokens": 455726,
       "outputTokens": 7208,
       "cacheTokens": 405450,
       "totalTokens": 462934,
       "durationSec": 812.581,
       "startedAt": "2026-09-25T19:39:00.954880Z",
       "finishedAt": "2026-09-25T19:52:33.535049Z",
       "taskChecksum": "1abf8912a1a885c218b97f347559dcba74804c675e5afe653aba1e24be5de6a8",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "rstan-to-pystan__BcqsZVz",
       "job": "r3-11-rstan-to-pystan",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 26,
       "agentRequests": 26,
       "costUsd": 0.013150510000000002,
       "inputTokens": 367150,
       "outputTokens": 6674,
       "cacheTokens": 309506,
       "totalTokens": 373824,
       "durationSec": 901.283,
       "startedAt": "2026-09-25T21:25:25.161223Z",
       "finishedAt": "2026-09-25T21:40:26.444662Z",
       "taskChecksum": "1abf8912a1a885c218b97f347559dcba74804c675e5afe653aba1e24be5de6a8",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.05876625,
     "qualifiedCompactions": 0
    }
   },
   "diff": 0
  },
  {
   "name": "distribution-search",
   "set": "B",
   "timeoutSec": 3600,
   "category": "machine-learning",
   "task": "distribution-search",
   "compactionRate": 0,
   "group": "placebo",
   "outcome": "Both always pass",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "distribution-search__WMEcLUo",
       "job": "r0-12-distribution-search",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 5,
       "agentRequests": 5,
       "costUsd": 0.0033369199999999997,
       "inputTokens": 22035,
       "outputTokens": 4683,
       "cacheTokens": 15292,
       "totalTokens": 26718,
       "durationSec": 103.742,
       "startedAt": "2026-09-25T15:17:19.325169Z",
       "finishedAt": "2026-09-25T15:19:03.067025Z",
       "taskChecksum": "dcd92ac5501340d190b2276e1f60ece558c8da07c104617b9d216b47cf5fc32f",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "distribution-search__Jj84zJf",
       "job": "r1-12-distribution-search",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 7,
       "agentRequests": 7,
       "costUsd": 0.003651765,
       "inputTokens": 35338,
       "outputTokens": 5003,
       "cacheTokens": 28404,
       "totalTokens": 40341,
       "durationSec": 88.941,
       "startedAt": "2026-09-25T16:50:30.612375Z",
       "finishedAt": "2026-09-25T16:51:59.553628Z",
       "taskChecksum": "dcd92ac5501340d190b2276e1f60ece558c8da07c104617b9d216b47cf5fc32f",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "distribution-search__r98NM5L",
       "job": "r2-12-distribution-search",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 5,
       "agentRequests": 5,
       "costUsd": 0.0035750000000000005,
       "inputTokens": 23205,
       "outputTokens": 5156,
       "cacheTokens": 16550,
       "totalTokens": 28361,
       "durationSec": 87.699,
       "startedAt": "2026-09-25T19:52:41.155987Z",
       "finishedAt": "2026-09-25T19:54:08.854650Z",
       "taskChecksum": "dcd92ac5501340d190b2276e1f60ece558c8da07c104617b9d216b47cf5fc32f",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "distribution-search__woCTWwT",
       "job": "r3-12-distribution-search",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 9,
       "agentRequests": 9,
       "costUsd": 0.00340142,
       "inputTokens": 40395,
       "outputTokens": 4404,
       "cacheTokens": 33472,
       "totalTokens": 44799,
       "durationSec": 77.768,
       "startedAt": "2026-09-25T21:24:04.286534Z",
       "finishedAt": "2026-09-25T21:25:22.054964Z",
       "taskChecksum": "dcd92ac5501340d190b2276e1f60ece558c8da07c104617b9d216b47cf5fc32f",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.013965105,
     "qualifiedCompactions": 0
    },
    "Native50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "distribution-search__kUyC9NX",
       "job": "r0-12-distribution-search",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 8,
       "agentRequests": 8,
       "costUsd": 0.004908015,
       "inputTokens": 38687,
       "outputTokens": 6100,
       "cacheTokens": 24454,
       "totalTokens": 44787,
       "durationSec": 134.815,
       "startedAt": "2026-09-25T15:17:19.359439Z",
       "finishedAt": "2026-09-25T15:19:34.174702Z",
       "taskChecksum": "dcd92ac5501340d190b2276e1f60ece558c8da07c104617b9d216b47cf5fc32f",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "distribution-search__HsJNeVe",
       "job": "r1-12-distribution-search",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 5,
       "agentRequests": 5,
       "costUsd": 0.0027361300000000002,
       "inputTokens": 16677,
       "outputTokens": 3461,
       "cacheTokens": 8938,
       "totalTokens": 20138,
       "durationSec": 74.138,
       "startedAt": "2026-09-25T16:50:30.579969Z",
       "finishedAt": "2026-09-25T16:51:44.717246Z",
       "taskChecksum": "dcd92ac5501340d190b2276e1f60ece558c8da07c104617b9d216b47cf5fc32f",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "distribution-search__vqE2pNd",
       "job": "r2-12-distribution-search",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 4,
       "agentRequests": 4,
       "costUsd": 0.00251864,
       "inputTokens": 14801,
       "outputTokens": 3112,
       "cacheTokens": 7324,
       "totalTokens": 17913,
       "durationSec": 60.408,
       "startedAt": "2026-09-25T19:52:41.189576Z",
       "finishedAt": "2026-09-25T19:53:41.597543Z",
       "taskChecksum": "dcd92ac5501340d190b2276e1f60ece558c8da07c104617b9d216b47cf5fc32f",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "distribution-search__bX5VQ8K",
       "job": "r3-12-distribution-search",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 6,
       "agentRequests": 6,
       "costUsd": 0.0034627850000000003,
       "inputTokens": 23086,
       "outputTokens": 4048,
       "cacheTokens": 10986,
       "totalTokens": 27134,
       "durationSec": 72.32,
       "startedAt": "2026-09-25T21:24:04.261322Z",
       "finishedAt": "2026-09-25T21:25:16.581185Z",
       "taskChecksum": "dcd92ac5501340d190b2276e1f60ece558c8da07c104617b9d216b47cf5fc32f",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.01362557,
     "qualifiedCompactions": 0
    }
   },
   "diff": 0
  },
  {
   "name": "mteb-leaderboard",
   "set": "B",
   "timeoutSec": 3600,
   "category": "data-science",
   "task": "mteb-leaderboard",
   "compactionRate": 0.875,
   "group": "primary",
   "outcome": "Textual ahead",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 2,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "mteb-leaderboard__QpcWSH5",
       "job": "r0-13-mteb-leaderboard",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 4,
       "qualifiedCompactions": 4,
       "compactions": [
        {
         "at": "2026-09-25T15:24:04.100Z",
         "tokensAt": 50222,
         "residualTokens": 24265,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T15:25:10.676Z",
         "tokensAt": 50578,
         "residualTokens": 23251,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T15:26:06.491Z",
         "tokensAt": 51250,
         "residualTokens": 14956,
         "qualified": true,
         "readsBefore": 0.8,
         "readsAfter": 1.4
        },
        {
         "at": "2026-09-25T15:28:38.563Z",
         "tokensAt": 51526,
         "residualTokens": 28322,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.8
        }
       ],
       "providerRequests": 97,
       "agentRequests": 90,
       "costUsd": 0.07987642,
       "inputTokens": 2868308,
       "outputTokens": 35793,
       "cacheTokens": 2578707,
       "totalTokens": 2904101,
       "durationSec": 757.244,
       "startedAt": "2026-09-25T15:18:30.277069Z",
       "finishedAt": "2026-09-25T15:31:07.521932Z",
       "taskChecksum": "4949cc7d135fa60c6fdd1e649cab556adaccce34839a8af2a6f4cf89ca9c5221",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-leaderboard__2aPCdeA",
       "job": "r1-13-mteb-leaderboard",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T16:48:56.812Z",
         "tokensAt": 50419,
         "residualTokens": 24593,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T16:50:01.501Z",
         "tokensAt": 51852,
         "residualTokens": 28007,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 49,
       "agentRequests": 46,
       "costUsd": 0.03243657,
       "inputTokens": 1166249,
       "outputTokens": 9531,
       "cacheTokens": 1027012,
       "totalTokens": 1175780,
       "durationSec": 212.293,
       "startedAt": "2026-09-25T16:47:09.297323Z",
       "finishedAt": "2026-09-25T16:50:41.590246Z",
       "taskChecksum": "4949cc7d135fa60c6fdd1e649cab556adaccce34839a8af2a6f4cf89ca9c5221",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-leaderboard__ta3ZQcb",
       "job": "r2-13-mteb-leaderboard",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 4,
       "qualifiedCompactions": 4,
       "compactions": [
        {
         "at": "2026-09-25T19:59:15.121Z",
         "tokensAt": 54831,
         "residualTokens": 22038,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:00:20.251Z",
         "tokensAt": 50327,
         "residualTokens": 29683,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.4
        },
        {
         "at": "2026-09-25T20:02:30.472Z",
         "tokensAt": 50620,
         "residualTokens": 23584,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T20:03:30.590Z",
         "tokensAt": 53012,
         "residualTokens": 32983,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        }
       ],
       "providerRequests": 110,
       "agentRequests": 105,
       "costUsd": 0.08095047999999996,
       "inputTokens": 3404025,
       "outputTokens": 27628,
       "cacheTokens": 3116008,
       "totalTokens": 3431653,
       "durationSec": 778.145,
       "startedAt": "2026-09-25T19:54:12.456130Z",
       "finishedAt": "2026-09-25T20:07:10.601170Z",
       "taskChecksum": "4949cc7d135fa60c6fdd1e649cab556adaccce34839a8af2a6f4cf89ca9c5221",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-leaderboard__jLmwL5k",
       "job": "r3-13-mteb-leaderboard",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 4,
       "qualifiedCompactions": 4,
       "compactions": [
        {
         "at": "2026-09-25T21:25:40.893Z",
         "tokensAt": 51138,
         "residualTokens": 27213,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T21:26:01.529Z",
         "tokensAt": 58236,
         "residualTokens": 19198,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T21:27:24.365Z",
         "tokensAt": 51692,
         "residualTokens": 34184,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T21:27:57.140Z",
         "tokensAt": 50313,
         "residualTokens": 24386,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 61,
       "agentRequests": 55,
       "costUsd": 0.05861050499999999,
       "inputTokens": 1725055,
       "outputTokens": 24808,
       "cacheTokens": 1473003,
       "totalTokens": 1749863,
       "durationSec": 497.157,
       "startedAt": "2026-09-25T21:23:52.858971Z",
       "finishedAt": "2026-09-25T21:32:10.015734Z",
       "taskChecksum": "4949cc7d135fa60c6fdd1e649cab556adaccce34839a8af2a6f4cf89ca9c5221",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.5,
     "costUsd": 0.25187397499999997,
     "qualifiedCompactions": 14
    },
    "Native50kPi": {
     "passes": 0,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "mteb-leaderboard__TvxTBrf",
       "job": "r0-13-mteb-leaderboard",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 19,
       "agentRequests": 19,
       "costUsd": 0.009218309999999997,
       "inputTokens": 300392,
       "outputTokens": 3357,
       "cacheTokens": 258116,
       "totalTokens": 303749,
       "durationSec": 141.341,
       "startedAt": "2026-09-25T15:18:30.250252Z",
       "finishedAt": "2026-09-25T15:20:51.591696Z",
       "taskChecksum": "4949cc7d135fa60c6fdd1e649cab556adaccce34839a8af2a6f4cf89ca9c5221",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-leaderboard__2DSWkjS",
       "job": "r1-13-mteb-leaderboard",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T16:50:10.827Z",
         "tokensAt": 64623,
         "residualTokens": 10715,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 65,
       "agentRequests": 65,
       "costUsd": 0.037627415,
       "inputTokens": 1261478,
       "outputTokens": 14640,
       "cacheTokens": 1086024,
       "totalTokens": 1276118,
       "durationSec": 398.982,
       "startedAt": "2026-09-25T16:47:09.338608Z",
       "finishedAt": "2026-09-25T16:53:48.320797Z",
       "taskChecksum": "4949cc7d135fa60c6fdd1e649cab556adaccce34839a8af2a6f4cf89ca9c5221",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-leaderboard__3kZiLnf",
       "job": "r2-13-mteb-leaderboard",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 3,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T19:57:54.261Z",
         "tokensAt": 51544,
         "residualTokens": 3350,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T19:58:38.764Z",
         "tokensAt": 58712,
         "residualTokens": 6222,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:02:16.245Z",
         "tokensAt": 51048,
         "residualTokens": 14779,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.8
        }
       ],
       "providerRequests": 96,
       "agentRequests": 96,
       "costUsd": 0.07283158000000002,
       "inputTokens": 2521833,
       "outputTokens": 29591,
       "cacheTokens": 2200428,
       "totalTokens": 2551424,
       "durationSec": 578.465,
       "startedAt": "2026-09-25T19:54:12.421334Z",
       "finishedAt": "2026-09-25T20:03:50.886490Z",
       "taskChecksum": "4949cc7d135fa60c6fdd1e649cab556adaccce34839a8af2a6f4cf89ca9c5221",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-leaderboard__2SHdfWD",
       "job": "r3-13-mteb-leaderboard",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T21:27:23.452Z",
         "tokensAt": 52739,
         "residualTokens": 2972,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 53,
       "agentRequests": 53,
       "costUsd": 0.03032091500000001,
       "inputTokens": 975844,
       "outputTokens": 13731,
       "cacheTokens": 837714,
       "totalTokens": 989575,
       "durationSec": 267.027,
       "startedAt": "2026-09-25T21:23:52.883295Z",
       "finishedAt": "2026-09-25T21:28:19.910959Z",
       "taskChecksum": "4949cc7d135fa60c6fdd1e649cab556adaccce34839a8af2a6f4cf89ca9c5221",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0,
     "costUsd": 0.14999822000000002,
     "qualifiedCompactions": 5
    }
   },
   "diff": -0.5
  },
  {
   "name": "portfolio-optimization",
   "set": "B",
   "timeoutSec": 3600,
   "category": "optimization",
   "task": "portfolio-optimization",
   "compactionRate": 0,
   "group": "placebo",
   "outcome": "Both always pass",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "portfolio-optimization__8YiBoGz",
       "job": "r0-14-portfolio-optimization",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 11,
       "agentRequests": 11,
       "costUsd": 0.005067320000000001,
       "inputTokens": 81802,
       "outputTokens": 5999,
       "cacheTokens": 70927,
       "totalTokens": 87801,
       "durationSec": 262.008,
       "startedAt": "2026-09-25T15:19:37.291107Z",
       "finishedAt": "2026-09-25T15:23:59.299368Z",
       "taskChecksum": "9e47f44ac76ab6ae52cbcadc85923ff253ed4f044b7b4e457d75af1d041f703b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "portfolio-optimization__bWhGVQN",
       "job": "r1-14-portfolio-optimization",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 8,
       "agentRequests": 8,
       "costUsd": 0.00342682,
       "inputTokens": 48547,
       "outputTokens": 4014,
       "cacheTokens": 40417,
       "totalTokens": 52561,
       "durationSec": 206.042,
       "startedAt": "2026-09-25T16:45:50.366572Z",
       "finishedAt": "2026-09-25T16:49:16.408047Z",
       "taskChecksum": "9e47f44ac76ab6ae52cbcadc85923ff253ed4f044b7b4e457d75af1d041f703b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "portfolio-optimization__BQjeshp",
       "job": "r2-14-portfolio-optimization",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 6,
       "agentRequests": 6,
       "costUsd": 0.002670295,
       "inputTokens": 30922,
       "outputTokens": 3128,
       "cacheTokens": 23987,
       "totalTokens": 34050,
       "durationSec": 176.479,
       "startedAt": "2026-09-25T19:57:47.691088Z",
       "finishedAt": "2026-09-25T20:00:44.170931Z",
       "taskChecksum": "9e47f44ac76ab6ae52cbcadc85923ff253ed4f044b7b4e457d75af1d041f703b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "portfolio-optimization__WgGzfys",
       "job": "r3-14-portfolio-optimization",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 9,
       "agentRequests": 9,
       "costUsd": 0.004650905,
       "inputTokens": 59592,
       "outputTokens": 5769,
       "cacheTokens": 49408,
       "totalTokens": 65361,
       "durationSec": 226.948,
       "startedAt": "2026-09-25T21:20:02.889598Z",
       "finishedAt": "2026-09-25T21:23:49.837778Z",
       "taskChecksum": "9e47f44ac76ab6ae52cbcadc85923ff253ed4f044b7b4e457d75af1d041f703b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.01581534,
     "qualifiedCompactions": 0
    },
    "Native50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "portfolio-optimization__uYrpDjL",
       "job": "r0-14-portfolio-optimization",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 6,
       "agentRequests": 6,
       "costUsd": 0.0035314599999999993,
       "inputTokens": 32580,
       "outputTokens": 4084,
       "cacheTokens": 21816,
       "totalTokens": 36664,
       "durationSec": 231.23,
       "startedAt": "2026-09-25T15:19:37.327808Z",
       "finishedAt": "2026-09-25T15:23:28.557769Z",
       "taskChecksum": "9e47f44ac76ab6ae52cbcadc85923ff253ed4f044b7b4e457d75af1d041f703b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "portfolio-optimization__mH54fQd",
       "job": "r1-14-portfolio-optimization",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 10,
       "agentRequests": 10,
       "costUsd": 0.00765616,
       "inputTokens": 84267,
       "outputTokens": 9456,
       "cacheTokens": 65136,
       "totalTokens": 93723,
       "durationSec": 277.402,
       "startedAt": "2026-09-25T16:45:50.340004Z",
       "finishedAt": "2026-09-25T16:50:27.742350Z",
       "taskChecksum": "9e47f44ac76ab6ae52cbcadc85923ff253ed4f044b7b4e457d75af1d041f703b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "portfolio-optimization__kXSwQBA",
       "job": "r2-14-portfolio-optimization",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 13,
       "agentRequests": 13,
       "costUsd": 0.0069527799999999996,
       "inputTokens": 110687,
       "outputTokens": 7363,
       "cacheTokens": 90458,
       "totalTokens": 118050,
       "durationSec": 260.547,
       "startedAt": "2026-09-25T19:57:47.716445Z",
       "finishedAt": "2026-09-25T20:02:08.263458Z",
       "taskChecksum": "9e47f44ac76ab6ae52cbcadc85923ff253ed4f044b7b4e457d75af1d041f703b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "portfolio-optimization__WxzJccs",
       "job": "r3-14-portfolio-optimization",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 6,
       "agentRequests": 6,
       "costUsd": 0.0045368349999999995,
       "inputTokens": 32501,
       "outputTokens": 5091,
       "cacheTokens": 16106,
       "totalTokens": 37592,
       "durationSec": 213.72,
       "startedAt": "2026-09-25T21:20:02.863872Z",
       "finishedAt": "2026-09-25T21:23:36.583894Z",
       "taskChecksum": "9e47f44ac76ab6ae52cbcadc85923ff253ed4f044b7b4e457d75af1d041f703b",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.022677234999999997,
     "qualifiedCompactions": 0
    }
   },
   "diff": 0
  },
  {
   "name": "reshard-c4-data",
   "set": "B",
   "timeoutSec": 3600,
   "category": "data-science",
   "task": "reshard-c4-data",
   "compactionRate": 0.125,
   "group": "other",
   "outcome": "Both always pass",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "reshard-c4-data__VD7FhXH",
       "job": "r0-15-reshard-c4-data",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 19,
       "agentRequests": 19,
       "costUsd": 0.019897724999999998,
       "inputTokens": 468924,
       "outputTokens": 22132,
       "cacheTokens": 432890,
       "totalTokens": 491056,
       "durationSec": 426.684,
       "startedAt": "2026-09-25T15:24:01.930175Z",
       "finishedAt": "2026-09-25T15:31:08.614649Z",
       "taskChecksum": "7822970dc6285d1a64b7338fd11be420d2157ab14f4b49c5542ad69934115dce",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "reshard-c4-data__yjtSg5u",
       "job": "r1-15-reshard-c4-data",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 18,
       "agentRequests": 18,
       "costUsd": 0.019427985000000002,
       "inputTokens": 395348,
       "outputTokens": 23942,
       "cacheTokens": 364871,
       "totalTokens": 419290,
       "durationSec": 361.175,
       "startedAt": "2026-09-25T16:41:05.038843Z",
       "finishedAt": "2026-09-25T16:47:06.213991Z",
       "taskChecksum": "7822970dc6285d1a64b7338fd11be420d2157ab14f4b49c5542ad69934115dce",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "reshard-c4-data__vobUTie",
       "job": "r2-15-reshard-c4-data",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 25,
       "agentRequests": 25,
       "costUsd": 0.018321615000000003,
       "inputTokens": 458955,
       "outputTokens": 21222,
       "cacheTokens": 431799,
       "totalTokens": 480177,
       "durationSec": 306.807,
       "startedAt": "2026-09-25T19:58:00.313480Z",
       "finishedAt": "2026-09-25T20:03:07.120035Z",
       "taskChecksum": "7822970dc6285d1a64b7338fd11be420d2157ab14f4b49c5542ad69934115dce",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "reshard-c4-data__jZaVCFD",
       "job": "r3-15-reshard-c4-data",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 19,
       "agentRequests": 19,
       "costUsd": 0.018601799999999995,
       "inputTokens": 413729,
       "outputTokens": 21763,
       "cacheTokens": 382560,
       "totalTokens": 435492,
       "durationSec": 315.641,
       "startedAt": "2026-09-25T21:18:43.641075Z",
       "finishedAt": "2026-09-25T21:23:59.282400Z",
       "taskChecksum": "7822970dc6285d1a64b7338fd11be420d2157ab14f4b49c5542ad69934115dce",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.076249125,
     "qualifiedCompactions": 0
    },
    "Native50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "reshard-c4-data__hwkys7S",
       "job": "r0-15-reshard-c4-data",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 17,
       "agentRequests": 17,
       "costUsd": 0.018449050000000005,
       "inputTokens": 380689,
       "outputTokens": 20744,
       "cacheTokens": 341650,
       "totalTokens": 401433,
       "durationSec": 394.407,
       "startedAt": "2026-09-25T15:24:01.899575Z",
       "finishedAt": "2026-09-25T15:30:36.306829Z",
       "taskChecksum": "7822970dc6285d1a64b7338fd11be420d2157ab14f4b49c5542ad69934115dce",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "reshard-c4-data__pkbE4cx",
       "job": "r1-15-reshard-c4-data",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 22,
       "agentRequests": 22,
       "costUsd": 0.01952179499999999,
       "inputTokens": 453026,
       "outputTokens": 21609,
       "cacheTokens": 414232,
       "totalTokens": 474635,
       "durationSec": 353.631,
       "startedAt": "2026-09-25T16:41:05.067860Z",
       "finishedAt": "2026-09-25T16:46:58.698987Z",
       "taskChecksum": "7822970dc6285d1a64b7338fd11be420d2157ab14f4b49c5542ad69934115dce",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "reshard-c4-data__b7Z4L3s",
       "job": "r2-15-reshard-c4-data",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T20:01:11.637Z",
         "tokensAt": 52027,
         "residualTokens": 7382,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 1
        }
       ],
       "providerRequests": 18,
       "agentRequests": 18,
       "costUsd": 0.021936185000000004,
       "inputTokens": 316615,
       "outputTokens": 18894,
       "cacheTokens": 222276,
       "totalTokens": 335509,
       "durationSec": 290.728,
       "startedAt": "2026-09-25T19:58:00.282113Z",
       "finishedAt": "2026-09-25T20:02:51.010892Z",
       "taskChecksum": "7822970dc6285d1a64b7338fd11be420d2157ab14f4b49c5542ad69934115dce",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "reshard-c4-data__FVytnur",
       "job": "r3-15-reshard-c4-data",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 16,
       "agentRequests": 16,
       "costUsd": 0.018542174999999998,
       "inputTokens": 374049,
       "outputTokens": 21174,
       "cacheTokens": 335940,
       "totalTokens": 395223,
       "durationSec": 317.6,
       "startedAt": "2026-09-25T21:18:43.669720Z",
       "finishedAt": "2026-09-25T21:24:01.269275Z",
       "taskChecksum": "7822970dc6285d1a64b7338fd11be420d2157ab14f4b49c5542ad69934115dce",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.078449205,
     "qualifiedCompactions": 1
    }
   },
   "diff": 0
  },
  {
   "name": "sam-cell-seg",
   "set": "B",
   "timeoutSec": 7200,
   "category": "data-science",
   "task": "sam-cell-seg",
   "compactionRate": 0,
   "group": "placebo",
   "outcome": "Encrypted ahead",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 3,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "sam-cell-seg__vBxXXXP",
       "job": "r0-16-sam-cell-seg",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 10,
       "agentRequests": 10,
       "costUsd": 0.01314259,
       "inputTokens": 160976,
       "outputTokens": 17710,
       "cacheTokens": 137684,
       "totalTokens": 178686,
       "durationSec": 361.666,
       "startedAt": "2026-09-25T15:31:10.111739Z",
       "finishedAt": "2026-09-25T15:37:11.777927Z",
       "taskChecksum": "9d6e0538e93d2fc3c8c8ba361125c08ece2c191b1b1148c5bcb5d94e77a622c5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "sam-cell-seg__7RU5P8t",
       "job": "r1-16-sam-cell-seg",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 10,
       "agentRequests": 10,
       "costUsd": 0.012150415000000001,
       "inputTokens": 152114,
       "outputTokens": 15801,
       "cacheTokens": 128379,
       "totalTokens": 167915,
       "durationSec": 283.457,
       "startedAt": "2026-09-25T16:41:03.832227Z",
       "finishedAt": "2026-09-25T16:45:47.289002Z",
       "taskChecksum": "9d6e0538e93d2fc3c8c8ba361125c08ece2c191b1b1148c5bcb5d94e77a622c5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "sam-cell-seg__Xt6kyHY",
       "job": "r2-16-sam-cell-seg",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 8,
       "agentRequests": 8,
       "costUsd": 0.009468365,
       "inputTokens": 81286,
       "outputTokens": 12867,
       "cacheTokens": 61959,
       "totalTokens": 94153,
       "durationSec": 217.08,
       "startedAt": "2026-09-25T20:02:11.069858Z",
       "finishedAt": "2026-09-25T20:05:48.149553Z",
       "taskChecksum": "9d6e0538e93d2fc3c8c8ba361125c08ece2c191b1b1148c5bcb5d94e77a622c5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "sam-cell-seg__dsfsBLz",
       "job": "r3-16-sam-cell-seg",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 9,
       "agentRequests": 9,
       "costUsd": 0.009564865,
       "inputTokens": 80929,
       "outputTokens": 13522,
       "cacheTokens": 63579,
       "totalTokens": 94451,
       "durationSec": 303.095,
       "startedAt": "2026-09-25T21:14:56.714418Z",
       "finishedAt": "2026-09-25T21:19:59.809330Z",
       "taskChecksum": "9d6e0538e93d2fc3c8c8ba361125c08ece2c191b1b1148c5bcb5d94e77a622c5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0.75,
     "costUsd": 0.044326235,
     "qualifiedCompactions": 0
    },
    "Native50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "sam-cell-seg__aw8W3jr",
       "job": "r0-16-sam-cell-seg",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 7,
       "agentRequests": 7,
       "costUsd": 0.009251144999999999,
       "inputTokens": 79870,
       "outputTokens": 12024,
       "cacheTokens": 57812,
       "totalTokens": 91894,
       "durationSec": 312.929,
       "startedAt": "2026-09-25T15:31:10.140571Z",
       "finishedAt": "2026-09-25T15:36:23.069244Z",
       "taskChecksum": "9d6e0538e93d2fc3c8c8ba361125c08ece2c191b1b1148c5bcb5d94e77a622c5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "sam-cell-seg__spxSJQk",
       "job": "r1-16-sam-cell-seg",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 6,
       "agentRequests": 6,
       "costUsd": 0.006986279999999999,
       "inputTokens": 36133,
       "outputTokens": 9844,
       "cacheTokens": 20948,
       "totalTokens": 45977,
       "durationSec": 241.322,
       "startedAt": "2026-09-25T16:41:03.796049Z",
       "finishedAt": "2026-09-25T16:45:05.118251Z",
       "taskChecksum": "9d6e0538e93d2fc3c8c8ba361125c08ece2c191b1b1148c5bcb5d94e77a622c5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "sam-cell-seg__DCAQvUr",
       "job": "r2-16-sam-cell-seg",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 7,
       "agentRequests": 7,
       "costUsd": 0.01215862,
       "inputTokens": 79785,
       "outputTokens": 17530,
       "cacheTokens": 56872,
       "totalTokens": 97315,
       "durationSec": 303.434,
       "startedAt": "2026-09-25T20:02:11.095909Z",
       "finishedAt": "2026-09-25T20:07:14.529607Z",
       "taskChecksum": "9d6e0538e93d2fc3c8c8ba361125c08ece2c191b1b1148c5bcb5d94e77a622c5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "sam-cell-seg__twAitJ6",
       "job": "r3-16-sam-cell-seg",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 7,
       "agentRequests": 7,
       "costUsd": 0.008534765,
       "inputTokens": 63931,
       "outputTokens": 11916,
       "cacheTokens": 46114,
       "totalTokens": 75847,
       "durationSec": 300.079,
       "startedAt": "2026-09-25T21:14:56.676389Z",
       "finishedAt": "2026-09-25T21:19:56.755861Z",
       "taskChecksum": "9d6e0538e93d2fc3c8c8ba361125c08ece2c191b1b1148c5bcb5d94e77a622c5",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.03693081,
     "qualifiedCompactions": 0
    }
   },
   "diff": 0.25
  },
  {
   "name": "train-fasttext",
   "set": "B",
   "timeoutSec": 3600,
   "category": "model-training",
   "task": "train-fasttext",
   "compactionRate": 0.375,
   "group": "other",
   "outcome": "Both always fail",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 0,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "train-fasttext__zasZYSE",
       "job": "r0-17-train-fasttext",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T16:05:17.770Z",
         "tokensAt": 51443,
         "residualTokens": 33510,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 25,
       "agentRequests": 24,
       "costUsd": 0.024349225,
       "inputTokens": 594174,
       "outputTokens": 15276,
       "cacheTokens": 500510,
       "totalTokens": 609450,
       "durationSec": 4137.69,
       "startedAt": "2026-09-25T15:31:11.358728Z",
       "finishedAt": "2026-09-25T16:40:09.048205Z",
       "taskChecksum": "be84205f314cc6c7318a200c4705fa61e4aba340b736de02a6ddaabcbbb4c524",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "train-fasttext__sHQdUMt",
       "job": "r1-17-train-fasttext",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 15,
       "agentRequests": 15,
       "costUsd": 0.00799212,
       "inputTokens": 162430,
       "outputTokens": 4678,
       "cacheTokens": 127387,
       "totalTokens": 167108,
       "durationSec": 3771.785,
       "startedAt": "2026-09-25T16:09:05.756815Z",
       "finishedAt": "2026-09-25T17:11:57.541579Z",
       "taskChecksum": "be84205f314cc6c7318a200c4705fa61e4aba340b736de02a6ddaabcbbb4c524",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "train-fasttext__Pjhz7xY",
       "job": "r2-17-train-fasttext",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T20:50:57.398Z",
         "tokensAt": 56463,
         "residualTokens": 29935,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0
        }
       ],
       "providerRequests": 26,
       "agentRequests": 25,
       "costUsd": 0.02417160500000001,
       "inputTokens": 575342,
       "outputTokens": 17141,
       "cacheTokens": 489693,
       "totalTokens": 592483,
       "durationSec": 4087.297,
       "startedAt": "2026-09-25T20:03:09.809629Z",
       "finishedAt": "2026-09-25T21:11:17.106641Z",
       "taskChecksum": "be84205f314cc6c7318a200c4705fa61e4aba340b736de02a6ddaabcbbb4c524",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "train-fasttext__GZb29vo",
       "job": "r3-17-train-fasttext",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 9,
       "agentRequests": 9,
       "costUsd": 0.004237365,
       "inputTokens": 77007,
       "outputTokens": 2272,
       "cacheTokens": 56729,
       "totalTokens": 79279,
       "durationSec": 4340.616,
       "startedAt": "2026-09-25T21:11:16.567578Z",
       "finishedAt": "2026-09-25T22:23:37.183744Z",
       "taskChecksum": "be84205f314cc6c7318a200c4705fa61e4aba340b736de02a6ddaabcbbb4c524",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0,
     "costUsd": 0.060750315000000006,
     "qualifiedCompactions": 2
    },
    "Native50kPi": {
     "passes": 0,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "train-fasttext__Lzk5pw6",
       "job": "r0-17-train-fasttext",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 16,
       "agentRequests": 16,
       "costUsd": 0.012981505,
       "inputTokens": 389641,
       "outputTokens": 5485,
       "cacheTokens": 331688,
       "totalTokens": 395126,
       "durationSec": 4189.219,
       "startedAt": "2026-09-25T15:31:11.316850Z",
       "finishedAt": "2026-09-25T16:41:00.535428Z",
       "taskChecksum": "be84205f314cc6c7318a200c4705fa61e4aba340b736de02a6ddaabcbbb4c524",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "train-fasttext__nEok7La",
       "job": "r1-17-train-fasttext",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 10,
       "agentRequests": 10,
       "costUsd": 0.004942725,
       "inputTokens": 64221,
       "outputTokens": 2413,
       "cacheTokens": 33670,
       "totalTokens": 66634,
       "durationSec": 4191.637,
       "startedAt": "2026-09-25T16:09:05.800678Z",
       "finishedAt": "2026-09-25T17:18:57.437988Z",
       "taskChecksum": "be84205f314cc6c7318a200c4705fa61e4aba340b736de02a6ddaabcbbb4c524",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "train-fasttext__ytWfVnN",
       "job": "r2-17-train-fasttext",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 17,
       "agentRequests": 17,
       "costUsd": 0.012486880000000002,
       "inputTokens": 379459,
       "outputTokens": 4799,
       "cacheTokens": 322628,
       "totalTokens": 384258,
       "durationSec": 4303.375,
       "startedAt": "2026-09-25T20:03:09.767571Z",
       "finishedAt": "2026-09-25T21:14:53.142478Z",
       "taskChecksum": "be84205f314cc6c7318a200c4705fa61e4aba340b736de02a6ddaabcbbb4c524",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "train-fasttext__HX2u5H4",
       "job": "r3-17-train-fasttext",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T22:02:16.566Z",
         "tokensAt": 52397,
         "residualTokens": 5052,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        }
       ],
       "providerRequests": 46,
       "agentRequests": 46,
       "costUsd": 0.042765779999999996,
       "inputTokens": 892844,
       "outputTokens": 34681,
       "cacheTokens": 727678,
       "totalTokens": 927525,
       "durationSec": 3706.753,
       "startedAt": "2026-09-25T21:11:16.613559Z",
       "finishedAt": "2026-09-25T22:13:03.366301Z",
       "taskChecksum": "be84205f314cc6c7318a200c4705fa61e4aba340b736de02a6ddaabcbbb4c524",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0,
     "costUsd": 0.07317689,
     "qualifiedCompactions": 1
    }
   },
   "diff": 0
  },
  {
   "name": "winning-avg-corewars",
   "set": "B",
   "timeoutSec": 3600,
   "category": "software-engineering",
   "task": "winning-avg-corewars",
   "compactionRate": 0.375,
   "group": "other",
   "outcome": "Both always pass",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "winning-avg-corewars__qZr99wf",
       "job": "r0-18-winning-avg-corewars",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 30,
       "agentRequests": 30,
       "costUsd": 0.020915345000000002,
       "inputTokens": 606846,
       "outputTokens": 22327,
       "cacheTokens": 574797,
       "totalTokens": 629173,
       "durationSec": 346.062,
       "startedAt": "2026-09-25T15:33:29.064409Z",
       "finishedAt": "2026-09-25T15:39:15.126391Z",
       "taskChecksum": "9b8a318b24bb5e38f5611b673c5cfbcdde1bad258c1e06df8fbbb0cf1f8ed9a6",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "winning-avg-corewars__FQsb2Rf",
       "job": "r1-18-winning-avg-corewars",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 39,
       "agentRequests": 39,
       "costUsd": 0.021393300000000004,
       "inputTokens": 751718,
       "outputTokens": 20379,
       "cacheTokens": 719635,
       "totalTokens": 772097,
       "durationSec": 367.135,
       "startedAt": "2026-09-25T16:05:01.770004Z",
       "finishedAt": "2026-09-25T16:11:08.905307Z",
       "taskChecksum": "9b8a318b24bb5e38f5611b673c5cfbcdde1bad258c1e06df8fbbb0cf1f8ed9a6",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "winning-avg-corewars__JbU8Ywk",
       "job": "r2-18-winning-avg-corewars",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 10,
       "qualifiedCompactions": 10,
       "compactions": [
        {
         "at": "2026-09-25T20:14:34.142Z",
         "tokensAt": 51139,
         "residualTokens": 43883,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:15:56.955Z",
         "tokensAt": 50589,
         "residualTokens": 51298,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:16:07.721Z",
         "tokensAt": 51476,
         "residualTokens": 50044,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:16:52.205Z",
         "tokensAt": 50576,
         "residualTokens": 49936,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T20:21:12.483Z",
         "tokensAt": 50423,
         "residualTokens": 48866,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T20:21:43.916Z",
         "tokensAt": 50054,
         "residualTokens": 48187,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.2
        },
        {
         "at": "2026-09-25T20:23:58.850Z",
         "tokensAt": 50623,
         "residualTokens": 45304,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.8
        },
        {
         "at": "2026-09-25T20:24:48.172Z",
         "tokensAt": 50301,
         "residualTokens": 41421,
         "qualified": true,
         "readsBefore": 0.8,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:28:12.743Z",
         "tokensAt": 50665,
         "residualTokens": 43957,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:30:00.266Z",
         "tokensAt": 50607,
         "residualTokens": 41984,
         "qualified": true,
         "readsBefore": 0.2,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 113,
       "agentRequests": 103,
       "costUsd": 0.14866229000000006,
       "inputTokens": 3767896,
       "outputTokens": 85523,
       "cacheTokens": 3174589,
       "totalTokens": 3853419,
       "durationSec": 1516.521,
       "startedAt": "2026-09-25T20:07:13.552154Z",
       "finishedAt": "2026-09-25T20:32:30.073490Z",
       "taskChecksum": "9b8a318b24bb5e38f5611b673c5cfbcdde1bad258c1e06df8fbbb0cf1f8ed9a6",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "winning-avg-corewars__BovvDym",
       "job": "r3-18-winning-avg-corewars",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 0,
       "compactions": [
        {
         "at": "2026-09-25T21:18:18.037Z",
         "tokensAt": 89261,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0,
         "readsAfter": null
        }
       ],
       "providerRequests": 71,
       "agentRequests": 70,
       "costUsd": 0.07499956999999997,
       "inputTokens": 3004937,
       "outputTokens": 66213,
       "cacheTokens": 2901902,
       "totalTokens": 3071150,
       "durationSec": 1008.858,
       "startedAt": "2026-09-25T21:01:51.715482Z",
       "finishedAt": "2026-09-25T21:18:40.573566Z",
       "taskChecksum": "9b8a318b24bb5e38f5611b673c5cfbcdde1bad258c1e06df8fbbb0cf1f8ed9a6",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.265970505,
     "qualifiedCompactions": 10
    },
    "Native50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "winning-avg-corewars__F7hnnqy",
       "job": "r0-18-winning-avg-corewars",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 33,
       "agentRequests": 33,
       "costUsd": 0.018165070000000005,
       "inputTokens": 467307,
       "outputTokens": 18063,
       "cacheTokens": 425252,
       "totalTokens": 485370,
       "durationSec": 363.62,
       "startedAt": "2026-09-25T15:33:29.091995Z",
       "finishedAt": "2026-09-25T15:39:32.711550Z",
       "taskChecksum": "9b8a318b24bb5e38f5611b673c5cfbcdde1bad258c1e06df8fbbb0cf1f8ed9a6",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "winning-avg-corewars__he6pN9F",
       "job": "r1-18-winning-avg-corewars",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 3,
       "qualifiedCompactions": 3,
       "compactions": [
        {
         "at": "2026-09-25T16:13:37.047Z",
         "tokensAt": 52171,
         "residualTokens": 3840,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 1.8
        },
        {
         "at": "2026-09-25T16:21:35.826Z",
         "tokensAt": 52836,
         "residualTokens": 3396,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 2
        },
        {
         "at": "2026-09-25T16:28:01.406Z",
         "tokensAt": 51641,
         "residualTokens": 6872,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 1
        }
       ],
       "providerRequests": 172,
       "agentRequests": 172,
       "costUsd": 0.15068883500000005,
       "inputTokens": 4668249,
       "outputTokens": 119190,
       "cacheTokens": 4228166,
       "totalTokens": 4787439,
       "durationSec": 2160.45,
       "startedAt": "2026-09-25T16:05:01.743883Z",
       "finishedAt": "2026-09-25T16:41:02.193616Z",
       "taskChecksum": "9b8a318b24bb5e38f5611b673c5cfbcdde1bad258c1e06df8fbbb0cf1f8ed9a6",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "winning-avg-corewars__VewHmW7",
       "job": "r2-18-winning-avg-corewars",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 18,
       "agentRequests": 18,
       "costUsd": 0.010339095,
       "inputTokens": 192497,
       "outputTokens": 11264,
       "cacheTokens": 166112,
       "totalTokens": 203761,
       "durationSec": 188.261,
       "startedAt": "2026-09-25T20:07:13.577579Z",
       "finishedAt": "2026-09-25T20:10:21.838520Z",
       "taskChecksum": "9b8a318b24bb5e38f5611b673c5cfbcdde1bad258c1e06df8fbbb0cf1f8ed9a6",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "winning-avg-corewars__rHBkd6c",
       "job": "r3-18-winning-avg-corewars",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T21:09:55.528Z",
         "tokensAt": 53433,
         "residualTokens": 3074,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.6
        }
       ],
       "providerRequests": 58,
       "agentRequests": 58,
       "costUsd": 0.05797468500000002,
       "inputTokens": 1424375,
       "outputTokens": 57213,
       "cacheTokens": 1274996,
       "totalTokens": 1481588,
       "durationSec": 717.009,
       "startedAt": "2026-09-25T21:01:51.677081Z",
       "finishedAt": "2026-09-25T21:13:48.686287Z",
       "taskChecksum": "9b8a318b24bb5e38f5611b673c5cfbcdde1bad258c1e06df8fbbb0cf1f8ed9a6",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.23716768500000007,
     "qualifiedCompactions": 4
    }
   },
   "diff": 0
  },
  {
   "name": "dna-assembly",
   "set": "B",
   "timeoutSec": 1800,
   "category": "scientific-computing",
   "task": "dna-assembly",
   "compactionRate": 0,
   "group": "placebo",
   "outcome": "Both always fail",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 0,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "dna-assembly__tcBoWyK",
       "job": "r0-19-dna-assembly",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 30,
       "agentRequests": 30,
       "costUsd": 0.028057214999999993,
       "inputTokens": 690014,
       "outputTokens": 31841,
       "cacheTokens": 644459,
       "totalTokens": 721855,
       "durationSec": 433.411,
       "startedAt": "2026-09-25T15:37:14.531907Z",
       "finishedAt": "2026-09-25T15:44:27.942852Z",
       "taskChecksum": "3087a6d15922343fcde2166be12274296557c0b9c230af3363dfb70668b463a2",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-assembly__XaMeKR7",
       "job": "r1-19-dna-assembly",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 31,
       "agentRequests": 31,
       "costUsd": 0.029599004999999994,
       "inputTokens": 710681,
       "outputTokens": 33041,
       "cacheTokens": 658733,
       "totalTokens": 743722,
       "durationSec": 383.538,
       "startedAt": "2026-09-25T16:02:25.523062Z",
       "finishedAt": "2026-09-25T16:08:49.061149Z",
       "taskChecksum": "3087a6d15922343fcde2166be12274296557c0b9c230af3363dfb70668b463a2",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-assembly__jXhhmJ2",
       "job": "r2-19-dna-assembly",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 30,
       "agentRequests": 30,
       "costUsd": 0.03640900999999999,
       "inputTokens": 827264,
       "outputTokens": 43561,
       "cacheTokens": 771976,
       "totalTokens": 870825,
       "durationSec": 438.366,
       "startedAt": "2026-09-25T20:07:17.691085Z",
       "finishedAt": "2026-09-25T20:14:36.057504Z",
       "taskChecksum": "3087a6d15922343fcde2166be12274296557c0b9c230af3363dfb70668b463a2",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-assembly__6mJbd7J",
       "job": "r3-19-dna-assembly",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 29,
       "agentRequests": 29,
       "costUsd": 0.02796984500000001,
       "inputTokens": 598149,
       "outputTokens": 33925,
       "cacheTokens": 554427,
       "totalTokens": 632074,
       "durationSec": 377.815,
       "startedAt": "2026-09-25T20:55:30.669763Z",
       "finishedAt": "2026-09-25T21:01:48.484132Z",
       "taskChecksum": "3087a6d15922343fcde2166be12274296557c0b9c230af3363dfb70668b463a2",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0,
     "costUsd": 0.12203507499999998,
     "qualifiedCompactions": 0
    },
    "Native50kPi": {
     "passes": 0,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "dna-assembly__pdJwK7v",
       "job": "r0-19-dna-assembly",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 25,
       "agentRequests": 25,
       "costUsd": 0.027154325000000003,
       "inputTokens": 464261,
       "outputTokens": 32634,
       "cacheTokens": 407810,
       "totalTokens": 496895,
       "durationSec": 438.85,
       "startedAt": "2026-09-25T15:37:14.491272Z",
       "finishedAt": "2026-09-25T15:44:33.341867Z",
       "taskChecksum": "3087a6d15922343fcde2166be12274296557c0b9c230af3363dfb70668b463a2",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-assembly__3TNyjJk",
       "job": "r1-19-dna-assembly",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 0,
       "compactions": [
        {
         "at": "2026-09-25T16:08:44.926Z",
         "tokensAt": 52037,
         "residualTokens": null,
         "qualified": false,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 34,
       "agentRequests": 34,
       "costUsd": 0.034354825,
       "inputTokens": 704380,
       "outputTokens": 29924,
       "cacheTokens": 580900,
       "totalTokens": 734304,
       "durationSec": 397.525,
       "startedAt": "2026-09-25T16:02:25.548711Z",
       "finishedAt": "2026-09-25T16:09:03.073889Z",
       "taskChecksum": "3087a6d15922343fcde2166be12274296557c0b9c230af3363dfb70668b463a2",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-assembly__VjDVK2B",
       "job": "r2-19-dna-assembly",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 26,
       "agentRequests": 26,
       "costUsd": 0.024270039999999996,
       "inputTokens": 506460,
       "outputTokens": 26129,
       "cacheTokens": 449794,
       "totalTokens": 532589,
       "durationSec": 307.141,
       "startedAt": "2026-09-25T20:07:17.665653Z",
       "finishedAt": "2026-09-25T20:12:24.806263Z",
       "taskChecksum": "3087a6d15922343fcde2166be12274296557c0b9c230af3363dfb70668b463a2",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-assembly__ZkjfG6B",
       "job": "r3-19-dna-assembly",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 28,
       "agentRequests": 28,
       "costUsd": 0.02783953,
       "inputTokens": 587449,
       "outputTokens": 30055,
       "cacheTokens": 523678,
       "totalTokens": 617504,
       "durationSec": 342.125,
       "startedAt": "2026-09-25T20:55:30.694267Z",
       "finishedAt": "2026-09-25T21:01:12.819100Z",
       "taskChecksum": "3087a6d15922343fcde2166be12274296557c0b9c230af3363dfb70668b463a2",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0,
     "costUsd": 0.11361871999999999,
     "qualifiedCompactions": 0
    }
   },
   "diff": 0
  },
  {
   "name": "dna-insert",
   "set": "B",
   "timeoutSec": 1800,
   "category": "scientific-computing",
   "task": "dna-insert",
   "compactionRate": 0.125,
   "group": "other",
   "outcome": "Both always fail",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 0,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "dna-insert__kh9U3kS",
       "job": "r0-20-dna-insert",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 16,
       "agentRequests": 16,
       "costUsd": 0.006616385000000001,
       "inputTokens": 144447,
       "outputTokens": 6896,
       "cacheTokens": 129446,
       "totalTokens": 151343,
       "durationSec": 133.202,
       "startedAt": "2026-09-25T15:39:35.468271Z",
       "finishedAt": "2026-09-25T15:41:48.670511Z",
       "taskChecksum": "f3ef2b366665ecdb95d351f9164d282cf0722495cea347090f79197aa51309ec",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-insert__XJeS5D6",
       "job": "r1-20-dna-insert",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 22,
       "agentRequests": 22,
       "costUsd": 0.011444935,
       "inputTokens": 249399,
       "outputTokens": 12787,
       "cacheTokens": 227146,
       "totalTokens": 262186,
       "durationSec": 193.445,
       "startedAt": "2026-09-25T16:01:45.692414Z",
       "finishedAt": "2026-09-25T16:04:59.137919Z",
       "taskChecksum": "f3ef2b366665ecdb95d351f9164d282cf0722495cea347090f79197aa51309ec",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-insert__FPmarJb",
       "job": "r2-20-dna-insert",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 22,
       "agentRequests": 22,
       "costUsd": 0.0071364950000000005,
       "inputTokens": 183177,
       "outputTokens": 7180,
       "cacheTokens": 168252,
       "totalTokens": 190357,
       "durationSec": 126.916,
       "startedAt": "2026-09-25T20:14:38.970550Z",
       "finishedAt": "2026-09-25T20:16:45.886380Z",
       "taskChecksum": "f3ef2b366665ecdb95d351f9164d282cf0722495cea347090f79197aa51309ec",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-insert__bXQKvdR",
       "job": "r3-20-dna-insert",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": "AgentTimeoutError",
       "timedOut": true,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 12,
       "agentRequests": 12,
       "costUsd": 0.00409938,
       "inputTokens": 63751,
       "outputTokens": 5040,
       "cacheTokens": 55553,
       "totalTokens": 68791,
       "durationSec": 1840.957,
       "startedAt": "2026-09-25T20:40:32.629778Z",
       "finishedAt": "2026-09-25T21:11:13.586105Z",
       "taskChecksum": "f3ef2b366665ecdb95d351f9164d282cf0722495cea347090f79197aa51309ec",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0,
     "costUsd": 0.029297194999999998,
     "qualifiedCompactions": 0
    },
    "Native50kPi": {
     "passes": 0,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "dna-insert__7LTSkeL",
       "job": "r0-20-dna-insert",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 28,
       "agentRequests": 28,
       "costUsd": 0.024970150000000003,
       "inputTokens": 843578,
       "outputTokens": 19835,
       "cacheTokens": 782750,
       "totalTokens": 863413,
       "durationSec": 296.024,
       "startedAt": "2026-09-25T15:39:35.493501Z",
       "finishedAt": "2026-09-25T15:44:31.517727Z",
       "taskChecksum": "f3ef2b366665ecdb95d351f9164d282cf0722495cea347090f79197aa51309ec",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-insert__5Wikruy",
       "job": "r1-20-dna-insert",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 17,
       "agentRequests": 17,
       "costUsd": 0.008265924999999999,
       "inputTokens": 156039,
       "outputTokens": 7205,
       "cacheTokens": 126020,
       "totalTokens": 163244,
       "durationSec": 133.523,
       "startedAt": "2026-09-25T16:01:45.666973Z",
       "finishedAt": "2026-09-25T16:03:59.189204Z",
       "taskChecksum": "f3ef2b366665ecdb95d351f9164d282cf0722495cea347090f79197aa51309ec",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-insert__pEZqd37",
       "job": "r2-20-dna-insert",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T20:17:22.553Z",
         "tokensAt": 50089,
         "residualTokens": 5415,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 31,
       "agentRequests": 31,
       "costUsd": 0.01974486,
       "inputTokens": 441270,
       "outputTokens": 14903,
       "cacheTokens": 368776,
       "totalTokens": 456173,
       "durationSec": 211.889,
       "startedAt": "2026-09-25T20:14:38.996791Z",
       "finishedAt": "2026-09-25T20:18:10.885724Z",
       "taskChecksum": "f3ef2b366665ecdb95d351f9164d282cf0722495cea347090f79197aa51309ec",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "dna-insert__wcZ44kc",
       "job": "r3-20-dna-insert",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 18,
       "agentRequests": 18,
       "costUsd": 0.012420120000000002,
       "inputTokens": 181770,
       "outputTokens": 11538,
       "cacheTokens": 134212,
       "totalTokens": 193308,
       "durationSec": 1290.884,
       "startedAt": "2026-09-25T20:40:32.593226Z",
       "finishedAt": "2026-09-25T21:02:03.477545Z",
       "taskChecksum": "f3ef2b366665ecdb95d351f9164d282cf0722495cea347090f79197aa51309ec",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0,
     "costUsd": 0.06540105500000001,
     "qualifiedCompactions": 1
    }
   },
   "diff": 0
  },
  {
   "name": "llm-inference-batching-scheduler",
   "set": "B",
   "timeoutSec": 1800,
   "category": "machine-learning",
   "task": "llm-inference-batching-scheduler",
   "compactionRate": 0.125,
   "group": "other",
   "outcome": "Both always pass",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "llm-inference-batching-scheduler__mty5Nmx",
       "job": "r0-21-llm-inference-batching-scheduler",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 2,
       "qualifiedCompactions": 2,
       "compactions": [
        {
         "at": "2026-09-25T15:49:01.439Z",
         "tokensAt": 51098,
         "residualTokens": 45376,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.6666666666666666
        },
        {
         "at": "2026-09-25T15:50:01.483Z",
         "tokensAt": 51747,
         "residualTokens": 40666,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0
        }
       ],
       "providerRequests": 18,
       "agentRequests": 16,
       "costUsd": 0.033636355,
       "inputTokens": 438436,
       "outputTokens": 23408,
       "cacheTokens": 285833,
       "totalTokens": 461844,
       "durationSec": 354.022,
       "startedAt": "2026-09-25T15:44:34.208937Z",
       "finishedAt": "2026-09-25T15:50:28.230004Z",
       "taskChecksum": "2635be7269a496a4edefb77c595bc3acf8782ee84b3dab2056dfb25ac9e317a1",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "llm-inference-batching-scheduler__gXqsNqM",
       "job": "r1-21-llm-inference-batching-scheduler",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 15,
       "agentRequests": 15,
       "costUsd": 0.013621355000000002,
       "inputTokens": 227888,
       "outputTokens": 16353,
       "cacheTokens": 200348,
       "totalTokens": 244241,
       "durationSec": 203.592,
       "startedAt": "2026-09-25T15:58:31.683425Z",
       "finishedAt": "2026-09-25T16:01:55.275381Z",
       "taskChecksum": "2635be7269a496a4edefb77c595bc3acf8782ee84b3dab2056dfb25ac9e317a1",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "llm-inference-batching-scheduler__We6nUVw",
       "job": "r2-21-llm-inference-batching-scheduler",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 8,
       "agentRequests": 8,
       "costUsd": 0.008481864999999998,
       "inputTokens": 94980,
       "outputTokens": 10035,
       "cacheTokens": 73109,
       "totalTokens": 105015,
       "durationSec": 120.903,
       "startedAt": "2026-09-25T20:18:13.920533Z",
       "finishedAt": "2026-09-25T20:20:14.823609Z",
       "taskChecksum": "2635be7269a496a4edefb77c595bc3acf8782ee84b3dab2056dfb25ac9e317a1",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "llm-inference-batching-scheduler__bsUQato",
       "job": "r3-21-llm-inference-batching-scheduler",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 12,
       "agentRequests": 12,
       "costUsd": 0.009975005,
       "inputTokens": 152549,
       "outputTokens": 11491,
       "cacheTokens": 129028,
       "totalTokens": 164040,
       "durationSec": 173.546,
       "startedAt": "2026-09-25T20:37:07.899465Z",
       "finishedAt": "2026-09-25T20:40:01.445804Z",
       "taskChecksum": "2635be7269a496a4edefb77c595bc3acf8782ee84b3dab2056dfb25ac9e317a1",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.06571458,
     "qualifiedCompactions": 2
    },
    "Native50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "llm-inference-batching-scheduler__yvLJskx",
       "job": "r0-21-llm-inference-batching-scheduler",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 12,
       "agentRequests": 12,
       "costUsd": 0.013056459999999999,
       "inputTokens": 272634,
       "outputTokens": 11394,
       "cacheTokens": 230746,
       "totalTokens": 284028,
       "durationSec": 168.257,
       "startedAt": "2026-09-25T15:44:34.178594Z",
       "finishedAt": "2026-09-25T15:47:22.435147Z",
       "taskChecksum": "2635be7269a496a4edefb77c595bc3acf8782ee84b3dab2056dfb25ac9e317a1",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "llm-inference-batching-scheduler__yp2H7VM",
       "job": "r1-21-llm-inference-batching-scheduler",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 14,
       "agentRequests": 14,
       "costUsd": 0.017274125,
       "inputTokens": 335159,
       "outputTokens": 17547,
       "cacheTokens": 289270,
       "totalTokens": 352706,
       "durationSec": 231.069,
       "startedAt": "2026-09-25T15:58:31.721225Z",
       "finishedAt": "2026-09-25T16:02:22.790165Z",
       "taskChecksum": "2635be7269a496a4edefb77c595bc3acf8782ee84b3dab2056dfb25ac9e317a1",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "llm-inference-batching-scheduler__cfRuN5i",
       "job": "r2-21-llm-inference-batching-scheduler",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 19,
       "agentRequests": 19,
       "costUsd": 0.02171309,
       "inputTokens": 474921,
       "outputTokens": 22057,
       "cacheTokens": 421244,
       "totalTokens": 496978,
       "durationSec": 374.199,
       "startedAt": "2026-09-25T20:18:13.882154Z",
       "finishedAt": "2026-09-25T20:24:28.081617Z",
       "taskChecksum": "2635be7269a496a4edefb77c595bc3acf8782ee84b3dab2056dfb25ac9e317a1",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "llm-inference-batching-scheduler__ihJC5uM",
       "job": "r3-21-llm-inference-batching-scheduler",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 17,
       "agentRequests": 17,
       "costUsd": 0.016191695,
       "inputTokens": 349420,
       "outputTokens": 16631,
       "cacheTokens": 310062,
       "totalTokens": 366051,
       "durationSec": 201.642,
       "startedAt": "2026-09-25T20:37:07.936857Z",
       "finishedAt": "2026-09-25T20:40:29.578681Z",
       "taskChecksum": "2635be7269a496a4edefb77c595bc3acf8782ee84b3dab2056dfb25ac9e317a1",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.06823536999999999,
     "qualifiedCompactions": 0
    }
   },
   "diff": 0
  },
  {
   "name": "mteb-retrieve",
   "set": "B",
   "timeoutSec": 1800,
   "category": "data-science",
   "task": "mteb-retrieve",
   "compactionRate": 0,
   "group": "placebo",
   "outcome": "Both always pass",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "mteb-retrieve__bBzfCVz",
       "job": "r0-22-mteb-retrieve",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 14,
       "agentRequests": 14,
       "costUsd": 0.0025863550000000003,
       "inputTokens": 58772,
       "outputTokens": 2362,
       "cacheTokens": 51653,
       "totalTokens": 61134,
       "durationSec": 239.133,
       "startedAt": "2026-09-25T15:44:36.024940Z",
       "finishedAt": "2026-09-25T15:48:35.157945Z",
       "taskChecksum": "7dad786323e0e2126c79c62d9d72f4929e8cc79270643931e20884c7bdbed231",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-retrieve__T683v6r",
       "job": "r1-22-mteb-retrieve",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 16,
       "agentRequests": 16,
       "costUsd": 0.00315639,
       "inputTokens": 77537,
       "outputTokens": 2918,
       "cacheTokens": 69509,
       "totalTokens": 80455,
       "durationSec": 215.357,
       "startedAt": "2026-09-25T15:54:46.781511Z",
       "finishedAt": "2026-09-25T15:58:22.138018Z",
       "taskChecksum": "7dad786323e0e2126c79c62d9d72f4929e8cc79270643931e20884c7bdbed231",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-retrieve__Lx2S5eV",
       "job": "r2-22-mteb-retrieve",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 25,
       "agentRequests": 25,
       "costUsd": 0.005462904999999999,
       "inputTokens": 177696,
       "outputTokens": 4531,
       "cacheTokens": 165328,
       "totalTokens": 182227,
       "durationSec": 265.539,
       "startedAt": "2026-09-25T20:24:30.987717Z",
       "finishedAt": "2026-09-25T20:28:56.526864Z",
       "taskChecksum": "7dad786323e0e2126c79c62d9d72f4929e8cc79270643931e20884c7bdbed231",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-retrieve__gpMbnep",
       "job": "r3-22-mteb-retrieve",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 13,
       "agentRequests": 13,
       "costUsd": 0.0030729650000000004,
       "inputTokens": 62846,
       "outputTokens": 2901,
       "cacheTokens": 54194,
       "totalTokens": 65747,
       "durationSec": 180.712,
       "startedAt": "2026-09-25T20:33:47.927989Z",
       "finishedAt": "2026-09-25T20:36:48.639488Z",
       "taskChecksum": "7dad786323e0e2126c79c62d9d72f4929e8cc79270643931e20884c7bdbed231",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.014278615,
     "qualifiedCompactions": 0
    },
    "Native50kPi": {
     "passes": 4,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "mteb-retrieve__BhQJhKZ",
       "job": "r0-22-mteb-retrieve",
       "round": 0,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 12,
       "agentRequests": 12,
       "costUsd": 0.00362779,
       "inputTokens": 50881,
       "outputTokens": 2354,
       "cacheTokens": 31344,
       "totalTokens": 53235,
       "durationSec": 210.035,
       "startedAt": "2026-09-25T15:44:36.050243Z",
       "finishedAt": "2026-09-25T15:48:06.085313Z",
       "taskChecksum": "7dad786323e0e2126c79c62d9d72f4929e8cc79270643931e20884c7bdbed231",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-retrieve__UgGDU5r",
       "job": "r1-22-mteb-retrieve",
       "round": 1,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 14,
       "agentRequests": 14,
       "costUsd": 0.004141745,
       "inputTokens": 69836,
       "outputTokens": 3031,
       "cacheTokens": 50522,
       "totalTokens": 72867,
       "durationSec": 222.121,
       "startedAt": "2026-09-25T15:54:46.756137Z",
       "finishedAt": "2026-09-25T15:58:28.877329Z",
       "taskChecksum": "7dad786323e0e2126c79c62d9d72f4929e8cc79270643931e20884c7bdbed231",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-retrieve__rDTEAVG",
       "job": "r2-22-mteb-retrieve",
       "round": 2,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 13,
       "agentRequests": 13,
       "costUsd": 0.0035036200000000007,
       "inputTokens": 57806,
       "outputTokens": 2181,
       "cacheTokens": 39102,
       "totalTokens": 59987,
       "durationSec": 175.502,
       "startedAt": "2026-09-25T20:24:31.028753Z",
       "finishedAt": "2026-09-25T20:27:26.530560Z",
       "taskChecksum": "7dad786323e0e2126c79c62d9d72f4929e8cc79270643931e20884c7bdbed231",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "mteb-retrieve__RXdMoPP",
       "job": "r3-22-mteb-retrieve",
       "round": 3,
       "state": "PASS",
       "reward": 1,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 13,
       "agentRequests": 13,
       "costUsd": 0.004201885,
       "inputTokens": 61129,
       "outputTokens": 3011,
       "cacheTokens": 40126,
       "totalTokens": 64140,
       "durationSec": 197.024,
       "startedAt": "2026-09-25T20:33:47.889253Z",
       "finishedAt": "2026-09-25T20:37:04.913487Z",
       "taskChecksum": "7dad786323e0e2126c79c62d9d72f4929e8cc79270643931e20884c7bdbed231",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 1,
     "costUsd": 0.015475039999999999,
     "qualifiedCompactions": 0
    }
   },
   "diff": 0
  },
  {
   "name": "protein-assembly",
   "set": "B",
   "timeoutSec": 1800,
   "category": "scientific-computing",
   "task": "protein-assembly",
   "compactionRate": 0.5,
   "group": "primary",
   "outcome": "Both always fail",
   "arms": {
    "TextualFixed50kPi": {
     "passes": 0,
     "scored": 2,
     "errors": 2,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "protein-assembly__WG3sGe6",
       "job": "r0-23-protein-assembly",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T15:53:22.139Z",
         "tokensAt": 50266,
         "residualTokens": 31496,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.6
        }
       ],
       "providerRequests": 22,
       "agentRequests": 21,
       "costUsd": 0.028279865000000008,
       "inputTokens": 518835,
       "outputTokens": 23745,
       "cacheTokens": 421264,
       "totalTokens": 542580,
       "durationSec": 365.947,
       "startedAt": "2026-09-25T15:48:38.031572Z",
       "finishedAt": "2026-09-25T15:54:43.978697Z",
       "taskChecksum": "b956dd5a941e6566a1e6d3531ca16b734152f0893555d8d42029b135e3c641c0",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "protein-assembly__mSxqHGV",
       "job": "r1-23-protein-assembly",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T16:01:00.952Z",
         "tokensAt": 51477,
         "residualTokens": 35697,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        }
       ],
       "providerRequests": 34,
       "agentRequests": 33,
       "costUsd": 0.03271864000000001,
       "inputTokens": 794892,
       "outputTokens": 27810,
       "cacheTokens": 700394,
       "totalTokens": 822702,
       "durationSec": 671.834,
       "startedAt": "2026-09-25T15:50:31.056976Z",
       "finishedAt": "2026-09-25T16:01:42.890598Z",
       "taskChecksum": "b956dd5a941e6566a1e6d3531ca16b734152f0893555d8d42029b135e3c641c0",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "protein-assembly__DtESU2i",
       "job": "r2-23-protein-assembly",
       "round": 2,
       "state": "ERROR",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": "Error: Stream ended without terminal usage",
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 25,
       "agentRequests": 25,
       "costUsd": 0.42020304,
       "inputTokens": 459447,
       "outputTokens": 21736,
       "cacheTokens": 418209,
       "totalTokens": 481183,
       "durationSec": 285.632,
       "startedAt": "2026-09-25T20:28:59.341070Z",
       "finishedAt": "2026-09-25T20:33:44.973368Z",
       "taskChecksum": "b956dd5a941e6566a1e6d3531ca16b734152f0893555d8d42029b135e3c641c0",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": true
      },
      {
       "trial": "protein-assembly__rErR9L9",
       "job": "r3-23-protein-assembly",
       "round": 3,
       "state": "ERROR",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": "Error: Stream ended without terminal usage",
       "compactionsObserved": 8,
       "qualifiedCompactions": 8,
       "compactions": [
        {
         "at": "2026-09-25T20:39:11.375Z",
         "tokensAt": 50992,
         "residualTokens": 32302,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0.6
        },
        {
         "at": "2026-09-25T20:41:23.688Z",
         "tokensAt": 50468,
         "residualTokens": 36828,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:45:22.931Z",
         "tokensAt": 58675,
         "residualTokens": 38867,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:46:16.843Z",
         "tokensAt": 50214,
         "residualTokens": 39133,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:47:47.435Z",
         "tokensAt": 53446,
         "residualTokens": 36759,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:50:17.761Z",
         "tokensAt": 52542,
         "residualTokens": 39418,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:51:45.702Z",
         "tokensAt": 50193,
         "residualTokens": 32647,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 0
        },
        {
         "at": "2026-09-25T20:53:43.282Z",
         "tokensAt": 61024,
         "residualTokens": 27966,
         "qualified": true,
         "readsBefore": 0.4,
         "readsAfter": 0.4
        }
       ],
       "providerRequests": 111,
       "agentRequests": 100,
       "costUsd": 0.5395631350000001,
       "inputTokens": 3709378,
       "outputTokens": 90409,
       "cacheTokens": 3211351,
       "totalTokens": 3799787,
       "durationSec": 1374.584,
       "startedAt": "2026-09-25T20:32:32.936975Z",
       "finishedAt": "2026-09-25T20:55:27.520145Z",
       "taskChecksum": "b956dd5a941e6566a1e6d3531ca16b734152f0893555d8d42029b135e3c641c0",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": true
      }
     ],
     "passRate": 0,
     "costUsd": 1.02076468,
     "qualifiedCompactions": 10
    },
    "Native50kPi": {
     "passes": 0,
     "scored": 4,
     "errors": 0,
     "running": 0,
     "queued": 0,
     "trials": [
      {
       "trial": "protein-assembly__HjCcdzU",
       "job": "r0-23-protein-assembly",
       "round": 0,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 16,
       "agentRequests": 16,
       "costUsd": 0.014896375,
       "inputTokens": 279008,
       "outputTokens": 14231,
       "cacheTokens": 234130,
       "totalTokens": 293239,
       "durationSec": 231.167,
       "startedAt": "2026-09-25T15:48:37.990733Z",
       "finishedAt": "2026-09-25T15:52:29.157324Z",
       "taskChecksum": "b956dd5a941e6566a1e6d3531ca16b734152f0893555d8d42029b135e3c641c0",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "protein-assembly__wpRzTaU",
       "job": "r1-23-protein-assembly",
       "round": 1,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 32,
       "agentRequests": 32,
       "costUsd": 0.02589721,
       "inputTokens": 657772,
       "outputTokens": 24593,
       "cacheTokens": 591296,
       "totalTokens": 682365,
       "durationSec": 388.234,
       "startedAt": "2026-09-25T15:50:31.082254Z",
       "finishedAt": "2026-09-25T15:56:59.316729Z",
       "taskChecksum": "b956dd5a941e6566a1e6d3531ca16b734152f0893555d8d42029b135e3c641c0",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "protein-assembly__W4GFXnG",
       "job": "r2-23-protein-assembly",
       "round": 2,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 0,
       "qualifiedCompactions": 0,
       "compactions": [],
       "providerRequests": 19,
       "agentRequests": 19,
       "costUsd": 0.01683407,
       "inputTokens": 278033,
       "outputTokens": 17410,
       "cacheTokens": 228142,
       "totalTokens": 295443,
       "durationSec": 238.903,
       "startedAt": "2026-09-25T20:28:59.314501Z",
       "finishedAt": "2026-09-25T20:32:58.217183Z",
       "taskChecksum": "b956dd5a941e6566a1e6d3531ca16b734152f0893555d8d42029b135e3c641c0",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      },
      {
       "trial": "protein-assembly__DdDGfFv",
       "job": "r3-23-protein-assembly",
       "round": 3,
       "state": "FAIL",
       "reward": 0,
       "exception": null,
       "timedOut": false,
       "error": null,
       "compactionsObserved": 1,
       "qualifiedCompactions": 1,
       "compactions": [
        {
         "at": "2026-09-25T20:36:22.964Z",
         "tokensAt": 47707,
         "residualTokens": 6265,
         "qualified": true,
         "readsBefore": 0,
         "readsAfter": 1
        }
       ],
       "providerRequests": 28,
       "agentRequests": 28,
       "costUsd": 0.02528403,
       "inputTokens": 656615,
       "outputTokens": 22248,
       "cacheTokens": 586888,
       "totalTokens": 678863,
       "durationSec": 316.306,
       "startedAt": "2026-09-25T20:32:32.963136Z",
       "finishedAt": "2026-09-25T20:37:49.269374Z",
       "taskChecksum": "b956dd5a941e6566a1e6d3531ca16b734152f0893555d8d42029b135e3c641c0",
       "actualModels": [
        "gpt-6-luna"
       ],
       "uncertainCost": false
      }
     ],
     "passRate": 0,
     "costUsd": 0.082911685,
     "qualifiedCompactions": 1
    }
   },
   "diff": 0
  }
 ],
 "warnings": [],
 "audit": {
  "ok": true,
  "warnings": [
   "make-mips-interpreter/TextualFixed50kPi/r0: agent timed out, verifier scored FAIL",
   "fix-ocaml-gc/TextualFixed50kPi/r0: agent timed out, verifier scored FAIL",
   "fix-ocaml-gc/Native50kPi/r1: agent timed out, verifier scored FAIL",
   "make-doom-for-mips/TextualFixed50kPi/r0: agent timed out, verifier scored FAIL",
   "make-doom-for-mips/TextualFixed50kPi/r1: agent timed out, verifier scored FAIL",
   "make-doom-for-mips/TextualFixed50kPi/r2: agent timed out, verifier scored FAIL",
   "make-doom-for-mips/TextualFixed50kPi/r3: agent timed out, verifier scored FAIL",
   "make-doom-for-mips/Native50kPi/r1: agent timed out, verifier scored FAIL",
   "make-doom-for-mips/Native50kPi/r2: agent timed out, verifier scored FAIL",
   "make-doom-for-mips/Native50kPi/r3: agent timed out, verifier scored FAIL",
   "caffe-cifar-10/TextualFixed50kPi/r3: Error: 10 minute request deadline",
   "caffe-cifar-10__zbduktH: uncertain cost reserved at upper bound",
   "caffe-cifar-10/Native50kPi/r1: agent timed out, verifier scored FAIL",
   "path-tracing-reverse/Native50kPi/r2: agent timed out, verifier scored PASS",
   "regex-chess/TextualFixed50kPi/r2: Error: Stream ended without terminal usage",
   "regex-chess__rHFuUz5: uncertain cost reserved at upper bound",
   "gpt2-codegolf/Native50kPi/r2: AgentTimeoutError",
   "gpt2-codegolf/Native50kPi/r2: agent timed out, verifier scored ERROR",
   "path-tracing/TextualFixed50kPi/r2: agent timed out, verifier scored FAIL",
   "path-tracing/Native50kPi/r0: agent timed out, verifier scored PASS",
   "path-tracing/Native50kPi/r1: agent timed out, verifier scored PASS",
   "path-tracing/Native50kPi/r2: Error: Stream ended without terminal usage",
   "path-tracing__QatasRC: uncertain cost reserved at upper bound",
   "path-tracing/Native50kPi/r3: agent timed out, verifier scored PASS",
   "build-pov-ray/TextualFixed50kPi/r2: Error: Stream ended without terminal usage",
   "build-pov-ray__EyEagA7: uncertain cost reserved at upper bound",
   "build-pov-ray/Native50kPi/r2: Error: Stream ended without terminal usage",
   "build-pov-ray__vbqcjNo: uncertain cost reserved at upper bound",
   "mcmc-sampling-stan/TextualFixed50kPi/r0: agent timed out, verifier scored FAIL",
   "train-fasttext/TextualFixed50kPi/r0: agent timed out, verifier scored FAIL",
   "train-fasttext/TextualFixed50kPi/r1: agent timed out, verifier scored FAIL",
   "train-fasttext/TextualFixed50kPi/r2: agent timed out, verifier scored FAIL",
   "train-fasttext/TextualFixed50kPi/r3: agent timed out, verifier scored FAIL",
   "train-fasttext/Native50kPi/r0: agent timed out, verifier scored FAIL",
   "train-fasttext/Native50kPi/r1: agent timed out, verifier scored FAIL",
   "train-fasttext/Native50kPi/r2: agent timed out, verifier scored FAIL",
   "train-fasttext/Native50kPi/r3: agent timed out, verifier scored FAIL",
   "dna-insert/TextualFixed50kPi/r3: agent timed out, verifier scored FAIL",
   "protein-assembly/TextualFixed50kPi/r2: Error: Stream ended without terminal usage",
   "protein-assembly__DtESU2i: uncertain cost reserved at upper bound",
   "protein-assembly/TextualFixed50kPi/r3: Error: Stream ended without terminal usage",
   "protein-assembly__rErR9L9: uncertain cost reserved at upper bound"
  ],
  "errors": []
 }
}