{ "runId": "luna6-compaction-20-v1", "model": "gpt-6-luna", "dataset": "Terminal-Bench 2.1", "updatedAt": "2026-09-25T10:48:03.127Z", "runState": "COMPLETE", "phase": "", "completedArms": 40, "plannedArms": 40, "totalCostUsd": 1.745391675, "preflightCostUsd": 0.08348906999999998, "reservedCostUsd": 1.1102230246251565e-16, "arms": { "TextualFixed50kPi": { "passes": 14, "fails": 6, "errors": 0, "running": 0, "complete": 20, "costUsd": 0.8971547800000003, "strictlyQualifiedCompactions": 41, "strictlyQualifiedTrials": 10 }, "Native50kPi": { "passes": 16, "fails": 4, "errors": 0, "running": 0, "complete": 20, "costUsd": 0.764747825, "strictlyQualifiedCompactions": 24, "strictlyQualifiedTrials": 11 } }, "comparisonAverages": { "pairedTaskCount": 20, "arms": { "TextualFixed50kPi": { "passes": 14, "averageCostUsd": 0.044857739000000015, "averageProviderRequests": 41.95, "averageTotalTokens": 1445256.35 }, "Native50kPi": { "passes": 16, "averageCostUsd": 0.03823739125, "averageProviderRequests": 40.1, "averageTotalTokens": 1030084.35 } } }, "tasks": [ { "task": "sanitize-git-repo", "outcome": "Both fail", "arms": { "TextualFixed50kPi": { "state": "FAIL", "trial": "sanitize-git-repo__wYHbyDG", "job": "pair-00", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 18, "agentRequests": 18, "costUsd": 0.020138595, "inputTokens": 508212, "outputTokens": 19569, "cacheTokens": 462357, "totalTokens": 527781, "reward": 0, "exception": null, "error": null, "startedAt": "2026-09-24T21:39:39.701088Z", "finishedAt": "2026-09-24T21:43:54.643864Z", "taskChecksum": "07778bfe6203b37854b0828de4f35f75f177fab35f5da441df39751835e1b2e6", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "FAIL", "trial": "sanitize-git-repo__tqwcR3z", "job": "pair-00", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 17, "agentRequests": 17, "costUsd": 0.020307455000000002, "inputTokens": 513180, "outputTokens": 17748, "cacheTokens": 456338, "totalTokens": 530928, "reward": 0, "exception": null, "error": null, "startedAt": "2026-09-24T21:39:39.727098Z", "finishedAt": "2026-09-24T21:43:21.329680Z", "taskChecksum": "07778bfe6203b37854b0828de4f35f75f177fab35f5da441df39751835e1b2e6", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "make-mips-interpreter", "outcome": "Encrypted only", "arms": { "TextualFixed50kPi": { "state": "FAIL", "trial": "make-mips-interpreter__Ngdquep", "job": "pair-01", "strictlyQualifiedCompactions": 4, "compactionsObserved": 4, "providerRequests": 49, "agentRequests": 48, "costUsd": 0.06527841999999998, "inputTokens": 1684013, "outputTokens": 37845, "cacheTokens": 1427322, "totalTokens": 1721858, "reward": 0, "exception": null, "error": null, "startedAt": "2026-09-24T21:39:39.728406Z", "finishedAt": "2026-09-24T21:48:06.587196Z", "taskChecksum": "17bee6fa30f7b36123f86b6b0594c89eb80060b423af1134adb4bf143fe29040", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "make-mips-interpreter__A37MvFx", "job": "pair-01", "strictlyQualifiedCompactions": 1, "compactionsObserved": 1, "providerRequests": 35, "agentRequests": 35, "costUsd": 0.036493120000000004, "inputTokens": 787889, "outputTokens": 26342, "cacheTokens": 637202, "totalTokens": 814231, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T21:39:39.702199Z", "finishedAt": "2026-09-24T21:47:23.405890Z", "taskChecksum": "17bee6fa30f7b36123f86b6b0594c89eb80060b423af1134adb4bf143fe29040", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "fix-ocaml-gc", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "fix-ocaml-gc__f9Sve2C", "job": "pair-02", "strictlyQualifiedCompactions": 2, "compactionsObserved": 2, "providerRequests": 39, "agentRequests": 38, "costUsd": 0.032151135, "inputTokens": 1017521, "outputTokens": 10357, "cacheTokens": 871431, "totalTokens": 1027878, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T21:43:57.060476Z", "finishedAt": "2026-09-24T22:10:57.783224Z", "taskChecksum": "b6d57afd1632658546bfd5eab8b6c90568850a4c0c4a153dfaf3e1c0aed0ae9b", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "fix-ocaml-gc__k4CkqUZ", "job": "pair-02", "strictlyQualifiedCompactions": 2, "compactionsObserved": 2, "providerRequests": 71, "agentRequests": 71, "costUsd": 0.05061784500000002, "inputTokens": 1727561, "outputTokens": 20963, "cacheTokens": 1503972, "totalTokens": 1748524, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T21:43:57.085054Z", "finishedAt": "2026-09-24T22:16:16.821134Z", "taskChecksum": "b6d57afd1632658546bfd5eab8b6c90568850a4c0c4a153dfaf3e1c0aed0ae9b", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "make-doom-for-mips", "outcome": "Both fail", "arms": { "TextualFixed50kPi": { "state": "FAIL", "trial": "make-doom-for-mips__nyVvVW7", "job": "pair-03", "strictlyQualifiedCompactions": 8, "compactionsObserved": 8, "providerRequests": 150, "agentRequests": 145, "costUsd": 0.150204935, "inputTokens": 5641363, "outputTokens": 71327, "cacheTokens": 5135806, "totalTokens": 5712690, "reward": 0, "exception": "AgentTimeoutError", "error": null, "startedAt": "2026-09-24T21:48:09.178922Z", "finishedAt": "2026-09-24T22:04:23.548647Z", "taskChecksum": "d21f1e251c43a2271fb9604613da1a6f5768be18b2c5a7f05bac5b620ac271f5", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "FAIL", "trial": "make-doom-for-mips__AcBRZXU", "job": "pair-03", "strictlyQualifiedCompactions": 3, "compactionsObserved": 3, "providerRequests": 100, "agentRequests": 100, "costUsd": 0.09147110999999998, "inputTokens": 2976868, "outputTokens": 36600, "cacheTokens": 2541576, "totalTokens": 3013468, "reward": 0, "exception": null, "error": null, "startedAt": "2026-09-24T21:48:09.152705Z", "finishedAt": "2026-09-24T22:01:29.397024Z", "taskChecksum": "d21f1e251c43a2271fb9604613da1a6f5768be18b2c5a7f05bac5b620ac271f5", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "caffe-cifar-10", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "caffe-cifar-10__rxFuina", "job": "pair-04", "strictlyQualifiedCompactions": 2, "compactionsObserved": 2, "providerRequests": 43, "agentRequests": 43, "costUsd": 0.037791284999999994, "inputTokens": 1263356, "outputTokens": 19909, "cacheTokens": 1131126, "totalTokens": 1283265, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T22:04:26.074191Z", "finishedAt": "2026-09-24T22:32:58.907108Z", "taskChecksum": "5462402281e0bcc11b81ade62488f1013f970c7c0679c27df3f87185f4de40c3", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "caffe-cifar-10__sERABou", "job": "pair-04", "strictlyQualifiedCompactions": 2, "compactionsObserved": 2, "providerRequests": 45, "agentRequests": 45, "costUsd": 0.04495321, "inputTokens": 1259197, "outputTokens": 8777, "cacheTokens": 984546, "totalTokens": 1267974, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T22:04:26.099106Z", "finishedAt": "2026-09-24T22:51:18.851850Z", "taskChecksum": "5462402281e0bcc11b81ade62488f1013f970c7c0679c27df3f87185f4de40c3", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "path-tracing-reverse", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "path-tracing-reverse__zEQZ93E", "job": "pair-05", "strictlyQualifiedCompactions": 3, "compactionsObserved": 4, "providerRequests": 101, "agentRequests": 100, "costUsd": 0.12719974000000003, "inputTokens": 6385600, "outputTokens": 59209, "cacheTokens": 6092064, "totalTokens": 6444809, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T22:16:19.264308Z", "finishedAt": "2026-09-24T22:30:27.906878Z", "taskChecksum": "2c4c5c4e76b6706f7a2e7f45efb8262fb3774ca92e27fbbcbb940e68ae5cd641", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "path-tracing-reverse__CjjWVAm", "job": "pair-05", "strictlyQualifiedCompactions": 6, "compactionsObserved": 6, "providerRequests": 79, "agentRequests": 79, "costUsd": 0.10663828000000004, "inputTokens": 2193383, "outputTokens": 51799, "cacheTokens": 1620308, "totalTokens": 2245182, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T22:16:19.238461Z", "finishedAt": "2026-09-24T22:30:09.405569Z", "taskChecksum": "2c4c5c4e76b6706f7a2e7f45efb8262fb3774ca92e27fbbcbb940e68ae5cd641", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "compile-compcert", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "compile-compcert__DjhqDEi", "job": "pair-06", "strictlyQualifiedCompactions": 1, "compactionsObserved": 1, "providerRequests": 34, "agentRequests": 34, "costUsd": 0.023545220000000006, "inputTokens": 862595, "outputTokens": 7909, "cacheTokens": 767227, "totalTokens": 870504, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T22:30:30.473851Z", "finishedAt": "2026-09-24T22:51:01.934878Z", "taskChecksum": "f7285aa10ae9a8d0c80a0a35bfcdb994a058f0f817eeaeef080c0dc817efb842", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "compile-compcert__PomCudX", "job": "pair-06", "strictlyQualifiedCompactions": 1, "compactionsObserved": 1, "providerRequests": 32, "agentRequests": 32, "costUsd": 0.02518264500000001, "inputTokens": 732843, "outputTokens": 6950, "cacheTokens": 592442, "totalTokens": 739793, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T22:30:30.502431Z", "finishedAt": "2026-09-24T22:51:31.523905Z", "taskChecksum": "f7285aa10ae9a8d0c80a0a35bfcdb994a058f0f817eeaeef080c0dc817efb842", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "circuit-fibsqrt", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "circuit-fibsqrt__vgCCXhr", "job": "pair-07", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 8, "agentRequests": 8, "costUsd": 0.01041132, "inputTokens": 106797, "outputTokens": 14449, "cacheTokens": 88367, "totalTokens": 121246, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T22:51:21.350699Z", "finishedAt": "2026-09-24T22:54:36.064352Z", "taskChecksum": "78e3e650669830d01d87116caebfb21c0ff6877ad815184ea108fa50034fc8f9", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "circuit-fibsqrt__ffTiS5w", "job": "pair-07", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 20, "agentRequests": 20, "costUsd": 0.03288326, "inputTokens": 625803, "outputTokens": 40427, "cacheTokens": 567676, "totalTokens": 666230, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T22:51:21.324933Z", "finishedAt": "2026-09-24T23:00:22.023361Z", "taskChecksum": "78e3e650669830d01d87116caebfb21c0ff6877ad815184ea108fa50034fc8f9", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "regex-chess", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "regex-chess__UqVP9NL", "job": "pair-08", "strictlyQualifiedCompactions": 1, "compactionsObserved": 1, "providerRequests": 25, "agentRequests": 26, "costUsd": 0.037589555000000004, "inputTokens": 848982, "outputTokens": 34234, "cacheTokens": 744768, "totalTokens": 883216, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T22:51:33.956328Z", "finishedAt": "2026-09-24T23:02:43.498687Z", "taskChecksum": "9125ddfbd11e611d4c9553432db49cc06380e0a37267d32f67cf57fe6f66c87b", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "regex-chess__P6hZNeC", "job": "pair-08", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 30, "agentRequests": 30, "costUsd": 0.029828925000000006, "inputTokens": 881668, "outputTokens": 28703, "cacheTokens": 820360, "totalTokens": 910371, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T22:51:33.981814Z", "finishedAt": "2026-09-24T23:01:05.878245Z", "taskChecksum": "9125ddfbd11e611d4c9553432db49cc06380e0a37267d32f67cf57fe6f66c87b", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "schemelike-metacircular-eval", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "schemelike-metacircular-eval__kjvX2Kh", "job": "pair-09", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 30, "agentRequests": 30, "costUsd": 0.024514265, "inputTokens": 909611, "outputTokens": 21216, "cacheTokens": 867764, "totalTokens": 930827, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:00:24.600768Z", "finishedAt": "2026-09-24T23:09:06.994478Z", "taskChecksum": "e7de1bcca18bd3a12c556963c57aabf7780094be76d5c06e8334eb4e25692365", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "schemelike-metacircular-eval__5drTLwe", "job": "pair-09", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 28, "agentRequests": 28, "costUsd": 0.01964374, "inputTokens": 808862, "outputTokens": 12219, "cacheTokens": 758764, "totalTokens": 821081, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:00:24.560869Z", "finishedAt": "2026-09-24T23:05:24.160750Z", "taskChecksum": "e7de1bcca18bd3a12c556963c57aabf7780094be76d5c06e8334eb4e25692365", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "write-compressor", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "write-compressor__fCZZVN5", "job": "pair-10", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 15, "agentRequests": 15, "costUsd": 0.00835464, "inputTokens": 140413, "outputTokens": 10718, "cacheTokens": 126564, "totalTokens": 151131, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:02:46.031635Z", "finishedAt": "2026-09-24T23:06:04.678689Z", "taskChecksum": "7aa3946dadb964150bc0e579ab3108a57e30d107d2167d59c986279b205b02b6", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "write-compressor__rSdcQnz", "job": "pair-10", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 14, "agentRequests": 14, "costUsd": 0.011417875, "inputTokens": 156056, "outputTokens": 14049, "cacheTokens": 129370, "totalTokens": 170105, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:02:46.057301Z", "finishedAt": "2026-09-24T23:06:10.183198Z", "taskChecksum": "7aa3946dadb964150bc0e579ab3108a57e30d107d2167d59c986279b205b02b6", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "gpt2-codegolf", "outcome": "Encrypted only", "arms": { "TextualFixed50kPi": { "state": "FAIL", "trial": "gpt2-codegolf__2WAz4YH", "job": "pair-11", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 14, "agentRequests": 14, "costUsd": 0.01371478, "inputTokens": 147818, "outputTokens": 19413, "cacheTokens": 125808, "totalTokens": 167231, "reward": 0, "exception": null, "error": null, "startedAt": "2026-09-24T23:06:12.695557Z", "finishedAt": "2026-09-24T23:10:25.059624Z", "taskChecksum": "384c32098bf0ee970715f65516726f9b17b8a797261f9a8dce44c99c371dd8f3", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "gpt2-codegolf__qDuh5FS", "job": "pair-11", "strictlyQualifiedCompactions": 1, "compactionsObserved": 1, "providerRequests": 68, "agentRequests": 68, "costUsd": 0.06054545499999999, "inputTokens": 1720515, "outputTokens": 50906, "cacheTokens": 1544498, "totalTokens": 1771421, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:06:12.670737Z", "finishedAt": "2026-09-24T23:17:05.585635Z", "taskChecksum": "384c32098bf0ee970715f65516726f9b17b8a797261f9a8dce44c99c371dd8f3", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "path-tracing", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "path-tracing__cvtwCX2", "job": "pair-12", "strictlyQualifiedCompactions": 17, "compactionsObserved": 17, "providerRequests": 149, "agentRequests": 144, "costUsd": 0.23534032000000005, "inputTokens": 5259116, "outputTokens": 141983, "cacheTokens": 4286917, "totalTokens": 5401099, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:09:09.441009Z", "finishedAt": "2026-09-24T23:34:35.247129Z", "taskChecksum": "c0059cc8b4f539405befee4386b355bff397f9dabe2610bdff279fcb0453e808", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "path-tracing__T2huC7B", "job": "pair-12", "strictlyQualifiedCompactions": 5, "compactionsObserved": 5, "providerRequests": 124, "agentRequests": 124, "costUsd": 0.141840075, "inputTokens": 3318360, "outputTokens": 122591, "cacheTokens": 2855010, "totalTokens": 3440951, "reward": 1, "exception": "AgentTimeoutError", "error": null, "startedAt": "2026-09-24T23:09:09.465902Z", "finishedAt": "2026-09-24T23:39:56.172300Z", "taskChecksum": "c0059cc8b4f539405befee4386b355bff397f9dabe2610bdff279fcb0453e808", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "custom-memory-heap-crash", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "custom-memory-heap-crash__kcq7qQR", "job": "pair-13", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 31, "agentRequests": 31, "costUsd": 0.01610979, "inputTokens": 694053, "outputTokens": 9999, "cacheTokens": 657774, "totalTokens": 704052, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:17:08.073463Z", "finishedAt": "2026-09-24T23:20:13.458966Z", "taskChecksum": "949f44d01f9411e5858fdfd0861e2c35e9622cbfe91542b761039a7a16286381", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "custom-memory-heap-crash__8KkLneN", "job": "pair-13", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 21, "agentRequests": 21, "costUsd": 0.012484765, "inputTokens": 381900, "outputTokens": 8236, "cacheTokens": 339324, "totalTokens": 390136, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:17:08.047478Z", "finishedAt": "2026-09-24T23:19:30.909093Z", "taskChecksum": "949f44d01f9411e5858fdfd0861e2c35e9622cbfe91542b761039a7a16286381", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "build-pov-ray", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "build-pov-ray__mtaZgEG", "job": "pair-14", "strictlyQualifiedCompactions": 1, "compactionsObserved": 1, "providerRequests": 55, "agentRequests": 55, "costUsd": 0.03621660500000001, "inputTokens": 1631667, "outputTokens": 14705, "cacheTokens": 1522523, "totalTokens": 1646372, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:20:15.893795Z", "finishedAt": "2026-09-24T23:25:09.041134Z", "taskChecksum": "62e9feaef83fb9febfe3fc2a55ab7ca2fd505231114155b2e55ed880bb2c58b4", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "build-pov-ray__J9V6n3x", "job": "pair-14", "strictlyQualifiedCompactions": 1, "compactionsObserved": 1, "providerRequests": 55, "agentRequests": 55, "costUsd": 0.032577290000000016, "inputTokens": 1177010, "outputTokens": 10929, "cacheTokens": 1024394, "totalTokens": 1187939, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:20:15.919698Z", "finishedAt": "2026-09-24T23:25:07.069580Z", "taskChecksum": "62e9feaef83fb9febfe3fc2a55ab7ca2fd505231114155b2e55ed880bb2c58b4", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "bn-fit-modify", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "bn-fit-modify__4VjDnMU", "job": "pair-15", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 8, "agentRequests": 8, "costUsd": 0.00342485, "inputTokens": 27798, "outputTokens": 4621, "cacheTokens": 20520, "totalTokens": 32419, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:25:11.530857Z", "finishedAt": "2026-09-24T23:26:23.004341Z", "taskChecksum": "454bfa8137cdac00b085ad27568437e1031ecaa2456669e5632f11aeb96a5c62", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "bn-fit-modify__gqtk8Sf", "job": "pair-15", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 8, "agentRequests": 8, "costUsd": 0.00375196, "inputTokens": 30220, "outputTokens": 4279, "cacheTokens": 17876, "totalTokens": 34499, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:25:11.506096Z", "finishedAt": "2026-09-24T23:26:29.182624Z", "taskChecksum": "454bfa8137cdac00b085ad27568437e1031ecaa2456669e5632f11aeb96a5c62", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "mcmc-sampling-stan", "outcome": "Textual only", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "mcmc-sampling-stan__LMaw5vg", "job": "pair-16", "strictlyQualifiedCompactions": 2, "compactionsObserved": 2, "providerRequests": 27, "agentRequests": 26, "costUsd": 0.028053684999999995, "inputTokens": 640296, "outputTokens": 13371, "cacheTokens": 510146, "totalTokens": 653667, "reward": 1, "exception": "AgentTimeoutError", "error": null, "startedAt": "2026-09-24T23:26:31.722048Z", "finishedAt": "2026-09-24T23:59:52.738262Z", "taskChecksum": "97dd19504c429a02e1dbec93ee9c0cc64d46fd886be8d88c59b84feb26144bef", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "FAIL", "trial": "mcmc-sampling-stan__bWjfeix", "job": "pair-16", "strictlyQualifiedCompactions": 1, "compactionsObserved": 1, "providerRequests": 17, "agentRequests": 17, "costUsd": 0.018065995, "inputTokens": 347304, "outputTokens": 6819, "cacheTokens": 236022, "totalTokens": 354123, "reward": 0, "exception": null, "error": null, "startedAt": "2026-09-24T23:26:31.746788Z", "finishedAt": "2026-09-24T23:54:22.759627Z", "taskChecksum": "97dd19504c429a02e1dbec93ee9c0cc64d46fd886be8d88c59b84feb26144bef", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "rstan-to-pystan", "outcome": "Both pass", "arms": { "TextualFixed50kPi": { "state": "PASS", "trial": "rstan-to-pystan__JugnMSB", "job": "pair-17", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 26, "agentRequests": 26, "costUsd": 0.01470632, "inputTokens": 490740, "outputTokens": 8461, "cacheTokens": 442302, "totalTokens": 499201, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:39:58.668914Z", "finishedAt": "2026-09-24T23:58:05.582383Z", "taskChecksum": "1abf8912a1a885c218b97f347559dcba74804c675e5afe653aba1e24be5de6a8", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "rstan-to-pystan__VQZJgsW", "job": "pair-17", "strictlyQualifiedCompactions": 1, "compactionsObserved": 1, "providerRequests": 26, "agentRequests": 26, "costUsd": 0.01552833, "inputTokens": 419788, "outputTokens": 8307, "cacheTokens": 353538, "totalTokens": 428095, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:39:58.641432Z", "finishedAt": "2026-09-24T23:53:40.819620Z", "taskChecksum": "1abf8912a1a885c218b97f347559dcba74804c675e5afe653aba1e24be5de6a8", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "torch-pipeline-parallelism", "outcome": "Encrypted only", "arms": { "TextualFixed50kPi": { "state": "FAIL", "trial": "torch-pipeline-parallelism__exLEFVo", "job": "pair-18", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 9, "agentRequests": 9, "costUsd": 0.00794851, "inputTokens": 67140, "outputTokens": 11207, "cacheTokens": 52581, "totalTokens": 78347, "reward": 0, "exception": null, "error": null, "startedAt": "2026-09-24T23:58:08.382024Z", "finishedAt": "2026-09-25T00:02:22.050639Z", "taskChecksum": "0c68fea1761a455dbd35e1cf6a7ad9b4fc433312d14259d084cf88016681bc82", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "PASS", "trial": "torch-pipeline-parallelism__kHBfs93", "job": "pair-18", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 7, "agentRequests": 7, "costUsd": 0.0071345800000000015, "inputTokens": 37480, "outputTokens": 9626, "cacheTokens": 19178, "totalTokens": 47106, "reward": 1, "exception": null, "error": null, "startedAt": "2026-09-24T23:58:08.411159Z", "finishedAt": "2026-09-25T00:01:51.972982Z", "taskChecksum": "0c68fea1761a455dbd35e1cf6a7ad9b4fc433312d14259d084cf88016681bc82", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } }, { "task": "torch-tensor-parallelism", "outcome": "Both fail", "arms": { "TextualFixed50kPi": { "state": "FAIL", "trial": "torch-tensor-parallelism__mXRhrZk", "job": "pair-19", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 8, "agentRequests": 8, "costUsd": 0.00446081, "inputTokens": 41372, "outputTokens": 6162, "cacheTokens": 32966, "totalTokens": 47534, "reward": 0, "exception": null, "error": null, "startedAt": "2026-09-24T23:59:55.194364Z", "finishedAt": "2026-09-25T00:03:23.501710Z", "taskChecksum": "3efbd1bc7719d9cbc147fe8942eb72f5c25d591d683a1bc521febf1316e57f88", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false }, "Native50kPi": { "state": "FAIL", "trial": "torch-tensor-parallelism__jiaSkd6", "job": "pair-19", "strictlyQualifiedCompactions": 0, "compactionsObserved": 0, "providerRequests": 5, "agentRequests": 5, "costUsd": 0.0033819099999999997, "inputTokens": 15307, "outputTokens": 4223, "cacheTokens": 4686, "totalTokens": 19530, "reward": 0, "exception": null, "error": null, "startedAt": "2026-09-24T23:59:55.168559Z", "finishedAt": "2026-09-25T00:03:15.122454Z", "taskChecksum": "3efbd1bc7719d9cbc147fe8942eb72f5c25d591d683a1bc521febf1316e57f88", "actualModels": [ "gpt-6-luna" ], "uncertainCost": false } } } ], "warnings": [], "audit": { "ok": true, "warnings": [] }, "interval": "+10.0 pp (95% paired bootstrap -10.0 to 30.0 pp; 20 tasks)" }