{ "schema_version": 3, "generated_at": "2026-10-19T17:39:10+00:00", "source_sha256": "2f7580a0e5a08577f7b9f141c94e42b0edd0e0639268498057f28d2008419692", "core_revision": "7a044a156c37bc1febd823344d6e6c97ff083019", "runtime": "1 × NVIDIA H200 SXM 141GB", "task": "lm-pretrain", "selection": "Latest authorized attempt per model/Track configuration; B6 and B12 retained separately; previous failed attempts excluded.", "excluded_attempts": [ { "model": "glm-5.3", "track": "B", "track_config": null, "outcome": "failed" }, { "model": "deepseek-v4-flash", "track": "A", "track_config": null, "outcome": "failed" }, { "model": "glm-5.3", "track": "B", "track_config": "preview-b-12x1h", "outcome": "failed" } ], "preview": true, "rows": [ { "model": "gpt-5.6-luna", "track": "B", "track_config": "preview-b-6x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-6x1h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": 6, "status": "finalized", "outcome": "success", "test_nll": 6.359444763140411, "score": 41.25172744871658, "seconds_per_task": 0.02905222325921386, "ms": 29.05222325921386, "ppl": 577.9253818194371, "evaluation_seconds": 59.49895323486999, "tasks_per_second": 34.420773621270165, "input_tokens_per_second": 23111.398188331586, "task_count": 2048, "budget_hours": 6.0, "lease_hours": 2.8550688083333333, "research_hours": 2.904826594722222, "leases": 4, "candidates": 2, "first_analysis": 20.7544659948457, "final_analysis": 6.462881619076471, "reasoning_effort": "medium", "points": [ { "index": 1, "candidate_id": "ae70e05d5695", "lease": "lease-0003", "hours": 1.9617612202777777, "research_hours": 1.998434834722222, "nll": 20.7544659948457, "score": 0.0, "ppl": 1031691984.5218021, "seconds_per_task": 0.030369924336355325, "ms": 30.369924336355325, "evaluation_seconds": 62.197605040855706, "tasks_per_second": 32.927312854807376, "input_tokens_per_second": 22108.632624949725, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "2873a9fb9349", "lease": "lease-0004", "hours": 2.855068808611111, "research_hours": 2.8996348541666666, "nll": 6.462881619076471, "score": 40.29618229804548, "ppl": 640.9052429463804, "seconds_per_task": 0.02943418453423874, "ms": 29.43418453423874, "evaluation_seconds": 60.28120992612094, "tasks_per_second": 33.974102419476566, "input_tokens_per_second": 22811.486393277297, "task_count": 2048, "final": true } ], "reasoning_tokens": 1990, "output_tokens": 5239, "total_tokens": 1136848, "agent_turns": 35 }, { "model": "gpt-5.6-terra", "track": "B", "track_config": "preview-b-6x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-6x1h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": 6, "status": "finalized", "outcome": "success", "test_nll": 3.788062542227833, "score": 65.00604392981427, "seconds_per_task": 0.004777960676392468, "ms": 4.777960676392468, "ppl": 44.17073837925039, "evaluation_seconds": 9.785263465251774, "tasks_per_second": 209.29431356371816, "input_tokens_per_second": 140528.050663439, "task_count": 2048, "budget_hours": 6.0, "lease_hours": 1.734797806388889, "research_hours": 1.7569193752777779, "leases": 2, "candidates": 1, "first_analysis": 3.8266165565840007, "final_analysis": 3.8266165565840007, "reasoning_effort": "medium", "points": [ { "index": 1, "candidate_id": "a4d587405d46", "lease": "lease-0002", "hours": 1.7347978063888887, "research_hours": 1.7513810866666666, "nll": 3.8266165565840007, "score": 64.64988363159607, "ppl": 45.90695160021351, "seconds_per_task": 0.004775839541025562, "ms": 4.775839541025562, "evaluation_seconds": 9.78091938002035, "tasks_per_second": 209.38726927690297, "input_tokens_per_second": 140590.46461511054, "task_count": 2048, "final": true } ], "reasoning_tokens": 2953, "output_tokens": 6642, "total_tokens": 788207, "agent_turns": 23 }, { "model": "gpt-5.6-sol", "track": "B", "track_config": "preview-b-6x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-6x1h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": 6, "status": "finalized", "outcome": "success", "test_nll": 3.229495537606164, "score": 70.16606148604242, "seconds_per_task": 0.005981626528864581, "ms": 5.981626528864581, "ppl": 25.26690755076386, "evaluation_seconds": 12.250371131114662, "tasks_per_second": 167.17860855646194, "input_tokens_per_second": 112249.98698262942, "task_count": 2048, "budget_hours": 6.0, "lease_hours": 5.26856855, "research_hours": 5.3230718575000004, "leases": 6, "candidates": 6, "first_analysis": 3.583498869310492, "final_analysis": 3.2175719170956056, "reasoning_effort": "medium", "points": [ { "index": 1, "candidate_id": "7953524659af", "lease": "lease-0001", "hours": 0.7993058638888889, "research_hours": 0.8137329155555556, "nll": 3.583498869310492, "score": 66.89579419233638, "ppl": 35.99927751800758, "seconds_per_task": 0.005979985827707424, "ms": 5.979985827707424, "evaluation_seconds": 12.247010975144804, "tasks_per_second": 167.22447658097124, "input_tokens_per_second": 112280.78449433588, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "5405aed86354", "lease": "lease-0002", "hours": 1.6917726336111112, "research_hours": 1.7130500522222223, "nll": 3.3930209645596427, "score": 68.65542074466677, "ppl": 29.755707566329168, "seconds_per_task": 0.005829052737681195, "ms": 5.829052737681195, "evaluation_seconds": 11.937900006771088, "tasks_per_second": 171.55446090504944, "input_tokens_per_second": 115188.09834393414, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "93415ef352b3", "lease": "lease-0003", "hours": 2.5829608986111112, "research_hours": 2.613718581111111, "nll": 3.331842280193082, "score": 69.22058675480768, "ppl": 27.98985939722665, "seconds_per_task": 0.006031939562490152, "ms": 6.031939562490152, "evaluation_seconds": 12.35341222397983, "tasks_per_second": 165.78415443989832, "input_tokens_per_second": 111313.69819673922, "task_count": 2048, "final": false }, { "index": 4, "candidate_id": "04b0ab94b00f", "lease": "lease-0004", "hours": 3.4768095316666665, "research_hours": 3.5142922975, "nll": 3.2607105545192345, "score": 69.87769852518602, "ppl": 26.06805333646252, "seconds_per_task": 0.005942903682580436, "ms": 5.942903682580436, "evaluation_seconds": 12.171066741924733, "tasks_per_second": 168.26791302897163, "input_tokens_per_second": 112981.38685439013, "task_count": 2048, "final": false }, { "index": 5, "candidate_id": "55112fadcdb3", "lease": "lease-0005", "hours": 4.3694552275000005, "research_hours": 4.413783918333333, "nll": 3.225357166697916, "score": 70.20429157547495, "ppl": 25.162559779427617, "seconds_per_task": 0.0056687125386361, "ms": 5.6687125386361, "evaluation_seconds": 11.609523279126734, "tasks_per_second": 176.40689895356755, "input_tokens_per_second": 118446.20721613601, "task_count": 2048, "final": false }, { "index": 6, "candidate_id": "a7cd0d287f5f", "lease": "lease-0006", "hours": 5.26856855, "research_hours": 5.319707358888889, "nll": 3.2175719170956056, "score": 70.27621137076387, "ppl": 24.967423548644657, "seconds_per_task": 0.005743525006892014, "ms": 5.743525006892014, "evaluation_seconds": 11.762739214114845, "tasks_per_second": 174.109105262019, "input_tokens_per_second": 116903.38236436687, "task_count": 2048, "final": true } ], "reasoning_tokens": 2211, "output_tokens": 8003, "total_tokens": 1816774, "agent_turns": 36 }, { "model": "claude-sonnet-5", "track": "B", "track_config": "preview-b-6x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-6x1h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": 6, "status": "finalized", "outcome": "success", "test_nll": 3.134502695392341, "score": 71.04360120729993, "seconds_per_task": 0.005241482888095561, "ms": 5.241482888095561, "ppl": 22.977206329716893, "evaluation_seconds": 10.734556954819709, "tasks_per_second": 190.78570346403242, "input_tokens_per_second": 128100.67576963127, "task_count": 2048, "budget_hours": 6.0, "lease_hours": 4.83820652888889, "research_hours": 4.871391920000001, "leases": 6, "candidates": 1, "first_analysis": 3.185325654525792, "final_analysis": 3.185325654525792, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "189e72fd1f91", "lease": "lease-0006", "hours": 4.83820652888889, "research_hours": 4.8687079452777775, "nll": 3.185325654525792, "score": 70.57410093388923, "ppl": 24.17515987431068, "seconds_per_task": 0.005480226780264275, "ms": 5.480226780264275, "evaluation_seconds": 11.223504445981234, "tasks_per_second": 182.4742004475546, "input_tokens_per_second": 122520.02096300494, "task_count": 2048, "final": true } ], "reasoning_tokens": 28449, "output_tokens": 30058, "total_tokens": 7455210, "agent_turns": 82 }, { "model": "claude-opus-5", "track": "B", "track_config": "preview-b-6x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-6x1h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": 6, "status": "finalized", "outcome": "success", "test_nll": 3.017565853133928, "score": 72.12385864110948, "seconds_per_task": 0.005750144274543345, "ms": 5.750144274543345, "ppl": 20.441473527710844, "evaluation_seconds": 11.77629547426477, "tasks_per_second": 173.9086798964564, "input_tokens_per_second": 116768.80925797693, "task_count": 2048, "budget_hours": 6.0, "lease_hours": 5.109524305000001, "research_hours": 5.145477649722222, "leases": 6, "candidates": 2, "first_analysis": 3.5329726321582053, "final_analysis": 3.0321317570884605, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "014f9376d0bc", "lease": "lease-0001", "hours": 0.831356956388889, "research_hours": 0.8419814022222223, "nll": 3.5329726321582053, "score": 67.36255336106412, "ppl": 34.22555653669179, "seconds_per_task": 0.006032888488789467, "ms": 6.032888488789467, "evaluation_seconds": 12.35535562504083, "tasks_per_second": 165.75807788561588, "input_tokens_per_second": 111296.18942032321, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "0b386f75efff", "lease": "lease-0006", "hours": 5.109524305000001, "research_hours": 5.142945618888889, "nll": 3.0321317570884605, "score": 71.9892994575096, "ppl": 20.74140112323943, "seconds_per_task": 0.005923672694962079, "ms": 5.923672694962079, "evaluation_seconds": 12.131681679282337, "tasks_per_second": 168.81418867900527, "input_tokens_per_second": 113348.1768111596, "task_count": 2048, "final": true } ], "reasoning_tokens": 25644, "output_tokens": 23382, "total_tokens": 2067084, "agent_turns": 43 }, { "model": "deepseek-v4-flash", "track": "B", "track_config": "preview-b-6x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-6x1h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": 6, "status": "finalized", "outcome": "success", "test_nll": 3.058128111567686, "score": 71.74914627194579, "seconds_per_task": 0.005485652499828575, "ms": 5.485652499828575, "ppl": 21.287671696702812, "evaluation_seconds": 11.234616319648921, "tasks_per_second": 182.29371985032768, "input_tokens_per_second": 122398.8395220044, "task_count": 2048, "budget_hours": 6.0, "lease_hours": 5.003047263333333, "research_hours": 5.0418284938888895, "leases": 6, "candidates": 2, "first_analysis": 3.2009684868958432, "final_analysis": 3.0890940126996425, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "d7417383bf9d", "lease": "lease-0005", "hours": 4.1915955205555555, "research_hours": 4.2217354049999996, "nll": 3.2009684868958432, "score": 70.42959313269309, "ppl": 24.556301140192538, "seconds_per_task": 0.005526260570377417, "ms": 5.526260570377417, "evaluation_seconds": 11.31778164813295, "tasks_per_second": 180.9541890515135, "input_tokens_per_second": 121499.42831127561, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "fce61eae7930", "lease": "lease-0006", "hours": 5.003047263333333, "research_hours": 5.040023001111112, "nll": 3.0890940126996425, "score": 71.46308463177866, "ppl": 21.957176039030692, "seconds_per_task": 0.005546785478827587, "ms": 5.546785478827587, "evaluation_seconds": 11.359816660638899, "tasks_per_second": 180.2845997590965, "input_tokens_per_second": 121049.84095074836, "task_count": 2048, "final": true } ], "reasoning_tokens": 28123, "output_tokens": 11401, "total_tokens": 2020298, "agent_turns": 43 }, { "model": "deepseek-v4-pro", "track": "B", "track_config": "preview-b-6x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-6x1h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": 6, "status": "finalized", "outcome": "success", "test_nll": 3.438659371901475, "score": 68.23381513392782, "seconds_per_task": 0.0059451668294059345, "ms": 5.9451668294059345, "ppl": 31.14517606926735, "evaluation_seconds": 12.175701666623354, "tasks_per_second": 168.2038584777484, "input_tokens_per_second": 112938.37822665319, "task_count": 2048, "budget_hours": 6.0, "lease_hours": 3.984912792222222, "research_hours": 4.022744548888888, "leases": 6, "candidates": 1, "first_analysis": 3.470289980848834, "final_analysis": 3.470289980848834, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "650b36b6f3b5", "lease": "lease-0006", "hours": 3.9849127925, "research_hours": 4.020548846111111, "nll": 3.470289980848834, "score": 67.9416129520372, "ppl": 32.1460628359082, "seconds_per_task": 0.0058945008686350775, "ms": 5.8945008686350775, "evaluation_seconds": 12.071937778964639, "tasks_per_second": 169.649648424186, "input_tokens_per_second": 113909.1358138144, "task_count": 2048, "final": true } ], "reasoning_tokens": 41884, "output_tokens": 13820, "total_tokens": 4380272, "agent_turns": 56 }, { "model": "kimi-k3", "track": "B", "track_config": "preview-b-6x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-6x1h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": 6, "status": "finalized", "outcome": "success", "test_nll": 3.0364002220492186, "score": 71.94986756491049, "seconds_per_task": 0.006242106470836006, "ms": 6.242106470836006, "ppl": 20.83012428832192, "evaluation_seconds": 12.783834052272141, "tasks_per_second": 160.20232988209025, "input_tokens_per_second": 107565.85187020597, "task_count": 2048, "budget_hours": 6.0, "lease_hours": 5.400805265833334, "research_hours": 5.473760193611111, "leases": 6, "candidates": 6, "first_analysis": 3.590522183491794, "final_analysis": 3.0649334698114874, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "76a0957519e9", "lease": "lease-0001", "hours": 0.9188271577777778, "research_hours": 0.9342546330555556, "nll": 3.590522183491794, "score": 66.83091312313866, "ppl": 36.25300170369986, "seconds_per_task": 0.005802400994070922, "ms": 5.802400994070922, "evaluation_seconds": 11.883317235857248, "tasks_per_second": 172.34244944839762, "input_tokens_per_second": 115717.18340150848, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "8be987258612", "lease": "lease-0002", "hours": 1.8117128397222222, "research_hours": 1.8397212983333333, "nll": 3.3754107811171576, "score": 68.8181028490155, "ppl": 29.236291031181658, "seconds_per_task": 0.005873021916158905, "ms": 5.873021916158905, "evaluation_seconds": 12.027948884293437, "tasks_per_second": 170.27009506785964, "input_tokens_per_second": 114325.72695712601, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "ac492aba36ce", "lease": "lease-0003", "hours": 2.7028213188888888, "research_hours": 2.738134278333333, "nll": 3.2545898160684796, "score": 69.9342416392648, "ppl": 25.908984904806573, "seconds_per_task": 0.00593496662418147, "ms": 5.93496662418147, "evaluation_seconds": 12.154811646323651, "tasks_per_second": 168.49294416005523, "input_tokens_per_second": 113132.4811944671, "task_count": 2048, "final": false }, { "index": 4, "candidate_id": "b0a9d6a539a4", "lease": "lease-0004", "hours": 3.597082975555556, "research_hours": 3.642166097777778, "nll": 3.239617306231056, "score": 70.07255703022534, "ppl": 25.52395202066686, "seconds_per_task": 0.0057030439024856605, "ms": 5.7030439024856605, "evaluation_seconds": 11.679833912290633, "tasks_per_second": 175.34495913036054, "input_tokens_per_second": 117733.18099609145, "task_count": 2048, "final": false }, { "index": 5, "candidate_id": "aa984785f644", "lease": "lease-0005", "hours": 4.493196662222222, "research_hours": 4.549453894166667, "nll": 3.1756899543711703, "score": 70.66311511043922, "ppl": 23.943333978667702, "seconds_per_task": 0.005767190767073771, "ms": 5.767190767073771, "evaluation_seconds": 11.811206690967083, "tasks_per_second": 173.39464574489747, "input_tokens_per_second": 116423.66745233959, "task_count": 2048, "final": false }, { "index": 6, "candidate_id": "23230bf86298", "lease": "lease-0006", "hours": 5.400805265555555, "research_hours": 5.468189479444444, "nll": 3.0649334698114874, "score": 71.68627867016501, "ppl": 21.433035995821218, "seconds_per_task": 0.005752141458060578, "ms": 5.752141458060578, "evaluation_seconds": 11.780385706108063, "tasks_per_second": 173.84829759335668, "input_tokens_per_second": 116728.26631533944, "task_count": 2048, "final": true } ], "reasoning_tokens": 33480, "output_tokens": 20857, "total_tokens": 4423432, "agent_turns": 75 }, { "model": "glm-5.3", "track": "B", "track_config": "preview-b-6x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-6x1h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": 6, "status": "finalized", "outcome": "success", "test_nll": 3.149064944548225, "score": 70.90907578657011, "seconds_per_task": 0.005226121973691988, "ms": 5.226121973691988, "ppl": 23.314254265049836, "evaluation_seconds": 10.703097802121192, "tasks_per_second": 191.34647163498005, "input_tokens_per_second": 128477.19654841191, "task_count": 2048, "budget_hours": 6.0, "lease_hours": 5.093102945277778, "research_hours": 5.218912179722222, "leases": 6, "candidates": 3, "first_analysis": 3.6054832246612007, "final_analysis": 3.2035448862840767, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "ef0bc607eb74", "lease": "lease-0002", "hours": 1.6443182897222222, "research_hours": 1.7366705208333333, "nll": 3.6054832246612007, "score": 66.6927036792316, "ppl": 36.799461969029565, "seconds_per_task": 0.005314896788377155, "ms": 5.314896788377155, "evaluation_seconds": 10.884908622596413, "tasks_per_second": 188.15040814844102, "input_tokens_per_second": 126331.23967116886, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "b156d2b3c0e8", "lease": "lease-0005", "hours": 4.372751926388888, "research_hours": 4.484669535, "nll": 3.309682952468253, "score": 69.42529365505108, "ppl": 27.376444461684923, "seconds_per_task": 0.005348094431155914, "ms": 5.348094431155914, "evaluation_seconds": 10.952897395007312, "tasks_per_second": 186.98248747710767, "input_tokens_per_second": 125547.05393541048, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "34fad622ad6e", "lease": "lease-0006", "hours": 5.093102945277778, "research_hours": 5.215570272222222, "nll": 3.2035448862840767, "score": 70.40579246783972, "ppl": 24.619649549787923, "seconds_per_task": 0.005254499657667111, "ms": 5.254499657667111, "evaluation_seconds": 10.761215298902243, "tasks_per_second": 190.31307739088888, "input_tokens_per_second": 127783.33690064495, "task_count": 2048, "final": true } ], "reasoning_tokens": 83364, "output_tokens": 26546, "total_tokens": 11579584, "agent_turns": 104 }, { "model": "gpt-6-astra", "track": "A", "track_config": "preview-a-8h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "A", "track_config": "preview-a-8h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": null, "status": "finalized", "outcome": "success", "test_nll": 2.986191461153635, "score": 72.41369390186563, "seconds_per_task": 0.004962810632264336, "ms": 4.962810632264336, "ppl": 19.810091134988937, "evaluation_seconds": 10.16383617487736, "tasks_per_second": 201.49872201425086, "input_tokens_per_second": 135293.79816244356, "task_count": 2048, "budget_hours": 8.0, "lease_hours": 7.470907205277778, "research_hours": 7.487047124166667, "leases": 1, "candidates": 1, "first_analysis": 3.0258160244890764, "final_analysis": 3.0258160244890764, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "c9e9f7ced8a2", "lease": "lease-0001", "hours": 7.470907205277778, "research_hours": 7.484534042500001, "nll": 3.0258160244890764, "score": 72.04764391900414, "ppl": 20.610816781966324, "seconds_per_task": 0.004926358596208047, "ms": 4.926358596208047, "evaluation_seconds": 10.08918240503408, "tasks_per_second": 202.98968913260342, "input_tokens_per_second": 136294.8893969724, "task_count": 2048, "final": true } ], "reasoning_tokens": 9192, "output_tokens": 13970, "total_tokens": 2798959, "agent_turns": 51 }, { "model": "claude-fable-5-1", "track": "A", "track_config": "preview-a-8h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "A", "track_config": "preview-a-8h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": null, "status": "finalized", "outcome": "success", "test_nll": 2.88529248349031, "score": 73.34579424406338, "seconds_per_task": 0.005995534990688611, "ms": 5.995534990688611, "ppl": 17.90880485932868, "evaluation_seconds": 12.278855660930276, "tasks_per_second": 166.79078706955323, "input_tokens_per_second": 111989.58909301314, "task_count": 2048, "budget_hours": 8.0, "lease_hours": 7.517945586388889, "research_hours": 7.548542665, "leases": 2, "candidates": 2, "first_analysis": 7.771034468087212, "final_analysis": 2.886063029217994, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "9911c103cf1e", "lease": "lease-0001", "hours": 0.27873582305555555, "research_hours": 0.28951273777777775, "nll": 7.771034468087212, "score": 28.211523499237067, "ppl": 2370.922664034981, "seconds_per_task": 0.0045752031177244135, "ms": 4.5752031177244135, "evaluation_seconds": 9.370015985099599, "tasks_per_second": 218.56953107196122, "input_tokens_per_second": 146755.77951912995, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "5cb9197ed179", "lease": "lease-0002", "hours": 7.517945586388889, "research_hours": 7.544805519722222, "nll": 2.886063029217994, "score": 73.33867597633574, "ppl": 17.922609730360108, "seconds_per_task": 0.006015095246652891, "ms": 6.015095246652891, "evaluation_seconds": 12.31891506514512, "tasks_per_second": 166.24840654958066, "input_tokens_per_second": 111625.41447263407, "task_count": 2048, "final": true } ], "reasoning_tokens": 34494, "output_tokens": 36430, "total_tokens": 3172864, "agent_turns": 58 }, { "model": "gpt-6-astra", "track": "B", "track_config": "preview-b-6x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-6x1h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": 6, "status": "finalized", "outcome": "success", "test_nll": 3.122259727831594, "score": 71.15670120610235, "seconds_per_task": 0.004683376078673973, "ms": 4.683376078673973, "ppl": 22.697612161512687, "evaluation_seconds": 9.591554209124297, "tasks_per_second": 213.52118283935354, "input_tokens_per_second": 143366.12920269844, "task_count": 2048, "budget_hours": 6.0, "lease_hours": 5.377279839444444, "research_hours": 5.423411071944444, "leases": 6, "candidates": 6, "first_analysis": 4.375368536329834, "final_analysis": 3.1322024045527033, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "8bfebc39cb53", "lease": "lease-0001", "hours": 0.8256679230555556, "research_hours": 0.8338855672222222, "nll": 4.375368536329834, "score": 59.58053684584858, "ppl": 79.46912141455611, "seconds_per_task": 0.005301226430674433, "ms": 5.301226430674433, "evaluation_seconds": 10.856911730021238, "tasks_per_second": 188.63559462650193, "input_tokens_per_second": 126657.01206703189, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "542595e18d2a", "lease": "lease-0002", "hours": 1.7483966361111112, "research_hours": 1.7647523052777778, "nll": 4.474357311632436, "score": 58.66608286950438, "ppl": 87.73819396915967, "seconds_per_task": 0.005432205831993997, "ms": 5.432205831993997, "evaluation_seconds": 11.125157543923706, "tasks_per_second": 184.0872807341563, "input_tokens_per_second": 123603.10355794006, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "3a3376bfe030", "lease": "lease-0003", "hours": 2.676583027777778, "research_hours": 2.6993877283333334, "nll": 4.164212934348758, "score": 61.531183060721695, "ppl": 64.34202111443108, "seconds_per_task": 0.0052369748045748565, "ms": 5.2369748045748565, "evaluation_seconds": 10.725324399769306, "tasks_per_second": 190.94993528065697, "input_tokens_per_second": 128210.94717000611, "task_count": 2048, "final": false }, { "index": 4, "candidate_id": "076638b0d8be", "lease": "lease-0004", "hours": 3.5994437930555554, "research_hours": 3.6293835225, "nll": 4.207483568490142, "score": 61.13145083523892, "ppl": 67.18725477737075, "seconds_per_task": 0.005328661324710993, "ms": 5.328661324710993, "evaluation_seconds": 10.913098393008113, "tasks_per_second": 187.66439431281296, "input_tokens_per_second": 126004.91175640936, "task_count": 2048, "final": false }, { "index": 5, "candidate_id": "cd58f7aab5d1", "lease": "lease-0005", "hours": 4.508453073055556, "research_hours": 4.543657536111111, "nll": 3.254495762351258, "score": 69.93511050339048, "ppl": 25.90654818306, "seconds_per_task": 0.005627136368502761, "ms": 5.627136368502761, "evaluation_seconds": 11.524375282693654, "tasks_per_second": 177.7102836173268, "input_tokens_per_second": 119321.34855630886, "task_count": 2048, "final": false }, { "index": 6, "candidate_id": "11d2816595a9", "lease": "lease-0006", "hours": 5.3772798391666665, "research_hours": 5.419334380833333, "nll": 3.1322024045527033, "score": 71.06485119345871, "ppl": 22.924412815973234, "seconds_per_task": 0.004517626364759053, "ms": 4.517626364759053, "evaluation_seconds": 9.25209879502654, "tasks_per_second": 221.35518063219354, "input_tokens_per_second": 148626.16909572843, "task_count": 2048, "final": true } ], "reasoning_tokens": 1159, "output_tokens": 6702, "total_tokens": 1601189, "agent_turns": 40 }, { "model": "claude-fable-5-1", "track": "B", "track_config": "preview-b-6x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-6x1h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": 6, "status": "finalized", "outcome": "success", "test_nll": 3.017252226719791, "score": 72.12675590820486, "seconds_per_task": 0.0062553301547723095, "ms": 6.2553301547723095, "ppl": 20.435063546890856, "evaluation_seconds": 12.81091615697369, "tasks_per_second": 159.86366430828292, "input_tokens_per_second": 107338.45910399272, "task_count": 2048, "budget_hours": 6.0, "lease_hours": 4.779901470555556, "research_hours": 4.867967242777778, "leases": 6, "candidates": 5, "first_analysis": 3.3912709365136458, "final_analysis": 3.044046764266998, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "cfc678d136a4", "lease": "lease-0002", "hours": 1.2716253341666668, "research_hours": 1.3170873819444444, "nll": 3.3912709365136458, "score": 68.67158742720714, "ppl": 29.703679781878936, "seconds_per_task": 0.006290217187370217, "ms": 6.290217187370217, "evaluation_seconds": 12.882364799734205, "tasks_per_second": 158.97702260708027, "input_tokens_per_second": 106743.13461674147, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "d4da811c8e3f", "lease": "lease-0003", "hours": 2.1475148619444444, "research_hours": 2.2052149697222223, "nll": 3.2464250203171936, "score": 70.00966766527613, "ppl": 25.69830458621688, "seconds_per_task": 0.00621913666736873, "ms": 6.21913666736873, "evaluation_seconds": 12.736791894771159, "tasks_per_second": 160.79402230327455, "input_tokens_per_second": 107963.13635025489, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "d307e8ec687b", "lease": "lease-0004", "hours": 3.0144274263888886, "research_hours": 3.0820062908333337, "nll": 3.1696913272763023, "score": 70.71853016515365, "ppl": 23.800136769446166, "seconds_per_task": 0.006248432998518183, "ms": 6.248432998518183, "evaluation_seconds": 12.796790780965239, "tasks_per_second": 160.04012529815878, "input_tokens_per_second": 107456.9416298825, "task_count": 2048, "final": false }, { "index": 4, "candidate_id": "239fd4a7ba2d", "lease": "lease-0005", "hours": 3.883104146111111, "research_hours": 3.958041688611111, "nll": 3.1374654593206923, "score": 71.01623132372508, "ppl": 23.045383314088596, "seconds_per_task": 0.006218034558060026, "ms": 6.218034558060026, "evaluation_seconds": 12.734534774906933, "tasks_per_second": 160.82252207874373, "input_tokens_per_second": 107982.27216824649, "task_count": 2048, "final": false }, { "index": 5, "candidate_id": "02ff3d0e5dbf", "lease": "lease-0006", "hours": 4.779901470555555, "research_hours": 4.862615659166667, "nll": 3.044046764266998, "score": 71.87922914237261, "ppl": 20.990013232823408, "seconds_per_task": 0.006155366227176273, "ms": 6.155366227176273, "evaluation_seconds": 12.606190033257008, "tasks_per_second": 162.45987047609714, "input_tokens_per_second": 109081.64928279446, "task_count": 2048, "final": true } ], "reasoning_tokens": 27505, "output_tokens": 52761, "total_tokens": 20523440, "agent_turns": 190 }, { "model": "kimi-k3", "track": "A", "track_config": "preview-a-8h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "A", "track_config": "preview-a-8h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": null, "status": "finalized", "outcome": "success", "test_nll": 3.161392606646895, "score": 70.795193383331, "seconds_per_task": 0.006348233416929361, "ms": 6.348233416929361, "ppl": 23.60344336409984, "evaluation_seconds": 13.001182037871331, "tasks_per_second": 157.52413850020338, "input_tokens_per_second": 105767.61374423032, "task_count": 2048, "budget_hours": 8.0, "lease_hours": 7.371758672777778, "research_hours": 7.47693991, "leases": 4, "candidates": 3, "first_analysis": 3.2000519050285434, "final_analysis": 3.1733102544637486, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "ca16706d3790", "lease": "lease-0002", "hours": 5.675278240277778, "research_hours": 5.714804040555556, "nll": 3.2000519050285434, "score": 70.43806047589068, "ppl": 24.533803591836982, "seconds_per_task": 0.006408895285176186, "ms": 6.408895285176186, "evaluation_seconds": 13.125417544040829, "tasks_per_second": 156.0331313749198, "input_tokens_per_second": 104766.49564754772, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "3452f193a0c4", "lease": "lease-0003", "hours": 7.366771342222223, "research_hours": 7.435580319166666, "nll": 3.1733102544637486, "score": 70.68509867409274, "ppl": 23.88642377048062, "seconds_per_task": 0.006442496038516765, "ms": 6.442496038516765, "evaluation_seconds": 13.194231886882335, "tasks_per_second": 155.21934263078364, "input_tokens_per_second": 104220.08736765679, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "1dcbf468c8cc", "lease": "lease-0004", "hours": 7.371758672777778, "research_hours": 7.4716574994444445, "nll": 3.1733102544637486, "score": 70.68509867409274, "ppl": 23.88642377048062, "seconds_per_task": 0.006529467514610587, "ms": 6.529467514610587, "evaluation_seconds": 13.372349469922483, "tasks_per_second": 153.1518455007796, "input_tokens_per_second": 102831.8922634297, "task_count": 2048, "final": true } ], "reasoning_tokens": 57309, "output_tokens": 27996, "total_tokens": 6859755, "agent_turns": 94 }, { "model": "glm-5.3", "track": "A", "track_config": "preview-a-8h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "A", "track_config": "preview-a-8h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": null, "status": "finalized", "outcome": "success", "test_nll": 3.247955036400625, "score": 69.99553344362232, "seconds_per_task": 0.006678744517557789, "ms": 6.678744517557789, "ppl": 25.73765350011012, "evaluation_seconds": 13.678068771958351, "tasks_per_second": 149.7287397910033, "input_tokens_per_second": 100533.49072342178, "task_count": 2048, "budget_hours": 8.0, "lease_hours": 5.712417235833334, "research_hours": 5.945129551666667, "leases": 2, "candidates": 1, "first_analysis": 3.2834213768823868, "final_analysis": 3.2834213768823868, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "854b869dfd03", "lease": "lease-0002", "hours": 5.712417235833334, "research_hours": 5.940207638333334, "nll": 3.2834213768823868, "score": 69.66789694159692, "ppl": 26.666854157368842, "seconds_per_task": 0.0067885132943956705, "ms": 6.7885132943956705, "evaluation_seconds": 13.902875226922333, "tasks_per_second": 147.30765878083506, "input_tokens_per_second": 98907.88614265695, "task_count": 2048, "final": true } ], "reasoning_tokens": 95105, "output_tokens": 25630, "total_tokens": 8064366, "agent_turns": 74 }, { "model": "gpt-5.6-luna", "track": "A", "track_config": "preview-a-8h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "A", "track_config": "preview-a-8h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": null, "status": "finalized", "outcome": "success", "test_nll": 3.7066179119344222, "score": 65.75842586196788, "seconds_per_task": 0.011390191824148133, "ms": 11.390191824148133, "ppl": 40.71586874584725, "evaluation_seconds": 23.327112855855376, "tasks_per_second": 87.79483396231473, "input_tokens_per_second": 58948.74382857169, "task_count": 2048, "budget_hours": 8.0, "lease_hours": 1.9637719319444444, "research_hours": 1.9801401305555555, "leases": 1, "candidates": 1, "first_analysis": 3.793098750428555, "final_analysis": 3.793098750428555, "reasoning_effort": "medium", "points": [ { "index": 1, "candidate_id": "f7e964c6a38c", "lease": "lease-0001", "hours": 1.9637719319444444, "research_hours": 1.976775925277778, "nll": 3.793098750428555, "score": 64.95951965874647, "ppl": 44.39375251555291, "seconds_per_task": 0.01147692753318097, "ms": 11.47692753318097, "evaluation_seconds": 23.504747587954625, "tasks_per_second": 87.1313334608847, "input_tokens_per_second": 58503.24471064278, "task_count": 2048, "final": true } ], "reasoning_tokens": 1157, "output_tokens": 4944, "total_tokens": 951006, "agent_turns": 21 }, { "model": "gpt-5.6-terra", "track": "A", "track_config": "preview-a-8h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "A", "track_config": "preview-a-8h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": null, "status": "finalized", "outcome": "success", "test_nll": 3.64037417043656, "score": 66.37038264833541, "seconds_per_task": 0.02178464821781745, "ms": 21.78464821781745, "ppl": 38.10609223140406, "evaluation_seconds": 44.614959550090134, "tasks_per_second": 45.903885617125084, "input_tokens_per_second": 30821.590199048424, "task_count": 2048, "budget_hours": 8.0, "lease_hours": 7.567380592777778, "research_hours": 7.586737426666667, "leases": 2, "candidates": 1, "first_analysis": 3.715562529614639, "final_analysis": 3.715562529614639, "reasoning_effort": "medium", "points": [ { "index": 1, "candidate_id": "0888849521bc", "lease": "lease-0002", "hours": 7.567380592777778, "research_hours": 7.580936770555555, "nll": 3.715562529614639, "score": 65.67579587508756, "ppl": 41.08169025306201, "seconds_per_task": 0.022030572419453165, "ms": 22.030572419453165, "evaluation_seconds": 45.11861231504008, "tasks_per_second": 45.391466956028445, "input_tokens_per_second": 30477.53309428835, "task_count": 2048, "final": true } ], "reasoning_tokens": 2629, "output_tokens": 7189, "total_tokens": 656265, "agent_turns": 27 }, { "model": "gpt-5.6-sol", "track": "A", "track_config": "preview-a-8h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "A", "track_config": "preview-a-8h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": null, "status": "finalized", "outcome": "success", "test_nll": 3.153931641503035, "score": 70.8641174529774, "seconds_per_task": 0.005515821313451852, "ms": 5.515821313451852, "ppl": 23.427994219739112, "evaluation_seconds": 11.296402049949393, "tasks_per_second": 181.29666339285214, "input_tokens_per_second": 121729.37842683816, "task_count": 2048, "budget_hours": 8.0, "lease_hours": 6.843067181388888, "research_hours": 6.8567462425, "leases": 1, "candidates": 1, "first_analysis": 3.1705159381380508, "final_analysis": 3.1705159381380508, "reasoning_effort": "medium", "points": [ { "index": 1, "candidate_id": "e5344a8a2605", "lease": "lease-0001", "hours": 6.843067181388888, "research_hours": 6.854362885555555, "nll": 3.1705159381380508, "score": 70.7109124460824, "ppl": 23.81977071480709, "seconds_per_task": 0.005449867834386168, "ms": 5.449867834386168, "evaluation_seconds": 11.161329324822873, "tasks_per_second": 183.49068828613755, "input_tokens_per_second": 123202.52901612349, "task_count": 2048, "final": true } ], "reasoning_tokens": 2565, "output_tokens": 6781, "total_tokens": 891949, "agent_turns": 22 }, { "model": "claude-sonnet-5", "track": "A", "track_config": "preview-a-8h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "A", "track_config": "preview-a-8h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": null, "status": "finalized", "outcome": "success", "test_nll": 3.4941142968437533, "score": 67.72152496298354, "seconds_per_task": 0.005543276047319523, "ms": 5.543276047319523, "ppl": 32.92111669940808, "evaluation_seconds": 11.352629344910383, "tasks_per_second": 180.39873740070271, "input_tokens_per_second": 121126.47724348433, "task_count": 2048, "budget_hours": 8.0, "lease_hours": 6.537855405, "research_hours": 6.558599476944444, "leases": 2, "candidates": 1, "first_analysis": 3.551453179774088, "final_analysis": 3.551453179774088, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "693aaf436cbd", "lease": "lease-0002", "hours": 6.5378554047222215, "research_hours": 6.552582671666666, "nll": 3.551453179774088, "score": 67.19183087055242, "ppl": 34.86394427244461, "seconds_per_task": 0.005645032651386828, "ms": 5.645032651386828, "evaluation_seconds": 11.561026870040223, "tasks_per_second": 177.14689387214221, "input_tokens_per_second": 118943.0675542765, "task_count": 2048, "final": true } ], "reasoning_tokens": 47015, "output_tokens": 30585, "total_tokens": 7086043, "agent_turns": 88 }, { "model": "claude-opus-5", "track": "A", "track_config": "preview-a-8h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "A", "track_config": "preview-a-8h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": null, "status": "finalized", "outcome": "success", "test_nll": 2.916379053399159, "score": 73.05861787101352, "seconds_per_task": 0.005186905573623335, "ms": 5.186905573623335, "ppl": 18.474271851135924, "evaluation_seconds": 10.62278261478059, "tasks_per_second": 192.7931761636921, "input_tokens_per_second": 129448.56822040901, "task_count": 2048, "budget_hours": 8.0, "lease_hours": 6.791675280833333, "research_hours": 6.829935034999999, "leases": 2, "candidates": 2, "first_analysis": 4.661999963745717, "final_analysis": 2.9361170224208113, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "fdd09e18ef04", "lease": "lease-0001", "hours": 0.7064001594444445, "research_hours": 0.7185653572222223, "nll": 4.661999963745717, "score": 56.93264825702211, "ppl": 105.84756189662814, "seconds_per_task": 0.005142097363773246, "ms": 5.142097363773246, "evaluation_seconds": 10.531015401007608, "tasks_per_second": 194.4731749042972, "input_tokens_per_second": 130576.58237480406, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "d13371377c0e", "lease": "lease-0002", "hours": 6.791675280833333, "research_hours": 6.826158202499999, "nll": 2.9361170224208113, "score": 72.87627937655662, "ppl": 18.842538922715907, "seconds_per_task": 0.004996623987267412, "ms": 4.996623987267412, "evaluation_seconds": 10.23308592592366, "tasks_per_second": 200.13513175060564, "input_tokens_per_second": 134378.23252479726, "task_count": 2048, "final": true } ], "reasoning_tokens": 67332, "output_tokens": 62565, "total_tokens": 9089014, "agent_turns": 93 }, { "model": "deepseek-v4-pro", "track": "A", "track_config": "preview-a-8h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "A", "track_config": "preview-a-8h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": null, "status": "finalized", "outcome": "success", "test_nll": 3.1074060893991597, "score": 71.29391846822321, "seconds_per_task": 0.006002029170872447, "ms": 6.002029170872447, "ppl": 22.362961579383267, "evaluation_seconds": 12.292155741946772, "tasks_per_second": 166.61031986531336, "input_tokens_per_second": 111868.41664456634, "task_count": 2048, "budget_hours": 8.0, "lease_hours": 7.57606954, "research_hours": 7.794683673055555, "leases": 8, "candidates": 4, "first_analysis": 10.836687763656013, "final_analysis": 3.191800861126631, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "4496ac61c6b7", "lease": "lease-0004", "hours": 0.3996331513888889, "research_hours": 0.5588556827777778, "nll": 10.836687763656013, "score": 0.0, "ppl": 50852.66268652917, "seconds_per_task": 0.003965667229522296, "ms": 3.965667229522296, "evaluation_seconds": 8.121686486061662, "tasks_per_second": 252.1643754058658, "input_tokens_per_second": 169312.617811576, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "d2bd202f4559", "lease": "lease-0006", "hours": 3.447408520555556, "research_hours": 3.6452925708333335, "nll": 3.4145236583375556, "score": 68.45677980010296, "ppl": 30.40246401758441, "seconds_per_task": 0.006291344190003656, "ms": 6.291344190003656, "evaluation_seconds": 12.884672901127487, "tasks_per_second": 158.94854419011193, "input_tokens_per_second": 106724.01313964828, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "0c9f7d59f00e", "lease": "lease-0007", "hours": 6.7680912475, "research_hours": 6.974888850555556, "nll": 3.1799817861989528, "score": 70.62346735574462, "ppl": 24.046315573268107, "seconds_per_task": 0.006129045676289024, "ms": 6.129045676289024, "evaluation_seconds": 12.552285545039922, "tasks_per_second": 163.15753753779717, "input_tokens_per_second": 109550.0891105347, "task_count": 2048, "final": false }, { "index": 4, "candidate_id": "eb50bfc4d90a", "lease": "lease-0008", "hours": 7.576069539722223, "research_hours": 7.791797518055556, "nll": 3.191800861126631, "score": 70.51428325854485, "ppl": 24.332206936465926, "seconds_per_task": 0.00612039860538971, "ms": 6.12039860538971, "evaluation_seconds": 12.534576343838125, "tasks_per_second": 163.3880510853306, "input_tokens_per_second": 109704.86455060665, "task_count": 2048, "final": true } ], "reasoning_tokens": 268880, "output_tokens": 29239, "total_tokens": 39382351, "agent_turns": 159 }, { "model": "deepseek-v4-flash", "track": "A", "track_config": "preview-a-8h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "A", "track_config": "preview-a-8h" }, "challenge_manifest_sha256": "a1940190ba00d0290dd7d6e39e330864fc8ddd36e4ffd47c7c0dd1c1e23d9b84", "max_leases": null, "status": "finalized", "outcome": "success", "test_nll": 3.0783356460307036, "score": 71.562469952481, "seconds_per_task": 0.005576860892574587, "ms": 5.576860892574587, "ppl": 21.722218832285126, "evaluation_seconds": 11.421411107992753, "tasks_per_second": 179.3123442134747, "input_tokens_per_second": 120397.03211783491, "task_count": 2048, "budget_hours": 8.0, "lease_hours": 7.683254876666666, "research_hours": 7.720291221388888, "leases": 5, "candidates": 3, "first_analysis": 3.1922240685001784, "final_analysis": 3.0933389509519023, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "9a7170c7a785", "lease": "lease-0002", "hours": 6.160158230277777, "research_hours": 6.172569628611111, "nll": 3.1922240685001784, "score": 70.51037368733968, "ppl": 24.342506685167187, "seconds_per_task": 0.005620221803269487, "ms": 5.620221803269487, "evaluation_seconds": 11.51021425309591, "tasks_per_second": 177.92892078000617, "input_tokens_per_second": 119468.14974622539, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "567c63cfc203", "lease": "lease-0003", "hours": 7.322455534166666, "research_hours": 7.342337705833334, "nll": 3.104102615758474, "score": 71.3244358132175, "ppl": 22.289208013830294, "seconds_per_task": 0.0054669981797133005, "ms": 5.4669981797133005, "evaluation_seconds": 11.19641227205284, "tasks_per_second": 182.915736776858, "input_tokens_per_second": 122816.4850121116, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "e8625f49cf3c", "lease": "lease-0004", "hours": 7.680716035833333, "research_hours": 7.710738369166666, "nll": 3.0933389509519023, "score": 71.42387007788355, "ppl": 22.050581004406418, "seconds_per_task": 0.005568542564446943, "ms": 5.568542564446943, "evaluation_seconds": 11.40437517198734, "tasks_per_second": 179.58020225698286, "input_tokens_per_second": 120576.88205292291, "task_count": 2048, "final": true } ], "reasoning_tokens": 50692, "output_tokens": 18937, "total_tokens": 5574220, "agent_turns": 86 }, { "model": "gpt-6-astra", "track": "B", "track_config": "preview-b-12x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-12x1h" }, "challenge_manifest_sha256": "051cc6cf3861ea4939ed2ff4632449af93aa764c7e2e4b83b870aaad04b945bc", "max_leases": 12, "status": "finalized", "outcome": "success", "test_nll": 2.9798472947759014, "score": 72.47230103336082, "seconds_per_task": 0.005764507180629153, "ms": 5.764507180629153, "ppl": 19.684810441846935, "evaluation_seconds": 11.805710705928504, "tasks_per_second": 173.4753672196584, "input_tokens_per_second": 116477.86687754939, "task_count": 2048, "budget_hours": 12.0, "lease_hours": 11.583489100555555, "research_hours": 11.71038553888889, "leases": 12, "candidates": 12, "first_analysis": 3.5047896327052794, "final_analysis": 3.0083542582488505, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "7490e13edb12", "lease": "lease-0001", "hours": 0.9729784102777778, "research_hours": 0.9817937838888888, "nll": 3.5047896327052794, "score": 67.62290667724821, "ppl": 33.274443261469635, "seconds_per_task": 0.0056132210200985355, "ms": 5.6132210200985355, "evaluation_seconds": 11.4958766491618, "tasks_per_second": 178.15083290314584, "input_tokens_per_second": 119617.14986740598, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "2fb9d0a5cb99", "lease": "lease-0002", "hours": 1.9324687005555556, "research_hours": 1.9506840449999998, "nll": 3.3595027244865596, "score": 68.96506078032934, "ppl": 28.774878278514546, "seconds_per_task": 0.005895656113352743, "ms": 5.895656113352743, "evaluation_seconds": 12.074303720146418, "tasks_per_second": 169.61640583736826, "input_tokens_per_second": 113886.81549442794, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "c3c16a364a4f", "lease": "lease-0003", "hours": 2.8925760422222226, "research_hours": 2.917775905555555, "nll": 3.257579779839019, "score": 69.90662048473759, "ppl": 25.986567758165307, "seconds_per_task": 0.005554823182706059, "ms": 5.554823182706059, "evaluation_seconds": 11.376277878182009, "tasks_per_second": 180.0237320088459, "input_tokens_per_second": 120874.68456068946, "task_count": 2048, "final": false }, { "index": 4, "candidate_id": "9f6be959921c", "lease": "lease-0004", "hours": 3.8526311205555555, "research_hours": 3.8888481061111113, "nll": 3.2209011296548034, "score": 70.24545625076621, "ppl": 25.05068392754073, "seconds_per_task": 0.00560117136228655, "ms": 5.60117136228655, "evaluation_seconds": 11.471198949962854, "tasks_per_second": 178.5340842690756, "input_tokens_per_second": 119874.47920641745, "task_count": 2048, "final": false }, { "index": 5, "candidate_id": "e9a45d944016", "lease": "lease-0005", "hours": 4.813551375277778, "research_hours": 4.857600954722222, "nll": 3.185727039147054, "score": 70.5703929603151, "ppl": 24.184865359386702, "seconds_per_task": 0.0056192266586094775, "ms": 5.6192266586094775, "evaluation_seconds": 11.50817619683221, "tasks_per_second": 177.96043134651876, "input_tokens_per_second": 119489.3071222282, "task_count": 2048, "final": false }, { "index": 6, "candidate_id": "fd698b9068af", "lease": "lease-0006", "hours": 5.774112215, "research_hours": 5.826728645555556, "nll": 3.170551045326791, "score": 70.71058812740846, "ppl": 23.8206069746726, "seconds_per_task": 0.005561238094287546, "ms": 5.561238094287546, "evaluation_seconds": 11.389415617100894, "tasks_per_second": 179.81607387520253, "input_tokens_per_second": 120735.2551025813, "task_count": 2048, "final": false }, { "index": 7, "candidate_id": "a827cdf34b35", "lease": "lease-0007", "hours": 6.732917225277778, "research_hours": 6.796826123055555, "nll": 3.13942773509017, "score": 70.99810390599922, "ppl": 23.0906491088255, "seconds_per_task": 0.005890129996487303, "ms": 5.890129996487303, "evaluation_seconds": 12.062986232805997, "tasks_per_second": 169.77553986013382, "input_tokens_per_second": 113993.6640448386, "task_count": 2048, "final": false }, { "index": 8, "candidate_id": "09700d1d14b9", "lease": "lease-0008", "hours": 7.695833071666667, "research_hours": 7.767009400277778, "nll": 3.095501247949304, "score": 71.4038948728033, "ppl": 22.098312495735904, "seconds_per_task": 0.00595213110500481, "ms": 5.95213110500481, "evaluation_seconds": 12.18996450304985, "tasks_per_second": 168.00705198834694, "input_tokens_per_second": 112806.2349694257, "task_count": 2048, "final": false }, { "index": 9, "candidate_id": "b03e3cf163d6", "lease": "lease-0009", "hours": 8.670947354166668, "research_hours": 8.749919761666666, "nll": 3.073820335285254, "score": 71.60418219564173, "ppl": 21.624357367904363, "seconds_per_task": 0.005795558878503471, "ms": 5.795558878503471, "evaluation_seconds": 11.869304583175108, "tasks_per_second": 172.5459133387702, "input_tokens_per_second": 115853.7966874005, "task_count": 2048, "final": false }, { "index": 10, "candidate_id": "c67ccb5fdf3d", "lease": "lease-0010", "hours": 9.634433482777778, "research_hours": 9.721296252777778, "nll": 3.0534608980746825, "score": 71.79226178604401, "ppl": 21.188549081260742, "seconds_per_task": 0.0060774398266403296, "ms": 6.07743982664033, "evaluation_seconds": 12.446596764959395, "tasks_per_second": 164.54297015274773, "input_tokens_per_second": 110480.32052193554, "task_count": 2048, "final": false }, { "index": 11, "candidate_id": "3592100221cf", "lease": "lease-0011", "hours": 10.608175547777778, "research_hours": 10.717140435833333, "nll": 3.0216443582836234, "score": 72.08618158893587, "ppl": 20.525014427953895, "seconds_per_task": 0.0053606049394829824, "ms": 5.3606049394829824, "evaluation_seconds": 10.978518916061148, "tasks_per_second": 186.5461102411415, "input_tokens_per_second": 125254.05389503643, "task_count": 2048, "final": false }, { "index": 12, "candidate_id": "3eac602d7a2e", "lease": "lease-0012", "hours": 11.583489100555555, "research_hours": 11.705513757222223, "nll": 3.0083542582488505, "score": 72.20895495173036, "ppl": 20.25403956296183, "seconds_per_task": 0.005480467770553332, "ms": 5.480467770553332, "evaluation_seconds": 11.223997994093224, "tasks_per_second": 182.4661765867908, "input_tokens_per_second": 122514.63344199334, "task_count": 2048, "final": true } ], "reasoning_tokens": 7079, "output_tokens": 14590, "total_tokens": 6403698, "agent_turns": 90 }, { "model": "claude-fable-5-1", "track": "B", "track_config": "preview-b-12x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-12x1h" }, "challenge_manifest_sha256": "051cc6cf3861ea4939ed2ff4632449af93aa764c7e2e4b83b870aaad04b945bc", "max_leases": 12, "status": "finalized", "outcome": "success", "test_nll": 2.8687905812539505, "score": 73.49823809510762, "seconds_per_task": 0.005295427327723701, "ms": 5.295427327723701, "ppl": 17.615700552981316, "evaluation_seconds": 10.845035167178139, "tasks_per_second": 188.84217233321212, "input_tokens_per_second": 126795.71608598111, "task_count": 2048, "budget_hours": 12.0, "lease_hours": 10.569369016666666, "research_hours": 10.664261931111112, "leases": 12, "candidates": 12, "first_analysis": 3.8086576881528513, "final_analysis": 2.894117040934942, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "27477d6cc81c", "lease": "lease-0001", "hours": 0.7952009336111111, "research_hours": 0.8127404938888888, "nll": 3.8086576881528513, "score": 64.81578687261813, "ppl": 45.08987355518571, "seconds_per_task": 0.006142569972212186, "ms": 6.142569972212186, "evaluation_seconds": 12.579983303090557, "tasks_per_second": 162.79830828526318, "input_tokens_per_second": 109308.8891192864, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "52c45a8c3ad5", "lease": "lease-0002", "hours": 1.688287758333333, "research_hours": 1.7140195144444446, "nll": 3.5593511099377015, "score": 67.11887013716698, "ppl": 35.140387497197786, "seconds_per_task": 0.006000811380886262, "ms": 6.000811380886262, "evaluation_seconds": 12.289661708055064, "tasks_per_second": 166.64413135616832, "input_tokens_per_second": 111891.11894745726, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "e95680128e53", "lease": "lease-0003", "hours": 2.6085974566666663, "research_hours": 2.640416853888889, "nll": 3.284571573625453, "score": 69.65727147439561, "ppl": 26.6975439324247, "seconds_per_task": 0.005885664631819054, "ms": 5.885664631819054, "evaluation_seconds": 12.053841165965423, "tasks_per_second": 169.90434599243125, "input_tokens_per_second": 114080.14931229306, "task_count": 2048, "final": false }, { "index": 4, "candidate_id": "8072ab436213", "lease": "lease-0004", "hours": 3.5185296066666667, "research_hours": 3.5563300749999995, "nll": 3.125570600187011, "score": 71.12611551210499, "ppl": 22.77288559976363, "seconds_per_task": 0.005214940570340332, "ms": 5.214940570340332, "evaluation_seconds": 10.680198288057, "tasks_per_second": 191.756739412802, "input_tokens_per_second": 128752.66571948324, "task_count": 2048, "final": false }, { "index": 5, "candidate_id": "ff3d5f5a7eeb", "lease": "lease-0005", "hours": 4.412784115555556, "research_hours": 4.45665199, "nll": 3.070586983103579, "score": 71.63405176166489, "ppl": 21.554551119740093, "seconds_per_task": 0.005104206349187734, "ms": 5.104206349187734, "evaluation_seconds": 10.45341460313648, "tasks_per_second": 195.91684418462756, "input_tokens_per_second": 131545.91606721588, "task_count": 2048, "final": false }, { "index": 6, "candidate_id": "4283d2b22a64", "lease": "lease-0006", "hours": 5.316528137222223, "research_hours": 5.366587822777778, "nll": 3.029124211793015, "score": 72.0170830293949, "ppl": 20.679114132294096, "seconds_per_task": 0.005099856678157266, "ms": 5.099856678157266, "evaluation_seconds": 10.444506476866081, "tasks_per_second": 196.08394178664065, "input_tokens_per_second": 131658.11166336754, "task_count": 2048, "final": false }, { "index": 7, "candidate_id": "a90e0748b6b3", "lease": "lease-0007", "hours": 6.2234843699999995, "research_hours": 6.279726836944445, "nll": 3.0005903650624366, "score": 72.2806774573834, "ppl": 20.0973982233515, "seconds_per_task": 0.005023196350634862, "ms": 5.023196350634862, "evaluation_seconds": 10.287506126100197, "tasks_per_second": 199.07643066224435, "input_tokens_per_second": 133667.3809127807, "task_count": 2048, "final": false }, { "index": 8, "candidate_id": "0e25a05bc6b3", "lease": "lease-0008", "hours": 7.128240450555555, "research_hours": 7.190635310277778, "nll": 2.977362054925696, "score": 72.49525957038931, "ppl": 19.635949706964524, "seconds_per_task": 0.005415361600626056, "ms": 5.415361600626056, "evaluation_seconds": 11.090660558082163, "tasks_per_second": 184.65987569221463, "input_tokens_per_second": 123987.56528509136, "task_count": 2048, "final": false }, { "index": 9, "candidate_id": "76c764267970", "lease": "lease-0009", "hours": 8.001568577777778, "research_hours": 8.070287231388889, "nll": 2.9608073673150512, "score": 72.6481910504123, "ppl": 19.313558598452026, "seconds_per_task": 0.0051048328379010854, "ms": 5.104832837901085, "evaluation_seconds": 10.454697652021423, "tasks_per_second": 195.8928003627171, "input_tokens_per_second": 131529.77214354186, "task_count": 2048, "final": false }, { "index": 10, "candidate_id": "3141bcc7ce32", "lease": "lease-0010", "hours": 8.911891596944445, "research_hours": 8.986859668333333, "nll": 2.9312602234692275, "score": 72.92114627282848, "ppl": 18.75124637366357, "seconds_per_task": 0.0053020091264670555, "ms": 5.3020091264670555, "evaluation_seconds": 10.85851469100453, "tasks_per_second": 188.6077477702006, "input_tokens_per_second": 126638.31464345407, "task_count": 2048, "final": false }, { "index": 11, "candidate_id": "0ce223b87528", "lease": "lease-0011", "hours": 9.821834835277778, "research_hours": 9.903271934444444, "nll": 2.9052824789774854, "score": 73.16112754014195, "ppl": 18.270403928360032, "seconds_per_task": 0.005241267212795719, "ms": 5.241267212795719, "evaluation_seconds": 10.734115251805633, "tasks_per_second": 190.7935541921349, "input_tokens_per_second": 128105.94704288158, "task_count": 2048, "final": false }, { "index": 12, "candidate_id": "4eae8dc3718d", "lease": "lease-0012", "hours": 10.569369016666666, "research_hours": 10.657171010555556, "nll": 2.894117040934942, "score": 73.26427336838779, "ppl": 18.067541496757926, "seconds_per_task": 0.005488741457497781, "ms": 5.488741457497781, "evaluation_seconds": 11.240942504955456, "tasks_per_second": 182.1911284660659, "input_tokens_per_second": 122329.95581943412, "task_count": 2048, "final": true } ], "reasoning_tokens": 44988, "output_tokens": 59387, "total_tokens": 11145438, "agent_turns": 116 }, { "model": "gpt-5.6-luna", "track": "B", "track_config": "preview-b-12x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-12x1h" }, "challenge_manifest_sha256": "051cc6cf3861ea4939ed2ff4632449af93aa764c7e2e4b83b870aaad04b945bc", "max_leases": 12, "status": "finalized", "outcome": "success", "test_nll": 5.671991233589605, "score": 47.60239308457137, "seconds_per_task": 0.003143889860893978, "ms": 3.143889860893978, "ppl": 290.61263627569065, "evaluation_seconds": 6.438686435110867, "tasks_per_second": 318.07730049285055, "input_tokens_per_second": 213569.02744966836, "task_count": 2048, "budget_hours": 12.0, "lease_hours": 0.05856560444444444, "research_hours": 0.13004843416666667, "leases": 1, "candidates": 1, "first_analysis": 5.751995881837667, "final_analysis": 5.751995881837667, "reasoning_effort": "medium", "points": [ { "index": 1, "candidate_id": "f386200aa94a", "lease": "lease-0001", "hours": 0.05856560444444444, "research_hours": 0.1263203386111111, "nll": 5.751995881837667, "score": 46.86331364356594, "ppl": 314.8183739308777, "seconds_per_task": 0.003170872048031015, "ms": 3.170872048031015, "evaluation_seconds": 6.493945954367518, "tasks_per_second": 315.37065666871047, "input_tokens_per_second": 211751.6852869973, "task_count": 2048, "final": true } ], "reasoning_tokens": 1879, "output_tokens": 4862, "total_tokens": 652747, "agent_turns": 25 }, { "model": "gpt-5.6-terra", "track": "B", "track_config": "preview-b-12x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-12x1h" }, "challenge_manifest_sha256": "051cc6cf3861ea4939ed2ff4632449af93aa764c7e2e4b83b870aaad04b945bc", "max_leases": 12, "status": "finalized", "outcome": "success", "test_nll": 4.064540093997045, "score": 62.451956400067665, "seconds_per_task": 0.0054828894208185375, "ms": 5.4828894208185375, "ppl": 58.23811830232459, "evaluation_seconds": 11.228957533836365, "tasks_per_second": 182.38558600197166, "input_tokens_per_second": 122460.52190119885, "task_count": 2048, "budget_hours": 12.0, "lease_hours": 1.6023312075, "research_hours": 1.6276828983333334, "leases": 2, "candidates": 1, "first_analysis": 4.078436688207989, "final_analysis": 4.078436688207989, "reasoning_effort": "medium", "points": [ { "index": 1, "candidate_id": "3ceb187a21d1", "lease": "lease-0002", "hours": 1.6023312075, "research_hours": 1.6231616111111111, "nll": 4.078436688207989, "score": 62.323580270602555, "ppl": 59.05307927587528, "seconds_per_task": 0.00535395826182139, "ms": 5.35395826182139, "evaluation_seconds": 10.964906520210207, "tasks_per_second": 186.77769812493923, "input_tokens_per_second": 125409.55068476389, "task_count": 2048, "final": true } ], "reasoning_tokens": 1670, "output_tokens": 5090, "total_tokens": 721410, "agent_turns": 27 }, { "model": "gpt-5.6-sol", "track": "B", "track_config": "preview-b-12x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-12x1h" }, "challenge_manifest_sha256": "051cc6cf3861ea4939ed2ff4632449af93aa764c7e2e4b83b870aaad04b945bc", "max_leases": 12, "status": "finalized", "outcome": "success", "test_nll": 3.1067920314864157, "score": 71.29959110836141, "seconds_per_task": 0.00634530802426525, "ms": 6.34530802426525, "ppl": 22.34923364117896, "evaluation_seconds": 12.995190833695233, "tasks_per_second": 157.59676223374422, "input_tokens_per_second": 105816.37604231964, "task_count": 2048, "budget_hours": 12.0, "lease_hours": 10.3661413825, "research_hours": 10.432393133333333, "leases": 12, "candidates": 1, "first_analysis": 3.156662634956278, "final_analysis": 3.156662634956278, "reasoning_effort": "medium", "points": [ { "index": 1, "candidate_id": "31f2ac5e86db", "lease": "lease-0012", "hours": 10.366141382499999, "research_hours": 10.426934074722222, "nll": 3.156662634956278, "score": 70.83888865491363, "ppl": 23.492063364963208, "seconds_per_task": 0.00659322065212109, "ms": 6.59322065212109, "evaluation_seconds": 13.502915895543993, "tasks_per_second": 151.670943953361, "input_tokens_per_second": 101837.5594306848, "task_count": 2048, "final": true } ], "reasoning_tokens": 4322, "output_tokens": 9430, "total_tokens": 2030271, "agent_turns": 59 }, { "model": "claude-sonnet-5", "track": "B", "track_config": "preview-b-12x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-12x1h" }, "challenge_manifest_sha256": "051cc6cf3861ea4939ed2ff4632449af93aa764c7e2e4b83b870aaad04b945bc", "max_leases": 12, "status": "finalized", "outcome": "success", "test_nll": 3.101838232154234, "score": 71.34535408990632, "seconds_per_task": 0.0071573377445020014, "ms": 7.157337744502001, "ppl": 22.238793796756852, "evaluation_seconds": 14.658227700740099, "tasks_per_second": 139.71675442704412, "input_tokens_per_second": 93811.06830060844, "task_count": 2048, "budget_hours": 12.0, "lease_hours": 7.2587802508333334, "research_hours": 7.371615345833334, "leases": 12, "candidates": 2, "first_analysis": 3.1226833883157874, "final_analysis": 3.1027547741720554, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "b5aed4c14167", "lease": "lease-0010", "hours": 6.196295706388889, "research_hours": 6.274013622222222, "nll": 3.1226833883157874, "score": 71.15278744908086, "ppl": 22.707230280135608, "seconds_per_task": 0.007350381064497924, "ms": 7.350381064497924, "evaluation_seconds": 15.053580420091748, "tasks_per_second": 136.04736832352324, "input_tokens_per_second": 91347.30486872565, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "f439a631afa8", "lease": "lease-0012", "hours": 7.2587802508333334, "research_hours": 7.368029894722222, "nll": 3.1027547741720554, "score": 71.33688711483646, "ppl": 22.25918592939291, "seconds_per_task": 0.007167745921833557, "ms": 7.167745921833557, "evaluation_seconds": 14.679543647915125, "tasks_per_second": 139.51387380430378, "input_tokens_per_second": 93674.84664247722, "task_count": 2048, "final": true } ], "reasoning_tokens": 51449, "output_tokens": 33550, "total_tokens": 8792481, "agent_turns": 97 }, { "model": "claude-opus-5", "track": "B", "track_config": "preview-b-12x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-12x1h" }, "challenge_manifest_sha256": "051cc6cf3861ea4939ed2ff4632449af93aa764c7e2e4b83b870aaad04b945bc", "max_leases": 12, "status": "finalized", "outcome": "success", "test_nll": 2.843612031836784, "score": 73.73083643328002, "seconds_per_task": 0.006220763531018747, "ms": 6.220763531018747, "ppl": 17.177700014491887, "evaluation_seconds": 12.740123711526394, "tasks_per_second": 160.75197120316105, "input_tokens_per_second": 107934.90166472243, "task_count": 2048, "budget_hours": 12.0, "lease_hours": 10.185028385277777, "research_hours": 10.254238075277778, "leases": 12, "candidates": 3, "first_analysis": 3.00897342493163, "final_analysis": 2.8259236885421797, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "2618f24b8925", "lease": "lease-0006", "hours": 5.819389105, "research_hours": 5.849830659999999, "nll": 3.00897342493163, "score": 72.20323511699803, "ppl": 20.266584072621367, "seconds_per_task": 0.006662470521860087, "ms": 6.662470521860087, "evaluation_seconds": 13.644739628769457, "tasks_per_second": 150.0944727213309, "input_tokens_per_second": 100779.05752782861, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "4296490ca675", "lease": "lease-0011", "hours": 10.100326319444443, "research_hours": 10.157837314722224, "nll": 2.8259236885421797, "score": 73.89424057492381, "ppl": 16.876526442881715, "seconds_per_task": 0.006083168002987804, "ms": 6.083168002987804, "evaluation_seconds": 12.458328070119023, "tasks_per_second": 164.38802931446915, "input_tokens_per_second": 110376.28743283388, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "ee3df937da65", "lease": "lease-0012", "hours": 10.185028384999999, "research_hours": 10.251176719999998, "nll": 2.8259236885421797, "score": 73.89424057492381, "ppl": 16.876526442881715, "seconds_per_task": 0.006379253887189407, "ms": 6.379253887189407, "evaluation_seconds": 13.064711960963905, "tasks_per_second": 156.75814408455585, "input_tokens_per_second": 105253.29636877397, "task_count": 2048, "final": true } ], "reasoning_tokens": 63112, "output_tokens": 49984, "total_tokens": 9082309, "agent_turns": 101 }, { "model": "deepseek-v4-flash", "track": "B", "track_config": "preview-b-12x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-12x1h" }, "challenge_manifest_sha256": "051cc6cf3861ea4939ed2ff4632449af93aa764c7e2e4b83b870aaad04b945bc", "max_leases": 12, "status": "finalized", "outcome": "success", "test_nll": 2.994324602608485, "score": 72.3385602968602, "seconds_per_task": 0.005876307781818468, "ms": 5.876307781818468, "ppl": 19.97186638717022, "evaluation_seconds": 12.034678337164223, "tasks_per_second": 170.1748848305802, "input_tokens_per_second": 114261.79923343268, "task_count": 2048, "budget_hours": 12.0, "lease_hours": 10.24964059611111, "research_hours": 10.357881725, "leases": 12, "candidates": 11, "first_analysis": 3.614510028096719, "final_analysis": 3.0051731386598104, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "c911d107882f", "lease": "lease-0002", "hours": 0.9746226380555556, "research_hours": 0.9914455191666667, "nll": 3.614510028096719, "score": 66.60931446393873, "ppl": 37.13314726875324, "seconds_per_task": 0.006135101407835464, "ms": 6.135101407835464, "evaluation_seconds": 12.56468768324703, "tasks_per_second": 162.99649077077143, "input_tokens_per_second": 109441.95627189985, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "456d034ad512", "lease": "lease-0003", "hours": 1.8544543666666666, "research_hours": 1.8809778636111112, "nll": 3.411208341851638, "score": 68.4874065487835, "ppl": 30.30183712458159, "seconds_per_task": 0.006025114841577306, "ms": 6.025114841577306, "evaluation_seconds": 12.339435195550323, "tasks_per_second": 165.9719401693946, "input_tokens_per_second": 111439.78457748788, "task_count": 2048, "final": false }, { "index": 3, "candidate_id": "78040e69127c", "lease": "lease-0004", "hours": 2.787246035, "research_hours": 2.8265486016666665, "nll": 3.3060687789490206, "score": 69.45868123193297, "ppl": 27.277679824835673, "seconds_per_task": 0.005814222734898067, "ms": 5.814222734898067, "evaluation_seconds": 11.907528161071241, "tasks_per_second": 171.9920349796389, "input_tokens_per_second": 115481.90198664129, "task_count": 2048, "final": false }, { "index": 4, "candidate_id": "ebb548cd6a05", "lease": "lease-0005", "hours": 3.720069257222222, "research_hours": 3.767063778333333, "nll": 3.266834234304885, "score": 69.8211282391841, "ppl": 26.228175515832582, "seconds_per_task": 0.005848404427979403, "ms": 5.848404427979403, "evaluation_seconds": 11.977532268501818, "tasks_per_second": 170.98680713937824, "input_tokens_per_second": 114806.95431864628, "task_count": 2048, "final": false }, { "index": 5, "candidate_id": "9d2f0674fbc2", "lease": "lease-0006", "hours": 4.652918748888888, "research_hours": 4.70792152, "nll": 3.237184261454949, "score": 70.09503339144241, "ppl": 25.46192658846197, "seconds_per_task": 0.0058257962909920025, "ms": 5.8257962909920025, "evaluation_seconds": 11.931230803951621, "tasks_per_second": 171.65035474141553, "input_tokens_per_second": 115252.48506168918, "task_count": 2048, "final": false }, { "index": 6, "candidate_id": "191a0ee5dee1", "lease": "lease-0007", "hours": 5.585693151388889, "research_hours": 5.649783705277778, "nll": 3.204359583612304, "score": 70.3982663295575, "ppl": 24.63971528513866, "seconds_per_task": 0.0057738114046514966, "ms": 5.773811404651497, "evaluation_seconds": 11.824765756726265, "tasks_per_second": 173.19581986941594, "input_tokens_per_second": 116290.16830357096, "task_count": 2048, "final": false }, { "index": 7, "candidate_id": "ded9441862e4", "lease": "lease-0008", "hours": 6.518581679722223, "research_hours": 6.590719441388889, "nll": 3.155327833012518, "score": 70.85121949688407, "ppl": 23.46072703167266, "seconds_per_task": 0.005811055315916747, "ms": 5.811055315916747, "evaluation_seconds": 11.901041286997497, "tasks_per_second": 172.0857822951632, "input_tokens_per_second": 115544.84744980863, "task_count": 2048, "final": false }, { "index": 8, "candidate_id": "eee049515983", "lease": "lease-0009", "hours": 7.451308951111111, "research_hours": 7.531788938055556, "nll": 3.1091322860995847, "score": 71.27797193861083, "ppl": 22.401597787126622, "seconds_per_task": 0.005890916801490675, "ms": 5.890916801490675, "evaluation_seconds": 12.064597609452903, "tasks_per_second": 169.7528642310741, "input_tokens_per_second": 113978.43877715182, "task_count": 2048, "final": false }, { "index": 9, "candidate_id": "b3902e093411", "lease": "lease-0010", "hours": 8.38405085, "research_hours": 8.473508575555556, "nll": 3.0596848605721276, "score": 71.73476509273704, "ppl": 21.320837066804042, "seconds_per_task": 0.005902429093112005, "ms": 5.902429093112005, "evaluation_seconds": 12.088174782693386, "tasks_per_second": 169.42177266762533, "input_tokens_per_second": 113756.13148551868, "task_count": 2048, "final": false }, { "index": 10, "candidate_id": "4319da58e709", "lease": "lease-0011", "hours": 9.316849079166667, "research_hours": 9.414204247222221, "nll": 3.007795899160788, "score": 72.21411304856252, "ppl": 20.24273369256334, "seconds_per_task": 0.005729739699290803, "ms": 5.729739699290803, "evaluation_seconds": 11.734506904147565, "tasks_per_second": 174.52799821321284, "input_tokens_per_second": 117184.6428002841, "task_count": 2048, "final": false }, { "index": 11, "candidate_id": "fde73befb054", "lease": "lease-0012", "hours": 10.24964059611111, "research_hours": 10.354951192777778, "nll": 3.0051731386598104, "score": 72.23834199534751, "ppl": 20.189711412967437, "seconds_per_task": 0.005748589970153262, "ms": 5.748589970153262, "evaluation_seconds": 11.77311225887388, "tasks_per_second": 173.9557013445054, "input_tokens_per_second": 116800.38122150135, "task_count": 2048, "final": true } ], "reasoning_tokens": 26922, "output_tokens": 14237, "total_tokens": 2392353, "agent_turns": 50 }, { "model": "deepseek-v4-pro", "track": "B", "track_config": "preview-b-12x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-12x1h" }, "challenge_manifest_sha256": "051cc6cf3861ea4939ed2ff4632449af93aa764c7e2e4b83b870aaad04b945bc", "max_leases": 12, "status": "finalized", "outcome": "success", "test_nll": 3.245675275360559, "score": 70.01659377639066, "seconds_per_task": 0.08424569574481211, "ms": 84.24569574481211, "ppl": 25.67904463288794, "evaluation_seconds": 172.5351848853752, "tasks_per_second": 11.870042631365893, "input_tokens_per_second": 7969.991749297737, "task_count": 2048, "budget_hours": 12.0, "lease_hours": 11.784627684444445, "research_hours": 11.856942313055555, "leases": 12, "candidates": 1, "first_analysis": 3.2617016712297926, "final_analysis": 3.2617016712297926, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "b788c1bd7f79", "lease": "lease-0012", "hours": 11.784627684444445, "research_hours": 11.853143955277778, "nll": 3.2617016712297926, "score": 69.86854263236665, "ppl": 26.093902627452128, "seconds_per_task": 0.08369561723202423, "ms": 83.69561723202423, "evaluation_seconds": 171.40862409118563, "tasks_per_second": 11.9480569362164, "input_tokens_per_second": 8022.373479110799, "task_count": 2048, "final": true } ], "reasoning_tokens": 52369, "output_tokens": 20308, "total_tokens": 7910151, "agent_turns": 89 }, { "model": "kimi-k3", "track": "B", "track_config": "preview-b-12x1h", "comparison_key": { "challenge_id": "lm-pretrain", "runtime_profile": "h200-sxm-141gb-x1", "track": "B", "track_config": "preview-b-12x1h" }, "challenge_manifest_sha256": "051cc6cf3861ea4939ed2ff4632449af93aa764c7e2e4b83b870aaad04b945bc", "max_leases": 12, "status": "finalized", "outcome": "success", "test_nll": 3.014810705797679, "score": 72.14931057168793, "seconds_per_task": 0.005191842980366346, "ms": 5.191842980366346, "ppl": 20.385231769114487, "evaluation_seconds": 10.632894423790276, "tasks_per_second": 192.60983118742897, "input_tokens_per_second": 129325.46352790935, "task_count": 2048, "budget_hours": 12.0, "lease_hours": 10.914635439166666, "research_hours": 11.099531710555556, "leases": 12, "candidates": 2, "first_analysis": 3.0477444395182536, "final_analysis": 3.0139275311561993, "reasoning_effort": "provider-default", "points": [ { "index": 1, "candidate_id": "7378e66b05ef", "lease": "lease-0010", "hours": 9.229540942500002, "research_hours": 9.355648016111111, "nll": 3.0477444395182536, "score": 71.84507018014277, "ppl": 21.06777115845918, "seconds_per_task": 0.005235196976173029, "ms": 5.235196976173029, "evaluation_seconds": 10.721683407202363, "tasks_per_second": 191.01478025589174, "input_tokens_per_second": 128254.48651806531, "task_count": 2048, "final": false }, { "index": 2, "candidate_id": "e56dbe2e1360", "lease": "lease-0012", "hours": 10.91463543861111, "research_hours": 11.092308199166666, "nll": 3.0139275311561993, "score": 72.15746930039468, "ppl": 20.367235997229727, "seconds_per_task": 0.005304723154040403, "ms": 5.304723154040403, "evaluation_seconds": 10.864073019474745, "tasks_per_second": 188.51125138139182, "input_tokens_per_second": 126573.52334939326, "task_count": 2048, "final": true } ], "reasoning_tokens": 64136, "output_tokens": 19973, "total_tokens": 5241646, "agent_turns": 71 } ] }