{
 "label": "stock",
 "model": "Qwen3.8-Flash-Next",
 "concurrency": 4,
 "effort": "xhigh",
 "n": 60,
 "wall_s": 1917.5051703453064,
 "latency_median_s": 13.417104840278625,
 "latency_mean_s": 101.00422356526057,
 "latency_p90_s": 256.7808928489685,
 "correct": 50,
 "completion_tokens_mean": 3227.4666666666667,
 "length_stops": 0,
 "items": [
  {
   "id": "mmlupro-3593",
   "suite": "mmlupro",
   "latency_s": 13.844464302062988,
   "ttft_s": 3.5439469814300537,
   "completion_tokens": 325,
   "finish": "stop",
   "reasoning_tokens": 316,
   "correct": true
  },
  {
   "id": "mmlupro-6249",
   "suite": "mmlupro",
   "latency_s": 111.20273518562317,
   "ttft_s": 0.5147731304168701,
   "completion_tokens": 3064,
   "finish": "stop",
   "reasoning_tokens": 3057,
   "correct": true
  },
  {
   "id": "mmlupro-6182",
   "suite": "mmlupro",
   "latency_s": 39.803414821624756,
   "ttft_s": 3.5432026386260986,
   "completion_tokens": 920,
   "finish": "stop",
   "reasoning_tokens": 913,
   "correct": true
  },
  {
   "id": "mmlupro-4993",
   "suite": "mmlupro",
   "latency_s": 10.72613525390625,
   "ttft_s": 3.5427284240722656,
   "completion_tokens": 189,
   "finish": "stop",
   "reasoning_tokens": 182,
   "correct": true
  },
  {
   "id": "mmlupro-12095",
   "suite": "mmlupro",
   "latency_s": 1142.4692659378052,
   "ttft_s": 0.48436689376831055,
   "completion_tokens": 36716,
   "finish": "stop",
   "reasoning_tokens": 36709,
   "correct": true
  },
  {
   "id": "mmlupro-2253",
   "suite": "mmlupro",
   "latency_s": 2.805438280105591,
   "ttft_s": 0.24726605415344238,
   "completion_tokens": 98,
   "finish": "stop",
   "reasoning_tokens": 91,
   "correct": true
  },
  {
   "id": "mmlupro-2570",
   "suite": "mmlupro",
   "latency_s": 6.856769561767578,
   "ttft_s": 0.2512969970703125,
   "completion_tokens": 233,
   "finish": "stop",
   "reasoning_tokens": 226,
   "correct": true
  },
  {
   "id": "mmlupro-2192",
   "suite": "mmlupro",
   "latency_s": 232.03433775901794,
   "ttft_s": 0.2850325107574463,
   "completion_tokens": 6324,
   "finish": "stop",
   "reasoning_tokens": 6317,
   "correct": true
  },
  {
   "id": "mmlupro-3354",
   "suite": "mmlupro",
   "latency_s": 15.521983861923218,
   "ttft_s": 0.4101710319519043,
   "completion_tokens": 491,
   "finish": "stop",
   "reasoning_tokens": 484,
   "correct": true
  },
  {
   "id": "mmlupro-4643",
   "suite": "mmlupro",
   "latency_s": 5.923410654067993,
   "ttft_s": 0.3193986415863037,
   "completion_tokens": 197,
   "finish": "stop",
   "reasoning_tokens": 190,
   "correct": true
  },
  {
   "id": "mmlupro-10930",
   "suite": "mmlupro",
   "latency_s": 32.20020055770874,
   "ttft_s": 0.2222907543182373,
   "completion_tokens": 972,
   "finish": "stop",
   "reasoning_tokens": 965,
   "correct": false
  },
  {
   "id": "mmlupro-10659",
   "suite": "mmlupro",
   "latency_s": 18.606993675231934,
   "ttft_s": 0.31696486473083496,
   "completion_tokens": 588,
   "finish": "stop",
   "reasoning_tokens": 581,
   "correct": true
  },
  {
   "id": "mmlupro-4834",
   "suite": "mmlupro",
   "latency_s": 16.131842851638794,
   "ttft_s": 0.42604684829711914,
   "completion_tokens": 458,
   "finish": "stop",
   "reasoning_tokens": 451,
   "correct": true
  },
  {
   "id": "mmlupro-2618",
   "suite": "mmlupro",
   "latency_s": 180.31332802772522,
   "ttft_s": 0.31105971336364746,
   "completion_tokens": 5080,
   "finish": "stop",
   "reasoning_tokens": 5073,
   "correct": true
  },
  {
   "id": "mmlupro-5034",
   "suite": "mmlupro",
   "latency_s": 9.508354902267456,
   "ttft_s": 0.25989580154418945,
   "completion_tokens": 309,
   "finish": "stop",
   "reasoning_tokens": 302,
   "correct": true
  },
  {
   "id": "mmlupro-9174",
   "suite": "mmlupro",
   "latency_s": 12.989745378494263,
   "ttft_s": 0.30300092697143555,
   "completion_tokens": 543,
   "finish": "stop",
   "reasoning_tokens": 536,
   "correct": true
  },
  {
   "id": "mmlupro-4212",
   "suite": "mmlupro",
   "latency_s": 7.356569290161133,
   "ttft_s": 0.2958977222442627,
   "completion_tokens": 319,
   "finish": "stop",
   "reasoning_tokens": 312,
   "correct": true
  },
  {
   "id": "mmlupro-2139",
   "suite": "mmlupro",
   "latency_s": 7.791964054107666,
   "ttft_s": 0.313967227935791,
   "completion_tokens": 182,
   "finish": "stop",
   "reasoning_tokens": 175,
   "correct": true
  },
  {
   "id": "mmlupro-7387",
   "suite": "mmlupro",
   "latency_s": 2.1477112770080566,
   "ttft_s": 0.28412389755249023,
   "completion_tokens": 69,
   "finish": "stop",
   "reasoning_tokens": 62,
   "correct": true
  },
  {
   "id": "mmlupro-10998",
   "suite": "mmlupro",
   "latency_s": 11.79932188987732,
   "ttft_s": 0.40303540229797363,
   "completion_tokens": 360,
   "finish": "stop",
   "reasoning_tokens": 353,
   "correct": true
  },
  {
   "id": "mmlupro-11035",
   "suite": "mmlupro",
   "latency_s": 12.56105661392212,
   "ttft_s": 0.3060636520385742,
   "completion_tokens": 353,
   "finish": "stop",
   "reasoning_tokens": 346,
   "correct": true
  },
  {
   "id": "mmlupro-1452",
   "suite": "mmlupro",
   "latency_s": 536.6673805713654,
   "ttft_s": 0.37474513053894043,
   "completion_tokens": 14796,
   "finish": "stop",
   "reasoning_tokens": 14789,
   "correct": true
  },
  {
   "id": "mmlupro-9852",
   "suite": "mmlupro",
   "latency_s": 14.259297847747803,
   "ttft_s": 0.308635950088501,
   "completion_tokens": 467,
   "finish": "stop",
   "reasoning_tokens": 460,
   "correct": true
  },
  {
   "id": "mmlupro-1840",
   "suite": "mmlupro",
   "latency_s": 825.5370903015137,
   "ttft_s": 0.42742204666137695,
   "completion_tokens": 23118,
   "finish": "stop",
   "reasoning_tokens": 23111,
   "correct": true
  },
  {
   "id": "mmlupro-9244",
   "suite": "mmlupro",
   "latency_s": 18.9079647064209,
   "ttft_s": 0.30471014976501465,
   "completion_tokens": 640,
   "finish": "stop",
   "reasoning_tokens": 633,
   "correct": true
  },
  {
   "id": "mmlupro-11200",
   "suite": "mmlupro",
   "latency_s": 19.007320165634155,
   "ttft_s": 0.27759742736816406,
   "completion_tokens": 497,
   "finish": "stop",
   "reasoning_tokens": 490,
   "correct": false
  },
  {
   "id": "mmlupro-804",
   "suite": "mmlupro",
   "latency_s": 39.4261269569397,
   "ttft_s": 0.32136034965515137,
   "completion_tokens": 1323,
   "finish": "stop",
   "reasoning_tokens": 1316,
   "correct": true
  },
  {
   "id": "mmlupro-11025",
   "suite": "mmlupro",
   "latency_s": 63.350680112838745,
   "ttft_s": 0.27361154556274414,
   "completion_tokens": 1739,
   "finish": "stop",
   "reasoning_tokens": 1732,
   "correct": true
  },
  {
   "id": "mmlupro-8937",
   "suite": "mmlupro",
   "latency_s": 252.01034092903137,
   "ttft_s": 0.26653480529785156,
   "completion_tokens": 7886,
   "finish": "stop",
   "reasoning_tokens": 7879,
   "correct": false
  },
  {
   "id": "mmlupro-8442",
   "suite": "mmlupro",
   "latency_s": 7.05310845375061,
   "ttft_s": 0.34116554260253906,
   "completion_tokens": 213,
   "finish": "stop",
   "reasoning_tokens": 206,
   "correct": true
  },
  {
   "id": "bbh-object_counting-28",
   "suite": "bbh",
   "latency_s": 4.570019006729126,
   "ttft_s": 0.2288684844970703,
   "completion_tokens": 133,
   "finish": "stop",
   "reasoning_tokens": 126,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-75",
   "suite": "bbh",
   "latency_s": 27.583275318145752,
   "ttft_s": 0.31671810150146484,
   "completion_tokens": 823,
   "finish": "stop",
   "reasoning_tokens": 816,
   "correct": false
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-136",
   "suite": "bbh",
   "latency_s": 7.307842493057251,
   "ttft_s": 0.3167073726654053,
   "completion_tokens": 267,
   "finish": "stop",
   "reasoning_tokens": 260,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-139",
   "suite": "bbh",
   "latency_s": 8.389260292053223,
   "ttft_s": 0.23311781883239746,
   "completion_tokens": 225,
   "finish": "stop",
   "reasoning_tokens": 218,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-207",
   "suite": "bbh",
   "latency_s": 351.15043544769287,
   "ttft_s": 0.22583937644958496,
   "completion_tokens": 9678,
   "finish": "stop",
   "reasoning_tokens": 9670,
   "correct": false
  },
  {
   "id": "bbh-movie_recommendation-26",
   "suite": "bbh",
   "latency_s": 44.23190498352051,
   "ttft_s": 0.25229501724243164,
   "completion_tokens": 1359,
   "finish": "stop",
   "reasoning_tokens": 1352,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_five_objects-223",
   "suite": "bbh",
   "latency_s": 8.255655527114868,
   "ttft_s": 0.3073256015777588,
   "completion_tokens": 295,
   "finish": "stop",
   "reasoning_tokens": 288,
   "correct": true
  },
  {
   "id": "bbh-reasoning_about_colored_objects-95",
   "suite": "bbh",
   "latency_s": 3.4743309020996094,
   "ttft_s": 0.26096367835998535,
   "completion_tokens": 114,
   "finish": "stop",
   "reasoning_tokens": 107,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-117",
   "suite": "bbh",
   "latency_s": 256.7808928489685,
   "ttft_s": 0.3252544403076172,
   "completion_tokens": 9629,
   "finish": "stop",
   "reasoning_tokens": 9622,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-67",
   "suite": "bbh",
   "latency_s": 20.25753688812256,
   "ttft_s": 0.31197643280029297,
   "completion_tokens": 671,
   "finish": "stop",
   "reasoning_tokens": 664,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-2",
   "suite": "bbh",
   "latency_s": 201.1933662891388,
   "ttft_s": 0.31523895263671875,
   "completion_tokens": 6098,
   "finish": "stop",
   "reasoning_tokens": 6091,
   "correct": false
  },
  {
   "id": "bbh-multistep_arithmetic_two-202",
   "suite": "bbh",
   "latency_s": 3.8131985664367676,
   "ttft_s": 0.17499637603759766,
   "completion_tokens": 169,
   "finish": "stop",
   "reasoning_tokens": 161,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-125",
   "suite": "bbh",
   "latency_s": 18.292163133621216,
   "ttft_s": 0.3164024353027344,
   "completion_tokens": 631,
   "finish": "stop",
   "reasoning_tokens": 624,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_seven_objects-9",
   "suite": "bbh",
   "latency_s": 6.441089153289795,
   "ttft_s": 0.2632572650909424,
   "completion_tokens": 226,
   "finish": "stop",
   "reasoning_tokens": 219,
   "correct": true
  },
  {
   "id": "bbh-snarks-87",
   "suite": "bbh",
   "latency_s": 12.473942995071411,
   "ttft_s": 0.2550990581512451,
   "completion_tokens": 406,
   "finish": "stop",
   "reasoning_tokens": 399,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-161",
   "suite": "bbh",
   "latency_s": 24.52378273010254,
   "ttft_s": 0.2905101776123047,
   "completion_tokens": 818,
   "finish": "stop",
   "reasoning_tokens": 811,
   "correct": false
  },
  {
   "id": "bbh-ruin_names-76",
   "suite": "bbh",
   "latency_s": 5.531009674072266,
   "ttft_s": 0.2574343681335449,
   "completion_tokens": 172,
   "finish": "stop",
   "reasoning_tokens": 165,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-9",
   "suite": "bbh",
   "latency_s": 7.099632024765015,
   "ttft_s": 0.310727596282959,
   "completion_tokens": 270,
   "finish": "stop",
   "reasoning_tokens": 263,
   "correct": true
  },
  {
   "id": "bbh-web_of_lies-160",
   "suite": "bbh",
   "latency_s": 12.168543815612793,
   "ttft_s": 0.22559833526611328,
   "completion_tokens": 436,
   "finish": "stop",
   "reasoning_tokens": 429,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-33",
   "suite": "bbh",
   "latency_s": 491.6821184158325,
   "ttft_s": 0.46134471893310547,
   "completion_tokens": 17646,
   "finish": "stop",
   "reasoning_tokens": 17639,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-21",
   "suite": "bbh",
   "latency_s": 7.670367479324341,
   "ttft_s": 0.3194589614868164,
   "completion_tokens": 202,
   "finish": "stop",
   "reasoning_tokens": 195,
   "correct": false
  },
  {
   "id": "bbh-reasoning_about_colored_objects-57",
   "suite": "bbh",
   "latency_s": 4.490676641464233,
   "ttft_s": 0.29398107528686523,
   "completion_tokens": 162,
   "finish": "stop",
   "reasoning_tokens": 155,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-197",
   "suite": "bbh",
   "latency_s": 65.0591185092926,
   "ttft_s": 0.21769404411315918,
   "completion_tokens": 1796,
   "finish": "stop",
   "reasoning_tokens": 1789,
   "correct": false
  },
  {
   "id": "bbh-salient_translation_error_detection-203",
   "suite": "bbh",
   "latency_s": 2.9966843128204346,
   "ttft_s": 0.38303685188293457,
   "completion_tokens": 137,
   "finish": "stop",
   "reasoning_tokens": 130,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_three_objects-227",
   "suite": "bbh",
   "latency_s": 2.4859073162078857,
   "ttft_s": 0.2607283592224121,
   "completion_tokens": 87,
   "finish": "stop",
   "reasoning_tokens": 80,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-84",
   "suite": "bbh",
   "latency_s": 758.8237166404724,
   "ttft_s": 0.29180026054382324,
   "completion_tokens": 31588,
   "finish": "stop",
   "reasoning_tokens": 31581,
   "correct": false
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-201",
   "suite": "bbh",
   "latency_s": 2.6820263862609863,
   "ttft_s": 0.29003405570983887,
   "completion_tokens": 93,
   "finish": "stop",
   "reasoning_tokens": 86,
   "correct": true
  },
  {
   "id": "bbh-penguins_in_a_table-74",
   "suite": "bbh",
   "latency_s": 3.9089229106903076,
   "ttft_s": 0.31616854667663574,
   "completion_tokens": 127,
   "finish": "stop",
   "reasoning_tokens": 120,
   "correct": true
  },
  {
   "id": "bbh-disambiguation_qa-168",
   "suite": "bbh",
   "latency_s": 10.70196533203125,
   "ttft_s": 0.2477428913116455,
   "completion_tokens": 315,
   "finish": "stop",
   "reasoning_tokens": 308,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_seven_objects-201",
   "suite": "bbh",
   "latency_s": 17.39967370033264,
   "ttft_s": 0.2903401851654053,
   "completion_tokens": 606,
   "finish": "stop",
   "reasoning_tokens": 599,
   "correct": true
  }
 ]
}