{
 "label": "ours",
 "model": "Qwen3.8-Flash-Next",
 "concurrency": 1,
 "effort": "xhigh",
 "n": 60,
 "wall_s": 1180.539073228836,
 "latency_median_s": 7.707489728927612,
 "latency_mean_s": 19.675426562627155,
 "latency_p90_s": 32.57245373725891,
 "correct": 50,
 "completion_tokens_mean": 930.6,
 "length_stops": 0,
 "items": [
  {
   "id": "mmlupro-3593",
   "suite": "mmlupro",
   "latency_s": 7.272158145904541,
   "ttft_s": 1.2327919006347656,
   "completion_tokens": 262,
   "finish": "stop",
   "reasoning_tokens": 253,
   "correct": true
  },
  {
   "id": "mmlupro-6249",
   "suite": "mmlupro",
   "latency_s": 14.418099641799927,
   "ttft_s": 1.2425472736358643,
   "completion_tokens": 509,
   "finish": "stop",
   "reasoning_tokens": 502,
   "correct": true
  },
  {
   "id": "mmlupro-6182",
   "suite": "mmlupro",
   "latency_s": 9.705151081085205,
   "ttft_s": 1.733017921447754,
   "completion_tokens": 314,
   "finish": "stop",
   "reasoning_tokens": 307,
   "correct": true
  },
  {
   "id": "mmlupro-4993",
   "suite": "mmlupro",
   "latency_s": 3.7868731021881104,
   "ttft_s": 1.1229214668273926,
   "completion_tokens": 112,
   "finish": "stop",
   "reasoning_tokens": 105,
   "correct": true
  },
  {
   "id": "mmlupro-12095",
   "suite": "mmlupro",
   "latency_s": 200.12191486358643,
   "ttft_s": 0.7331292629241943,
   "completion_tokens": 9873,
   "finish": "stop",
   "reasoning_tokens": 9866,
   "correct": true
  },
  {
   "id": "mmlupro-2253",
   "suite": "mmlupro",
   "latency_s": 2.2991275787353516,
   "ttft_s": 0.6225042343139648,
   "completion_tokens": 88,
   "finish": "stop",
   "reasoning_tokens": 81,
   "correct": true
  },
  {
   "id": "mmlupro-2570",
   "suite": "mmlupro",
   "latency_s": 3.943478584289551,
   "ttft_s": 0.5009212493896484,
   "completion_tokens": 162,
   "finish": "stop",
   "reasoning_tokens": 155,
   "correct": true
  },
  {
   "id": "mmlupro-2192",
   "suite": "mmlupro",
   "latency_s": 15.275721311569214,
   "ttft_s": 0.623009204864502,
   "completion_tokens": 661,
   "finish": "stop",
   "reasoning_tokens": 654,
   "correct": true
  },
  {
   "id": "mmlupro-3354",
   "suite": "mmlupro",
   "latency_s": 8.51788330078125,
   "ttft_s": 1.1838700771331787,
   "completion_tokens": 365,
   "finish": "stop",
   "reasoning_tokens": 358,
   "correct": true
  },
  {
   "id": "mmlupro-4643",
   "suite": "mmlupro",
   "latency_s": 3.901780366897583,
   "ttft_s": 0.3895864486694336,
   "completion_tokens": 199,
   "finish": "stop",
   "reasoning_tokens": 192,
   "correct": true
  },
  {
   "id": "mmlupro-10930",
   "suite": "mmlupro",
   "latency_s": 76.62430238723755,
   "ttft_s": 0.29620981216430664,
   "completion_tokens": 3560,
   "finish": "stop",
   "reasoning_tokens": 3553,
   "correct": false
  },
  {
   "id": "mmlupro-10659",
   "suite": "mmlupro",
   "latency_s": 12.467932939529419,
   "ttft_s": 0.6149067878723145,
   "completion_tokens": 531,
   "finish": "stop",
   "reasoning_tokens": 524,
   "correct": true
  },
  {
   "id": "mmlupro-4834",
   "suite": "mmlupro",
   "latency_s": 6.589696168899536,
   "ttft_s": 1.561018943786621,
   "completion_tokens": 195,
   "finish": "stop",
   "reasoning_tokens": 188,
   "correct": true
  },
  {
   "id": "mmlupro-2618",
   "suite": "mmlupro",
   "latency_s": 19.672127723693848,
   "ttft_s": 0.7209982872009277,
   "completion_tokens": 835,
   "finish": "stop",
   "reasoning_tokens": 828,
   "correct": true
  },
  {
   "id": "mmlupro-5034",
   "suite": "mmlupro",
   "latency_s": 7.6839210987091064,
   "ttft_s": 0.4841454029083252,
   "completion_tokens": 373,
   "finish": "stop",
   "reasoning_tokens": 366,
   "correct": true
  },
  {
   "id": "mmlupro-9174",
   "suite": "mmlupro",
   "latency_s": 8.840696334838867,
   "ttft_s": 0.5712800025939941,
   "completion_tokens": 534,
   "finish": "stop",
   "reasoning_tokens": 527,
   "correct": true
  },
  {
   "id": "mmlupro-4212",
   "suite": "mmlupro",
   "latency_s": 3.4809048175811768,
   "ttft_s": 0.3954651355743408,
   "completion_tokens": 208,
   "finish": "stop",
   "reasoning_tokens": 201,
   "correct": true
  },
  {
   "id": "mmlupro-2139",
   "suite": "mmlupro",
   "latency_s": 4.637468099594116,
   "ttft_s": 0.6770706176757812,
   "completion_tokens": 171,
   "finish": "stop",
   "reasoning_tokens": 164,
   "correct": true
  },
  {
   "id": "mmlupro-7387",
   "suite": "mmlupro",
   "latency_s": 1.9902632236480713,
   "ttft_s": 0.5071506500244141,
   "completion_tokens": 80,
   "finish": "stop",
   "reasoning_tokens": 73,
   "correct": true
  },
  {
   "id": "mmlupro-10998",
   "suite": "mmlupro",
   "latency_s": 6.58513879776001,
   "ttft_s": 0.6352531909942627,
   "completion_tokens": 270,
   "finish": "stop",
   "reasoning_tokens": 263,
   "correct": true
  },
  {
   "id": "mmlupro-11035",
   "suite": "mmlupro",
   "latency_s": 8.168367147445679,
   "ttft_s": 0.9982082843780518,
   "completion_tokens": 314,
   "finish": "stop",
   "reasoning_tokens": 307,
   "correct": true
  },
  {
   "id": "mmlupro-1452",
   "suite": "mmlupro",
   "latency_s": 77.56019568443298,
   "ttft_s": 1.2273497581481934,
   "completion_tokens": 3539,
   "finish": "stop",
   "reasoning_tokens": 3532,
   "correct": true
  },
  {
   "id": "mmlupro-9852",
   "suite": "mmlupro",
   "latency_s": 7.146798133850098,
   "ttft_s": 0.44434356689453125,
   "completion_tokens": 350,
   "finish": "stop",
   "reasoning_tokens": 343,
   "correct": true
  },
  {
   "id": "mmlupro-1840",
   "suite": "mmlupro",
   "latency_s": 42.65364980697632,
   "ttft_s": 0.5030307769775391,
   "completion_tokens": 2210,
   "finish": "stop",
   "reasoning_tokens": 2203,
   "correct": true
  },
  {
   "id": "mmlupro-9244",
   "suite": "mmlupro",
   "latency_s": 16.695698022842407,
   "ttft_s": 0.4704601764678955,
   "completion_tokens": 865,
   "finish": "stop",
   "reasoning_tokens": 858,
   "correct": true
  },
  {
   "id": "mmlupro-11200",
   "suite": "mmlupro",
   "latency_s": 32.57245373725891,
   "ttft_s": 0.2974276542663574,
   "completion_tokens": 1415,
   "finish": "stop",
   "reasoning_tokens": 1408,
   "correct": false
  },
  {
   "id": "mmlupro-804",
   "suite": "mmlupro",
   "latency_s": 26.51653814315796,
   "ttft_s": 0.4173409938812256,
   "completion_tokens": 1430,
   "finish": "stop",
   "reasoning_tokens": 1423,
   "correct": true
  },
  {
   "id": "mmlupro-11025",
   "suite": "mmlupro",
   "latency_s": 16.967180728912354,
   "ttft_s": 0.36400890350341797,
   "completion_tokens": 728,
   "finish": "stop",
   "reasoning_tokens": 721,
   "correct": false
  },
  {
   "id": "mmlupro-8937",
   "suite": "mmlupro",
   "latency_s": 14.666766881942749,
   "ttft_s": 0.2642655372619629,
   "completion_tokens": 705,
   "finish": "stop",
   "reasoning_tokens": 698,
   "correct": false
  },
  {
   "id": "mmlupro-8442",
   "suite": "mmlupro",
   "latency_s": 4.9780144691467285,
   "ttft_s": 0.7035775184631348,
   "completion_tokens": 212,
   "finish": "stop",
   "reasoning_tokens": 205,
   "correct": true
  },
  {
   "id": "bbh-object_counting-28",
   "suite": "bbh",
   "latency_s": 1.9051063060760498,
   "ttft_s": 0.24679923057556152,
   "completion_tokens": 81,
   "finish": "stop",
   "reasoning_tokens": 74,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-75",
   "suite": "bbh",
   "latency_s": 10.98507285118103,
   "ttft_s": 0.6501588821411133,
   "completion_tokens": 486,
   "finish": "stop",
   "reasoning_tokens": 479,
   "correct": false
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-136",
   "suite": "bbh",
   "latency_s": 7.731058359146118,
   "ttft_s": 0.5525705814361572,
   "completion_tokens": 388,
   "finish": "stop",
   "reasoning_tokens": 381,
   "correct": false
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-139",
   "suite": "bbh",
   "latency_s": 3.081817865371704,
   "ttft_s": 0.3508296012878418,
   "completion_tokens": 133,
   "finish": "stop",
   "reasoning_tokens": 126,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-207",
   "suite": "bbh",
   "latency_s": 150.24479579925537,
   "ttft_s": 0.2431790828704834,
   "completion_tokens": 6963,
   "finish": "stop",
   "reasoning_tokens": 6952,
   "correct": false
  },
  {
   "id": "bbh-movie_recommendation-26",
   "suite": "bbh",
   "latency_s": 16.76141381263733,
   "ttft_s": 0.27428746223449707,
   "completion_tokens": 809,
   "finish": "stop",
   "reasoning_tokens": 802,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_five_objects-223",
   "suite": "bbh",
   "latency_s": 5.307890892028809,
   "ttft_s": 0.3745715618133545,
   "completion_tokens": 321,
   "finish": "stop",
   "reasoning_tokens": 314,
   "correct": true
  },
  {
   "id": "bbh-reasoning_about_colored_objects-95",
   "suite": "bbh",
   "latency_s": 1.6469738483428955,
   "ttft_s": 0.31827807426452637,
   "completion_tokens": 73,
   "finish": "stop",
   "reasoning_tokens": 66,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-117",
   "suite": "bbh",
   "latency_s": 11.966024398803711,
   "ttft_s": 0.37076759338378906,
   "completion_tokens": 813,
   "finish": "stop",
   "reasoning_tokens": 806,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_seven_objects-67",
   "suite": "bbh",
   "latency_s": 10.458592176437378,
   "ttft_s": 0.4016110897064209,
   "completion_tokens": 600,
   "finish": "stop",
   "reasoning_tokens": 593,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-2",
   "suite": "bbh",
   "latency_s": 20.512701511383057,
   "ttft_s": 0.3081674575805664,
   "completion_tokens": 971,
   "finish": "stop",
   "reasoning_tokens": 964,
   "correct": false
  },
  {
   "id": "bbh-multistep_arithmetic_two-202",
   "suite": "bbh",
   "latency_s": 2.7375667095184326,
   "ttft_s": 0.2157282829284668,
   "completion_tokens": 169,
   "finish": "stop",
   "reasoning_tokens": 161,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-125",
   "suite": "bbh",
   "latency_s": 11.137370824813843,
   "ttft_s": 0.506237268447876,
   "completion_tokens": 608,
   "finish": "stop",
   "reasoning_tokens": 601,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_seven_objects-9",
   "suite": "bbh",
   "latency_s": 5.130329370498657,
   "ttft_s": 0.37009310722351074,
   "completion_tokens": 253,
   "finish": "stop",
   "reasoning_tokens": 246,
   "correct": true
  },
  {
   "id": "bbh-snarks-87",
   "suite": "bbh",
   "latency_s": 5.615600824356079,
   "ttft_s": 0.2560575008392334,
   "completion_tokens": 320,
   "finish": "stop",
   "reasoning_tokens": 313,
   "correct": true
  },
  {
   "id": "bbh-geometric_shapes-161",
   "suite": "bbh",
   "latency_s": 9.271058082580566,
   "ttft_s": 0.2572484016418457,
   "completion_tokens": 589,
   "finish": "stop",
   "reasoning_tokens": 582,
   "correct": false
  },
  {
   "id": "bbh-ruin_names-76",
   "suite": "bbh",
   "latency_s": 2.642474412918091,
   "ttft_s": 0.2533707618713379,
   "completion_tokens": 116,
   "finish": "stop",
   "reasoning_tokens": 109,
   "correct": true
  },
  {
   "id": "bbh-formal_fallacies-9",
   "suite": "bbh",
   "latency_s": 6.959958791732788,
   "ttft_s": 0.3408846855163574,
   "completion_tokens": 384,
   "finish": "stop",
   "reasoning_tokens": 377,
   "correct": true
  },
  {
   "id": "bbh-web_of_lies-160",
   "suite": "bbh",
   "latency_s": 2.697458028793335,
   "ttft_s": 0.2288074493408203,
   "completion_tokens": 124,
   "finish": "stop",
   "reasoning_tokens": 117,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-33",
   "suite": "bbh",
   "latency_s": 174.96992087364197,
   "ttft_s": 1.310821533203125,
   "completion_tokens": 8049,
   "finish": "stop",
   "reasoning_tokens": 8042,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-21",
   "suite": "bbh",
   "latency_s": 4.985917091369629,
   "ttft_s": 0.5196554660797119,
   "completion_tokens": 193,
   "finish": "stop",
   "reasoning_tokens": 186,
   "correct": false
  },
  {
   "id": "bbh-reasoning_about_colored_objects-57",
   "suite": "bbh",
   "latency_s": 3.0767276287078857,
   "ttft_s": 0.3716137409210205,
   "completion_tokens": 166,
   "finish": "stop",
   "reasoning_tokens": 159,
   "correct": true
  },
  {
   "id": "bbh-dyck_languages-197",
   "suite": "bbh",
   "latency_s": 7.992469072341919,
   "ttft_s": 0.1715254783630371,
   "completion_tokens": 396,
   "finish": "stop",
   "reasoning_tokens": 387,
   "correct": true
  },
  {
   "id": "bbh-salient_translation_error_detection-203",
   "suite": "bbh",
   "latency_s": 1.893455982208252,
   "ttft_s": 0.359344482421875,
   "completion_tokens": 125,
   "finish": "stop",
   "reasoning_tokens": 118,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_three_objects-227",
   "suite": "bbh",
   "latency_s": 1.7485013008117676,
   "ttft_s": 0.2659187316894531,
   "completion_tokens": 87,
   "finish": "stop",
   "reasoning_tokens": 80,
   "correct": true
  },
  {
   "id": "bbh-causal_judgement-84",
   "suite": "bbh",
   "latency_s": 13.184863567352295,
   "ttft_s": 0.34646081924438477,
   "completion_tokens": 587,
   "finish": "stop",
   "reasoning_tokens": 580,
   "correct": true
  },
  {
   "id": "bbh-tracking_shuffled_objects_three_objects-201",
   "suite": "bbh",
   "latency_s": 3.2729852199554443,
   "ttft_s": 0.22023463249206543,
   "completion_tokens": 176,
   "finish": "stop",
   "reasoning_tokens": 169,
   "correct": true
  },
  {
   "id": "bbh-penguins_in_a_table-74",
   "suite": "bbh",
   "latency_s": 2.1344480514526367,
   "ttft_s": 0.47934508323669434,
   "completion_tokens": 81,
   "finish": "stop",
   "reasoning_tokens": 74,
   "correct": true
  },
  {
   "id": "bbh-disambiguation_qa-168",
   "suite": "bbh",
   "latency_s": 5.426037549972534,
   "ttft_s": 0.2818870544433594,
   "completion_tokens": 255,
   "finish": "stop",
   "reasoning_tokens": 248,
   "correct": true
  },
  {
   "id": "bbh-logical_deduction_seven_objects-201",
   "suite": "bbh",
   "latency_s": 9.306700229644775,
   "ttft_s": 0.34582018852233887,
   "completion_tokens": 450,
   "finish": "stop",
   "reasoning_tokens": 443,
   "correct": true
  }
 ]
}