{
  "status": "RUN_ON_CPU",
  "seeds": {
    "7": {
      "sft": {
        "pass@1": 0.803125,
        "pass@4": 1.0,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.1625,
        "pass@1_ci95": [
          0.803125,
          0.7796875,
          0.8265625
        ]
      },
      "bon": {
        "pass@1": 1.0,
        "samples_per_task": 16,
        "deployment_budget_multiplier": 16
      },
      "rejection": {
        "pass@1": 0.9140625,
        "pass@4": 1.0,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.1265625,
        "pass@1_ci95": [
          0.9140625,
          0.8921875,
          0.934375
        ]
      },
      "relative_strict": {
        "pass@1": 0.959375,
        "pass@4": 1.0,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.1140625,
        "pass@1_ci95": [
          0.959375,
          0.9421875,
          0.975
        ]
      },
      "relative_weak": {
        "pass@1": 0.4890625,
        "pass@4": 0.925,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.4109375,
        "pass@1_ci95": [
          0.4890625,
          0.440625,
          0.5421875
        ]
      }
    },
    "17": {
      "sft": {
        "pass@1": 0.80625,
        "pass@4": 1.0,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.1890625,
        "pass@1_ci95": [
          0.80625,
          0.778125,
          0.834375
        ]
      },
      "bon": {
        "pass@1": 1.0,
        "samples_per_task": 16,
        "deployment_budget_multiplier": 16
      },
      "rejection": {
        "pass@1": 0.9171875,
        "pass@4": 1.0,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.125,
        "pass@1_ci95": [
          0.9171875,
          0.896875,
          0.9375
        ]
      },
      "relative_strict": {
        "pass@1": 0.95,
        "pass@4": 1.0,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.11875,
        "pass@1_ci95": [
          0.95,
          0.93125,
          0.965625
        ]
      },
      "relative_weak": {
        "pass@1": 0.346875,
        "pass@4": 0.95,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.6265625,
        "pass@1_ci95": [
          0.346875,
          0.290625,
          0.4109765624999998
        ]
      }
    },
    "29": {
      "sft": {
        "pass@1": 0.7859375,
        "pass@4": 1.0,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.25625,
        "pass@1_ci95": [
          0.7859375,
          0.753125,
          0.81875
        ]
      },
      "bon": {
        "pass@1": 1.0,
        "samples_per_task": 16,
        "deployment_budget_multiplier": 16
      },
      "rejection": {
        "pass@1": 0.8984375,
        "pass@4": 1.0,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.24375,
        "pass@1_ci95": [
          0.8984375,
          0.875,
          0.921875
        ]
      },
      "relative_strict": {
        "pass@1": 0.9484375,
        "pass@4": 1.0,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.2390625,
        "pass@1_ci95": [
          0.9484375,
          0.93125,
          0.9640625
        ]
      },
      "relative_weak": {
        "pass@1": 0.6515625,
        "pass@4": 0.95,
        "pass@16": 1.0,
        "best_of_n_oracle": 1.0,
        "proxy_reward": 0.3296875,
        "pass@1_ci95": [
          0.6515625,
          0.6046875,
          0.696875
        ]
      }
    }
  },
  "aggregate": {
    "sft": {
      "pass@1_or_coverage_ci95": [
        0.7984375,
        0.78125,
        0.8156380208333333
      ],
      "seed_means": [
        0.803125,
        0.80625,
        0.7859375
      ]
    },
    "rejection": {
      "pass@1_or_coverage_ci95": [
        0.9098958333333333,
        0.8973958333333333,
        0.9229166666666667
      ],
      "seed_means": [
        0.9140625,
        0.9171875,
        0.8984375
      ],
      "paired_delta_vs_sft_ci95": [
        0.11145833333333334,
        0.096875,
        0.12604166666666666
      ]
    },
    "bon": {
      "pass@1_or_coverage_ci95": [
        1.0,
        1.0,
        1.0
      ],
      "seed_means": [
        1.0,
        1.0,
        1.0
      ]
    },
    "relative_strict": {
      "pass@1_or_coverage_ci95": [
        0.9526041666666667,
        0.9432291666666667,
        0.9619791666666667
      ],
      "seed_means": [
        0.959375,
        0.95,
        0.9484375
      ],
      "paired_delta_vs_sft_ci95": [
        0.15416666666666667,
        0.13697916666666668,
        0.17083333333333334
      ]
    },
    "relative_weak": {
      "pass@1_or_coverage_ci95": [
        0.49583333333333335,
        0.4583333333333333,
        0.5343880208333333
      ],
      "seed_means": [
        0.4890625,
        0.346875,
        0.6515625
      ],
      "paired_delta_vs_sft_ci95": [
        -0.3026041666666667,
        -0.33751302083333334,
        -0.2687369791666667
      ]
    }
  },
  "reward_model": {
    "clean": {
      "noise": 0.0,
      "weights": [
        4.030782748406872,
        2.1920405444858377,
        0.0
      ],
      "loss_initial": 0.6931471805579453,
      "loss_final": 0.23479660657619159,
      "agreement": 1.0,
      "constant_gradient_norm": 0.0,
      "shift_invariance_demo": {
        "score": 2.1920405444858377,
        "score_plus_7": 9.192040544485838,
        "pair_probability_unchanged": true
      },
      "loss_curve": [
        0.6931471805579453,
        0.6754872253094316,
        0.6588519520016536,
        0.6431788842364771,
        0.6284075083514578,
        0.6144796962078194,
        0.6013400061897247,
        0.5889358779680572,
        0.5772177375981862,
        0.5661390292778041,
        0.5556561889934106,
        0.5457285736778136,
        0.5363183576566289,
        0.5273904062750499,
        0.5189121347975034,
        0.5108533590398395,
        0.5031861427645649,
        0.4958846456549461,
        0.48892497467633317,
        0.4822850408151542,
        0.47594442253492164,
        0.46988423677998054,
        0.4640870179679913,
        0.45853660511964867,
        0.45321803705984187,
        0.44811745547206755,
        0.44322201548380263,
        0.4385198033935664,
        0.4339997611115244,
        0.4296516168675206,
        0.42546582173772246,
        0.4214334915491925,
        0.41754635373732246,
        0.4137966987515496,
        0.41017733562823233,
        0.4066815513745381,
        0.4033030738326587,
        0.40003603771888024,
        0.3968749535564814,
        0.39381467924477365,
        0.3908503940286381,
        0.3879775746535161,
        0.38519197350995427,
        0.38248959858947434,
        0.3798666950897882,
        0.37731972852225276,
        0.37484536918804506,
        0.3724404779019162,
        0.37010209285362694,
        0.36782741750738895,
        0.36561380944888755,
        0.36345877009785615,
        0.3613599352117604,
        0.3593150661130208,
        0.35732204157841624,
        0.35537885033492644,
        0.3534835841113524,
        0.351634431199646,
        0.3498296704840345,
        0.34806766589978244,
        0.3463468612868317,
        0.3446657756066411,
        0.34302299849332796,
        0.3414171861127422,
        0.33984705730538795,
        0.3383113899911841,
        0.3368090178159337,
        0.33533882702108514,
        0.33389975351991197,
        0.332490780164658,
        0.33111093419046767,
        0.32975928482309347,
        0.3284349410384339,
        0.32713704946292227,
        0.32586479240467126,
        0.32461738600608037,
        0.3233940785093534,
        0.32219414862703966,
        0.32101690401032784,
        0.3198616798083884,
        0.3187278373125656,
        0.31761476267969785,
        0.3165218657292748,
        0.3154485788095355,
        0.3143943557279741,
        0.31335867074205626,
        0.3123410176062531,
        0.31134090867178393,
        0.3103578740357162,
        0.3093914607363136,
        0.3084412319917393,
        0.30750676647942704,
        0.3065876576536208,
        0.30568351309875214,
        0.3047939539164911,
        0.30391861414444654,
        0.30305714020463426,
        0.3022091903799546,
        0.3013744343170376,
        0.3005525525539254,
        0.2997432360711574,
        0.2989461858649213,
        0.2981611125410159,
        0.2973877359284548,
        0.29662578471161133,
        0.29587499607987805,
        0.29513511539387627,
        0.2944058958673102,
        0.2936870982636188,
        0.29297849060662945,
        0.2922798479044632,
        0.29159095188599155,
        0.2909115907491826,
        0.29024155892071635,
        0.28958065682628303,
        0.28892869067101673,
        0.28828547222954387,
        0.2876508186451601,
        0.28702455223767437,
        0.28640650031948717,
        0.28579649501949417,
        0.28519437311442725,
        0.2845999758672707,
        0.2840131488724065,
        0.28343374190716497,
        0.2828616087894716,
        0.28229660724130096,
        0.28173859875766166,
        0.28118744848085264,
        0.28064302507974453,
        0.2801052006338518,
        0.27957385052197725,
        0.27904885331521584,
        0.2785300906741228,
        0.2780174472498556,
        0.27751081058911226,
        0.2770100710426956,
        0.2765151216775443,
        0.2760258581920755,
        0.27554217883469645,
        0.27506398432534546,
        0.27459117777993214,
        0.2741236646375512,
        0.27366135259035324,
        0.2732041515159575,
        0.27275197341230095,
        0.2723047323348205,
        0.2718623443358719,
        0.2714247274062921,
        0.27099180141901613,
        0.2705634880746668,
        0.2701397108490335,
        0.26972039494236605,
        0.2693054672304106,
        0.26889485621711595,
        0.26848849198894587,
        0.26808630617073304,
        0.2676882318830143,
        0.2672942037007882,
        0.26690415761364183,
        0.2665180309871917,
        0.266135762525789,
        0.2657572922364424,
        0.2653825613939086,
        0.2650115125069105,
        0.2646440892854376,
        0.26428023660908984,
        0.26391990049642516,
        0.2635630280752743,
        0.2632095675539869,
        0.26285946819357453,
        0.26251268028071906,
        0.26216915510161387,
        0.26182884491660813,
        0.26149170293562657,
        0.26115768329433525,
        0.2608267410310281,
        0.26049883206420843,
        0.26017391317084204,
        0.25985194196525546,
        0.2595328768786612,
        0.2592166771392849,
        0.2589033027530754,
        0.2585927144849766,
        0.25828487384074417,
        0.25797974304928545,
        0.25767728504550785,
        0.25737746345365714,
        0.25708024257112955,
        0.2567855873527419,
        0.25649346339544554,
        0.25620383692346727,
        0.2559166747738663,
        0.2556319443824909,
        0.25534961377032384,
        0.25506965153020256,
        0.254792026813903,
        0.25451670931957543,
        0.25424366927952036,
        0.2539728774482945,
        0.2537043050911359,
        0.25343792397269876,
        0.2531737063460876,
        0.25291162494218195,
        0.25265165295924275,
        0.25239376405279057,
        0.25213793232574927,
        0.2518841323188456,
        0.2516323390012565,
        0.25138252776149883,
        0.2511346743985517,
        0.250888755113206,
        0.2506447464996337,
        0.2504026255371706,
        0.2501623695823065,
        0.24992395636087597,
        0.24968736396044444,
        0.24945257082288314,
        0.2492195557371292,
        0.24898829783212298,
        0.2487587765699194,
        0.248530971738968,
        0.24830486344755598,
        0.2480804321174108,
        0.24785765847745667,
        0.24763652355772184,
        0.247417008683391,
        0.24719909546900007,
        0.24698276581276862,
        0.246768001891066,
        0.24655478615300855,
        0.24634310131518267,
        0.24613293035649159,
        0.24592425651312197,
        0.245717063273627,
        0.24551133437412317,
        0.24530705379359707,
        0.2451042057493199,
        0.2449027746923666,
        0.24470274530323616,
        0.24450410248757198,
        0.2443068313719775,
        0.24411091729992676,
        0.24391634582776553,
        0.24372310272080222,
        0.24353117394948479,
        0.24334054568566274,
        0.243151204298931,
        0.242963136353054,
        0.24277632860246826,
        0.24259076798886023,
        0.24240644163781927,
        0.2422233368555622,
        0.24204144112572873,
        0.24186074210624525,
        0.24168122762625596,
        0.24150288568311878,
        0.24132570443946558,
        0.24114967222032366,
        0.24097477751029875,
        0.24080100895081624,
        0.24062835533742022,
        0.2404568056171293,
        0.24028634888584616,
        0.24011697438582189,
        0.23994867150317142,
        0.23978142976544034,
        0.23961523883922198,
        0.23945008852782176,
        0.23928596876897004,
        0.23912286963258103,
        0.23896078131855622,
        0.2387996941546327,
        0.23863959859427383,
        0.23848048521460266,
        0.2383223447143759,
        0.23816516791199802,
        0.2380089457435748,
        0.23785366926100493,
        0.23769932963010926,
        0.23754591812879616,
        0.23739342614526304,
        0.23724184517623229,
        0.2370911668252218,
        0.23694138280084848,
        0.23679248491516428,
        0.23664446508202444,
        0.23649731531548643,
        0.23635102772823957,
        0.2362055945300644,
        0.23606100802632066,
        0.23591726061646512,
        0.23577434479259476,
        0.23563225313801997,
        0.23549097832586252,
        0.23535051311767974,
        0.23521085036211492,
        0.23507198299357213,
        0.23493390403091532,
        0.23479660657619159
      ]
    },
    "noisy_15pct": {
      "noise": 0.15,
      "weights": [
        1.9835384992355825,
        0.9997547330662366,
        0.0
      ],
      "loss_initial": 0.6931471805579453,
      "loss_final": 0.5132583510575813,
      "agreement": 0.8491666666666666,
      "constant_gradient_norm": 0.0,
      "shift_invariance_demo": {
        "score": 0.9997547330662366,
        "score_plus_7": 7.999754733066236,
        "pair_probability_unchanged": true
      },
      "loss_curve": [
        0.6931471805579453,
        0.6842288042810601,
        0.6758270822082718,
        0.6679113125424546,
        0.6604521498132104,
        0.6534216567482194,
        0.6467933295595795,
        0.6405421009647015,
        0.6346443249908372,
        0.6290777472519181,
        0.6238214639776651,
        0.6188558726501036,
        0.6141626166851555,
        0.6097245262029443,
        0.6055255565697542,
        0.6015507260725034,
        0.5977860538048497,
        0.5942184986019591,
        0.5908358996562577,
        0.5876269192758987,
        0.5845809881075187,
        0.5816882530312891,
        0.5789395278455737,
        0.5763262467871656,
        0.5738404208779738,
        0.5714745970473167,
        0.5692218199482384,
        0.5670755963644003,
        0.5650298620893198,
        0.563078951150581,
        0.5612175672468545,
        0.5594407572641785,
        0.5577438867390903,
        0.5561226171392387,
        0.554572884836504,
        0.553090881652973,
        0.551673036866053,
        0.5503160005652826,
        0.5490166282598291,
        0.5477719666420874,
        0.5465792404191159,
        0.5454358401297774,
        0.5443393108713335,
        0.5432873418648505,
        0.5422777567940734,
        0.5413085048574082,
        0.5403776524773274,
        0.5394833756158647,
        0.5386239526489188,
        0.5377977577558426,
        0.5370032547842718,
        0.5362389915533606,
        0.5355035945615548,
        0.5347957640677605,
        0.5341142695172821,
        0.5334579452862104,
        0.532825686720062,
        0.5322164464444245,
        0.5316292309271423,
        0.5310630972732259,
        0.5305171502351682,
        0.5299905394227321,
        0.5294824566975386,
        0.5289921337389489,
        0.5285188397687938,
        0.528061879423481,
        0.5276205907629115,
        0.527194343406448,
        0.5267825367869446,
        0.5263845985145269,
        0.5259999828424561,
        0.5256281692279915,
        0.5252686609816999,
        0.5249209839991557,
        0.5245846855694293,
        0.5242593332551757,
        0.5239445138395195,
        0.5236398323352873,
        0.5233449110524621,
        0.5230593887200317,
        0.5227829196586855,
        0.5225151730010602,
        0.5222558319564765,
        0.5220045931173227,
        0.5217611658044413,
        0.5215252714490599,
        0.5212966430089785,
        0.5210750244168834,
        0.5208601700588029,
        0.520651844280859,
        0.520449820922586,
        0.5202538828752151,
        0.5200638216634208,
        0.519879437049129,
        0.5197005366560808,
        0.5195269356139279,
        0.5193584562207183,
        0.5191949276227031,
        0.5190361855104652,
        0.518882071830433,
        0.5187324345109048,
        0.5185871272017601,
        0.5184460090270907,
        0.518308944350028,
        0.5181758025490913,
        0.5180464578054218,
        0.5179207889003038,
        0.5177986790224163,
        0.5176800155842854,
        0.5175646900474464,
        0.5174525977558473,
        0.5173436377770583,
        0.517237712750874,
        0.5171347287449227,
        0.5170345951169162,
        0.516937224383196,
        0.5168425320932533,
        0.5167504367099163,
        0.5166608594949174,
        0.5165737243995652,
        0.5164889579602712,
        0.5164064891986822,
        0.5163262495261933,
        0.5162481726526247,
        0.5161721944988572,
        0.5160982531132347,
        0.5160262885915492,
        0.5159562430004371,
        0.5158880603040222,
        0.5158216862936512,
        0.5157570685205745,
        0.5156941562314344,
        0.5156329003064288,
        0.5155732532000231,
        0.5155151688840983,
        0.5154586027934148,
        0.5154035117732938,
        0.5153498540294085,
        0.5152975890795938,
        0.5152466777075818,
        0.5151970819185764,
        0.5151487648965843,
        0.5151016909634275,
        0.515055825539359,
        0.5150111351052139,
        0.5149675871660279,
        0.5149251502160594,
        0.5148837937051548,
        0.5148434880063978,
        0.5148042043849906,
        0.5147659149683115,
        0.5147285927171001,
        0.5146922113977235,
        0.5146567455554761,
        0.514622170488873,
        0.5145884622248912,
        0.5145555974951228,
        0.5145235537128032,
        0.5144923089506748,
        0.5144618419196548,
        0.5144321319482753,
        0.5144031589628617,
        0.514374903468421,
        0.5143473465302135,
        0.5143204697559772,
        0.5142942552787814,
        0.5142686857404842,
        0.5142437442757697,
        0.5142194144967425,
        0.5141956804780582,
        0.5141725267425686,
        0.5141499382474614,
        0.5141279003708764,
        0.5141063988989798,
        0.5140854200134779,
        0.514064950279555,
        0.514044976634219,
        0.5140254863750393,
        0.514006467149262,
        0.513987906943289,
        0.5139697940725069,
        0.5139521171714536,
        0.5139348651843089,
        0.513918027355698,
        0.5139015932217973,
        0.5138855526017292,
        0.5138698955892379,
        0.5138546125446349,
        0.5138396940870045,
        0.5138251310866592,
        0.5138109146578387,
        0.5137970361516413,
        0.5137834871491804,
        0.5137702594549576,
        0.5137573450904466,
        0.5137447362878781,
        0.5137324254842206,
        0.5137204053153493,
        0.5137086686103963,
        0.5136972083862782,
        0.5136860178423908,
        0.5136750903554694,
        0.5136644194746075,
        0.5136539989164266,
        0.5136438225603955,
        0.5136338844442914,
        0.5136241787597985,
        0.5136146998482425,
        0.5136054421964493,
        0.5135964004327336,
        0.5135875693230033,
        0.5135789437669842,
        0.5135705187945553,
        0.5135622895621923,
        0.5135542513495198,
        0.5135463995559616,
        0.513538729697493,
        0.5135312374034863,
        0.51352391841365,
        0.5135167685750566,
        0.5135097838392587,
        0.5135029602594866,
        0.513496293987929,
        0.5134897812730908,
        0.513483418457229,
        0.51347720197386,
        0.5134711283453406,
        0.513465194180516,
        0.5134593961724369,
        0.5134537310961402,
        0.5134481958064926,
        0.5134427872360969,
        0.5134375023932547,
        0.5134323383599887,
        0.5134272922901196,
        0.5134223614073974,
        0.5134175430036834,
        0.5134128344371849,
        0.513408233130738,
        0.5134037365701375,
        0.5133993423025142,
        0.5133950479347555,
        0.5133908511319717,
        0.5133867496160017,
        0.5133827411639618,
        0.5133788236068332,
        0.5133749948280882,
        0.5133712527623544,
        0.5133675953941134,
        0.5133640207564372,
        0.513360526929757,
        0.5133571120406657,
        0.513353774260753,
        0.5133505118054713,
        0.5133473229330321,
        0.5133442059433319,
        0.5133411591769076,
        0.513338181013918,
        0.5133352698731543,
        0.5133324242110747,
        0.5133296425208674,
        0.5133269233315352,
        0.5133242652070067,
        0.5133216667452692,
        0.5133191265775264,
        0.5133166433673754,
        0.513314215810007,
        0.5133118426314279,
        0.5133095225877005,
        0.5133072544642039,
        0.5133050370749148,
        0.5133028692617058,
        0.5133007498936624,
        0.5132986778664173,
        0.5132966521015027,
        0.5132946715457178,
        0.5132927351705143,
        0.5132908419713963,
        0.5132889909673359,
        0.5132871812002042,
        0.5132854117342165,
        0.513283681655391,
        0.5132819900710225,
        0.5132803361091686,
        0.5132787189181489,
        0.5132771376660576,
        0.5132755915402873,
        0.5132740797470661,
        0.5132726015110044,
        0.513271156074656,
        0.5132697426980877,
        0.5132683606584599,
        0.5132670092496199,
        0.5132656877817023,
        0.5132643955807417,
        0.513263131988294,
        0.5132618963610674,
        0.5132606880705621,
        0.51325950650272,
        0.5132583510575813
      ]
    }
  },
  "grader_agreement": {
    "strict_vs_human": 0.975,
    "weak_vs_human": 0.475,
    "strict_vs_weak": 0.475,
    "items": 160
  },
  "budget_contract": {
    "training_rollouts_each_online_or_rejection": 38400,
    "bon_deployment_samples_per_task": 16,
    "warning": "BoN deployment compute differs; equal training rollout count is not equal end-to-end compute."
  },
  "split_contract": {
    "train_hidden_audit_disjoint": true,
    "hidden_used_only_for_final_evaluation": true
  }
}