{ "metric": "Forward reference-conditioned coding-rate Wasserstein-1", "units": "bits / UTF-8 byte", "metrics": { "conditional": { "name": "Conditional compression W\u2081", "units": "bits / UTF-8 byte", "sequences": 256 }, "sol": { "name": "SOL", "units": "SOL score", "sequences": 512 }, "ed": { "name": "Energy distance", "units": "feature-space ED", "sequences": 512 }, "gm": { "name": "Gradient Moment", "units": "gradient score", "sequences": 512 }, "nll": { "name": "Qwen3-8B-Base NLL", "units": "bits / Pythia token", "sequences": 1024 } }, "tasks": [ "arc_challenge_acc", "arc_easy_acc", "lambada_openai_acc", "logiqa_acc", "piqa_acc", "sciq_acc", "winogrande_acc" ], "sizes": [ "160m", "410m", "1b", "6.9b" ], "settings": [ "t07", "t10", "t10-p095", "t10-k20", "t13" ], "points": [ { "setting": "t07", "size": "160m", "step": 3000, "distance": 2.0745210551666204, "accuracy": 0.382655743432671, "tasks": { "arc_challenge_acc": 0.17064846416382254, "arc_easy_acc": 0.3792087542087542, "lambada_openai_acc": 0.15253250533669707, "logiqa_acc": 0.2350230414746544, "piqa_acc": 0.5903155603917302, "sciq_acc": 0.626, "winogrande_acc": 0.5248618784530387 }, "job_id": "350bdb5e3e53a6794073e623", "scores": { "conditional": 2.0745210551666204, "gm": 8.93957137791972, "ed": 5.726738118344308, "sol": 5.757841177676538, "nll": 1.031813580339099 } }, { "setting": "t07", "size": "160m", "step": 13000, "distance": 1.606602317808693, "accuracy": 0.42397207457676045, "tasks": { "arc_challenge_acc": 0.19027303754266212, "arc_easy_acc": 0.41792929292929293, "lambada_openai_acc": 0.22705220260042694, "logiqa_acc": 0.21351766513056836, "piqa_acc": 0.6164309031556039, "sciq_acc": 0.788, "winogrande_acc": 0.5146014206787688 }, "job_id": "4f1bef96d7ef142a5daa325d", "scores": { "conditional": 1.606602317808693, "gm": 20.135154434429296, "ed": 3.4307268052094564, "sol": 4.348264568030308, "nll": 1.3296614032920662 } }, { "setting": "t07", "size": "160m", "step": 33000, "distance": 1.5344529801116493, "accuracy": 0.44079549853155203, "tasks": { "arc_challenge_acc": 0.19880546075085323, "arc_easy_acc": 0.4393939393939394, "lambada_openai_acc": 0.2555792742091985, "logiqa_acc": 0.2227342549923195, "piqa_acc": 0.6365614798694232, "sciq_acc": 0.81, "winogrande_acc": 0.5224940805051302 }, "job_id": "62f7640a09d5b29bddf1c3b1", "scores": { "conditional": 1.5344529801116493, "gm": 23.61541280321444, "ed": 3.075262814090138, "sol": 4.307854825010704, "nll": 1.3389807801433566 } }, { "setting": "t07", "size": "160m", "step": 63000, "distance": 1.7078545247166648, "accuracy": 0.4510944971867945, "tasks": { "arc_challenge_acc": 0.1945392491467577, "arc_easy_acc": 0.4473905723905724, "lambada_openai_acc": 0.3070056277896371, "logiqa_acc": 0.2350230414746544, "piqa_acc": 0.6289445048966268, "sciq_acc": 0.827, "winogrande_acc": 0.5177584846093133 }, "job_id": "4bf5658628b501d7a9027a6d", "scores": { "conditional": 1.7078545247166648, "gm": 35.97245105731531, "ed": 3.392624062467746, "sol": 4.6670832183169715, "nll": 1.2359010883109396 } }, { "setting": "t07", "size": "160m", "step": 103000, "distance": 1.5086954128785872, "accuracy": 0.44066173637581074, "tasks": { "arc_challenge_acc": 0.189419795221843, "arc_easy_acc": 0.45791245791245794, "lambada_openai_acc": 0.2718804579856394, "logiqa_acc": 0.21505376344086022, "piqa_acc": 0.6218715995647442, "sciq_acc": 0.806, "winogrande_acc": 0.5224940805051302 }, "job_id": "56221344898d79126bd591ee", "scores": { "conditional": 1.5086954128785872, "gm": 28.182895140648725, "ed": 2.7515551093047357, "sol": 4.352556652344444, "nll": 1.353201390058175 } }, { "setting": "t07", "size": "160m", "step": 143000, "distance": 1.4711760045626006, "accuracy": 0.4309219653678175, "tasks": { "arc_challenge_acc": 0.18600682593856654, "arc_easy_acc": 0.4494949494949495, "lambada_openai_acc": 0.25732582961381717, "logiqa_acc": 0.21658986175115208, "piqa_acc": 0.6213275299238302, "sciq_acc": 0.779, "winogrande_acc": 0.5067087608524072 }, "job_id": "7773413b2397ee53786ae2a2", "scores": { "conditional": 1.4711760045626006, "gm": 37.439567273418106, "ed": 2.689682623146025, "sol": 3.772298489167469, "nll": 1.5177546077281006 } }, { "setting": "t07", "size": "410m", "step": 3000, "distance": 1.982406069968575, "accuracy": 0.4045096758199565, "tasks": { "arc_challenge_acc": 0.17832764505119453, "arc_easy_acc": 0.4137205387205387, "lambada_openai_acc": 0.18358237919658452, "logiqa_acc": 0.250384024577573, "piqa_acc": 0.5935799782372143, "sciq_acc": 0.71, "winogrande_acc": 0.5019731649565904 }, "job_id": "ac5d56fb22e955b4d55bb8c3", "scores": { "conditional": 1.982406069968575, "gm": 7.514404850922347, "ed": 5.157377552493017, "sol": 5.909044477579302, "nll": 1.047645993694833 } }, { "setting": "t07", "size": "410m", "step": 13000, "distance": 1.4982641072865357, "accuracy": 0.46013927731831844, "tasks": { "arc_challenge_acc": 0.2022184300341297, "arc_easy_acc": 0.4739057239057239, "lambada_openai_acc": 0.33863768678439743, "logiqa_acc": 0.23348694316436253, "piqa_acc": 0.6327529923830251, "sciq_acc": 0.838, "winogrande_acc": 0.5019731649565904 }, "job_id": "b2a3d0198c238ee731f39a6a", "scores": { "conditional": 1.4982641072865357, "gm": 23.253426472181218, "ed": 2.9498445740057697, "sol": 4.448684833378616, "nll": 1.373145525776136 } }, { "setting": "t07", "size": "410m", "step": 33000, "distance": 1.369201858534241, "accuracy": 0.4857382433145336, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5117845117845118, "lambada_openai_acc": 0.38327188045798566, "logiqa_acc": 0.23809523809523808, "piqa_acc": 0.6692056583242655, "sciq_acc": 0.868, "winogrande_acc": 0.5122336227308603 }, "job_id": "53ae98b2bf00fbb6b6fc72d3", "scores": { "conditional": 1.369201858534241, "gm": 11.65466418556457, "ed": 2.3258788721758457, "sol": 3.7855249297871234, "nll": 1.4636418511282092 } }, { "setting": "t07", "size": "410m", "step": 63000, "distance": 1.1566396777606935, "accuracy": 0.4923627266517646, "tasks": { "arc_challenge_acc": 0.21331058020477817, "arc_easy_acc": 0.5416666666666666, "lambada_openai_acc": 0.4189792353968562, "logiqa_acc": 0.2073732718894009, "piqa_acc": 0.6670293797606094, "sciq_acc": 0.882, "winogrande_acc": 0.516179952644041 }, "job_id": "30fc400425fddaaf6532dfb7", "scores": { "conditional": 1.1566396777606935, "gm": 25.214120537739745, "ed": 1.9843370694849654, "sol": 3.3328317068294577, "nll": 1.5584537165472288 } }, { "setting": "t07", "size": "410m", "step": 103000, "distance": 0.9955069140685294, "accuracy": 0.5066002946739234, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5555555555555556, "lambada_openai_acc": 0.4315932466524355, "logiqa_acc": 0.22119815668202766, "piqa_acc": 0.6789989118607181, "sciq_acc": 0.903, "winogrande_acc": 0.5382794001578532 }, "job_id": "510d4adaa334a43f343c8e81", "scores": { "conditional": 0.9955069140685294, "gm": 25.652949159131822, "ed": 1.6354074308217648, "sol": 2.9637036035617994, "nll": 1.6431706414466603 } }, { "setting": "t07", "size": "410m", "step": 143000, "distance": 1.0649191384029137, "accuracy": 0.5071258254183677, "tasks": { "arc_challenge_acc": 0.22098976109215018, "arc_easy_acc": 0.5547138047138047, "lambada_openai_acc": 0.45546283718222397, "logiqa_acc": 0.2196620583717358, "piqa_acc": 0.6784548422198041, "sciq_acc": 0.891, "winogrande_acc": 0.5295974743488555 }, "job_id": "1ebd7f6336cf7d06dfe8393e", "scores": { "conditional": 1.0649191384029137, "gm": 19.960698425281862, "ed": 1.5342455037486609, "sol": 3.0294558050091296, "nll": 1.6468160306886932 } }, { "setting": "t07", "size": "1b", "step": 3000, "distance": 1.7018756172453315, "accuracy": 0.41411263962670136, "tasks": { "arc_challenge_acc": 0.19368600682593856, "arc_easy_acc": 0.41792929292929293, "lambada_openai_acc": 0.2066757228798758, "logiqa_acc": 0.22734254992319508, "piqa_acc": 0.6120783460282916, "sciq_acc": 0.732, "winogrande_acc": 0.5090765588003157 }, "job_id": "3c0c69ac1464fed15ec861fa", "scores": { "conditional": 1.7018756172453315, "gm": 40.52240808691931, "ed": 3.2085074912184037, "sol": 4.231849233135139, "nll": 1.3541280597104892 } }, { "setting": "t07", "size": "1b", "step": 13000, "distance": 1.3532827005723922, "accuracy": 0.479511269800246, "tasks": { "arc_challenge_acc": 0.20648464163822525, "arc_easy_acc": 0.5042087542087542, "lambada_openai_acc": 0.36231321560256163, "logiqa_acc": 0.23963133640552994, "piqa_acc": 0.6572361262241567, "sciq_acc": 0.865, "winogrande_acc": 0.5217048145224941 }, "job_id": "5601453fdc1ec5233f3882cc", "scores": { "conditional": 1.3532827005723922, "gm": 22.399737332812947, "ed": 2.166462383577457, "sol": 3.6989011274934325, "nll": 1.4540377160260471 } }, { "setting": "t07", "size": "1b", "step": 33000, "distance": 1.2509880956777462, "accuracy": 0.4982874070398225, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5492424242424242, "lambada_openai_acc": 0.4321754317873084, "logiqa_acc": 0.2196620583717358, "piqa_acc": 0.6811751904243744, "sciq_acc": 0.872, "winogrande_acc": 0.516179952644041 }, "job_id": "bbb9e04d9c18f985910118cf", "scores": { "conditional": 1.2509880956777462, "gm": 31.76047871244782, "ed": 2.0052278234005367, "sol": 3.4928224790834457, "nll": 1.4504224864096846 } }, { "setting": "t07", "size": "1b", "step": 63000, "distance": 0.9391220336487134, "accuracy": 0.5153026239542494, "tasks": { "arc_challenge_acc": 0.23122866894197952, "arc_easy_acc": 0.5593434343434344, "lambada_openai_acc": 0.4723462060935377, "logiqa_acc": 0.22580645161290322, "piqa_acc": 0.6936887921653971, "sciq_acc": 0.903, "winogrande_acc": 0.5217048145224941 }, "job_id": "9706b2ac57141d8f874e51e8", "scores": { "conditional": 0.9391220336487134, "gm": 36.865050908377725, "ed": 1.478849648544971, "sol": 2.901036422807481, "nll": 1.6481432507480598 } }, { "setting": "t07", "size": "1b", "step": 103000, "distance": 0.8757414839336353, "accuracy": 0.5333274933224145, "tasks": { "arc_challenge_acc": 0.25, "arc_easy_acc": 0.5909090909090909, "lambada_openai_acc": 0.4975742286046963, "logiqa_acc": 0.22580645161290322, "piqa_acc": 0.6996735582154516, "sciq_acc": 0.92, "winogrande_acc": 0.5493291239147593 }, "job_id": "38867853ef66f5bb218b698f", "scores": { "conditional": 0.8757414839336353, "gm": 45.03845175517745, "ed": 1.4674643275977974, "sol": 2.9472848433924823, "nll": 1.630785303523126 } }, { "setting": "t07", "size": "1b", "step": 143000, "distance": 0.9911402258437132, "accuracy": 0.5349914679163845, "tasks": { "arc_challenge_acc": 0.2593856655290102, "arc_easy_acc": 0.5938552188552189, "lambada_openai_acc": 0.5072773141859112, "logiqa_acc": 0.22734254992319508, "piqa_acc": 0.705114254624592, "sciq_acc": 0.92, "winogrande_acc": 0.531965272296764 }, "job_id": "0750fd9f285277a07148ec3c", "scores": { "conditional": 0.9911402258437132, "gm": 28.911003685565902, "ed": 1.2806290499215685, "sol": 2.8081019069241027, "nll": 1.6103914290793335 } }, { "setting": "t07", "size": "6.9b", "step": 3000, "distance": 1.8665055873040395, "accuracy": 0.42391127208163926, "tasks": { "arc_challenge_acc": 0.1766211604095563, "arc_easy_acc": 0.4246632996632997, "lambada_openai_acc": 0.22549970890743257, "logiqa_acc": 0.23963133640552994, "piqa_acc": 0.6175190424374319, "sciq_acc": 0.772, "winogrande_acc": 0.5114443567482242 }, "job_id": "bdaec95af131a1b3f516f027", "scores": { "conditional": 1.8665055873040395, "gm": 16.21022122666318, "ed": 4.325135493670391, "sol": 5.283757941680363, "nll": 1.0566597498885637 } }, { "setting": "t07", "size": "6.9b", "step": 13000, "distance": 1.07233426027317, "accuracy": 0.5130790719439721, "tasks": { "arc_challenge_acc": 0.21928327645051193, "arc_easy_acc": 0.563973063973064, "lambada_openai_acc": 0.4511934795264894, "logiqa_acc": 0.2488479262672811, "piqa_acc": 0.6958650707290533, "sciq_acc": 0.897, "winogrande_acc": 0.5153906866614049 }, "job_id": "8fd3366a3a865609f322ed3e", "scores": { "conditional": 1.07233426027317, "gm": 28.758889108054518, "ed": 1.7081488378067125, "sol": 3.172551810029403, "nll": 1.5395220048388223 } }, { "setting": "t07", "size": "6.9b", "step": 33000, "distance": 1.0880890070007685, "accuracy": 0.5533024681765826, "tasks": { "arc_challenge_acc": 0.2883959044368601, "arc_easy_acc": 0.6334175084175084, "lambada_openai_acc": 0.5389093731806714, "logiqa_acc": 0.2119815668202765, "piqa_acc": 0.720348204570185, "sciq_acc": 0.926, "winogrande_acc": 0.5540647198105761 }, "job_id": "d258bfa086a51ee116b225fc", "scores": { "conditional": 1.0880890070007685, "gm": 17.544555066143857, "ed": 1.280732644381862, "sol": 2.8239488493193763, "nll": 1.5869294206211817 } }, { "setting": "t07", "size": "6.9b", "step": 63000, "distance": 0.6798127583355351, "accuracy": 0.5820065388502569, "tasks": { "arc_challenge_acc": 0.318259385665529, "arc_easy_acc": 0.6813973063973064, "lambada_openai_acc": 0.5753929749660393, "logiqa_acc": 0.2304147465437788, "piqa_acc": 0.733949945593036, "sciq_acc": 0.934, "winogrande_acc": 0.6006314127861089 }, "job_id": "09d10d6dc06fea56fcfcf60e", "scores": { "conditional": 0.6798127583355351, "gm": 66.35189213218928, "ed": 1.174634615686056, "sol": 2.7620159987165365, "nll": 1.6919899541022567 } }, { "setting": "t07", "size": "6.9b", "step": 103000, "distance": 0.6850872907277851, "accuracy": 0.607810860185633, "tasks": { "arc_challenge_acc": 0.3370307167235495, "arc_easy_acc": 0.6919191919191919, "lambada_openai_acc": 0.6258490199883563, "logiqa_acc": 0.27035330261136714, "piqa_acc": 0.7453754080522307, "sciq_acc": 0.948, "winogrande_acc": 0.6361483820047356 }, "job_id": "bd6eec245185425e3a55478d", "scores": { "conditional": 0.6850872907277851, "gm": 27.821691617901706, "ed": 0.823983793956943, "sol": 2.2840021333466907, "nll": 1.7254530114616204 } }, { "setting": "t07", "size": "6.9b", "step": 143000, "distance": 0.6641200045960238, "accuracy": 0.6155622706270009, "tasks": { "arc_challenge_acc": 0.35580204778157, "arc_easy_acc": 0.7015993265993266, "lambada_openai_acc": 0.638074907820687, "logiqa_acc": 0.27035330261136714, "piqa_acc": 0.7551686615886833, "sciq_acc": 0.951, "winogrande_acc": 0.6369376479873717 }, "job_id": "c6a5229c1344264c56bb661c", "scores": { "conditional": 0.6641200045960238, "gm": 27.65707118734099, "ed": 0.8377630029002496, "sol": 2.177779301382227, "nll": 1.7664953245259163 } }, { "setting": "t10", "size": "160m", "step": 3000, "distance": 0.6421388132145984, "accuracy": 0.382655743432671, "tasks": { "arc_challenge_acc": 0.17064846416382254, "arc_easy_acc": 0.3792087542087542, "lambada_openai_acc": 0.15253250533669707, "logiqa_acc": 0.2350230414746544, "piqa_acc": 0.5903155603917302, "sciq_acc": 0.626, "winogrande_acc": 0.5248618784530387 }, "job_id": "8527ac52dfdd6e093c2d4454", "scores": { "conditional": 0.6421388132145984, "gm": 7.66339686019046, "ed": 0.9895467164140745, "sol": 2.586664972504809, "nll": 6.59721742193891 } }, { "setting": "t10", "size": "160m", "step": 13000, "distance": 0.622158112158971, "accuracy": 0.42397207457676045, "tasks": { "arc_challenge_acc": 0.19027303754266212, "arc_easy_acc": 0.41792929292929293, "lambada_openai_acc": 0.22705220260042694, "logiqa_acc": 0.21351766513056836, "piqa_acc": 0.6164309031556039, "sciq_acc": 0.788, "winogrande_acc": 0.5146014206787688 }, "job_id": "39786a7ba26eb8b088255b10", "scores": { "conditional": 0.622158112158971, "gm": 5.232280288219499, "ed": 0.8123503369220533, "sol": 2.3884467399563007, "nll": 6.3181479705957075 } }, { "setting": "t10", "size": "160m", "step": 33000, "distance": 0.4963054869634836, "accuracy": 0.44079549853155203, "tasks": { "arc_challenge_acc": 0.19880546075085323, "arc_easy_acc": 0.4393939393939394, "lambada_openai_acc": 0.2555792742091985, "logiqa_acc": 0.2227342549923195, "piqa_acc": 0.6365614798694232, "sciq_acc": 0.81, "winogrande_acc": 0.5224940805051302 }, "job_id": "18473b93e60f47335d18045c", "scores": { "conditional": 0.4963054869634836, "gm": 2.825969758547444, "ed": 0.5102190104599709, "sol": 1.7918686688043377, "nll": 5.838900856068905 } }, { "setting": "t10", "size": "160m", "step": 63000, "distance": 0.4505824214747117, "accuracy": 0.4510944971867945, "tasks": { "arc_challenge_acc": 0.1945392491467577, "arc_easy_acc": 0.4473905723905724, "lambada_openai_acc": 0.3070056277896371, "logiqa_acc": 0.2350230414746544, "piqa_acc": 0.6289445048966268, "sciq_acc": 0.827, "winogrande_acc": 0.5177584846093133 }, "job_id": "86b1ff400267ca3362b5d8cb", "scores": { "conditional": 0.4505824214747117, "gm": 2.2313279508407504, "ed": 0.5283509982992705, "sol": 2.2067214436827576, "nll": 6.100241293935113 } }, { "setting": "t10", "size": "160m", "step": 103000, "distance": 0.6488965894124582, "accuracy": 0.44066173637581074, "tasks": { "arc_challenge_acc": 0.189419795221843, "arc_easy_acc": 0.45791245791245794, "lambada_openai_acc": 0.2718804579856394, "logiqa_acc": 0.21505376344086022, "piqa_acc": 0.6218715995647442, "sciq_acc": 0.806, "winogrande_acc": 0.5224940805051302 }, "job_id": "8aed53f4addf291d6da57f16", "scores": { "conditional": 0.6488965894124582, "gm": 3.554537137561738, "ed": 0.7693872316213426, "sol": 2.182177900307515, "nll": 6.453481787700647 } }, { "setting": "t10", "size": "160m", "step": 143000, "distance": 0.719188926256739, "accuracy": 0.4309219653678175, "tasks": { "arc_challenge_acc": 0.18600682593856654, "arc_easy_acc": 0.4494949494949495, "lambada_openai_acc": 0.25732582961381717, "logiqa_acc": 0.21658986175115208, "piqa_acc": 0.6213275299238302, "sciq_acc": 0.779, "winogrande_acc": 0.5067087608524072 }, "job_id": "6077079d5283ce28a710c8a7", "scores": { "conditional": 0.719188926256739, "gm": 4.188464400150679, "ed": 1.2365814235430257, "sol": 2.7704614323347863, "nll": 7.177437292762421 } }, { "setting": "t10", "size": "410m", "step": 3000, "distance": 0.5424942647136833, "accuracy": 0.4045096758199565, "tasks": { "arc_challenge_acc": 0.17832764505119453, "arc_easy_acc": 0.4137205387205387, "lambada_openai_acc": 0.18358237919658452, "logiqa_acc": 0.250384024577573, "piqa_acc": 0.5935799782372143, "sciq_acc": 0.71, "winogrande_acc": 0.5019731649565904 }, "job_id": "a49da5f14abaa3197ca0b08d", "scores": { "conditional": 0.5424942647136833, "gm": 4.898531273638355, "ed": 0.5502502944068421, "sol": 2.259520008088459, "nll": 6.1541714218756844 } }, { "setting": "t10", "size": "410m", "step": 13000, "distance": 0.45878725399643533, "accuracy": 0.46013927731831844, "tasks": { "arc_challenge_acc": 0.2022184300341297, "arc_easy_acc": 0.4739057239057239, "lambada_openai_acc": 0.33863768678439743, "logiqa_acc": 0.23348694316436253, "piqa_acc": 0.6327529923830251, "sciq_acc": 0.838, "winogrande_acc": 0.5019731649565904 }, "job_id": "ec362921faae32d99714b010", "scores": { "conditional": 0.45878725399643533, "gm": 2.283036924686826, "ed": 0.5530521371214174, "sol": 1.995369365451676, "nll": 5.863194023544934 } }, { "setting": "t10", "size": "410m", "step": 33000, "distance": 0.5001073823411795, "accuracy": 0.4857382433145336, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5117845117845118, "lambada_openai_acc": 0.38327188045798566, "logiqa_acc": 0.23809523809523808, "piqa_acc": 0.6692056583242655, "sciq_acc": 0.868, "winogrande_acc": 0.5122336227308603 }, "job_id": "1f9ac65478459e63b63a9c10", "scores": { "conditional": 0.5001073823411795, "gm": 2.473414912311796, "ed": 0.43843854684256467, "sol": 1.8149561496935578, "nll": 5.725981679896157 } }, { "setting": "t10", "size": "410m", "step": 63000, "distance": 0.5173941338188874, "accuracy": 0.4923627266517646, "tasks": { "arc_challenge_acc": 0.21331058020477817, "arc_easy_acc": 0.5416666666666666, "lambada_openai_acc": 0.4189792353968562, "logiqa_acc": 0.2073732718894009, "piqa_acc": 0.6670293797606094, "sciq_acc": 0.882, "winogrande_acc": 0.516179952644041 }, "job_id": "38a8a87d5eec57f9a45c3bd0", "scores": { "conditional": 0.5173941338188874, "gm": 3.005226870266142, "ed": 0.4129823514917961, "sol": 1.776200794609296, "nll": 5.71834045263525 } }, { "setting": "t10", "size": "410m", "step": 103000, "distance": 0.4267649046008422, "accuracy": 0.5066002946739234, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5555555555555556, "lambada_openai_acc": 0.4315932466524355, "logiqa_acc": 0.22119815668202766, "piqa_acc": 0.6789989118607181, "sciq_acc": 0.903, "winogrande_acc": 0.5382794001578532 }, "job_id": "f4153d18cd56b23f9c265a69", "scores": { "conditional": 0.4267649046008422, "gm": 2.5085663144450336, "ed": 0.4672160213572365, "sol": 1.747062577528207, "nll": 5.483164275178929 } }, { "setting": "t10", "size": "410m", "step": 143000, "distance": 0.40910917119528695, "accuracy": 0.5071258254183677, "tasks": { "arc_challenge_acc": 0.22098976109215018, "arc_easy_acc": 0.5547138047138047, "lambada_openai_acc": 0.45546283718222397, "logiqa_acc": 0.2196620583717358, "piqa_acc": 0.6784548422198041, "sciq_acc": 0.891, "winogrande_acc": 0.5295974743488555 }, "job_id": "6a31b85c8540ecc2feec5b44", "scores": { "conditional": 0.40910917119528695, "gm": 2.4566772753501027, "ed": 0.39045834514998834, "sol": 1.631357052526126, "nll": 5.45291576067825 } }, { "setting": "t10", "size": "1b", "step": 3000, "distance": 0.5556923381002267, "accuracy": 0.41411263962670136, "tasks": { "arc_challenge_acc": 0.19368600682593856, "arc_easy_acc": 0.41792929292929293, "lambada_openai_acc": 0.2066757228798758, "logiqa_acc": 0.22734254992319508, "piqa_acc": 0.6120783460282916, "sciq_acc": 0.732, "winogrande_acc": 0.5090765588003157 }, "job_id": "a7036c55b68ffed9dcc5c5b8", "scores": { "conditional": 0.5556923381002267, "gm": 3.5709786180184437, "ed": 0.7217319829343101, "sol": 2.1343288250053427, "nll": 6.1124221225743085 } }, { "setting": "t10", "size": "1b", "step": 13000, "distance": 0.45728938793751694, "accuracy": 0.479511269800246, "tasks": { "arc_challenge_acc": 0.20648464163822525, "arc_easy_acc": 0.5042087542087542, "lambada_openai_acc": 0.36231321560256163, "logiqa_acc": 0.23963133640552994, "piqa_acc": 0.6572361262241567, "sciq_acc": 0.865, "winogrande_acc": 0.5217048145224941 }, "job_id": "068d69aa5d40fe9fe32c8722", "scores": { "conditional": 0.45728938793751694, "gm": 1.9686075296939407, "ed": 0.4385054104701984, "sol": 1.7552478574686108, "nll": 5.588010300223098 } }, { "setting": "t10", "size": "1b", "step": 33000, "distance": 0.3518818622036406, "accuracy": 0.4982874070398225, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5492424242424242, "lambada_openai_acc": 0.4321754317873084, "logiqa_acc": 0.2196620583717358, "piqa_acc": 0.6811751904243744, "sciq_acc": 0.872, "winogrande_acc": 0.516179952644041 }, "job_id": "a1487a681952c542bf21c290", "scores": { "conditional": 0.3518818622036406, "gm": 2.341521849409446, "ed": 0.26854343628850064, "sol": 1.6053842191502534, "nll": 5.380393636498143 } }, { "setting": "t10", "size": "1b", "step": 63000, "distance": 0.40554914070667547, "accuracy": 0.5153026239542494, "tasks": { "arc_challenge_acc": 0.23122866894197952, "arc_easy_acc": 0.5593434343434344, "lambada_openai_acc": 0.4723462060935377, "logiqa_acc": 0.22580645161290322, "piqa_acc": 0.6936887921653971, "sciq_acc": 0.903, "winogrande_acc": 0.5217048145224941 }, "job_id": "4069c9e61160195633c8f494", "scores": { "conditional": 0.40554914070667547, "gm": 1.6033445473890993, "ed": 0.33996400220693523, "sol": 1.5638270150352975, "nll": 5.25391294618959 } }, { "setting": "t10", "size": "1b", "step": 103000, "distance": 0.3982607000904188, "accuracy": 0.5333274933224145, "tasks": { "arc_challenge_acc": 0.25, "arc_easy_acc": 0.5909090909090909, "lambada_openai_acc": 0.4975742286046963, "logiqa_acc": 0.22580645161290322, "piqa_acc": 0.6996735582154516, "sciq_acc": 0.92, "winogrande_acc": 0.5493291239147593 }, "job_id": "e30a1c258db5d68b502213fd", "scores": { "conditional": 0.3982607000904188, "gm": 1.419481952992328, "ed": 0.27125997684703407, "sol": 1.4688039481155821, "nll": 5.007177490663779 } }, { "setting": "t10", "size": "1b", "step": 143000, "distance": 0.4271918303055753, "accuracy": 0.5349914679163845, "tasks": { "arc_challenge_acc": 0.2593856655290102, "arc_easy_acc": 0.5938552188552189, "lambada_openai_acc": 0.5072773141859112, "logiqa_acc": 0.22734254992319508, "piqa_acc": 0.705114254624592, "sciq_acc": 0.92, "winogrande_acc": 0.531965272296764 }, "job_id": "e36220de88b9456f9b386018", "scores": { "conditional": 0.4271918303055753, "gm": 1.5345794487225817, "ed": 0.30176882512661773, "sol": 1.5069004046419205, "nll": 5.045491852257875 } }, { "setting": "t10", "size": "6.9b", "step": 3000, "distance": 0.5617063272412396, "accuracy": 0.42391127208163926, "tasks": { "arc_challenge_acc": 0.1766211604095563, "arc_easy_acc": 0.4246632996632997, "lambada_openai_acc": 0.22549970890743257, "logiqa_acc": 0.23963133640552994, "piqa_acc": 0.6175190424374319, "sciq_acc": 0.772, "winogrande_acc": 0.5114443567482242 }, "job_id": "5827516b5b6a7e5e4001f02d", "scores": { "conditional": 0.5617063272412396, "gm": 2.8511824505019376, "ed": 0.573223367792087, "sol": 1.8733964247536508, "nll": 5.777233867869601 } }, { "setting": "t10", "size": "6.9b", "step": 13000, "distance": 0.3105641212931143, "accuracy": 0.5130790719439721, "tasks": { "arc_challenge_acc": 0.21928327645051193, "arc_easy_acc": 0.563973063973064, "lambada_openai_acc": 0.4511934795264894, "logiqa_acc": 0.2488479262672811, "piqa_acc": 0.6958650707290533, "sciq_acc": 0.897, "winogrande_acc": 0.5153906866614049 }, "job_id": "69087fefe37e4ef9f2e4da38", "scores": { "conditional": 0.3105641212931143, "gm": 1.734834694817664, "ed": 0.22953912181345615, "sol": 1.4728181850500233, "nll": 4.952632510680409 } }, { "setting": "t10", "size": "6.9b", "step": 33000, "distance": 0.29138288728025735, "accuracy": 0.5533024681765826, "tasks": { "arc_challenge_acc": 0.2883959044368601, "arc_easy_acc": 0.6334175084175084, "lambada_openai_acc": 0.5389093731806714, "logiqa_acc": 0.2119815668202765, "piqa_acc": 0.720348204570185, "sciq_acc": 0.926, "winogrande_acc": 0.5540647198105761 }, "job_id": "6b4e32f9f645c555ca459662", "scores": { "conditional": 0.29138288728025735, "gm": 1.5705062846494116, "ed": 0.17643474649638247, "sol": 1.4289745934158273, "nll": 4.907077885351917 } }, { "setting": "t10", "size": "6.9b", "step": 63000, "distance": 0.29803553716892633, "accuracy": 0.5820065388502569, "tasks": { "arc_challenge_acc": 0.318259385665529, "arc_easy_acc": 0.6813973063973064, "lambada_openai_acc": 0.5753929749660393, "logiqa_acc": 0.2304147465437788, "piqa_acc": 0.733949945593036, "sciq_acc": 0.934, "winogrande_acc": 0.6006314127861089 }, "job_id": "17f23fae1333eac665199ee7", "scores": { "conditional": 0.29803553716892633, "gm": 2.45125017173973, "ed": 0.17608098006974693, "sol": 1.3283616974570474, "nll": 4.567884708046838 } }, { "setting": "t10", "size": "6.9b", "step": 103000, "distance": 0.3298904933616853, "accuracy": 0.607810860185633, "tasks": { "arc_challenge_acc": 0.3370307167235495, "arc_easy_acc": 0.6919191919191919, "lambada_openai_acc": 0.6258490199883563, "logiqa_acc": 0.27035330261136714, "piqa_acc": 0.7453754080522307, "sciq_acc": 0.948, "winogrande_acc": 0.6361483820047356 }, "job_id": "62d26b1e941c27f828be55c0", "scores": { "conditional": 0.3298904933616853, "gm": 1.4099501261840715, "ed": 0.1721210646486968, "sol": 1.2884986821613806, "nll": 4.489357605172519 } }, { "setting": "t10", "size": "6.9b", "step": 143000, "distance": 0.3070592903542448, "accuracy": 0.6155622706270009, "tasks": { "arc_challenge_acc": 0.35580204778157, "arc_easy_acc": 0.7015993265993266, "lambada_openai_acc": 0.638074907820687, "logiqa_acc": 0.27035330261136714, "piqa_acc": 0.7551686615886833, "sciq_acc": 0.951, "winogrande_acc": 0.6369376479873717 }, "job_id": "c79c7db3ad789e7d508eba0f", "scores": { "conditional": 0.3070592903542448, "gm": 1.4801229619369565, "ed": 0.17738498893299148, "sol": 1.3083979667712076, "nll": 4.522492982367509 } }, { "setting": "t10-p095", "size": "160m", "step": 3000, "distance": 0.3303990315258933, "accuracy": 0.382655743432671, "tasks": { "arc_challenge_acc": 0.17064846416382254, "arc_easy_acc": 0.3792087542087542, "lambada_openai_acc": 0.15253250533669707, "logiqa_acc": 0.2350230414746544, "piqa_acc": 0.5903155603917302, "sciq_acc": 0.626, "winogrande_acc": 0.5248618784530387 }, "job_id": "8df4d91af506dd05ea53a47d", "scores": { "conditional": 0.3303990315258933, "gm": 5.103049137859671, "ed": 0.4516504343957184, "sol": 2.1934900883588915, "nll": 5.253145203438638 } }, { "setting": "t10-p095", "size": "160m", "step": 13000, "distance": 0.209939810433953, "accuracy": 0.42397207457676045, "tasks": { "arc_challenge_acc": 0.19027303754266212, "arc_easy_acc": 0.41792929292929293, "lambada_openai_acc": 0.22705220260042694, "logiqa_acc": 0.21351766513056836, "piqa_acc": 0.6164309031556039, "sciq_acc": 0.788, "winogrande_acc": 0.5146014206787688 }, "job_id": "0d3eca8fa817f3df11327962", "scores": { "conditional": 0.209939810433953, "gm": 4.298357956612108, "ed": 0.33727153379111385, "sol": 1.756381001949452, "nll": 4.959252737027903 } }, { "setting": "t10-p095", "size": "160m", "step": 33000, "distance": 0.17597803184338168, "accuracy": 0.44079549853155203, "tasks": { "arc_challenge_acc": 0.19880546075085323, "arc_easy_acc": 0.4393939393939394, "lambada_openai_acc": 0.2555792742091985, "logiqa_acc": 0.2227342549923195, "piqa_acc": 0.6365614798694232, "sciq_acc": 0.81, "winogrande_acc": 0.5224940805051302 }, "job_id": "0c3af991a5e4aa1dfebb4e83", "scores": { "conditional": 0.17597803184338168, "gm": 3.438543704782205, "ed": 0.13896020392549246, "sol": 1.2858950581588997, "nll": 4.544975804553598 } }, { "setting": "t10-p095", "size": "160m", "step": 63000, "distance": 0.15590997817097832, "accuracy": 0.4510944971867945, "tasks": { "arc_challenge_acc": 0.1945392491467577, "arc_easy_acc": 0.4473905723905724, "lambada_openai_acc": 0.3070056277896371, "logiqa_acc": 0.2350230414746544, "piqa_acc": 0.6289445048966268, "sciq_acc": 0.827, "winogrande_acc": 0.5177584846093133 }, "job_id": "1dc3974b2f87c29e89640fde", "scores": { "conditional": 0.15590997817097832, "gm": 2.3363495646032106, "ed": 0.15449405262397775, "sol": 1.763727526589497, "nll": 4.497645362632913 } }, { "setting": "t10-p095", "size": "160m", "step": 103000, "distance": 0.508979567472853, "accuracy": 0.44066173637581074, "tasks": { "arc_challenge_acc": 0.189419795221843, "arc_easy_acc": 0.45791245791245794, "lambada_openai_acc": 0.2718804579856394, "logiqa_acc": 0.21505376344086022, "piqa_acc": 0.6218715995647442, "sciq_acc": 0.806, "winogrande_acc": 0.5224940805051302 }, "job_id": "bfcbd483a8d0ee85694a7821", "scores": { "conditional": 0.508979567472853, "gm": 4.632658802045853, "ed": 0.5736688436725732, "sol": 2.017953299632533, "nll": 6.032778782448932 } }, { "setting": "t10-p095", "size": "160m", "step": 143000, "distance": 0.7126344358012477, "accuracy": 0.4309219653678175, "tasks": { "arc_challenge_acc": 0.18600682593856654, "arc_easy_acc": 0.4494949494949495, "lambada_openai_acc": 0.25732582961381717, "logiqa_acc": 0.21658986175115208, "piqa_acc": 0.6213275299238302, "sciq_acc": 0.779, "winogrande_acc": 0.5067087608524072 }, "job_id": "4d89fd02e245e5167f18f43f", "scores": { "conditional": 0.7126344358012477, "gm": 4.483484051285814, "ed": 1.2817338716744242, "sol": 2.8420514589633328, "nll": 7.214597410556155 } }, { "setting": "t10-p095", "size": "410m", "step": 3000, "distance": 0.2515222665340676, "accuracy": 0.4045096758199565, "tasks": { "arc_challenge_acc": 0.17832764505119453, "arc_easy_acc": 0.4137205387205387, "lambada_openai_acc": 0.18358237919658452, "logiqa_acc": 0.250384024577573, "piqa_acc": 0.5935799782372143, "sciq_acc": 0.71, "winogrande_acc": 0.5019731649565904 }, "job_id": "da94bbdfdabdb51c1c2af078", "scores": { "conditional": 0.2515222665340676, "gm": 4.408898585339557, "ed": 0.30924346574474804, "sol": 1.982901171026546, "nll": 4.67685589206495 } }, { "setting": "t10-p095", "size": "410m", "step": 13000, "distance": 0.16221565329521884, "accuracy": 0.46013927731831844, "tasks": { "arc_challenge_acc": 0.2022184300341297, "arc_easy_acc": 0.4739057239057239, "lambada_openai_acc": 0.33863768678439743, "logiqa_acc": 0.23348694316436253, "piqa_acc": 0.6327529923830251, "sciq_acc": 0.838, "winogrande_acc": 0.5019731649565904 }, "job_id": "da6eb41ab9ddcfb94cd1d035", "scores": { "conditional": 0.16221565329521884, "gm": 2.1725331094878584, "ed": 0.1869403464008048, "sol": 1.5042567928614567, "nll": 4.32965300724812 } }, { "setting": "t10-p095", "size": "410m", "step": 33000, "distance": 0.1445376795215686, "accuracy": 0.4857382433145336, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5117845117845118, "lambada_openai_acc": 0.38327188045798566, "logiqa_acc": 0.23809523809523808, "piqa_acc": 0.6692056583242655, "sciq_acc": 0.868, "winogrande_acc": 0.5122336227308603 }, "job_id": "7bb72c5d8d083dd8db49637c", "scores": { "conditional": 0.1445376795215686, "gm": 1.976953791816868, "ed": 0.13370051307665776, "sol": 1.3614420327815027, "nll": 4.221174866397954 } }, { "setting": "t10-p095", "size": "410m", "step": 63000, "distance": 0.15584046479236693, "accuracy": 0.4923627266517646, "tasks": { "arc_challenge_acc": 0.21331058020477817, "arc_easy_acc": 0.5416666666666666, "lambada_openai_acc": 0.4189792353968562, "logiqa_acc": 0.2073732718894009, "piqa_acc": 0.6670293797606094, "sciq_acc": 0.882, "winogrande_acc": 0.516179952644041 }, "job_id": "ec7664f2215929783abafb6e", "scores": { "conditional": 0.15584046479236693, "gm": 3.2883451000436708, "ed": 0.10669920147466172, "sol": 1.3430651149832356, "nll": 4.1916307391431165 } }, { "setting": "t10-p095", "size": "410m", "step": 103000, "distance": 0.11954824651684895, "accuracy": 0.5066002946739234, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5555555555555556, "lambada_openai_acc": 0.4315932466524355, "logiqa_acc": 0.22119815668202766, "piqa_acc": 0.6789989118607181, "sciq_acc": 0.903, "winogrande_acc": 0.5382794001578532 }, "job_id": "1e55a7a91ce11361295bd055", "scores": { "conditional": 0.11954824651684895, "gm": 2.879469764094438, "ed": 0.14530581660437036, "sol": 1.3261873100940886, "nll": 4.142665869274325 } }, { "setting": "t10-p095", "size": "410m", "step": 143000, "distance": 0.15737290593732456, "accuracy": 0.5071258254183677, "tasks": { "arc_challenge_acc": 0.22098976109215018, "arc_easy_acc": 0.5547138047138047, "lambada_openai_acc": 0.45546283718222397, "logiqa_acc": 0.2196620583717358, "piqa_acc": 0.6784548422198041, "sciq_acc": 0.891, "winogrande_acc": 0.5295974743488555 }, "job_id": "d921dc56f5e0c20e396dce43", "scores": { "conditional": 0.15737290593732456, "gm": 2.399840212464369, "ed": 0.11472718159227568, "sol": 1.2010087025241423, "nll": 4.125894032315678 } }, { "setting": "t10-p095", "size": "1b", "step": 3000, "distance": 0.2249769331004934, "accuracy": 0.41411263962670136, "tasks": { "arc_challenge_acc": 0.19368600682593856, "arc_easy_acc": 0.41792929292929293, "lambada_openai_acc": 0.2066757228798758, "logiqa_acc": 0.22734254992319508, "piqa_acc": 0.6120783460282916, "sciq_acc": 0.732, "winogrande_acc": 0.5090765588003157 }, "job_id": "d4ccab331ac3af38469b59d4", "scores": { "conditional": 0.2249769331004934, "gm": 3.4057260377767307, "ed": 0.23657565487747334, "sol": 1.5753082891532202, "nll": 4.807563643482387 } }, { "setting": "t10-p095", "size": "1b", "step": 13000, "distance": 0.14313341701725996, "accuracy": 0.479511269800246, "tasks": { "arc_challenge_acc": 0.20648464163822525, "arc_easy_acc": 0.5042087542087542, "lambada_openai_acc": 0.36231321560256163, "logiqa_acc": 0.23963133640552994, "piqa_acc": 0.6572361262241567, "sciq_acc": 0.865, "winogrande_acc": 0.5217048145224941 }, "job_id": "23503d9d2b44015ab8e15861", "scores": { "conditional": 0.14313341701725996, "gm": 2.122544659808682, "ed": 0.10128241105230096, "sol": 1.3415093324899257, "nll": 4.107707438994052 } }, { "setting": "t10-p095", "size": "1b", "step": 33000, "distance": 0.055621718577177535, "accuracy": 0.4982874070398225, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5492424242424242, "lambada_openai_acc": 0.4321754317873084, "logiqa_acc": 0.2196620583717358, "piqa_acc": 0.6811751904243744, "sciq_acc": 0.872, "winogrande_acc": 0.516179952644041 }, "job_id": "522c7170f59ea100ee5405b9", "scores": { "conditional": 0.055621718577177535, "gm": 2.6370672438883958, "ed": 0.054156190845128904, "sol": 1.2127325495961176, "nll": 3.953899296464816 } }, { "setting": "t10-p095", "size": "1b", "step": 63000, "distance": 0.11261734951846127, "accuracy": 0.5153026239542494, "tasks": { "arc_challenge_acc": 0.23122866894197952, "arc_easy_acc": 0.5593434343434344, "lambada_openai_acc": 0.4723462060935377, "logiqa_acc": 0.22580645161290322, "piqa_acc": 0.6936887921653971, "sciq_acc": 0.903, "winogrande_acc": 0.5217048145224941 }, "job_id": "73fcbfc237280886433a1576", "scores": { "conditional": 0.11261734951846127, "gm": 2.9788252363632637, "ed": 0.09969588897261694, "sol": 1.327704229641457, "nll": 3.9837372439808747 } }, { "setting": "t10-p095", "size": "1b", "step": 103000, "distance": 0.09098016293388654, "accuracy": 0.5333274933224145, "tasks": { "arc_challenge_acc": 0.25, "arc_easy_acc": 0.5909090909090909, "lambada_openai_acc": 0.4975742286046963, "logiqa_acc": 0.22580645161290322, "piqa_acc": 0.6996735582154516, "sciq_acc": 0.92, "winogrande_acc": 0.5493291239147593 }, "job_id": "ef3d90904ed86724d5b936f8", "scores": { "conditional": 0.09098016293388654, "gm": 1.7976380625210595, "ed": 0.07795115315513268, "sol": 1.1845556260824224, "nll": 3.8453572961543956 } }, { "setting": "t10-p095", "size": "1b", "step": 143000, "distance": 0.08288978540613015, "accuracy": 0.5349914679163845, "tasks": { "arc_challenge_acc": 0.2593856655290102, "arc_easy_acc": 0.5938552188552189, "lambada_openai_acc": 0.5072773141859112, "logiqa_acc": 0.22734254992319508, "piqa_acc": 0.705114254624592, "sciq_acc": 0.92, "winogrande_acc": 0.531965272296764 }, "job_id": "1cfcaa16d4019c21498b8a0e", "scores": { "conditional": 0.08288978540613015, "gm": 2.2254772054900513, "ed": 0.06991829056496446, "sol": 1.2508832117358162, "nll": 3.8010593009452758 } }, { "setting": "t10-p095", "size": "6.9b", "step": 3000, "distance": 0.21916048690347872, "accuracy": 0.42391127208163926, "tasks": { "arc_challenge_acc": 0.1766211604095563, "arc_easy_acc": 0.4246632996632997, "lambada_openai_acc": 0.22549970890743257, "logiqa_acc": 0.23963133640552994, "piqa_acc": 0.6175190424374319, "sciq_acc": 0.772, "winogrande_acc": 0.5114443567482242 }, "job_id": "9e14c7e294daa6104004a1de", "scores": { "conditional": 0.21916048690347872, "gm": 2.6111705586777134, "ed": 0.2834429339971827, "sol": 1.702281037625816, "nll": 4.283165799611843 } }, { "setting": "t10-p095", "size": "6.9b", "step": 13000, "distance": 0.1311572426893367, "accuracy": 0.5130790719439721, "tasks": { "arc_challenge_acc": 0.21928327645051193, "arc_easy_acc": 0.563973063973064, "lambada_openai_acc": 0.4511934795264894, "logiqa_acc": 0.2488479262672811, "piqa_acc": 0.6958650707290533, "sciq_acc": 0.897, "winogrande_acc": 0.5153906866614049 }, "job_id": "8b82d08285726303bd559318", "scores": { "conditional": 0.1311572426893367, "gm": 2.041709707710993, "ed": 0.10480379655640526, "sol": 1.2876527462920508, "nll": 3.8510988147580134 } }, { "setting": "t10-p095", "size": "6.9b", "step": 33000, "distance": 0.061760810056771864, "accuracy": 0.5533024681765826, "tasks": { "arc_challenge_acc": 0.2883959044368601, "arc_easy_acc": 0.6334175084175084, "lambada_openai_acc": 0.5389093731806714, "logiqa_acc": 0.2119815668202765, "piqa_acc": 0.720348204570185, "sciq_acc": 0.926, "winogrande_acc": 0.5540647198105761 }, "job_id": "5c6ffb87adb1c0ba9c3aba01", "scores": { "conditional": 0.061760810056771864, "gm": 1.6672665593855527, "ed": 0.056373969741857266, "sol": 1.2212782658887058, "nll": 3.652516670095789 } }, { "setting": "t10-p095", "size": "6.9b", "step": 63000, "distance": 0.06140426968412173, "accuracy": 0.5820065388502569, "tasks": { "arc_challenge_acc": 0.318259385665529, "arc_easy_acc": 0.6813973063973064, "lambada_openai_acc": 0.5753929749660393, "logiqa_acc": 0.2304147465437788, "piqa_acc": 0.733949945593036, "sciq_acc": 0.934, "winogrande_acc": 0.6006314127861089 }, "job_id": "83e4c5e21a727165aaf0db0d", "scores": { "conditional": 0.06140426968412173, "gm": 5.4126023851279115, "ed": 0.0662844420120674, "sol": 1.250954225672577, "nll": 3.472630334091902 } }, { "setting": "t10-p095", "size": "6.9b", "step": 103000, "distance": 0.08899791239000965, "accuracy": 0.607810860185633, "tasks": { "arc_challenge_acc": 0.3370307167235495, "arc_easy_acc": 0.6919191919191919, "lambada_openai_acc": 0.6258490199883563, "logiqa_acc": 0.27035330261136714, "piqa_acc": 0.7453754080522307, "sciq_acc": 0.948, "winogrande_acc": 0.6361483820047356 }, "job_id": "69f62f35014df8f09a4025bc", "scores": { "conditional": 0.08899791239000965, "gm": 2.270191196138967, "ed": 0.05452296029316983, "sol": 1.1955283977530309, "nll": 3.4386456299074486 } }, { "setting": "t10-p095", "size": "6.9b", "step": 143000, "distance": 0.10349051282540767, "accuracy": 0.6155622706270009, "tasks": { "arc_challenge_acc": 0.35580204778157, "arc_easy_acc": 0.7015993265993266, "lambada_openai_acc": 0.638074907820687, "logiqa_acc": 0.27035330261136714, "piqa_acc": 0.7551686615886833, "sciq_acc": 0.951, "winogrande_acc": 0.6369376479873717 }, "job_id": "933d7740c4013c90ae2ced84", "scores": { "conditional": 0.10349051282540767, "gm": 1.5130550866372232, "ed": 0.06523750246181059, "sol": 1.2291466907956794, "nll": 3.451090308778791 } }, { "setting": "t10-k20", "size": "160m", "step": 3000, "distance": 1.017486226225539, "accuracy": 0.382655743432671, "tasks": { "arc_challenge_acc": 0.17064846416382254, "arc_easy_acc": 0.3792087542087542, "lambada_openai_acc": 0.15253250533669707, "logiqa_acc": 0.2350230414746544, "piqa_acc": 0.5903155603917302, "sciq_acc": 0.626, "winogrande_acc": 0.5248618784530387 }, "job_id": "3147c68cae5eac10fe9f31d2", "scores": { "conditional": 1.017486226225539, "gm": 12.665245026617413, "ed": 1.6179191334155743, "sol": 2.4851671594748597, "nll": 2.461973271752223 } }, { "setting": "t10-k20", "size": "160m", "step": 13000, "distance": 0.6289372781789949, "accuracy": 0.42397207457676045, "tasks": { "arc_challenge_acc": 0.19027303754266212, "arc_easy_acc": 0.41792929292929293, "lambada_openai_acc": 0.22705220260042694, "logiqa_acc": 0.21351766513056836, "piqa_acc": 0.6164309031556039, "sciq_acc": 0.788, "winogrande_acc": 0.5146014206787688 }, "job_id": "6adc40d57d879bbc547fb0bf", "scores": { "conditional": 0.6289372781789949, "gm": 20.98162595826534, "ed": 1.0565406844385876, "sol": 2.097312516323563, "nll": 2.625025996957505 } }, { "setting": "t10-k20", "size": "160m", "step": 33000, "distance": 0.5619194737426361, "accuracy": 0.44079549853155203, "tasks": { "arc_challenge_acc": 0.19880546075085323, "arc_easy_acc": 0.4393939393939394, "lambada_openai_acc": 0.2555792742091985, "logiqa_acc": 0.2227342549923195, "piqa_acc": 0.6365614798694232, "sciq_acc": 0.81, "winogrande_acc": 0.5224940805051302 }, "job_id": "00ca6049e0bc37cd219970ac", "scores": { "conditional": 0.5619194737426361, "gm": 24.306175152802098, "ed": 0.8518670752937467, "sol": 1.9769987320646503, "nll": 2.6412466244264734 } }, { "setting": "t10-k20", "size": "160m", "step": 63000, "distance": 0.8086440716265832, "accuracy": 0.4510944971867945, "tasks": { "arc_challenge_acc": 0.1945392491467577, "arc_easy_acc": 0.4473905723905724, "lambada_openai_acc": 0.3070056277896371, "logiqa_acc": 0.2350230414746544, "piqa_acc": 0.6289445048966268, "sciq_acc": 0.827, "winogrande_acc": 0.5177584846093133 }, "job_id": "3a7ae688180b9c97cc4e1185", "scores": { "conditional": 0.8086440716265832, "gm": 31.241888919373196, "ed": 1.0767927538103468, "sol": 2.4365388197341513, "nll": 2.5232566158043492 } }, { "setting": "t10-k20", "size": "160m", "step": 103000, "distance": 0.5740517256074712, "accuracy": 0.44066173637581074, "tasks": { "arc_challenge_acc": 0.189419795221843, "arc_easy_acc": 0.45791245791245794, "lambada_openai_acc": 0.2718804579856394, "logiqa_acc": 0.21505376344086022, "piqa_acc": 0.6218715995647442, "sciq_acc": 0.806, "winogrande_acc": 0.5224940805051302 }, "job_id": "bf517fbccd300502c5778dee", "scores": { "conditional": 0.5740517256074712, "gm": 23.322545758097476, "ed": 0.955027544995044, "sol": 2.0602826377131693, "nll": 2.5952144726327586 } }, { "setting": "t10-k20", "size": "160m", "step": 143000, "distance": 0.49466733676832214, "accuracy": 0.4309219653678175, "tasks": { "arc_challenge_acc": 0.18600682593856654, "arc_easy_acc": 0.4494949494949495, "lambada_openai_acc": 0.25732582961381717, "logiqa_acc": 0.21658986175115208, "piqa_acc": 0.6213275299238302, "sciq_acc": 0.779, "winogrande_acc": 0.5067087608524072 }, "job_id": "5026b0e9e3a38be2951085f3", "scores": { "conditional": 0.49466733676832214, "gm": 31.538044466044084, "ed": 0.8225054657657846, "sol": 2.093765132270266, "nll": 2.7542547094811067 } }, { "setting": "t10-k20", "size": "410m", "step": 3000, "distance": 0.96643134089724, "accuracy": 0.4045096758199565, "tasks": { "arc_challenge_acc": 0.17832764505119453, "arc_easy_acc": 0.4137205387205387, "lambada_openai_acc": 0.18358237919658452, "logiqa_acc": 0.250384024577573, "piqa_acc": 0.5935799782372143, "sciq_acc": 0.71, "winogrande_acc": 0.5019731649565904 }, "job_id": "0fcb87310f2a4d080122367b", "scores": { "conditional": 0.96643134089724, "gm": 15.543866366094335, "ed": 1.5060872381445627, "sol": 2.458354116354065, "nll": 2.4450348631930523 } }, { "setting": "t10-k20", "size": "410m", "step": 13000, "distance": 0.598989668621076, "accuracy": 0.46013927731831844, "tasks": { "arc_challenge_acc": 0.2022184300341297, "arc_easy_acc": 0.4739057239057239, "lambada_openai_acc": 0.33863768678439743, "logiqa_acc": 0.23348694316436253, "piqa_acc": 0.6327529923830251, "sciq_acc": 0.838, "winogrande_acc": 0.5019731649565904 }, "job_id": "bc5b10f4ed2ddd067d896c09", "scores": { "conditional": 0.598989668621076, "gm": 24.811786143049176, "ed": 0.7861294261488148, "sol": 2.0044326227707145, "nll": 2.585455683202501 } }, { "setting": "t10-k20", "size": "410m", "step": 33000, "distance": 0.46785732091976334, "accuracy": 0.4857382433145336, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5117845117845118, "lambada_openai_acc": 0.38327188045798566, "logiqa_acc": 0.23809523809523808, "piqa_acc": 0.6692056583242655, "sciq_acc": 0.868, "winogrande_acc": 0.5122336227308603 }, "job_id": "ff544823cdcda7e9e55773b5", "scores": { "conditional": 0.46785732091976334, "gm": 13.593461972421665, "ed": 0.5915591761770393, "sol": 1.7582403654740562, "nll": 2.6549579619167103 } }, { "setting": "t10-k20", "size": "410m", "step": 63000, "distance": 0.4093493174087839, "accuracy": 0.4923627266517646, "tasks": { "arc_challenge_acc": 0.21331058020477817, "arc_easy_acc": 0.5416666666666666, "lambada_openai_acc": 0.4189792353968562, "logiqa_acc": 0.2073732718894009, "piqa_acc": 0.6670293797606094, "sciq_acc": 0.882, "winogrande_acc": 0.516179952644041 }, "job_id": "e45a81d76dce159453cf2980", "scores": { "conditional": 0.4093493174087839, "gm": 12.962388710772302, "ed": 0.6760739712994379, "sol": 1.8267726452284698, "nll": 2.656388991156558 } }, { "setting": "t10-k20", "size": "410m", "step": 103000, "distance": 0.385123744636859, "accuracy": 0.5066002946739234, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5555555555555556, "lambada_openai_acc": 0.4315932466524355, "logiqa_acc": 0.22119815668202766, "piqa_acc": 0.6789989118607181, "sciq_acc": 0.903, "winogrande_acc": 0.5382794001578532 }, "job_id": "ba417affad92bc8fc691f21b", "scores": { "conditional": 0.385123744636859, "gm": 29.119012946403018, "ed": 0.49988615012274895, "sol": 1.9386944182232562, "nll": 2.673056637909806 } }, { "setting": "t10-k20", "size": "410m", "step": 143000, "distance": 0.35971542962625014, "accuracy": 0.5071258254183677, "tasks": { "arc_challenge_acc": 0.22098976109215018, "arc_easy_acc": 0.5547138047138047, "lambada_openai_acc": 0.45546283718222397, "logiqa_acc": 0.2196620583717358, "piqa_acc": 0.6784548422198041, "sciq_acc": 0.891, "winogrande_acc": 0.5295974743488555 }, "job_id": "41bfcdda1a7ddd9e3ef9ef65", "scores": { "conditional": 0.35971542962625014, "gm": 15.84070124643631, "ed": 0.4081596897239015, "sol": 1.699735311507962, "nll": 2.6513228637379833 } }, { "setting": "t10-k20", "size": "1b", "step": 3000, "distance": 0.6683131964410047, "accuracy": 0.41411263962670136, "tasks": { "arc_challenge_acc": 0.19368600682593856, "arc_easy_acc": 0.41792929292929293, "lambada_openai_acc": 0.2066757228798758, "logiqa_acc": 0.22734254992319508, "piqa_acc": 0.6120783460282916, "sciq_acc": 0.732, "winogrande_acc": 0.5090765588003157 }, "job_id": "bea9970c2d3b2407514f2af0", "scores": { "conditional": 0.6683131964410047, "gm": 23.62378376428751, "ed": 0.9763059085495467, "sol": 2.155622311183041, "nll": 2.6799267880725277 } }, { "setting": "t10-k20", "size": "1b", "step": 13000, "distance": 0.5682824670167823, "accuracy": 0.479511269800246, "tasks": { "arc_challenge_acc": 0.20648464163822525, "arc_easy_acc": 0.5042087542087542, "lambada_openai_acc": 0.36231321560256163, "logiqa_acc": 0.23963133640552994, "piqa_acc": 0.6572361262241567, "sciq_acc": 0.865, "winogrande_acc": 0.5217048145224941 }, "job_id": "1e16a815f00763f00dca218e", "scores": { "conditional": 0.5682824670167823, "gm": 28.79340363117626, "ed": 0.6431848714275592, "sol": 2.0645150515874686, "nll": 2.5383674382117167 } }, { "setting": "t10-k20", "size": "1b", "step": 33000, "distance": 0.5213863234371299, "accuracy": 0.4982874070398225, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5492424242424242, "lambada_openai_acc": 0.4321754317873084, "logiqa_acc": 0.2196620583717358, "piqa_acc": 0.6811751904243744, "sciq_acc": 0.872, "winogrande_acc": 0.516179952644041 }, "job_id": "03be4a80aa4ae09384a86316", "scores": { "conditional": 0.5213863234371299, "gm": 29.24397921669879, "ed": 0.6420152785300495, "sol": 2.0749122196752223, "nll": 2.493244695025369 } }, { "setting": "t10-k20", "size": "1b", "step": 63000, "distance": 0.3355195829199768, "accuracy": 0.5153026239542494, "tasks": { "arc_challenge_acc": 0.23122866894197952, "arc_easy_acc": 0.5593434343434344, "lambada_openai_acc": 0.4723462060935377, "logiqa_acc": 0.22580645161290322, "piqa_acc": 0.6936887921653971, "sciq_acc": 0.903, "winogrande_acc": 0.5217048145224941 }, "job_id": "98990e63fb58a76265154138", "scores": { "conditional": 0.3355195829199768, "gm": 24.17870731648145, "ed": 0.5007495037779854, "sol": 2.1002289946625203, "nll": 2.669608674469164 } }, { "setting": "t10-k20", "size": "1b", "step": 103000, "distance": 0.327846479971823, "accuracy": 0.5333274933224145, "tasks": { "arc_challenge_acc": 0.25, "arc_easy_acc": 0.5909090909090909, "lambada_openai_acc": 0.4975742286046963, "logiqa_acc": 0.22580645161290322, "piqa_acc": 0.6996735582154516, "sciq_acc": 0.92, "winogrande_acc": 0.5493291239147593 }, "job_id": "d05a9ebbbf287ab19f0eeb72", "scores": { "conditional": 0.327846479971823, "gm": 27.178019372658127, "ed": 0.44688993828563817, "sol": 1.9383908585900043, "nll": 2.564022905835893 } }, { "setting": "t10-k20", "size": "1b", "step": 143000, "distance": 0.30655848420642684, "accuracy": 0.5349914679163845, "tasks": { "arc_challenge_acc": 0.2593856655290102, "arc_easy_acc": 0.5938552188552189, "lambada_openai_acc": 0.5072773141859112, "logiqa_acc": 0.22734254992319508, "piqa_acc": 0.705114254624592, "sciq_acc": 0.92, "winogrande_acc": 0.531965272296764 }, "job_id": "be9a0b47dfd2097761c8c0ef", "scores": { "conditional": 0.30655848420642684, "gm": 26.791510352662044, "ed": 0.40469360625395545, "sol": 1.8372479079867685, "nll": 2.53930545665633 } }, { "setting": "t10-k20", "size": "6.9b", "step": 3000, "distance": 0.9741125955527723, "accuracy": 0.42391127208163926, "tasks": { "arc_challenge_acc": 0.1766211604095563, "arc_easy_acc": 0.4246632996632997, "lambada_openai_acc": 0.22549970890743257, "logiqa_acc": 0.23963133640552994, "piqa_acc": 0.6175190424374319, "sciq_acc": 0.772, "winogrande_acc": 0.5114443567482242 }, "job_id": "fd6cd63b7761c51a72cf3c89", "scores": { "conditional": 0.9741125955527723, "gm": 19.233178767273785, "ed": 1.4259889566048596, "sol": 2.330323832524362, "nll": 2.362408452056622 } }, { "setting": "t10-k20", "size": "6.9b", "step": 13000, "distance": 0.4018489924197613, "accuracy": 0.5130790719439721, "tasks": { "arc_challenge_acc": 0.21928327645051193, "arc_easy_acc": 0.563973063973064, "lambada_openai_acc": 0.4511934795264894, "logiqa_acc": 0.2488479262672811, "piqa_acc": 0.6958650707290533, "sciq_acc": 0.897, "winogrande_acc": 0.5153906866614049 }, "job_id": "b60424fa1f52e5ad076f5af8", "scores": { "conditional": 0.4018489924197613, "gm": 29.989730577176488, "ed": 0.5505749354221239, "sol": 2.081449362573418, "nll": 2.5284095193031755 } }, { "setting": "t10-k20", "size": "6.9b", "step": 33000, "distance": 0.4571693010002693, "accuracy": 0.5533024681765826, "tasks": { "arc_challenge_acc": 0.2883959044368601, "arc_easy_acc": 0.6334175084175084, "lambada_openai_acc": 0.5389093731806714, "logiqa_acc": 0.2119815668202765, "piqa_acc": 0.720348204570185, "sciq_acc": 0.926, "winogrande_acc": 0.5540647198105761 }, "job_id": "fb137937c36c273121a40079", "scores": { "conditional": 0.4571693010002693, "gm": 20.926797754192194, "ed": 0.451205902675075, "sol": 1.9811774895098575, "nll": 2.5234942176672694 } }, { "setting": "t10-k20", "size": "6.9b", "step": 63000, "distance": 0.2525590679491211, "accuracy": 0.5820065388502569, "tasks": { "arc_challenge_acc": 0.318259385665529, "arc_easy_acc": 0.6813973063973064, "lambada_openai_acc": 0.5753929749660393, "logiqa_acc": 0.2304147465437788, "piqa_acc": 0.733949945593036, "sciq_acc": 0.934, "winogrande_acc": 0.6006314127861089 }, "job_id": "7db94a28a2e5833d3dcaacc0", "scores": { "conditional": 0.2525590679491211, "gm": 44.95384589704918, "ed": 0.4931008467819771, "sol": 2.255600796699871, "nll": 2.4982963389203823 } }, { "setting": "t10-k20", "size": "6.9b", "step": 103000, "distance": 0.2916282281017145, "accuracy": 0.607810860185633, "tasks": { "arc_challenge_acc": 0.3370307167235495, "arc_easy_acc": 0.6919191919191919, "lambada_openai_acc": 0.6258490199883563, "logiqa_acc": 0.27035330261136714, "piqa_acc": 0.7453754080522307, "sciq_acc": 0.948, "winogrande_acc": 0.6361483820047356 }, "job_id": "b808ee35f69c379e2c72444a", "scores": { "conditional": 0.2916282281017145, "gm": 18.596686019906883, "ed": 0.29921000235540696, "sol": 1.873640596781573, "nll": 2.4872086102813054 } }, { "setting": "t10-k20", "size": "6.9b", "step": 143000, "distance": 0.23805852116041598, "accuracy": 0.6155622706270009, "tasks": { "arc_challenge_acc": 0.35580204778157, "arc_easy_acc": 0.7015993265993266, "lambada_openai_acc": 0.638074907820687, "logiqa_acc": 0.27035330261136714, "piqa_acc": 0.7551686615886833, "sciq_acc": 0.951, "winogrande_acc": 0.6369376479873717 }, "job_id": "ee429049c8eeb243b3d02e64", "scores": { "conditional": 0.23805852116041598, "gm": 18.0700459192808, "ed": 0.2720807163135266, "sol": 1.7909372319884063, "nll": 2.4951072182707343 } }, { "setting": "t13", "size": "160m", "step": 3000, "distance": 1.4434032308969424, "accuracy": 0.382655743432671, "tasks": { "arc_challenge_acc": 0.17064846416382254, "arc_easy_acc": 0.3792087542087542, "lambada_openai_acc": 0.15253250533669707, "logiqa_acc": 0.2350230414746544, "piqa_acc": 0.5903155603917302, "sciq_acc": 0.626, "winogrande_acc": 0.5248618784530387 }, "job_id": "0a88a9191c64b00bb3216ea4", "scores": { "conditional": 1.4434032308969424, "gm": 37.727204297196295, "ed": 6.2079390065734295, "sol": 6.537235323097982, "nll": 13.741850256983469 } }, { "setting": "t13", "size": "160m", "step": 13000, "distance": 1.507621164285741, "accuracy": 0.42397207457676045, "tasks": { "arc_challenge_acc": 0.19027303754266212, "arc_easy_acc": 0.41792929292929293, "lambada_openai_acc": 0.22705220260042694, "logiqa_acc": 0.21351766513056836, "piqa_acc": 0.6164309031556039, "sciq_acc": 0.788, "winogrande_acc": 0.5146014206787688 }, "job_id": "0bb72bc8288cdb54e46261b1", "scores": { "conditional": 1.507621164285741, "gm": 35.371492206471885, "ed": 6.987822394391595, "sol": 7.129057823722283, "nll": 14.10680482254659 } }, { "setting": "t13", "size": "160m", "step": 33000, "distance": 1.5526264300694976, "accuracy": 0.44079549853155203, "tasks": { "arc_challenge_acc": 0.19880546075085323, "arc_easy_acc": 0.4393939393939394, "lambada_openai_acc": 0.2555792742091985, "logiqa_acc": 0.2227342549923195, "piqa_acc": 0.6365614798694232, "sciq_acc": 0.81, "winogrande_acc": 0.5224940805051302 }, "job_id": "bfd6a7006589c25b7a886891", "scores": { "conditional": 1.5526264300694976, "gm": 34.13606860614891, "ed": 6.667648765574607, "sol": 6.837999739556647, "nll": 14.153776874635058 } }, { "setting": "t13", "size": "160m", "step": 63000, "distance": 1.4426994357487248, "accuracy": 0.4510944971867945, "tasks": { "arc_challenge_acc": 0.1945392491467577, "arc_easy_acc": 0.4473905723905724, "lambada_openai_acc": 0.3070056277896371, "logiqa_acc": 0.2350230414746544, "piqa_acc": 0.6289445048966268, "sciq_acc": 0.827, "winogrande_acc": 0.5177584846093133 }, "job_id": "644c151d52a269b79e624f23", "scores": { "conditional": 1.4426994357487248, "gm": 36.65043659498042, "ed": 6.692954607863746, "sol": 7.143964709658832, "nll": 14.203208689776892 } }, { "setting": "t13", "size": "160m", "step": 103000, "distance": 1.3853849196818677, "accuracy": 0.44066173637581074, "tasks": { "arc_challenge_acc": 0.189419795221843, "arc_easy_acc": 0.45791245791245794, "lambada_openai_acc": 0.2718804579856394, "logiqa_acc": 0.21505376344086022, "piqa_acc": 0.6218715995647442, "sciq_acc": 0.806, "winogrande_acc": 0.5224940805051302 }, "job_id": "49f05ee111b4419ef69bb347", "scores": { "conditional": 1.3853849196818677, "gm": 36.23467858006916, "ed": 6.798418914946668, "sol": 7.334339613694089, "nll": 14.189942578244516 } }, { "setting": "t13", "size": "160m", "step": 143000, "distance": 1.3551483700678826, "accuracy": 0.4309219653678175, "tasks": { "arc_challenge_acc": 0.18600682593856654, "arc_easy_acc": 0.4494949494949495, "lambada_openai_acc": 0.25732582961381717, "logiqa_acc": 0.21658986175115208, "piqa_acc": 0.6213275299238302, "sciq_acc": 0.779, "winogrande_acc": 0.5067087608524072 }, "job_id": "d361aa28ff8dd406d6d323b8", "scores": { "conditional": 1.3551483700678826, "gm": 36.716217957800836, "ed": 6.81358480438166, "sol": 7.481009123874007, "nll": 14.362345238284828 } }, { "setting": "t13", "size": "410m", "step": 3000, "distance": 1.4306930556923776, "accuracy": 0.4045096758199565, "tasks": { "arc_challenge_acc": 0.17832764505119453, "arc_easy_acc": 0.4137205387205387, "lambada_openai_acc": 0.18358237919658452, "logiqa_acc": 0.250384024577573, "piqa_acc": 0.5935799782372143, "sciq_acc": 0.71, "winogrande_acc": 0.5019731649565904 }, "job_id": "cebe01e0dd3c37adea936501", "scores": { "conditional": 1.4306930556923776, "gm": 33.601696875237394, "ed": 5.610558208167639, "sol": 6.597681966303726, "nll": 13.59871578917199 } }, { "setting": "t13", "size": "410m", "step": 13000, "distance": 1.3468576094574782, "accuracy": 0.46013927731831844, "tasks": { "arc_challenge_acc": 0.2022184300341297, "arc_easy_acc": 0.4739057239057239, "lambada_openai_acc": 0.33863768678439743, "logiqa_acc": 0.23348694316436253, "piqa_acc": 0.6327529923830251, "sciq_acc": 0.838, "winogrande_acc": 0.5019731649565904 }, "job_id": "2547f77eb5620beb6b679b39", "scores": { "conditional": 1.3468576094574782, "gm": 36.459914121929614, "ed": 6.609483051418923, "sol": 7.172893929161671, "nll": 14.324760987508885 } }, { "setting": "t13", "size": "410m", "step": 33000, "distance": 1.7685253729060977, "accuracy": 0.4857382433145336, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5117845117845118, "lambada_openai_acc": 0.38327188045798566, "logiqa_acc": 0.23809523809523808, "piqa_acc": 0.6692056583242655, "sciq_acc": 0.868, "winogrande_acc": 0.5122336227308603 }, "job_id": "e506e6eb2e1086e8b50a5255", "scores": { "conditional": 1.7685253729060977, "gm": 34.13426489763515, "ed": 8.449304691829024, "sol": 7.030551021491878, "nll": 14.297772815595252 } }, { "setting": "t13", "size": "410m", "step": 63000, "distance": 1.5085597810150548, "accuracy": 0.4923627266517646, "tasks": { "arc_challenge_acc": 0.21331058020477817, "arc_easy_acc": 0.5416666666666666, "lambada_openai_acc": 0.4189792353968562, "logiqa_acc": 0.2073732718894009, "piqa_acc": 0.6670293797606094, "sciq_acc": 0.882, "winogrande_acc": 0.516179952644041 }, "job_id": "8fdd583f06fc23a5c2afee0b", "scores": { "conditional": 1.5085597810150548, "gm": 36.1161663420462, "ed": 6.7673617224251625, "sol": 7.106939812091696, "nll": 14.447165877140002 } }, { "setting": "t13", "size": "410m", "step": 103000, "distance": 1.5386767266832728, "accuracy": 0.5066002946739234, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5555555555555556, "lambada_openai_acc": 0.4315932466524355, "logiqa_acc": 0.22119815668202766, "piqa_acc": 0.6789989118607181, "sciq_acc": 0.903, "winogrande_acc": 0.5382794001578532 }, "job_id": "b5aa1cc7171ca9621f1ec2b1", "scores": { "conditional": 1.5386767266832728, "gm": 35.89411251492493, "ed": 6.668922267119607, "sol": 6.9794119984093985, "nll": 14.338068384954576 } }, { "setting": "t13", "size": "410m", "step": 143000, "distance": 1.4473884431126671, "accuracy": 0.5071258254183677, "tasks": { "arc_challenge_acc": 0.22098976109215018, "arc_easy_acc": 0.5547138047138047, "lambada_openai_acc": 0.45546283718222397, "logiqa_acc": 0.2196620583717358, "piqa_acc": 0.6784548422198041, "sciq_acc": 0.891, "winogrande_acc": 0.5295974743488555 }, "job_id": "e8f2630f84f218705d1684ca", "scores": { "conditional": 1.4473884431126671, "gm": 36.50855012274042, "ed": 6.753089997842857, "sol": 7.083951691193226, "nll": 14.363648515087283 } }, { "setting": "t13", "size": "1b", "step": 3000, "distance": 1.4536057207582753, "accuracy": 0.41411263962670136, "tasks": { "arc_challenge_acc": 0.19368600682593856, "arc_easy_acc": 0.41792929292929293, "lambada_openai_acc": 0.2066757228798758, "logiqa_acc": 0.22734254992319508, "piqa_acc": 0.6120783460282916, "sciq_acc": 0.732, "winogrande_acc": 0.5090765588003157 }, "job_id": "b36faea053c31f61cf3aecd4", "scores": { "conditional": 1.4536057207582753, "gm": 36.900740926881554, "ed": 6.094003765142589, "sol": 6.8877794802240375, "nll": 13.877256498203849 } }, { "setting": "t13", "size": "1b", "step": 13000, "distance": 1.4082800703124738, "accuracy": 0.479511269800246, "tasks": { "arc_challenge_acc": 0.20648464163822525, "arc_easy_acc": 0.5042087542087542, "lambada_openai_acc": 0.36231321560256163, "logiqa_acc": 0.23963133640552994, "piqa_acc": 0.6572361262241567, "sciq_acc": 0.865, "winogrande_acc": 0.5217048145224941 }, "job_id": "25798b2638642a75ae5ee475", "scores": { "conditional": 1.4082800703124738, "gm": 35.53064674559573, "ed": 7.092724887502366, "sol": 7.309234100889768, "nll": 14.365663005988164 } }, { "setting": "t13", "size": "1b", "step": 33000, "distance": 1.6232993435735588, "accuracy": 0.4982874070398225, "tasks": { "arc_challenge_acc": 0.2175767918088737, "arc_easy_acc": 0.5492424242424242, "lambada_openai_acc": 0.4321754317873084, "logiqa_acc": 0.2196620583717358, "piqa_acc": 0.6811751904243744, "sciq_acc": 0.872, "winogrande_acc": 0.516179952644041 }, "job_id": "96f9133d67c7682bede07d57", "scores": { "conditional": 1.6232993435735588, "gm": 35.268611588981, "ed": 8.790503185968088, "sol": 7.197348205097749, "nll": 14.513255393055008 } }, { "setting": "t13", "size": "1b", "step": 63000, "distance": 1.4019889374607082, "accuracy": 0.5153026239542494, "tasks": { "arc_challenge_acc": 0.23122866894197952, "arc_easy_acc": 0.5593434343434344, "lambada_openai_acc": 0.4723462060935377, "logiqa_acc": 0.22580645161290322, "piqa_acc": 0.6936887921653971, "sciq_acc": 0.903, "winogrande_acc": 0.5217048145224941 }, "job_id": "6f421fa28d1973d86964223d", "scores": { "conditional": 1.4019889374607082, "gm": 38.32642702604062, "ed": 6.494489430499503, "sol": 7.259187979310572, "nll": 14.534897490984644 } }, { "setting": "t13", "size": "1b", "step": 103000, "distance": 1.492228933891445, "accuracy": 0.5333274933224145, "tasks": { "arc_challenge_acc": 0.25, "arc_easy_acc": 0.5909090909090909, "lambada_openai_acc": 0.4975742286046963, "logiqa_acc": 0.22580645161290322, "piqa_acc": 0.6996735582154516, "sciq_acc": 0.92, "winogrande_acc": 0.5493291239147593 }, "job_id": "0950c1354fd7bfd9de0d08fb", "scores": { "conditional": 1.492228933891445, "gm": 38.05461053891486, "ed": 7.507394457386639, "sol": 7.262646929616703, "nll": 14.680308116891556 } }, { "setting": "t13", "size": "1b", "step": 143000, "distance": 1.4272845388938034, "accuracy": 0.5349914679163845, "tasks": { "arc_challenge_acc": 0.2593856655290102, "arc_easy_acc": 0.5938552188552189, "lambada_openai_acc": 0.5072773141859112, "logiqa_acc": 0.22734254992319508, "piqa_acc": 0.705114254624592, "sciq_acc": 0.92, "winogrande_acc": 0.531965272296764 }, "job_id": "f5ef95dda2625d0ac8110625", "scores": { "conditional": 1.4272845388938034, "gm": 38.88399681437295, "ed": 7.061359393499244, "sol": 7.336777176769608, "nll": 14.554149071719335 } }, { "setting": "t13", "size": "6.9b", "step": 3000, "distance": 1.8746465929358411, "accuracy": 0.42391127208163926, "tasks": { "arc_challenge_acc": 0.1766211604095563, "arc_easy_acc": 0.4246632996632997, "lambada_openai_acc": 0.22549970890743257, "logiqa_acc": 0.23963133640552994, "piqa_acc": 0.6175190424374319, "sciq_acc": 0.772, "winogrande_acc": 0.5114443567482242 }, "job_id": "28296fca5b6f4863c708fc84", "scores": { "conditional": 1.8746465929358411, "gm": 30.079040245778742, "ed": 7.356703101603135, "sol": 6.625768073622666, "nll": 13.385419103143013 } }, { "setting": "t13", "size": "6.9b", "step": 13000, "distance": 1.375090513137328, "accuracy": 0.5130790719439721, "tasks": { "arc_challenge_acc": 0.21928327645051193, "arc_easy_acc": 0.563973063973064, "lambada_openai_acc": 0.4511934795264894, "logiqa_acc": 0.2488479262672811, "piqa_acc": 0.6958650707290533, "sciq_acc": 0.897, "winogrande_acc": 0.5153906866614049 }, "job_id": "d1a8216cf2cdc3e8ed0eb1ed", "scores": { "conditional": 1.375090513137328, "gm": 34.38684751906112, "ed": 6.3546714071802315, "sol": 7.20808063915932, "nll": 14.024249932847617 } }, { "setting": "t13", "size": "6.9b", "step": 33000, "distance": 1.4538553123162172, "accuracy": 0.5533024681765826, "tasks": { "arc_challenge_acc": 0.2883959044368601, "arc_easy_acc": 0.6334175084175084, "lambada_openai_acc": 0.5389093731806714, "logiqa_acc": 0.2119815668202765, "piqa_acc": 0.720348204570185, "sciq_acc": 0.926, "winogrande_acc": 0.5540647198105761 }, "job_id": "77b369a4b00a32e524e16f67", "scores": { "conditional": 1.4538553123162172, "gm": 34.584721584804356, "ed": 6.259479648333988, "sol": 7.093555297321005, "nll": 14.06232716383057 } }, { "setting": "t13", "size": "6.9b", "step": 63000, "distance": 1.507055044114232, "accuracy": 0.5820065388502569, "tasks": { "arc_challenge_acc": 0.318259385665529, "arc_easy_acc": 0.6813973063973064, "lambada_openai_acc": 0.5753929749660393, "logiqa_acc": 0.2304147465437788, "piqa_acc": 0.733949945593036, "sciq_acc": 0.934, "winogrande_acc": 0.6006314127861089 }, "job_id": "753f02fed31d29d454473466", "scores": { "conditional": 1.507055044114232, "gm": 34.27148393041716, "ed": 6.2612939696315495, "sol": 7.00830350052931, "nll": 14.01115701993719 } }, { "setting": "t13", "size": "6.9b", "step": 103000, "distance": 1.5433100390869794, "accuracy": 0.607810860185633, "tasks": { "arc_challenge_acc": 0.3370307167235495, "arc_easy_acc": 0.6919191919191919, "lambada_openai_acc": 0.6258490199883563, "logiqa_acc": 0.27035330261136714, "piqa_acc": 0.7453754080522307, "sciq_acc": 0.948, "winogrande_acc": 0.6361483820047356 }, "job_id": "7048b5445ed2c919dc8c8976", "scores": { "conditional": 1.5433100390869794, "gm": 34.66847435565796, "ed": 6.476794175506437, "sol": 6.985680333113113, "nll": 14.16765448660595 } }, { "setting": "t13", "size": "6.9b", "step": 143000, "distance": 1.4894466311985561, "accuracy": 0.6155622706270009, "tasks": { "arc_challenge_acc": 0.35580204778157, "arc_easy_acc": 0.7015993265993266, "lambada_openai_acc": 0.638074907820687, "logiqa_acc": 0.27035330261136714, "piqa_acc": 0.7551686615886833, "sciq_acc": 0.951, "winogrande_acc": 0.6369376479873717 }, "job_id": "a119ae57de4f6d7a3217ef4e", "scores": { "conditional": 1.4894466311985561, "gm": 34.46835873076634, "ed": 6.0444148093192, "sol": 6.981891284029926, "nll": 14.021840311470473 } } ], "correlations": { "t07": { "pooled": 0.951304347826087, "within_size": { "160m": 0.2, "410m": 0.9428571428571428, "1b": 0.8285714285714286, "6.9b": 0.8857142857142857 }, "leave_one_size_out": { "160m": 0.9339525283797729, "410m": 0.9277605779153767, "1b": 0.9401444788441693, "6.9b": 0.9401444788441693 } }, "t10": { "pooled": 0.8921739130434783, "within_size": { "160m": 0.5428571428571428, "410m": 0.7714285714285715, "1b": 0.4857142857142857, "6.9b": 0.3142857142857143 }, "leave_one_size_out": { "160m": 0.8679050567595459, "410m": 0.8720330237358102, "1b": 0.8823529411764706, "6.9b": 0.8328173374613003 } }, "t10-p095": { "pooled": 0.88, "within_size": { "160m": 0.5428571428571428, "410m": 0.6, "1b": 0.6571428571428571, "6.9b": 0.4857142857142857 }, "leave_one_size_out": { "160m": 0.803921568627451, "410m": 0.8328173374613003, "1b": 0.9009287925696594, "6.9b": 0.8679050567595459 } }, "t10-k20": { "pooled": 0.9278260869565217, "within_size": { "160m": 0.2571428571428571, "410m": 1.0, "1b": 1.0, "6.9b": 0.8857142857142857 }, "leave_one_size_out": { "160m": 0.9195046439628483, "410m": 0.913312693498452, "1b": 0.9236326109391125, "6.9b": 0.9050567595459237 } }, "t13": { "pooled": -0.10782608695652174, "within_size": { "160m": 0.02857142857142857, "410m": -0.42857142857142855, "1b": 0.02857142857142857, "6.9b": 0.02857142857142857 }, "leave_one_size_out": { "160m": -0.05263157894736842, "410m": -0.07946336429308566, "1b": -0.14551083591331268, "6.9b": -0.1434468524251806 } } }, "correlations_by_metric": { "conditional": { "t07": { "pooled": 0.951304347826087, "within_size": { "160m": 0.2, "410m": 0.9428571428571428, "1b": 0.8285714285714286, "6.9b": 0.8857142857142857 }, "leave_one_size_out": { "160m": 0.9339525283797729, "410m": 0.9277605779153767, "1b": 0.9401444788441693, "6.9b": 0.9401444788441693 } }, "t10": { "pooled": 0.8921739130434783, "within_size": { "160m": 0.5428571428571428, "410m": 0.7714285714285715, "1b": 0.4857142857142857, "6.9b": 0.3142857142857143 }, "leave_one_size_out": { "160m": 0.8679050567595459, "410m": 0.8720330237358102, "1b": 0.8823529411764706, "6.9b": 0.8328173374613003 } }, "t10-p095": { "pooled": 0.88, "within_size": { "160m": 0.5428571428571428, "410m": 0.6, "1b": 0.6571428571428571, "6.9b": 0.4857142857142857 }, "leave_one_size_out": { "160m": 0.803921568627451, "410m": 0.8328173374613003, "1b": 0.9009287925696594, "6.9b": 0.8679050567595459 } }, "t10-k20": { "pooled": 0.9278260869565217, "within_size": { "160m": 0.2571428571428571, "410m": 1.0, "1b": 1.0, "6.9b": 0.8857142857142857 }, "leave_one_size_out": { "160m": 0.9195046439628483, "410m": 0.913312693498452, "1b": 0.9236326109391125, "6.9b": 0.9050567595459237 } }, "t13": { "pooled": -0.10782608695652174, "within_size": { "160m": 0.02857142857142857, "410m": -0.42857142857142855, "1b": 0.02857142857142857, "6.9b": 0.02857142857142857 }, "leave_one_size_out": { "160m": -0.05263157894736842, "410m": -0.07946336429308566, "1b": -0.14551083591331268, "6.9b": -0.1434468524251806 } } }, "sol": { "t07": { "pooled": 0.94, "within_size": { "160m": 0.14285714285714285, "410m": 0.9428571428571428, "1b": 0.9428571428571428, "6.9b": 1.0 }, "leave_one_size_out": { "160m": 0.977296181630547, "410m": 0.9277605779153767, "1b": 0.9195046439628483, "6.9b": 0.8782249742002064 } }, "t10": { "pooled": 0.9426086956521739, "within_size": { "160m": 0.6571428571428571, "410m": 1.0, "1b": 0.9428571428571428, "6.9b": 0.9428571428571428 }, "leave_one_size_out": { "160m": 0.9731682146542827, "410m": 0.9174406604747162, "1b": 0.9256965944272446, "6.9b": 0.9236326109391125 } }, "t10-p095": { "pooled": 0.8408695652173913, "within_size": { "160m": 0.42857142857142855, "410m": 1.0, "1b": 0.7142857142857143, "6.9b": 0.7142857142857143 }, "leave_one_size_out": { "160m": 0.803921568627451, "410m": 0.7956656346749226, "1b": 0.8617131062951496, "6.9b": 0.8369453044375645 } }, "t10-k20": { "pooled": 0.5834782608695652, "within_size": { "160m": 0.42857142857142855, "410m": 0.7714285714285715, "1b": 0.7714285714285715, "6.9b": 0.8285714285714286 }, "leave_one_size_out": { "160m": 0.3993808049535604, "410m": 0.6346749226006192, "1b": 0.6429308565531475, "6.9b": 0.6842105263157895 } }, "t13": { "pooled": -0.2530434782608696, "within_size": { "160m": -0.3142857142857143, "410m": -0.14285714285714285, "1b": -0.6571428571428571, "6.9b": 0.14285714285714285 }, "leave_one_size_out": { "160m": -0.25696594427244585, "410m": -0.1867905056759546, "1b": -0.13312693498452013, "6.9b": -0.4324045407636739 } } }, "ed": { "t07": { "pooled": 0.971304347826087, "within_size": { "160m": 0.42857142857142855, "410m": 1.0, "1b": 1.0, "6.9b": 0.9428571428571428 }, "leave_one_size_out": { "160m": 0.9834881320949432, "410m": 0.9566563467492261, "1b": 0.9566563467492261, "6.9b": 0.9484004127966976 } }, "t10": { "pooled": 0.9234782608695652, "within_size": { "160m": 0.7714285714285715, "410m": 0.7714285714285715, "1b": 0.6, "6.9b": 0.6571428571428571 }, "leave_one_size_out": { "160m": 0.9050567595459237, "410m": 0.9112487100103199, "1b": 0.9153766769865841, "6.9b": 0.8823529411764706 } }, "t10-p095": { "pooled": 0.8791304347826087, "within_size": { "160m": 0.4857142857142857, "410m": 0.7142857142857143, "1b": 0.6571428571428571, "6.9b": 0.7142857142857143 }, "leave_one_size_out": { "160m": 0.8204334365325078, "410m": 0.8513931888544891, "1b": 0.9050567595459237, "6.9b": 0.849329205366357 } }, "t10-k20": { "pooled": 0.9495652173913044, "within_size": { "160m": 0.2571428571428571, "410m": 0.9428571428571428, "1b": 1.0, "6.9b": 0.9428571428571428 }, "leave_one_size_out": { "160m": 0.9236326109391125, "410m": 0.936016511867905, "1b": 0.9525283797729618, "6.9b": 0.9401444788441693 } }, "t13": { "pooled": 0.05478260869565217, "within_size": { "160m": 0.08571428571428572, "410m": -0.4857142857142857, "1b": -0.2571428571428571, "6.9b": 0.6 }, "leave_one_size_out": { "160m": 0.18472652218782248, "410m": 0.15995872033023736, "1b": 0.2982456140350877, "6.9b": -0.4695562435500516 } } }, "gm": { "t07": { "pooled": -0.36, "within_size": { "160m": -0.6, "410m": -0.5428571428571428, "1b": -0.02857142857142857, "6.9b": -0.3142857142857143 }, "leave_one_size_out": { "160m": -0.44685242518059853, "410m": -0.24458204334365324, "1b": -0.43653250773993807, "6.9b": -0.32920536635706915 } }, "t10": { "pooled": 0.8565217391304348, "within_size": { "160m": 1.0, "410m": 0.2571428571428571, "1b": 0.8857142857142857, "6.9b": 0.7714285714285715 }, "leave_one_size_out": { "160m": 0.7894736842105263, "410m": 0.8761609907120743, "1b": 0.8617131062951496, "6.9b": 0.8369453044375645 } }, "t10-p095": { "pooled": 0.5617391304347826, "within_size": { "160m": 0.7714285714285715, "410m": 0.14285714285714285, "1b": 0.4857142857142857, "6.9b": 0.37142857142857144 }, "leave_one_size_out": { "160m": 0.37048503611971106, "410m": 0.5686274509803921, "1b": 0.5686274509803921, "6.9b": 0.7048503611971104 } }, "t10-k20": { "pooled": -0.18521739130434783, "within_size": { "160m": -0.6571428571428571, "410m": -0.2, "1b": -0.08571428571428572, "6.9b": 0.4857142857142857 }, "leave_one_size_out": { "160m": -0.20123839009287925, "410m": -0.08978328173374613, "1b": -0.15170278637770898, "6.9b": -0.2899896800825593 } }, "t13": { "pooled": 0.00782608695652174, "within_size": { "160m": 0.42857142857142855, "410m": -0.6, "1b": -0.7142857142857143, "6.9b": -0.6 }, "leave_one_size_out": { "160m": -0.11867905056759546, "410m": 0.15789473684210525, "1b": 0.12899896800825594, "6.9b": -0.29205366357069146 } } }, "nll": { "t07": { "pooled": -0.908695652173913, "within_size": { "160m": -0.2, "410m": -1.0, "1b": -0.7142857142857143, "6.9b": -1.0 }, "leave_one_size_out": { "160m": -0.8947368421052632, "410m": -0.8926728586171311, "1b": -0.9298245614035088, "6.9b": -0.8513931888544891 } }, "t10": { "pooled": 0.9417391304347826, "within_size": { "160m": 0.6571428571428571, "410m": 1.0, "1b": 0.9428571428571428, "6.9b": 0.9428571428571428 }, "leave_one_size_out": { "160m": 0.9690402476780186, "410m": 0.9174406604747162, "1b": 0.9153766769865841, "6.9b": 0.9339525283797729 } }, "t10-p095": { "pooled": 0.9321739130434783, "within_size": { "160m": 0.5428571428571428, "410m": 1.0, "1b": 0.9428571428571428, "6.9b": 0.9428571428571428 }, "leave_one_size_out": { "160m": 0.9525283797729618, "410m": 0.9174406604747162, "1b": 0.9112487100103199, "6.9b": 0.9050567595459237 } }, "t10-k20": { "pooled": 0.07130434782608695, "within_size": { "160m": -0.14285714285714285, "410m": -0.6571428571428571, "1b": 0.14285714285714285, "6.9b": 0.08571428571428572 }, "leave_one_size_out": { "160m": 0.16821465428276575, "410m": 0.23219814241486067, "1b": -0.07946336429308566, "6.9b": -0.13725490196078433 } }, "t13": { "pooled": -0.3617391304347826, "within_size": { "160m": -0.6, "410m": -0.7714285714285715, "1b": -0.9428571428571428, "6.9b": -0.37142857142857144 }, "leave_one_size_out": { "160m": -0.2218782249742002, "410m": -0.2961816305469556, "1b": -0.1867905056759546, "6.9b": -0.9009287925696594 } } } }, "sources": [ { "path": ".cache/analysis/bidirectional-savings/dd9d3ca581adc948cb9e/conditional-directions/8630a5e84115aef93a3a/summary.json", "sha256": "3fe87bae5a277e60b252384946c1c1b7407d14c2accca653e88921bd0971014c" }, { "path": ".cache/pythia/published-evals/checkpoint_summary.csv", "sha256": "5822160b1bbadbf7f816abf1a1a449de5735dc5da08db564e9ccafc315bd4b14" }, { "path": ".cache/pythia/published-evals/manifest.json", "sha256": "39f9395167daa2980670f84c24803b06f71f220aa6f16e48fed7ad2cebc61b86" }, { "path": ".cache/analysis/pythia-baseline-training/points.csv", "sha256": "9bd859ea2736a87ad11a571c075ed59ef3c3a320a57998be32df1a623147998e" }, { "path": ".cache/pythia/analysis/qwen8b-pile/all_settings.csv", "sha256": "3fa21770552b3801bcfd8088f8a5c5b3db07ebac0a96ac77a19561cdc1a3bd7d" } ], "published_eval_commit": "a19eecb807ec2c79a39ebf18108816e6ffffc1d5" }