{ "benchmark": "JevBench", "revision": "v1.5.2", "protocol": "jevbench::v1.5", "status": "released", "run_kind": "official", "source_sha256": "452885de2a84cd5b9ed393d541fd8d6c6a540f9d7f762f9d2df9349383f26173", "base_results_sha256": "1cea4d6018da4c7e61132e932452b20ddea5758dcb8714a57a858e078dd6af12", "addenda_sources_sha256": { "A1": "2101bff95e6056fdebbcc6cfdb9e86c634ac8fb4a39419389c38bb47732e8c88", "A1_protocol": "fadbcd6ab18cc4576deab9f5dd7bc7e0af1eb93d111648719083e4093125b7ef", "A2": "eea496b1a549963084db76659fe32d3a67c2e9ca281e1d6a231df9be759343a7", "A2_protocol": "1fb818a123eedf68c49295fd860f62b27c79b80fb85953b2a2e86599f861f0f3", "base": "1cea4d6018da4c7e61132e932452b20ddea5758dcb8714a57a858e078dd6af12", "pricing_disclosure_correction": "1b660648bd498645a0888bd365bb24cfc616c79542862594d7a0077ed1db51fb", "pricing_disclosure_normalizer": "2cbc7bd5fdd9c249d526e4269ee492261feafecabd0cc662251437223776f586" }, "method_sha256": "c25d3d8b8512e4d93370a9e0c99705d19b2a9389956ca33b8a4bd2b0ec501c07", "pricing_addendum_sha256": "2fc44459ef801d0627062f7eefd973df40772e8ac117727479748e4be4c220cc", "headline_method_addendum_sha256": "752ddccc4e191c81412631ee3121563421c4b9077cb6c3f2243ed1d1545cfab0", "pricing_disclosure_correction_sha256": "1b660648bd498645a0888bd365bb24cfc616c79542862594d7a0077ed1db51fb", "pricing_disclosure_normalizer_sha256": "2cbc7bd5fdd9c249d526e4269ee492261feafecabd0cc662251437223776f586", "sample": { "open": 904, "sealed": 720, "total": 1624, "published_open": 601 }, "types": { "choice": 0.3333333333333333, "noul": 0.3333333333333333, "score": 0.3333333333333333 }, "tier_weights": { "easy": 0.1, "standard": 0.2, "judge": 0.3, "hard": 0.4 }, "sealed_share_of_intelligence": 0.5, "G_med": 5.186627500079243, "G_med_flag_gt10": false, "headline": "A", "options": { "A": { "weights": { "intelligence": 25, "calibration": 25, "speed": 25, "cost": 25 }, "intelligence_floor": 50 }, "B": { "weights": { "intelligence": 40, "calibration": 20, "speed": 20, "cost": 20 }, "intelligence_floor": 50 }, "C": { "weights": { "intelligence": 25, "calibration": 25, "speed": 25, "cost": 25 }, "intelligence_floor": 60 } }, "views": [ "Balanced 33:33:33", "Emphasis on Accuracy 60:20:20", "Emphasis on Cost 20:20:60", "Emphasis on Speed 20:60:20", "Intelligence only" ], "bootstrap": { "B": 2000, "bootstrap_seed": 15 }, "n_ranked": 99, "board": { "A": { "order": [ "cygnet", "winnow-12b", "jev-1.13.0", "jevk5-v0.3-4b", "plumb-4b", "jev-omni", "decider-4b-v2", "decision-4b-v12", "imajev-4b-rtx5090-a2", "decision-4b-v11", "semif-qwen3.5-4b", "spark-s1-4b-v6", "metask-jev-4b", "hopper", "malkuth-4b", "surogate-rune-26b-a4b-v3-rtxpro6000-a2", "reflex-4b", "jev-local", "djev", "raw-qwen3-4b-instruct-2507", "jqv", "jevk5-v02", "qwen35-9b-jev-data-mix-v2", "standardone-8b", "ninfer-qwen3.8-flash-next", "swanone", "raw-qwen3-8b", "decider-2b", "system-one-sg", "system-one-open", "kushal-gemma4-31b-it-autoloops", "gpt-6-luna-low", "gpt-6-luna", "jevone", "kev-4b", "kev-8b", "open-alternative-jev", "nimble-9b", "malkuth-2b", "openjev-sglang", "decider-35b-a3b", "localjev-qwen3.5-4b", "nemotron-diffusion-8b", "open-jev-zefan-9b", "decision-2b", "gpt-5.6-luna", "typecastlm", "jev-qwen3.5-9b-base-nvfp4", "gemini-3.1-flash-lite", "autojev-27b", "autojev-27b-rtxpro6000-a2", "ninfer-qwen3.8-27b", "eikos-27b", "ninfer-qwen3.8-27b-t1.5", "instinct", "openjev-thinking", "djev-thinking", "litjev", "raw-phi-4-mini", "opensourcejev-qwen35-4b-q4km", "reflex-27b", "open-jev-zefan-2b", "gliner2-large", "qwen3-reranker-4b", "deepseek-flash", "simplejev-qwen3.5-0.8b", "simplejev-qwen3.8-27b", "decision-machine-1", "gliner2.5-multi", "gliner2", "jevact", "clm-8b", "kev-0.6b", "raw-qwen3-0.6b", "gliner2.5-small", "raw-qwen3-1.7b", "mirror", "zerank-2", "jeff", "smalljev", "opendecision", "bge-reranker-v2-m3", "certo", "decision-fast", "gte-reranker-modernbert-base", "kev-0.5b", "laya", "lev-350m", "mghafiri-qwen3.5-0.8b-decision-model", "mxbai-rerank-base-v2", "needle-3", "needle-3-tools", "open-jev-deberta-v3-large", "open-jev-json-canvas-joshuasp", "openjev-verdict", "openjev-verdict-1.4", "qwen3.8-27b", "verdict-small", "von-395m" ], "leader_wording": "joint leaders (statistical tie)", "markers": [ { "upper": "cygnet", "lower": "winnow-12b", "tie": true, "diff_ci95": [ -0.46125459419296033, 1.2374474782799036 ], "p_upper_wins": 0.8345 }, { "upper": "winnow-12b", "lower": "jev-1.13.0", "tie": false, "diff_ci95": [ 0.2845795431300456, 2.0792169124533744 ], "p_upper_wins": 0.995 }, { "upper": "jev-omni", "lower": "decider-4b-v2", "tie": true, "diff_ci95": [ -1.038582204987227, 2.199403871434391 ], "p_upper_wins": 0.7245 }, { "upper": "semif-qwen3.5-4b", "lower": "spark-s1-4b-v6", "tie": true, "diff_ci95": [ -7.2683777036922965, 2.1182629795515853 ], "p_upper_wins": 0.5735 }, { "upper": "spark-s1-4b-v6", "lower": "metask-jev-4b", "tie": true, "diff_ci95": [ -0.8854612486139711, 2.800075681219657 ], "p_upper_wins": 0.8275 }, { "upper": "metask-jev-4b", "lower": "hopper", "tie": true, "diff_ci95": [ -1.8537605222526754, 9.941454259279055 ], "p_upper_wins": 0.5785 }, { "upper": "hopper", "lower": "malkuth-4b", "tie": true, "diff_ci95": [ -9.382473047380726, 2.590058496634971 ], "p_upper_wins": 0.5305 }, { "upper": "reflex-4b", "lower": "jev-local", "tie": true, "diff_ci95": [ -6.041152723112148, 1.492373069358983 ], "p_upper_wins": 0.4325 }, { "upper": "jev-local", "lower": "djev", "tie": true, "diff_ci95": [ -0.5525291189700304, 2.5296882748744998 ], "p_upper_wins": 0.8925 }, { "upper": "djev", "lower": "raw-qwen3-4b-instruct-2507", "tie": true, "diff_ci95": [ -0.23307398561861295, 4.86568485012425 ], "p_upper_wins": 0.9605 }, { "upper": "raw-qwen3-4b-instruct-2507", "lower": "jqv", "tie": true, "diff_ci95": [ -2.9153289030224725, 10.293731051459305 ], "p_upper_wins": 0.656 }, { "upper": "jqv", "lower": "jevk5-v02", "tie": true, "diff_ci95": [ -6.493803455404365, 10.969443174861983 ], "p_upper_wins": 0.656 }, { "upper": "jevk5-v02", "lower": "qwen35-9b-jev-data-mix-v2", "tie": true, "diff_ci95": [ -5.0456850000987785, 14.787465294682384 ], "p_upper_wins": 0.827 }, { "upper": "qwen35-9b-jev-data-mix-v2", "lower": "standardone-8b", "tie": false, "diff_ci95": [ 4.309767791178995, 6.238131399932088 ], "p_upper_wins": 1 }, { "upper": "standardone-8b", "lower": "ninfer-qwen3.8-flash-next", "tie": true, "diff_ci95": [ -0.6416244151701841, 1.2262471900961953 ], "p_upper_wins": 0.749 }, { "upper": "ninfer-qwen3.8-flash-next", "lower": "swanone", "tie": false, "diff_ci95": [ 0.39497981009657546, 1.4306319415544606 ], "p_upper_wins": 1 }, { "upper": "swanone", "lower": "raw-qwen3-8b", "tie": true, "diff_ci95": [ -0.32051802578132094, 9.204043926273682 ], "p_upper_wins": 0.941 }, { "upper": "raw-qwen3-8b", "lower": "decider-2b", "tie": true, "diff_ci95": [ -9.716705520853559, 9.766722902363652 ], "p_upper_wins": 0.513 }, { "upper": "decider-2b", "lower": "system-one-sg", "tie": true, "diff_ci95": [ -8.46504370655965, 10.581308485732498 ], "p_upper_wins": 0.583 }, { "upper": "system-one-sg", "lower": "system-one-open", "tie": true, "diff_ci95": [ -8.708966141268096, 9.288384655250024 ], "p_upper_wins": 0.553 }, { "upper": "system-one-open", "lower": "kushal-gemma4-31b-it-autoloops", "tie": true, "diff_ci95": [ -6.531864021404802, 11.396874568395233 ], "p_upper_wins": 0.6615 }, { "upper": "kushal-gemma4-31b-it-autoloops", "lower": "gpt-6-luna-low", "tie": true, "diff_ci95": [ -0.4144644818337552, 0.32506232006125557 ], "p_upper_wins": 0.439 }, { "upper": "gpt-6-luna-low", "lower": "gpt-6-luna", "tie": false, "diff_ci95": [ 1.5583286196261574, 1.8857336022517284 ], "p_upper_wins": 1 }, { "upper": "gpt-6-luna", "lower": "jevone", "tie": true, "diff_ci95": [ -0.03151412517146692, 1.3843962177828855 ], "p_upper_wins": 0.967 }, { "upper": "jevone", "lower": "kev-4b", "tie": true, "diff_ci95": [ -8.509048186025899, 10.610625427507967 ], "p_upper_wins": 0.579 }, { "upper": "kev-4b", "lower": "kev-8b", "tie": true, "diff_ci95": [ -3.7433056342930913, 11.397615769700971 ], "p_upper_wins": 0.8385 }, { "upper": "kev-8b", "lower": "open-alternative-jev", "tie": true, "diff_ci95": [ -8.771759347590297, 7.6265715775051195 ], "p_upper_wins": 0.485 }, { "upper": "open-alternative-jev", "lower": "nimble-9b", "tie": true, "diff_ci95": [ -5.855456245563838, 9.74872432947998 ], "p_upper_wins": 0.6465 }, { "upper": "nimble-9b", "lower": "malkuth-2b", "tie": true, "diff_ci95": [ -7.142707535889598, 10.586944134669313 ], "p_upper_wins": 0.662 }, { "upper": "malkuth-2b", "lower": "openjev-sglang", "tie": true, "diff_ci95": [ -8.006951980847019, 10.100145938288167 ], "p_upper_wins": 0.5605 }, { "upper": "openjev-sglang", "lower": "decider-35b-a3b", "tie": false, "diff_ci95": [ 0.9680817033605356, 2.051678046459891 ], "p_upper_wins": 1 }, { "upper": "decider-35b-a3b", "lower": "localjev-qwen3.5-4b", "tie": true, "diff_ci95": [ -5.352522919587847, 7.400630850160767 ], "p_upper_wins": 0.6905 }, { "upper": "open-jev-zefan-9b", "lower": "decision-2b", "tie": true, "diff_ci95": [ -4.4136769003767755, 7.468775417922205 ], "p_upper_wins": 0.738 }, { "upper": "decision-2b", "lower": "gpt-5.6-luna", "tie": true, "diff_ci95": [ -5.633641938976553, 6.477057055322495 ], "p_upper_wins": 0.499 }, { "upper": "gpt-5.6-luna", "lower": "typecastlm", "tie": true, "diff_ci95": [ -6.241861739411629, 6.126494390998944 ], "p_upper_wins": 0.58 }, { "upper": "typecastlm", "lower": "jev-qwen3.5-9b-base-nvfp4", "tie": true, "diff_ci95": [ -4.540142594145304, 8.509187230312083 ], "p_upper_wins": 0.706 }, { "upper": "jev-qwen3.5-9b-base-nvfp4", "lower": "gemini-3.1-flash-lite", "tie": true, "diff_ci95": [ -4.178309047666805, 5.746017130776264 ], "p_upper_wins": 0.565 }, { "upper": "ninfer-qwen3.8-27b-t1.5", "lower": "instinct", "tie": true, "diff_ci95": [ -0.024202650340603427, 0.3015667277965538 ], "p_upper_wins": 0.957 }, { "upper": "instinct", "lower": "openjev-thinking", "tie": false, "diff_ci95": [ 0.10228253132511417, 0.5988127921420549 ], "p_upper_wins": 0.9965 }, { "upper": "openjev-thinking", "lower": "djev-thinking", "tie": false, "diff_ci95": [ 0.3544604920141397, 0.6977343610857955 ], "p_upper_wins": 1 }, { "upper": "djev-thinking", "lower": "litjev", "tie": false, "diff_ci95": [ 0.8933699543947561, 1.3761614082369105 ], "p_upper_wins": 1 }, { "upper": "litjev", "lower": "raw-phi-4-mini", "tie": true, "diff_ci95": [ -4.407182033300111, 5.911981192696572 ], "p_upper_wins": 0.678 }, { "upper": "raw-phi-4-mini", "lower": "opensourcejev-qwen35-4b-q4km", "tie": true, "diff_ci95": [ -4.188941906621974, 8.152834924030318 ], "p_upper_wins": 0.727 }, { "upper": "opensourcejev-qwen35-4b-q4km", "lower": "reflex-27b", "tie": true, "diff_ci95": [ -4.086740231332742, 5.091597668154737 ], "p_upper_wins": 0.5045 }, { "upper": "reflex-27b", "lower": "open-jev-zefan-2b", "tie": false, "diff_ci95": [ 1.6762420468103854, 6.464898360709222 ], "p_upper_wins": 0.9995 }, { "upper": "open-jev-zefan-2b", "lower": "gliner2-large", "tie": true, "diff_ci95": [ -4.020653839372584, 4.574881867858114 ], "p_upper_wins": 0.613 }, { "upper": "gliner2-large", "lower": "qwen3-reranker-4b", "tie": true, "diff_ci95": [ -2.6235372779379844, 5.458934880708517 ], "p_upper_wins": 0.754 }, { "upper": "qwen3-reranker-4b", "lower": "deepseek-flash", "tie": true, "diff_ci95": [ -2.3971447414709712, 4.087291868438116 ], "p_upper_wins": 0.5605 }, { "upper": "deepseek-flash", "lower": "simplejev-qwen3.5-0.8b", "tie": true, "diff_ci95": [ -0.016820216423917355, 4.583231154534746 ], "p_upper_wins": 0.974 }, { "upper": "simplejev-qwen3.5-0.8b", "lower": "simplejev-qwen3.8-27b", "tie": true, "diff_ci95": [ -1.2913920287617369, 3.288144558865908 ], "p_upper_wins": 0.7195 }, { "upper": "simplejev-qwen3.8-27b", "lower": "decision-machine-1", "tie": true, "diff_ci95": [ -2.043185939748674, 1.621607948589032 ], "p_upper_wins": 0.5625 }, { "upper": "decision-machine-1", "lower": "gliner2.5-multi", "tie": true, "diff_ci95": [ -2.0466439303901143, 3.0151451587566616 ], "p_upper_wins": 0.653 }, { "upper": "gliner2.5-multi", "lower": "gliner2", "tie": true, "diff_ci95": [ -1.8543985948833104, 2.732411778791044 ], "p_upper_wins": 0.6605 }, { "upper": "gliner2", "lower": "jevact", "tie": true, "diff_ci95": [ -1.37742137346713, 3.224283034579049 ], "p_upper_wins": 0.756 }, { "upper": "jevact", "lower": "clm-8b", "tie": true, "diff_ci95": [ -1.702477083526242, 1.8005172541805738 ], "p_upper_wins": 0.5205 }, { "upper": "clm-8b", "lower": "kev-0.6b", "tie": true, "diff_ci95": [ -1.2082033404420878, 1.8475007171872655 ], "p_upper_wins": 0.6055 }, { "upper": "kev-0.6b", "lower": "raw-qwen3-0.6b", "tie": true, "diff_ci95": [ -1.4493108577709668, 1.7053664084996474 ], "p_upper_wins": 0.5925 }, { "upper": "raw-qwen3-0.6b", "lower": "gliner2.5-small", "tie": true, "diff_ci95": [ -1.1563285979099487, 1.732142807021534 ], "p_upper_wins": 0.599 }, { "upper": "gliner2.5-small", "lower": "raw-qwen3-1.7b", "tie": true, "diff_ci95": [ -1.4041473705371919, 1.4426281090931599 ], "p_upper_wins": 0.5165 }, { "upper": "raw-qwen3-1.7b", "lower": "mirror", "tie": true, "diff_ci95": [ -0.31468926986976725, 2.0745338529312134 ], "p_upper_wins": 0.892 }, { "upper": "mirror", "lower": "zerank-2", "tie": true, "diff_ci95": [ -0.27793465343426604, 0.755911422270539 ], "p_upper_wins": 0.666 }, { "upper": "zerank-2", "lower": "jeff", "tie": true, "diff_ci95": [ -0.3853413905470331, 0.3522016103947976 ], "p_upper_wins": 0.561 }, { "upper": "jeff", "lower": "smalljev", "tie": true, "diff_ci95": [ -0.2533892169562256, 0.41879249587200007 ], "p_upper_wins": 0.6425 }, { "upper": "smalljev", "lower": "opendecision", "tie": true, "diff_ci95": [ -0.12045273710115305, 0.3475332681246026 ], "p_upper_wins": 0.7875 }, { "upper": "opendecision", "lower": "bge-reranker-v2-m3", "tie": true, "diff_ci95": [ 0, 0.19294313592439996 ], "p_upper_wins": 0.892 }, { "upper": "bge-reranker-v2-m3", "lower": "certo", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "certo", "lower": "decision-fast", "tie": true, "diff_ci95": [ -0.0001584994708377039, 0 ], "p_upper_wins": 0 }, { "upper": "decision-fast", "lower": "gte-reranker-modernbert-base", "tie": true, "diff_ci95": [ 0, 0.00015849947083770386 ], "p_upper_wins": 0.0505 }, { "upper": "gte-reranker-modernbert-base", "lower": "kev-0.5b", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "kev-0.5b", "lower": "laya", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "laya", "lower": "lev-350m", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "lev-350m", "lower": "mghafiri-qwen3.5-0.8b-decision-model", "tie": true, "diff_ci95": [ -0.022833461476708714, 0 ], "p_upper_wins": 0 }, { "upper": "mghafiri-qwen3.5-0.8b-decision-model", "lower": "mxbai-rerank-base-v2", "tie": true, "diff_ci95": [ 0, 0.022833461476708693 ], "p_upper_wins": 0.475 }, { "upper": "mxbai-rerank-base-v2", "lower": "needle-3", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "needle-3", "lower": "needle-3-tools", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "needle-3-tools", "lower": "open-jev-deberta-v3-large", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "open-jev-deberta-v3-large", "lower": "open-jev-json-canvas-joshuasp", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "open-jev-json-canvas-joshuasp", "lower": "openjev-verdict", "tie": true, "diff_ci95": [ -0.014760916697897619, 0 ], "p_upper_wins": 0 }, { "upper": "openjev-verdict", "lower": "openjev-verdict-1.4", "tie": true, "diff_ci95": [ 0, 0.014760916697897527 ], "p_upper_wins": 0.325 }, { "upper": "openjev-verdict-1.4", "lower": "qwen3.8-27b", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "qwen3.8-27b", "lower": "verdict-small", "tie": true, "diff_ci95": [ -0.0051515709883764756, 0 ], "p_upper_wins": 0 }, { "upper": "verdict-small", "lower": "von-395m", "tie": true, "diff_ci95": [ 0, 0.005151570988376465 ], "p_upper_wins": 0.1255 } ] }, "B": { "order": [ "winnow-12b", "cygnet", "jev-1.13.0", "jev-omni", "jevk5-v0.3-4b", "plumb-4b", "decider-4b-v2", "spark-s1-4b-v6", "decision-4b-v12", "surogate-rune-26b-a4b-v3-rtxpro6000-a2", "imajev-4b-rtx5090-a2", "decision-4b-v11", "djev", "semif-qwen3.5-4b", "metask-jev-4b", "malkuth-4b", "jev-local", "hopper", "reflex-4b", "raw-qwen3-4b-instruct-2507", "jqv", "jevk5-v02", "qwen35-9b-jev-data-mix-v2", "ninfer-qwen3.8-flash-next", "swanone", "standardone-8b", "raw-qwen3-8b", "system-one-sg", "gpt-6-luna-low", "kushal-gemma4-31b-it-autoloops", "gpt-6-luna", "decider-2b", "system-one-open", "jevone", "kev-4b", "kev-8b", "nimble-9b", "open-alternative-jev", "openjev-sglang", "decider-35b-a3b", "malkuth-2b", "open-jev-zefan-9b", "gpt-5.6-luna", "localjev-qwen3.5-4b", "nemotron-diffusion-8b", "gemini-3.1-flash-lite", "autojev-27b", "autojev-27b-rtxpro6000-a2", "eikos-27b", "ninfer-qwen3.8-27b", "decision-2b", "openjev-thinking", "ninfer-qwen3.8-27b-t1.5", "instinct", "typecastlm", "djev-thinking", "jev-qwen3.5-9b-base-nvfp4", "litjev", "reflex-27b", "raw-phi-4-mini", "opensourcejev-qwen35-4b-q4km", "open-jev-zefan-2b", "deepseek-flash", "gliner2-large", "qwen3-reranker-4b", "simplejev-qwen3.8-27b", "simplejev-qwen3.5-0.8b", "decision-machine-1", "gliner2.5-multi", "gliner2", "clm-8b", "jevact", "kev-0.6b", "raw-qwen3-0.6b", "raw-qwen3-1.7b", "gliner2.5-small", "mirror", "zerank-2", "jeff", "smalljev", "opendecision", "bge-reranker-v2-m3", "certo", "decision-fast", "gte-reranker-modernbert-base", "kev-0.5b", "laya", "lev-350m", "mghafiri-qwen3.5-0.8b-decision-model", "mxbai-rerank-base-v2", "needle-3", "needle-3-tools", "open-jev-deberta-v3-large", "open-jev-json-canvas-joshuasp", "openjev-verdict", "openjev-verdict-1.4", "qwen3.8-27b", "verdict-small", "von-395m" ], "leader_wording": "joint leaders (statistical tie)", "markers": [ { "upper": "winnow-12b", "lower": "cygnet", "tie": true, "diff_ci95": [ -0.7084696230272624, 1.4585613409064304 ], "p_upper_wins": 0.731 }, { "upper": "cygnet", "lower": "jev-1.13.0", "tie": true, "diff_ci95": [ -0.06610426174216853, 2.3130221909080624 ], "p_upper_wins": 0.9645 }, { "upper": "jev-1.13.0", "lower": "jev-omni", "tie": true, "diff_ci95": [ -0.5431497817752661, 1.9720661401099062 ], "p_upper_wins": 0.851 }, { "upper": "decider-4b-v2", "lower": "spark-s1-4b-v6", "tie": true, "diff_ci95": [ -2.1946707969666193, 2.596028382828773 ], "p_upper_wins": 0.6075 }, { "upper": "djev", "lower": "semif-qwen3.5-4b", "tie": true, "diff_ci95": [ -1.5767426990756088, 8.620787305020604 ], "p_upper_wins": 0.673 }, { "upper": "semif-qwen3.5-4b", "lower": "metask-jev-4b", "tie": true, "diff_ci95": [ -6.664966544405195, 2.275274677429068 ], "p_upper_wins": 0.5525 }, { "upper": "metask-jev-4b", "lower": "malkuth-4b", "tie": true, "diff_ci95": [ -1.8071467456760701, 2.0510200547009267 ], "p_upper_wins": 0.5855 }, { "upper": "malkuth-4b", "lower": "jev-local", "tie": true, "diff_ci95": [ -1.8461747180925758, 2.909436298771614 ], "p_upper_wins": 0.7015 }, { "upper": "jev-local", "lower": "hopper", "tie": true, "diff_ci95": [ -2.075070320770288, 10.280103386220935 ], "p_upper_wins": 0.6285 }, { "upper": "hopper", "lower": "reflex-4b", "tie": true, "diff_ci95": [ -8.003191618773977, 4.910504591450937 ], "p_upper_wins": 0.599 }, { "upper": "reflex-4b", "lower": "raw-qwen3-4b-instruct-2507", "tie": true, "diff_ci95": [ -4.527115700264317, 4.123040901228487 ], "p_upper_wins": 0.78 }, { "upper": "raw-qwen3-4b-instruct-2507", "lower": "jqv", "tie": true, "diff_ci95": [ -1.8101263442107425, 11.774500744579674 ], "p_upper_wins": 0.819 }, { "upper": "jqv", "lower": "jevk5-v02", "tie": true, "diff_ci95": [ -5.2238636055052465, 12.095254186677929 ], "p_upper_wins": 0.7445 }, { "upper": "jevk5-v02", "lower": "qwen35-9b-jev-data-mix-v2", "tie": true, "diff_ci95": [ -8.729909910429436, 10.644321838901877 ], "p_upper_wins": 0.5755 }, { "upper": "qwen35-9b-jev-data-mix-v2", "lower": "ninfer-qwen3.8-flash-next", "tie": false, "diff_ci95": [ 3.5266856947312375, 6.155260792180568 ], "p_upper_wins": 1 }, { "upper": "ninfer-qwen3.8-flash-next", "lower": "swanone", "tie": true, "diff_ci95": [ -0.3282641122399742, 1.1334253178856313 ], "p_upper_wins": 0.85 }, { "upper": "swanone", "lower": "standardone-8b", "tie": true, "diff_ci95": [ -1.0728694554302445, 1.427460514019379 ], "p_upper_wins": 0.6055 }, { "upper": "standardone-8b", "lower": "raw-qwen3-8b", "tie": false, "diff_ci95": [ 0.5776097362594756, 10.613504124081201 ], "p_upper_wins": 0.9945 }, { "upper": "raw-qwen3-8b", "lower": "system-one-sg", "tie": true, "diff_ci95": [ -6.207458587592837, 8.061834283019177 ], "p_upper_wins": 0.731 }, { "upper": "system-one-sg", "lower": "gpt-6-luna-low", "tie": true, "diff_ci95": [ -7.198905174854513, 2.13399902678581 ], "p_upper_wins": 0.5555 }, { "upper": "gpt-6-luna-low", "lower": "kushal-gemma4-31b-it-autoloops", "tie": false, "diff_ci95": [ 0.790173617458304, 1.8994744112441106 ], "p_upper_wins": 1 }, { "upper": "kushal-gemma4-31b-it-autoloops", "lower": "gpt-6-luna", "tie": true, "diff_ci95": [ -0.15165210624559347, 0.9136093152573668 ], "p_upper_wins": 0.9195 }, { "upper": "gpt-6-luna", "lower": "decider-2b", "tie": true, "diff_ci95": [ -8.662717657197522, 11.152558684506745 ], "p_upper_wins": 0.5975 }, { "upper": "decider-2b", "lower": "system-one-open", "tie": true, "diff_ci95": [ -8.289063466492474, 10.675191193322593 ], "p_upper_wins": 0.5955 }, { "upper": "system-one-open", "lower": "jevone", "tie": true, "diff_ci95": [ -6.488940144153428, 10.430755862969878 ], "p_upper_wins": 0.627 }, { "upper": "jevone", "lower": "kev-4b", "tie": true, "diff_ci95": [ -5.578786299067803, 12.834833786237505 ], "p_upper_wins": 0.7725 }, { "upper": "kev-4b", "lower": "kev-8b", "tie": true, "diff_ci95": [ -5.811453110718897, 8.711967125065511 ], "p_upper_wins": 0.663 }, { "upper": "kev-8b", "lower": "nimble-9b", "tie": true, "diff_ci95": [ -6.447837630202695, 4.257101970983761 ], "p_upper_wins": 0.521 }, { "upper": "nimble-9b", "lower": "open-alternative-jev", "tie": true, "diff_ci95": [ -5.207725142199323, 9.53438373679304 ], "p_upper_wins": 0.746 }, { "upper": "open-alternative-jev", "lower": "openjev-sglang", "tie": true, "diff_ci95": [ -6.187693085736844, 8.28265429977841 ], "p_upper_wins": 0.5605 }, { "upper": "openjev-sglang", "lower": "decider-35b-a3b", "tie": false, "diff_ci95": [ 0.6782238560410832, 2.246661067234712 ], "p_upper_wins": 1 }, { "upper": "decider-35b-a3b", "lower": "malkuth-2b", "tie": true, "diff_ci95": [ -7.279882145048464, 9.488440765133443 ], "p_upper_wins": 0.62 }, { "upper": "malkuth-2b", "lower": "open-jev-zefan-9b", "tie": true, "diff_ci95": [ -6.745064614893428, 10.115493583501802 ], "p_upper_wins": 0.5985 }, { "upper": "open-jev-zefan-9b", "lower": "gpt-5.6-luna", "tie": false, "diff_ci95": [ 0.2337847255866267, 1.2804668975038942 ], "p_upper_wins": 0.9975 }, { "upper": "gpt-5.6-luna", "lower": "localjev-qwen3.5-4b", "tie": true, "diff_ci95": [ -5.431958325926747, 6.928887144748287 ], "p_upper_wins": 0.672 }, { "upper": "ninfer-qwen3.8-27b", "lower": "decision-2b", "tie": true, "diff_ci95": [ -5.751623996759211, 5.086876526976401 ], "p_upper_wins": 0.516 }, { "upper": "decision-2b", "lower": "openjev-thinking", "tie": true, "diff_ci95": [ -5.123266312043672, 5.864039614968194 ], "p_upper_wins": 0.4925 }, { "upper": "openjev-thinking", "lower": "ninfer-qwen3.8-27b-t1.5", "tie": false, "diff_ci95": [ 0.052523558620332, 0.7926896300206859 ], "p_upper_wins": 0.987 }, { "upper": "ninfer-qwen3.8-27b-t1.5", "lower": "instinct", "tie": true, "diff_ci95": [ -0.23440520735511094, 0.27219883350117824 ], "p_upper_wins": 0.5565 }, { "upper": "instinct", "lower": "typecastlm", "tie": true, "diff_ci95": [ -6.126751728238739, 4.832016778084619 ], "p_upper_wins": 0.502 }, { "upper": "typecastlm", "lower": "djev-thinking", "tie": true, "diff_ci95": [ -4.653677430027907, 6.744113169361638 ], "p_upper_wins": 0.552 }, { "upper": "djev-thinking", "lower": "jev-qwen3.5-9b-base-nvfp4", "tie": true, "diff_ci95": [ -4.457785665220995, 5.193343451686781 ], "p_upper_wins": 0.62 }, { "upper": "jev-qwen3.5-9b-base-nvfp4", "lower": "litjev", "tie": true, "diff_ci95": [ -3.259373791201761, 6.0464640970184105 ], "p_upper_wins": 0.661 }, { "upper": "litjev", "lower": "reflex-27b", "tie": false, "diff_ci95": [ 2.7348000637933807, 3.200328573928312 ], "p_upper_wins": 1 }, { "upper": "reflex-27b", "lower": "raw-phi-4-mini", "tie": true, "diff_ci95": [ -4.521568447923932, 5.072568179434511 ], "p_upper_wins": 0.631 }, { "upper": "raw-phi-4-mini", "lower": "opensourcejev-qwen35-4b-q4km", "tie": true, "diff_ci95": [ -3.659380215310137, 7.717511235132574 ], "p_upper_wins": 0.746 }, { "upper": "opensourcejev-qwen35-4b-q4km", "lower": "open-jev-zefan-2b", "tie": true, "diff_ci95": [ -0.7450788449351966, 6.868849362594874 ], "p_upper_wins": 0.9405 }, { "upper": "open-jev-zefan-2b", "lower": "deepseek-flash", "tie": true, "diff_ci95": [ -1.3674561632732893, 3.525168581204988 ], "p_upper_wins": 0.7915 }, { "upper": "deepseek-flash", "lower": "gliner2-large", "tie": true, "diff_ci95": [ -3.341068194853924, 3.041084700863563 ], "p_upper_wins": 0.5715 }, { "upper": "gliner2-large", "lower": "qwen3-reranker-4b", "tie": true, "diff_ci95": [ -2.2260314902761027, 5.016575343994072 ], "p_upper_wins": 0.779 }, { "upper": "qwen3-reranker-4b", "lower": "simplejev-qwen3.8-27b", "tie": true, "diff_ci95": [ -0.273819661294457, 5.436019165960417 ], "p_upper_wins": 0.9505 }, { "upper": "simplejev-qwen3.8-27b", "lower": "simplejev-qwen3.5-0.8b", "tie": true, "diff_ci95": [ -1.8829055280393932, 2.1008171350669484 ], "p_upper_wins": 0.66 }, { "upper": "simplejev-qwen3.5-0.8b", "lower": "decision-machine-1", "tie": true, "diff_ci95": [ -1.4217893897136948, 3.2162038741644787 ], "p_upper_wins": 0.7435 }, { "upper": "decision-machine-1", "lower": "gliner2.5-multi", "tie": true, "diff_ci95": [ -1.6973912035421361, 2.436169521373138 ], "p_upper_wins": 0.6445 }, { "upper": "gliner2.5-multi", "lower": "gliner2", "tie": true, "diff_ci95": [ -1.5932857318617524, 2.180827374451306 ], "p_upper_wins": 0.63 }, { "upper": "gliner2", "lower": "clm-8b", "tie": true, "diff_ci95": [ -1.129600446088771, 2.8034533953497283 ], "p_upper_wins": 0.7655 }, { "upper": "clm-8b", "lower": "jevact", "tie": true, "diff_ci95": [ -1.4176142087453059, 1.4066417670225329 ], "p_upper_wins": 0.4975 }, { "upper": "jevact", "lower": "kev-0.6b", "tie": true, "diff_ci95": [ -0.9110435978334234, 1.5433406071188291 ], "p_upper_wins": 0.6335 }, { "upper": "kev-0.6b", "lower": "raw-qwen3-0.6b", "tie": true, "diff_ci95": [ -1.2753644639076047, 1.2663424961142864 ], "p_upper_wins": 0.548 }, { "upper": "raw-qwen3-0.6b", "lower": "raw-qwen3-1.7b", "tie": true, "diff_ci95": [ -0.5569164664604941, 0.9749357000561392 ], "p_upper_wins": 0.7 }, { "upper": "raw-qwen3-1.7b", "lower": "gliner2.5-small", "tie": true, "diff_ci95": [ -1.0679165999301004, 1.240171000364177 ], "p_upper_wins": 0.518 }, { "upper": "gliner2.5-small", "lower": "mirror", "tie": true, "diff_ci95": [ -0.14467571549131245, 1.4839676642058017 ], "p_upper_wins": 0.9305 }, { "upper": "mirror", "lower": "zerank-2", "tie": true, "diff_ci95": [ -0.19884477053933702, 0.566713544498838 ], "p_upper_wins": 0.6705 }, { "upper": "zerank-2", "lower": "jeff", "tie": true, "diff_ci95": [ -0.2748776230456189, 0.2529313789425559 ], "p_upper_wins": 0.5635 }, { "upper": "jeff", "lower": "smalljev", "tie": true, "diff_ci95": [ -0.18024600201515434, 0.29883890068258734 ], "p_upper_wins": 0.6425 }, { "upper": "smalljev", "lower": "opendecision", "tie": true, "diff_ci95": [ -0.08188668460115738, 0.24471420587888623 ], "p_upper_wins": 0.7885 }, { "upper": "opendecision", "lower": "bge-reranker-v2-m3", "tie": true, "diff_ci95": [ 0, 0.13147202005412764 ], "p_upper_wins": 0.892 }, { "upper": "bge-reranker-v2-m3", "lower": "certo", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "certo", "lower": "decision-fast", "tie": true, "diff_ci95": [ -0.0000999013286476343, 0 ], "p_upper_wins": 0 }, { "upper": "decision-fast", "lower": "gte-reranker-modernbert-base", "tie": true, "diff_ci95": [ 0, 0.00009990132864763427 ], "p_upper_wins": 0.0505 }, { "upper": "gte-reranker-modernbert-base", "lower": "kev-0.5b", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "kev-0.5b", "lower": "laya", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "laya", "lower": "lev-350m", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "lev-350m", "lower": "mghafiri-qwen3.5-0.8b-decision-model", "tie": true, "diff_ci95": [ -0.01495240866961929, 0 ], "p_upper_wins": 0 }, { "upper": "mghafiri-qwen3.5-0.8b-decision-model", "lower": "mxbai-rerank-base-v2", "tie": true, "diff_ci95": [ 0, 0.014952408669619277 ], "p_upper_wins": 0.475 }, { "upper": "mxbai-rerank-base-v2", "lower": "needle-3", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "needle-3", "lower": "needle-3-tools", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "needle-3-tools", "lower": "open-jev-deberta-v3-large", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "open-jev-deberta-v3-large", "lower": "open-jev-json-canvas-joshuasp", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "open-jev-json-canvas-joshuasp", "lower": "openjev-verdict", "tie": true, "diff_ci95": [ -0.00962844029451886, 0 ], "p_upper_wins": 0 }, { "upper": "openjev-verdict", "lower": "openjev-verdict-1.4", "tie": true, "diff_ci95": [ 0, 0.0096284402945188 ], "p_upper_wins": 0.325 }, { "upper": "openjev-verdict-1.4", "lower": "qwen3.8-27b", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "qwen3.8-27b", "lower": "verdict-small", "tie": true, "diff_ci95": [ -0.003310895006967587, 0 ], "p_upper_wins": 0 }, { "upper": "verdict-small", "lower": "von-395m", "tie": true, "diff_ci95": [ 0, 0.003310895006967579 ], "p_upper_wins": 0.1255 } ] }, "C": { "order": [ "cygnet", "winnow-12b", "jev-1.13.0", "jev-omni", "spark-s1-4b-v6", "surogate-rune-26b-a4b-v3-rtxpro6000-a2", "djev", "jevk5-v0.3-4b", "plumb-4b", "decider-4b-v2", "jev-local", "decision-4b-v12", "imajev-4b-rtx5090-a2", "decision-4b-v11", "malkuth-4b", "metask-jev-4b", "qwen35-9b-jev-data-mix-v2", "raw-qwen3-4b-instruct-2507", "semif-qwen3.5-4b", "reflex-4b", "ninfer-qwen3.8-flash-next", "standardone-8b", "hopper", "swanone", "jqv", "kushal-gemma4-31b-it-autoloops", "gpt-6-luna-low", "jevk5-v02", "gpt-6-luna", "raw-qwen3-8b", "nimble-9b", "decider-2b", "system-one-sg", "jevone", "system-one-open", "openjev-sglang", "decider-35b-a3b", "kev-4b", "open-jev-zefan-9b", "kev-8b", "open-alternative-jev", "gpt-5.6-luna", "malkuth-2b", "gemini-3.1-flash-lite", "autojev-27b", "autojev-27b-rtxpro6000-a2", "ninfer-qwen3.8-27b", "eikos-27b", "ninfer-qwen3.8-27b-t1.5", "instinct", "openjev-thinking", "localjev-qwen3.5-4b", "nemotron-diffusion-8b", "djev-thinking", "decision-2b", "litjev", "typecastlm", "jev-qwen3.5-9b-base-nvfp4", "reflex-27b", "raw-phi-4-mini", "opensourcejev-qwen35-4b-q4km", "deepseek-flash", "open-jev-zefan-2b", "gliner2-large", "qwen3-reranker-4b", "simplejev-qwen3.8-27b", "simplejev-qwen3.5-0.8b", "decision-machine-1", "gliner2.5-multi", "gliner2", "jevact", "clm-8b", "kev-0.6b", "raw-qwen3-0.6b", "gliner2.5-small", "raw-qwen3-1.7b", "mirror", "zerank-2", "jeff", "smalljev", "opendecision", "bge-reranker-v2-m3", "certo", "decision-fast", "gte-reranker-modernbert-base", "kev-0.5b", "laya", "lev-350m", "mghafiri-qwen3.5-0.8b-decision-model", "mxbai-rerank-base-v2", "needle-3", "needle-3-tools", "open-jev-deberta-v3-large", "open-jev-json-canvas-joshuasp", "openjev-verdict", "openjev-verdict-1.4", "qwen3.8-27b", "verdict-small", "von-395m" ], "leader_wording": "joint leaders (statistical tie)", "markers": [ { "upper": "cygnet", "lower": "winnow-12b", "tie": true, "diff_ci95": [ -0.46125459419296033, 1.2374474782799036 ], "p_upper_wins": 0.8345 }, { "upper": "winnow-12b", "lower": "jev-1.13.0", "tie": false, "diff_ci95": [ 0.2845795431300456, 2.0792169124533744 ], "p_upper_wins": 0.995 }, { "upper": "jev-1.13.0", "lower": "jev-omni", "tie": true, "diff_ci95": [ -0.4111511855486413, 1.4641041378552804 ], "p_upper_wins": 0.837 }, { "upper": "jev-omni", "lower": "spark-s1-4b-v6", "tie": false, "diff_ci95": [ 1.7852740064338062, 7.931344922135727 ], "p_upper_wins": 1 }, { "upper": "decider-4b-v2", "lower": "jev-local", "tie": true, "diff_ci95": [ -7.673520278253025, 13.685000613835673 ], "p_upper_wins": 0.73 }, { "upper": "malkuth-4b", "lower": "metask-jev-4b", "tie": true, "diff_ci95": [ -5.765562613471671, 8.507608985220587 ], "p_upper_wins": 0.645 }, { "upper": "metask-jev-4b", "lower": "qwen35-9b-jev-data-mix-v2", "tie": true, "diff_ci95": [ -5.540261470682385, 8.474042015286878 ], "p_upper_wins": 0.639 }, { "upper": "qwen35-9b-jev-data-mix-v2", "lower": "raw-qwen3-4b-instruct-2507", "tie": true, "diff_ci95": [ -6.9286842199874945, 10.012447217823107 ], "p_upper_wins": 0.6435 }, { "upper": "raw-qwen3-4b-instruct-2507", "lower": "semif-qwen3.5-4b", "tie": true, "diff_ci95": [ -9.439724268362463, 9.589503250154001 ], "p_upper_wins": 0.5355 }, { "upper": "semif-qwen3.5-4b", "lower": "reflex-4b", "tie": true, "diff_ci95": [ -4.192089985162522, 8.547530948665214 ], "p_upper_wins": 0.741 }, { "upper": "reflex-4b", "lower": "ninfer-qwen3.8-flash-next", "tie": true, "diff_ci95": [ -6.340779395292697, 8.523949882705208 ], "p_upper_wins": 0.564 }, { "upper": "ninfer-qwen3.8-flash-next", "lower": "standardone-8b", "tie": true, "diff_ci95": [ -1.2262471900961955, 6.302532174322392 ], "p_upper_wins": 0.553 }, { "upper": "standardone-8b", "lower": "hopper", "tie": true, "diff_ci95": [ -7.501300874741987, 6.497417848468792 ], "p_upper_wins": 0.4465 }, { "upper": "hopper", "lower": "swanone", "tie": true, "diff_ci95": [ -7.003211547109533, 8.200550060680202 ], "p_upper_wins": 0.5185 }, { "upper": "swanone", "lower": "jqv", "tie": true, "diff_ci95": [ -2.58882077686001, 10.54440365024724 ], "p_upper_wins": 0.881 }, { "upper": "jqv", "lower": "kushal-gemma4-31b-it-autoloops", "tie": true, "diff_ci95": [ -4.631558003521666, 8.724095922018243 ], "p_upper_wins": 0.671 }, { "upper": "kushal-gemma4-31b-it-autoloops", "lower": "gpt-6-luna-low", "tie": true, "diff_ci95": [ -0.4144644818337552, 0.32506232006125557 ], "p_upper_wins": 0.439 }, { "upper": "gpt-6-luna-low", "lower": "jevk5-v02", "tie": true, "diff_ci95": [ -7.521983670107476, 7.470648883641279 ], "p_upper_wins": 0.5305 }, { "upper": "jevk5-v02", "lower": "gpt-6-luna", "tie": true, "diff_ci95": [ -5.761754010481389, 9.27723766534513 ], "p_upper_wins": 0.6435 }, { "upper": "gpt-6-luna", "lower": "raw-qwen3-8b", "tie": true, "diff_ci95": [ -0.18601217083374647, 13.10257722534698 ], "p_upper_wins": 0.973 }, { "upper": "raw-qwen3-8b", "lower": "nimble-9b", "tie": true, "diff_ci95": [ -6.017058086414951, 6.996061920676411 ], "p_upper_wins": 0.5805 }, { "upper": "nimble-9b", "lower": "decider-2b", "tie": true, "diff_ci95": [ -5.860093258204187, 8.150724279487909 ], "p_upper_wins": 0.621 }, { "upper": "decider-2b", "lower": "system-one-sg", "tie": true, "diff_ci95": [ -7.49504104204474, 6.615573973373475 ], "p_upper_wins": 0.444 }, { "upper": "system-one-sg", "lower": "jevone", "tie": true, "diff_ci95": [ -5.871274270932872, 6.7250623291265 ], "p_upper_wins": 0.5085 }, { "upper": "jevone", "lower": "system-one-open", "tie": true, "diff_ci95": [ -4.537793515948867, 7.619546193636626 ], "p_upper_wins": 0.6765 }, { "upper": "system-one-open", "lower": "openjev-sglang", "tie": true, "diff_ci95": [ -3.6544717773204343, 8.00475244548645 ], "p_upper_wins": 0.7405 }, { "upper": "openjev-sglang", "lower": "decider-35b-a3b", "tie": true, "diff_ci95": [ -2.7292836938423526, 3.3392878330668863 ], "p_upper_wins": 0.654 }, { "upper": "decider-35b-a3b", "lower": "kev-4b", "tie": true, "diff_ci95": [ -5.098303909355384, 7.785814324991241 ], "p_upper_wins": 0.6335 }, { "upper": "kev-4b", "lower": "open-jev-zefan-9b", "tie": true, "diff_ci95": [ -5.00591212889903, 8.123144862184763 ], "p_upper_wins": 0.664 }, { "upper": "open-jev-zefan-9b", "lower": "kev-8b", "tie": true, "diff_ci95": [ -3.399919251554369, 5.4998644396219465 ], "p_upper_wins": 0.6785 }, { "upper": "kev-8b", "lower": "open-alternative-jev", "tie": true, "diff_ci95": [ -6.07645256840151, 5.5380481089055795 ], "p_upper_wins": 0.501 }, { "upper": "open-alternative-jev", "lower": "gpt-5.6-luna", "tie": true, "diff_ci95": [ -4.457570780521773, 6.57872272540185 ], "p_upper_wins": 0.599 }, { "upper": "gpt-5.6-luna", "lower": "malkuth-2b", "tie": true, "diff_ci95": [ -4.723271895196073, 7.7518902560895695 ], "p_upper_wins": 0.696 }, { "upper": "malkuth-2b", "lower": "gemini-3.1-flash-lite", "tie": true, "diff_ci95": [ -5.018588926275203, 7.504508923007906 ], "p_upper_wins": 0.615 }, { "upper": "ninfer-qwen3.8-27b-t1.5", "lower": "instinct", "tie": true, "diff_ci95": [ -1.043786059846809, 0.3114062434715871 ], "p_upper_wins": 0.7675 }, { "upper": "instinct", "lower": "openjev-thinking", "tie": true, "diff_ci95": [ -0.2902045192155084, 0.5988127921420549 ], "p_upper_wins": 0.957 }, { "upper": "openjev-thinking", "lower": "localjev-qwen3.5-4b", "tie": true, "diff_ci95": [ -4.976315285912141, 4.108935218874537 ], "p_upper_wins": 0.51 }, { "upper": "djev-thinking", "lower": "decision-2b", "tie": true, "diff_ci95": [ -2.5997320186339024, 5.799242166885696 ], "p_upper_wins": 0.803 }, { "upper": "decision-2b", "lower": "litjev", "tie": true, "diff_ci95": [ -3.284084292567798, 4.3094539735288135 ], "p_upper_wins": 0.56 }, { "upper": "litjev", "lower": "typecastlm", "tie": true, "diff_ci95": [ -4.338513656085368, 3.871297060471404 ], "p_upper_wins": 0.539 }, { "upper": "typecastlm", "lower": "jev-qwen3.5-9b-base-nvfp4", "tie": true, "diff_ci95": [ -3.1528768014897937, 5.9091577988278345 ], "p_upper_wins": 0.706 }, { "upper": "jev-qwen3.5-9b-base-nvfp4", "lower": "reflex-27b", "tie": true, "diff_ci95": [ -2.4728097206030606, 4.444981395697316 ], "p_upper_wins": 0.6655 }, { "upper": "reflex-27b", "lower": "raw-phi-4-mini", "tie": true, "diff_ci95": [ -1.2853399369371623, 5.969916191132811 ], "p_upper_wins": 0.914 }, { "upper": "raw-phi-4-mini", "lower": "opensourcejev-qwen35-4b-q4km", "tie": true, "diff_ci95": [ -2.908987435154151, 5.661690919465499 ], "p_upper_wins": 0.727 }, { "upper": "opensourcejev-qwen35-4b-q4km", "lower": "deepseek-flash", "tie": true, "diff_ci95": [ -0.41382649340273375, 6.063108082558174 ], "p_upper_wins": 0.9545 }, { "upper": "deepseek-flash", "lower": "open-jev-zefan-2b", "tie": true, "diff_ci95": [ -1.3658465119638772, 2.049709591399384 ], "p_upper_wins": 0.6735 }, { "upper": "open-jev-zefan-2b", "lower": "gliner2-large", "tie": true, "diff_ci95": [ -2.792120721786518, 3.1770012971236907 ], "p_upper_wins": 0.613 }, { "upper": "gliner2-large", "lower": "qwen3-reranker-4b", "tie": true, "diff_ci95": [ -1.8219008874569327, 3.7909270004920264 ], "p_upper_wins": 0.754 }, { "upper": "qwen3-reranker-4b", "lower": "simplejev-qwen3.8-27b", "tie": true, "diff_ci95": [ -0.38877761491280066, 4.091506480440268 ], "p_upper_wins": 0.9235 }, { "upper": "simplejev-qwen3.8-27b", "lower": "simplejev-qwen3.5-0.8b", "tie": true, "diff_ci95": [ -1.2606831760696502, 1.9236208095030196 ], "p_upper_wins": 0.751 }, { "upper": "simplejev-qwen3.5-0.8b", "lower": "decision-machine-1", "tie": true, "diff_ci95": [ -1.2839445281521746, 2.4801358570056755 ], "p_upper_wins": 0.7065 }, { "upper": "decision-machine-1", "lower": "gliner2.5-multi", "tie": true, "diff_ci95": [ -1.4212805072153578, 2.0938508046921256 ], "p_upper_wins": 0.653 }, { "upper": "gliner2.5-multi", "lower": "gliner2", "tie": true, "diff_ci95": [ -1.287776802002299, 1.8975081797160034 ], "p_upper_wins": 0.6605 }, { "upper": "gliner2", "lower": "jevact", "tie": true, "diff_ci95": [ -0.9565426204632848, 2.2390854406798963 ], "p_upper_wins": 0.756 }, { "upper": "jevact", "lower": "clm-8b", "tie": true, "diff_ci95": [ -1.1822757524487788, 1.2503592042920653 ], "p_upper_wins": 0.5205 }, { "upper": "clm-8b", "lower": "kev-0.6b", "tie": true, "diff_ci95": [ -0.8390300975292277, 1.2829866091578235 ], "p_upper_wins": 0.6055 }, { "upper": "kev-0.6b", "lower": "raw-qwen3-0.6b", "tie": true, "diff_ci95": [ -1.00646587345206, 1.1842822281247551 ], "p_upper_wins": 0.5925 }, { "upper": "raw-qwen3-0.6b", "lower": "gliner2.5-small", "tie": true, "diff_ci95": [ -0.8030059707707977, 1.2028769493205096 ], "p_upper_wins": 0.599 }, { "upper": "gliner2.5-small", "lower": "raw-qwen3-1.7b", "tie": true, "diff_ci95": [ -0.9751023406508271, 1.0018250757591387 ], "p_upper_wins": 0.5165 }, { "upper": "raw-qwen3-1.7b", "lower": "mirror", "tie": true, "diff_ci95": [ -0.2185342151873383, 1.4406485089800096 ], "p_upper_wins": 0.892 }, { "upper": "mirror", "lower": "zerank-2", "tie": true, "diff_ci95": [ -0.1930101759960182, 0.5249384876878743 ], "p_upper_wins": 0.666 }, { "upper": "zerank-2", "lower": "jeff", "tie": true, "diff_ci95": [ -0.26759818787988415, 0.24458445166305384 ], "p_upper_wins": 0.561 }, { "upper": "jeff", "lower": "smalljev", "tie": true, "diff_ci95": [ -0.17596473399737889, 0.2908281221333333 ], "p_upper_wins": 0.6425 }, { "upper": "smalljev", "lower": "opendecision", "tie": true, "diff_ci95": [ -0.08364773409802294, 0.24134254730875176 ], "p_upper_wins": 0.7875 }, { "upper": "opendecision", "lower": "bge-reranker-v2-m3", "tie": true, "diff_ci95": [ 0, 0.13398828883638889 ], "p_upper_wins": 0.892 }, { "upper": "bge-reranker-v2-m3", "lower": "certo", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "certo", "lower": "decision-fast", "tie": true, "diff_ci95": [ -0.00011006907697062774, 0 ], "p_upper_wins": 0 }, { "upper": "decision-fast", "lower": "gte-reranker-modernbert-base", "tie": true, "diff_ci95": [ 0, 0.00011006907697062771 ], "p_upper_wins": 0.0505 }, { "upper": "gte-reranker-modernbert-base", "lower": "kev-0.5b", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "kev-0.5b", "lower": "laya", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "laya", "lower": "lev-350m", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "lev-350m", "lower": "mghafiri-qwen3.5-0.8b-decision-model", "tie": true, "diff_ci95": [ -0.015856570469936604, 0 ], "p_upper_wins": 0 }, { "upper": "mghafiri-qwen3.5-0.8b-decision-model", "lower": "mxbai-rerank-base-v2", "tie": true, "diff_ci95": [ 0, 0.01585657046993659 ], "p_upper_wins": 0.475 }, { "upper": "mxbai-rerank-base-v2", "lower": "needle-3", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "needle-3", "lower": "needle-3-tools", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "needle-3-tools", "lower": "open-jev-deberta-v3-large", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "open-jev-deberta-v3-large", "lower": "open-jev-json-canvas-joshuasp", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "open-jev-json-canvas-joshuasp", "lower": "openjev-verdict", "tie": true, "diff_ci95": [ -0.010250636595762235, 0 ], "p_upper_wins": 0 }, { "upper": "openjev-verdict", "lower": "openjev-verdict-1.4", "tie": true, "diff_ci95": [ 0, 0.010250636595762173 ], "p_upper_wins": 0.325 }, { "upper": "openjev-verdict-1.4", "lower": "qwen3.8-27b", "tie": true, "diff_ci95": [ 0, 0 ], "p_upper_wins": 0 }, { "upper": "qwen3.8-27b", "lower": "verdict-small", "tie": true, "diff_ci95": [ -0.0035774798530392193, 0 ], "p_upper_wins": 0 }, { "upper": "verdict-small", "lower": "von-395m", "tie": true, "diff_ci95": [ 0, 0.0035774798530392115 ], "p_upper_wins": 0.1255 } ] } }, "systems": [ { "key": "cygnet", "display": "Cygnet (blockbrain, frozen Gemma-4-12B-it)", "addendum": null, "author": "blockbrain", "repo": "https://github.com/blockbrain-ai/cygnet-recipe", "class": "system-one-open", "licence": "shim MIT; weights Apache-2.0 with Google's Gemma Prohibited Use Policy", "open": "yes", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "gpu": "RTXPRO6000", "adapter_id": "images/cygnet/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 71.09241537850588, "calibration": 87.00732784869052, "speed": 90.9725824502543, "cost": 56.42912569481113 }, "scores": { "A": 73.7012674756234, "B": 73.16429088621824, "C": 73.7012674756234 }, "views": { "Balanced 33:33:33": 70.12644697753653, "Emphasis on Accuracy 60:20:20": 70.50966710035375, "Emphasis on Cost 20:20:60": 63.92018198047105, "Emphasis on Speed 20:60:20": 77.2027746144365, "Intelligence only": 71.09241537850588 }, "composite_ci95": { "A": [ 72.3623865819622, 74.45948779199418 ], "B": [ 71.50599842979109, 74.36519444510661 ], "C": [ 72.3623865819622, 74.45948779199418 ] }, "jevbench_score": 73.7012674756234, "intelligence": { "I_open": 72.46391414608965, "I_sealed": 69.7209166109221, "base": 71.09241537850588, "gap": 2.7429975351675466, "excess": -2.443629964911696, "penalty": 1, "per_type_split": { "open|choice": { "cc": 82.9228973908834, "tiers": { "easy": 98.19113656918904, "standard": 89.08240187158823, "judge": 92.73858921161826, "hard": 68.66431649040345 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 57.86443298155948, "tiers": { "easy": 76.47058823529412, "standard": 69.6969696969697, "judge": 72.54901960784315, "hard": 36.28318584070796 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 76.6044120658261, "tiers": { "easy": 79.71124459421121, "standard": 87.82244564147533, "judge": 75.83786659638186, "hard": 70.79359624798836 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 80.28732165371557, "tiers": { "easy": 94.140625, "standard": 82.34095539959247, "judge": 78.79858657243817, "hard": 76.91373025516404 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 55.67272727272728, "tiers": { "easy": 84.00000000000001, "standard": 52, "judge": 72, "hard": 38.18181818181819 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 73.20270090632351, "tiers": { "easy": 82.23234484152947, "standard": 77.3304059414898, "judge": 72.01127703284678, "hard": 69.77500531004645 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 87.00732784869052, "parts": { "choice": { "ece_hard": 0.04126117192486663, "ece_score": 91.74776561502668, "mean_tvd": 0.28928215362572635, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 81.40977512622702 }, "noul": { "ece": 0.059567446143809336, "brier": 0.09627388769694645, "n": 495, "score": 88.08651077123814, "abstention_rate": 0.1393939393939394 }, "score": { "nrps": 0.055629168051636785, "top_ece": 0.056928439488117065, "n": 330, "interval90_coverage": 0.9666666666666667, "interval90_width": 0.3088383838383839, "score": 91.52569764860645 } } }, "speed": { "p50_s_raw": 0.03980663896072656, "p95_s_raw": 0.09906632518395769, "p50_s_adjusted": 0.2296132779214531, "p95_s_adjusted": 0.3481326503679154, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.028337561576354687, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 1, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 1, "B": 2, "C": 1 } }, { "key": "winnow-12b", "display": "Winnow-12B Q8", "addendum": null, "author": "Eldan Ring", "repo": "https://huggingface.co/EldanRing/Winnow-12B", "class": "jev-rebuild", "licence": "Apache-2.0, including the applicable Gemma 4 base/derivative licence terms", "open": "yes", "underlying": "google/gemma-4-12B-it LoRA fine-tune, merged and exported as Q8_0 GGUF", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "gpu": "RTX6000", "adapter_id": "images/winnow-12b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 74.43468593005443, "calibration": 84.06987570289192, "speed": 86.13960582383429, "cost": 56.556023424877196 }, "scores": { "A": 73.23310265651646, "B": 73.47030575448954, "C": 73.23310265651646 }, "views": { "Balanced 33:33:33": 70.21610446721299, "Emphasis on Accuracy 60:20:20": 71.8448249649829, "Emphasis on Cost 20:20:60": 64.02999151081272, "Emphasis on Speed 20:60:20": 75.8226400442452, "Intelligence only": 74.43468593005443 }, "composite_ci95": { "A": [ 72.02170416109966, 73.99468092883008 ], "B": [ 71.92399627008938, 74.62805392246878 ], "C": [ 72.02170416109966, 73.99468092883008 ] }, "jevbench_score": 73.23310265651646, "intelligence": { "I_open": 72.47593928648939, "I_sealed": 76.39343257361945, "base": 74.43468593005443, "gap": -3.917493287130057, "excess": -9.1041207872093, "penalty": 1, "per_type_split": { "open|choice": { "cc": 77.61378737459853, "tiers": { "easy": 98.19113656918904, "standard": 90.64205874707565, "judge": 83.40248962655602, "hard": 61.61378770074421 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 58.608048207215305, "tiers": { "easy": 82.35294117647058, "standard": 75.75757575757575, "judge": 66.66666666666667, "hard": 38.053097345132734 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 81.20598227765439, "tiers": { "easy": 90.95264588576535, "standard": 92.03169612810453, "judge": 81.13037013155842, "hard": 73.41316855997353 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 82.04454751825295, "tiers": { "easy": 97.0703125, "standard": 83.69934344577767, "judge": 82.773851590106, "hard": 76.91373025516404 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 66.14545454545454, "tiers": { "easy": 100, "standard": 60.00000000000001, "judge": 72, "hard": 56.363636363636374 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 80.99029565715088, "tiers": { "easy": 87.91654862427384, "standard": 81.96986364427659, "judge": 82.4600539087451, "hard": 77.66662973311162 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 84.06987570289192, "parts": { "choice": { "ece_hard": 0.08490714326828491, "ece_score": 83.01857134634302, "mean_tvd": 0.3393316979020659, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 74.54270077806822 }, "noul": { "ece": 0.06312829271927016, "brier": 0.08320230663204263, "n": 495, "score": 87.37434145614597, "abstention_rate": 0.13131313131313127 }, "score": { "nrps": 0.04821302833936116, "top_ece": 0.0729676370857035, "n": 330, "interval90_coverage": 0.9303030303030303, "interval90_width": 0.1404040404040404, "score": 90.2925848744616 } } }, "speed": { "p50_s_raw": 0.09506642795167863, "p95_s_raw": 0.2825698039145205, "p50_s_adjusted": 0.3401328559033573, "p95_s_adjusted": 0.7151396078290411, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.028062900246305422, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 2, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 2, "B": 1, "C": 2 } }, { "key": "jev-1.13.0", "display": "Jev 1.13.0 (TypeSafe AI)", "addendum": null, "author": "TypeSafe AI", "repo": "https://docs.typesafe.ai", "class": "jev", "licence": "proprietary API", "open": "no", "underlying": "closed", "endpoint_kind": "api", "endpoint_condition": "the operator's hosted API", "gpu": null, "adapter_id": "harness/specs/jev-1.13.0.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 71.99974990450261, "calibration": 88.02903108992821, "speed": 83.8140999985063, "cost": 54.72500234484654 }, "scores": { "A": 72.13292897895896, "B": 72.10625376369363, "C": 72.13292897895896 }, "views": { "Balanced 33:33:33": 68.03756745894525, "Emphasis on Accuracy 60:20:20": 69.56893280213254, "Emphasis on Cost 20:20:60": 62.00423444459753, "Emphasis on Speed 20:60:20": 73.57743121846089, "Intelligence only": 71.99974990450261 }, "composite_ci95": { "A": [ 71.01427970926099, 72.61481293732982 ], "B": [ 70.60279184971247, 73.05372079639332 ], "C": [ 71.01427970926099, 72.61481293732982 ] }, "jevbench_score": 72.13292897895896, "intelligence": { "I_open": 71.54923917772135, "I_sealed": 72.45026063128388, "base": 71.99974990450261, "gap": -0.9010214535625352, "excess": -6.087648953641778, "penalty": 1, "per_type_split": { "open|choice": { "cc": 85.73455614675619, "tiers": { "easy": 98.19113656918904, "standard": 98.44034312451261, "judge": 92.73858921161826, "hard": 71.01449275362319 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 47.75448393355734, "tiers": { "easy": 76.47058823529412, "standard": 54.54545454545454, "judge": 72.54901960784315, "hard": 18.584070796460182 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 81.15867745285051, "tiers": { "easy": 81.8188800446865, "standard": 86.17932862190813, "judge": 85.66186107470512, "hard": 75.10591350397176 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 87.59399876813663, "tiers": { "easy": 97.0703125, "standard": 89.13289563051845, "judge": 81.4487632508834, "hard": 89.0643985419198 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 48.618181818181824, "tiers": { "easy": 84.00000000000001, "standard": 60.00000000000001, "judge": 48, "hard": 34.54545454545455 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 81.13860130753321, "tiers": { "easy": 86.53079710144928, "standard": 77.45413836093937, "judge": 83.82451303260066, "hard": 79.61835003855052 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 88.02903108992821, "parts": { "choice": { "ece_hard": 0.027114101301826996, "ece_score": 94.5771797396346, "mean_tvd": 0.3155481665500016, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 81.51118154231722 }, "noul": { "ece": 0.05597979797979819, "brier": 0.07014484848484845, "n": 495, "score": 88.80404040404036, "abstention_rate": 0.2282828282828283 }, "score": { "nrps": 0.041089662604001445, "top_ece": 0.04173645546372849, "n": 330, "interval90_coverage": 0.9696969696969697, "interval90_width": 0.19823232323232326, "score": 93.77187132342706 } } }, "speed": { "p50_s_raw": 0.6164783202111721, "p95_s_raw": 0.674019081890583, "p50_s_adjusted": 0.6164783202111721, "p95_s_adjusted": 0.674019081890583, "n": 450, "adjustment": "none (hosted API, measured as is)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.032297327586206896, "basis": "operator standard launch list price (interpretation I-1); no exact base-model floor applies" }, "rank": 3, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 3, "B": 3, "C": 3 } }, { "key": "jev-omni", "display": "Jev-Omni (akhilaaa3, Gemma-4-12B merged)", "addendum": null, "author": "akhilaaa3", "repo": "https://huggingface.co/akhilaaa3/Jev-Omni", "class": "jev-rebuild", "licence": "Apache-2.0, following Gemma 4; dataset rights stated separately by the author", "open": "yes", "underlying": "google/gemma-4-12B-it fine-tuned and merged, with a trained 256-way decision head", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "gpu": "RTX6000", "adapter_id": "images/jev-omni/spec.json:omni_v15.JevOmni", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 70.49063617669168, "calibration": 82.59813256382532, "speed": 84.67623138886884, "cost": 56.05113155939739 }, "scores": { "A": 71.5005378093621, "B": 71.29624911998478, "C": 71.5005378093621 }, "views": { "Balanced 33:33:33": 68.43561492071707, "Emphasis on Accuracy 60:20:20": 69.24307531155604, "Emphasis on Cost 20:20:60": 62.87842598462993, "Emphasis on Speed 20:60:20": 74.1221679560491, "Intelligence only": 70.49063617669168 }, "composite_ci95": { "A": [ 70.21419464201573, 72.40139832609583 ], "B": [ 69.61520330245101, 72.61670152902757 ], "C": [ 70.21419464201573, 72.40139832609583 ] }, "jevbench_score": 71.5005378093621, "intelligence": { "I_open": 70.48105345190129, "I_sealed": 70.50021890148207, "base": 70.49063617669168, "gap": -0.01916544958078248, "excess": -5.205792949660025, "penalty": 1, "per_type_split": { "open|choice": { "cc": 83.36657120240034, "tiers": { "easy": 96.38227313837805, "standard": 89.08240187158823, "judge": 93.7759336099585, "hard": 69.44770857814335 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 55.282444433928035, "tiers": { "easy": 76.47058823529412, "standard": 69.6969696969697, "judge": 56.86274509803921, "hard": 41.5929203539823 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 72.7941447193755, "tiers": { "easy": 79.99649950944763, "standard": 86.06273734032244, "judge": 73.12472472755394, "hard": 64.11132470525015 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 78.70317394647086, "tiers": { "easy": 88.28125, "standard": 76.9074032148517, "judge": 77.47349823321555, "hard": 78.12879708383961 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 56.727272727272734, "tiers": { "easy": 84.00000000000001, "standard": 56.00000000000001, "judge": 68, "hard": 41.81818181818182 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 76.07021003070264, "tiers": { "easy": 83.58858680697834, "standard": 73.02819652611512, "judge": 77.2106969349196, "hard": 74.85625741076477 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 82.59813256382532, "parts": { "choice": { "ece_hard": 0.08520662096002907, "ece_score": 82.95867580799418, "mean_tvd": 0.34815184544266786, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 74.0717456318637 }, "noul": { "ece": 0.06459588018096656, "brier": 0.08581972089584758, "n": 495, "score": 87.08082396380668, "abstention_rate": 0.17575757575757578 }, "score": { "nrps": 0.06883415898624487, "top_ece": 0.09916463954882188, "n": 330, "interval90_coverage": 0.9333333333333333, "interval90_width": 0.18914141414141417, "score": 86.64182809580556 } } }, "speed": { "p50_s_raw": 0.112679542042315, "p95_s_raw": 0.37883703656261736, "p50_s_adjusted": 0.37535908408463003, "p95_s_adjusted": 0.9076740731252347, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.02917173645320197, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 6, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 6, "B": 4, "C": 4 } }, { "key": "decider-4b-v2", "display": "decider-4b v2 (Mapika)", "addendum": null, "author": "Mapika", "repo": "https://github.com/Mapika/decider", "class": "system-one-open", "licence": "Apache-2.0 (package and weights)", "open": "yes", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/decider-4b-v2/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 55.7709920864861, "calibration": 85.59400811608691, "speed": 90.85503597842234, "cost": 64.54376873042901 }, "scores": { "A": 71.28417523518583, "B": 67.5275035635358, "C": 61.58959786188317 }, "views": { "Balanced 33:33:33": 67.52137451850446, "Emphasis on Accuracy 60:20:20": 62.273238587557444, "Emphasis on Cost 20:20:60": 66.29796164251424, "Emphasis on Speed 20:60:20": 75.25194627944218, "Intelligence only": 55.7709920864861 }, "composite_ci95": { "A": [ 69.11045953172504, 72.34691401220934 ], "B": [ 64.68875564545965, 69.20814174207187 ], "C": [ 50.887685275957445, 69.72615382450459 ] }, "jevbench_score": 71.28417523518583, "intelligence": { "I_open": 61.74364566303797, "I_sealed": 49.79833850993424, "base": 55.7709920864861, "gap": 11.945307153103727, "excess": 6.758679653024484, "penalty": 1, "per_type_split": { "open|choice": { "cc": 80.43472694605762, "tiers": { "easy": 98.19113656918904, "standard": 92.20171562256304, "judge": 82.36514522821577, "hard": 68.66431649040345 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 42.918461028820204, "tiers": { "easy": 76.47058823529412, "standard": 54.54545454545454, "judge": 70.58823529411764, "hard": 7.964601769911495 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 61.87774901423609, "tiers": { "easy": 47.07476958525345, "standard": 70.18886925795053, "judge": 58.8005897771953, "hard": 63.730803177405114 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 77.31230284100064, "tiers": { "easy": 85.3515625, "standard": 78.2657912610369, "judge": 76.14840989399293, "hard": 75.69866342648847 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 18.727272727272723, "tiers": { "easy": 60.00000000000001, "standard": 24, "judge": 24, "hard": 1.8181818181818077 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 53.35543996152937, "tiers": { "easy": 66.72291666666668, "standard": 61.24424853064653, "judge": 52.99970513900589, "hard": 46.335967617579065 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 85.59400811608691, "parts": { "choice": { "ece_hard": 0.04920685920577605, "ece_score": 90.15862815884479, "mean_tvd": 0.22729216655000173, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 83.71470575192231 }, "noul": { "ece": 0.05604383838383825, "brier": 0.10456726022222225, "n": 495, "score": 88.79123232323235, "abstention_rate": 0.26868686868686864 }, "score": { "nrps": 0.09234675938636361, "top_ece": 0.11106575757575754, "n": 330, "interval90_coverage": 0.9757575757575757, "interval90_width": 0.5386363636363639, "score": 84.27608627310607 } } }, "speed": { "p50_s_raw": 0.026405358512420207, "p95_s_raw": 0.12747691074036993, "p50_s_adjusted": 0.2028107170248404, "p95_s_adjusted": 0.4049538214807399, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.015201077586206898, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 7, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 7, "B": 7, "C": 10 } }, { "key": "semif-qwen3.5-4b", "display": "SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ)", "addendum": null, "author": "Theodore Lee (TheoLeeCJ)", "repo": "https://github.com/TheoLeeCJ/openjev", "class": "jev-rebuild", "licence": "MIT (code); Qwen3.5 weights Apache-2.0", "open": "yes", "underlying": "Qwen/Qwen3.5-4B (frozen, BF16)", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/semif-qwen3.5-4b/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 51.31227416736609, "calibration": 83.9694498943646, "speed": 90.890351971217, "cost": 63.07359395281681 }, "scores": { "A": 68.66437558511035, "B": 64.31456686948948, "C": 50.21939777970113 }, "views": { "Balanced 33:33:33": 64.73151414788424, "Emphasis on Accuracy 60:20:20": 58.60132233439368, "Emphasis on Cost 20:20:60": 64.0579958758946, "Emphasis on Speed 20:60:20": 73.15308917543874, "Intelligence only": 51.31227416736609 }, "composite_ci95": { "A": [ 60.05966147761982, 70.15877841526802 ], "B": [ 55.6917195697546, 66.46721380623629 ], "C": [ 41.7080982483471, 58.662504296040126 ] }, "jevbench_score": 68.66437558511035, "intelligence": { "I_open": 53.996219382373376, "I_sealed": 48.62832895235881, "base": 51.31227416736609, "gap": 5.367890430014569, "excess": 0.18126292993532633, "penalty": 1, "per_type_split": { "open|choice": { "cc": 73.79929570880981, "tiers": { "easy": 94.57340970756708, "standard": 82.84377436963868, "judge": 89.6265560165975, "hard": 52.21308264786526 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 30.297824680958463, "tiers": { "easy": 58.823529411764696, "standard": 51.515151515151516, "judge": 52.941176470588225, "hard": -4.424778761061942 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 57.891537757351855, "tiers": { "easy": 38.14197517572985, "standard": 75.65692943620333, "judge": 57.516282294121936, "hard": 54.227674160754056 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 70.95783711255453, "tiers": { "easy": 79.4921875, "standard": 72.83223907629615, "judge": 58.92226148409894, "hard": 76.91373025516404 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 13.927272727272719, "tiers": { "easy": 52, "standard": 15.999999999999993, "judge": 15.999999999999993, "hard": 1.8181818181818077 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 60.99987701724918, "tiers": { "easy": 76.14710035023134, "standard": 63.67894900857971, "judge": 60.24526285169589, "hard": 56.43949581250334 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 83.9694498943646, "parts": { "choice": { "ece_hard": 0.06314318276148391, "ece_score": 87.37136344770322, "mean_tvd": 0.2735131150426244, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 80.01002597172038 }, "noul": { "ece": 0.06720214043427124, "brier": 0.1307654521663549, "n": 495, "score": 86.55957191314575, "abstention_rate": 0.2909090909090909 }, "score": { "nrps": 0.10609041896052611, "top_ece": 0.09356727253746004, "n": 330, "interval90_coverage": 0.9454545454545454, "interval90_width": 0.5229797979797979, "score": 85.33875179822769 } } }, "speed": { "p50_s_raw": 0.039298868738114834, "p95_s_raw": 0.10318161030299944, "p50_s_adjusted": 0.22859773747622966, "p95_s_adjusted": 0.3563632206059989, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.01701689039408867, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 11, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 11, "B": 14, "C": 19 } }, { "key": "spark-s1-4b-v6", "display": "spark-s1-4b-v6 (Open Spark Jev, abhishek085)", "addendum": null, "author": "Abhishek Rai (abhishek085)", "repo": "https://github.com/abhishek085/open-spark-jev", "class": "jev-rebuild", "licence": "Apache-2.0 (code and weights); base Qwen/Qwen3.5-4B Apache-2.0", "open": "yes", "underlying": "Qwen/Qwen3.5-4B with a LoRA, read at the option-letter logits with a fitted temperature (no extra head)", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "gpu": "RTX6000", "adapter_id": "images/spark-s1-4b-v6/spec.json:spark_v15.Spark", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 62.11757005795365, "calibration": 69.70580960824081, "speed": 85.52672016213519, "cost": 60.422063196854964 }, "scores": { "A": 68.15818403751406, "B": 66.85786733898678, "C": 68.15818403751406 }, "views": { "Balanced 33:33:33": 67.65746810827054, "Emphasis on Accuracy 60:20:20": 65.32701629026283, "Emphasis on Cost 20:20:60": 64.56486948416433, "Emphasis on Speed 20:60:20": 73.82742716682645, "Intelligence only": 62.11757005795365 }, "composite_ci95": { "A": [ 66.27191706094285, 69.70470030753586 ], "B": [ 64.54463023423625, 68.79399704219207 ], "C": [ 62.94461884952748, 69.70470030753586 ] }, "jevbench_score": 68.15818403751406, "intelligence": { "I_open": 62.460678071910536, "I_sealed": 61.77446204399676, "base": 62.11757005795365, "gap": 0.6862160279137726, "excess": -4.50041147216547, "penalty": 1, "per_type_split": { "open|choice": { "cc": 72.55232891520762, "tiers": { "easy": 94.57340970756708, "standard": 82.84377436963868, "judge": 86.51452282157676, "hard": 51.42969056012534 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 50.43601028504724, "tiers": { "easy": 64.70588235294117, "standard": 69.6969696969697, "judge": 70.58823529411764, "hard": 22.123893805309724 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 64.39369501547677, "tiers": { "easy": 87.17913149779508, "standard": 74.65259596167797, "judge": 61.1941893149865, "hard": 55.967514697164255 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 73.1148440968712, "tiers": { "easy": 79.4921875, "standard": 71.47385103011095, "judge": 73.49823321554771, "hard": 72.05346294046174 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 46.472727272727276, "tiers": { "easy": 76, "standard": 28.000000000000004, "judge": 60.00000000000001, "hard": 38.18181818181819 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 65.73581476239181, "tiers": { "easy": 68.26278642742467, "standard": 64.68169284072869, "judge": 62.18712724711797, "hard": 68.29264844342052 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 69.70580960824081, "parts": { "choice": { "ece_hard": 0.16689755974842224, "ece_score": 66.62048805031556, "mean_tvd": 0.35435283393590955, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 65.5926023283623 }, "noul": { "ece": 0.1283910387499207, "brier": 0.1442645809272019, "n": 495, "score": 74.32179225001586, "abstention_rate": 0.1333333333333333 }, "score": { "nrps": 0.11390986143211532, "top_ece": 0.2510147268204994, "n": 330, "interval90_coverage": 0.7515151515151515, "interval90_width": 0.08106060606060606, "score": 69.20303424634429 } } }, "speed": { "p50_s_raw": 0.1279726339271292, "p95_s_raw": 0.2700090640457343, "p50_s_adjusted": 0.40594526785425844, "p95_s_adjusted": 0.6900181280914687, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.020857610837438423, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 12, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 12, "B": 8, "C": 5 } }, { "key": "metask-jev-4b", "display": "metask-jev-4b", "addendum": null, "author": "Wayfind (metask-ai)", "repo": "https://github.com/metask-ai/metask-jev", "class": "jev-rebuild", "licence": "Apache-2.0", "open": "yes", "underlying": "Qwen3.5-4B merged r16 LoRA, candidate-logit readout", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/metask-jev-4b/spec.json:metask_v15.MetaskJevWarm", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1622 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 53.478671436751014, "calibration": 82.74360182521858, "speed": 89.49053928541517, "cost": 57.7311327584882 }, "scores": { "A": 67.47568667137665, "B": 64.11928820546953, "C": 53.6050902247502 }, "views": { "Balanced 33:33:33": 63.565940709101675, "Emphasis on Accuracy 60:20:20": 59.10642509551968, "Emphasis on Cost 20:20:60": 61.095985088875466, "Emphasis on Speed 20:60:20": 71.89711825951035, "Intelligence only": 53.478671436751014 }, "composite_ci95": { "A": [ 65.39080983744157, 68.69256296806242 ], "B": [ 61.58172699270945, 65.90468584974295 ], "C": [ 45.52555578171516, 61.50937969724669 ] }, "jevbench_score": 67.47568667137665, "intelligence": { "I_open": 56.74073420693068, "I_sealed": 50.216608666571354, "base": 53.478671436751014, "gap": 6.524125540359329, "excess": 1.337498040280086, "penalty": 1, "per_type_split": { "open|choice": { "cc": 70.10893404347104, "tiers": { "easy": 98.19113656918904, "standard": 78.1648037431765, "judge": 81.32780082987551, "hard": 50.64629847238542 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 43.229220891896574, "tiers": { "easy": 76.47058823529412, "standard": 66.66666666666667, "judge": 58.823529411764696, "hard": 11.504424778761058 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 56.88404768542444, "tiers": { "easy": 32.011664274608734, "standard": 69.38352511074332, "judge": 53.48129432018609, "hard": 59.4044698493977 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 72.41221362515527, "tiers": { "easy": 82.421875, "standard": 75.5490151686665, "judge": 64.22261484098941, "hard": 74.48359659781289 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 25.2, "tiers": { "easy": 52, "standard": 24, "judge": 24, "hard": 19.999999999999996 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 53.03761237455879, "tiers": { "easy": 67.18940475884378, "standard": 57.50830908544155, "judge": 54.88502353109698, "hard": 45.87875755564252 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.0025031289111389077 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 82.74360182521858, "parts": { "choice": { "ece_hard": 0.04753605370252558, "ece_score": 90.49278925949488, "mean_tvd": 0.2525412810871565, "n_tvd": 50, "n": 797, "n_ece_hard": 275, "score": 82.61933057538961 }, "noul": { "ece": 0.07502700184001239, "brier": 0.13271058288209991, "n": 495, "score": 84.99459963199753, "abstention_rate": 0.21212121212121215 }, "score": { "nrps": 0.11350190609753398, "top_ece": 0.1370802942685472, "n": 330, "interval90_coverage": 0.9606060606060606, "interval90_width": 0.5760101010101012, "score": 80.61687526826859 } } }, "speed": { "p50_s_raw": 0.06768286094302312, "p95_s_raw": 0.12202179493033323, "p50_s_adjusted": 0.28536572188604625, "p95_s_adjusted": 0.39404358986066645, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.025642610837438424, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 13, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 13, "B": 15, "C": 16 } }, { "key": "hopper", "display": "Hopper", "addendum": null, "author": "HopitAI", "repo": "https://huggingface.co/HopitAI/hopper", "class": "jev-rebuild", "licence": "Component-specific terms recorded in RESULT.md; submitted adapter release and Qwen base retain their respective terms", "open": "yes", "underlying": "Qwen3.5-4B plus HopitAI/hopper LoRA", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/hopper/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 49.86320336082703, "calibration": 87.86903672712391, "speed": 87.23386153386807, "cost": 62.25627699896333 }, "scores": { "A": 67.47015061443733, "B": 62.932238759063466, "C": 46.85427126002592 }, "views": { "Balanced 33:33:33": 62.70591910312661, "Emphasis on Accuracy 60:20:20": 56.70702953180434, "Emphasis on Cost 20:20:60": 62.38757962057503, "Emphasis on Speed 20:60:20": 70.52661677934104, "Intelligence only": 49.590731857742696 }, "composite_ci95": { "A": [ 56.33592438734226, 69.15394144849903 ], "B": [ 51.98455577329755, 65.27281473616978 ], "C": [ 39.12216971343212, 54.929785035322034 ] }, "jevbench_score": 67.47015061443733, "intelligence": { "I_open": 48.7045625282528, "I_sealed": 51.02184419340125, "base": 49.86320336082703, "gap": -2.3172816651484496, "excess": -7.503909165227692, "penalty": 1, "per_type_split": { "open|choice": { "cc": 70.1240393038206, "tiers": { "easy": 98.19113656918904, "standard": 82.84377436963868, "judge": 71.99170124481329, "hard": 55.34665099882491 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 10.233937500985915, "tiers": { "easy": 70.58823529411764, "standard": 42.42424242424243, "judge": 0, "hard": -13.27433628318584 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 65.7557107799519, "tiers": { "easy": 48.33854622640297, "standard": 78.65058411480577, "judge": 69.38652302338251, "hard": 60.93945606833921 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 75.48169096739608, "tiers": { "easy": 85.3515625, "standard": 74.19062712248135, "judge": 69.52296819787986, "hard": 78.12879708383961 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 12.763636363636362, "tiers": { "easy": 60.00000000000001, "standard": 15.999999999999993, "judge": 24, "hard": -9.090909090909093 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 64.82020524917131, "tiers": { "easy": 79.23209224282715, "standard": 67.59502487034688, "judge": 65.64745705143598, "hard": 59.20938483847106 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 87.86903672712391, "parts": { "choice": { "ece_hard": 0.047132288889173826, "ece_score": 90.57354222216524, "mean_tvd": 0.15466886536122398, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 87.55332784302142 }, "noul": { "ece": 0.07041272261623366, "brier": 0.10425515982817155, "n": 495, "score": 85.91745547675328, "abstention_rate": 0.406060606060606 }, "score": { "nrps": 0.0863650042965921, "top_ece": 0.05545422923573362, "n": 330, "interval90_coverage": 0.9515151515151515, "interval90_width": 0.41489898989898993, "score": 90.13632686159704 } } }, "speed": { "p50_s_raw": 0.12245992664247751, "p95_s_raw": 0.16437284634448568, "p50_s_adjusted": 0.39491985328495505, "p95_s_adjusted": 0.4787456926889714, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.018118577586206896, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 14, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 14, "B": 18, "C": 23 } }, { "key": "malkuth-4b", "display": "Malkuth-4B (newfull5, Kev post-train)", "addendum": null, "author": "newfull5 (dhtocks)", "repo": "https://github.com/newfull5/malkuth", "class": "jev-rebuild", "licence": "CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)", "open": "yes", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/malkuth-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 54.45108712587745, "calibration": 83.20783168346826, "speed": 86.15537413790446, "cost": 55.82433433950372 }, "scores": { "A": 66.77408647378738, "B": 63.88259181735141, "C": 54.994412178529096 }, "views": { "Balanced 33:33:33": 62.64960437870624, "Emphasis on Accuracy 60:20:20": 59.090765182620935, "Emphasis on Cost 20:20:60": 59.72855893426261, "Emphasis on Speed 20:60:20": 70.32422767798526, "Intelligence only": 54.45108712587746 }, "composite_ci95": { "A": [ 64.936284491473, 67.89009495158388 ], "B": [ 61.571100423606644, 65.54784266334441 ], "C": [ 46.8770023830624, 62.78108641487751 ] }, "jevbench_score": 66.77408647378738, "intelligence": { "I_open": 58.362772871720175, "I_sealed": 50.53940138003472, "base": 54.45108712587745, "gap": 7.823371491685457, "excess": 2.636743991606214, "penalty": 1, "per_type_split": { "open|choice": { "cc": 71.16861524959509, "tiers": { "easy": 96.38227313837805, "standard": 87.52274499610087, "judge": 83.40248962655602, "hard": 47.51273012142576 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 39.480542015679966, "tiers": { "easy": 76.47058823529412, "standard": 51.515151515151516, "judge": 70.58823529411764, "hard": 0.8849557522123908 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 64.43916134988548, "tiers": { "easy": 36.32776497695851, "standard": 78.3743816254417, "judge": 66.15386631716908, "hard": 63.21337157987643 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 69.55316310303347, "tiers": { "easy": 76.56249999999999, "standard": 71.47385103011095, "judge": 64.22261484098941, "hard": 70.83839611178615 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 22.836363636363632, "tiers": { "easy": 76, "standard": 15.999999999999993, "judge": 28.000000000000004, "hard": 9.090909090909083 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 59.228677400707056, "tiers": { "easy": 77.18079710144929, "standard": 67.53711167086482, "judge": 51.13374052232519, "hard": 56.657632999229 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 83.20783168346826, "parts": { "choice": { "ece_hard": 0.043919494584837536, "ece_score": 91.2161010830325, "mean_tvd": 0.2473586666000066, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 83.24011721151592 }, "noul": { "ece": 0.10944020202020216, "brier": 0.14711016430303037, "n": 495, "score": 78.11195959595956, "abstention_rate": 0.2282828282828283 }, "score": { "nrps": 0.09267163514141413, "top_ece": 0.07094999999999994, "n": 330, "interval90_coverage": 0.9606060606060606, "interval90_width": 0.43181818181818166, "score": 88.2714182429293 } } }, "speed": { "p50_s_raw": 0.14595738402567804, "p95_s_raw": 0.19921681064297442, "p50_s_adjusted": 0.4419147680513561, "p95_s_adjusted": 0.5484336212859489, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.02968398399014778, "basis": "price floor: base-model reference price applied; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 15, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 15, "B": 16, "C": 15 } }, { "key": "reflex-4b", "display": "reflex 4B (kshetrajna12)", "addendum": null, "author": "kshetrajna12", "repo": "https://github.com/kshetrajna12/reflex", "class": "jev-rebuild", "licence": "MIT (code, adapter); Apache-2.0 (base)", "open": "yes", "underlying": "Qwen/Qwen3.5-4B + kshetrajna12/reflex-qwen3.5-4b-lora", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/reflex-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 51.49107492301285, "calibration": 86.84320054497626, "speed": 68.77483167692569, "cost": 63.142712976263866 }, "scores": { "A": 65.23881652418177, "B": 61.93175130880487, "C": 48.04713427572403 }, "views": { "Balanced 33:33:33": 60.24316014490931, "Emphasis on Accuracy 60:20:20": 56.408026714558744, "Emphasis on Cost 20:20:60": 61.370427494798925, "Emphasis on Speed 20:60:20": 63.38855699417101, "Intelligence only": 51.49107492301285 }, "composite_ci95": { "A": [ 58.56011597033304, 66.34964401610556 ], "B": [ 55.04915488817, 63.722007658620804 ], "C": [ 40.66674720162017, 55.96422378625981 ] }, "jevbench_score": 65.23881652418177, "intelligence": { "I_open": 55.20563662766551, "I_sealed": 47.77651321836019, "base": 51.49107492301285, "gap": 7.429123409305319, "excess": 2.2424959092260766, "penalty": 1, "per_type_split": { "open|choice": { "cc": 73.66464105000873, "tiers": { "easy": 96.38227313837805, "standard": 78.1648037431765, "judge": 92.73858921161826, "hard": 51.42969056012534 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 33.33349107945673, "tiers": { "easy": 70.58823529411764, "standard": 45.45454545454546, "judge": 49.01960784313726, "hard": 6.194690265486735 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 58.618777753531084, "tiers": { "easy": 30.865999999999982, "standard": 72.12921687279152, "judge": 57.9468256880734, "hard": 59.30571668137689 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 69.3128758568003, "tiers": { "easy": 82.421875, "standard": 74.19062712248135, "judge": 62.897526501766784, "hard": 68.408262454435 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 19.454545454545457, "tiers": { "easy": 52, "standard": 12.00000000000001, "judge": 8.000000000000007, "hard": 23.636363636363633 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 54.56211834373482, "tiers": { "easy": 70.11713768115942, "standard": 60.228924013434096, "judge": 50.31963774220725, "hard": 51.021821125674684 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 86.84320054497626, "parts": { "choice": { "ece_hard": 0.06095504332129953, "ece_score": 87.80899133574009, "mean_tvd": 0.24011401655000167, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 81.89879484036996 }, "noul": { "ece": 0.03688168282828298, "brier": 0.1222939817592503, "n": 495, "score": 92.6236634343434, "abstention_rate": 0.2909090909090909 }, "score": { "nrps": 0.10425772067447935, "top_ece": 0.08779970606060608, "n": 330, "interval90_coverage": 0.9666666666666667, "interval90_width": 0.6209595959595962, "score": 86.00714336021542 } } }, "speed": { "p50_s_raw": 1.3576662300620228, "p95_s_raw": 2.238725413940847, "p50_s_adjusted": 2.8653324601240455, "p95_s_adjusted": 4.627450827881694, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.016926853448275863, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 17, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 17, "B": 19, "C": 20 } }, { "key": "jev-local", "display": "jev-local (Qwen3.5-9B)", "addendum": null, "author": "us (GitHub)", "repo": "https://github.com/us/jev-local", "class": "jev-rebuild", "licence": "no licence stated in the repository (public code); Apache-2.0 base weights", "open": "yes", "underlying": "Qwen/Qwen3.5-9B, frozen, per-option mean log-probability", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/jev-local/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 56.27600615219267, "calibration": 77.82254243294076, "speed": 72.97538812549428, "cost": 58.85554343293177 }, "scores": { "A": 65.23868668964282, "B": 63.224813137015204, "C": 57.391720704958274 }, "views": { "Balanced 33:33:33": 61.902172672191675, "Emphasis on Accuracy 60:20:20": 59.52190326603423, "Emphasis on Cost 20:20:60": 60.64643903259494, "Emphasis on Speed 20:60:20": 65.90214393582374, "Intelligence only": 56.27600615219267 }, "composite_ci95": { "A": [ 63.46433946165809, 66.49644591562449 ], "B": [ 61.014275918200745, 65.01397981407818 ], "C": [ 49.030955191671445, 65.92078049729584 ] }, "jevbench_score": 65.23868668964282, "intelligence": { "I_open": 58.180064462983424, "I_sealed": 54.37194784140192, "base": 56.27600615219267, "gap": 3.8081166215815045, "excess": -1.378510878497738, "penalty": 1, "per_type_split": { "open|choice": { "cc": 66.88423583037286, "tiers": { "easy": 90.95568284594513, "standard": 71.92617624122693, "judge": 81.32780082987551, "hard": 47.51273012142576 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 46.944615336078115, "tiers": { "easy": 94.11764705882352, "standard": 57.57575757575757, "judge": 66.66666666666667, "hard": 15.04424778761062 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 60.7113422224993, "tiers": { "easy": 34.579953917050645, "standard": 76.4059628975265, "judge": 59.867169069462655, "hard": 60.030008826125325 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 65.39152308484063, "tiers": { "easy": 61.91406249999999, "standard": 61.96513470681459, "judge": 61.572438162544174, "hard": 70.83839611178615 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 37.01818181818182, "tiers": { "easy": 52, "standard": 12.00000000000001, "judge": 52, "hard": 34.54545454545455 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 60.7061386211833, "tiers": { "easy": 77.73496376811593, "standard": 72.42779177162049, "judge": 53.71099410278012, "hard": 55.83446414803394 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 77.82254243294076, "parts": { "choice": { "ece_hard": 0.10431407942238266, "ece_score": 79.13718411552347, "mean_tvd": 0.2741831665500017, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 75.85943373026166 }, "noul": { "ece": 0.131251515151515, "brier": 0.15351713593939395, "n": 495, "score": 73.749696969697, "abstention_rate": 0.1515151515151515 }, "score": { "nrps": 0.10504582559848477, "top_ece": 0.10889212121212123, "n": 330, "interval90_coverage": 0.8757575757575757, "interval90_width": 0.26313131313131316, "score": 83.85849659886364 } } }, "speed": { "p50_s_raw": 0.3970644664950669, "p95_s_raw": 2.594315356318838, "p50_s_adjusted": 0.9441289329901338, "p95_s_adjusted": 5.338630712637676, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.023522413793103446, "basis": "price floor: base-model reference price applied" }, "rank": 18, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 18, "B": 17, "C": 11 } }, { "key": "djev", "display": "djev (Maisa, diffusion-gemma)", "addendum": null, "author": "Maisa (David Villalón)", "repo": "https://github.com/Davipar/djev-dev", "class": "jev-rebuild", "licence": "Apache-2.0 code; Google DiffusionGemma Apache-2.0 weights; no djev-specific weights", "open": "yes", "underlying": "inference method on google/diffusiongemma-26B-A4B-it (one structured denoising read), not a separately trained model", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/djev/spec.json:jevbench.adapters.djev.DjevAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 72.32562987104993, "calibration": 80.4134255723618, "speed": 91.0029245322729, "cost": 48.22031380545122 }, "scores": { "A": 64.15569234453473, "B": 64.75500708450079, "C": 64.15569234453473 }, "views": { "Balanced 33:33:33": 61.25235241364572, "Emphasis on Accuracy 60:20:20": 63.52491736662031, "Emphasis on Cost 20:20:60": 53.43474942542091, "Emphasis on Speed 20:60:20": 68.86368074385312, "Intelligence only": 67.26858316507415 }, "composite_ci95": { "A": [ 63.06972830768684, 65.00414152390454 ], "B": [ 63.369610740199924, 65.92487018708866 ], "C": [ 63.06972830768684, 65.00414152390454 ] }, "jevbench_score": 64.15569234453473, "intelligence": { "I_open": 72.70481821582241, "I_sealed": 71.94644152627744, "base": 72.32562987104993, "gap": 0.7583766895449742, "excess": -4.428250810534268, "penalty": 1, "per_type_split": { "open|choice": { "cc": 76.61741501146423, "tiers": { "easy": 94.57340970756708, "standard": 92.20171562256304, "judge": 86.51452282157676, "hard": 56.91343517430473 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 66.16361427917909, "tiers": { "easy": 82.35294117647058, "standard": 78.78787878787878, "judge": 80.3921568627451, "hard": 45.13274336283186 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 75.33342535682394, "tiers": { "easy": 85.959739463197, "standard": 85.35101065815556, "judge": 72.92196305695782, "hard": 69.47665090446445 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 72.7798797008079, "tiers": { "easy": 85.3515625, "standard": 72.83223907629615, "judge": 69.52296819787986, "hard": 72.05346294046174 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 70.25454545454545, "tiers": { "easy": 92, "standard": 63.99999999999999, "judge": 76, "hard": 63.63636363636365 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 72.80489942347899, "tiers": { "easy": 76.59060844866048, "standard": 71.952289347639, "judge": 70.41666194673873, "hard": 74.07595531265878 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 80.4134255723618, "parts": { "choice": { "ece_hard": 0.12015275137219117, "ece_score": 75.96944972556176, "mean_tvd": 0.3136562430742985, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 72.30191270906596 }, "noul": { "ece": 0.0897244771539998, "brier": 0.0956660081741808, "n": 495, "score": 82.05510456920004, "abstention_rate": 0.07070707070707072 }, "score": { "nrps": 0.0693604150006043, "top_ece": 0.09648719811150377, "n": 330, "interval90_coverage": 0.9, "interval90_width": 0.1881313131313132, "score": 86.88325943881941 } } }, "speed": { "p50_s_raw": 0.050276658963412046, "p95_s_raw": 0.08340810290537767, "p50_s_adjusted": 0.2505533179268241, "p95_s_adjusted": 0.3168162058107553, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.05320954433497537, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 19, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 19, "B": 13, "C": 7 } }, { "key": "raw-qwen3-4b-instruct-2507", "display": "Raw Qwen3 4B Instruct 2507 direct logits", "addendum": null, "author": "Alibaba Qwen / neutral reproduction", "repo": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507", "class": "raw-logit-control", "licence": "Apache-2.0", "open": "yes", "underlying": "Qwen3-4B-Instruct-2507 BF16", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/raw-qwen3-4b-instruct-2507/spec.json:raw_adapter.RawControlAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 54.06280185198723, "calibration": 52.94631488507357, "speed": 89.228241580316, "cost": 63.36068112101963 }, "scores": { "A": 62.13695038108371, "B": 60.334779733788544, "C": 50.44806735123544 }, "views": { "Balanced 33:33:33": 65.95308478978349, "Emphasis on Accuracy 60:20:20": 60.62010074266458, "Emphasis on Cost 20:20:60": 64.8910768522085, "Emphasis on Speed 20:60:20": 73.63628483823207, "Intelligence only": 54.06280185198723 }, "composite_ci95": { "A": [ 59.35165730733617, 64.21780469584392 ], "B": [ 57.342469645630096, 62.736547541702414 ], "C": [ 41.992589504404044, 59.11349905782746 ] }, "jevbench_score": 62.13695038108371, "intelligence": { "I_open": 57.42840197206909, "I_sealed": 50.69720173190538, "base": 54.06280185198723, "gap": 6.731200240163709, "excess": 1.5445727400844662, "penalty": 1, "per_type_split": { "open|choice": { "cc": 65.49764059357105, "tiers": { "easy": 92.7645462767561, "standard": 75.04548999220171, "judge": 80.29045643153528, "hard": 42.81237759498628 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 49.80707649109523, "tiers": { "easy": 82.35294117647058, "standard": 42.42424242424243, "judge": 78.43137254901961, "hard": 23.893805309734507 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 56.980488831541, "tiers": { "easy": 37.16373879798242, "standard": 74.5345292714069, "judge": 52.065630628239326, "hard": 56.84379977247396 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 63.45679070007781, "tiers": { "easy": 76.56249999999999, "standard": 71.47385103011095, "judge": 53.621908127208485, "hard": 63.54799513973269 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 38.036363636363646, "tiers": { "easy": 12.00000000000001, "standard": 32.00000000000001, "judge": 36.00000000000001, "hard": 49.09090909090909 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 50.598450859274706, "tiers": { "easy": 71.82574190481856, "standard": 64.809117602594, "judge": 45.36051487860803, "hard": 42.11474671172911 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 52.94631488507357, "parts": { "choice": { "ece_hard": 0.32878554147980255, "ece_score": 34.242891704039494, "mean_tvd": 0.40842296047137, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 46.70029782845124 }, "noul": { "ece": 0.22707429711655797, "brier": 0.23134890441078815, "n": 495, "score": 54.58514057668841, "abstention_rate": 0.0444444444444444 }, "score": { "nrps": 0.15100753264429193, "top_ece": 0.3489611711770434, "n": 330, "interval90_coverage": 0.6848484848484848, "interval90_width": 0.11994949494949493, "score": 57.55350625008107 } } }, "speed": { "p50_s_raw": 0.05585077637806535, "p95_s_raw": 0.153212562156841, "p50_s_adjusted": 0.2617015527561307, "p95_s_adjusted": 0.456425124313682, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.01664602832512315, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 20, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 20, "B": 20, "C": 18 } }, { "key": "jqv", "display": "jqv (Qwen3-32B zero-shot)", "addendum": null, "author": "hjmurmur (Octalab)", "repo": "https://github.com/Octalab-Inc/jqv", "class": "jev-rebuild", "licence": "Apache-2.0 (Qwen3-32B weights); serving code public", "open": "yes", "underlying": "Qwen/Qwen3-32B, bf16, read as a direct-logit classifier (no fine-tuning)", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/jqv/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 49.086814605278015, "calibration": 86.76753584348495, "speed": 83.36030040456406, "cost": 51.172592607797476 }, "scores": { "A": 60.77445997498618, "B": 57.50151945806778, "C": 42.2044860937404 }, "views": { "Balanced 33:33:33": 55.70071491439261, "Emphasis on Accuracy 60:20:20": 52.01101730934296, "Emphasis on Cost 20:20:60": 52.96026958495173, "Emphasis on Speed 20:60:20": 63.49005579723604, "Intelligence only": 47.310173664858446 }, "composite_ci95": { "A": [ 51.25244150522343, 64.26095073970949 ], "B": [ 48.000035915340774, 61.47780194813299 ], "C": [ 35.591973267516266, 49.209015623794464 ] }, "jevbench_score": 60.77445997498618, "intelligence": { "I_open": 51.3394552913734, "I_sealed": 46.83417391918263, "base": 49.086814605278015, "gap": 4.505281372190765, "excess": -0.6813461278884771, "penalty": 1, "per_type_split": { "open|choice": { "cc": 72.99961235350925, "tiers": { "easy": 92.7645462767561, "standard": 85.96308812061348, "judge": 84.43983402489627, "hard": 52.996474735605155 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 18.673039610051582, "tiers": { "easy": 64.70588235294117, "standard": 36.36363636363635, "judge": 29.41176470588236, "hard": -9.734513274336287 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 62.345713910559354, "tiers": { "easy": 32.955006050581126, "standard": 73.96017267622133, "judge": 63.217955635894754, "hard": 63.23198019872138 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 73.08890597287122, "tiers": { "easy": 85.3515625, "standard": 72.83223907629615, "judge": 72.1731448763251, "hard": 70.83839611178615 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 10.181818181818176, "tiers": { "easy": 43.99999999999999, "standard": -12, "judge": 19.999999999999996, "hard": 5.454545454545445 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 57.23179760285851, "tiers": { "easy": 69.61177215806845, "standard": 62.693423998098766, "judge": 53.08672551735734, "hard": 54.51479483056176 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 86.76753584348495, "parts": { "choice": { "ece_hard": 0.06104189082173234, "ece_score": 87.79162183565353, "mean_tvd": 0.2675015176296743, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 80.52073503634305 }, "noul": { "ece": 0.04726762280733596, "brier": 0.11782616492468487, "n": 495, "score": 90.54647543853281, "abstention_rate": 0.37575757575757573 }, "score": { "nrps": 0.09595485271258676, "top_ece": 0.059668603087916515, "n": 330, "interval90_coverage": 0.9333333333333333, "interval90_width": 0.4224747474747473, "score": 89.235397055579 } } }, "speed": { "p50_s_raw": 0.07847566052805632, "p95_s_raw": 0.6763987306097987, "p50_s_adjusted": 0.30695132105611267, "p95_s_adjusted": 1.5027974612195973, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.042420935960591134, "basis": "documented hosted-model estimate" }, "rank": 21, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 21, "B": 21, "C": 25 } }, { "key": "jevk5-v02", "display": "JevK5 v0.2.0", "addendum": null, "author": "allebee", "repo": "https://github.com/allebee/jevk5", "class": "jev-rebuild", "licence": "Apache-2.0 (code/adapter); Apache-2.0 (Qwen3.5-4B base)", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/jevk5-v02/spec.json:jevk5_direct.JevK5DirectAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 46.704186767134146, "calibration": 84.87188332789893, "speed": 90.85646347150488, "cost": 63.07359395281681 }, "scores": { "A": 58.11452490392985, "B": 53.55068001120491, "C": 40.35730896106238 }, "views": { "Balanced 33:33:33": 54.22450179173148, "Emphasis on Accuracy 60:20:20": 47.89028098056264, "Emphasis on Cost 20:20:60": 54.54483488475008, "Emphasis on Speed 20:60:20": 62.069633409460685, "Intelligence only": 40.74998323664581 }, "composite_ci95": { "A": [ 47.559542661909845, 68.24036195737175 ], "B": [ 43.288013328987475, 63.73326751910801 ], "C": [ 33.027460181881835, 47.99518550145878 ] }, "jevbench_score": 58.11452490392985, "intelligence": { "I_open": 50.008514935705065, "I_sealed": 43.39985859856322, "base": 46.704186767134146, "gap": 6.608656337141845, "excess": 1.4220288370626024, "penalty": 1, "per_type_split": { "open|choice": { "cc": 77.60514266103064, "tiers": { "easy": 98.19113656918904, "standard": 85.96308812061348, "judge": 89.6265560165975, "hard": 59.263611437524474 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 12.186992254665341, "tiers": { "easy": 52.941176470588225, "standard": 15.15151515151516, "judge": 35.29411764705883, "hard": -16.814159292035402 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 60.23340989141923, "tiers": { "easy": 28.60779652081231, "standard": 74.75649985421431, "judge": 62.07745703499336, "hard": 59.49523289499283 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 71.7603783528002, "tiers": { "easy": 82.421875, "standard": 68.75707493774055, "judge": 68.19787985865725, "hard": 73.2685297691373 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -3.890909090909095, "tiers": { "easy": 43.99999999999999, "standard": -16.000000000000004, "judge": 0, "hard": -12.727272727272732 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 62.330106533798556, "tiers": { "easy": 74.06669119758907, "standard": 67.10954514563969, "judge": 62.80257002389328, "hard": 56.651893444359324 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 84.87188332789893, "parts": { "choice": { "ece_hard": 0.05857257507337988, "ece_score": 88.28548498532402, "mean_tvd": 0.2615167615672721, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 81.0669044142984 }, "noul": { "ece": 0.06765822430583412, "brier": 0.11693544190853578, "n": 495, "score": 86.46835513883318, "abstention_rate": 0.4242424242424242 }, "score": { "nrps": 0.0947385599255009, "top_ece": 0.08182681573159765, "n": 330, "interval90_coverage": 0.9424242424242424, "interval90_width": 0.4585858585858586, "score": 87.08039043056519 } } }, "speed": { "p50_s_raw": 0.033216340001672506, "p95_s_raw": 0.11467095024418086, "p50_s_adjusted": 0.216432680003345, "p95_s_adjusted": 0.3793419004883617, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.01701689039408867, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 22, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 22, "B": 22, "C": 28 } }, { "key": "qwen35-9b-jev-data-mix-v2", "display": "Qwen3.5-9B Jev-like data-mix v2", "addendum": null, "author": "jsaurabh", "repo": "https://huggingface.co/jsaurabh/qwen3.5-9b-jev-data-mix-v2", "class": "jev-rebuild", "licence": "Apache-2.0 (adapter code and weights; Qwen3.5-9B base is Apache-2.0)", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/qwen35-9b-jev-data-mix-v2/spec.json:qwen35_nimble_lora.Qwen35NimbleLoraAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 60.448690411881714, "calibration": 80.85511159800245, "speed": 81.99549950557196, "cost": 45.690218065856605 }, "scores": { "A": 53.02414313946459, "B": 52.494350100112975, "C": 53.02414313946459 }, "views": { "Balanced 33:33:33": 49.483494063390474, "Emphasis on Accuracy 60:20:20": 49.87615766345389, "Emphasis on Cost 20:20:60": 44.22951871313033, "Emphasis on Speed 20:60:20": 55.65672146393036, "Intelligence only": 50.476978374194644 }, "composite_ci95": { "A": [ 51.87649654035859, 53.88222182087126 ], "B": [ 50.98151113113002, 53.70330555388338 ], "C": [ 46.887046030978055, 53.88222182087126 ] }, "jevbench_score": 53.02414313946459, "intelligence": { "I_open": 62.6138776717486, "I_sealed": 58.28350315201483, "base": 60.448690411881714, "gap": 4.330374519733766, "excess": -0.8562529803454764, "penalty": 1, "per_type_split": { "open|choice": { "cc": 72.91194808577592, "tiers": { "easy": 98.19113656918904, "standard": 82.84377436963868, "judge": 87.55186721991701, "hard": 50.64629847238542 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 42.36587635858848, "tiers": { "easy": 64.70588235294117, "standard": 60.606060606060595, "judge": 68.62745098039215, "hard": 7.964601769911495 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 72.5638085708814, "tiers": { "easy": 88.96008632590154, "standard": 82.11242483530464, "judge": 73.89712850993713, "hard": 62.69044104562293 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 74.7723778131831, "tiers": { "easy": 82.421875, "standard": 79.6241793072221, "judge": 77.47349823321555, "hard": 68.408262454435 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 31.67272727272728, "tiers": { "easy": 43.99999999999999, "standard": 12.00000000000001, "judge": 32.00000000000001, "hard": 38.18181818181819 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 68.40540437013414, "tiers": { "easy": 76.24269933016151, "standard": 78.78101956011632, "judge": 66.09094517211264, "hard": 62.99411743365233 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 80.85511159800245, "parts": { "choice": { "ece_hard": 0.11208446592845642, "ece_score": 77.5831068143087, "mean_tvd": 0.310202368163371, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 73.2814349989858 }, "noul": { "ece": 0.08670530591940094, "brier": 0.12223515496642685, "n": 495, "score": 82.65893881611981, "abstention_rate": 0.23232323232323238 }, "score": { "nrps": 0.07503667025703346, "top_ece": 0.09623205508246563, "n": 330, "interval90_coverage": 0.906060606060606, "interval90_width": 0.1997474747474748, "score": 86.62496097890177 } } }, "speed": { "p50_s_raw": 0.21541375573724508, "p95_s_raw": 0.4687169534619888, "p50_s_adjusted": 0.5808275114744902, "p95_s_adjusted": 1.0874339069239776, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.06461391625615764, "basis": "documented hosted-model estimate" }, "rank": 23, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 23, "B": 23, "C": 17 } }, { "key": "standardone-8b", "display": "Standard One 8B (Standard Thinking)", "addendum": null, "author": "Standard Thinking (myeongho12)", "repo": "https://huggingface.co/StandardThinking/StandardOne-8B", "class": "jev-rebuild", "licence": "Apache-2.0 (jev-adapter server, LoRA and merged weights; base Ministral 3 8B Apache-2.0)", "open": "yes", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/standardone-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 59.59798327565973, "calibration": 83.15842292777755, "speed": 92.47668217708318, "cost": 43.283250331904235 }, "scores": { "A": 47.79069264973117, "B": 47.13021336388438, "C": 47.15241622866449 }, "views": { "Balanced 33:33:33": 44.34504647890239, "Emphasis on Accuracy 60:20:20": 44.47100595116002, "Emphasis on Cost 20:20:60": 38.6660890601248, "Emphasis on Speed 20:60:20": 51.807344644154206, "Intelligence only": 44.661292576950785 }, "composite_ci95": { "A": [ 46.67227934534719, 48.54237759923712 ], "B": [ 45.66560169780469, 48.273280957146895 ], "C": [ 40.81880744242586, 48.54237759923712 ] }, "jevbench_score": 47.79069264973117, "intelligence": { "I_open": 59.95626170147317, "I_sealed": 59.239704849846284, "base": 59.59798327565973, "gap": 0.7165568516268834, "excess": -4.470070648452359, "penalty": 1, "per_type_split": { "open|choice": { "cc": 73.71524106786116, "tiers": { "easy": 100, "standard": 82.84377436963868, "judge": 89.6265560165975, "hard": 50.64629847238542 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 45.04235483412995, "tiers": { "easy": 70.58823529411764, "standard": 54.54545454545454, "judge": 72.54901960784315, "hard": 13.27433628318584 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 61.11118920242842, "tiers": { "easy": 35.9733866269639, "standard": 75.16804575299989, "judge": 63.69908035768006, "hard": 58.426293204570065 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 72.17810329904881, "tiers": { "easy": 88.28125, "standard": 79.6241793072221, "judge": 66.87279151943464, "hard": 68.408262454435 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 42.981818181818184, "tiers": { "easy": 60.00000000000001, "standard": 39.99999999999999, "judge": 36.00000000000001, "hard": 45.45454545454546 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 62.55919306867187, "tiers": { "easy": 67.88458997347536, "standard": 63.500144415233116, "judge": 57.21070882928115, "hard": 64.7687313487334 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 83.15842292777755, "parts": { "choice": { "ece_hard": 0.05968117462976331, "ece_score": 88.06376507404734, "mean_tvd": 0.24762246288483555, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 81.6507593927819 }, "noul": { "ece": 0.0904581989687968, "brier": 0.1212055351648418, "n": 495, "score": 81.90836020624064, "abstention_rate": 0.1898989898989899 }, "score": { "nrps": 0.10385562133359677, "top_ece": 0.0889106974901002, "n": 330, "interval90_coverage": 0.8787878787878788, "interval90_width": 0.28232323232323225, "score": 85.91614918431014 } } }, "speed": { "p50_s_raw": 0.026684600859880447, "p95_s_raw": 0.06400057689752427, "p50_s_adjusted": 0.2033692017197609, "p95_s_adjusted": 0.27800115379504853, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.07772456896551724, "basis": "documented hosted-model estimate" }, "rank": 24, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 24, "B": 26, "C": 22 } }, { "key": "ninfer-qwen3.8-flash-next", "display": "NInfer Qwen3.8-Flash-Next mixed", "addendum": null, "author": "Igor L. / NInfer contributors", "repo": "https://github.com/igorls/ninfer", "class": "native-logit", "licence": "Qwen Community License 1.0 (model); Apache-2.0 (engine)", "open": "yes", "underlying": "Qwen/Qwen3.8-Flash-Next, mixed NVFP4/FP8/INT4 NInfer artifact", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "gpu": "RTXPRO6000", "adapter_id": "images/ninfer-qwen3.8-flash-next/spec.json:ninfer_native.NInferNativeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 67.19957888400617, "calibration": 88.54350643788759, "speed": 88.61299548829486, "cost": 42.53362878447125 }, "scores": { "A": 47.474808502448944, "B": 47.7011582677789, "C": 47.474808502448944 }, "views": { "Balanced 33:33:33": 43.7010429741045, "Emphasis on Accuracy 60:20:20": 45.54715488980916, "Emphasis on Cost 20:20:60": 37.41753230615858, "Emphasis on Speed 20:60:20": 50.08128343228141, "Intelligence only": 48.62856070723032 }, "composite_ci95": { "A": [ 46.69503698220878, 47.881123014444135 ], "B": [ 46.62486661589989, 48.43899187781872 ], "C": [ 46.69503698220878, 47.881123014444135 ] }, "jevbench_score": 47.474808502448944, "intelligence": { "I_open": 70.25732348162926, "I_sealed": 64.14183428638309, "base": 67.19957888400617, "gap": 6.11548919524617, "excess": 0.9288616951669271, "penalty": 1, "per_type_split": { "open|choice": { "cc": 84.92619742669542, "tiers": { "easy": 96.38227313837805, "standard": 92.20171562256304, "judge": 95.850622406639, "hard": 70.23110066588326 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 51.40125881406464, "tiers": { "easy": 82.35294117647058, "standard": 72.72727272727273, "judge": 58.823529411764696, "hard": 27.43362831858407 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 74.44451420412774, "tiers": { "easy": 59.42108431739883, "standard": 88.84655227920712, "judge": 71.8468031376396, "hard": 72.9476359381364 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 79.10854505246607, "tiers": { "easy": 94.140625, "standard": 85.05773149196287, "judge": 68.19787985865725, "hard": 80.55893074119078 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 43.01818181818182, "tiers": { "easy": 52, "standard": 36.00000000000001, "judge": 56.00000000000001, "hard": 34.54545454545455 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 70.29877598850139, "tiers": { "easy": 76.762949788409, "standard": 74.31336416172283, "judge": 71.11674673491447, "hard": 66.06196039210394 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 88.54350643788759, "parts": { "choice": { "ece_hard": 0.05943674097877588, "ece_score": 88.11265180424482, "mean_tvd": 0.2663371154061145, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 80.73947013181669 }, "noul": { "ece": 0.032862434581237034, "brier": 0.06073012931047522, "n": 495, "score": 93.42751308375259, "abstention_rate": 0.2404040404040404 }, "score": { "nrps": 0.062258834532185996, "top_ece": 0.054235221752971935, "n": 330, "interval90_coverage": 0.9818181818181818, "interval90_width": 0.34166666666666656, "score": 91.4635360980935 } } }, "speed": { "p50_s_raw": 0.07975747575983405, "p95_s_raw": 0.14732526696752724, "p50_s_adjusted": 0.3095149515196681, "p95_s_adjusted": 0.4446505339350545, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.08232764778325123, "basis": "documented hosted-model estimate" }, "rank": 25, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 25, "B": 24, "C": 21 } }, { "key": "swanone", "display": "swanOne (blockbrain, Qwen3.8-Flash-Next NVFP4)", "addendum": null, "author": "blockbrain", "repo": "https://github.com/blockbrain-ai/swanone-recipe", "class": "system-one-open", "licence": "patches Apache-2.0, shim MIT; weights under the Qwen licence (LICENSE-NOTICE.md)", "open": "yes", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "gpu": "RTXPRO6000", "adapter_id": "images/swanone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 71.22352028588858, "calibration": 87.05674154866381, "speed": 84.74079430681516, "cost": 42.15074720483203 }, "scores": { "A": 46.56179491157818, "B": 47.31995862079193, "C": 46.56179491157818 }, "views": { "Balanced 33:33:33": 43.014358880893774, "Emphasis on Accuracy 60:20:20": 45.76374642916793, "Emphasis on Cost 20:20:60": 36.627361387652215, "Emphasis on Speed 20:60:20": 48.56537743728146, "Intelligence only": 50.61671801334552 }, "composite_ci95": { "A": [ 45.81356430623499, 46.94332947688013 ], "B": [ 46.35320829142545, 48.000773651251464 ], "C": [ 45.81356430623499, 46.94332947688013 ] }, "jevbench_score": 46.56179491157818, "intelligence": { "I_open": 70.49798699750956, "I_sealed": 71.9490535742676, "base": 71.22352028588858, "gap": -1.4510665767580377, "excess": -6.63769407683728, "penalty": 1, "per_type_split": { "open|choice": { "cc": 88.10818162872131, "tiers": { "easy": 100, "standard": 92.20171562256304, "judge": 95.850622406639, "hard": 77.2816294555425 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 45.54950862082565, "tiers": { "easy": 76.47058823529412, "standard": 57.57575757575757, "judge": 49.01960784313726, "hard": 29.20353982300885 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 77.83627074298171, "tiers": { "easy": 65.05719927474456, "standard": 89.87197767107257, "judge": 74.955420484181, "hard": 77.1738228400961 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 82.64893490554081, "tiers": { "easy": 94.140625, "standard": 86.41611953814805, "judge": 77.47349823321555, "hard": 81.77399756986635 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 59.89090909090909, "tiers": { "easy": 92, "standard": 52, "judge": 63.99999999999999, "hard": 52.727272727272734 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 73.30731672635291, "tiers": { "easy": 84.42086253601506, "standard": 74.97120272471354, "judge": 71.05934999875308, "hard": 71.38296232045693 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 87.05674154866381, "parts": { "choice": { "ece_hard": 0.028102447139622583, "ece_score": 94.37951057207549, "mean_tvd": 0.30445967398626356, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 81.96677158672458 }, "noul": { "ece": 0.06737763907474524, "brier": 0.08484464490362868, "n": 495, "score": 86.52447218505095, "abstention_rate": 0.19999999999999996 }, "score": { "nrps": 0.05807570109811426, "top_ece": 0.0441723407087838, "n": 330, "interval90_coverage": 0.9787878787878788, "interval90_width": 0.25883838383838387, "score": 92.67898087421591 } } }, "speed": { "p50_s_raw": 0.2087587924906984, "p95_s_raw": 0.22074080496095116, "p50_s_adjusted": 0.5675175849813968, "p95_s_adjusted": 0.5914816099219024, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.08478293103448274, "basis": "price floor: base-model reference price applied" }, "rank": 26, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 26, "B": 25, "C": 24 } }, { "key": "raw-qwen3-8b", "display": "Raw Qwen3 8B direct logits", "addendum": null, "author": "Alibaba Qwen / neutral reproduction", "repo": "https://huggingface.co/Qwen/Qwen3-8B", "class": "raw-logit-control", "licence": "Apache-2.0", "open": "yes", "underlying": "Qwen3-8B BF16", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/raw-qwen3-8b/spec.json:raw_adapter.RawControlAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 51.1392760971673, "calibration": 49.19480442556255, "speed": 86.84465631896776, "cost": 45.53515592313027 }, "scores": { "A": 45.223917256408555, "B": 44.632516465694714, "C": 32.85298453350857 }, "views": { "Balanced 33:33:33": 46.9192160309914, "Emphasis on Accuracy 60:20:20": 45.0069155506764, "Emphasis on Cost 20:20:60": 42.77253755222896, "Emphasis on Speed 20:60:20": 54.52148806396662, "Intelligence only": 42.41390150199564 }, "composite_ci95": { "A": [ 36.887258965736855, 46.74342614714434 ], "B": [ 36.02835612521688, 46.43665511058838 ], "C": [ 25.616152059539484, 38.86132200528848 ] }, "jevbench_score": 45.223917256408555, "intelligence": { "I_open": 56.77526351111453, "I_sealed": 45.503288683220056, "base": 51.1392760971673, "gap": 11.271974827894475, "excess": 6.085347327815232, "penalty": 1, "per_type_split": { "open|choice": { "cc": 64.85654971862496, "tiers": { "easy": 92.7645462767561, "standard": 71.92617624122693, "judge": 88.58921161825727, "hard": 36.54524089306697 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 48.067609988484534, "tiers": { "easy": 82.35294117647058, "standard": 48.484848484848484, "judge": 80.3921568627451, "hard": 15.04424778761062 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 57.40163082623409, "tiers": { "easy": 47.131947467639016, "standard": 77.03790981221292, "judge": 53.06730608793813, "hard": 53.40165572661542 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 56.54764008742353, "tiers": { "easy": 73.63281249999999, "standard": 64.68191079918498, "judge": 52.296819787985875, "hard": 51.39732685297693 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 38.4, "tiers": { "easy": 60.00000000000001, "standard": 43.99999999999999, "judge": 52, "hard": 19.999999999999996 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 41.56222596223666, "tiers": { "easy": 56.8339149235689, "standard": 42.20719309928661, "judge": 29.4866104671199, "hard": 46.47853177471619 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 49.19480442556255, "parts": { "choice": { "ece_hard": 0.3824646871164692, "ece_score": 23.50706257670616, "mean_tvd": 0.4363539849414789, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 39.935832041279134 }, "noul": { "ece": 0.2219333154858661, "brier": 0.2249852735368055, "n": 495, "score": 55.61333690282678, "abstention_rate": 0.04040404040404044 }, "score": { "nrps": 0.17281393617880628, "top_ece": 0.3932405885847793, "n": 330, "interval90_coverage": 0.6393939393939394, "interval90_width": 0.12853535353535353, "score": 52.03524433258176 } } }, "speed": { "p50_s_raw": 0.09141041804105043, "p95_s_raw": 0.23566609979607117, "p50_s_adjusted": 0.3328208360821009, "p95_s_adjusted": 0.6213321995921424, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.0653875104679803, "basis": "documented hosted-model estimate" }, "rank": 27, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 27, "B": 27, "C": 30 } }, { "key": "decider-2b", "display": "decider-2b (Mapika)", "addendum": null, "author": "Mapika", "repo": "https://huggingface.co/Mapika/decider-2b", "class": "jev-rebuild", "licence": "Apache-2.0", "open": "yes", "underlying": "Qwen3.5-2B-Base with a trained decision readout, 1.9B", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/decider-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 42.33871878833063, "calibration": 71.52659723196619, "speed": 94.40334833224873, "cost": 64.91766177740826 }, "scores": { "A": 45.09827651852322, "B": 41.10644266575027, "C": 31.318247582307787 }, "views": { "Balanced 33:33:33": 43.35456011050343, "Emphasis on Accuracy 60:20:20": 37.015822693274224, "Emphasis on Cost 20:20:60": 44.5777637422437, "Emphasis on Speed 20:60:20": 50.6362696545408, "Intelligence only": 30.357997888722927 }, "composite_ci95": { "A": [ 33.81613238500261, 54.263223032815134 ], "B": [ 30.279563783540407, 50.04341291318229 ], "C": [ 23.48342526736293, 37.68279377278829 ] }, "jevbench_score": 45.09827651852322, "intelligence": { "I_open": 48.790563557043, "I_sealed": 35.88687401961826, "base": 42.33871878833063, "gap": 12.90368953742474, "excess": 7.717062037345497, "penalty": 1, "per_type_split": { "open|choice": { "cc": 52.01934595406228, "tiers": { "easy": 89.14681941513415, "standard": 68.80686249025213, "judge": 61.6182572614108, "hard": 27.144535840188002 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 35.238117773255716, "tiers": { "easy": 82.35294117647058, "standard": 30.303030303030297, "judge": 68.62745098039215, "hard": 0.8849557522123908 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 59.114226943811005, "tiers": { "easy": 26.393778801843315, "standard": 74.94920494699646, "judge": 59.75314547837483, "hard": 58.8976610767873 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 53.48470768046778, "tiers": { "easy": 79.4921875, "standard": 59.2483586144442, "judge": 46.99646643109541, "hard": 48.96719319562577 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 5.7818181818181795, "tiers": { "easy": 36.00000000000001, "standard": 0, "judge": 0, "hard": 5.454545454545445 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 48.39409619656884, "tiers": { "easy": 73.47898550724638, "standard": 60.693366918555846, "judge": 39.21284751474303, "hard": 42.8591750192753 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 71.52659723196619, "parts": { "choice": { "ece_hard": 0.2555180505415162, "ece_score": 48.89638989169676, "mean_tvd": 0.38142566660000665, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 55.376911615848044 }, "noul": { "ece": 0.14005999999999988, "brier": 0.18936641214141414, "n": 495, "score": 71.98800000000003, "abstention_rate": 0.22424242424242424 }, "score": { "nrps": 0.11813754991414144, "top_ece": 0.06878242424242426, "n": 330, "interval90_coverage": 0.9030303030303031, "interval90_width": 0.4565656565656566, "score": 87.2148800800505 } } }, "speed": { "p50_s_raw": 0.01366500393487513, "p95_s_raw": 0.02729465400916524, "p50_s_adjusted": 0.17733000786975026, "p95_s_adjusted": 0.20458930801833047, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.014771046798029557, "basis": "documented hosted-model estimate" }, "rank": 28, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 28, "B": 32, "C": 32 } }, { "key": "system-one-sg", "display": "system-one (Qwen3-8B, Sean Goedecke)", "addendum": null, "author": "Sean Goedecke", "repo": "https://github.com/sgoedecke/system-one", "class": "jev-rebuild", "licence": "no licence file in the repository as of 19 Sep; Qwen3 weights Apache-2.0", "open": "yes", "underlying": "Qwen/Qwen3-8B (frozen, BF16), the model of the author's demos", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/system-one-sg/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 50.47245577952833, "calibration": 49.37850478055448, "speed": 90.87700150484226, "cost": 44.96855803039237 }, "scores": { "A": 44.13645499190376, "B": 43.43200453121163, "C": 31.232289361046593 }, "views": { "Balanced 33:33:33": 45.73803572130066, "Emphasis on Accuracy 60:20:20": 43.637702179733125, "Emphasis on Cost 20:20:60": 41.46769673439668, "Emphasis on Speed 20:60:20": 53.879881804933696, "Intelligence only": 40.82557881705132 }, "composite_ci95": { "A": [ 36.78500148194342, 45.662322394284914 ], "B": [ 35.915897669029526, 45.22132798708415 ], "C": [ 25.545139918016265, 36.98532780405549 ] }, "jevbench_score": 44.13645499190376, "intelligence": { "I_open": 53.53755450895149, "I_sealed": 47.40735705010517, "base": 50.47245577952833, "gap": 6.130197458846318, "excess": 0.9435699587670756, "penalty": 1, "per_type_split": { "open|choice": { "cc": 63.17740474707069, "tiers": { "easy": 94.57340970756708, "standard": 73.48583311671433, "judge": 78.21576763485477, "hard": 38.895417156286705 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 49.88579180666636, "tiers": { "easy": 100, "standard": 57.57575757575757, "judge": 74.50980392156863, "hard": 15.04424778761062 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 47.54946697311742, "tiers": { "easy": 22.44824791409641, "standard": 72.50561783397852, "judge": 42.37602381862431, "hard": 45.22677867331196 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 56.6787225986529, "tiers": { "easy": 79.4921875, "standard": 61.96513470681459, "judge": 50.97173144876326, "hard": 52.61239368165249 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 40.03636363636364, "tiers": { "easy": 43.99999999999999, "standard": 19.999999999999996, "judge": 39.99999999999999, "hard": 49.09090909090909 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 45.50698491529897, "tiers": { "easy": 67.19757471079686, "standard": 60.6717253542459, "judge": 36.61887846690121, "hard": 39.16804708324935 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 49.37850478055448, "parts": { "choice": { "ece_hard": 0.3414165000468772, "ece_score": 31.716699990624562, "mean_tvd": 0.43927763866315545, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 43.894468062154516 }, "noul": { "ece": 0.22457895725345312, "brier": 0.22985988066791246, "n": 495, "score": 55.08420854930938, "abstention_rate": 0.04040404040404044 }, "score": { "nrps": 0.17575051481605997, "top_ece": 0.42055636528997464, "n": 330, "interval90_coverage": 0.5787878787878787, "interval90_width": 0.09949494949494948, "score": 49.15683773019954 } } }, "speed": { "p50_s_raw": 0.0384387425146997, "p95_s_raw": 0.10508537914138283, "p50_s_adjusted": 0.2268774850293994, "p95_s_adjusted": 0.3601707582827657, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.06829381896551724, "basis": "documented hosted-model estimate" }, "rank": 29, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 29, "B": 28, "C": 33 } }, { "key": "system-one-open", "display": "system-one-open (Gemma 4 E2B LoRA on an L4)", "addendum": null, "author": "mithalouni", "repo": "https://github.com/mithalouni/system-one-open", "class": "jev-rebuild", "licence": "MIT (repository LICENSE; Gemma weights keep Google’s terms)", "open": "yes", "underlying": "google/gemma-4-E2B-it + LoRA", "endpoint_kind": "demo", "endpoint_condition": "the author's public demo endpoint", "gpu": null, "adapter_id": "harness/specs/system-one-open.json:jevbench.adapters.systemone_list.SystemOneListAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 41.64356524223511, "calibration": 71.62757593487629, "speed": 78.27072904753155, "cost": 68.33006522881325 }, "scores": { "A": 42.43603455732882, "B": 38.796663536397624, "C": 29.469468442589456 }, "views": { "Balanced 33:33:33": 40.46769109259382, "Emphasis on Accuracy 60:20:20": 34.87521081607888, "Emphasis on Cost 20:20:60": 42.98178498751218, "Emphasis on Speed 20:60:20": 45.05746080516129, "Intelligence only": 28.887083896478522 }, "composite_ci95": { "A": [ 33.838584413879616, 51.9229885095774 ], "B": [ 30.532950806264438, 48.14184520284253 ], "C": [ 23.499016954083064, 36.05763090942875 ] }, "jevbench_score": 42.43603455732882, "intelligence": { "I_open": 44.43803383833533, "I_sealed": 38.849096646134896, "base": 41.64356524223511, "gap": 5.588937192200433, "excess": 0.4023096921211904, "penalty": 1, "per_type_split": { "open|choice": { "cc": 59.92067737087481, "tiers": { "easy": 96.38227313837805, "standard": 70.36651936573953, "judge": 75.10373443983403, "hard": 34.19506462984723 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 15.40091177259319, "tiers": { "easy": 76.47058823529412, "standard": 39.393939393939405, "judge": 7.843137254901955, "hard": -6.194690265486724 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 57.992512371538, "tiers": { "easy": 41.64728311554955, "standard": 71.89847009741732, "judge": 63.21182539395285, "hard": 51.21135605578431 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 52.042685008120415, "tiers": { "easy": 76.56249999999999, "standard": 59.2483586144442, "judge": 49.64664310954064, "hard": 44.10692588092346 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 14.981818181818177, "tiers": { "easy": 36.00000000000001, "standard": 15.999999999999993, "judge": 19.999999999999996, "hard": 5.454545454545445 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 49.5227867484661, "tiers": { "easy": 61.75933432475533, "standard": 61.00129485100315, "judge": 40.54257912619954, "hard": 47.45955151982518 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 71.62757593487629, "parts": { "choice": { "ece_hard": 0.19489049120715382, "ece_score": 61.02190175856923, "mean_tvd": 0.3319398583447739, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 63.91395796204592 }, "noul": { "ece": 0.12032569308344548, "brier": 0.17386092858386082, "n": 495, "score": 75.9348613833109, "abstention_rate": 0.2848484848484848 }, "score": { "nrps": 0.12647097490053033, "top_ece": 0.18642542795701458, "n": 330, "interval90_coverage": 0.8151515151515152, "interval90_width": 0.3361111111111109, "score": 75.03390845927203 } } }, "speed": { "p50_s_raw": 0.5750190168619156, "p95_s_raw": 0.647418188303709, "p50_s_adjusted": 1.1500380337238312, "p95_s_adjusted": 1.294836376607418, "n": 450, "adjustment": "x2 demo-endpoint adjustment (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.011367487684729064, "basis": "ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies" }, "rank": 30, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 30, "B": 33, "C": 35 } }, { "key": "kushal-gemma4-31b-it-autoloops", "display": "Autoloops – Gemma 4 31B IT", "addendum": null, "author": "Autoloops", "repo": null, "class": "jev", "licence": "Gemma terms of use (Google) for the base weights; hosted Autoloops service", "open": "no", "underlying": null, "endpoint_kind": "api", "endpoint_condition": "the operator's hosted API", "gpu": null, "adapter_id": "harness/specs/kushal-gemma4-31b-it-autoloops.json:extra.autoloops_systemone.AutoloopsSystemOneAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 76.68463397202072, "calibration": 85.84316449763286, "speed": 83.9240668357571, "cost": 39.586359989532696 }, "scores": { "A": 40.52814707638268, "B": 41.84081656494916, "C": 40.52814707638268 }, "views": { "Balanced 33:33:33": 37.44726037707129, "Emphasis on Accuracy 60:20:20": 41.07786216973585, "Emphasis on Cost 20:20:60": 31.111486702915485, "Emphasis on Speed 20:60:20": 42.32591427429326, "Intelligence only": 48.06837932931786 }, "composite_ci95": { "A": [ 40.08903740319995, 40.76240789696144 ], "B": [ 41.21534979263599, 42.28834759900149 ], "C": [ 40.08903740319995, 40.76240789696144 ] }, "jevbench_score": 40.52814707638268, "intelligence": { "I_open": 76.87888067003877, "I_sealed": 76.49038727400266, "base": 76.68463397202072, "gap": 0.388493396036111, "excess": -4.798134104043132, "penalty": 1, "per_type_split": { "open|choice": { "cc": 86.35693213456801, "tiers": { "easy": 98.19113656918904, "standard": 93.76137249805042, "judge": 96.88796680497926, "hard": 71.7978848413631 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 68.73077469121196, "tiers": { "easy": 76.47058823529412, "standard": 72.72727272727273, "judge": 78.43137254901961, "hard": 57.52212389380531 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 75.54893518433636, "tiers": { "easy": 79.44374053586364, "standard": 84.43635298314938, "judge": 75.86669966490899, "hard": 69.89320158661853 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 87.5068098550186, "tiers": { "easy": 100, "standard": 83.69934344577767, "judge": 85.42402826855124, "hard": 87.84933171324423 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 67.45454545454545, "tiers": { "easy": 100, "standard": 63.99999999999999, "judge": 63.99999999999999, "hard": 63.63636363636365 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 74.50980651244393, "tiers": { "easy": 77.31828535179173, "standard": 74.77474662968355, "judge": 74.55125067882011, "hard": 73.64413361920504 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 85.84316449763286, "parts": { "choice": { "ece_hard": 0.11069466505831069, "ece_score": 77.86106698833785, "mean_tvd": 0.24261659079045955, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 76.79970395464595 }, "noul": { "ece": 0.024641134940521763, "brier": 0.04314422942385484, "n": 495, "score": 95.07177301189564, "abstention_rate": 0.13535353535353534 }, "score": { "nrps": 0.05012237564914895, "top_ece": 0.11835864691185555, "n": 330, "interval90_coverage": 0.9696969696969697, "interval90_width": 0.33282828282828275, "score": 85.658016526357 } } }, "speed": { "p50_s_raw": 0.6074120700359344, "p95_s_raw": 0.6669755268841981, "p50_s_adjusted": 0.6074120700359344, "p95_s_adjusted": 0.6669755268841981, "n": 450, "adjustment": "none (hosted API, measured as is)" }, "cost": { "kind": "tariff", "usd_per_1000": 0.10322573891625615, "basis": "operator standard launch list price (interpretation I-1)" }, "rank": 31, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 31, "B": 30, "C": 26 } }, { "key": "gpt-6-luna-low", "display": "GPT-6 Luna (low reasoning effort)", "addendum": null, "author": "OpenAI", "repo": null, "class": "llm-baseline", "licence": "proprietary API", "open": "unknown", "underlying": null, "endpoint_kind": "api", "endpoint_condition": "the operator's hosted API", "gpu": null, "adapter_id": "harness/specs/gpt-6-luna-low.json:jevbench.adapters.openai_compat.OpenAICompatAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 95.29107650041868, "calibration": 94.92401460047215, "speed": 73.19737721417388, "cost": 39.05771622064802 }, "scores": { "A": 40.481958334046816, "B": 43.10073731729786, "C": 40.481958334046816 }, "views": { "Balanced 33:33:33": 36.78945782049289, "Emphasis on Accuracy 60:20:20": 43.125459438148994, "Emphasis on Cost 20:20:60": 30.21841757014399, "Emphasis on Speed 20:60:20": 39.58116868050534, "Intelligence only": 58.146812947733785 }, "composite_ci95": { "A": [ 40.268976996595626, 40.66385155794427 ], "B": [ 42.806610244055186, 43.35101400348243 ], "C": [ 40.268976996595626, 40.66385155794427 ] }, "jevbench_score": 40.481958334046816, "intelligence": { "I_open": 94.72456284907499, "I_sealed": 95.8575901517624, "base": 95.29107650041868, "gap": -1.1330273026874096, "excess": -6.319654802766652, "penalty": 1, "per_type_split": { "open|choice": { "cc": 99.0613549547106, "tiers": { "easy": 100, "standard": 98.44034312451261, "judge": 100, "hard": 98.43321582452018 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 85.24505860268484, "tiers": { "easy": 100, "standard": 96.96969696969697, "judge": 88.23529411764706, "hard": 73.45132743362832 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 99.86727498982954, "tiers": { "easy": 99.87903225806451, "standard": 99.90724381625442, "judge": 99.96068152031454, "hard": 99.77429636169461 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 97.87271569458548, "tiers": { "easy": 100, "standard": 98.64161195381482, "judge": 98.67491166077738, "hard": 96.35479951397326 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 90.18181818181819, "tiers": { "easy": 100, "standard": 92, "judge": 92, "hard": 85.45454545454545 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 99.51823657888352, "tiers": { "easy": 99.90942028985508, "standard": 99.93073047858942, "judge": 98.60362257792755, "hard": 99.90015420200463 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 94.92401460047215, "parts": { "choice": { "ece_hard": 0.04035662449133609, "ece_score": 91.92867510173278, "mean_tvd": 0.023317194080665005, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 94.79847784683314 }, "noul": { "ece": 0.04876064279151522, "brier": 0.04657367170495567, "n": 495, "score": 90.24787144169696, "abstention_rate": 0.014141414141414121 }, "score": { "nrps": 0.0008295440859084673, "top_ece": 0.0023282828281823414, "n": 330, "interval90_coverage": 0.996969696969697, "interval90_width": 0.00202020202020202, "score": 99.72569451288635 } } }, "speed": { "p50_s_raw": 1.5798842646181583, "p95_s_raw": 3.0313564356416416, "p50_s_adjusted": 1.5798842646181583, "p95_s_adjusted": 3.0313564356416416, "n": 450, "adjustment": "none (hosted API, measured as is)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.10750024630541874, "basis": "operator standard launch list price (interpretation I-1); no exact base-model floor applies" }, "rank": 32, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 32, "B": 29, "C": 27 } }, { "key": "gpt-6-luna", "display": "GPT-6 Luna (default medium reasoning effort)", "addendum": null, "author": "OpenAI", "repo": null, "class": "llm-baseline", "licence": "proprietary API", "open": "unknown", "underlying": null, "endpoint_kind": "api", "endpoint_condition": "the operator's hosted API", "gpu": null, "adapter_id": "harness/specs/gpt-6-luna.json:jevbench.adapters.openai_compat.OpenAICompatAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 96.23569044084438, "calibration": 95.56003846585782, "speed": 73.18530319155485, "cost": 38.316528790869064 }, "scores": { "A": 38.751615238726444, "B": 41.35111355834929, "C": 38.751615238726444 }, "views": { "Balanced 33:33:33": 35.127911556506724, "Emphasis on Accuracy 60:20:20": 41.393941998009545, "Emphasis on Cost 20:20:60": 28.688855301298666, "Emphasis on Speed 20:60:20": 37.897006955989184, "Intelligence only": 56.51561710716807 }, "composite_ci95": { "A": [ 38.56772553955819, 38.92820298954891 ], "B": [ 41.101933604560855, 41.59723975450365 ], "C": [ 38.56772553955819, 38.92820298954891 ] }, "jevbench_score": 38.751615238726444, "intelligence": { "I_open": 96.31846029492732, "I_sealed": 96.15292058676144, "base": 96.23569044084438, "gap": 0.16553970816588048, "excess": -5.021087791913362, "penalty": 1, "per_type_split": { "open|choice": { "cc": 99.37543984540196, "tiers": { "easy": 100, "standard": 100, "judge": 98.96265560165975, "hard": 99.21660791226009 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 90.68714211348257, "tiers": { "easy": 100, "standard": 100, "judge": 90.19607843137254, "hard": 84.070796460177 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 98.89279892589744, "tiers": { "easy": 100, "standard": 99.886925795053, "judge": 99.89777195281782, "hard": 97.3652054526037 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 99.51397326852977, "tiers": { "easy": 100, "standard": 100, "judge": 100.00000000000001, "hard": 98.78493317132441 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 89.92727272727274, "tiers": { "easy": 92, "standard": 96, "judge": 96, "hard": 81.81818181818181 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 99.01751576448186, "tiers": { "easy": 99.9375, "standard": 99.8942065491184, "judge": 98.51263689974726, "hard": 98.727833461835 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 95.56003846585782, "parts": { "choice": { "ece_hard": 0.03585405941841228, "ece_score": 92.82918811631754, "mean_tvd": 0.031053666112664996, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 94.86191075252552 }, "noul": { "ece": 0.03508381739474774, "brier": 0.03327200606740464, "n": 495, "score": 92.98323652105044, "abstention_rate": 0.012121212121212088 }, "score": { "nrps": 0.0028561930758074566, "top_ece": 0.010222222222121494, "n": 330, "interval90_coverage": 0.990909090909091, "interval90_width": 0.0027777777777777775, "score": 98.83496812399747 } } }, "speed": { "p50_s_raw": 1.5554233863949776, "p95_s_raw": 3.0876001715660073, "p50_s_adjusted": 1.5554233863949776, "p95_s_adjusted": 3.0876001715660073, "n": 450, "adjustment": "none (hosted API, measured as is)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.1137930418719212, "basis": "operator standard launch list price (interpretation I-1); no exact base-model floor applies" }, "rank": 33, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 33, "B": 31, "C": 29 } }, { "key": "jevone", "display": "JevOne", "addendum": null, "author": "Juspay", "repo": "https://huggingface.co/juspay/jev-one", "class": "jev-rebuild", "licence": "Component-specific JevOne/Qwen/SGLang terms recorded in RESULT.md", "open": "yes", "underlying": "Qwen3.6-35B-A3B BF16 with JevOne bidirectional option-logit mapping", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "gpu": "RTXPRO6000", "adapter_id": "images/jevone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 54.13082284597005, "calibration": 84.94489158761552, "speed": 90.43197361602677, "cost": 39.84212731591276 }, "scores": { "A": 38.24864204152178, "B": 37.40461666043465, "C": 31.131695776173597 }, "views": { "Balanced 33:33:33": 34.86809688509843, "Emphasis on Accuracy 60:20:20": 34.66746178277511, "Emphasis on Cost 20:20:60": 30.28542415522889, "Emphasis on Speed 20:60:20": 41.366998522047716, "Intelligence only": 34.37080137397821 }, "composite_ci95": { "A": [ 37.37790294644138, 38.7857176460762 ], "B": [ 36.19750772097442, 38.28517425236358 ], "C": [ 26.636189982602396, 35.41851297457285 ] }, "jevbench_score": 38.24864204152178, "intelligence": { "I_open": 53.90312873058646, "I_sealed": 54.358516961353644, "base": 54.13082284597005, "gap": -0.45538823076718415, "excess": -5.642015730846427, "penalty": 1, "per_type_split": { "open|choice": { "cc": 82.42577264151704, "tiers": { "easy": 96.38227313837805, "standard": 87.52274499610087, "judge": 94.81327800829875, "hard": 67.09753231492361 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 9.375956335873049, "tiers": { "easy": 64.70588235294117, "standard": 15.15151515151516, "judge": 7.843137254901955, "hard": -6.194690265486724 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 69.9076572143693, "tiers": { "easy": 40.016129032258064, "standard": 83.07703180212015, "judge": 72.52889908256881, "hard": 68.82992056487203 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 77.14053600029337, "tiers": { "easy": 91.2109375, "standard": 76.9074032148517, "judge": 76.14840989399293, "hard": 74.48359659781289 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 17.127272727272725, "tiers": { "easy": 68, "standard": 0, "judge": 32.00000000000001, "hard": 1.8181818181818077 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 68.80774215649485, "tiers": { "easy": 81.35172101449275, "standard": 72.98148614609572, "judge": 64.51078348778432, "hard": 66.80759444872784 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 84.94489158761552, "parts": { "choice": { "ece_hard": 0.032576534296028775, "ece_score": 93.48469314079423, "mean_tvd": 0.30537816655000166, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 81.47343824289703 }, "noul": { "ece": 0.07786626262626258, "brier": 0.09677940056565656, "n": 495, "score": 84.42674747474749, "abstention_rate": 0.42020202020202024 }, "score": { "nrps": 0.07153506758080812, "top_ece": 0.07488757575757568, "n": 330, "interval90_coverage": 0.9484848484848485, "interval90_width": 0.33308080808080787, "score": 88.93448904520203 } } }, "speed": { "p50_s_raw": 0.05845375219359994, "p95_s_raw": 0.09459453681483865, "p50_s_adjusted": 0.2669075043871999, "p95_s_adjusted": 0.3391890736296773, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.10121908866995075, "basis": "documented hosted-model estimate" }, "rank": 34, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 34, "B": 34, "C": 34 } }, { "key": "kev-4b", "display": "kev 4B (research preview)", "addendum": null, "author": "Jared Palmer", "repo": "https://github.com/jaredpalmer/kev", "class": "jev-rebuild", "licence": "Apache-2.0", "open": true, "underlying": "Qwen3-4B-Base + LoRA + learned pointer head; jaredpalmer/kev-4b", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/kev-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 39.86088085724986, "calibration": 67.64154808465537, "speed": 85.47775575216679, "cost": 65.78036497095258 }, "scores": { "A": 38.073131188431404, "B": 34.59396156532314, "C": 26.439674436410698 }, "views": { "Balanced 33:33:33": 36.67493457142576, "Emphasis on Accuracy 60:20:20": 31.10505213714635, "Emphasis on Cost 20:20:60": 38.56878935020766, "Emphasis on Speed 20:60:20": 42.15332921905585, "Intelligence only": 25.33381916742963 }, "composite_ci95": { "A": [ 27.476430952821357, 46.767301073415496 ], "B": [ 24.5320197349573, 43.08613240484619 ], "C": [ 19.080854828348166, 32.477292412094094 ] }, "jevbench_score": 38.073131188431404, "intelligence": { "I_open": 47.060039168272525, "I_sealed": 33.20122670594433, "base": 40.13063293710843, "gap": 13.858812462328196, "excess": 8.672184962248952, "penalty": 0.9932781503775104, "per_type_split": { "open|choice": { "cc": 53.88729392667013, "tiers": { "easy": 89.14681941513415, "standard": 70.36651936573953, "judge": 66.80497925311204, "hard": 27.144535840188002 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 29.05746691275062, "tiers": { "easy": 64.70588235294117, "standard": 42.42424242424243, "judge": 64.70588235294117, "hard": -13.27433628318584 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 58.23535666539682, "tiers": { "easy": 33.1008573207551, "standard": 69.32495502200398, "judge": 59.779114215484476, "hard": 57.81636416068792 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 46.761713448034214, "tiers": { "easy": 85.3515625, "standard": 55.17319447588863, "judge": 35.07067137809188, "hard": 41.67679222357231 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 4.2909090909090875, "tiers": { "easy": 12.00000000000001, "standard": -4.0000000000000036, "judge": -4.0000000000000036, "hard": 12.72727272727272 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 48.55105757888969, "tiers": { "easy": 69.97776498664359, "standard": 55.93544744902701, "judge": 42.91280939382903, "hard": 43.73087193067805 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.003754693366708417 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0.00303030303030305 } }, "calibration": { "score": 67.64154808465537, "parts": { "choice": { "ece_hard": 0.2777823341900978, "ece_score": 44.44353316198044, "mean_tvd": 0.3226589404273894, "n_tvd": 50, "n": 796, "n_ece_hard": 277, "score": 56.08881955962075 }, "noul": { "ece": 0.1829696969696968, "brier": 0.22143454545454574, "n": 495, "score": 63.40606060606064, "abstention_rate": 0.20404040404040402 }, "score": { "nrps": 0.11772776999236903, "top_ece": 0.10683847412096825, "n": 329, "interval90_coverage": 0.9027355623100304, "interval90_width": 0.4435157041540021, "score": 83.42976408828473 } } }, "speed": { "p50_s_raw": 0.1715529104694724, "p95_s_raw": 0.21224620514549314, "p50_s_adjusted": 0.49310582093894484, "p95_s_adjusted": 0.5744924102909863, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.013824661330049262, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 35, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 35, "B": 35, "C": 38 } }, { "key": "kev-8b", "display": "kev 8B (research preview)", "addendum": null, "author": "Jared Palmer", "repo": "https://github.com/jaredpalmer/kev", "class": "jev-rebuild", "licence": "Apache-2.0", "open": true, "underlying": "Qwen3-8B-Base + LoRA + learned pointer head; jaredpalmer/kev-8b", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/kev-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 48.295292360614766, "calibration": 71.01729114829811, "speed": 84.14101453293591, "cost": 40.42054386073457 }, "scores": { "A": 34.15264116882949, "B": 33.094884395798175, "C": 23.717111922798257 }, "views": { "Balanced 33:33:33": 31.905696022137963, "Emphasis on Accuracy 60:20:20": 30.874469599373526, "Emphasis on Cost 20:20:60": 28.542378501445935, "Emphasis on Speed 20:60:20": 37.590798427003946, "Intelligence only": 29.446839307001515 }, "composite_ci95": { "A": [ 26.78663166842091, 37.48283011878216 ], "B": [ 25.56048047867171, 36.685764855628186 ], "C": [ 18.601827547514524, 27.88353014576553 ] }, "jevbench_score": 34.15264116882949, "intelligence": { "I_open": 54.3060710172369, "I_sealed": 42.28451370399262, "base": 48.295292360614766, "gap": 12.02155731324428, "excess": 6.834929813165037, "penalty": 1, "per_type_split": { "open|choice": { "cc": 60.672703581896855, "tiers": { "easy": 94.57340970756708, "standard": 73.48583311671433, "judge": 77.17842323651452, "hard": 33.41167254210731 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 38.20216743173536, "tiers": { "easy": 76.47058823529412, "standard": 54.54545454545454, "judge": 66.66666666666667, "hard": -0.8849557522123908 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 64.04334203807852, "tiers": { "easy": 49.42575709183821, "standard": 73.94927434439558, "judge": 65.26071217868838, "hard": 61.831744516022646 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 58.716195357778545, "tiers": { "easy": 76.56249999999999, "standard": 68.75707493774055, "judge": 50.97173144876326, "hard": 55.04252733900366 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 14.000000000000004, "tiers": { "easy": 36.00000000000001, "standard": 0, "judge": 8.000000000000007, "hard": 19.999999999999996 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 54.13734575419933, "tiers": { "easy": 68.61186209200632, "standard": 57.078521488434596, "judge": 49.05877571750435, "hard": 52.85705633015117 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.0012515644555695093 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0.00303030303030305 } }, "calibration": { "score": 71.01729114829811, "parts": { "choice": { "ece_hard": 0.226086608660866, "ece_score": 54.7826782678268, "mean_tvd": 0.34816829856320297, "n_tvd": 50, "n": 798, "n_ece_hard": 277, "score": 59.98292420575325 }, "noul": { "ece": 0.14919191919191907, "brier": 0.18700707070707095, "n": 495, "score": 70.16161616161618, "abstention_rate": 0.19797979797979803 }, "score": { "nrps": 0.10820997597418104, "top_ece": 0.11682168123766032, "n": 329, "interval90_coverage": 0.9209726443768997, "interval90_width": 0.36119554204660564, "score": 82.90733307752492 } } }, "speed": { "p50_s_raw": 0.1806289586238563, "p95_s_raw": 0.3019020542968048, "p50_s_adjusted": 0.5112579172477126, "p95_s_adjusted": 0.7538041085936096, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.09682374384236453, "basis": "price floor: base-model reference price applied" }, "rank": 36, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 36, "B": 36, "C": 40 } }, { "key": "open-alternative-jev", "display": "open-alternative-jev (Qwen3.5-4B, IkerMoel)", "addendum": null, "author": "IkerMoel", "repo": "https://github.com/ikermoel/open-alternative-jev", "class": "jev-rebuild", "licence": "Apache-2.0 (code and weights)", "open": "yes", "underlying": "Qwen/Qwen3.5-4B (frozen, BF16)", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/open-alternative-jev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 37.357784479174825, "calibration": 76.86762970590799, "speed": 91.29116775952656, "cost": 63.27859776193043 }, "scores": { "A": 33.5609546866917, "B": 29.915576313185298, "C": 23.306218532424786 }, "views": { "Balanced 33:33:33": 31.288486249897765, "Emphasis on Accuracy 60:20:20": 26.071037925172767, "Emphasis on Cost 20:20:60": 32.78700402897485, "Emphasis on Speed 20:60:20": 37.00238217276661, "Intelligence only": 20.854670294515845 }, "composite_ci95": { "A": [ 25.75468612694063, 41.663182217360564 ], "B": [ 22.57747692929389, 37.672922118345966 ], "C": [ 17.885198699264325, 28.932765428722615 ] }, "jevbench_score": 33.5609546866917, "intelligence": { "I_open": 42.59591148247294, "I_sealed": 32.11965747587671, "base": 37.357784479174825, "gap": 10.476254006596236, "excess": 5.289626506516993, "penalty": 1, "per_type_split": { "open|choice": { "cc": 63.280143623961266, "tiers": { "easy": 89.14681941513415, "standard": 76.6051468676891, "judge": 67.84232365145229, "hard": 46.729338033685856 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 5.2909627245910436, "tiers": { "easy": 70.58823529411764, "standard": 24.242424242424242, "judge": 9.80392156862746, "hard": -23.893805309734518 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 59.21662809886652, "tiers": { "easy": 47.81087316628233, "standard": 75.15891797938407, "judge": 57.036466908840055, "hard": 55.73204278427366 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 65.00076013866314, "tiers": { "easy": 76.56249999999999, "standard": 61.96513470681459, "judge": 60.24734982332155, "hard": 67.19319562575942 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -16.763636363636362, "tiers": { "easy": 12.00000000000001, "standard": -16.000000000000004, "judge": -7.9999999999999964, "hard": -30.909090909090907 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 48.12184865260336, "tiers": { "easy": 65.6992634007358, "standard": 51.34907394770446, "judge": 42.577076821697965, "hard": 46.27246119119873 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 76.86762970590799, "parts": { "choice": { "ece_hard": 0.1316919776022638, "ece_score": 73.66160447954724, "mean_tvd": 0.3123444214771244, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 71.21358116591739 }, "noul": { "ece": 0.09963701989459954, "brier": 0.17149596469772813, "n": 495, "score": 80.07259602108009, "abstention_rate": 0.4242424242424242 }, "score": { "nrps": 0.11664431905900818, "top_ece": 0.14851072116323072, "n": 330, "interval90_coverage": 0.8909090909090909, "interval90_width": 0.4803030303030303, "score": 79.31671193072651 } } }, "speed": { "p50_s_raw": 0.03633533196989447, "p95_s_raw": 0.09179776395903874, "p50_s_adjusted": 0.22267066393978893, "p95_s_adjusted": 0.33359552791807745, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.0167512315270936, "basis": "ESTIMATE (proxy tokens, I-2): exact base-model market reference; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 37, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 37, "B": 38, "C": 41 } }, { "key": "nimble-9b", "display": "Bespoke Nimble 9B (Bespoke Labs)", "addendum": null, "author": "Bespoke Labs", "repo": "https://github.com/bespokelabsai/nimble", "class": "jev-rebuild", "licence": "Apache-2.0 (weights); repository without a licence file as of 19 Sep", "open": "yes", "underlying": "bespokelabs/Bespoke-Nimble-9B (LoRA, adapter unchanged since 93ec5d6), merged into Qwen/Qwen3.5-9B@c202236 with the author's PEFT safe-merge", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/nimble-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 63.74597067366744, "calibration": 77.16259866302579, "speed": 82.95342573220694, "cost": 36.75037423027614 }, "scores": { "A": 31.8195434029124, "B": 32.310865011937956, "C": 31.8195434029124 }, "views": { "Balanced 33:33:33": 29.492712053149084, "Emphasis on Accuracy 60:20:20": 31.289963360792562, "Emphasis on Cost 20:20:60": 24.696718592776953, "Emphasis on Speed 20:60:20": 34.16501680960333, "Intelligence only": 34.43786836751591 }, "composite_ci95": { "A": [ 31.188890807449496, 32.274270442759345 ], "B": [ 31.4950648517445, 32.969723723528006 ], "C": [ 31.176926835399797, 32.274270442759345 ] }, "jevbench_score": 31.8195434029124, "intelligence": { "I_open": 65.1457495138704, "I_sealed": 62.34619183346448, "base": 63.74597067366744, "gap": 2.7995576804059183, "excess": -2.387069819673324, "penalty": 1, "per_type_split": { "open|choice": { "cc": 71.68287617013561, "tiers": { "easy": 98.19113656918904, "standard": 75.04548999220171, "judge": 79.25311203319502, "hard": 57.69682726204465 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 52.85520483334122, "tiers": { "easy": 64.70588235294117, "standard": 60.606060606060595, "judge": 82.35294117647058, "hard": 23.893805309734507 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 70.89916753813438, "tiers": { "easy": 82.31172489122478, "standard": 79.54763619054714, "judge": 70.02436327449513, "hard": 64.37789707138484 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 72.81744698393845, "tiers": { "easy": 91.2109375, "standard": 78.2657912610369, "judge": 72.1731448763251, "hard": 65.97812879708384 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 44.545454545454554, "tiers": { "easy": 68, "standard": 28.000000000000004, "judge": 32.00000000000001, "hard": 56.363636363636374 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 69.67567397100045, "tiers": { "easy": 71.08838672560954, "standard": 78.70853527327498, "judge": 69.64710519169674, "hard": 64.8274917156887 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 77.16259866302579, "parts": { "choice": { "ece_hard": 0.12758339853798056, "ece_score": 74.48332029240389, "mean_tvd": 0.30830640533409903, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 71.826339879497 }, "noul": { "ece": 0.09389240135955254, "brier": 0.12388675340423497, "n": 495, "score": 81.22151972808949, "abstention_rate": 0.15353535353535352 }, "score": { "nrps": 0.08139624087095425, "top_ece": 0.17490251574961402, "n": 330, "interval90_coverage": 0.8575757575757575, "interval90_width": 0.16565656565656572, "score": 78.43993638149088 } } }, "speed": { "p50_s_raw": 0.198314493522048, "p95_s_raw": 0.38837731960229444, "p50_s_adjusted": 0.546628987044096, "p95_s_adjusted": 0.9267546392045889, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.1283278325123153, "basis": "documented hosted-model estimate" }, "rank": 38, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 38, "B": 37, "C": 31 } }, { "key": "malkuth-2b", "display": "Malkuth-2B (newfull5, Kev post-train)", "addendum": null, "author": "newfull5 (dhtocks)", "repo": "https://github.com/newfull5/malkuth", "class": "jev-rebuild", "licence": "CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)", "open": "yes", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/malkuth-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 35.53647228073529, "calibration": 75.18587487053827, "speed": 91.80105854155826, "cost": 65.5661191189412 }, "scores": { "A": 29.896192877577803, "B": 26.384634413814368, "C": 20.761245053873477 }, "views": { "Balanced 33:33:33": 27.915795106016912, "Emphasis on Accuracy 60:20:20": 22.84336748921099, "Emphasis on Cost 20:20:60": 29.78800020397366, "Emphasis on Speed 20:60:20": 33.20153017322748, "Intelligence only": 17.95076371724391 }, "composite_ci95": { "A": [ 20.96423569566723, 38.978586221197304 ], "B": [ 18.086333645410093, 35.04119294166098 ], "C": [ 14.558497010880016, 27.068462653609238 ] }, "jevbench_score": 29.896192877577803, "intelligence": { "I_open": 44.24414236221776, "I_sealed": 28.611125630269996, "base": 36.42763399624388, "gap": 15.633016731947762, "excess": 10.446389231868519, "penalty": 0.9755361076813148, "per_type_split": { "open|choice": { "cc": 54.828061836361115, "tiers": { "easy": 89.14681941513415, "standard": 67.24720561476475, "judge": 67.84232365145229, "hard": 30.27810419114765 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 26.009338570504628, "tiers": { "easy": 76.47058823529412, "standard": 24.242424242424242, "judge": 62.745098039215684, "hard": -13.27433628318584 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 51.89502667978753, "tiers": { "easy": 39.241589861751144, "standard": 67.96974381625442, "judge": 52.00314547837483, "hard": 46.93993821712269 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 48.49372586969443, "tiers": { "easy": 82.421875, "standard": 53.81480642970343, "judge": 44.34628975265018, "hard": 40.46172539489673 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -5.781818181818181, "tiers": { "easy": 12.00000000000001, "standard": -12, "judge": -7.9999999999999964, "hard": -5.454545454545457 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 43.12146920293375, "tiers": { "easy": 59.144112318840605, "standard": 52.48518052057095, "judge": 37.35101095197979, "hard": 38.7617964533539 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 75.18587487053827, "parts": { "choice": { "ece_hard": 0.14623212996389884, "ece_score": 70.75357400722024, "mean_tvd": 0.29562866660000664, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 70.5953536736098 }, "noul": { "ece": 0.1561735353535354, "brier": 0.19303849723232322, "n": 495, "score": 68.76529292929293, "abstention_rate": 0.29292929292929293 }, "score": { "nrps": 0.13080165194696963, "top_ece": 0.07262939393939392, "n": 330, "interval90_coverage": 0.9303030303030303, "interval90_width": 0.561868686868687, "score": 86.19697800871212 } } }, "speed": { "p50_s_raw": 0.04066141549265012, "p95_s_raw": 0.06777285743592065, "p50_s_adjusted": 0.23132283098530024, "p95_s_adjusted": 0.2855457148718413, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.014053873152709357, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 39, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 39, "B": 41, "C": 43 } }, { "key": "openjev-sglang", "display": "openjev-sglang (Qwen3.6-35B-A3B on SGLang)", "addendum": null, "author": "ekzhang", "repo": "https://github.com/ekzhang/openjev-sglang", "class": "jev-rebuild", "licence": "no licence file in the repository as of 2026-09-19; Qwen3.6 weights keep their own terms", "open": "yes", "underlying": "Qwen3.6-35B-A3B", "endpoint_kind": "demo", "endpoint_condition": "the author's public demo endpoint", "gpu": null, "adapter_id": "harness/specs/openjev-sglang.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 58.648935534041364, "calibration": 82.87127167957337, "speed": 78.0657837921845, "cost": 35.63382653259791 }, "scores": { "A": 29.02726618573374, "B": 29.17633727531063, "C": 27.734727492696795 }, "views": { "Balanced 33:33:33": 26.30577347713018, "Emphasis on Accuracy 60:20:20": 27.59626123075179, "Emphasis on Cost 20:20:60": 22.26689638309311, "Emphasis on Speed 20:60:20": 30.39800407427992, "Intelligence only": 29.788254009511864 }, "composite_ci95": { "A": [ 28.422562526211088, 29.40494742988873 ], "B": [ 28.346647426334666, 29.79994598846062 ], "C": [ 24.04751115494833, 29.40494742988873 ] }, "jevbench_score": 29.02726618573374, "intelligence": { "I_open": 61.35777167533219, "I_sealed": 55.94009939275054, "base": 58.648935534041364, "gap": 5.417672282581648, "excess": 0.2310447825024058, "penalty": 1, "per_type_split": { "open|choice": { "cc": 80.55491244652758, "tiers": { "easy": 96.38227313837805, "standard": 79.7244606186639, "judge": 93.7759336099585, "hard": 67.09753231492361 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 36.38193491394949, "tiers": { "easy": 58.823529411764696, "standard": 21.212121212121215, "judge": 62.745098039215684, "hard": 18.584070796460182 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 67.13646766551949, "tiers": { "easy": 36.18600197266465, "standard": 79.38766508799597, "judge": 67.58495177114636, "hard": 68.4121222982748 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 74.59813773415907, "tiers": { "easy": 85.3515625, "standard": 74.19062712248135, "judge": 68.19787985865725, "hard": 76.91373025516404 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 27.200000000000003, "tiers": { "easy": 52, "standard": 15.999999999999993, "judge": 36.00000000000001, "hard": 19.999999999999996 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 66.02216044409255, "tiers": { "easy": 76.97326685759016, "standard": 68.10605355143156, "judge": 61.5517431466956, "hard": 65.59525026009632 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 82.87127167957337, "parts": { "choice": { "ece_hard": 0.04011216296140964, "ece_score": 91.97756740771807, "mean_tvd": 0.3284542337752234, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 79.56607201509786 }, "noul": { "ece": 0.08274897091682233, "brier": 0.12604658057209445, "n": 495, "score": 83.45020581663553, "abstention_rate": 0.2666666666666667 }, "score": { "nrps": 0.0869064310144005, "top_ece": 0.1005714124229327, "n": 330, "interval90_coverage": 0.9151515151515152, "interval90_width": 0.3108585858585858, "score": 85.5975372069867 } } }, "speed": { "p50_s_raw": 0.5980852171778679, "p95_s_raw": 0.6525271125137806, "p50_s_adjusted": 1.1961704343557358, "p95_s_adjusted": 1.3050542250275612, "n": 450, "adjustment": "x2 demo-endpoint adjustment (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.13981028325123152, "basis": "ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)" }, "rank": 40, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 40, "B": 39, "C": 36 } }, { "key": "decider-35b-a3b", "display": "decider-35b-a3b (Mapika)", "addendum": null, "author": "Mapika", "repo": "https://huggingface.co/Mapika/decider-35b-a3b", "class": "jev-rebuild", "licence": "Apache-2.0", "open": "yes", "underlying": "Qwen3.5-35B-A3B-Base with a trained decision readout, 34.7B parameters / 3B active", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/decider-35b-a3b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 60.45951150408739, "calibration": 82.03924708540572, "speed": 91.00421380212515, "cost": 34.38579876859529 }, "scores": { "A": 27.497459411252194, "B": 27.710085669874296, "C": 27.497459411252194 }, "views": { "Balanced 33:33:33": 25.063646358718533, "Emphasis on Accuracy 60:20:20": 26.365921213247887, "Emphasis on Cost 20:20:60": 20.603709005545024, "Emphasis on Speed 20:60:20": 30.090957262150802, "Intelligence only": 28.59452323204778 }, "composite_ci95": { "A": [ 26.93393630242337, 27.871066050370203 ], "B": [ 26.935050076468837, 28.28516241072057 ], "C": [ 24.14994839287159, 27.871066050370203 ] }, "jevbench_score": 27.497459411252194, "intelligence": { "I_open": 64.142449771148, "I_sealed": 56.77657323702679, "base": 60.45951150408739, "gap": 7.365876534121206, "excess": 2.1792490340419635, "penalty": 1, "per_type_split": { "open|choice": { "cc": 74.29787656936696, "tiers": { "easy": 96.38227313837805, "standard": 87.52274499610087, "judge": 85.47717842323651, "hard": 53.779866823345074 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 51.34273500228732, "tiers": { "easy": 70.58823529411764, "standard": 42.42424242424243, "judge": 80.3921568627451, "hard": 29.20353982300885 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 66.7867377417897, "tiers": { "easy": 25.374308755760357, "standard": 80.14023851590106, "judge": 66.86474442988204, "hard": 70.4045895851721 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 76.47002057270414, "tiers": { "easy": 88.28125, "standard": 80.98256735340728, "judge": 72.1731448763251, "hard": 74.48359659781289 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 31.309090909090912, "tiers": { "easy": 60.00000000000001, "standard": 24, "judge": 32.00000000000001, "hard": 27.27272727272727 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 62.55060822928531, "tiers": { "easy": 75.58623188405798, "standard": 71.18681780016796, "judge": 62.38618365627634, "hard": 55.096915959907534 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 82.03924708540572, "parts": { "choice": { "ece_hard": 0.12633935018050504, "ece_score": 74.732129963899, "mean_tvd": 0.2638981665500017, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 74.17115665444942 }, "noul": { "ece": 0.0707870707070704, "brier": 0.11012826391919195, "n": 495, "score": 85.84258585858592, "abstention_rate": 0.2080808080808081 }, "score": { "nrps": 0.08997396453030303, "top_ece": 0.09397303030303017, "n": 330, "interval90_coverage": 0.9272727272727272, "interval90_width": 0.33005050505050504, "score": 86.10399874318183 } } }, "speed": { "p50_s_raw": 0.045338188065215945, "p95_s_raw": 0.08985994872637092, "p50_s_adjusted": 0.24067637613043188, "p95_s_adjusted": 0.32971989745274183, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.1538650708128079, "basis": "price floor: base-model reference price applied" }, "rank": 41, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 41, "B": 40, "C": 37 } }, { "key": "localjev-qwen3.5-4b", "display": "local-jev Qwen3.5-4B", "addendum": null, "author": "Amith Chandrappa (amithgc)", "repo": "https://github.com/amithgc/local-jev", "class": "jev-rebuild", "licence": "MIT code; Apache-2.0 Qwen weights", "open": "yes", "underlying": "Qwen/Qwen3.5-4B text model, zero-shot next-token letter probabilities", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/localjev-qwen3.5-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 33.74767380040539, "calibration": 82.44516243154092, "speed": 83.73771259157968, "cost": 59.269627658320445 }, "scores": { "A": 25.82001585149664, "B": 22.731120666929975, "C": 17.93056656353933 }, "views": { "Balanced 33:33:33": 23.38384064773755, "Emphasis on Accuracy 60:20:20": 19.351186049964124, "Emphasis on Cost 20:20:60": 24.707823779766677, "Emphasis on Speed 20:60:20": 27.666883561690653, "Intelligence only": 15.374164345077862 }, "composite_ci95": { "A": [ 19.886176149800825, 32.96453052800093 ], "B": [ 17.19694397415245, 29.530260078234154 ], "C": [ 13.809844548472798, 22.892035088889532 ] }, "jevbench_score": 25.82001585149664, "intelligence": { "I_open": 36.702724854533855, "I_sealed": 30.79262274627692, "base": 33.74767380040539, "gap": 5.910102108256936, "excess": 0.7234746081776935, "penalty": 1, "per_type_split": { "open|choice": { "cc": 73.6696761367919, "tiers": { "easy": 96.38227313837805, "standard": 79.7244606186639, "judge": 89.6265560165975, "hard": 52.996474735605155 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -23.237896928682975, "tiers": { "easy": 41.176470588235304, "standard": 0, "judge": -21.568627450980394, "hard": -52.212389380530965 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 59.67639535549263, "tiers": { "easy": 47.47615207373271, "standard": 75.60719964664311, "judge": 59.13073394495414, "hard": 55.170300088261236 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 69.06095945143655, "tiers": { "easy": 76.56249999999999, "standard": 70.11546298392577, "judge": 60.24734982332155, "hard": 73.2685297691373 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -38.290909090909096, "tiers": { "easy": -19.999999999999996, "standard": -40, "judge": -24, "hard": -52.727272727272734 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 61.60781787830332, "tiers": { "easy": 73.30018115942029, "standard": 64.31759026028547, "judge": 61.74161752316767, "hard": 57.22949113338475 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 82.44516243154092, "parts": { "choice": { "ece_hard": 0.08462599277978346, "ece_score": 83.07480144404332, "mean_tvd": 0.230396499916665, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 80.01757572618841 }, "noul": { "ece": 0.09167757575757576, "brier": 0.13444001785858598, "n": 495, "score": 81.66448484848485, "abstention_rate": 0.6161616161616161 }, "score": { "nrps": 0.10311631408585854, "top_ece": 0.0919075757575757, "n": 330, "interval90_coverage": 0.9454545454545454, "interval90_width": 0.5088383838383839, "score": 85.65342671994951 } } }, "speed": { "p50_s_raw": 0.138433791231364, "p95_s_raw": 0.42034247601404706, "p50_s_adjusted": 0.42686758246272805, "p95_s_adjusted": 0.9906849520280941, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.022786576354679802, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 42, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 42, "B": 44, "C": 52 } }, { "key": "open-jev-zefan-9b", "display": "Open-Jev 9B (Zefan Cai)", "addendum": null, "author": "Zefan Cai (@Zefan_Cai)", "repo": "https://github.com/Zefan-Cai/Open-Jev", "class": "jev-rebuild", "licence": "MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection", "open": "yes", "underlying": "Qwen3.5-9B plus rank-8 LoRA and trained scalar decision head", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/open-jev-zefan-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 63.82052863874054, "calibration": 81.52933105829675, "speed": 73.58523755072761, "cost": 33.05490478272432 }, "scores": { "A": 24.35608345131106, "B": 24.98980978448973, "C": 24.35608345131106 }, "views": { "Balanced 33:33:33": 22.031978386347873, "Emphasis on Accuracy 60:20:20": 24.053635358614148, "Emphasis on Cost 20:20:60": 18.207909890804228, "Emphasis on Speed 20:60:20": 25.20747778317762, "Intelligence only": 27.89280621034464 }, "composite_ci95": { "A": [ 23.927044930797685, 24.64472014937102 ], "B": [ 24.396050570509544, 25.44585523866238 ], "C": [ 23.926502881115063, 24.64472014937102 ] }, "jevbench_score": 24.35608345131106, "intelligence": { "I_open": 61.78593118508326, "I_sealed": 65.85512609239782, "base": 63.82052863874054, "gap": -4.069194907314568, "excess": -9.255822407393811, "penalty": 1, "per_type_split": { "open|choice": { "cc": 66.19142369957882, "tiers": { "easy": 96.38227313837805, "standard": 73.48583311671433, "judge": 71.99170124481329, "hard": 50.64629847238542 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 48.95919107787926, "tiers": { "easy": 76.47058823529412, "standard": 57.57575757575757, "judge": 62.745098039215684, "hard": 27.43362831858407 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 70.20717877779171, "tiers": { "easy": 58.95041897615663, "standard": 81.3647374208426, "judge": 69.73772163647239, "hard": 67.79468226266454 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 71.45157070104113, "tiers": { "easy": 88.28125, "standard": 75.5490151686665, "judge": 65.54770318021201, "hard": 69.62332928311056 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 58.690909090909095, "tiers": { "easy": 84.00000000000001, "standard": 43.99999999999999, "judge": 68, "hard": 52.727272727272734 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 67.42289848524325, "tiers": { "easy": 79.5403850699516, "standard": 72.65294815521406, "judge": 69.55506458520617, "hard": 60.17937742910857 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 81.52933105829675, "parts": { "choice": { "ece_hard": 0.1106033375245877, "ece_score": 77.87933249508245, "mean_tvd": 0.28171887066993206, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 74.85372271404462 }, "noul": { "ece": 0.09714938165037393, "brier": 0.12342402792417258, "n": 495, "score": 80.57012366992522, "abstention_rate": 0.13131313131313127 }, "score": { "nrps": 0.0776873899926604, "top_ece": 0.06951483709446572, "n": 330, "interval90_coverage": 0.9303030303030303, "interval90_width": 0.28939393939393937, "score": 89.1641467909204 } } }, "speed": { "p50_s_raw": 0.5631460719741881, "p95_s_raw": 1.640916511422256, "p50_s_adjusted": 1.276292143948376, "p95_s_adjusted": 3.4318330228445117, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.17041317733990147, "basis": "documented hosted-model estimate" }, "rank": 44, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 44, "B": 42, "C": 39 } }, { "key": "decision-2b", "display": "Decision 2B (FlyMy.AI, v59)", "addendum": null, "author": "FlyMy.AI (@denti)", "repo": "https://huggingface.co/flymy-ai/decision-2b-preview", "class": "jev-rebuild", "licence": "Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release", "open": "yes", "underlying": "openbmb/MiniCPM5-2B with a trained LoRA adapter and pointer head (26.2M trainable parameters)", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "gpu": "RTX6000", "adapter_id": "images/decision-2b/spec.json:flymy_v15.FlyMyPackage", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 31.312216031505486, "calibration": 86.21585443681741, "speed": 90.35616211568993, "cost": 66.37275756963365 }, "scores": { "A": 22.51688642767283, "B": 19.299273543136604, "C": 15.636726685883911 }, "views": { "Balanced 33:33:33": 20.26076694157444, "Emphasis on Accuracy 60:20:20": 16.080545286324668, "Emphasis on Cost 20:20:60": 22.231781041422483, "Emphasis on Speed 20:60:20": 24.44878868072661, "Intelligence only": 12.280085914548287 }, "composite_ci95": { "A": [ 16.675348522547402, 28.773372628345562 ], "B": [ 14.076770422866533, 25.124356365137665 ], "C": [ 11.580103140657918, 19.98150876968442 ] }, "jevbench_score": 22.51688642767283, "intelligence": { "I_open": 34.645695446630285, "I_sealed": 27.978736616380687, "base": 31.312216031505486, "gap": 6.666958830249598, "excess": 1.4803313301703556, "penalty": 1, "per_type_split": { "open|choice": { "cc": 58.265190419661025, "tiers": { "easy": 98.19113656918904, "standard": 67.24720561476475, "judge": 55.39419087136931, "hard": 45.94594594594594 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -16.10777215149938, "tiers": { "easy": 58.823529411764696, "standard": 12.12121212121211, "judge": -11.764705882352944, "hard": -52.212389380530965 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 61.77966807172922, "tiers": { "easy": 33.683871034295954, "standard": 71.51419167650035, "judge": 69.32108617250525, "hard": 58.280291953119935 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 67.70410134214103, "tiers": { "easy": 85.3515625, "standard": 64.68191079918498, "judge": 62.897526501766784, "hard": 68.408262454435 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -35.127272727272725, "tiers": { "easy": 12.00000000000001, "standard": -31.999999999999996, "judge": -43.99999999999999, "hard": -41.81818181818182 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 51.359381234273755, "tiers": { "easy": 62.88746852162508, "standard": 58.88458396911487, "judge": 45.314711352785054, "hard": 49.24826045613189 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 86.21585443681741, "parts": { "choice": { "ece_hard": 0.052297166829797445, "ece_score": 89.5405666340405, "mean_tvd": 0.2250098243022926, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 83.51979210190561 }, "noul": { "ece": 0.06815406545421997, "brier": 0.15106906880962623, "n": 495, "score": 86.369186909156, "abstention_rate": 0.5797979797979798 }, "score": { "nrps": 0.10797650023219298, "top_ece": 0.05842590688999745, "n": 330, "interval90_coverage": 0.9545454545454546, "interval90_width": 0.5267676767676768, "score": 88.7585842993906 } } }, "speed": { "p50_s_raw": 0.07489005557727069, "p95_s_raw": 0.07865651411702856, "p50_s_adjusted": 0.2997801111545414, "p95_s_adjusted": 0.30731302823405715, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.013210160098522166, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 45, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 45, "B": 51, "C": 55 } }, { "key": "gpt-5.6-luna", "display": "GPT-5.6 Luna (low reasoning effort)", "addendum": null, "author": "OpenAI", "repo": null, "class": "llm-baseline", "licence": "proprietary API", "open": "no", "underlying": "closed", "endpoint_kind": "api", "endpoint_condition": "the operator's hosted API", "gpu": null, "adapter_id": "harness/specs/gpt-5.6-luna.json:jevbench.adapters.openai_compat.OpenAICompatAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 94.33090883309077, "calibration": 94.71076759662373, "speed": 74.06258245488996, "cost": 30.66824417593942 }, "scores": { "A": 22.371016433424646, "B": 24.156847114641927, "C": 22.371016433424646 }, "views": { "Balanced 33:33:33": 19.902091456402744, "Emphasis on Accuracy 60:20:20": 24.143664413958902, "Emphasis on Cost 20:20:60": 15.42830299146938, "Emphasis on Speed 20:60:20": 22.470268722064183, "Intelligence only": 35.48884250789443 }, "composite_ci95": { "A": [ 22.260921433578396, 22.46343987903718 ], "B": [ 24.004102294684866, 24.28792813200078 ], "C": [ 22.260921433578396, 22.46343987903718 ] }, "jevbench_score": 22.371016433424646, "intelligence": { "I_open": 93.15592763913932, "I_sealed": 95.50589002704224, "base": 94.33090883309077, "gap": -2.34996238790292, "excess": -7.536589887982163, "penalty": 1, "per_type_split": { "open|choice": { "cc": 95.9370980705296, "tiers": { "easy": 100, "standard": 95.32102937353783, "judge": 96.88796680497926, "hard": 94.51625538582061 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 87.11166848074708, "tiers": { "easy": 100, "standard": 93.93939393939394, "judge": 94.11764705882352, "hard": 75.2212389380531 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 96.41901636614128, "tiers": { "easy": 92.19354838709677, "standard": 99.09187279151944, "judge": 95.78636959370904, "hard": 96.61344022753751 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 93.85225740986291, "tiers": { "easy": 94.140625, "standard": 91.84967172288883, "judge": 93.37455830388691, "hard": 95.13973268529769 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 94.14545454545454, "tiers": { "easy": 100, "standard": 96, "judge": 87.99999999999999, "hard": 96.36363636363636 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 98.5199581258093, "tiers": { "easy": 99.64673913043478, "standard": 99.45340050377834, "judge": 99.59519797809604, "hard": 96.96511179645336 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 94.71076759662373, "parts": { "choice": { "ece_hard": 0.059550371644044475, "ece_score": 88.08992567119111, "mean_tvd": 0.04208186321201166, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 91.94086967499497 }, "noul": { "ece": 0.030852003673131213, "brier": 0.034674078902818234, "n": 495, "score": 93.82959926537376, "abstention_rate": 0.016161616161616155 }, "score": { "nrps": 0.005979484626312506, "top_ece": 0.013391919191818978, "n": 330, "interval90_coverage": 0.9787878787878788, "interval90_width": 0.006818181818181818, "score": 98.36183384950247 } } }, "speed": { "p50_s_raw": 1.3211710378527641, "p95_s_raw": 2.9701795924454926, "p50_s_adjusted": 1.3211710378527641, "p95_s_adjusted": 2.9701795924454926, "n": 450, "adjustment": "none (hosted API, measured as is)" }, "cost": { "kind": "tariff", "usd_per_1000": 0.20467204433497538, "basis": "operator list price" }, "rank": 46, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 46, "B": 43, "C": 42 } }, { "key": "typecastlm", "display": "typecastlm (Mikhail Gribov, Qwen3.5-4B computed head)", "addendum": null, "author": "Mikhail Gribov", "repo": "https://github.com/mihail-gribov/typecastlm", "class": "system-one-open", "licence": "Apache-2.0 (package and weights)", "open": "yes", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/typecastlm/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 31.24114221399841, "calibration": 76.71047738548705, "speed": 92.0380396136627, "cost": 63.95900966831593 }, "scores": { "A": 21.826965316555995, "B": 18.850194162174848, "C": 15.157614803163888 }, "views": { "Balanced 33:33:33": 20.017562533976484, "Emphasis on Accuracy 60:20:20": 15.93128916525674, "Emphasis on Cost 20:20:60": 21.742432680793506, "Emphasis on Speed 20:60:20": 24.32745311425757, "Intelligence only": 12.196653974015371 }, "composite_ci95": { "A": [ 16.231764971521944, 28.575247201608622 ], "B": [ 13.768179560326216, 25.18177166866868 ], "C": [ 11.272059008001348, 19.84392166778376 ] }, "jevbench_score": 21.826965316555995, "intelligence": { "I_open": 23.699002495040826, "I_sealed": 38.78328193295599, "base": 31.24114221399841, "gap": -15.08427943791516, "excess": -20.270906937994404, "penalty": 1, "per_type_split": { "open|choice": { "cc": 66.45284701040082, "tiers": { "easy": 92.7645462767561, "standard": 84.40343124512606, "judge": 68.87966804979253, "hard": 49.0795142969056 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -52.089347404287544, "tiers": { "easy": -11.764705882352944, "standard": -45.45454545454546, "judge": -74.50980392156863, "hard": -48.67256637168141 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 56.73350787900921, "tiers": { "easy": 32.6273750480858, "standard": 64.69949789703816, "judge": 60.03354979086002, "hard": 56.30201464383748 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 72.00706113946435, "tiers": { "easy": 82.421875, "standard": 74.19062712248135, "judge": 73.49823321554771, "hard": 67.19319562575942 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -7.490909090909094, "tiers": { "easy": 12.00000000000001, "standard": -12, "judge": -4.0000000000000036, "hard": -12.727272727272732 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 51.83369375031271, "tiers": { "easy": 68.33244307522665, "standard": 53.5615139862305, "judge": 48.685030196219714, "hard": 49.20659396669508 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 76.71047738548705, "parts": { "choice": { "ece_hard": 0.08098920458616882, "ece_score": 83.80215908276624, "mean_tvd": 0.3165516236083484, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 76.0734983609657 }, "noul": { "ece": 0.13493836400492118, "brier": 0.21792086351560655, "n": 495, "score": 73.01232719901576, "abstention_rate": 0.5858585858585859 }, "score": { "nrps": 0.10993623801474207, "top_ece": 0.13457581502783178, "n": 330, "interval90_coverage": 0.9757575757575757, "interval90_width": 0.8330808080808081, "score": 81.04560659647973 } } }, "speed": { "p50_s_raw": 0.03754384949570522, "p95_s_raw": 0.06393583689932711, "p50_s_adjusted": 0.22508769899141043, "p95_s_adjusted": 0.2778716737986542, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.01589887315270936, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 47, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 47, "B": 55, "C": 57 } }, { "key": "jev-qwen3.5-9b-base-nvfp4", "display": "JEV Qwen3.5-9B Base NVFP4", "addendum": null, "author": "WilfLin", "repo": "https://huggingface.co/WIlfLin/JEV-Qwen3.5-9B-Base-NVFP4", "class": "jev-rebuild", "licence": "Apache-2.0 entrant and upstream checkpoint", "open": "yes", "underlying": "Qwen3.5-9B Base NVFP4 with compact BF16 decision head", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/jev-qwen3.5-9b-base-nvfp4/spec.json:jev_choice_http.Adapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 32.24194556070419, "calibration": 80.93292191627495, "speed": 93.53733828267468, "cost": 47.592235403520306 }, "scores": { "A": 20.07262599814848, "B": 17.755447982336538, "C": 13.939323609825337 }, "views": { "Balanced 33:33:33": 18.020299995518467, "Emphasis on Accuracy 60:20:20": 15.099645707236771, "Emphasis on Cost 20:20:60": 17.983913005615214, "Emphasis on Speed 20:60:20": 22.39794913832311, "Intelligence only": 12.146633055154606 }, "composite_ci95": { "A": [ 15.1940766297855, 25.505644598215827 ], "B": [ 13.195795378879625, 22.94062856057887 ], "C": [ 10.551442104017708, 17.712253193205427 ] }, "jevbench_score": 20.07262599814848, "intelligence": { "I_open": 32.876088752154395, "I_sealed": 31.60780236925398, "base": 32.24194556070419, "gap": 1.2682863829004134, "excess": -3.918341117178829, "penalty": 1, "per_type_split": { "open|choice": { "cc": 71.16709783601964, "tiers": { "easy": 96.38227313837805, "standard": 75.04548999220171, "judge": 89.6265560165975, "hard": 49.0795142969056 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -34.85432145505024, "tiers": { "easy": 5.882352941176472, "standard": -39.39393939393939, "judge": -52.94117647058824, "hard": -29.20353982300885 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 62.3154898754938, "tiers": { "easy": 31.71997571338293, "standard": 73.94858718392696, "judge": 57.44464977819541, "hard": 67.80094983477873 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 59.86656059671132, "tiers": { "easy": 79.4921875, "standard": 63.32352275299977, "judge": 50.97173144876326, "hard": 59.90279465370596 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -22.472727272727273, "tiers": { "easy": 12.00000000000001, "standard": -12, "judge": -19.999999999999996, "hard": -38.18181818181819 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 57.42957378377791, "tiers": { "easy": 72.8641754647496, "standard": 62.32447323157109, "judge": 53.13911824245885, "hard": 54.34131529562769 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 80.93292191627495, "parts": { "choice": { "ece_hard": 0.07197345707175538, "ece_score": 85.60530858564891, "mean_tvd": 0.24738028711851867, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 80.43363993689852 }, "noul": { "ece": 0.13810909353384762, "brier": 0.1516578693794212, "n": 495, "score": 72.37818129323048, "abstention_rate": 0.6161616161616161 }, "score": { "nrps": 0.09632665434693449, "top_ece": 0.05196722763957396, "n": 330, "interval90_coverage": 0.9424242424242424, "interval90_width": 0.49343434343434345, "score": 89.98694451869588 } } }, "speed": { "p50_s_raw": 0.019789331941865385, "p95_s_raw": 0.04180104220286005, "p50_s_adjusted": 0.18957866388373076, "p95_s_adjusted": 0.2336020844057201, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.05583743842364531, "basis": "ESTIMATE (proxy tokens, I-2): exact base-model market reference" }, "rank": 48, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 48, "B": 57, "C": 58 } }, { "key": "gemini-3.1-flash-lite", "display": "Gemini 3.1 Flash-Lite", "addendum": null, "author": "Google", "repo": null, "class": "llm-baseline", "licence": "proprietary API", "open": "no", "underlying": "closed", "endpoint_kind": "api", "endpoint_condition": "the operator's hosted API", "gpu": null, "adapter_id": "harness/specs/gemini-3.1-flash-lite.json:jevbench.adapters.openai_compat.OpenAICompatAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 77.58771823015074, "calibration": 74.67587927109267, "speed": 80.05303137744917, "cost": 29.76239303895973 }, "scores": { "A": 19.583532784444895, "B": 20.778880829756616, "C": 19.583532784444895 }, "views": { "Balanced 33:33:33": 18.022440572612815, "Emphasis on Accuracy 60:20:20": 20.90208919224148, "Emphasis on Cost 20:20:60": 14.040404841077358, "Emphasis on Speed 20:60:20": 21.099709608322517, "Intelligence only": 27.490881546260717 }, "composite_ci95": { "A": [ 19.303624954036483, 19.83181713878822 ], "B": [ 20.410436729614165, 21.113109439966298 ], "C": [ 19.303624954036483, 19.83181713878822 ] }, "jevbench_score": 19.583532784444895, "intelligence": { "I_open": 76.61549398603603, "I_sealed": 78.55994247426545, "base": 77.58771823015074, "gap": -1.9444484882294262, "excess": -7.131075988308669, "penalty": 1, "per_type_split": { "open|choice": { "cc": 83.98327600141056, "tiers": { "easy": 96.38227313837805, "standard": 95.32102937353783, "judge": 95.850622406639, "hard": 66.31414022718371 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 73.60986228763429, "tiers": { "easy": 88.23529411764706, "standard": 81.81818181818181, "judge": 82.35294117647058, "hard": 59.29203539823009 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 72.25334366906326, "tiers": { "easy": 65.81797235023042, "standard": 97.773851590106, "judge": 69.40366972477065, "hard": 63.239187996469546 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 84.09321219590066, "tiers": { "easy": 94.140625, "standard": 83.69934344577767, "judge": 84.09893992932862, "hard": 81.77399756986635 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 75.05454545454546, "tiers": { "easy": 92, "standard": 76, "judge": 84.00000000000001, "hard": 63.63636363636365 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 76.53206977235027, "tiers": { "easy": 75.89673913043478, "standard": 76.44836272040303, "judge": 72.95703454085931, "hard": 79.41403238242097 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 74.67587927109267, "parts": { "choice": { "ece_hard": 0.2320206406509653, "ece_score": 53.59587186980694, "mean_tvd": 0.0602453253801704, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 73.78566966589494 }, "noul": { "ece": 0.10700429752066044, "brier": 0.11326300106119025, "n": 495, "score": 78.59914049586791, "abstention_rate": 0.008080808080808133 }, "score": { "nrps": 0.08817375000000002, "top_ece": 0.23948484848484825, "n": 330, "interval90_coverage": 0.7606060606060606, "interval90_width": 0.015404040404040406, "score": 71.64282765151518 } } }, "speed": { "p50_s_raw": 0.8633867427706718, "p95_s_raw": 1.1441724501550194, "p50_s_adjusted": 0.8633867427706718, "p95_s_adjusted": 1.1441724501550194, "n": 450, "adjustment": "none (hosted API, measured as is)" }, "cost": { "kind": "tariff", "usd_per_1000": 0.21940855911330048, "basis": "operator list price" }, "rank": 49, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 49, "B": 46, "C": 44 } }, { "key": "ninfer-qwen3.8-27b", "display": "NInfer Qwen3.8-27B NVFP4", "addendum": null, "author": "Igor L. / NInfer contributors", "repo": "https://github.com/igorls/ninfer", "class": "native-logit", "licence": "Apache-2.0 (engine and submitted 27B artifact/base)", "open": "yes", "underlying": "Qwen/Qwen3.8-27B, NInfer NVFP4/FP8 artifact", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/ninfer-qwen3.8-27b/spec.json:ninfer_native.NInferNativeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 65.47395819302696, "calibration": 85.91195579796471, "speed": 89.8610845330764, "cost": 29.118887844204664 }, "scores": { "A": 18.742649798238652, "B": 19.34617225634836, "C": 18.742649798238652 }, "views": { "Balanced 33:33:33": 16.750622611223033, "Emphasis on Accuracy 60:20:20": 18.576177137127328, "Emphasis on Cost 20:20:60": 13.102469983124392, "Emphasis on Speed 20:60:20": 20.431537156152054, "Intelligence only": 22.20639984766575 }, "composite_ci95": { "A": [ 18.45104757628145, 18.922589561307248 ], "B": [ 18.92985911551215, 19.64828120753878 ], "C": [ 18.45104757628145, 18.922589561307248 ] }, "jevbench_score": 18.742649798238652, "intelligence": { "I_open": 66.97566042468935, "I_sealed": 63.97225596136457, "base": 65.47395819302696, "gap": 3.0034044633247845, "excess": -2.183223036754458, "penalty": 1, "per_type_split": { "open|choice": { "cc": 80.86187003722651, "tiers": { "easy": 96.38227313837805, "standard": 89.08240187158823, "judge": 92.73858921161826, "hard": 63.963963963963955 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 47.772309245500296, "tiers": { "easy": 82.35294117647058, "standard": 57.57575757575757, "judge": 68.62745098039215, "hard": 18.584070796460182 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 72.29280199134126, "tiers": { "easy": 56.96217831131241, "standard": 83.55170403037255, "judge": 77.20210812013953, "hard": 66.81402729523414 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 80.00292218906678, "tiers": { "easy": 85.3515625, "standard": 86.41611953814805, "judge": 74.82332155477033, "hard": 79.34386391251519 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 44.727272727272734, "tiers": { "easy": 76, "standard": 24, "judge": 52, "hard": 41.81818181818182 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 67.18657296775422, "tiers": { "easy": 76.74621676344742, "standard": 69.58612480739778, "judge": 63.90748423596986, "hard": 66.05620264784741 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 85.91195579796471, "parts": { "choice": { "ece_hard": 0.07320507155126452, "ece_score": 85.35898568974709, "mean_tvd": 0.29594025310350514, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 77.88248018969828 }, "noul": { "ece": 0.061787820545642955, "brier": 0.09997090924891203, "n": 495, "score": 87.64243589087141, "abstention_rate": 0.20404040404040402 }, "score": { "nrps": 0.07114015360017155, "top_ece": 0.042320410066669624, "n": 330, "interval90_coverage": 0.9606060606060606, "interval90_width": 0.3474747474747474, "score": 92.21095131332446 } } }, "speed": { "p50_s_raw": 0.0491448815446347, "p95_s_raw": 0.13292309776879818, "p50_s_adjusted": 0.2482897630892694, "p95_s_adjusted": 0.4158461955375964, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.2305174137931034, "basis": "price floor: base-model reference price applied" }, "rank": 52, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 52, "B": 50, "C": 47 } }, { "key": "ninfer-qwen3.8-27b-t1.5", "display": "NInfer Qwen3.8-27B NVFP4 (T=1.5)", "addendum": null, "author": "Igor L. / NInfer contributors", "repo": "https://github.com/igorls/ninfer", "class": "native-logit", "licence": "Apache-2.0 (engine and submitted 27B artifact/base)", "open": "yes", "underlying": "Qwen/Qwen3.8-27B, NInfer NVFP4/FP8 artifact", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "gpu": "RTX5090", "adapter_id": "images/ninfer-qwen3.8-27b-t1.5/derive_from_27b.py:derive_t15", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 61.07759382196298, "calibration": 86.49259657807198, "speed": 89.8610845330764, "cost": 29.118887844204664 }, "scores": { "A": 18.48189600644971, "B": 18.88920290519583, "C": 18.48189600644971 }, "views": { "Balanced 33:33:33": 16.452850206736613, "Emphasis on Accuracy 60:20:20": 17.928461730599007, "Emphasis on Cost 20:20:60": 12.99211382052128, "Emphasis on Speed 20:60:20": 20.164451202362645, "Intelligence only": 20.715311974040358 }, "composite_ci95": { "A": [ 18.17854646267274, 18.648169861543884 ], "B": [ 18.444739178298107, 19.20540289130532 ], "C": [ 16.867965556382803, 18.648169861543884 ] }, "jevbench_score": 18.48189600644971, "intelligence": { "I_open": 62.432554991885866, "I_sealed": 59.72263265204009, "base": 61.07759382196298, "gap": 2.7099223398457752, "excess": -2.4767051602334673, "penalty": 1, "per_type_split": { "open|choice": { "cc": 80.86187003722651, "tiers": { "easy": 96.38227313837805, "standard": 89.08240187158823, "judge": 92.73858921161826, "hard": 63.963963963963955 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 36.82725222027669, "tiers": { "easy": 76.47058823529412, "standard": 39.393939393939405, "judge": 62.745098039215684, "hard": 6.194690265486735 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 69.60854271815441, "tiers": { "easy": 53.10192902290834, "standard": 79.39270470091643, "judge": 74.3533274635788, "hard": 65.28452659151662 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 80.00292218906678, "tiers": { "easy": 85.3515625, "standard": 86.41611953814805, "judge": 74.82332155477033, "hard": 79.34386391251519 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 36.36363636363636, "tiers": { "easy": 68, "standard": 8.000000000000007, "judge": 52, "hard": 30.909090909090907 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 62.801339403417145, "tiers": { "easy": 74.35069080100433, "standard": 66.19026700915842, "judge": 59.795755472073495, "hard": 60.47372569965746 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 86.49259657807198, "parts": { "choice": { "ece_hard": 0.046565725487101334, "ece_score": 90.68685490257973, "mean_tvd": 0.2652805775311959, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 82.07939857473006 }, "noul": { "ece": 0.050195049032060304, "brier": 0.09459883054585931, "n": 495, "score": 89.96099019358795, "abstention_rate": 0.2828282828282829 }, "score": { "nrps": 0.07411353495537272, "top_ece": 0.08856922286333445, "n": 330, "interval90_coverage": 0.9848484848484849, "interval90_width": 0.4818181818181818, "score": 87.43740096589792 } } }, "speed": { "p50_s_raw": 0.0491448815446347, "p95_s_raw": 0.13292309776879818, "p50_s_adjusted": 0.2482897630892694, "p95_s_adjusted": 0.4158461955375964, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.2305174137931034, "basis": "price floor: base-model reference price applied" }, "rank": 54, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 54, "B": 53, "C": 49 } }, { "key": "instinct", "display": "Instinct (ZooWork, Qwen3.8-27B)", "addendum": null, "author": "rayrain-srp (ZooWork)", "repo": "https://github.com/fstandhartinger/jevbench/issues/69", "class": "jev-rebuild", "licence": "proprietary service over Apache-2.0 base weights (Qwen3.8-27B)", "open": "no", "underlying": "Qwen3.8-27B (Apache-2.0 base weights) with no additional fine-tuning; their own serving stack reads candidate-token scores in one forward pass per question and normalizes them into a decision probability, with no autoregressive decoding. The serving implementation is not public.", "endpoint_kind": "demo", "endpoint_condition": "the author's public demo endpoint", "gpu": null, "adapter_id": "harness/specs/instinct.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 62.74758857686837, "calibration": 84.95658090858146, "speed": 81.68029758765765, "cost": 29.16007113367894 }, "scores": { "A": 18.324676123308297, "B": 18.85789359077411, "C": 18.324676123308297 }, "views": { "Balanced 33:33:33": 16.33295307078797, "Emphasis on Accuracy 60:20:20": 18.02517125337854, "Emphasis on Cost 20:20:60": 12.975872256759336, "Emphasis on Speed 20:60:20": 19.55658931728601, "Intelligence only": 21.341954505243862 }, "composite_ci95": { "A": [ 18.03251754045729, 18.509314642890757 ], "B": [ 18.423335678285298, 19.174218341638475 ], "C": [ 17.615351400843892, 18.509314642890757 ] }, "jevbench_score": 18.324676123308297, "intelligence": { "I_open": 61.912899747276896, "I_sealed": 63.582277406459845, "base": 62.74758857686837, "gap": -1.669377659182949, "excess": -6.856005159262192, "penalty": 1, "per_type_split": { "open|choice": { "cc": 84.48610259077087, "tiers": { "easy": 98.19113656918904, "standard": 90.64205874707565, "judge": 93.7759336099585, "hard": 71.01449275362319 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 28.500465351063998, "tiers": { "easy": 70.58823529411764, "standard": 42.42424242424243, "judge": 25.49019607843137, "hard": 13.27433628318584 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 72.75213129999584, "tiers": { "easy": 42.05506928611551, "standard": 82.48342931697461, "judge": 74.99860827508381, "hard": 73.87589006366055 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 79.66534680747468, "tiers": { "easy": 91.2109375, "standard": 78.2657912610369, "judge": 78.79858657243817, "hard": 78.12879708383961 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 39.56363636363636, "tiers": { "easy": 68, "standard": 36.00000000000001, "judge": 43.99999999999999, "hard": 30.909090909090907 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 71.51784904826852, "tiers": { "easy": 78.33556811423234, "standard": 72.1281289243048, "judge": 68.0999921739055, "hard": 72.07167199953165 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 84.95658090858146, "parts": { "choice": { "ece_hard": 0.041775188617254565, "ece_score": 91.6449622765491, "mean_tvd": 0.27112430615970573, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 82.26626583028926 }, "noul": { "ece": 0.09012706875176714, "brier": 0.08283135625470438, "n": 495, "score": 81.97458624964658, "abstention_rate": 0.3171717171717172 }, "score": { "nrps": 0.06592061964396308, "top_ece": 0.06075078371993284, "n": 330, "interval90_coverage": 0.9636363636363636, "interval90_width": 0.3207070707070707, "score": 90.62889064580857 } } }, "speed": { "p50_s_raw": 0.259529784321785, "p95_s_raw": 0.6542188361287111, "p50_s_adjusted": 0.51905956864357, "p95_s_adjusted": 1.3084376722574222, "n": 450, "adjustment": "x2 demo-endpoint adjustment (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.22978991379310343, "basis": "ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)" }, "rank": 55, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 55, "B": 54, "C": 50 } }, { "key": "openjev-thinking", "display": "OpenJev (thinking, BF16)", "addendum": null, "author": "razorback16", "repo": "https://github.com/razorback16/openjev", "class": "jev-rebuild", "licence": "Apache-2.0", "open": "yes", "underlying": "google/diffusiongemma-26b-a4b-it, BF16; OpenJev think=512", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/openjev-thinking/spec.json:openjev_thinking.ThinkingOpenJev", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 84.20554946501295, "calibration": 83.10226165901825, "speed": 73.86286987310874, "cost": 28.51480924396573 }, "scores": { "A": 17.939903136855065, "B": 19.269266611296747, "C": 17.939903136855065 }, "views": { "Balanced 33:33:33": 16.131810682141285, "Emphasis on Accuracy 60:20:20": 19.305333345098983, "Emphasis on Cost 20:20:60": 12.449501761513527, "Emphasis on Speed 20:60:20": 18.57206532391989, "Intelligence only": 27.386822476131922 }, "composite_ci95": { "A": [ 17.75987145107232, 18.078147584373728 ], "B": [ 19.028507279349867, 19.473239838287018 ], "C": [ 17.75987145107232, 18.078147584373728 ] }, "jevbench_score": 17.939903136855065, "intelligence": { "I_open": 82.51224259391168, "I_sealed": 85.89885633611422, "base": 84.20554946501295, "gap": -3.3866137422025417, "excess": -8.573241242281785, "penalty": 1, "per_type_split": { "open|choice": { "cc": 81.66804459050107, "tiers": { "easy": 98.19113656918904, "standard": 90.64205874707565, "judge": 92.73858921161826, "hard": 64.74735605170387 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 80.61284368936633, "tiers": { "easy": 100, "standard": 90.90909090909092, "judge": 86.27450980392157, "hard": 66.3716814159292 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 85.25583950186767, "tiers": { "easy": 96.95557988178955, "standard": 94.28149345965278, "judge": 83.79455455177514, "hard": 78.91404114056402 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 78.51164590189032, "tiers": { "easy": 100, "standard": 78.2657912610369, "judge": 80.12367491166077, "hard": 72.05346294046174 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 88.61818181818181, "tiers": { "easy": 100, "standard": 100, "judge": 96, "hard": 74.54545454545453 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 90.56674128827055, "tiers": { "easy": 99.95508182759957, "standard": 85.36939599599363, "judge": 94.12127816756136, "hard": 88.15242614010864 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 83.10226165901825, "parts": { "choice": { "ece_hard": 0.14617801394299762, "ece_score": 70.76439721140048, "mean_tvd": 0.34613564131126073, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 68.07541654013721 }, "noul": { "ece": 0.046908044598036464, "brier": 0.048015879012664964, "n": 495, "score": 90.61839108039271, "abstention_rate": 0.036363636363636376 }, "score": { "nrps": 0.03461684887491551, "top_ece": 0.07656180199729376, "n": 330, "interval90_coverage": 0.9272727272727272, "interval90_width": 0.04924242424242424, "score": 90.61297735652485 } } }, "speed": { "p50_s_raw": 0.7056447945069522, "p95_s_raw": 1.2408293015789242, "p50_s_adjusted": 1.5612895890139042, "p95_s_adjusted": 2.6316586031578484, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.24145693965517237, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 56, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 56, "B": 52, "C": 51 } }, { "key": "djev-thinking", "display": "djev (thinking)", "addendum": null, "author": "David Villalon / Maisa", "repo": "https://github.com/Davipar/djev-dev", "class": "jev-rebuild", "licence": "Apache-2.0", "open": "yes", "underlying": "google/diffusiongemma-26b-a4b-it, BF16; full generation with thinking enabled", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/djev-thinking/spec.json:djev_thinking_v15.DjevThinkingV15Adapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1357 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 77.34831993443525, "calibration": 95.66103904664365, "speed": 72.31810288828999, "cost": 28.131942787244554 }, "scores": { "A": 17.403898604323025, "B": 18.472410565292908, "C": 17.403898604323025 }, "views": { "Balanced 33:33:33": 15.243019264957107, "Emphasis on Accuracy 60:20:20": 17.95382784931123, "Emphasis on Cost 20:20:60": 11.86545691772876, "Emphasis on Speed 20:60:20": 17.59487888695899, "Intelligence only": 24.485576136504655 }, "composite_ci95": { "A": [ 17.276321176749658, 17.507045027661402 ], "B": [ 18.270980597540277, 18.646681574730966 ], "C": [ 17.276321176749658, 17.507045027661402 ] }, "jevbench_score": 17.403898604323025, "intelligence": { "I_open": 76.55371800448589, "I_sealed": 78.14292186438462, "base": 77.34831993443525, "gap": -1.5892038598987313, "excess": -6.775831359977974, "penalty": 1, "per_type_split": { "open|choice": { "cc": 64.76721084087255, "tiers": { "easy": 94.57340970756708, "standard": 81.28411749415127, "judge": 63.6929460580913, "hard": 49.8629063846455 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 74.62858675248056, "tiers": { "easy": 100, "standard": 93.93939393939394, "judge": 66.66666666666667, "hard": 64.60176991150442 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 90.26535642010458, "tiers": { "easy": 61.73963133640552, "standard": 95.26956582584054, "judge": 89.46264744429882, "hard": 95.4967147200157 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 48.29304190430006, "tiers": { "easy": 85.3515625, "standard": 52.45641838351824, "judge": 54.94699646643111, "hard": 31.956257594167692 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 95.74545454545455, "tiers": { "easy": 100, "standard": 92, "judge": 96, "hard": 96.36363636363636 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 90.39026914339927, "tiers": { "easy": 93.8768115942029, "standard": 92.1662468513854, "judge": 81.55855096882898, "hard": 95.25443330763301 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.2916145181476846 }, "noul": { "n": 495, "invalid_rate": 0.0505050505050505 }, "score": { "n": 330, "invalid_rate": 0.036363636363636376 } }, "calibration": { "score": 95.66103904664365, "parts": { "choice": { "ece_hard": 0.04855384588187193, "ece_score": 90.28923082362562, "mean_tvd": 0.007095392065954789, "n_tvd": 39, "n": 566, "n_ece_hard": 164, "score": 94.78984580851507 }, "noul": { "ece": 0.02890577741407529, "brier": 0.030226985540601766, "n": 470, "score": 94.21884451718495, "abstention_rate": 0.008510638297872353 }, "score": { "nrps": 0.008094957913858556, "top_ece": 0.01620825290076149, "n": 318, "interval90_coverage": 0.9779874213836478, "interval90_width": 0.01493710691823899, "score": 97.97442681423092 } } }, "speed": { "p50_s_raw": 0.6655631910543889, "p95_s_raw": 1.9045551142189638, "p50_s_adjusted": 1.4811263821087777, "p95_s_adjusted": 3.9591102284379276, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.248657697044335, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 57, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 57, "B": 56, "C": 54 } }, { "key": "litjev", "display": "LitJev (Qwen3.8-27B)", "addendum": null, "author": "Zhengxu Yu", "repo": "https://github.com/zhengxuyu/litjev", "class": "jev-rebuild", "licence": "Apache-2.0 (code); Apache-2.0 base weights", "open": "yes", "underlying": "Qwen/Qwen3.8-27B, frozen, read at the output head", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/litjev/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 58.324628215248765, "calibration": 84.45282357688976, "speed": 68.2244816164631, "cost": 28.35809912639418 }, "scores": { "A": 16.306394195686046, "B": 16.74462103865969, "C": 15.408465642675305 }, "views": { "Balanced 33:33:33": 14.3890375127424, "Emphasis on Accuracy 60:20:20": 15.868297092774801, "Emphasis on Cost 20:20:60": 11.689300893996586, "Emphasis on Speed 20:60:20": 16.687524645731486, "Intelligence only": 18.761441475825645 }, "composite_ci95": { "A": [ 16.031824256739885, 16.48126738701614 ], "B": [ 16.332888760119257, 17.052026385960268 ], "C": [ 13.385786071253271, 16.48126738701614 ] }, "jevbench_score": 16.306394195686046, "intelligence": { "I_open": 58.061823774277066, "I_sealed": 58.58743265622046, "base": 58.324628215248765, "gap": -0.5256088819433913, "excess": -5.712236382022634, "penalty": 1, "per_type_split": { "open|choice": { "cc": 80.4361524060561, "tiers": { "easy": 98.19113656918904, "standard": 90.64205874707565, "judge": 82.36514522821577, "hard": 69.44770857814335 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 23.485873834650512, "tiers": { "easy": 76.47058823529412, "standard": 54.54545454545454, "judge": 29.41176470588236, "hard": -9.734513274336287 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 70.2634450821246, "tiers": { "easy": 54.66237167016875, "standard": 81.32857378919948, "judge": 70.73340284898273, "hard": 68.27868075643254 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 79.03218004271174, "tiers": { "easy": 88.28125, "standard": 80.98256735340728, "judge": 77.47349823321555, "hard": 76.91373025516404 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 33.709090909090904, "tiers": { "easy": 76, "standard": 15.999999999999993, "judge": 39.99999999999999, "hard": 27.27272727272727 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 63.02102701685875, "tiers": { "easy": 74.79973205795622, "standard": 66.01457217051639, "judge": 60.65620381864931, "hard": 60.353195578412645 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 84.45282357688976, "parts": { "choice": { "ece_hard": 0.04970492013031636, "ece_score": 90.05901597393672, "mean_tvd": 0.2614886387339129, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 81.95507605027271 }, "noul": { "ece": 0.08802838392984348, "brier": 0.09245053724684392, "n": 495, "score": 82.3943232140313, "abstention_rate": 0.34545454545454546 }, "score": { "nrps": 0.07045292120710725, "top_ece": 0.07468282473279364, "n": 330, "interval90_coverage": 0.9727272727272728, "interval90_width": 0.469949494949495, "score": 89.00907146636527 } } }, "speed": { "p50_s_raw": 1.4589347330620512, "p95_s_raw": 2.3779289319296364, "p50_s_adjusted": 3.0678694661241024, "p95_s_adjusted": 4.905857863859273, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.2443787068965517, "basis": "price floor: base-model reference price applied" }, "rank": 58, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 58, "B": 58, "C": 56 } }, { "key": "raw-phi-4-mini", "display": "Raw Phi-4 mini direct logits", "addendum": null, "author": "Microsoft / neutral reproduction", "repo": "https://huggingface.co/microsoft/Phi-4-mini-instruct", "class": "raw-logit-control", "licence": "MIT", "open": "yes", "underlying": "Phi-4-mini-instruct BF16", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/raw-phi-4-mini/spec.json:raw_adapter.RawControlAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 27.62403248828664, "calibration": 71.41052135550255, "speed": 89.17438865550707, "cost": 53.22051180195924 }, "scores": { "A": 15.222235703588067, "B": 13.110562970892774, "C": 10.5709970163806 }, "views": { "Balanced 33:33:33": 13.831540696125431, "Emphasis on Accuracy 60:20:20": 11.010976074923198, "Emphasis on Cost 20:20:60": 14.705353331656267, "Emphasis on Speed 20:60:20": 17.219213805677136, "Intelligence only": 8.431817920288301 }, "composite_ci95": { "A": [ 10.258712918386285, 20.737336979219407 ], "B": [ 8.602535584899657, 18.245435235297943 ], "C": [ 7.124106193323811, 14.400928457791256 ] }, "jevbench_score": 15.222235703588067, "intelligence": { "I_open": 38.273825809008926, "I_sealed": 19.96000189388855, "base": 29.116913851448736, "gap": 18.313823915120377, "excess": 13.127196415041134, "penalty": 0.9487280358495886, "per_type_split": { "open|choice": { "cc": 54.09535596329945, "tiers": { "easy": 90.95568284594513, "standard": 54.76995061086561, "judge": 60.580912863070544, "hard": 39.678809244026624 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 6.0318962661492606, "tiers": { "easy": 64.70588235294117, "standard": -9.090909090909093, "judge": 41.176470588235304, "hard": -27.43362831858407 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 54.694225197578064, "tiers": { "easy": 24.723976188382103, "standard": 70.2841282845953, "judge": 59.057398525133834, "hard": 51.1194559107016 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 56.668623376411986, "tiers": { "easy": 76.56249999999999, "standard": 56.53158252207382, "judge": 52.296819787985875, "hard": 55.04252733900366 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -43.63636363636364, "tiers": { "easy": -36, "standard": -36, "judge": -43.99999999999999, "hard": -49.09090909090909 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 46.847745941617305, "tiers": { "easy": 60.677416027791274, "standard": 54.691931112074776, "judge": 49.34671366256982, "hard": 37.59401004413068 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 71.41052135550255, "parts": { "choice": { "ece_hard": 0.15855047915699413, "ece_score": 68.28990416860117, "mean_tvd": 0.31305162444341283, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 68.49237086212995 }, "noul": { "ece": 0.176680478068443, "brier": 0.24340647285719444, "n": 495, "score": 64.6639043863114, "abstention_rate": 0.3858585858585859 }, "score": { "nrps": 0.12796697355859052, "top_ece": 0.1252636250400414, "n": 330, "interval90_coverage": 0.906060606060606, "interval90_width": 0.5098484848484847, "score": 81.07528881806633 } } }, "speed": { "p50_s_raw": 0.06535653909668326, "p95_s_raw": 0.14041130524128673, "p50_s_adjusted": 0.28071307819336655, "p95_s_adjusted": 0.4308226104825734, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.03625068965517241, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 59, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 59, "B": 60, "C": 60 } }, { "key": "opensourcejev-qwen35-4b-q4km", "display": "OpenSourceJev (Qwen3.5-4B Q4_K_M, native llama.cpp)", "addendum": null, "author": "sabeel111", "repo": "https://github.com/sabeel111/OpenSourceJev", "class": "jev-rebuild", "licence": "MIT (repository code); Apache-2.0 (Qwen/Qwen3.5-4B base and unsloth/Qwen3.5-4B-GGUF Q4_K_M conversion)", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/opensourcejev-qwen35-4b-q4km/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 25.769029299779596, "calibration": 76.0354906185011, "speed": 72.50784557479969, "cost": 69.13657649738097 }, "scores": { "A": 13.245465302463623, "B": 11.158505729571912, "C": 9.198239793377518 }, "views": { "Balanced 33:33:33": 11.882308110018135, "Emphasis on Accuracy 60:20:20": 9.179815610247537, "Emphasis on Cost 20:20:60": 13.835630815721885, "Emphasis on Speed 20:60:20": 14.032252211105229, "Intelligence only": 6.84469608018877 }, "composite_ci95": { "A": [ 8.979165053864623, 18.308517022614893 ], "B": [ 7.3854187708120556, 15.78442530630126 ], "C": [ 6.235531287405989, 12.714247932371459 ] }, "jevbench_score": 13.245465302463623, "intelligence": { "I_open": 27.857284804877462, "I_sealed": 23.680773794681734, "base": 25.769029299779596, "gap": 4.176511010195728, "excess": -1.0101164898835142, "penalty": 1, "per_type_split": { "open|choice": { "cc": 64.38244688758064, "tiers": { "easy": 90.95568284594513, "standard": 78.1648037431765, "judge": 76.14107883817428, "hard": 42.02898550724636 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -36.117867903396274, "tiers": { "easy": 47.058823529411775, "standard": -30.303030303030297, "judge": -50.98039215686274, "hard": -48.67256637168141 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 55.307275430448016, "tiers": { "easy": 27.21197926267279, "standard": 76.67532199646642, "judge": 50.515131061598964, "hard": 55.24118446601941 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 51.48566463280753, "tiers": { "easy": 73.63281249999999, "standard": 59.2483586144442, "judge": 39.045936395759725, "hard": 51.39732685297693 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -37.23636363636364, "tiers": { "easy": -28.000000000000004, "standard": -31.999999999999996, "judge": -28.000000000000004, "hard": -49.09090909090909 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 56.793020387601324, "tiers": { "easy": 72.95325452898551, "standard": 64.46683753148615, "judge": 49.08188711036227, "hard": 54.69940323824214 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 76.0354906185011, "parts": { "choice": { "ece_hard": 0.14908980866425994, "ece_score": 70.18203826714802, "mean_tvd": 0.3192922165500016, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 69.12640830607393 }, "noul": { "ece": 0.1400313272727273, "brier": 0.15711699719054362, "n": 495, "score": 71.99373454545454, "abstention_rate": 0.6626262626262627 }, "score": { "nrps": 0.11148831082959457, "top_ece": 0.07439255454545449, "n": 330, "interval90_coverage": 0.9242424242424242, "interval90_width": 0.4464646464646464, "score": 86.98632900397483 } } }, "speed": { "p50_s_raw": 0.6195944650098681, "p95_s_raw": 1.9453384675551155, "p50_s_adjusted": 1.3891889300197362, "p95_s_adjusted": 4.040676935110231, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.0106851539408867, "basis": "price floor: base-model reference price applied; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 60, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 60, "B": 61, "C": 61 } }, { "key": "reflex-27b", "display": "reflex-27b (Qwen3.8-27B)", "addendum": null, "author": "kshetrajna12", "repo": "https://github.com/kshetrajna12/reflex", "class": "jev-rebuild", "licence": "MIT code; Apache-2.0 Qwen weights", "open": "yes", "underlying": "Qwen3.8-27B, frozen, direct-logit readout averaged across two option orders", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/reflex-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 62.794986197868965, "calibration": 85.93457237686198, "speed": 69.19534736783416, "cost": 25.802305452315423 }, "scores": { "A": 13.188297397843682, "B": 13.770358241149133, "C": 13.188297397843682 }, "views": { "Balanced 33:33:33": 11.556158023973218, "Emphasis on Accuracy 60:20:20": 13.185620031521559, "Emphasis on Cost 20:20:60": 9.079850594306789, "Emphasis on Speed 20:60:20": 13.581847895497212, "Intelligence only": 16.722530048872894 }, "composite_ci95": { "A": [ 12.992358682861784, 13.300762882643161 ], "B": [ 13.473043870540051, 13.98155386873936 ], "C": [ 12.664340288901577, 13.300762882643161 ] }, "jevbench_score": 13.188297397843682, "intelligence": { "I_open": 63.0906276726272, "I_sealed": 62.499344723110724, "base": 62.794986197868965, "gap": 0.5912829495164758, "excess": -4.595344550562767, "penalty": 1, "per_type_split": { "open|choice": { "cc": 84.80373580586229, "tiers": { "easy": 98.19113656918904, "standard": 85.96308812061348, "judge": 93.7759336099585, "hard": 74.14806110458284 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 30.922026091208803, "tiers": { "easy": 70.58823529411764, "standard": 15.15151515151516, "judge": 58.823529411764696, "hard": 7.964601769911495 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 73.54612112081051, "tiers": { "easy": 48.2951624423963, "standard": 84.68773586572438, "judge": 76.40678899082567, "hard": 72.14255251544573 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 80.29589093906678, "tiers": { "easy": 88.28125, "standard": 86.41611953814805, "judge": 74.82332155477033, "hard": 79.34386391251519 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 35.709090909090904, "tiers": { "easy": 68, "standard": 24, "judge": 43.99999999999999, "hard": 27.27272727272727 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 71.49305232117449, "tiers": { "easy": 82.4410579710145, "standard": 74.76243366918558, "judge": 67.25852569502949, "hard": 70.29725520431766 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 85.93457237686198, "parts": { "choice": { "ece_hard": 0.03788619494584862, "ece_score": 92.42276101083029, "mean_tvd": 0.2503876265500017, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 83.69199917791505 }, "noul": { "ece": 0.09199937373737374, "brier": 0.08261424794053736, "n": 495, "score": 81.60012525252525, "abstention_rate": 0.31919191919191914 }, "score": { "nrps": 0.06417932175466212, "top_ece": 0.04279441212121212, "n": 330, "interval90_coverage": 0.9757575757575757, "interval90_width": 0.3825757575757574, "score": 92.51159270014568 } } }, "speed": { "p50_s_raw": 1.2991375599522144, "p95_s_raw": 2.1146518142428246, "p50_s_adjusted": 2.7482751199044286, "p95_s_adjusted": 4.3793036284856495, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.2973421551724138, "basis": "price floor: base-model reference price applied" }, "rank": 61, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 61, "B": 59, "C": 59 } }, { "key": "open-jev-zefan-2b", "display": "Open-Jev 2B (Zefan Cai)", "addendum": null, "author": "Zefan Cai (@Zefan_Cai)", "repo": "https://github.com/Zefan-Cai/Open-Jev", "class": "jev-rebuild", "licence": "MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection", "open": "yes", "underlying": "Qwen3.5-2B plus rank-8 LoRA and trained scalar decision head", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/open-jev-zefan-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 33.564173076571095, "calibration": 73.62274281306851, "speed": 75.76421649699464, "cost": 33.05490478272432 }, "scores": { "A": 9.072856494116836, "B": 8.443734103226019, "C": 6.300594787581135 }, "views": { "Balanced 33:33:33": 8.066510781336797, "Emphasis on Accuracy 60:20:20": 7.413258997675015, "Emphasis on Cost 20:20:60": 7.362375355630084, "Emphasis on Speed 20:60:20": 9.8825122020589, "Intelligence only": 6.610277008881134 }, "composite_ci95": { "A": [ 6.618399852202896, 11.536977019372797 ], "B": [ 6.032267746205537, 10.927897940214102 ], "C": [ 4.5961110084742325, 8.011789596786665 ] }, "jevbench_score": 9.072856494116836, "intelligence": { "I_open": 38.78572545166409, "I_sealed": 28.34262070147809, "base": 33.564173076571095, "gap": 10.443104750186002, "excess": 5.25647725010676, "penalty": 1, "per_type_split": { "open|choice": { "cc": 51.51924767992967, "tiers": { "easy": 87.33795598432319, "standard": 51.650636859890824, "judge": 71.99170124481329, "hard": 27.144535840188002 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 17.336141214329654, "tiers": { "easy": 76.47058823529412, "standard": 9.090909090909083, "judge": 39.2156862745098, "hard": -9.734513274336287 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 47.50178746073294, "tiers": { "easy": 35.09794173421018, "standard": 65.72344820125355, "judge": 48.51920193657173, "hard": 40.72885766522424 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 45.4862239976945, "tiers": { "easy": 70.703125, "standard": 57.889970568258995, "judge": 40.37102473498235, "hard": 36.816524908869994 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -8.14545454545455, "tiers": { "easy": 19.999999999999996, "standard": -12, "judge": -4.0000000000000036, "hard": -16.36363636363637 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 47.68709265219432, "tiers": { "easy": 60.087358156308326, "standard": 58.822783994022785, "judge": 44.278631641034735, "hard": 41.575526363621286 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 73.62274281306851, "parts": { "choice": { "ece_hard": 0.13443745135490978, "ece_score": 73.11250972901804, "mean_tvd": 0.31141793250204397, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 70.98535823940682 }, "noul": { "ece": 0.15698662520519996, "brier": 0.21726970854511088, "n": 495, "score": 68.60267495896001, "abstention_rate": 0.29292929292929293 }, "score": { "nrps": 0.13193684670715047, "top_ece": 0.12122962423803765, "n": 330, "interval90_coverage": 0.9303030303030303, "interval90_width": 0.5454545454545454, "score": 81.28019524083871 } } }, "speed": { "p50_s_raw": 0.42904718848876655, "p95_s_raw": 1.2403676671441641, "p50_s_adjusted": 1.008094376977533, "p95_s_adjusted": 2.630735334288328, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.17041317733990147, "basis": "documented hosted-model estimate" }, "rank": 62, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 62, "B": 62, "C": 63 } }, { "key": "gliner2-large", "display": "GLiNER2 large (Fastino)", "addendum": null, "author": "Fastino", "repo": "https://huggingface.co/fastino/gliner2-large-v1", "class": "classifier", "licence": "Apache-2.0", "open": "yes", "underlying": "GLiNER2 large schema-conditioned extractor (DeBERTa-v3-large encoder)", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/gliner2-large/spec.json:warm_adapter.WarmLoaded", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 22.49372087756688, "calibration": 42.37995459053843, "speed": 64.775339304984, "cost": 77.5922354035203 }, "scores": { "A": 8.399649998514048, "B": 7.1852200696572694, "C": 5.833090276745868 }, "views": { "Balanced 33:33:33": 8.342104237606092, "Emphasis on Accuracy 60:20:20": 6.258239920284327, "Emphasis on Cost 20:20:60": 10.267354778838929, "Emphasis on Speed 20:60:20": 9.762184401986273, "Intelligence only": 4.552436497562072 }, "composite_ci95": { "A": [ 5.206602086900395, 12.336461805804452 ], "B": [ 4.351719703950813, 10.741887051792189 ], "C": [ 3.615695893680829, 8.56698736514198 ] }, "jevbench_score": 8.399649998514048, "intelligence": { "I_open": 26.030302249344764, "I_sealed": 18.957139505788998, "base": 22.49372087756688, "gap": 7.073162743555766, "excess": 1.8865352434765237, "penalty": 1, "per_type_split": { "open|choice": { "cc": 26.574056711930325, "tiers": { "easy": 87.33795598432319, "standard": 34.4944112295295, "judge": 20.124481327800847, "hard": 12.260086173129631 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 16.353067373369296, "tiers": { "easy": 64.70588235294117, "standard": 3.0303030303030276, "judge": 50.98039215686274, "hard": -15.04424778761062 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 35.163782662734675, "tiers": { "easy": 8.23002435297282, "standard": 62.58297647158422, "judge": 35.395996727022364, "hard": 28.013464787534602 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 28.41569323833785, "tiers": { "easy": 35.54687499999999, "standard": 32.08059769074034, "judge": 20.49469964664312, "hard": 30.741190765492117 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -0.6181818181818213, "tiers": { "easy": -36, "standard": 4.0000000000000036, "judge": 0, "hard": 5.454545454545445 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 29.073907097210963, "tiers": { "easy": 33.32924379728206, "standard": 30.537722630646826, "judge": 22.797688869828836, "hard": 31.98532882601185 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 42.37995459053843, "parts": { "choice": { "ece_hard": 0.35899151845529187, "ece_score": 28.201696308941628, "mean_tvd": 0.44204858856583795, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 41.99841872617891 }, "noul": { "ece": 0.34577794089535685, "brier": 0.35859397977345603, "n": 495, "score": 30.84441182092863, "abstention_rate": 0.10303030303030303 }, "score": { "nrps": 0.20017276385688332, "top_ece": 0.3569432858264807, "n": 330, "interval90_coverage": 0.6909090909090909, "interval90_width": 0.3994949494949492, "score": 54.29703322450776 } } }, "speed": { "p50_s_raw": 0.8647058844799176, "p95_s_raw": 8.784600352274692, "p50_s_adjusted": 1.879411768959835, "p95_s_adjusted": 17.719200704549383, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.005583743842364532, "basis": "ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies" }, "rank": 63, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 63, "B": 64, "C": 64 } }, { "key": "qwen3-reranker-4b", "display": "Qwen3-Reranker-4B", "addendum": null, "author": "Qwen", "repo": "https://huggingface.co/Qwen/Qwen3-Reranker-4B", "class": "reranker", "licence": "Apache-2.0", "open": "yes", "underlying": "4B instruction-aware generative reranker", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/qwen3-reranker-4b/spec.json:reranker_adapter.RerankerAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 21.025680453051585, "calibration": 76.2573027508077, "speed": 79.60936519080639, "cost": 48.39770052432463 }, "scores": { "A": 7.0578732911112105, "B": 5.856117186678418, "C": 4.901300896605006 }, "views": { "Balanced 33:33:33": 6.152668509566967, "Emphasis on Accuracy 60:20:20": 4.709324078073576, "Emphasis on Cost 20:20:60": 6.78411455677089, "Emphasis on Speed 20:60:20": 7.821945004625668, "Intelligence only": 3.4835305028101717 }, "composite_ci95": { "A": [ 4.256683239685833, 10.735647697538731 ], "B": [ 3.4274523885192507, 9.149689058545547 ], "C": [ 2.956030027559606, 7.455310901068563 ] }, "jevbench_score": 7.0578732911112105, "intelligence": { "I_open": 30.348285824317287, "I_sealed": 13.363369857977867, "base": 21.855827841147576, "gap": 16.984915966339422, "excess": 11.798288466260178, "penalty": 0.9620171153373982, "per_type_split": { "open|choice": { "cc": 56.69813267337053, "tiers": { "easy": 89.14681941513415, "standard": 64.12789186378996, "judge": 74.06639004149378, "hard": 31.84488836662748 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -10.056946350543434, "tiers": { "easy": 17.647058823529417, "standard": -42.42424242424242, "judge": 37.254901960784316, "hard": -36.283185840707965 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 44.40367115012478, "tiers": { "easy": 13.096620724978504, "standard": 54.28179029670248, "judge": 42.537424561879064, "hard": 48.69105912430677 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 45.59936268245455, "tiers": { "easy": 67.7734375, "standard": 41.58931401403669, "judge": 50.97173144876326, "hard": 38.031591737545575 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -45.92727272727273, "tiers": { "easy": -43.99999999999999, "standard": -40, "judge": -56.00000000000001, "hard": -41.81818181818182 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 40.418019618751785, "tiers": { "easy": 55.03666193584841, "standard": 42.80674647447732, "judge": 36.240240687009674, "hard": 38.702329810421446 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 76.2573027508077, "parts": { "choice": { "ece_hard": 0.09906655742890301, "ece_score": 80.1866885142194, "mean_tvd": 0.25225164327791294, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 77.48076209321405 }, "noul": { "ece": 0.16823664778156078, "brier": 0.21948898953258267, "n": 495, "score": 66.35267044368784, "abstention_rate": 0.49696969696969695 }, "score": { "nrps": 0.14741060021939884, "top_ece": 0.07690994273508868, "n": 330, "interval90_coverage": 0.9272727272727272, "interval90_width": 0.6116161616161616, "score": 84.9384757155212 } } }, "speed": { "p50_s_raw": 0.18062184238806367, "p95_s_raw": 0.9950535912532349, "p50_s_adjusted": 0.5112436847761274, "p95_s_adjusted": 2.14010718250647, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.05249000923645321, "basis": "ESTIMATE: hosted exact-model reference (deepinfra:Qwen/Qwen3-Reranker-4B); no exact base-model floor applies" }, "rank": 64, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 64, "B": 65, "C": 65 } }, { "key": "deepseek-flash", "display": "DeepSeek V4.1 Flash (thinking default)", "addendum": null, "author": "DeepSeek", "repo": null, "class": "llm-baseline", "licence": "open weights, proprietary API route", "open": "weights", "underlying": "DeepSeek-V4.1-Flash", "endpoint_kind": "api", "endpoint_condition": "the operator's hosted API", "gpu": null, "adapter_id": "harness/specs/deepseek-flash.json:jevbench.adapters.openai_compat.OpenAICompatAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1587 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 93.69043763543172, "calibration": 96.91785139608032, "speed": 69.39902343452796, "cost": 19.093609002972528 }, "scores": { "A": 6.645505039231519, "B": 7.406272143196756, "C": 6.645505039231519 }, "views": { "Balanced 33:33:33": 5.648067608779863, "Emphasis on Accuracy 60:20:20": 7.3796291044149225, "Emphasis on Cost 20:20:60": 4.001744057440232, "Emphasis on Speed 20:60:20": 6.860785464950723, "Intelligence only": 13.662535665508406 }, "composite_ci95": { "A": [ 6.624148909040629, 6.663488366039578 ], "B": [ 7.368735537079963, 7.438725543018248 ], "C": [ 6.624148909040629, 6.663488366039578 ] }, "jevbench_score": 6.645505039231519, "intelligence": { "I_open": 91.84671609723216, "I_sealed": 95.53415917363128, "base": 93.69043763543172, "gap": -3.68744307639912, "excess": -8.874070576478363, "penalty": 1, "per_type_split": { "open|choice": { "cc": 96.20131099896392, "tiers": { "easy": 96.38227313837805, "standard": 93.76137249805042, "judge": 97.9253112033195, "hard": 96.08303956130042 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 86.17465650781632, "tiers": { "easy": 100, "standard": 93.93939393939394, "judge": 86.27450980392157, "hard": 78.76106194690264 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 93.16418078491628, "tiers": { "easy": 100, "standard": 90.99779151943463, "judge": 93.83879423328965, "hard": 92.03246052760616 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 98.23289353352598, "tiers": { "easy": 100, "standard": 100, "judge": 97.34982332155477, "hard": 97.56986634264887 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 96.54545454545455, "tiers": { "easy": 92, "standard": 100, "judge": 96, "hard": 96.36363636363636 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 91.82412944191336, "tiers": { "easy": 99.83695652173913, "standard": 97.22292191435768, "judge": 94.2005054759899, "hard": 85.33924441017734 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.020025031289111372 }, "noul": { "n": 495, "invalid_rate": 0.018181818181818188 }, "score": { "n": 330, "invalid_rate": 0.036363636363636376 } }, "calibration": { "score": 96.91785139608032, "parts": { "choice": { "ece_hard": 0.04581060173808424, "ece_score": 90.83787965238315, "mean_tvd": 0.012719194915753415, "n_tvd": 47, "n": 783, "n_ece_hard": 270, "score": 94.7829800804039 }, "noul": { "ece": 0.01675268517489696, "brier": 0.019247259932996506, "n": 486, "score": 96.6494629650206, "abstention_rate": 0.012345679012345734 }, "score": { "nrps": 0.00022347944975023944, "top_ece": 0.006677148846960353, "n": 318, "interval90_coverage": 1, "interval90_width": 0.0047169811320754715, "score": 99.32111114281645 } } }, "speed": { "p50_s_raw": 1.7762285992503166, "p95_s_raw": 6.465450605377541, "p50_s_adjusted": 1.7762285992503166, "p95_s_adjusted": 6.465450605377541, "n": 450, "adjustment": "none (hosted API, measured as is)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.4975992302955664, "basis": "ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)" }, "rank": 65, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 65, "B": 63, "C": 62 } }, { "key": "simplejev-qwen3.5-0.8b", "display": "SimpleJev (Qwen3.5-0.8B, CPU)", "addendum": null, "author": "sabeel111 / Featherless AI", "repo": "https://github.com/featherless-ai/simple-jev", "class": "jev-rebuild", "licence": "Apache-2.0 server; Apache-2.0 Qwen3.5-0.8B checkpoint", "open": "yes", "underlying": "Qwen3.5-0.8B through SimpleJev native assistant-prefill option-logit scorer", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/simplejev-qwen3.5-0.8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1617 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 16.7457779502415, "calibration": 46.67276047458846, "speed": 59.07352652626999, "cost": 70.31175519898045 }, "scores": { "A": 3.995571708038271, "B": 3.2605340633066584, "C": 2.7747025750265775 }, "views": { "Balanced 33:33:33": 3.7032739070208662, "Emphasis on Accuracy 60:20:20": 2.66686632240896, "Emphasis on Cost 20:20:60": 4.700645988538198, "Emphasis on Speed 20:60:20": 4.496698803590565, "Intelligence only": 1.8783476496641063 }, "composite_ci95": { "A": [ 2.0618884680283314, 6.661268630105466 ], "B": [ 1.6182691638588302, 5.613466175208515 ], "C": [ 1.431866991686341, 4.625880993128795 ] }, "jevbench_score": 3.995571708038271, "intelligence": { "I_open": 13.326965757246013, "I_sealed": 20.164590143236985, "base": 16.7457779502415, "gap": -6.837624385990972, "excess": -12.024251886070214, "penalty": 1, "per_type_split": { "open|choice": { "cc": 28.275464895018914, "tiers": { "easy": 85.52909255351221, "standard": 34.4944112295295, "judge": 19.0871369294606, "hard": 17.74383078730903 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -12.834066852807085, "tiers": { "easy": -5.882352941176472, "standard": -3.0303030303030276, "judge": -47.05882352941176, "hard": 6.194690265486735 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 24.539499229526214, "tiers": { "easy": -1.6354458020304774, "standard": 23.475110530688738, "judge": 16.31938444589648, "hard": 37.780515924556425 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 27.771564822999164, "tiers": { "easy": 44.335937499999986, "standard": 28.00543355218476, "judge": 31.09540636042403, "hard": 21.0206561360875 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 4.254545454545452, "tiers": { "easy": 36.00000000000001, "standard": 12.00000000000001, "judge": 15.999999999999993, "hard": -16.36363636363637 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 28.46766015216634, "tiers": { "easy": 40.82324942776366, "standard": 35.51984055110081, "judge": 24.57770993391708, "hard": 24.770135297486707 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.007509386733416723 }, "noul": { "n": 495, "invalid_rate": 0.002020202020202033 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 46.67276047458846, "parts": { "choice": { "ece_hard": 0.20237649586807754, "ece_score": 59.52470082638449, "mean_tvd": 0.29457199268122575, "n_tvd": 50, "n": 793, "n_ece_hard": 271, "score": 65.03375077913095 }, "noul": { "ece": 0.5196194263761825, "brier": 0.5189770017458518, "n": 494, "score": 0, "abstention_rate": 0 }, "score": { "nrps": 0.2023654034513244, "top_ece": 0.1489719918279937, "n": 330, "interval90_coverage": 0.8575757575757575, "interval90_width": 0.6909090909090907, "score": 74.98453064463442 } } }, "speed": { "p50_s_raw": 3.6104603729909286, "p95_s_raw": 8.321448143641462, "p50_s_adjusted": 7.3709207459818575, "p95_s_adjusted": 16.79289628728292, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.009763559113300492, "basis": "documented hosted-model estimate" }, "rank": 66, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 66, "B": 67, "C": 67 } }, { "key": "simplejev-qwen3.8-27b", "display": "SimpleJev Qwen3.8-27B", "addendum": null, "author": "Featherless AI", "repo": "https://github.com/featherless-ai/simple-jev", "class": "jev-rebuild", "licence": "Apache-2.0 (Qwen weights); repository licence not stated", "open": "yes", "underlying": "Qwen3.8-27B through SimpleJev's direct-logit classifier", "endpoint_kind": "demo", "endpoint_condition": "the author's public demo endpoint", "gpu": null, "adapter_id": "harness/specs/simplejev-qwen3.8-27b.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 72.75482343968588, "calibration": 87.1969052856583, "speed": 74.91530661775406, "cost": 14.895025990507506 }, "scores": { "A": 3.3584255994370715, "B": 3.7149465104642747, "C": 3.3584255994370715 }, "views": { "Balanced 33:33:33": 2.8253739760349332, "Emphasis on Accuracy 60:20:20": 3.645466466370485, "Emphasis on Cost 20:20:60": 1.9418744899793077, "Emphasis on Speed 20:60:20": 3.669365485962498, "Intelligence only": 6.456606413207646 }, "composite_ci95": { "A": [ 3.3298853690492023, 3.3731363629589097 ], "B": [ 3.6686384808568335, 3.7449217201789837 ], "C": [ 3.3298853690492023, 3.3731363629589097 ] }, "jevbench_score": 3.3584255994370715, "intelligence": { "I_open": 71.90840452793111, "I_sealed": 73.60124235144063, "base": 72.75482343968588, "gap": -1.692837823509521, "excess": -6.8794653235887635, "penalty": 1, "per_type_split": { "open|choice": { "cc": 85.42687050046183, "tiers": { "easy": 98.19113656918904, "standard": 87.52274499610087, "judge": 94.81327800829875, "hard": 74.14806110458284 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 51.10091019513195, "tiers": { "easy": 82.35294117647058, "standard": 60.606060606060595, "judge": 58.823529411764696, "hard": 32.74336283185841 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 79.19743288819956, "tiers": { "easy": 79.17508201475533, "standard": 84.86964706071151, "judge": 82.73204775623967, "hard": 73.71595236927453 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 80.67343761665602, "tiers": { "easy": 88.28125, "standard": 82.34095539959247, "judge": 78.79858657243817, "hard": 79.34386391251519 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 60.94545454545455, "tiers": { "easy": 76, "standard": 63.99999999999999, "judge": 60.00000000000001, "hard": 56.363636363636374 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 79.18483489221134, "tiers": { "easy": 82.34156829293816, "standard": 78.53194588489751, "judge": 79.57425039229116, "hard": 78.4300344206267 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 87.1969052856583, "parts": { "choice": { "ece_hard": 0.03969125406621597, "ece_score": 92.0617491867568, "mean_tvd": 0.3276173629231819, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 79.65000644721931 }, "noul": { "ece": 0.0371275107796666, "brier": 0.0685308410661177, "n": 495, "score": 92.57449784406668, "abstention_rate": 0.19393939393939397 }, "score": { "nrps": 0.043365328698605106, "top_ece": 0.08465521999380805, "n": 330, "interval90_coverage": 0.990909090909091, "interval90_width": 0.2717171717171717, "score": 89.36621156568894 } } }, "speed": { "p50_s_raw": 0.8395581245422363, "p95_s_raw": 0.9601930942386389, "p50_s_adjusted": 1.6791162490844727, "p95_s_adjusted": 1.9203861884772777, "n": 450, "adjustment": "x2 demo-endpoint adjustment (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.686803448275862, "basis": "ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)" }, "rank": 67, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 67, "B": 66, "C": 66 } }, { "key": "decision-machine-1", "display": "decision-machine-1 (milliseconds.ai)", "addendum": null, "author": "milliseconds.ai (Baptiste Laget)", "repo": "https://www.milliseconds.ai", "class": "decision-api", "licence": "proprietary API, closed weights", "open": "no", "underlying": "decision-machine-1 (weights not published)", "endpoint_kind": "api", "endpoint_condition": "the operator's hosted API", "gpu": null, "adapter_id": "harness/specs/decision-machine-1.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 14.816438176464153, "calibration": 81.18169307356939, "speed": 92.77166935344088, "cost": 56.297500611451596 }, "scores": { "A": 3.2416665489183343, "B": 2.4968194273457973, "C": 2.2511573256377324 }, "views": { "Balanced 33:33:33": 2.7430986994891735, "Emphasis on Accuracy 60:20:20": 1.9005028454194188, "Emphasis on Cost 20:20:60": 3.3372862355103208, "Emphasis on Speed 20:60:20": 3.7336769644480805, "Intelligence only": 1.3010423425783209 }, "composite_ci95": { "A": [ 1.730049026826611, 5.392829369228498 ], "B": [ 1.2909980314827887, 4.283648012064621 ], "C": [ 1.2014229352962575, 3.745020395297569 ] }, "jevbench_score": 3.2416665489183343, "intelligence": { "I_open": 27.50038667381044, "I_sealed": 5.129340447229145, "base": 16.314863560519793, "gap": 22.371046226581292, "excess": 17.18441872650205, "penalty": 0.9081558127349795, "per_type_split": { "open|choice": { "cc": 53.2654927384921, "tiers": { "easy": 89.14681941513415, "standard": 53.21029373537822, "judge": 71.99170124481329, "hard": 30.27810419114765 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -17.549256227028224, "tiers": { "easy": 52.941176470588225, "standard": -51.515151515151516, "judge": 37.254901960784316, "hard": -59.29203539823009 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 46.78492350996744, "tiers": { "easy": 29.733926322635995, "standard": 61.068096883467916, "judge": 44.82424292253913, "hard": 45.37659656062129 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 44.942650158916166, "tiers": { "easy": 64.84374999999999, "standard": 48.381254244962655, "judge": 40.37102473498235, "hard": 41.67679222357231 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -67.96363636363635, "tiers": { "easy": -43.99999999999999, "standard": -68, "judge": -72, "hard": -70.9090909090909 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 38.40900754640762, "tiers": { "easy": 51.11240578541303, "standard": 49.00817958829449, "judge": 34.66912890206911, "hard": 32.73848094896671 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 81.18169307356939, "parts": { "choice": { "ece_hard": 0.08149957836600445, "ece_score": 83.7000843267991, "mean_tvd": 0.23287647233513212, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 80.20621854664294 }, "noul": { "ece": 0.10654949494949495, "brier": 0.2044606949494951, "n": 495, "score": 78.69010101010102, "abstention_rate": 0.6060606060606061 }, "score": { "nrps": 0.14849255922750398, "top_ece": 0.07926612374660588, "n": 330, "interval90_coverage": 0.906060606060606, "interval90_width": 0.6166666666666666, "score": 84.64875966396421 } } }, "speed": { "p50_s_raw": 0.18023579940199852, "p95_s_raw": 0.29308393076062134, "p50_s_adjusted": 0.18023579940199852, "p95_s_adjusted": 0.29308393076062134, "n": 450, "adjustment": "none (hosted API, measured as is)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.02862529556650246, "basis": "operator standard launch list price (interpretation I-1); no exact base-model floor applies" }, "rank": 68, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 68, "B": 68, "C": 68 } }, { "key": "gliner2.5-multi", "display": "GLiNER2.5 multi (Fastino, 287M)", "addendum": null, "author": "Fastino", "repo": "https://huggingface.co/fastino/gliner2.5-multi-v1", "class": "classifier", "licence": "Apache-2.0", "open": "yes", "underlying": "GLiNER2.5 multilingual schema-conditioned extractor, 287M", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/gliner2.5-multi/spec.json:warm_adapter.WarmLoaded", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 13.995189304514808, "calibration": 57.96384531072885, "speed": 66.5730877741724, "cost": 86.62313527343974 }, "scores": { "A": 2.7186855157112975, "B": 2.0979157487811895, "C": 1.88797605257729 }, "views": { "Balanced 33:33:33": 2.3978989928514545, "Emphasis on Accuracy 60:20:20": 1.6259462759138177, "Emphasis on Cost 20:20:60": 3.234583320343537, "Emphasis on Speed 20:60:20": 3.0589465345482507, "Intelligence only": 1.096468913176239 }, "composite_ci95": { "A": [ 1.196852251293987, 5.101047430526342 ], "B": [ 0.8901107605029109, 4.071135024636648 ], "C": [ 0.8311473967319353, 3.542394048976626 ] }, "jevbench_score": 2.7186855157112975, "intelligence": { "I_open": 15.77937087068345, "I_sealed": 12.211007738346165, "base": 13.995189304514808, "gap": 3.5683631323372857, "excess": -1.6182643677419568, "penalty": 1, "per_type_split": { "open|choice": { "cc": 19.436913892915463, "tiers": { "easy": 78.29363883026832, "standard": 29.815440603067323, "judge": 5.601659751037364, "hard": 9.909909909909892 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 0.15995456911646766, "tiers": { "easy": 23.529411764705888, "standard": -9.090909090909093, "judge": 23.529411764705888, "hard": -18.58407079646017 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 27.741244150018424, "tiers": { "easy": -5.203924413066585, "standard": 42.01767546895648, "judge": 33.347933189662704, "hard": 24.634303851587436 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 17.23700918907234, "tiers": { "easy": 32.61718749999999, "standard": 10.346388951777246, "judge": 16.519434628975272, "hard": 17.375455650060772 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -2.5818181818181833, "tiers": { "easy": 4.0000000000000036, "standard": -4.0000000000000036, "judge": 0, "hard": -5.454545454545457 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 21.977832207784342, "tiers": { "easy": 32.451026939341574, "standard": 22.802673394719065, "judge": 13.23783973949041, "hard": 25.502107282648122 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 57.96384531072885, "parts": { "choice": { "ece_hard": 0.19331665541505985, "ece_score": 61.33666891698803, "mean_tvd": 0.3324832180692498, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 64.04417355503152 }, "noul": { "ece": 0.3081204742930519, "brier": 0.3316772483381491, "n": 495, "score": 38.37590514138962, "abstention_rate": 0.18585858585858583 }, "score": { "nrps": 0.20624840028309196, "top_ece": 0.18216122750079994, "n": 330, "interval90_coverage": 0.896969696969697, "interval90_width": 0.8267676767676767, "score": 71.47145723576541 } } }, "speed": { "p50_s_raw": 0.6099890284240246, "p95_s_raw": 7.9592920757830115, "p50_s_adjusted": 1.369978056848049, "p95_s_adjusted": 16.06858415156602, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.002791871921182266, "basis": "ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies" }, "rank": 69, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 69, "B": 69, "C": 69 } }, { "key": "gliner2", "display": "GLiNER2 (Fastino, gliner2.5-base)", "addendum": null, "author": "Fastino", "repo": "https://github.com/fastino-ai/GLiNER2", "class": "classifier", "licence": "Apache-2.0", "open": "yes", "underlying": "DeBERTa-v3-base schema-conditioned extractor (GLiNER2.5 boundary architecture), 194M", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/gliner2/spec.json:warm_adapter.WarmLoaded", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 13.475160169985848, "calibration": 35.566185234425355, "speed": 70.33059310938175, "cost": 86.62313527343974 }, "scores": { "A": 2.2681537932105402, "B": 1.7951492831599238, "C": 1.575106800840653 }, "views": { "Balanced 33:33:33": 2.1795380936293123, "Emphasis on Accuracy 60:20:20": 1.462027588988605, "Emphasis on Cost 20:20:60": 2.951032720407862, "Emphasis on Speed 20:60:20": 2.8281151639239033, "Intelligence only": 0.9787275187231775 }, "composite_ci95": { "A": [ 0.9029935487473513, 4.50206881847512 ], "B": [ 0.682640869013673, 3.698158199520721 ], "C": [ 0.6270788532967718, 3.1264366794966114 ] }, "jevbench_score": 2.2681537932105402, "intelligence": { "I_open": 17.24017625553533, "I_sealed": 9.710144084436365, "base": 13.475160169985848, "gap": 7.530032171098966, "excess": 2.343404671019724, "penalty": 1, "per_type_split": { "open|choice": { "cc": 24.79595717384202, "tiers": { "easy": 81.91136569189027, "standard": 40.73303873147908, "judge": 8.71369294605811, "hard": 14.61026243634937 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 1.5045825248844507, "tiers": { "easy": 41.176470588235304, "standard": -21.212121212121215, "judge": 25.49019607843137, "hard": -15.04424778761062 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 25.419989067879534, "tiers": { "easy": 3.1212615250359788, "standard": 53.252372656757444, "judge": 27.812826480825635, "hard": 15.283851099441891 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 20.989344151513734, "tiers": { "easy": 32.61718749999999, "standard": 28.00543355218476, "judge": 20.49469964664312, "hard": 14.94532199270962 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -10, "tiers": { "easy": 28.000000000000004, "standard": -24, "judge": 0, "hard": -19.999999999999996 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 18.14108810179536, "tiers": { "easy": 31.15651420104033, "standard": 19.75534412338449, "judge": 9.97925546483749, "hard": 20.20147804390796 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 35.566185234425355, "parts": { "choice": { "ece_hard": 0.4157658562010376, "ece_score": 16.846828759792476, "mean_tvd": 0.4761788956904523, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 34.61446959537363 }, "noul": { "ece": 0.3247925844352796, "brier": 0.34914475920123894, "n": 495, "score": 35.041483112944086, "abstention_rate": 0.18181818181818177 }, "score": { "nrps": 0.25914794010083303, "top_ece": 0.502127007914312, "n": 330, "interval90_coverage": 0.5121212121212121, "interval90_width": 0.22702020202020204, "score": 37.04260299495835 } } }, "speed": { "p50_s_raw": 0.4240573085844517, "p95_s_raw": 4.567268735170364, "p50_s_adjusted": 0.9981146171689034, "p95_s_adjusted": 9.284537470340728, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.002791871921182266, "basis": "ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies" }, "rank": 70, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 70, "B": 70, "C": 70 } }, { "key": "jevact", "display": "JevAct (einptein, jev1-2b-v2)", "addendum": null, "author": "einptein", "repo": "https://github.com/fstandhartinger/jevbench/issues/66", "class": "jev-rebuild", "licence": "not stated (the author published only an endpoint and the evaluation client)", "open": "no", "underlying": "the author's own checkpoint model/jev1-2b-v2/checkpoint-45000-67.53 over a Qwen3.5-2B base, served by the author on his own machine; one request per decision returns a native probability for every offered option", "endpoint_kind": "demo", "endpoint_condition": "the author's public demo endpoint", "gpu": null, "adapter_id": "harness/specs/jevact.json:jevbench.adapters.jevact_api.JevActApiAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1496 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 11.23787871976056, "calibration": 62.54294670001093, "speed": 76.43004589872001, "cost": 68.56133553360087 }, "scores": { "A": 1.523364886742046, "B": 1.139657303464623, "C": 1.0578922824597539 }, "views": { "Balanced 33:33:33": 1.2991218913085527, "Emphasis on Accuracy 60:20:20": 0.8572965832446319, "Emphasis on Cost 20:20:60": 1.7320742174222945, "Emphasis on Speed 20:60:20": 1.768496074540878, "Intelligence only": 0.5676923133446331 }, "composite_ci95": { "A": [ 0.5244924655946405, 3.1790628916081416 ], "B": [ 0.37525621735250697, 2.472834711495382 ], "C": [ 0.36423087888516703, 2.207682563616765 ] }, "jevbench_score": 1.523364886742046, "intelligence": { "I_open": 19.718374313549337, "I_sealed": 3.4677959832723193, "base": 11.593085148410829, "gap": 16.250578330277015, "excess": 11.063950830197772, "penalty": 0.9693604916980223, "per_type_split": { "open|choice": { "cc": 31.064163034954287, "tiers": { "easy": 85.52909255351221, "standard": 45.41200935794125, "judge": 29.460580912863083, "hard": 11.476694085389722 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -10.650545012856309, "tiers": { "easy": 29.41176470588236, "standard": -21.212121212121215, "judge": 31.372549019607842, "hard": -46.90265486725663 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 38.741504918550035, "tiers": { "easy": 10.57001976596731, "standard": 49.03532322866402, "judge": 47.75306339910301, "hard": 33.87879819122398 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 35.28176715628971, "tiers": { "easy": 56.054687499999986, "standard": 32.08059769074034, "judge": 28.44522968197881, "hard": 36.816524908869994 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -55.345454545454544, "tiers": { "easy": -52, "standard": -60, "judge": -52, "hard": -56.363636363636374 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 30.467075338981786, "tiers": { "easy": 51.70725102450335, "standard": 34.736082929723736, "judge": 28.810676983883265, "hard": 24.264826388554305 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.12515644555694616 }, "noul": { "n": 495, "invalid_rate": 0.0444444444444444 }, "score": { "n": 330, "invalid_rate": 0.018181818181818188 } }, "calibration": { "score": 62.54294670001093, "parts": { "choice": { "ece_hard": 0.21664219000489238, "ece_score": 56.67156199902153, "mean_tvd": 0.35928157130262306, "n_tvd": 42, "n": 699, "n_ece_hard": 207, "score": 60.371702434379614 }, "noul": { "ece": 0.23128113451258264, "brier": 0.29823669613993214, "n": 473, "score": 53.74377309748347, "abstention_rate": 0.39746300211416485 }, "score": { "nrps": 0.1749893478993819, "top_ece": 0.17737168036861187, "n": 324, "interval90_coverage": 0.8950617283950617, "interval90_width": 0.6769547325102881, "score": 73.51336456816972 } } }, "speed": { "p50_s_raw": 0.4011184424161911, "p95_s_raw": 1.4179561108350751, "p50_s_adjusted": 0.8022368848323822, "p95_s_adjusted": 2.8359122216701502, "n": 450, "adjustment": "x2 demo-endpoint adjustment (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.011167487684729064, "basis": "ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies" }, "rank": 71, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 71, "B": 72, "C": 71 } }, { "key": "clm-8b", "display": "CLM-8B (Contrastive-LM, clm-latest)", "addendum": null, "author": "Contrastive-LM (Kwok, Kang, Suresh, Saad-Falcon, Pavone, Ré, Mirhoseini)", "repo": "https://github.com/Contrastive-LM/CLM", "class": "system-one-open", "licence": "Apache-2.0 (code and CLM-v0.1-8B head); Qwen3-8B encoder Apache-2.0", "open": "yes", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "gpu": "RTXPRO6000", "adapter_id": "images/clm-8b/spec.json:clm_adapter.CLMAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 11.4306380303076, "calibration": 48.48351103482105, "speed": 93.28574041500174, "cost": 50.501662240594015 }, "scores": { "A": 1.5079991770819052, "B": 1.1556905102757526, "C": 1.0472216507513232 }, "views": { "Balanced 33:33:33": 1.328681296240889, "Emphasis on Accuracy 60:20:20": 0.8919539241243533, "Emphasis on Cost 20:20:60": 1.6580320036736744, "Emphasis on Speed 20:60:20": 1.8739962877475922, "Intelligence only": 0.5974085148705298 }, "composite_ci95": { "A": [ 0.4792660492923253, 3.1684642573866175 ], "B": [ 0.34885034913271257, 2.51564375381825 ], "C": [ 0.3328236453418925, 2.2003224009629285 ] }, "jevbench_score": 1.5079991770819052, "intelligence": { "I_open": 17.087609508025615, "I_sealed": 5.773666552589584, "base": 11.4306380303076, "gap": 11.31394295543603, "excess": 6.127315455356787, "penalty": 1, "per_type_split": { "open|choice": { "cc": 19.35896876830928, "tiers": { "easy": 40.30750678323787, "standard": 9.539901221731215, "judge": 33.609958506224075, "hard": 8.343125734430064 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 6.948716735286226, "tiers": { "easy": 17.647058823529417, "standard": 6.060606060606055, "judge": 45.09803921568627, "hard": -23.893805309734518 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 24.95514302048134, "tiers": { "easy": 8.445647112288901, "standard": 35.35112046654424, "judge": 23.00066710619347, "hard": 25.3503852102139 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 8.287901229781768, "tiers": { "easy": 0.39062499999998146, "standard": 4.912836767036478, "judge": 20.49469964664312, "hard": 2.7946537059538477 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -13.490909090909096, "tiers": { "easy": -4.0000000000000036, "standard": -16.000000000000004, "judge": -16.000000000000004, "hard": -12.727272727272732 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 22.52400751889608, "tiers": { "easy": 24.610220611754297, "standard": 27.203564617800758, "judge": 19.93825422047939, "hard": 21.601990670041705 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 48.48351103482105, "parts": { "choice": { "ece_hard": 0.37154133110131926, "ece_score": 25.691733779736147, "mean_tvd": 0.4915353240795216, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 38.26910068589199 }, "noul": { "ece": 0.2160041611187395, "brier": 0.2602310957635408, "n": 495, "score": 56.799167776252105, "abstention_rate": 0.296969696969697 }, "score": { "nrps": 0.23115691335442487, "top_ece": 0.38059889689959703, "n": 330, "interval90_coverage": 0.7181818181818181, "interval90_width": 0.5527777777777781, "score": 50.38226464231905 } } }, "speed": { "p50_s_raw": 0.016555813956074417, "p95_s_raw": 0.0531386190210469, "p50_s_adjusted": 0.18311162791214883, "p95_s_adjusted": 0.2562772380420938, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.044662660714285714, "basis": "price floor: base-model reference price applied" }, "rank": 72, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 72, "B": 71, "C": 72 } }, { "key": "kev-0.6b", "display": "kev 0.6B (research preview)", "addendum": null, "author": "Jared Palmer", "repo": "https://github.com/jaredpalmer/kev", "class": "jev-rebuild", "licence": "Apache-2.0", "open": true, "underlying": "Qwen3-0.6B-Base + LoRA + learned pointer head; jaredpalmer/kev-0.6b", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/kev-0.6b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 10.335956592880297, "calibration": 67.65469262034678, "speed": 87.21666696915125, "cost": 80.09400261254245 }, "scores": { "A": 1.2616561727973594, "B": 0.920048498843254, "C": 0.8761501199981665 }, "views": { "Balanced 33:33:33": 1.0621184537440649, "Emphasis on Accuracy 60:20:20": 0.6800256161399014, "Emphasis on Cost 20:20:60": 1.4667540546597662, "Emphasis on Speed 20:60:20": 1.487578762118429, "Intelligence only": 0.4416843604758039 }, "composite_ci95": { "A": [ 0.4895664854415589, 2.5047442312337607 ], "B": [ 0.34458355059432716, 1.8834459713121963 ], "C": [ 0.33997672600108253, 1.739405716134556 ] }, "jevbench_score": 1.2616561727973594, "intelligence": { "I_open": 25.463748221534246, "I_sealed": -1.491216071552195, "base": 11.986266074991025, "gap": 26.95496429308644, "excess": 21.768336793007197, "penalty": 0.862316632069928, "per_type_split": { "open|choice": { "cc": 33.250533580633615, "tiers": { "easy": 85.52909255351221, "standard": 53.21029373537822, "judge": 29.460580912863083, "hard": 13.043478260869541 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 1.9626772672061623, "tiers": { "easy": 41.176470588235304, "standard": -21.212121212121215, "judge": 41.176470588235304, "hard": -25.663716814159287 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 41.17803381676297, "tiers": { "easy": 37.126562886703404, "standard": 56.85277405832456, "judge": 39.56389282440696, "hard": 35.564137172764056 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 21.314071279675943, "tiers": { "easy": 50.1953125, "standard": 23.930269413629183, "judge": 15.194346289752657, "hard": 17.375455650060772 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -57.6, "tiers": { "easy": -43.99999999999999, "standard": -56.00000000000001, "judge": -60, "hard": -60 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 31.812280505667474, "tiers": { "easy": 43.407238368764425, "standard": 37.62475155995868, "judge": 28.11632890331999, "hard": 28.77926921450824 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.003754693366708417 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0.0060606060606061 } }, "calibration": { "score": 67.65469262034678, "parts": { "choice": { "ece_hard": 0.22409167003656888, "ece_score": 55.18166599268623, "mean_tvd": 0.3545293454678935, "n_tvd": 50, "n": 796, "n_ece_hard": 276, "score": 59.864365722948435 }, "noul": { "ece": 0.18842424242424205, "brier": 0.2551985858585859, "n": 495, "score": 62.31515151515159, "abstention_rate": 0.42828282828282827 }, "score": { "nrps": 0.1609289678519071, "top_ece": 0.1116899098446431, "n": 328, "interval90_coverage": 0.948170731707317, "interval90_width": 0.7632113821138208, "score": 80.78456062294033 } } }, "speed": { "p50_s_raw": 0.13577913912013173, "p95_s_raw": 0.15013638343662017, "p50_s_adjusted": 0.4215582782402635, "p95_s_adjusted": 0.45027276687324036, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.004608220443349753, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 73, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 73, "B": 73, "C": 73 } }, { "key": "raw-qwen3-0.6b", "display": "Raw Qwen3 0.6B direct logits", "addendum": null, "author": "Alibaba Qwen / neutral reproduction", "repo": "https://huggingface.co/Qwen/Qwen3-0.6B", "class": "raw-logit-control", "licence": "Apache-2.0", "open": "yes", "underlying": "Qwen3-0.6B BF16", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/raw-qwen3-0.6b/spec.json:raw_adapter.RawControlAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 10.561366022298714, "calibration": 21.45426398042649, "speed": 90.39019014190508, "cost": 77.58073177551512 }, "scores": { "A": 1.079997575465549, "B": 0.8582394645786061, "C": 0.7499983162955202 }, "views": { "Balanced 33:33:33": 1.1282330954187632, "Emphasis on Accuracy 60:20:20": 0.7242849176442601, "Emphasis on Cost 20:20:60": 1.5447246246138013, "Emphasis on Speed 20:60:20": 1.5848154665864036, "Intelligence only": 0.47121626612423784 }, "composite_ci95": { "A": [ 0.27446277660594287, 2.612621391882716 ], "B": [ 0.20450823859446796, 2.166446961168533 ], "C": [ 0.1905991504207937, 1.8143204110296636 ] }, "jevbench_score": 1.079997575465549, "intelligence": { "I_open": 10.0105735585918, "I_sealed": 11.112158486005626, "base": 10.561366022298714, "gap": -1.1015849274138265, "excess": -6.288212427493069, "penalty": 1, "per_type_split": { "open|choice": { "cc": 17.1202876731998, "tiers": { "easy": 80.10250226107928, "standard": 31.37509747855471, "judge": 0.4149377593361205, "hard": 6.776341558950235 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 13.422302146924743, "tiers": { "easy": 11.764705882352944, "standard": 3.0303030303030276, "judge": 47.058823529411775, "hard": -6.194690265486724 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": -0.510869144349142, "tiers": { "easy": -9.384680024557767, "standard": -5.394614995523739, "judge": -16.620508057527793, "hard": 16.231685686174302 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 23.798922438502146, "tiers": { "easy": 50.1953125, "standard": 17.138329182703217, "judge": 23.144876325088344, "hard": 21.0206561360875 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -4.254545454545457, "tiers": { "easy": -36, "standard": -12, "judge": -16.000000000000004, "hard": 16.36363636363636 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 13.792098474060186, "tiers": { "easy": 14.761469685852314, "standard": 12.256011452425764, "judge": 16.30759642915529, "hard": 12.431175715608035 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 21.45426398042649, "parts": { "choice": { "ece_hard": 0.5472022432112024, "ece_score": 0, "mean_tvd": 0.5290557394596033, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 23.547213027019836 }, "noul": { "ece": 0.46457443043863456, "brier": 0.4635923548486851, "n": 495, "score": 7.085113912273089, "abstention_rate": 0.004040404040404066 }, "score": { "nrps": 0.3253906999602692, "top_ece": 0.5113738710605674, "n": 330, "interval90_coverage": 0.5363636363636364, "interval90_width": 0.3712121212121212, "score": 33.730465001986545 } } }, "speed": { "p50_s_raw": 0.06320727849379182, "p95_s_raw": 0.09034462417475878, "p50_s_adjusted": 0.27641455698758366, "p95_s_adjusted": 0.3306892483495175, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.005588676108374384, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 74, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 74, "B": 74, "C": 74 } }, { "key": "gliner2.5-small", "display": "GLiNER2.5 small (Fastino, 74M)", "addendum": null, "author": "Fastino", "repo": "https://huggingface.co/fastino/gliner2.5-small-v1", "class": "classifier", "licence": "Apache-2.0", "open": "yes", "underlying": "GLiNER2.5 small schema-conditioned extractor, 74M", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/gliner2.5-small/spec.json:warm_adapter.WarmLoaded", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 9.18733556299694, "calibration": 55.90409648375797, "speed": 77.35928845375412, "cost": 86.62313527343974 }, "scores": { "A": 0.893172274587778, "B": 0.6491615526073343, "C": 0.6202585240192903 }, "views": { "Balanced 33:33:33": 0.7597604657587358, "Emphasis on Accuracy 60:20:20": 0.4809421780175859, "Emphasis on Cost 20:20:60": 1.0793408837724145, "Emphasis on Speed 20:60:20": 1.0605921103009057, "Intelligence only": 0.31019066833311315 }, "composite_ci95": { "A": [ 0.23120097407975104, 2.1950672601130257 ], "B": [ 0.1602676167296053, 1.6603502833457342 ], "C": [ 0.1605562319998271, 1.5243522639673786 ] }, "jevbench_score": 0.893172274587778, "intelligence": { "I_open": 17.164169957518137, "I_sealed": 1.648634129566223, "base": 9.40640204354218, "gap": 15.515535827951915, "excess": 10.328908327872671, "penalty": 0.9767109167212733, "per_type_split": { "open|choice": { "cc": 15.71439352604785, "tiers": { "easy": 69.24932167621344, "standard": 25.136469976605138, "judge": 6.639004149377612, "hard": 4.426165295730488 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 4.308204375877466, "tiers": { "easy": 17.647058823529417, "standard": -24.242424242424242, "judge": 47.058823529411775, "hard": -16.814159292035402 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 31.4699119706291, "tiers": { "easy": 12.58206671454689, "standard": 38.491751312752264, "judge": 34.830715051559395, "hard": 30.160351302890355 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 11.865639447234665, "tiers": { "easy": 23.828124999999982, "standard": 13.063165044147636, "judge": 4.593639575971737, "hard": 13.730255164034043 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -34.10909090909091, "tiers": { "easy": -43.99999999999999, "standard": -52, "judge": -28.000000000000004, "hard": -27.27272727272727 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 27.189353850554916, "tiers": { "easy": 35.38743812791506, "standard": 28.853182750666086, "judge": 23.661402666683927, "hard": 26.953881719062533 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 55.90409648375797, "parts": { "choice": { "ece_hard": 0.14635085613073423, "ece_score": 70.72982877385316, "mean_tvd": 0.3114951726171251, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 69.79015575607032 }, "noul": { "ece": 0.37137972333206415, "brier": 0.3843988929195394, "n": 495, "score": 25.72405533358717, "abstention_rate": 0.19191919191919193 }, "score": { "nrps": 0.2018497964611113, "top_ece": 0.1770943181532802, "n": 330, "interval90_coverage": 0.8787878787878788, "interval90_width": 0.7929292929292928, "score": 72.19807836161641 } } }, "speed": { "p50_s_raw": 0.16102993860840797, "p95_s_raw": 1.8705574963241733, "p50_s_adjusted": 0.47205987721681597, "p95_s_adjusted": 3.8911149926483466, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.002791871921182266, "basis": "ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies" }, "rank": 75, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 75, "B": 76, "C": 75 } }, { "key": "raw-qwen3-1.7b", "display": "Raw Qwen3 1.7B direct logits", "addendum": null, "author": "Alibaba Qwen / neutral reproduction", "repo": "https://huggingface.co/Qwen/Qwen3-1.7B", "class": "raw-logit-control", "licence": "Apache-2.0", "open": "yes", "underlying": "Qwen3-1.7B BF16", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/raw-qwen3-1.7b/spec.json:raw_adapter.RawControlAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 9.67276350131027, "calibration": 21.572869316943088, "speed": 90.22368770839446, "cost": 68.54983190559568 }, "scores": { "A": 0.8534322366651821, "B": 0.6711978961938931, "C": 0.5926612754619321 }, "views": { "Balanced 33:33:33": 0.869979501977677, "Emphasis on Accuracy 60:20:20": 0.5572161102672888, "Emphasis on Cost 20:20:60": 1.1826089384941272, "Emphasis on Speed 20:60:20": 1.2374197554240198, "Intelligence only": 0.36200260818869395 }, "composite_ci95": { "A": [ 0.14697567748045587, 2.284188740424065 ], "B": [ 0.10677588994673629, 1.8948794983135524 ], "C": [ 0.10206644269476102, 1.5862421808500455 ] }, "jevbench_score": 0.8534322366651821, "intelligence": { "I_open": 12.266077251349891, "I_sealed": 7.079449751270649, "base": 9.67276350131027, "gap": 5.186627500079243, "excess": 0, "penalty": 1, "per_type_split": { "open|choice": { "cc": 41.03570373377122, "tiers": { "easy": 72.86704853783539, "standard": 40.73303873147908, "judge": 52.282157676348554, "hard": 24.794359576968255 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 11.657596264571797, "tiers": { "easy": 5.882352941176472, "standard": 3.0303030303030276, "judge": 43.13725490196079, "hard": -6.194690265486724 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": -15.895068244293338, "tiers": { "easy": -33.71272549854139, "standard": -16.03158197732668, "judge": -35.13756131391654, "hard": 3.059472738002744 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 23.292766488000304, "tiers": { "easy": 47.265624999999986, "standard": 28.00543355218476, "judge": 29.77031802120142, "hard": 10.085054678007312 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -3.454545454545456, "tiers": { "easy": -36, "standard": -7.9999999999999964, "judge": -16.000000000000004, "hard": 16.36363636363636 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 1.4001282203570966, "tiers": { "easy": 3.8269954562980213, "standard": 12.432023947196736, "judge": 3.6923303625473913, "hard": -6.4416880586906755 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 21.572869316943088, "parts": { "choice": { "ece_hard": 0.5445983642845623, "ece_score": 0, "mean_tvd": 0.4309651586811289, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 28.451742065943552 }, "noul": { "ece": 0.46974567785208526, "brier": 0.4695975444848059, "n": 495, "score": 6.050864429582948, "abstention_rate": 0 }, "score": { "nrps": 0.39567997089394474, "top_ece": 0.6812260889715819, "n": 330, "interval90_coverage": 0.3393939393939394, "interval90_width": 0.1431818181818182, "score": 30.216001455302766 } } }, "speed": { "p50_s_raw": 0.06600576639175415, "p95_s_raw": 0.09339700550772245, "p50_s_adjusted": 0.2820115327835083, "p95_s_adjusted": 0.3367940110154449, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.011177352216748768, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 76, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 76, "B": 75, "C": 76 } }, { "key": "mirror", "display": "Mirror", "addendum": null, "author": "Bluusun", "repo": "https://github.com/Bluusun/Decision-API", "class": "jev-rebuild", "licence": "Apache-2.0 wrapper and Mirror release; upstream DeBERTa/model assets retain their own terms", "open": "yes", "underlying": "Mirror DeBERTa-v3-large 436M classification-span scorer", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/mirror/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1253 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 5.595966086663697, "calibration": 43.210973211362244, "speed": 63.64487086148672, "cost": 89.27300805786774 }, "scores": { "A": 0.21902679657747523, "B": 0.15370891108333642, "C": 0.15210194206769112 }, "views": { "Balanced 33:33:33": 0.1827590337457119, "Emphasis on Accuracy 60:20:20": 0.11123994452772705, "Emphasis on Cost 20:20:60": 0.2746709218182245, "Emphasis on Speed 20:60:20": 0.2642175207718493, "Intelligence only": 0.07009470509798131 }, "composite_ci95": { "A": [ 0.014890703888250692, 0.9013602095664797 ], "B": [ 0.009792378233239713, 0.6688203363147419 ], "C": [ 0.010340766589062977, 0.6259445899767221 ] }, "jevbench_score": 0.21902679657747523, "intelligence": { "I_open": 6.74523751875524, "I_sealed": 4.4466946545721555, "base": 5.595966086663697, "gap": 2.2985428641830845, "excess": -2.888084635896158, "penalty": 1, "per_type_split": { "open|choice": { "cc": 1.6362965626667982, "tiers": { "easy": 69.24932167621344, "standard": 23.576813101117754, "judge": -3.7344398340248746, "hard": -22.209165687426584 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -0.6142634044768318, "tiers": { "easy": 47.058823529411775, "standard": 0, "judge": 11.764705882352944, "hard": -22.123893805309734 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 19.213679398075755, "tiers": { "easy": -3.1763500509754294, "standard": 27.541232473948863, "judge": 34.02986858621417, "hard": 9.535268331298186 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": -9.371752896132213, "tiers": { "easy": 6.249999999999979, "standard": 4.912836767036478, "judge": -4.681978798586563, "hard": -23.936816524908842 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -4.545454545454547, "tiers": { "easy": 19.999999999999996, "standard": -12, "judge": 8.000000000000007, "hard": -16.36363636363637 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 27.257291405303228, "tiers": { "easy": 38.978075851334324, "standard": 26.205089036101235, "judge": 26.98713233165959, "hard": 25.055815783629175 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.39424280350438046 }, "noul": { "n": 495, "invalid_rate": 0.09696969696969693 }, "score": { "n": 330, "invalid_rate": 0.024242424242424288 } }, "calibration": { "score": 43.210973211362244, "parts": { "choice": { "ece_hard": 0.3664447619629182, "ece_score": 26.71104760741636, "mean_tvd": 0.4119599465354314, "n_tvd": 26, "n": 484, "n_ece_hard": 68, "score": 42.75752647693662 }, "noul": { "ece": 0.40004330431713914, "brier": 0.40198105556852537, "n": 447, "score": 19.991339136572172, "abstention_rate": 0.046979865771812124 }, "score": { "nrps": 0.20196336767408615, "top_ece": 0.2301777759571773, "n": 322, "interval90_coverage": 0.8913043478260869, "interval90_width": 0.8027950310559004, "score": 66.88405402057796 } } }, "speed": { "p50_s_raw": 2.2142462366027758, "p95_s_raw": 4.643026909977197, "p50_s_adjusted": 4.578492473205552, "p95_s_adjusted": 9.436053819954394, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.0022780665024630543, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 77, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 77, "B": 77, "C": 77 } }, { "key": "zerank-2", "display": "ZeroEntropy zerank-2", "addendum": null, "author": "ZeroEntropy", "repo": "https://huggingface.co/zeroentropy/zerank-2-reranker", "class": "reranker", "licence": "Apache-2.0", "open": "yes", "underlying": "Qwen3-Reranker-derived 4B cross-encoder", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/zerank-2/spec.json:reranker_adapter.RerankerAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 4.764817160947821, "calibration": 81.70676532515279, "speed": 80.27703860657243, "cost": 48.39770052432463 }, "scores": { "A": 0.13334937595297605, "B": 0.09147122359399754, "C": 0.0926037333006778 }, "views": { "Balanced 33:33:33": 0.10504941162244104, "Emphasis on Accuracy 60:20:20": 0.06419374610133761, "Emphasis on Cost 20:20:60": 0.14963468464765028, "Emphasis on Speed 20:60:20": 0.15880059700680335, "Intelligence only": 0.04054227910564448 }, "composite_ci95": { "A": [ 0.012109200548825812, 0.48542181877192503 ], "B": [ 0.007905764526842694, 0.3479463361796913 ], "C": [ 0.008409167047795702, 0.33709848525828134 ] }, "jevbench_score": 0.13334937595297605, "intelligence": { "I_open": 9.967956631928743, "I_sealed": -0.43832231003310085, "base": 4.764817160947821, "gap": 10.406278941961844, "excess": 5.219651441882601, "penalty": 1, "per_type_split": { "open|choice": { "cc": 56.57069726813885, "tiers": { "easy": 90.95568284594513, "standard": 65.68754873927736, "judge": 70.95435684647303, "hard": 32.6282804543674 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -68.06887195747166, "tiers": { "easy": -41.17647058823529, "standard": -78.78787878787878, "judge": -74.50980392156863, "hard": -64.60176991150442 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 41.402044585119036, "tiers": { "easy": 8.991045588911728, "standard": 45.39451542604084, "judge": 41.19594702284447, "hard": 47.66313208541588 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 46.961454410449676, "tiers": { "easy": 64.84374999999999, "standard": 45.66447815259227, "judge": 45.67137809187279, "hard": 44.10692588092346 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -85.63636363636364, "tiers": { "easy": -76, "standard": -92, "judge": -80, "hard": -89.0909090909091 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 37.35994229581467, "tiers": { "easy": 48.814492412237634, "standard": 41.111792494891034, "judge": 35.220481360549726, "hard": 34.22497536861945 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 81.70676532515279, "parts": { "choice": { "ece_hard": 0.09639665979788783, "ece_score": 80.72066804042244, "mean_tvd": 0.26735237395866535, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 76.99271532227795 }, "noul": { "ece": 0.08297814786406878, "brier": 0.21755062612673598, "n": 495, "score": 83.40437042718625, "abstention_rate": 0.8363636363636364 }, "score": { "nrps": 0.15146723973315543, "top_ece": 0.07703427787348029, "n": 330, "interval90_coverage": 0.9515151515151515, "interval90_width": 0.8166666666666661, "score": 84.7232102259942 } } }, "speed": { "p50_s_raw": 0.16138588916510344, "p95_s_raw": 0.9172351316083216, "p50_s_adjusted": 0.4727717783302069, "p95_s_adjusted": 1.984470263216643, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.05249000923645321, "basis": "ESTIMATE: size-class proxy (deepinfra:Qwen/Qwen3-Reranker-4B); no exact base-model floor applies" }, "rank": 78, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 78, "B": 78, "C": 78 } }, { "key": "jeff", "display": "jeff (Logan Markewich, GLiFormer 400M)", "addendum": null, "author": "Logan Markewich", "repo": "https://github.com/logan-markewich/jeff", "class": "jev-rebuild", "licence": "MIT (code); GLiFormer weights per their model card", "open": "yes", "underlying": "GLiFormer large (knowledgator/gliformer-large-v1, ~400M) behind a TypeSafe-compatible /v1/systemone server", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/jeff/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1620 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 4.434526796394629, "calibration": 80.26674639624511, "speed": 55.762758324634696, "cost": 81.08682556098348 }, "scores": { "A": 0.11730368929731415, "B": 0.07965896456941576, "C": 0.08146089534535705 }, "views": { "Balanced 33:33:33": 0.09226315098361863, "Emphasis on Accuracy 60:20:20": 0.0556471938321315, "Emphasis on Cost 20:20:60": 0.14024725289968296, "Emphasis on Speed 20:60:20": 0.13486054271524178, "Intelligence only": 0.034882037283846826 }, "composite_ci95": { "A": [ 0.003040184337069344, 0.5116303479621337 ], "B": [ 0.0019501985520175905, 0.36371663996890347 ], "C": [ 0.0021112391229648223, 0.35529885275148176 ] }, "jevbench_score": 0.11730368929731415, "intelligence": { "I_open": 12.711903221404976, "I_sealed": -3.5604602603254047, "base": 4.5757214805397854, "gap": 16.27236348173038, "excess": 11.085735981651137, "penalty": 0.9691426401834886, "per_type_split": { "open|choice": { "cc": 33.014740188512285, "tiers": { "easy": 80.10250226107928, "standard": 48.53132310891603, "judge": 34.64730290456433, "hard": 12.260086173129631 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -28.086381777167823, "tiers": { "easy": 41.176470588235304, "standard": -24.242424242424242, "judge": -21.568627450980394, "hard": -52.212389380530965 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 33.20735125287047, "tiers": { "easy": 19.455069124423964, "standard": 47.54514134275618, "judge": 33.55851900393184, "hard": 29.213150926743193 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 21.788319457161514, "tiers": { "easy": 35.54687499999999, "standard": 32.08059769074034, "judge": 17.84452296819789, "hard": 16.160388821385194 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -60.763636363636365, "tiers": { "easy": -36, "standard": -60, "judge": -56.00000000000001, "hard": -70.9090909090909 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 28.29393612549864, "tiers": { "easy": 30.703170289855073, "standard": 31.066624685138567, "judge": 24.574894692502124, "hard": 29.09456437933695 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.0025031289111389077 }, "noul": { "n": 495, "invalid_rate": 0.004040404040404066 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 80.26674639624511, "parts": { "choice": { "ece_hard": 0.10291963636363644, "ece_score": 79.4160727272727, "mean_tvd": 0.32331250001667494, "n_tvd": 50, "n": 797, "n_ece_hard": 275, "score": 73.5424113628026 }, "noul": { "ece": 0.10606815415821512, "brier": 0.2296393906288032, "n": 493, "score": 78.78636916835697, "abstention_rate": 0.5841784989858012 }, "score": { "nrps": 0.17250476624242422, "top_ece": 0.02903303030303039, "n": 330, "interval90_coverage": 0.9878787878787879, "interval90_width": 0.9186868686868686, "score": 88.47145865757575 } } }, "speed": { "p50_s_raw": 3.4936436004936695, "p95_s_raw": 18.50993296541273, "p50_s_adjusted": 7.137287200987339, "p95_s_adjusted": 37.16986593082546, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.004270110837438424, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 79, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 79, "B": 79, "C": 79 } }, { "key": "smalljev", "display": "smalljev semantic-v9", "addendum": null, "author": "Aditya (isHeSatoshi)", "repo": "https://github.com/isHeSatoshi/smalljev", "class": "jev-rebuild", "licence": "Apache-2.0", "open": "yes", "underlying": "MiniCPM5-2B-Base, 2.5B dense, with LoRA and native semantic decision heads", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/smalljev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 3.659593452439375, "calibration": 73.08838344851645, "speed": 86.46386328362564, "cost": 60.77469204714053 }, "scores": { "A": 0.06803548717120994, "B": 0.04553683400675546, "C": 0.047246866091118014 }, "views": { "Balanced 33:33:33": 0.05334393826915162, "Emphasis on Accuracy 60:20:20": 0.031594465353347985, "Emphasis on Cost 20:20:60": 0.08015154153596153, "Emphasis on Speed 20:60:20": 0.08256726252157968, "Intelligence only": 0.019604623987683183 }, "composite_ci95": { "A": [ 0.00020953618518935847, 0.37562154070461956 ], "B": [ 0.00013237047164362027, 0.2637008130971111 ], "C": [ 0.0001455112397148323, 0.2608482921559858 ] }, "jevbench_score": 0.06803548717120994, "intelligence": { "I_open": 10.933407092326295, "I_sealed": -3.5202907865888893, "base": 3.706558152868703, "gap": 14.453697878915184, "excess": 9.267070378835943, "penalty": 0.9873292962116406, "per_type_split": { "open|choice": { "cc": 31.14027036673812, "tiers": { "easy": 80.10250226107928, "standard": 37.61372498050429, "judge": 36.721991701244825, "hard": 11.476694085389722 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -40.403672329752496, "tiers": { "easy": 41.176470588235304, "standard": -18.181818181818176, "judge": -66.66666666666667, "hard": -52.212389380530965 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 42.06362323999326, "tiers": { "easy": 8.729240242477642, "standard": 50.205876167206775, "judge": 42.3552220235646, "hard": 46.107393438086916 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 18.27162475771627, "tiers": { "easy": 38.47656249999998, "standard": 11.70477699796244, "judge": 13.869257950530042, "hard": 19.805589307411925 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -63.81818181818182, "tiers": { "easy": -36, "standard": -43.99999999999999, "judge": -72, "hard": -74.54545454545455 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 34.98568470069888, "tiers": { "easy": 51.90848980395907, "standard": 38.71916059833131, "judge": 35.11399183902504, "hard": 28.792015122323 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 73.08838344851645, "parts": { "choice": { "ece_hard": 0.1714385460645283, "ece_score": 65.71229078709435, "mean_tvd": 0.38015266094584205, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 63.84851234625506 }, "noul": { "ece": 0.15899537362210042, "brier": 0.24502031712995248, "n": 495, "score": 68.20092527557992, "abstention_rate": 0.6646464646464647 }, "score": { "nrps": 0.1554815062367462, "top_ece": 0.05010211964448292, "n": 330, "interval90_coverage": 0.9606060606060606, "interval90_width": 0.8315656565656561, "score": 87.2157127237144 } } }, "speed": { "p50_s_raw": 0.15031232591718435, "p95_s_raw": 0.17547750547528257, "p50_s_adjusted": 0.45062465183436873, "p95_s_adjusted": 0.5009550109505652, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.020300665024630543, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 80, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 80, "B": 80, "C": 80 } }, { "key": "opendecision", "display": "OpenDecision (ModernBERT-large zero-shot)", "addendum": null, "author": "Deepan Wadhwa", "repo": "https://github.com/deepanwadhwa/OpenDecision", "class": "classifier", "licence": "Apache-2.0", "open": "yes", "underlying": "MoritzLaurer/ModernBERT-large-zeroshot-v2.0 (~400M) used as a zero-shot NLI decision engine", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/opendecision/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 2.0712188139108285, "calibration": 72.52157722321965, "speed": 86.5838203210334, "cost": 79.10704862374335 }, "scores": { "A": 0.013179888010630534, "B": 0.008549163739426961, "C": 0.009152700007382313 }, "views": { "Balanced 33:33:33": 0.010153759421615222, "Emphasis on Accuracy 60:20:20": 0.005826306011620282, "Emphasis on Cost 20:20:60": 0.01611912861066923, "Emphasis on Speed 20:60:20": 0.01618551488693281, "Intelligence only": 0.00355416788559629 }, "composite_ci95": { "A": [ 0, 0.19294313592439996 ], "B": [ 0, 0.13147202005412764 ], "C": [ 0, 0.13398828883638889 ] }, "jevbench_score": 0.013179888010630534, "intelligence": { "I_open": 9.469704530612013, "I_sealed": -5.262248023254946, "base": 2.1037282536785336, "gap": 14.73195255386696, "excess": 9.545325053787717, "penalty": 0.9845467494621228, "per_type_split": { "open|choice": { "cc": 34.171161143191846, "tiers": { "easy": 85.52909255351221, "standard": 43.85235248245385, "judge": 42.946058091286325, "hard": 9.909909909909892 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -47.144006436041835, "tiers": { "easy": -5.882352941176472, "standard": -45.45454545454546, "judge": -64.70588235294117, "hard": -45.13274336283186 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 41.38195888468603, "tiers": { "easy": 4.952451296489713, "standard": 48.63539699723373, "judge": 45.60107321577297, "hard": 43.698280977146055 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 20.037419907604885, "tiers": { "easy": 35.54687499999999, "standard": 17.138329182703217, "judge": 13.869257950530042, "hard": 22.235722964763077 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -70.9090909090909, "tiers": { "easy": -52, "standard": -92, "judge": -68, "hard": -67.27272727272727 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 35.08492693172118, "tiers": { "easy": 42.30553527502682, "standard": 34.52664950313588, "judge": 33.423748326181055, "hard": 34.80479751434251 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 72.52157722321965, "parts": { "choice": { "ece_hard": 0.15882113167094838, "ece_score": 68.23577366581033, "mean_tvd": 0.3451301543433275, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 66.8613791157388 }, "noul": { "ece": 0.14797539912879165, "brier": 0.2628657210726989, "n": 495, "score": 70.40492017424167, "abstention_rate": 0.6646464646464647 }, "score": { "nrps": 0.16442792259044475, "top_ece": 0.11480171490799297, "n": 330, "interval90_coverage": 0.9515151515151515, "interval90_width": 0.747474747474747, "score": 80.29843237967847 } } }, "speed": { "p50_s_raw": 0.07342815498122945, "p95_s_raw": 0.29486368095385834, "p50_s_adjusted": 0.29685630996245893, "p95_s_adjusted": 0.7397273619077167, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.004970862068965518, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 81, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 81, "B": 81, "C": 81 } }, { "key": "bge-reranker-v2-m3", "display": "BAAI bge-reranker-v2-m3", "addendum": null, "author": "BAAI", "repo": "https://huggingface.co/BAAI/bge-reranker-v2-m3", "class": "reranker", "licence": "Apache-2.0", "open": "yes", "underlying": "568M multilingual cross-encoder", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/bge-reranker-v2-m3/spec.json:reranker_adapter.RerankerAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 83.32638523205513, "speed": 90.54571136520154, "cost": 59.325605115070715 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": -21.098300472845224, "I_sealed": -22.804972321036335, "base": -21.951636396940778, "gap": 1.7066718481911103, "excess": -3.4799556518881323, "penalty": 1, "per_type_split": { "open|choice": { "cc": 4.407461395812141, "tiers": { "easy": 18.601145613506187, "standard": 9.539901221731215, "judge": 1.452282157676369, "hard": 0.5092048570309202 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -100, "tiers": { "easy": -100, "standard": -100, "judge": -100, "hard": -100 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 32.29763718565218, "tiers": { "easy": 7.168115899391115, "standard": 33.2624979548925, "judge": 39.87814650710333, "hard": 32.41220513150892 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 4.202706795987319, "tiers": { "easy": 23.828124999999982, "standard": 8.98800090559206, "judge": -2.0318021201413345, "hard": 1.5795868772782722 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -100, "tiers": { "easy": -100, "standard": -100, "judge": -100, "hard": -100 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 27.382376240903668, "tiers": { "easy": 39.25333793379111, "standard": 27.66412381361546, "judge": 26.017247074792348, "hard": 25.297608905909396 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 83.32638523205513, "parts": { "choice": { "ece_hard": 0.03376010460233937, "ece_score": 93.24797907953213, "mean_tvd": 0.33300482957646693, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 79.97374806094271 }, "noul": { "ece": 0.08650888354314805, "brier": 0.2560880956241559, "n": 495, "score": 82.69822329137038, "abstention_rate": 1 }, "score": { "nrps": 0.18556025788685246, "top_ece": 0.03414802761805064, "n": 330, "interval90_coverage": 1, "interval90_width": 1, "score": 87.30718434385231 } } }, "speed": { "p50_s_raw": 0.02942055370658636, "p95_s_raw": 0.1361460106447339, "p50_s_adjusted": 0.20884110741317272, "p95_s_adjusted": 0.42229202128946774, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.022688885467980296, "basis": "ESTIMATE: base-model market reference (deepinfra:BAAI/bge-m3)" }, "rank": 82, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 82, "B": 82, "C": 82 } }, { "key": "certo", "display": "Certo v1 (AltSlate Labs)", "addendum": null, "author": "AltSlate Labs", "repo": "https://huggingface.co/altslate/certo-decision-model", "class": "jev-rebuild", "licence": "MIT", "open": "yes", "underlying": "ModernBERT-large with a per-option query/scoring head, ~400M parameters", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/certo/spec.json:certo_v15.CertoLocalV15Adapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 87.52088082714991, "speed": 91.42520786881005, "cost": 96.9130801359906 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": -24.120274249716694, "I_sealed": -24.930324931123515, "base": -24.525299590420104, "gap": 0.8100506814068211, "excess": -4.376576818672421, "penalty": 1, "per_type_split": { "open|choice": { "cc": -1.8459740781458573, "tiers": { "easy": 5.939101597829376, "standard": -2.9373537821679307, "judge": -5.809128630705371, "hard": -0.2741872307089941 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -100, "tiers": { "easy": -100, "standard": -100, "judge": -100, "hard": -100 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 29.48515132899575, "tiers": { "easy": 11.787672811059903, "standard": 32.471201413427586, "judge": 37.48872870249018, "hard": 26.413812886142974 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": -2.0457518184464063, "tiers": { "easy": -8.39843750000002, "standard": -4.595879556259874, "judge": -4.681978798586563, "hard": 2.7946537059538477 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -100, "tiers": { "easy": -100, "standard": -100, "judge": -100, "hard": -100 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 27.254777025075853, "tiers": { "easy": 38.20869565217393, "standard": 26.696263643996645, "judge": 25.918702611625953, "hard": 25.79760986892836 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 87.52088082714991, "parts": { "choice": { "ece_hard": 0.029833574007220198, "ece_score": 94.03328519855596, "mean_tvd": 0.3310568334499983, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 80.46380092677806 }, "noul": { "ece": 0.034504242424242486, "brier": 0.24896424921212137, "n": 495, "score": 93.0991515151515, "abstention_rate": 1 }, "score": { "nrps": 0.18789589617929317, "top_ece": 0.01605515151515168, "n": 330, "interval90_coverage": 1, "interval90_width": 0.9944444444444445, "score": 88.99969003952017 } } }, "speed": { "p50_s_raw": 0.05686726560816169, "p95_s_raw": 0.061547018261626354, "p50_s_adjusted": 0.2637345312163234, "p95_s_adjusted": 0.2730940365232527, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.0012673522167487683, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 83, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 83, "B": 83, "C": 83 } }, { "key": "decision-fast", "display": "Decision Fast (FlyMy.AI, v53a)", "addendum": null, "author": "FlyMy.AI (@denti)", "repo": "https://huggingface.co/flymy-ai/decision-fast-preview", "class": "jev-rebuild", "licence": "Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release", "open": "yes", "underlying": "Qwen/Qwen3-0.6B-Base with a trained LoRA adapter and pointer head (10.6M trainable parameters)", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "gpu": "RTX6000", "adapter_id": "images/decision-fast/spec.json:flymy_v15.FlyMyPackage", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 76.07766497651991, "speed": 91.44048940449588, "cost": 80.08651867745036 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0.00015849947083770386 ], "B": [ 0, 0.00009990132864763427 ], "C": [ 0, 0.00011006907697062771 ] }, "jevbench_score": 0, "intelligence": { "I_open": 7.310364004745869, "I_sealed": -11.695325345216284, "base": -2.1924806702352075, "gap": 19.005689349962154, "excess": 13.81906184988291, "penalty": 0.9418093815011709, "per_type_split": { "open|choice": { "cc": 38.3565246934954, "tiers": { "easy": 83.72022912270124, "standard": 54.76995061086561, "judge": 48.13278008298756, "hard": 11.476694085389722 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -59.06535421892006, "tiers": { "easy": -11.764705882352944, "standard": -81.81818181818181, "judge": -9.80392156862745, "hard": -96.46017699115043 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 42.63992153966227, "tiers": { "easy": 9.858070139613762, "standard": 59.88463036349702, "judge": 46.091382095862066, "hard": 39.624434560607156 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 18.49353134211178, "tiers": { "easy": 44.335937499999986, "standard": 28.00543355218476, "judge": 24.46996466431096, "hard": 2.7946537059538477 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -86.83636363636364, "tiers": { "easy": -76, "standard": -80, "judge": -92, "hard": -89.0909090909091 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 33.25685625860301, "tiers": { "easy": 46.23191862887507, "standard": 41.0924552565227, "judge": 25.409301485791623, "hard": 31.980957246683673 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 76.07766497651991, "parts": { "choice": { "ece_hard": 0.19382906987142728, "ece_score": 61.23418602571454, "mean_tvd": 0.29804180436481853, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 65.71500279461634 }, "noul": { "ece": 0.09637970835882076, "brier": 0.22661488853990605, "n": 495, "score": 80.72405832823584, "abstention_rate": 0.8080808080808081 }, "score": { "nrps": 0.16085481262204096, "top_ece": 0.10163325562190413, "n": 330, "interval90_coverage": 0.9606060606060606, "interval90_width": 0.7941919191919187, "score": 81.79393380670754 } } }, "speed": { "p50_s_raw": 0.058117716456763446, "p95_s_raw": 0.05978923523798585, "p50_s_adjusted": 0.2662354329135269, "p95_s_adjusted": 0.2695784704759717, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.004610868226600985, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 84, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 84, "B": 84, "C": 84 } }, { "key": "gte-reranker-modernbert-base", "display": "Alibaba GTE Reranker ModernBERT-base", "addendum": null, "author": "Alibaba-NLP", "repo": "https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base", "class": "reranker", "licence": "Apache-2.0", "open": "yes", "underlying": "149M ModernBERT cross-encoder", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/gte-reranker-modernbert-base/spec.json:reranker_adapter.RerankerAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 75.19518902157515, "speed": 91.48515951011706, "cost": 69.26965431492843 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": -18.627164187197426, "I_sealed": -23.156535387423837, "base": -20.891849787310633, "gap": 4.529371200226411, "excess": -0.6572562998528317, "penalty": 1, "per_type_split": { "open|choice": { "cc": 13.023655886527008, "tiers": { "easy": 11.365691890262292, "standard": 17.33818559916818, "judge": 27.385892116182585, "hard": 0.5092048570309202 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -100, "tiers": { "easy": -100, "standard": -100, "judge": -100, "hard": -100 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 31.09485155188071, "tiers": { "easy": 7.263813138813424, "standard": 30.66862041644295, "judge": 38.1165550700604, "hard": 31.999449084231635 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 2.3457082085803536, "tiers": { "easy": -11.328125000000023, "standard": 4.912836767036478, "judge": 4.593639575971737, "hard": 2.7946537059538477 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -100, "tiers": { "easy": -100, "standard": -100, "judge": -100, "hard": -100 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 28.184685629148124, "tiers": { "easy": 38.44342935494295, "standard": 28.268568534260474, "judge": 27.431807026686517, "hard": 26.14271719698945 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 75.19518902157515, "parts": { "choice": { "ece_hard": 0.054310813389460755, "ece_score": 89.13783732210786, "mean_tvd": 0.32663092997297427, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 78.23737216240522 }, "noul": { "ece": 0.19477931930463183, "brier": 0.29107023790736486, "n": 495, "score": 61.044136139073636, "abstention_rate": 1 }, "score": { "nrps": 0.18591656289063668, "top_ece": 0.044001130922215366, "n": 330, "interval90_coverage": 1, "interval90_width": 1, "score": 86.30405876324662 } } }, "speed": { "p50_s_raw": 0.030790242832154036, "p95_s_raw": 0.09287206921726458, "p50_s_adjusted": 0.21158048566430807, "p95_s_adjusted": 0.33574413843452916, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.010576570197044334, "basis": "ESTIMATE: size-class proxy (deepinfra:thenlper/gte-base); no exact base-model floor applies" }, "rank": 85, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 85, "B": 85, "C": 85 } }, { "key": "kev-0.5b", "display": "kev 0.5B", "addendum": null, "author": "Jared Palmer", "repo": "https://github.com/jaredpalmer/kev", "class": "jev-rebuild", "licence": "Apache-2.0", "open": true, "underlying": "Qwen2.5-0.5B + LoRA + learned pointer head; jaredpalmer/kev-0.5b", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/kev-0.5b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 64.98295390603997, "speed": 88.45344742064806, "cost": 80.09398520305423 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": 3.443416406201637, "I_sealed": -22.2626245077547, "base": -9.409604050776533, "gap": 25.706040913956336, "excess": 20.519413413877093, "penalty": 0.8748058658612291, "per_type_split": { "open|choice": { "cc": 25.23699856419025, "tiers": { "easy": 83.72022912270124, "standard": 28.255783727579928, "judge": 39.834024896265575, "hard": -1.8409714061888192 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -48.02344107393561, "tiers": { "easy": -29.411764705882348, "standard": -87.87878787878788, "judge": 15.686274509803933, "hard": -80.53097345132744 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 33.11669172835026, "tiers": { "easy": 27.747986780244837, "standard": 51.74438424407812, "judge": 36.650781722995376, "hard": 22.494454211528847 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 11.194986272433392, "tiers": { "easy": 23.828124999999982, "standard": 11.70477699796244, "judge": 17.84452296819789, "hard": 2.7946537059538477 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -94.54545454545455, "tiers": { "easy": -100, "standard": -92, "judge": -92, "hard": -96.36363636363636 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 16.562594749757046, "tiers": { "easy": 36.13676041517198, "standard": 15.847928201720284, "judge": 11.217165945743702, "hard": 16.035458210431706 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.006257822277847325 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0.00303030303030305 } }, "calibration": { "score": 64.98295390603997, "parts": { "choice": { "ece_hard": 0.3040174962950841, "ece_score": 39.196500740983176, "mean_tvd": 0.4046701342800813, "n_tvd": 50, "n": 794, "n_ece_hard": 275, "score": 49.36474365648752 }, "noul": { "ece": 0.10218181818181817, "brier": 0.23746343434343445, "n": 495, "score": 79.56363636363636, "abstention_rate": 0.7656565656565657 }, "score": { "nrps": 0.20914361923773894, "top_ece": 0.23522337340116994, "n": 329, "interval90_coverage": 0.8723404255319149, "interval90_width": 0.7150455927051664, "score": 66.02048169799606 } } }, "speed": { "p50_s_raw": 0.10358892846852541, "p95_s_raw": 0.12486683493480086, "p50_s_adjusted": 0.35717785693705084, "p95_s_adjusted": 0.3997336698696017, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.004608226600985222, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 86, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 86, "B": 86, "C": 86 } }, { "key": "laya", "display": "Laya (Convai Innovations, ModernBERT-large 421M)", "addendum": null, "author": "Convai Innovations", "repo": "https://huggingface.co/convaiinnovations/laya", "class": "jev-rebuild", "licence": "Apache-2.0", "open": "yes", "underlying": "ModernBERT-large encoder + option-marker decision head, 421M, RLCD-trained", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/laya/spec.json:warm_adapter.WarmLoaded", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 73.70996996254986, "speed": 73.89376469233966, "cost": 84.89402784895724 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": -0.47796953967189104, "I_sealed": -13.186035258388955, "base": -6.832002399030423, "gap": 12.708065718717064, "excess": 7.5214382186378215, "penalty": 1, "per_type_split": { "open|choice": { "cc": 37.096544852752906, "tiers": { "easy": 83.72022912270124, "standard": 40.73303873147908, "judge": 57.4688796680498, "hard": 8.343125734430064 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -75.73675326928841, "tiers": { "easy": -41.17647058823529, "standard": -78.78787878787878, "judge": -76.47058823529412, "hard": -82.30088495575221 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 37.20629979751983, "tiers": { "easy": 14.670391705069118, "standard": 50.6155477031802, "judge": 36.02994757536041, "hard": 37.0179170344219 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 12.772232378428598, "tiers": { "easy": 41.406249999999986, "standard": 19.8551052750736, "judge": 8.568904593639585, "hard": 5.224787363305005 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -81.52727272727273, "tiers": { "easy": -84, "standard": -76, "judge": -84, "hard": -81.81818181818181 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 29.19693457367726, "tiers": { "easy": 41.28777173913043, "standard": 36.50554156171282, "judge": 21.725989890480236, "hard": 28.123130300693955 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 73.70996996254986, "parts": { "choice": { "ece_hard": 0.14936750902527077, "ece_score": 70.12649819494585, "mean_tvd": 0.2982611665500017, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 70.15019076997285 }, "noul": { "ece": 0.11330686868686872, "brier": 0.2578809390101011, "n": 495, "score": 77.33862626262625, "abstention_rate": 0.8303030303030303 }, "score": { "nrps": 0.17930905198989894, "top_ece": 0.17393454545454545, "n": 330, "interval90_coverage": 0.8424242424242424, "interval90_width": 0.6704545454545449, "score": 73.6410928550505 } } }, "speed": { "p50_s_raw": 0.6676969870459288, "p95_s_raw": 1.2982571767410263, "p50_s_adjusted": 1.4853939740918576, "p95_s_adjusted": 2.7465143534820524, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.0031881034482758625, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 87, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 87, "B": 87, "C": 87 } }, { "key": "lev-350m", "display": "lev-350m (Franck Verrot, LFM2.5-350M)", "addendum": null, "author": "Franck Verrot (franckverrot)", "repo": "https://github.com/franckverrot/lev", "class": "jev-rebuild", "licence": "Apache-2.0 (code); weights under LiquidAI's LFM1.0 licence, following the LFM2.5-350M base", "open": "yes", "underlying": "LiquidAI/LFM2.5-350M with a LoRA and a 6.5M-parameter pointer head (a kev clone)", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "gpu": "RTX6000", "adapter_id": "images/lev-350m/spec.json:lev_v15.LevAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 77.67886674901663, "speed": 93.63413423056228, "cost": 80.03797760430007 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": 5.528286983062985, "I_sealed": -18.014062268821647, "base": -6.242887642879331, "gap": 23.54234925188463, "excess": 18.355721751805387, "penalty": 0.8964427824819461, "per_type_split": { "open|choice": { "cc": 33.06242798398311, "tiers": { "easy": 83.72022912270124, "standard": 46.97166623342864, "judge": 35.68464730290457, "hard": 11.476694085389722 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -52.199927436783234, "tiers": { "easy": -41.17647058823529, "standard": -75.75757575757575, "judge": 11.764705882352944, "hard": -91.1504424778761 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 35.72236040198907, "tiers": { "easy": 10.21198156682026, "standard": 48.57199646643111, "judge": 37.38335517693317, "hard": 34.429390997352186 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 15.013955037546538, "tiers": { "easy": 35.54687499999999, "standard": 19.8551052750736, "judge": 9.8939929328622, "hard": 11.300121506682888 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -100, "tiers": { "easy": -100, "standard": -100, "judge": -100, "hard": -100 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 30.943858155988508, "tiers": { "easy": 50.182971014492765, "standard": 30.069689336691873, "judge": 31.40985678180286, "hard": 26.221665381649995 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 77.67886674901663, "parts": { "choice": { "ece_hard": 0.07782310469314083, "ece_score": 84.43537906137183, "mean_tvd": 0.33867733339999334, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 75.28382286068626 }, "noul": { "ece": 0.12583434343434344, "brier": 0.23619623030303005, "n": 495, "score": 74.83313131313132, "abstention_rate": 0.8 }, "score": { "nrps": 0.17304344217171722, "top_ece": 0.0842818181818181, "n": 330, "interval90_coverage": 0.9878787878787879, "interval90_width": 0.941666666666667, "score": 82.91964607323233 } } }, "speed": { "p50_s_raw": 0.02207189949695021, "p95_s_raw": 0.03654062089044591, "p50_s_adjusted": 0.1941437989939004, "p95_s_adjusted": 0.2230812417808918, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.00462807881773399, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 88, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 88, "B": 88, "C": 88 } }, { "key": "mghafiri-qwen3.5-0.8b-decision-model", "display": "Qwen3.5-0.8B Decision Model (Mourad Ghafiri)", "addendum": null, "author": "Mourad Ghafiri", "repo": "https://huggingface.co/mghafiri/qwen3.5-0.8B-decision-model", "class": "jev-rebuild", "licence": "MIT code and training data; Apache-2.0 model weights", "open": "yes", "underlying": "Qwen3.5-0.8B-Base fine-tuned as a JevLite decision model with per-question calibration", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/mghafiri-qwen3.5-0.8b-decision-model/spec.json:mghafiri_adapter.JevLite", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 73.56687971516169, "speed": 71.82266186099122, "cost": 79.5180630553356 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0.022833461476708693 ], "B": [ 0, 0.014952408669619277 ], "C": [ 0, 0.01585657046993659 ] }, "jevbench_score": 0, "intelligence": { "I_open": 3.3975872914845855, "I_sealed": -3.7143568602260615, "base": -0.158384784370738, "gap": 7.111944151710647, "excess": 1.9253166516314044, "penalty": 1, "per_type_split": { "open|choice": { "cc": 34.92594009023165, "tiers": { "easy": 89.14681941513415, "standard": 39.17338185599168, "judge": 6.639004149377612, "hard": 40.462201331766536 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -69.94589307967757, "tiers": { "easy": 5.882352941176472, "standard": -75.75757575757575, "judge": -84.31372549019608, "hard": -75.2212389380531 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 45.212714863899684, "tiers": { "easy": 28.590783410138254, "standard": 48.11060070671379, "judge": 43.39331585845348, "hard": 49.283804060017644 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 40.43820114043159, "tiers": { "easy": 50.1953125, "standard": 32.08059769074034, "judge": 44.34628975265018, "hard": 39.24665856622116 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -86.03636363636365, "tiers": { "easy": -92, "standard": -80, "judge": -84, "hard": -89.0909090909091 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 34.455091915253874, "tiers": { "easy": 43.77771739130435, "standard": 33.51389588581023, "judge": 30.51832350463354, "hard": 35.54760986892834 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 73.56687971516169, "parts": { "choice": { "ece_hard": 0.056884476534296013, "ece_score": 88.6231046931408, "mean_tvd": 0.29926050001665505, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 79.34852734573765 }, "noul": { "ece": 0.16277070707070712, "brier": 0.2519389916969697, "n": 495, "score": 67.44585858585857, "abstention_rate": 0.8282828282828283 }, "score": { "nrps": 0.16518099632828284, "top_ece": 0.17834696969696973, "n": 330, "interval90_coverage": 0.8818181818181818, "interval90_width": 0.6093434343434344, "score": 73.90625321388889 } } }, "speed": { "p50_s_raw": 0.579218426020816, "p95_s_raw": 2.436603333125822, "p50_s_adjusted": 1.3084368520416318, "p95_s_adjusted": 5.023206666251644, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.00481649630541872, "basis": "documented hosted-model estimate" }, "rank": 89, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 89, "B": 89, "C": 89 } }, { "key": "mxbai-rerank-base-v2", "display": "Mixedbread mxbai-rerank-base-v2", "addendum": null, "author": "Mixedbread", "repo": "https://huggingface.co/mixedbread-ai/mxbai-rerank-base-v2", "class": "reranker", "licence": "Apache-2.0", "open": "yes", "underlying": "494M cross-encoder", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "gpu": "A6000", "adapter_id": "images/mxbai-rerank-base-v2/spec.json:reranker_adapter.RerankerAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 86.61210205397524, "speed": 89.34435302808978, "cost": 60.33589314238483 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": -20.99032740064011, "I_sealed": -24.038818140610356, "base": -22.514572770625232, "gap": 3.048490739970248, "excess": -2.1381367601089947, "penalty": 1, "per_type_split": { "open|choice": { "cc": 5.843867292486349, "tiers": { "easy": 20.410009044317164, "standard": 0.18195996880685458, "judge": 4.564315352697115, "hard": 5.992949471210316 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -100, "tiers": { "easy": -100, "standard": -100, "judge": -100, "hard": -100 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 31.185150505593313, "tiers": { "easy": 6.799755544497577, "standard": 31.007951785898857, "judge": 37.69384193308352, "hard": 32.48858003509682 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 0.6243932922656859, "tiers": { "easy": 15.03906249999998, "standard": -10.029431741000643, "judge": 3.268551236749129, "hard": 0.36452004860269366 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -100, "tiers": { "easy": -100, "standard": -100, "judge": -100, "hard": -100 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 27.259152285903227, "tiers": { "easy": 38.52221935703126, "standard": 27.179768385394233, "judge": 25.838359064936366, "hard": 25.548672384100858 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 86.61210205397524, "parts": { "choice": { "ece_hard": 0.028303254999639538, "ece_score": 94.33934900007209, "mean_tvd": 0.3377944081077518, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 80.27995409464846 }, "noul": { "ece": 0.04816304808672505, "brier": 0.2514061975269887, "n": 495, "score": 90.36739038265499, "abstention_rate": 1 }, "score": { "nrps": 0.18687570257877734, "top_ece": 0.014672531864388688, "n": 330, "interval90_coverage": 1, "interval90_width": 1, "score": 89.18896168462227 } } }, "speed": { "p50_s_raw": 0.04247652366757393, "p95_s_raw": 0.1724876998458056, "p50_s_adjusted": 0.23495304733514785, "p95_s_adjusted": 0.4949753996916112, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.020996016009852216, "basis": "ESTIMATE: size-class proxy (deepinfra:Qwen/Qwen3-Reranker-0.6B); no exact base-model floor applies" }, "rank": 90, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 90, "B": 90, "C": 90 } }, { "key": "needle-3", "display": "Needle 3 (Cactus, 2-bit, local CPU)", "addendum": null, "author": "Cactus Compute", "repo": "https://github.com/cactus-compute/needle", "class": "small-tool-model", "licence": "Apache-2.0 (model and package)", "open": "yes", "underlying": "Needle 3 (121M parameters, 2-bit)", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/needle-3/spec.json:jevbench.adapters.needle_local.NeedleLocalAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1101 }, "full_coverage": true, "support": { "choice": "label", "noul": "label", "score": "label" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 0, "speed": 34.1252252793452, "cost": 61.57165387848559 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": -11.117169094537822, "I_sealed": -21.113658670994404, "base": -16.115413882766113, "gap": 9.996489576456582, "excess": 4.80986207637734, "penalty": 1, "per_type_split": { "open|choice": { "cc": -3.5679770931519377, "tiers": { "easy": 16.792282182695214, "standard": -7.616324408630107, "judge": -9.958506224066367, "hard": -1.8409714061888192 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -14.520372911835695, "tiers": { "easy": -17.647058823529417, "standard": -45.45454545454546, "judge": 7.843137254901955, "hard": -15.04424778761062 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": -15.263157278625831, "tiers": { "easy": -2.5345622119815614, "standard": -18.816254416961108, "judge": -22.542595019659206, "hard": -11.20917917034423 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": -8.13855828680416, "tiers": { "easy": -8.39843750000002, "standard": -12.746207833371031, "judge": -3.356890459363947, "hard": -9.356014580801922 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -23.599999999999998, "tiers": { "easy": -19.999999999999996, "standard": -31.999999999999996, "judge": -24, "hard": -19.999999999999996 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": -31.60241772617906, "tiers": { "easy": -28.623188405797073, "standard": -27.20403022670026, "judge": -30.58129738837403, "hard": -35.312259059367726 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.33917396745932415 }, "noul": { "n": 495, "invalid_rate": 0.29292929292929293 }, "score": { "n": 330, "invalid_rate": 0.3242424242424242 } }, "calibration": { "score": 0, "parts": {} }, "speed": { "p50_s_raw": 67.68173758895136, "p95_s_raw": 142.63847913655917, "p50_s_adjusted": 135.51347517790273, "p95_s_adjusted": 285.4269582731183, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.019096108374384236, "basis": "ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies" }, "rank": 91, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 91, "B": 91, "C": 91 } }, { "key": "needle-3-tools", "display": "Needle 3, options as tools (post-hoc adapter mode)", "addendum": null, "author": "Cactus Compute", "repo": "https://github.com/cactus-compute/needle", "class": "small-tool-model", "licence": "Apache-2.0 (model and package)", "open": "yes", "underlying": "Needle 3 (121M parameters, 2-bit)", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/needle-3-tools/spec.json:jevbench.adapters.needle_local.NeedleLocalAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1306 }, "full_coverage": true, "support": { "choice": "label", "noul": "label", "score": "label" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 0, "speed": 40.9964345452206, "cost": 61.57165387848559 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": -5.786504055687074, "I_sealed": -19.897677485520102, "base": -12.842090770603589, "gap": 14.111173429833027, "excess": 8.924545929753783, "penalty": 0.9907545407024622, "per_type_split": { "open|choice": { "cc": 10.871090994383101, "tiers": { "easy": 45.7340970756708, "standard": 25.136469976605138, "judge": -0.6224066390041282, "hard": 3.6427732079905777 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -12.378653794582998, "tiers": { "easy": -17.647058823529417, "standard": -42.42424242424242, "judge": 5.882352941176472, "hard": -9.734513274336287 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": -15.851949366861326, "tiers": { "easy": 11.290322580645162, "standard": -27.20848056537102, "judge": -20.576671035386607, "hard": -13.41571050308914 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 2.204169809145612, "tiers": { "easy": 0.39062499999998146, "standard": -1.8791034638894857, "judge": -3.356890459363947, "hard": 8.869987849331737 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -30.10909090909091, "tiers": { "easy": -43.99999999999999, "standard": -31.999999999999996, "judge": -28.000000000000004, "hard": -27.27272727272727 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": -31.788111356615012, "tiers": { "easy": -31.3405797101449, "standard": -30.982367758186413, "judge": -29.317607413647817, "hard": -34.15574402467223 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.07759699624530658 }, "noul": { "n": 495, "invalid_rate": 0.29292929292929293 }, "score": { "n": 330, "invalid_rate": 0.3363636363636363 } }, "calibration": { "score": 0, "parts": {} }, "speed": { "p50_s_raw": 29.004772314452566, "p95_s_raw": 68.26981186334741, "p50_s_adjusted": 58.15954462890513, "p95_s_adjusted": 136.68962372669483, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.019096108374384236, "basis": "ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies" }, "rank": 92, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 92, "B": 92, "C": 92 } }, { "key": "open-jev-deberta-v3-large", "display": "open-jev-deberta-v3-large (local CPU)", "addendum": null, "author": "Kotoba Labs", "repo": "https://github.com/kotoba-lang/typed-decisions", "class": "jev-rebuild", "licence": "Apache-2.0 (model card); DeBERTa-v3 keeps its own terms", "open": "yes", "underlying": "microsoft/deberta-v3-large", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/open-jev-deberta-v3-large/spec.json:warm_adapter.WarmLoaded", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1610 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 77.05564904241504, "speed": 68.25469890461562, "cost": 77.5922354035203 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": -9.303260223347305, "I_sealed": -14.60833621498483, "base": -11.955798219166066, "gap": 5.3050759916375245, "excess": 0.11844849155828197, "penalty": 1, "per_type_split": { "open|choice": { "cc": 21.004312742803485, "tiers": { "easy": 90.95568284594513, "standard": 36.05406810501689, "judge": 8.71369294605811, "hard": 5.209557383470406 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -79.1568469704857, "tiers": { "easy": 17.647058823529417, "standard": -78.78787878787878, "judge": -98.0392156862745, "hard": -89.38053097345133 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 30.242753557640295, "tiers": { "easy": -0.4921783980686101, "standard": 33.09975231507328, "judge": 31.573146619732697, "hard": 35.50019237128173 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 16.66342552932574, "tiers": { "easy": 26.757812499999982, "standard": 13.063165044147636, "judge": 24.46996466431096, "hard": 10.085054678007312 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -87.38181818181818, "tiers": { "easy": -84, "standard": -92, "judge": -88, "hard": -85.45454545454545 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 26.893384007537954, "tiers": { "easy": 40.48922199863888, "standard": 27.38158587682259, "judge": 23.62051138932101, "hard": 25.70497803878311 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.017521902377972465 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 77.05564904241504, "parts": { "choice": { "ece_hard": 0.18483169170354635, "ece_score": 63.03366165929073, "mean_tvd": 0.2886090069457545, "n_tvd": 50, "n": 785, "n_ece_hard": 267, "score": 67.08638048235764 }, "noul": { "ece": 0.04918332987587257, "brier": 0.22354630250714677, "n": 495, "score": 90.16333402482547, "abstention_rate": 0.9030303030303031 }, "score": { "nrps": 0.18992507423278554, "top_ece": 0.16586513668298722, "n": 330, "interval90_coverage": 0.8424242424242424, "interval90_width": 0.7098484848484842, "score": 73.91723262006201 } } }, "speed": { "p50_s_raw": 1.3912787401350215, "p95_s_raw": 2.47331775734201, "p50_s_adjusted": 2.932557480270043, "p95_s_adjusted": 5.096635514684021, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.005583743842364532, "basis": "ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies" }, "rank": 93, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 93, "B": 93, "C": 93 } }, { "key": "open-jev-json-canvas-joshuasp", "display": "Open Jev JSON Canvas (JoshuaSP)", "addendum": null, "author": "JoshuaSP", "repo": "https://github.com/JoshuaSP/open-jev", "class": "jev-rebuild", "licence": "MIT code; Apache-2.0 DiffusionGemma weights", "open": "yes", "underlying": "google/diffusiongemma-26B-A4B-it BF16, one-step JSON canvas", "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/open-jev-json-canvas-joshuasp/spec.json:joshuasp_adapter.JoshuaSPCanvasAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "label", "noul": "label", "score": "label" }, "planned_weight_denominator": 1, "axes": { "intelligence": 77.13565407855228, "calibration": 0, "speed": 85.57288828383295, "cost": 49.295219536461964 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 64.89270256009785, "Emphasis on Accuracy 60:20:20": 68.58220536636948, "Emphasis on Cost 20:20:60": 56.837259850231455, "Emphasis on Speed 20:60:20": 71.14892802685067, "Intelligence only": 74.97643178712907 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": 79.15477691929314, "I_sealed": 75.1165312378114, "base": 77.13565407855228, "gap": 4.03824568148174, "excess": -1.1483818185975023, "penalty": 1, "per_type_split": { "open|choice": { "cc": 81.04636600709232, "tiers": { "easy": 98.19113656918904, "standard": 92.20171562256304, "judge": 89.6265560165975, "hard": 64.74735605170387 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 74.9238874954648, "tiers": { "easy": 82.35294117647058, "standard": 84.84848484848484, "judge": 84.31372549019606, "hard": 61.061946902654874 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 81.49407725532232, "tiers": { "easy": 93.08755760368663, "standard": 90.7243816254417, "judge": 80.34076015727392, "hard": 74.84554280670785 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 76.36059721739608, "tiers": { "easy": 94.140625, "standard": 74.19062712248135, "judge": 69.52296819787986, "hard": 78.12879708383961 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 74, "tiers": { "easy": 100, "standard": 68, "judge": 87.99999999999999, "hard": 60.00000000000001 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 74.98899649603815, "tiers": { "easy": 86.41304347826086, "standard": 72.29219143576826, "judge": 73.0412805391744, "hard": 74.94217424826523 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 0, "parts": {} }, "speed": { "p50_s_raw": 0.14337781799258664, "p95_s_raw": 0.2422799716354348, "p50_s_adjusted": 0.4367556359851733, "p95_s_adjusted": 0.6345599432708696, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.04899585591133005, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 94, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 94, "B": 94, "C": 94 } }, { "key": "openjev-verdict", "display": "openJev Verdict (heman10x, ModernBERT-base 151M)", "addendum": null, "author": "Hemant (heman10x)", "repo": "https://github.com/Heman10x-NGU/openJev-verdict-2.0", "class": "jev-rebuild", "licence": "Apache-2.0", "open": "yes", "underlying": "GLiClass ModernBERT-base (knowledgator/gliclass-modern-base-v2.0) fine-tuned, 151M", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/openjev-verdict/spec.json:warm_adapter.WarmLoaded", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 52.22236594343488, "speed": 83.87645390988475, "cost": 86.62313527343974 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0.014760916697897527 ], "B": [ 0, 0.0096284402945188 ], "C": [ 0, 0.010250636595762173 ] }, "jevbench_score": 0, "intelligence": { "I_open": 12.572554325229365, "I_sealed": -14.08853844775725, "base": -0.7579920612639421, "gap": 26.661092772986613, "excess": 21.47446527290737, "penalty": 0.8652553472709263, "per_type_split": { "open|choice": { "cc": 26.77838880692589, "tiers": { "easy": 80.10250226107928, "standard": 36.05406810501689, "judge": 25.311203319502088, "hard": 9.909909909909892 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -15.152619374378874, "tiers": { "easy": -11.764705882352944, "standard": -27.27272727272727, "judge": 29.41176470588236, "hard": -43.36283185840708 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 26.091893543141083, "tiers": { "easy": -5.338195756790021, "standard": 54.452603607718906, "judge": 22.574273323838835, "hard": 22.407276000311626 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 16.635451407617545, "tiers": { "easy": 41.406249999999986, "standard": 23.930269413629183, "judge": 13.869257950530042, "hard": 8.869987849331737 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -79.16363636363636, "tiers": { "easy": -68, "standard": -88, "judge": -88, "hard": -70.9090909090909 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 20.26256961274706, "tiers": { "easy": 45.33950093608078, "standard": 22.809338245529144, "judge": 12.49636803642975, "hard": 18.54460364776057 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 52.22236594343488, "parts": { "choice": { "ece_hard": 0.31863114738135034, "ece_score": 36.27377052372993, "mean_tvd": 0.4069134621098454, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 47.791212156372694 }, "noul": { "ece": 0.2689141261497282, "brier": 0.32052442894859673, "n": 495, "score": 46.217174770054356, "abstention_rate": 0.4747474747474747 }, "score": { "nrps": 0.21716081893888597, "top_ece": 0.26483248149178096, "n": 330, "interval90_coverage": 0.7848484848484848, "interval90_width": 0.6196969696969695, "score": 62.658710903877605 } } }, "speed": { "p50_s_raw": 0.12610384542495012, "p95_s_raw": 0.43418340694624924, "p50_s_adjusted": 0.40220769084990027, "p95_s_adjusted": 1.0183668138924984, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.002791871921182266, "basis": "ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies" }, "rank": 95, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 95, "B": 95, "C": 95 } }, { "key": "openjev-verdict-1.4", "display": "openJev Verdict 1.4", "addendum": null, "author": "Hemant (heman10x)", "repo": "https://huggingface.co/heman10x/rlcd-modernbert-151m", "class": "jev-rebuild", "licence": "Apache-2.0", "open": "yes", "underlying": "GLiClass ModernBERT-base fine-tuned decision model, 151M; v1.4 fixed inference engine", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/openjev-verdict-1.4/spec.json:warm_adapter.WarmLoaded", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 80.33031422397818, "speed": 80.63780728589471, "cost": 86.62313527343974 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": -14.746381485338418, "I_sealed": -18.28076253742016, "base": -16.51357201137929, "gap": 3.5343810520817414, "excess": -1.6522464479975012, "penalty": 1, "per_type_split": { "open|choice": { "cc": 24.287336790910803, "tiers": { "easy": 80.10250226107928, "standard": 37.61372498050429, "judge": 17.012448132780097, "hard": 9.126517822169975 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -100, "tiers": { "easy": -100, "standard": -100, "judge": -100, "hard": -100 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 31.47351875307393, "tiers": { "easy": 0.8001436810262552, "standard": 51.52721614568101, "judge": 30.4854576989947, "hard": 29.856059615341724 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 17.859202655617498, "tiers": { "easy": 41.406249999999986, "standard": 21.213493321258795, "judge": 16.519434628975272, "hard": 11.300121506682888 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -100, "tiers": { "easy": -100, "standard": -100, "judge": -100, "hard": -100 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 27.298509732122007, "tiers": { "easy": 48.17842864048421, "standard": 26.99406380774041, "judge": 23.76842509153304, "hard": 24.878316447663973 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 80.33031422397818, "parts": { "choice": { "ece_hard": 0.1319904885190996, "ece_score": 73.60190229618007, "mean_tvd": 0.3092785057985509, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 71.33702585816249 }, "noul": { "ece": 0.06400173539651974, "brier": 0.24791482740527485, "n": 495, "score": 87.19965292069605, "abstention_rate": 1 }, "score": { "nrps": 0.18447119454353428, "top_ece": 0.08322176379747256, "n": 330, "interval90_coverage": 0.9575757575757575, "interval90_width": 0.8568181818181818, "score": 82.45426389307602 } } }, "speed": { "p50_s_raw": 0.31029939255677164, "p95_s_raw": 0.48522302696947006, "p50_s_adjusted": 0.7705987851135433, "p95_s_adjusted": 1.12044605393894, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.002791871921182266, "basis": "ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies" }, "rank": 96, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 96, "B": 96, "C": 96 } }, { "key": "qwen3.8-27b", "display": "Qwen3.8 27B (Chutes TEE)", "addendum": null, "author": "Qwen / Chutes", "repo": null, "class": "llm-baseline", "licence": "open weights", "open": "weights", "underlying": "Qwen3.8-27B", "endpoint_kind": "api", "endpoint_condition": "the operator's hosted API", "gpu": null, "adapter_id": "harness/specs/qwen3.8-27b.json:jevbench.adapters.openai_compat.OpenAICompatAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1600 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 95.56146727664041, "calibration": 98.08678902254937, "speed": 56.846295929381085, "cost": 0 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": 92.89676067022268, "I_sealed": 98.22617388305815, "base": 95.56146727664041, "gap": -5.329413212835476, "excess": -10.51604071291472, "penalty": 1, "per_type_split": { "open|choice": { "cc": 97.18121394413481, "tiers": { "easy": 100, "standard": 98.44034312451261, "judge": 100, "hard": 93.73286329808069 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 86.67881311816762, "tiers": { "easy": 100, "standard": 100, "judge": 86.27450980392157, "hard": 76.99115044247789 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 94.83025494836559, "tiers": { "easy": 100, "standard": 99.6731448763251, "judge": 99.81651376146789, "hard": 87.37667961165049 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 97.65836657235229, "tiers": { "easy": 100, "standard": 100, "judge": 98.67491166077738, "hard": 95.13973268529769 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 98.8, "tiers": { "easy": 100, "standard": 100, "judge": 96, "hard": 100 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 98.22015507682215, "tiers": { "easy": 99.875, "standard": 94.60537363560033, "judge": 98.50968828980623, "hard": 99.39668465690053 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.01251564455569465 }, "noul": { "n": 495, "invalid_rate": 0.02020202020202022 }, "score": { "n": 330, "invalid_rate": 0.012121212121212088 } }, "calibration": { "score": 98.08678902254937, "parts": { "choice": { "ece_hard": 0.032551433891663095, "ece_score": 93.48971322166739, "mean_tvd": 0.01021755439138628, "n_tvd": 47, "n": 789, "n_ece_hard": 268, "score": 96.23397889126437 }, "noul": { "ece": 0.008304134010308824, "brier": 0.013199678015332589, "n": 485, "score": 98.33917319793824, "abstention_rate": 0.010309278350515427 }, "score": { "nrps": 0.0008364795721932518, "top_ece": 0.0027096104294481674, "n": 326, "interval90_coverage": 0.9969325153374233, "interval90_width": 0.003067484662576687, "score": 99.68721497844552 } } }, "speed": { "p50_s_raw": 6.489119492471218, "p95_s_raw": 31.855515367165104, "p50_s_adjusted": 6.489119492471218, "p95_s_adjusted": 31.855515367165104, "n": 450, "adjustment": "none (hosted API, measured as is)" }, "cost": { "kind": "estimate", "usd_per_1000": 2.1783887438423646, "basis": "ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)" }, "rank": 97, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 97, "B": 97, "C": 97 } }, { "key": "verdict-small", "display": "verdict-small (Manavarya09, multilingual-e5-small 118M)", "addendum": null, "author": "Manavarya09 (Manav)", "repo": "https://github.com/Manavarya09/verdict", "class": "jev-rebuild", "licence": "Apache-2.0 (verdictml code and the Manav2op/verdict-small checkpoint over intfloat/multilingual-e5-small)", "open": "yes", "underlying": "intfloat/multilingual-e5-small (118M multilingual bi-encoder) fine-tuned on a typed-decision mix; each option is scored against the rendered state by cosine similarity, with temperature scaling and a conformal abstain set on top. One encoder pass per option, no tokens generated.", "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/verdict-small/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 59.45216430058828, "speed": 82.05988082033076, "cost": 100 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0.005151570988376465 ], "B": [ 0, 0.003310895006967579 ], "C": [ 0, 0.0035774798530392115 ] }, "jevbench_score": 0, "intelligence": { "I_open": -0.9494028915653612, "I_sealed": -2.94840779086587, "base": -1.9489053412156156, "gap": 1.999004899300509, "excess": -3.1876226007787336, "penalty": 1, "per_type_split": { "open|choice": { "cc": 26.454347123855356, "tiers": { "easy": 67.44045824540248, "standard": 26.696126852092533, "judge": 28.423236514522838, "hard": 14.61026243634937 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -55.15025318252803, "tiers": { "easy": -47.05882352941176, "standard": -45.45454545454546, "judge": -70.58823529411764, "hard": -50.442477876106196 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 25.847697383976595, "tiers": { "easy": 14.758410138248824, "standard": 37.14880742049469, "judge": 29.26782437745743, "hard": 20.404368932038853 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 12.709233032116586, "tiers": { "easy": 23.828124999999982, "standard": 4.912836767036478, "judge": 11.219081272084816, "hard": 14.94532199270962 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -43.89090909090909, "tiers": { "easy": -28.000000000000004, "standard": -28.000000000000004, "judge": -48, "hard": -52.727272727272734 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 22.336452686194896, "tiers": { "easy": 28.33206521739132, "standard": 23.274979009235974, "judge": 20.83593091828141, "hard": 21.493677717810357 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 59.45216430058828, "parts": { "choice": { "ece_hard": 0.09814801444043317, "ece_score": 80.37039711191336, "mean_tvd": 0.3715248334499983, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 71.60895688345677 }, "noul": { "ece": 0.3252131313131313, "brier": 0.37707818664646464, "n": 495, "score": 34.95737373737374, "abstention_rate": 0.45252525252525255 }, "score": { "nrps": 0.2101906937752525, "top_ece": 0.17700303030303038, "n": 330, "interval90_coverage": 0.8757575757575757, "interval90_width": 0.7696969696969704, "score": 71.79016228093434 } } }, "speed": { "p50_s_raw": 0.03403579245787114, "p95_s_raw": 1.3518648592871614, "p50_s_adjusted": 0.21807158491574227, "p95_s_adjusted": 2.8537297185743227, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.000866152709359606, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 98, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 98, "B": 98, "C": 98 } }, { "key": "von-395m", "display": "Von (wfzyx, Option-Marker 395M)", "addendum": null, "author": "wfzyx (Victor Hugo)", "repo": "https://github.com/wfzyx/von", "class": "jev-rebuild", "licence": "Apache-2.0 code; Apache-2.0 weights (answerdotai/ModernBERT-large base)", "open": "unknown", "underlying": null, "endpoint_kind": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "gpu": "cpu", "adapter_id": "images/von-395m/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 83.4757036120557, "speed": 75.72186940216335, "cost": 82.71254751426194 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": -1.8892921869454486, "I_sealed": -16.55442797958771, "base": -9.22186008326658, "gap": 14.665135792642262, "excess": 9.47850829256302, "penalty": 0.9852149170743698, "per_type_split": { "open|choice": { "cc": 34.419086270896464, "tiers": { "easy": 90.95568284594513, "standard": 37.61372498050429, "judge": 36.721991701244825, "hard": 16.960438699569117 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -78.58138911236256, "tiers": { "easy": -17.647058823529417, "standard": -60.60606060606061, "judge": -94.11764705882352, "hard": -91.1504424778761 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 38.494426280629746, "tiers": { "easy": 6.3441244239631045, "standard": 56.45971731448762, "judge": 39.8554390563565, "hard": 36.5285966460724 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 17.857166657909275, "tiers": { "easy": 50.1953125, "standard": 30.722209644555146, "judge": 7.24381625441697, "hard": 11.300121506682888 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -98, "tiers": { "easy": -92, "standard": -100, "judge": -96, "hard": -100 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 30.47954940332759, "tiers": { "easy": 45.005797101449296, "standard": 29.559109991603684, "judge": 29.550168491996665, "hard": 28.00524286815731 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 83.4757036120557, "parts": { "choice": { "ece_hard": 0.02654584837545126, "ece_score": 94.69083032490974, "mean_tvd": 0.3211038334499984, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 81.29022348995495 }, "noul": { "ece": 0.09470949494949493, "brier": 0.24186929741414154, "n": 495, "score": 81.05810101010101, "abstention_rate": 0.8949494949494949 }, "score": { "nrps": 0.1641370005505052, "top_ece": 0.03714363636363633, "n": 330, "interval90_coverage": 0.996969696969697, "interval90_width": 0.9454545454545454, "score": 88.07878633611111 } } }, "speed": { "p50_s_raw": 0.3857649624114856, "p95_s_raw": 1.3780267895897842, "p50_s_adjusted": 0.9215299248229712, "p95_s_adjusted": 2.9060535791795683, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.003769193349753695, "basis": "documented hosted-model estimate; no exact base-model floor applies" }, "rank": 99, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 99, "B": 99, "C": 99 } }, { "key": "classifier-dev-fast", "display": "classifier.dev (fast tier)", "addendum": null, "author": "mrmps (@michael_chomsky)", "repo": "https://classifier.dev", "class": "jev-service", "licence": "MIT (code); hosted service", "open": "no", "underlying": "Jev (TypeSafe) behind classifier.dev's zero-shot classification API", "endpoint_kind": "api", "endpoint_condition": "the operator's hosted API", "gpu": null, "adapter_id": "harness/specs/classifier-dev-fast.json:jevbench.adapters.classifier_dev.ClassifierDevAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 75.80855995014088, "calibration": 89.18872642455774, "speed": 81.99020200942311, "cost": 58.89475669158328 }, "scores": { "A": 74.65566746650249, "B": 74.88343210437176, "C": 74.65566746650249 }, "views": { "Balanced 33:33:33": 70.80959108698245, "Emphasis on Accuracy 60:20:20": 72.7279212277414, "Emphasis on Cost 20:20:60": 65.50845755891696, "Emphasis on Speed 20:60:20": 74.89480959179811, "Intelligence only": 75.80855995014088 }, "composite_ci95": { "A": [ 73.50372056154933, 75.20247331380175 ], "B": [ 73.39160786384885, 75.83326867034351 ], "C": [ 73.50372056154933, 75.20247331380175 ] }, "jevbench_score": 74.65566746650249, "intelligence": { "I_open": 74.48059742656369, "I_sealed": 77.13652247371806, "base": 75.80855995014088, "gap": -2.655925047154369, "excess": -7.842552547233612, "penalty": 1, "per_type_split": { "open|choice": { "cc": 84.16704391568094, "tiers": { "easy": 98.19113656918904, "standard": 96.88068624902522, "judge": 93.7759336099585, "hard": 67.09753231492361 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 56.779455144889816, "tiers": { "easy": 76.47058823529412, "standard": 66.66666666666667, "judge": 80.3921568627451, "hard": 29.20353982300885 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 82.49529321912031, "tiers": { "easy": 77.05069124423962, "standard": 87.68109540636043, "judge": 88.3224115334207, "hard": 76.89320388349515 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 85.10653662378164, "tiers": { "easy": 97.0703125, "standard": 86.41611953814805, "judge": 81.4487632508834, "hard": 84.2041312272175 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 64.43636363636364, "tiers": { "easy": 84.00000000000001, "standard": 80, "judge": 68, "hard": 49.09090909090909 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 81.86666716100889, "tiers": { "easy": 86.22282608695653, "standard": 77.18316667938836, "judge": 84.51979780960403, "hard": 81.12952968388589 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 89.18872642455774, "parts": { "choice": { "ece_hard": 0.051448419210152085, "ece_score": 89.71031615796959, "mean_tvd": 0.3067600453378805, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 79.51715581209078 }, "noul": { "ece": 0.03414141414141409, "brier": 0.07133171717171723, "n": 495, "score": 93.17171717171718, "abstention_rate": 0.15353535353535352 }, "score": { "nrps": 0.04169966544848548, "top_ece": 0.03037710437710453, "n": 330, "interval90_coverage": 0.9818181818181818, "interval90_width": 0.21111111111111108, "score": 94.87730628986527 } } }, "speed": { "p50_s_raw": 0.5177607946097851, "p95_s_raw": 1.221379529684782, "p50_s_adjusted": 0.5177607946097851, "p95_s_adjusted": 1.221379529684782, "n": 450, "adjustment": "none (hosted API, measured as is)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.02345172413793104, "basis": "ESTIMATE (proxy tokens, I-2): operator list price: higher of 19 Sep plan cost and 26 Sep usage tariff USD 0.042/M input (rule 1.2); no exact base-model floor applies" }, "rank": null, "ranked": false, "listing": "honorable_mention", "not_ranked_because": "runs on Jev (TypeSafe) - listed, not ranked (honorable mention, as in v1.4.2)", "ranks": { "A": null, "B": null, "C": null } }, { "key": "jevk5-v0.3-4b", "display": "JevK5 v0.3 (4B)", "addendum": { "id": "A1", "release": "v1.5", "label": "v1.5 roster addendum A1" }, "author": "unknown", "repo": null, "class": "unclassified", "licence": "not recorded", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/jevk5-v0.3-4b/spec.json:jevk5_v15.JevK5Warm", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 56.27051532364801, "calibration": 88.34394935502905, "speed": 93.55214886599305, "cost": 63.07359395281681 }, "scores": { "A": 71.89794783714967, "B": 68.11459443579027, "C": 63.237658174056016 }, "views": { "Balanced 33:33:33": 67.69714212183655, "Emphasis on Accuracy 60:20:20": 62.611440918908144, "Emphasis on Cost 20:20:60": 65.76869806491888, "Emphasis on Speed 20:60:20": 76.11106829175051, "Intelligence only": 56.270515323648006 }, "composite_ci95": { "A": [ 69.39190737629329, 72.94757443195746 ], "B": [ 64.93425490805247, 69.81993373931475 ], "C": [ 51.342990288339934, 72.12390553992212 ] }, "jevbench_score": 71.89794783714967, "intelligence": { "I_open": 61.599649589732365, "I_sealed": 50.94138105756366, "base": 56.27051532364801, "gap": 10.658268532168705, "excess": 5.471641032089463, "penalty": 1, "per_type_split": { "open|choice": { "cc": 80.25232318940107, "tiers": { "easy": 96.38227313837805, "standard": 79.7244606186639, "judge": 88.58921161825727, "hard": 70.23110066588326 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 42.32817503509851, "tiers": { "easy": 64.70588235294117, "standard": 30.303030303030297, "judge": 62.745098039215684, "hard": 27.43362831858407 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 62.21845054469753, "tiers": { "easy": 27.990399176120206, "standard": 78.608761269458, "judge": 61.898387855725666, "hard": 62.82035504119053 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 74.2487837985823, "tiers": { "easy": 76.56249999999999, "standard": 74.19062712248135, "judge": 74.82332155477033, "hard": 73.2685297691373 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 14.981818181818179, "tiers": { "easy": 60.00000000000001, "standard": -7.9999999999999964, "judge": 28.000000000000004, "hard": 5.454545454545445 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 63.593541192290516, "tiers": { "easy": 78.74669580536042, "standard": 69.23663981627274, "judge": 60.357844150093996, "hard": 59.410476008679325 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 88.34394935502905, "parts": { "choice": { "ece_hard": 0.03780043689998049, "ece_score": 92.43991262000391, "mean_tvd": 0.2639746135435604, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 83.02122563282393 }, "noul": { "ece": 0.030562476858007503, "brier": 0.09874192080011364, "n": 495, "score": 93.8875046283985, "abstention_rate": 0.2909090909090909 }, "score": { "nrps": 0.09431639573309676, "top_ece": 0.0716106240948041, "n": 330, "interval90_coverage": 0.9363636363636364, "interval90_width": 0.38863636363636356, "score": 88.12311780386474 } } }, "speed": { "p50_s_raw": 0.01584504608763382, "p95_s_raw": 0.04645736991078593, "p50_s_adjusted": 0.18169009217526763, "p95_s_adjusted": 0.24291473982157186, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.01701689039408867, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 4, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 4, "B": 5, "C": 8 } }, { "key": "plumb-4b", "display": "Plumb-4B (crh225, JevK5 v0.2 + LoRA)", "addendum": { "id": "A1", "release": "v1.5", "label": "v1.5 roster addendum A1" }, "author": "unknown", "repo": null, "class": "unclassified", "licence": "not recorded", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/plumb-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 55.84843638193568, "calibration": 87.44416146719458, "speed": 93.45235823893108, "cost": 63.07359395281681 }, "scores": { "A": 71.56069240662383, "B": 67.74864628766687, "C": 62.00033986975725 }, "views": { "Balanced 33:33:33": 67.47526154024382, "Emphasis on Accuracy 60:20:20": 62.288260696856106, "Emphasis on Cost 20:20:60": 65.64287436007193, "Emphasis on Speed 20:60:20": 75.91628888044532, "Intelligence only": 55.84843638193568 }, "composite_ci95": { "A": [ 69.19832449519649, 72.72798975349464 ], "B": [ 64.67900356778624, 69.55675259096279 ], "C": [ 50.654423777900476, 70.8909332025414 ] }, "jevbench_score": 71.56069240662383, "intelligence": { "I_open": 60.882134008279415, "I_sealed": 50.81473875559194, "base": 55.84843638193568, "gap": 10.067395252687476, "excess": 4.880767752608233, "penalty": 1, "per_type_split": { "open|choice": { "cc": 80.42963055688979, "tiers": { "easy": 98.19113656918904, "standard": 81.28411749415127, "judge": 89.6265560165975, "hard": 68.66431649040345 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 42.95206095310208, "tiers": { "easy": 64.70588235294117, "standard": 36.36363636363635, "judge": 60.7843137254902, "hard": 27.43362831858407 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 59.26471051484638, "tiers": { "easy": 28.14668835825308, "standard": 75.0123339949518, "judge": 58.475823481817436, "hard": 59.762069588713686 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 73.29707817497771, "tiers": { "easy": 85.3515625, "standard": 70.11546298392577, "judge": 68.19787985865725, "hard": 75.69866342648847 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 16.58181818181818, "tiers": { "easy": 68, "standard": 8.000000000000007, "judge": 19.999999999999996, "hard": 5.454545454545445 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 62.565319909979934, "tiers": { "easy": 73.45705646633482, "standard": 68.81432190455739, "judge": 63.67451290834598, "hard": 55.88599002482795 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 87.44416146719458, "parts": { "choice": { "ece_hard": 0.06834988430518965, "ece_score": 86.33002313896208, "mean_tvd": 0.17367101892281844, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 84.48146062334013 }, "noul": { "ece": 0.04175922105446306, "brier": 0.09947893375488488, "n": 495, "score": 91.64815578910739, "abstention_rate": 0.2828282828282829 }, "score": { "nrps": 0.09880470839715763, "top_ece": 0.08856896591005901, "n": 330, "interval90_coverage": 0.906060606060606, "interval90_width": 0.4032828282828283, "score": 86.20286798913622 } } }, "speed": { "p50_s_raw": 0.01726541091920808, "p95_s_raw": 0.0473672681429889, "p50_s_adjusted": 0.18453082183841615, "p95_s_adjusted": 0.2447345362859778, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.01701689039408867, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 5, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 5, "B": 6, "C": 9 } }, { "key": "decision-4b-v12", "display": "Decision 4B v1.2 (FlyMyJev, Qwen3.5-4B + LoRA)", "addendum": { "id": "A1", "release": "v1.5", "label": "v1.5 roster addendum A1" }, "author": "unknown", "repo": null, "class": "unclassified", "licence": "not recorded", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/decision-4b-v12/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 53.663766457491015, "calibration": 88.56416661568062, "speed": 93.50125558915451, "cost": 63.07359395281681 }, "scores": { "A": 70.82738025747577, "B": 66.56913664963011, "C": 56.657966014846686 }, "views": { "Balanced 33:33:33": 66.39505737102407, "Emphasis on Accuracy 60:20:20": 60.64047420136369, "Emphasis on Cost 20:20:60": 65.02535990714459, "Emphasis on Speed 20:60:20": 75.10419953993227, "Intelligence only": 53.66376645749102 }, "composite_ci95": { "A": [ 68.51591643379686, 71.96768836941042 ], "B": [ 63.67943198254703, 68.39175169030159 ], "C": [ 47.580497523470044, 65.08783016151382 ] }, "jevbench_score": 70.82738025747577, "intelligence": { "I_open": 56.26310318837419, "I_sealed": 51.06442972660784, "base": 53.663766457491015, "gap": 5.198673461766347, "excess": 0.0120459616871047, "penalty": 1, "per_type_split": { "open|choice": { "cc": 80.86757187722047, "tiers": { "easy": 96.38227313837805, "standard": 82.84377436963868, "judge": 92.73858921161826, "hard": 67.09753231492361 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 20.127616613821715, "tiers": { "easy": 58.823529411764696, "standard": 39.393939393939405, "judge": 5.882352941176472, "hard": 11.504424778761058 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 67.79412107408038, "tiers": { "easy": 47.702439113741114, "standard": 79.25132012928306, "judge": 71.68112016355602, "hard": 64.1731927194571 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 73.75526883039994, "tiers": { "easy": 73.63281249999999, "standard": 78.2657912610369, "judge": 68.19787985865725, "hard": 75.69866342648847 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 16.436363636363634, "tiers": { "easy": 60.00000000000001, "standard": 4.0000000000000036, "judge": 19.999999999999996, "hard": 9.090909090909083 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 63.00165671305996, "tiers": { "easy": 78.49758935921852, "standard": 68.57556229972187, "judge": 66.47629048096655, "hard": 53.73474543225941 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 88.56416661568062, "parts": { "choice": { "ece_hard": 0.04734383561120563, "ece_score": 90.53123287775887, "mean_tvd": 0.21448997597778782, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 84.54111763999003 }, "noul": { "ece": 0.05692384040660478, "brier": 0.0977795596104323, "n": 495, "score": 88.61523191867904, "abstention_rate": 0.36969696969696975 }, "score": { "nrps": 0.07819475027704766, "top_ece": 0.035541121977748294, "n": 330, "interval90_coverage": 0.9696969696969697, "interval90_width": 0.4462121212121212, "score": 92.5361502883728 } } }, "speed": { "p50_s_raw": 0.016381043009459972, "p95_s_raw": 0.04716824610950424, "p50_s_adjusted": 0.18276208601891994, "p95_s_adjusted": 0.2443364922190085, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.01701689039408867, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 8, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 8, "B": 9, "C": 12 } }, { "key": "imajev-4b-rtx5090-a2", "display": "Imajev-4B (RTX 5090)", "addendum": { "id": "A2", "release": "v1.5", "label": "v1.5 roster addendum A2" }, "author": "unknown", "repo": null, "class": "unclassified", "licence": "not recorded", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX 5090), offline read-only container", "gpu": "RTX 5090", "adapter_id": "A2 reviewed bundle", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 53.47383727839022, "calibration": 88.11694149062471, "speed": 91.13208929836159, "cost": 63.26458220826236 }, "scores": { "A": 70.38659212275981, "B": 66.19909019518123, "C": 55.907508463046334 }, "views": { "Balanced 33:33:33": 65.96240864336546, "Emphasis on Accuracy 60:20:20": 60.32678975105616, "Emphasis on Cost 20:20:60": 64.85613036338354, "Emphasis on Speed 20:60:20": 74.15469284216614, "Intelligence only": 53.47383727839022 }, "composite_ci95": { "A": [ 67.79966861439989, 71.61191717463264 ], "B": [ 63.11559140704828, 68.08825963056043 ], "C": [ 47.08310320444435, 64.45534727420464 ] }, "jevbench_score": 70.38659212275981, "intelligence": { "I_open": 56.24619253993687, "I_sealed": 50.70148201684356, "base": 53.47383727839022, "gap": 5.544710523093308, "excess": 0.358083023014065, "penalty": 1, "per_type_split": { "open|choice": { "cc": 80.11776048417694, "tiers": { "easy": 98.19113656918904, "standard": 89.08240187158823, "judge": 85.47717842323651, "hard": 67.09753231492361 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 21.823071948006877, "tiers": { "easy": 64.70588235294117, "standard": 51.515151515151516, "judge": 27.450980392156854, "hard": -7.964601769911505 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 66.79774518762679, "tiers": { "easy": 36.12484114152319, "standard": 79.14114403997071, "judge": 68.57269580952628, "hard": 66.96305880655612 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 82.32588710912236, "tiers": { "easy": 88.28125, "standard": 76.9074032148517, "judge": 81.4487632508834, "hard": 84.2041312272175 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 7.563636363636362, "tiers": { "easy": 43.99999999999999, "standard": 4.0000000000000036, "judge": 19.999999999999996, "hard": -9.090909090909093 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 62.21492257777197, "tiers": { "easy": 73.37790947410274, "standard": 65.41303393458439, "judge": 62.0389806565921, "hard": 57.95707661616796 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 88.11694149062471, "parts": { "choice": { "ece_hard": 0.06801188748719926, "ece_score": 86.39762250256014, "mean_tvd": 0.2029350870949496, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 83.05205689653259 }, "noul": { "ece": 0.04795614076750405, "brier": 0.11127612689924851, "n": 495, "score": 90.40877184649919, "abstention_rate": 0.3737373737373737 }, "score": { "nrps": 0.08668615613253966, "top_ece": 0.04775696464530638, "n": 330, "interval90_coverage": 0.9393939393939394, "interval90_width": 0.3345959595959594, "score": 90.88999572884238 } } }, "speed": { "p50_s_raw": 0.041955091524869204, "p95_s_raw": 0.08970695780590177, "p50_s_adjusted": 0.2339101830497384, "p95_s_adjusted": 0.32941391561180355, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.016769261083743846, "basis": "ESTIMATE (I-2): measured input tokens; zero generated output tokens for signed logits readout. M2 floor uses the 25 Sep 2026 DeepInfra Qwen3.5-4B snapshot rates (USD 0.03/M input, USD 0.15/M output); the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen3.5-9B." }, "rank": 9, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 9, "B": 11, "C": 13 } }, { "key": "decision-4b-v11", "display": "Decision 4B v1.1 (FlyMyJev, Qwen3.5-4B + LoRA)", "addendum": { "id": "A1", "release": "v1.5", "label": "v1.5 roster addendum A1" }, "author": "unknown", "repo": null, "class": "unclassified", "licence": "not recorded", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/decision-4b-v11/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 53.10782790674597, "calibration": 87.31445358951783, "speed": 93.52161857471674, "cost": 63.07359395281681 }, "scores": { "A": 70.38575169491591, "B": 66.08572948524592, "C": 55.144135275761755 }, "views": { "Balanced 33:33:33": 66.11304184835826, "Emphasis on Accuracy 60:20:20": 60.21480463822364, "Emphasis on Cost 20:20:60": 64.8627752105624, "Emphasis on Speed 20:60:20": 74.89261623851299, "Intelligence only": 53.107827906745975 }, "composite_ci95": { "A": [ 66.85836636993423, 71.56623969374822 ], "B": [ 62.10775126922529, 67.92588259628072 ], "C": [ 46.42942109023209, 63.50687834004045 ] }, "jevbench_score": 70.38575169491591, "intelligence": { "I_open": 54.6540553015887, "I_sealed": 51.561600511903244, "base": 53.10782790674597, "gap": 3.092454789685455, "excess": -2.0941727103937877, "penalty": 1, "per_type_split": { "open|choice": { "cc": 80.1205807529816, "tiers": { "easy": 98.19113656918904, "standard": 81.28411749415127, "judge": 87.55186721991701, "hard": 69.44770857814335 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 18.021548120454945, "tiers": { "easy": 52.941176470588225, "standard": 42.42424242424243, "judge": 5.882352941176472, "hard": 6.194690265486735 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 65.82003703132958, "tiers": { "easy": 46.64708904955269, "standard": 78.7058833142304, "judge": 67.933060317906, "hard": 62.585583420391046 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 74.76525759162931, "tiers": { "easy": 88.28125, "standard": 75.5490151686665, "judge": 66.87279151943464, "hard": 76.91373025516404 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 15.745454545454546, "tiers": { "easy": 60.00000000000001, "standard": 4.0000000000000036, "judge": 8.000000000000007, "hard": 16.36363636363636 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 64.17408939862588, "tiers": { "easy": 82.70764582802245, "standard": 70.37001442323314, "judge": 64.58457567971891, "hard": 56.134873068153325 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 87.31445358951783, "parts": { "choice": { "ece_hard": 0.061003907977028454, "ece_score": 87.7992184045943, "mean_tvd": 0.2162431550144764, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 83.08745145157333 }, "noul": { "ece": 0.05942903872334748, "brier": 0.09970492807158093, "n": 495, "score": 88.11419225533051, "abstention_rate": 0.38181818181818183 }, "score": { "nrps": 0.08303471757831037, "top_ece": 0.05106547059434831, "n": 330, "interval90_coverage": 0.9666666666666667, "interval90_width": 0.4497474747474746, "score": 90.74171706164965 } } }, "speed": { "p50_s_raw": 0.016276915499474853, "p95_s_raw": 0.04673548646387644, "p50_s_adjusted": 0.1825538309989497, "p95_s_adjusted": 0.24347097292775288, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.01701689039408867, "basis": "documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)" }, "rank": 10, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 10, "B": 12, "C": 14 } }, { "key": "surogate-rune-26b-a4b-v3-rtxpro6000-a2", "display": "Surogate Rune 26B-A4B v3 (RTX PRO 6000)", "addendum": { "id": "A2", "release": "v1.5", "label": "v1.5 roster addendum A2" }, "author": "unknown", "repo": null, "class": "unclassified", "licence": "not recorded", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX PRO 6000), offline read-only container", "gpu": "RTX PRO 6000", "adapter_id": "A2 reviewed bundle", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 69.7384372659144, "calibration": 88.29679782691991, "speed": 85.9674611821838, "cost": 48.9678579741414 }, "scores": { "A": 66.47011066179378, "B": 66.55345952957438, "C": 66.47011066179378 }, "views": { "Balanced 33:33:33": 62.02253148171242, "Emphasis on Accuracy 60:20:20": 63.881582091786825, "Emphasis on Cost 20:20:60": 54.973713258577675, "Emphasis on Speed 20:60:20": 68.84661497656214, "Intelligence only": 66.88895581073571 }, "composite_ci95": { "A": [ 65.43169342866265, 67.00710945239213 ], "B": [ 65.10706397319322, 67.52972503849682 ], "C": [ 65.43169342866265, 67.00710945239213 ] }, "jevbench_score": 66.47011066179378, "intelligence": { "I_open": 68.95080025326385, "I_sealed": 70.52607427856495, "base": 69.7384372659144, "gap": -1.5752740253010984, "excess": -6.761901525380341, "penalty": 1, "per_type_split": { "open|choice": { "cc": 81.98573910797714, "tiers": { "easy": 98.19113656918904, "standard": 95.32102937353783, "judge": 88.58921161825727, "hard": 66.31414022718371 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 45.58752543656239, "tiers": { "easy": 76.47058823529412, "standard": 45.45454545454546, "judge": 66.66666666666667, "hard": 22.123893805309724 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 79.27913621525204, "tiers": { "easy": 77.45483328730258, "standard": 87.3570213774979, "judge": 80.61432644653624, "hard": 74.69487669265334 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 82.17410009259264, "tiers": { "easy": 94.140625, "standard": 79.6241793072221, "judge": 78.79858657243817, "hard": 82.98906439854193 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 49.52727272727273, "tiers": { "easy": 84.00000000000001, "standard": 43.99999999999999, "judge": 52, "hard": 41.81818181818182 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 79.87685001582949, "tiers": { "easy": 81.44270010435835, "standard": 80.09762263276632, "judge": 83.36030214546362, "hard": 76.76241208800323 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 88.29679782691991, "parts": { "choice": { "ece_hard": 0.048130505772540376, "ece_score": 90.37389884549192, "mean_tvd": 0.26981249787409634, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 81.69632452904114 }, "noul": { "ece": 0.041444379611227856, "brier": 0.06554161668204889, "n": 495, "score": 91.71112407775442, "abstention_rate": 0.24444444444444446 }, "score": { "nrps": 0.04234138893172694, "top_ece": 0.06399985679449458, "n": 330, "interval90_coverage": 0.9939393939393939, "interval90_width": 0.28257575757575754, "score": 91.4829448739642 } } }, "speed": { "p50_s_raw": 0.10046782437711954, "p95_s_raw": 0.28557565337978286, "p50_s_adjusted": 0.3509356487542391, "p95_s_adjusted": 0.7211513067595657, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.050242518472906394, "basis": "ESTIMATE: M2 base-model market reference from the frozen 25 Sep 2026 OpenRouter snapshot applied to measured tokens." }, "rank": 16, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 16, "B": 10, "C": 6 } }, { "key": "autojev-27b", "display": "AutoJev-27B (denis-pplx, Qwen3.8-27B)", "addendum": { "id": "A1", "release": "v1.5", "label": "v1.5 roster addendum A1" }, "author": "unknown", "repo": null, "class": "unclassified", "licence": "not recorded", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/autojev-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 72.78028107753738, "calibration": 87.70317647540496, "speed": 87.62166952486342, "cost": 29.365937020749882 }, "scores": { "A": 19.54135920374164, "B": 20.447669591289714, "C": 19.54135920374164 }, "views": { "Balanced 33:33:33": 17.478538208811404, "Emphasis on Accuracy 60:20:20": 19.89620159601279, "Emphasis on Cost 20:20:60": 13.547178051841165, "Emphasis on Speed 20:60:20": 21.025141031925013, "Intelligence only": 25.10507053670285 }, "composite_ci95": { "A": [ 19.299804948470786, 19.631386859392 ], "B": [ 20.095989263006373, 20.648563859347476 ], "C": [ 19.299804948470786, 19.631386859392 ] }, "jevbench_score": 19.54135920374164, "intelligence": { "I_open": 71.91293867189668, "I_sealed": 73.64762348317807, "base": 72.78028107753738, "gap": -1.734684811281383, "excess": -6.921312311360626, "penalty": 1, "per_type_split": { "open|choice": { "cc": 85.2388568573883, "tiers": { "easy": 96.38227313837805, "standard": 95.32102937353783, "judge": 94.81327800829875, "hard": 70.23110066588326 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 50.436010285047246, "tiers": { "easy": 76.47058823529412, "standard": 69.6969696969697, "judge": 66.66666666666667, "hard": 22.123893805309724 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 80.06394887325453, "tiers": { "easy": 80.58788363302166, "standard": 86.65872826406437, "judge": 80.03010394222157, "hard": 76.66095918618254 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 82.24130918153313, "tiers": { "easy": 91.2109375, "standard": 80.98256735340728, "judge": 77.47349823321555, "hard": 84.2041312272175 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 62, "tiers": { "easy": 84.00000000000001, "standard": 52, "judge": 63.99999999999999, "hard": 60.00000000000001 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 76.70156126800109, "tiers": { "easy": 81.85626924529423, "standard": 75.87669579367935, "judge": 76.44648330538433, "hard": 76.01662548280125 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 87.70317647540496, "parts": { "choice": { "ece_hard": 0.04212943593705838, "ece_score": 91.57411281258833, "mean_tvd": 0.25162423089203245, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 83.20584486169254 }, "noul": { "ece": 0.04052854529071123, "brier": 0.05503997046028658, "n": 495, "score": 91.89429094185775, "abstention_rate": 0.21010101010101012 }, "score": { "nrps": 0.04502967345008432, "top_ece": 0.09739122704831175, "n": 330, "interval90_coverage": 0.9878787878787879, "interval90_width": 0.29267676767676765, "score": 88.00939362266462 } } }, "speed": { "p50_s_raw": 0.10088440257823095, "p95_s_raw": 0.17077954066335213, "p50_s_adjusted": 0.3517688051564619, "p95_s_adjusted": 0.49155908132670423, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.22618758620689655, "basis": "documented hosted-model estimate" }, "rank": 50, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 50, "B": 47, "C": 45 } }, { "key": "autojev-27b-rtxpro6000-a2", "display": "AutoJev-27B (RTX PRO 6000)", "addendum": { "id": "A2", "release": "v1.5", "label": "v1.5 roster addendum A2" }, "author": "unknown", "repo": null, "class": "unclassified", "licence": "not recorded", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX PRO 6000), offline read-only container", "gpu": "RTX PRO 6000", "adapter_id": "A2 reviewed bundle", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 72.7504220700132, "calibration": 86.68041666712992, "speed": 87.06975289886662, "cost": 29.365937020749882 }, "scores": { "A": 19.482719520484327, "B": 20.394920235576496, "C": 19.482719520484327 }, "views": { "Balanced 33:33:33": 17.455546885699814, "Emphasis on Accuracy 60:20:20": 19.875735473140935, "Emphasis on Cost 20:20:60": 13.538885114405412, "Emphasis on Speed 20:60:20": 20.968224367337445, "Intelligence only": 25.094770872027855 }, "composite_ci95": { "A": [ 19.262439246036728, 19.59873421101785 ], "B": [ 20.06322075545375, 20.611081472646248 ], "C": [ 19.262439246036728, 19.59873421101785 ] }, "jevbench_score": 19.482719520484327, "intelligence": { "I_open": 72.0148638915178, "I_sealed": 73.4859802485086, "base": 72.7504220700132, "gap": -1.471116356990791, "excess": -6.657743857070034, "penalty": 1, "per_type_split": { "open|choice": { "cc": 84.92550002229234, "tiers": { "easy": 96.38227313837805, "standard": 95.32102937353783, "judge": 94.81327800829875, "hard": 69.44770857814335 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 51.14397488681716, "tiers": { "easy": 76.47058823529412, "standard": 69.6969696969697, "judge": 66.66666666666667, "hard": 23.893805309734507 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 79.97511676544391, "tiers": { "easy": 80.15537374983703, "standard": 86.72749010669713, "judge": 79.87342491381291, "hard": 76.63013473744229 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 81.69795396305905, "tiers": { "easy": 91.2109375, "standard": 78.2657912610369, "judge": 77.47349823321555, "hard": 84.2041312272175 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 62, "tiers": { "easy": 84.00000000000001, "standard": 52, "judge": 63.99999999999999, "hard": 60.00000000000001 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 76.75998678246677, "tiers": { "easy": 81.77890543489788, "standard": 76.16408820390532, "judge": 76.5872741824193, "hard": 75.9327408586753 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 86.68041666712992, "parts": { "choice": { "ece_hard": 0.05563197319122527, "ece_score": 88.87360536175495, "mean_tvd": 0.25077706263870864, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 81.89794954894205 }, "noul": { "ece": 0.04353587849034292, "brier": 0.05491962545051717, "n": 495, "score": 91.29282430193142, "abstention_rate": 0.2080808080808081 }, "score": { "nrps": 0.04503821273971415, "top_ece": 0.10897613212498017, "n": 330, "interval90_coverage": 0.9878787878787879, "interval90_width": 0.29318181818181815, "score": 86.85047615051627 } } }, "speed": { "p50_s_raw": 0.08811413822695613, "p95_s_raw": 0.22593528539873534, "p50_s_adjusted": 0.3262282764539123, "p95_s_adjusted": 0.6018705707974706, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.22618758620689655, "basis": "ESTIMATE: M2 base-model market reference from the frozen 25 Sep 2026 OpenRouter snapshot applied to measured tokens." }, "rank": 51, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 51, "B": 48, "C": 46 } }, { "key": "eikos-27b", "display": "Eikos-27B (caiovicentino1, Qwen3.8-27B)", "addendum": { "id": "A1", "release": "v1.5", "label": "v1.5 roster addendum A1" }, "author": "unknown", "repo": null, "class": "unclassified", "licence": "not recorded", "open": "unknown", "underlying": null, "endpoint_kind": "gpu", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "gpu": "H100", "adapter_id": "images/eikos-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": false, "api_exposure_note": null, "status": { "status": "complete", "rows": 1624, "missing": 0, "answered_ok": 1624 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 75.08487940331634, "calibration": 86.28582452360858, "speed": 87.62506251841538, "cost": 28.689752882469676 }, "scores": { "A": 18.503023570753697, "B": 19.483126908668613, "C": 18.503023570753697 }, "views": { "Balanced 33:33:33": 16.576368026746824, "Emphasis on Accuracy 60:20:20": 19.092466134747955, "Emphasis on Cost 20:20:60": 12.731910464092147, "Emphasis on Speed 20:60:20": 19.97561012269975, "Intelligence only": 24.721003373666086 }, "composite_ci95": { "A": [ 18.282206247053008, 18.598674719666235 ], "B": [ 19.15860082154746, 19.66517783572774 ], "C": [ 18.282206247053008, 18.598674719666235 ] }, "jevbench_score": 18.503023570753697, "intelligence": { "I_open": 74.1460564615501, "I_sealed": 76.02370234508257, "base": 75.08487940331634, "gap": -1.877645883532466, "excess": -7.064273383611709, "penalty": 1, "per_type_split": { "open|choice": { "cc": 87.62039899732592, "tiers": { "easy": 98.19113656918904, "standard": 92.20171562256304, "judge": 92.73858921161826, "hard": 78.84841363102232 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": 56.46932626630701, "tiers": { "easy": 76.47058823529412, "standard": 69.6969696969697, "judge": 49.01960784313726, "hard": 50.44247787610618 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": 78.34844412101737, "tiers": { "easy": 77.79806316637085, "standard": 83.74739299176998, "judge": 80.10505471948588, "hard": 74.46910697545131 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 88.26916117896289, "tiers": { "easy": 91.2109375, "standard": 85.05773149196287, "judge": 86.74911660777386, "hard": 90.27946537059537 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": 64.14545454545456, "tiers": { "easy": 92, "standard": 60.00000000000001, "judge": 68, "hard": 56.363636363636374 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": 75.65649131083029, "tiers": { "easy": 79.54885680667138, "standard": 74.95659667272105, "judge": 76.56023976602533, "hard": 74.35553591452833 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0 }, "noul": { "n": 495, "invalid_rate": 0 }, "score": { "n": 330, "invalid_rate": 0 } }, "calibration": { "score": 86.28582452360858, "parts": { "choice": { "ece_hard": 0.03693915188097349, "ece_score": 92.6121696238053, "mean_tvd": 0.2546374930060419, "n_tvd": 50, "n": 799, "n_ece_hard": 277, "score": 83.57421016160055 }, "noul": { "ece": 0.07420246840426414, "brier": 0.054408630663178775, "n": 495, "score": 85.15950631914717, "abstention_rate": 0.1838383838383838 }, "score": { "nrps": 0.04704954457108033, "top_ece": 0.0752376568136793, "n": 330, "interval90_coverage": 0.9636363636363636, "interval90_width": 0.24343434343434348, "score": 90.12375709007804 } } }, "speed": { "p50_s_raw": 0.10209655796643347, "p95_s_raw": 0.16890664728125523, "p50_s_adjusted": 0.35419311593286695, "p95_s_adjusted": 0.48781329456251044, "n": 450, "adjustment": "x2 + 0.15 s (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.23823646551724137, "basis": "documented hosted-model estimate" }, "rank": 53, "ranked": true, "listing": "ranked", "not_ranked_because": null, "ranks": { "A": 53, "B": 49, "C": 48 } }, { "key": "simplejev-qwen3.6-35b-a3b", "display": "SimpleJev Qwen3.6-35B-A3B", "addendum": null, "author": "Featherless AI", "repo": "https://github.com/featherless-ai/simple-jev", "class": "jev-rebuild", "licence": "Apache-2.0 (Qwen weights); repository licence not stated", "open": "yes", "underlying": "Qwen3.6-35B-A3B through SimpleJev's direct-logit classifier", "endpoint_kind": "demo", "endpoint_condition": "the author's public demo endpoint", "gpu": null, "adapter_id": "harness/specs/simplejev-qwen3.6-35b-a3b.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_flag": true, "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "status": { "status": "partial", "rows": 678, "missing": 946, "answered_ok": 677 }, "full_coverage": true, "support": { "choice": "native", "noul": "native", "score": "native" }, "planned_weight_denominator": 1, "axes": { "intelligence": 0, "calibration": 78.49852428212563, "speed": 75.18371345389433, "cost": 35.1485010409676 }, "scores": { "A": 0, "B": 0, "C": 0 }, "views": { "Balanced 33:33:33": 0, "Emphasis on Accuracy 60:20:20": 0, "Emphasis on Cost 20:20:60": 0, "Emphasis on Speed 20:60:20": 0, "Intelligence only": 0 }, "composite_ci95": { "A": [ 0, 0 ], "B": [ 0, 0 ], "C": [ 0, 0 ] }, "jevbench_score": 0, "intelligence": { "I_open": -17.361123164086187, "I_sealed": -22.966144581980604, "base": -20.163633873033397, "gap": 5.605021417894417, "excess": 0.4183939178151741, "penalty": 1, "per_type_split": { "open|choice": { "cc": 16.21556622973974, "tiers": { "easy": 27.645462767561053, "standard": 20.457499350142964, "judge": 27.385892116182585, "hard": 2.8593811202506667 }, "n": { "easy": 72, "standard": 84, "judge": 114, "hard": 169 } }, "open|noul": { "cc": -36.28602527092897, "tiers": { "easy": -47.05882352941176, "standard": -18.181818181818176, "judge": -23.529411764705888, "hard": -52.212389380530965 }, "n": { "easy": 34, "standard": 66, "judge": 102, "hard": 113 } }, "open|score": { "cc": -32.012910451069345, "tiers": { "easy": -61.6763097560751, "standard": -30.179560922548788, "judge": -26.90162155364877, "hard": -29.34720206214361 }, "n": { "easy": 16, "standard": 50, "judge": 34, "hard": 50 } }, "sealed|choice": { "cc": 8.941634454184864, "tiers": { "easy": 17.968749999999982, "standard": 4.912836767036478, "judge": 15.194346289752657, "hard": 4.00972053462943 }, "n": { "easy": 50, "standard": 101, "judge": 101, "hard": 108 } }, "sealed|noul": { "cc": -42.690909090909095, "tiers": { "easy": -12, "standard": -48, "judge": -36, "hard": -52.727272727272734 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } }, "sealed|score": { "cc": -35.149159109217585, "tiers": { "easy": -7.190320322368482, "standard": -36.15560725047939, "judge": -37.42010434672649, "hard": -39.93243580716728 }, "n": { "easy": 25, "standard": 50, "judge": 50, "hard": 55 } } } }, "validity": { "choice": { "n": 799, "invalid_rate": 0.5794743429286608 }, "noul": { "n": 495, "invalid_rate": 0.5717171717171717 }, "score": { "n": 330, "invalid_rate": 0.6090909090909091 } }, "calibration": { "score": 78.49852428212563, "parts": { "choice": { "ece_hard": 0.13284368943242192, "ece_score": 73.43126211351561, "mean_tvd": 0.3785573581540708, "n_tvd": 19, "n": 336, "n_ece_hard": 103, "score": 67.78776314905426 }, "noul": { "ece": 0.06196859662122321, "brier": 0.10697271930761937, "n": 212, "score": 87.60628067575536, "abstention_rate": 0.21698113207547165 }, "score": { "nrps": 0.08336163853547077, "top_ece": 0.1573038905165916, "n": 129, "interval90_coverage": 0.8682170542635659, "interval90_width": 0.2034883720930232, "score": 80.1015290215673 } } }, "speed": { "p50_s_raw": 0.8297080211341381, "p95_s_raw": 0.9133629161864518, "p50_s_adjusted": 1.6594160422682762, "p95_s_adjusted": 1.8267258323729036, "n": 198, "adjustment": "x2 demo-endpoint adjustment (assumption, not measured)" }, "cost": { "kind": "estimate", "usd_per_1000": 0.1451164454277286, "basis": "ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)" }, "rank": null, "ranked": false, "listing": "partial", "not_ranked_because": "Partial run: 677 of 1,624 decisions answered; the missing ones count wrong and the row is not ranked.", "ranks": { "A": null, "B": null, "C": null } }, { "adapter_id": "a3/candidates/nemotron-diffusion-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "addendum": { "id": "A3", "label": "v1.5 roster addendum A3", "release": "v1.5" }, "api_exposure_note": null, "api_flag": false, "author": "pst2154", "axes": { "calibration": 76.24688293519233, "cost": 55.49170925809556, "intelligence": 33.84367245974849, "speed": 93.74324086172008 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09594518072709125, "ece_score": 80.81096385458176, "mean_tvd": 0.3020534686259666, "n": 799, "n_ece_hard": 277, "n_tvd": 50, "score": 75.30280849599255 }, "noul": { "abstention_rate": 0.3656565656565657, "brier": 0.18855886879495318, "ece": 0.1293343161293865, "n": 495, "score": 74.1331367741227 }, "score": { "interval90_coverage": 0.906060606060606, "interval90_width": 0.5267676767676769, "n": 330, "nrps": 0.1366270109569611, "score": 79.30470353546178, "top_ece": 0.1386394591669016 } }, "score": 76.24688293519233 }, "class": "system-one-open", "composite_ci95": { "A": [ 18.600485994675733, 32.782504356457665 ], "B": [ 16.093431518950908, 29.451782528235324 ], "C": [ 12.91700416296926, 22.765628025317827 ] }, "cost": { "basis": "documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usd_per_1000": 0.03045157019704434 }, "display": "Nemotron Diffusion 8B (pst2154, optimized vLLM)", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "endpoint_kind": "gpu", "full_coverage": true, "gpu": "H100", "intelligence": { "I_open": 39.537877662793036, "I_sealed": 28.14946725670395, "base": 33.84367245974849, "excess": 6.201782906009844, "gap": 11.388410406089086, "penalty": 1, "per_type_split": { "open|choice": { "cc": 57.04524835356567, "n": { "easy": 72, "hard": 169, "judge": 114, "standard": 84 }, "tiers": { "easy": 89.14681941513415, "hard": 44.3791617704661, "judge": 56.43153526970955, "standard": 67.24720561476475 } }, "open|noul": { "cc": 10.773429242976352, "n": { "easy": 34, "hard": 113, "judge": 102, "standard": 66 }, "tiers": { "easy": 52.941176470588225, "hard": -27.43362831858407, "judge": 56.86274509803921, "standard": -3.0303030303030276 } }, "open|score": { "cc": 50.79495539183709, "n": { "easy": 16, "hard": 50, "judge": 34, "standard": 50 }, "tiers": { "easy": 24.69718225067451, "hard": 53.55396011795156, "judge": 45.622339029237644, "standard": 66.08475705408861 } }, "sealed|choice": { "cc": 58.80957283866533, "n": { "easy": 50, "hard": 108, "judge": 101, "standard": 101 }, "tiers": { "easy": 79.4921875, "hard": 51.39732685297693, "judge": 64.22261484098941, "standard": 55.17319447588863 } }, "sealed|noul": { "cc": -16.909090909090907, "n": { "easy": 25, "hard": 55, "judge": 50, "standard": 50 }, "tiers": { "easy": 4.0000000000000036, "hard": -27.27272727272727, "judge": 8.000000000000007, "standard": -43.99999999999999 } }, "sealed|score": { "cc": 42.54791984053743, "n": { "easy": 25, "hard": 55, "judge": 50, "standard": 50 }, "tiers": { "easy": 59.566799606231854, "hard": 36.06447446405748, "judge": 38.947749671897505, "standard": 52.405625963610014 } } } }, "jevbench_score": 25.684702471457477, "key": "nemotron-diffusion-8b", "licence": "MIT (application code); modified vLLM runtime Apache-2.0; nvidia/Nemotron-Labs-Diffusion-8B keeps its upstream licence", "listing": "ranked", "not_ranked_because": null, "open": "yes", "planned_weight_denominator": 1, "rank": 43, "ranked": true, "ranks": { "A": 43, "B": 45, "C": 53 }, "repo": "https://github.com/pst2154/Nemotron_Jev", "scores": { "A": 25.684702471457477, "B": 22.70384890919707, "C": 17.836598938512132 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 450, "p50_s_adjusted": 0.18745839605107903, "p50_s_raw": 0.018729198025539517, "p95_s_adjusted": 0.22530509717762465, "p95_s_raw": 0.03765254858881232 }, "status": { "answered_ok": 1624, "missing": 0, "rows": 1624, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "underlying": "nvidia/Nemotron-Labs-Diffusion-8B @ 16c67f05 (unchanged weights) served by github.com/pst2154/Nemotron_Jev @ cdc36d60 on the author's digest-pinned modified vLLM runtime: BF16, candidate-only output projection, direct candidate logits, CUDA graphs, one masked forward pass per decision, nothing generated. Disclosed by the author: the inherited option-ordering rule was selected using public benchmark examples in earlier 14B experiments. Our earlier v1.4.3 run of this system used a recipe that left the base image's older 14B application in place; this v1.5 row is a fresh measurement of the 8B.", "validity": { "choice": { "invalid_rate": 0, "n": 799 }, "noul": { "invalid_rate": 0, "n": 495 }, "score": { "invalid_rate": 0, "n": 330 } }, "views": { "Balanced 33:33:33": 23.60187012246317, "Emphasis on Accuracy 60:20:20": 19.52415109652738, "Emphasis on Cost 20:20:60": 24.29843756293519, "Emphasis on Speed 20:60:20": 28.78936736571111, "Intelligence only": 15.505737990644903 } } ], "not_measured": [ { "key": "jobe-qwen3.5-4b", "display": "Jobe Qwen3.5-4B (frozen)", "author": "MantisShrimpdev", "addendum": null, "status": "not measured", "reason": null }, { "key": "mica-v01-4b", "display": "mica-v01-4b", "author": "unknown", "addendum": { "id": "A1", "release": "v1.5", "label": "v1.5 roster addendum A1" }, "status": "partial run", "reason": "The frozen refusal policy stopped the run after 1,088 of 1,624 rows: 27 documented refusals were mapped to HTTP 422, then three consecutive passthrough HTTP 400 refusals triggered exit 6. The remaining 536 rows have no scores, so this system is not eligible for an official rank.", "rows": 1088, "missing": 536 }, { "key": "openjev-razorback16", "display": "OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16)", "author": "razorback16 / Codiv", "addendum": null, "status": "not measured", "reason": null } ], "roster_count": 104, "headline_hold": null, "parent_release": { "revision": "v1.5.1", "sha256": "6f2fa547454b1108fad701ef302f48450742562393d532d45eccd048f736a9e2" }, "revision_note": "Nemotron Diffusion 8B joins the official A/B/C rankings as a complete A3 roster-addendum system. Existing scores and intervals are unchanged; ranks are recalculated against v1.5.1. Wity-1 is held pending production-build identity confirmation and a clean rerun. The method and frozen v1.5.0 G_med are unchanged.", "paired_comparison_note": "No new paired-bootstrap comparisons were computed for A3 addendum systems. Existing tie markers are retained only for previously marked adjacent pairs that remain adjacent in this revision; missing markers do not imply a tie or a separation.", "addendum_sources_sha256": { "A3_aggregate": "424be19ec82f8f784fc264955ae0e4a76dbfb1b71486a21741b91c450c747c8c", "A3_protocol": "369d916b4ba00e48e6bc4669c7d05feaa9614554887ec9a3c721c231ed734648", "merged_aggregate": "f4c0a2d74018d49f115eea6b2cee88b7a45dad6e4524e3a34de8565ca05692ff", "projected_rows": "ddb1c8c84ec409a4146c05c772efa2ffbe1adcd8bbe92b5bb3006b34c3890d03", "independent_numeric_comparison": "c87301cd91e27d796775f51b8c5f7947294c1812b31b4211d78e5a81472cc080" } }