{"results_version":"94aef0c1","questions":2198,"sure_threshold":0.99,"units":[{"number":1,"name":"SÖZCÜKTE VE SÖZ ÖBEKLERİNDE ANLAM","english":"Word and phrase meanings","questions":131},{"number":2,"name":"CÜMLE ANLAMI","english":"Sentence meanings","questions":154},{"number":3,"name":"ANLATIM TEKNİKLERİ VE DÜŞÜNCEYİ GELİŞTİRME YOLLARI","english":"How a writer explains and develops ideas","questions":70},{"number":4,"name":"PARAGRAFTA KONU VE ANA DÜŞÜNCE","english":"Topic and main point of a paragraph","questions":98},{"number":5,"name":"PARAGRAFIN YAPISI","english":"How a paragraph fits together","questions":106},{"number":6,"name":"PARAGRAFTA YARDIMCI DÜŞÜNCELER","english":"Details that support a paragraph’s main point","questions":97},{"number":7,"name":"SÖZCÜK TÜRLERİ","english":"Word types, such as nouns and verbs","questions":140},{"number":8,"name":"TAMLAMALAR","english":"Groups of words built around a noun","questions":65},{"number":9,"name":"FİİLLER (EYLEMLER)","english":"Verbs and their forms","questions":91},{"number":10,"name":"FİİLİMSİLER","english":"Verb forms used like nouns or modifiers","questions":72},{"number":11,"name":"EKLER","english":"Word endings and what they do","questions":79},{"number":12,"name":"SÖZCÜK YAPISI","english":"How words are built","questions":69},{"number":13,"name":"CÜMLENİN ÖGELERİ","english":"The roles of words and phrases in a sentence","questions":145},{"number":14,"name":"FİİL ÇATISI","english":"How verbs show who acts and who receives the action","questions":74},{"number":15,"name":"CÜMLE ÇEŞİTLERİ","english":"Types of sentences","questions":85},{"number":16,"name":"KARMA DİL BİLGİSİ","english":"Questions combining several grammar topics","questions":220},{"number":17,"name":"SES BİLGİSİ","english":"Sound patterns and changes","questions":77},{"number":18,"name":"YAZIM KURALLARI","english":"Spelling","questions":133},{"number":19,"name":"NOKTALAMA İŞARETLERİ","english":"Punctuation","questions":117},{"number":20,"name":"ANLATIM BOZUKLUKLARI","english":"Problems in sentence wording","questions":175}],"prices":[{"provider":"agbench","model":"gemini-4-carbon","perplexity_mode":null,"basis":"Google's Gemini API regular price (from 1 December 2026)","source":"https://ai.google.dev/gemini-api/docs/pricing","date":"2026-11-12","change":"introductory until 30 November 2026: $2.00 input, $10.00 output and $0.10 cached input per 1M tokens","applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":4.0,"cached_input_tokens":0.2,"cache_write_input_tokens":4.0,"cache_write_1h_input_tokens":4.0,"output_tokens":20.0},"text":"$4 input, $0.20 cached input, $20 output per 1M tokens"},{"provider":"agbench","model":"gemini-4-hydrogen","perplexity_mode":null,"basis":"Google's Gemini API price","source":"https://ai.google.dev/gemini-api/docs/pricing","date":"2026-11-12","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":1.5,"cached_input_tokens":0.075,"cache_write_input_tokens":1.5,"cache_write_1h_input_tokens":1.5,"output_tokens":7.5},"text":"$1.50 input, $0.075 cached input, $7.50 output per 1M tokens"},{"provider":"baselabs","model":"baselabs/elephant-1","perplexity_mode":null,"basis":"Baseten's list price for the same model","source":"https://www.baseten.co/library/elephant-1/","date":"2026-11-01","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.035,"cached_input_tokens":0.035,"cache_write_input_tokens":0.035,"cache_write_1h_input_tokens":0.035,"output_tokens":0.0},"text":"$0.035 input, output free per 1M tokens"},{"provider":"baselabs","model":"baselabs/mammoth-1","perplexity_mode":null,"basis":"Baseten's list price for the same model","source":"https://www.baseten.co/library/mammoth-1/","date":"2026-11-01","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.1,"cached_input_tokens":0.1,"cache_write_input_tokens":0.1,"cache_write_1h_input_tokens":0.1,"output_tokens":0.0},"text":"$0.10 input, output free per 1M tokens"},{"provider":"celeris","model":"celeris-1-decision","perplexity_mode":null,"basis":"Celeris's decision model price","source":"https://docs.celeris.ai/pricing","date":"2026-10-10","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.04,"cached_input_tokens":0.04,"cache_write_input_tokens":0.04,"cache_write_1h_input_tokens":0.04,"output_tokens":0.0},"text":"$0.04 input, output free per 1M tokens"},{"provider":"cerno","model":"apertixai/cerno-9b-v1.2","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"claude","model":"claude-fable-5-1","perplexity_mode":null,"basis":"Anthropic's API list price","source":"https://platform.claude.com/docs/en/about-claude/pricing","date":"2026-10-28","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":10.0,"cached_input_tokens":0.25,"cache_write_input_tokens":12.5,"cache_write_1h_input_tokens":20.0,"output_tokens":50.0},"text":"$10 input, $0.25 cached input, $12.50 5-minute cache writes, $20 1-hour cache writes, $50 output per 1M tokens"},{"provider":"claude","model":"claude-fable-5-2","perplexity_mode":null,"basis":"Anthropic's API list price","source":"https://platform.claude.com/docs/en/about-claude/pricing","date":"2026-11-11","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":8.0,"cached_input_tokens":0.2,"cache_write_input_tokens":10.0,"cache_write_1h_input_tokens":16.0,"output_tokens":40.0},"text":"$8 input, $0.20 cached input, $10 5-minute cache writes, $16 1-hour cache writes, $40 output per 1M tokens"},{"provider":"claude","model":"claude-opus-5-5","perplexity_mode":null,"basis":"Anthropic's API list price","source":"https://platform.claude.com/docs/en/about-claude/pricing","date":"2026-10-08","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":4.0,"cached_input_tokens":0.2,"cache_write_input_tokens":5.0,"cache_write_1h_input_tokens":8.0,"output_tokens":20.0},"text":"$4 input, $0.20 cached input, $5 5-minute cache writes, $8 1-hour cache writes, $20 output per 1M tokens"},{"provider":"claude","model":"claude-sonnet-5-5","perplexity_mode":null,"basis":"Anthropic's API list price","source":"https://platform.claude.com/docs/en/about-claude/pricing","date":"2026-10-08","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":2.0,"cached_input_tokens":0.1,"cache_write_input_tokens":2.5,"cache_write_1h_input_tokens":4.0,"output_tokens":10.0},"text":"$2 input, $0.10 cached input, $2.50 5-minute cache writes, $4 1-hour cache writes, $10 output per 1M tokens"},{"provider":"clef","model":"clef","perplexity_mode":null,"basis":"Cloudflare's list price for the same model","source":"https://developers.cloudflare.com/workers-ai/models/clef/","date":"2026-10-18","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.24,"cached_input_tokens":0.24,"cache_write_input_tokens":0.24,"cache_write_1h_input_tokens":0.24,"output_tokens":0.0},"text":"$0.24 input, output free per 1M tokens"},{"provider":"clef","model":"clef-flash","perplexity_mode":null,"basis":"Cloudflare's list price for the same model","source":"https://developers.cloudflare.com/workers-ai/models/clef-flash/","date":"2026-10-18","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.038,"cached_input_tokens":0.038,"cache_write_input_tokens":0.038,"cache_write_1h_input_tokens":0.038,"output_tokens":0.0},"text":"$0.038 input, output free per 1M tokens"},{"provider":"copernicus","model":"Copernicus-31B","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"cygnet","model":"cygnet","perplexity_mode":null,"basis":"SiliconFlow's list price for the same weights (Gemma 4 12B IT; precision not stated, ours BF16)","source":"https://www.siliconflow.com/models/gemma-4-12b-it","date":"2026-11-05","change":null,"applies_to":"weights","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.1,"cached_input_tokens":0.1,"cache_write_input_tokens":0.1,"cache_write_1h_input_tokens":0.1,"output_tokens":0.3},"text":"$0.10 input, $0.30 output per 1M tokens"},{"provider":"d1-server","model":"LiquidAI/d1-2B","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"d3-server","model":"vllm-sr/d3-ultra","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"di-serve","model":"Chinese-Decision-Pointer-4B","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"di-serve","model":"Dexy-Decision-9B-V2","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"di-serve","model":"Jeffan-9B-1211","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"di-serve","model":"Logits-Sigma-1-Duo","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"di-serve","model":"Verdin-4B-v1.1","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"di-serve","model":"vera-medium-omni","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"drex-max","model":"drex-max-v1","perplexity_mode":null,"basis":"Nace.AI's list price for the same model","source":"https://nace.ai/drex","date":"2026-10-30","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.06,"cached_input_tokens":0.06,"cache_write_input_tokens":0.06,"cache_write_1h_input_tokens":0.06,"output_tokens":0.0},"text":"$0.06 input, output free per 1M tokens"},{"provider":"fastino","model":"fastino/GLiDE","perplexity_mode":null,"basis":"Fastino's GLiDE price","source":"https://docs.fastino.ai/pricing","date":"2026-10-08","change":"$0.04 per 1M input tokens since, as listed on 22 October 2026","applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.15,"cached_input_tokens":0.15,"cache_write_input_tokens":0.15,"cache_write_1h_input_tokens":0.15,"output_tokens":0.0},"text":"$0.15 input, output free per 1M tokens"},{"provider":"gemini","model":"gemini-3.8-flash","perplexity_mode":null,"basis":"Google's Gemini API price, introductory","source":"https://ai.google.dev/gemini-api/docs/pricing","date":"2026-10-08","change":"doubles on 1 January 2027, to $1.50 input and $7.50 output per 1M tokens","applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.75,"cached_input_tokens":0.075,"cache_write_input_tokens":0.75,"cache_write_1h_input_tokens":0.75,"output_tokens":3.75},"text":"$0.75 input, $0.075 cached input, $3.75 output per 1M tokens"},{"provider":"gemini","model":"gemini-4-argon","perplexity_mode":null,"basis":"Google's Gemini API regular price (from 1 December 2026)","source":"https://ai.google.dev/gemini-api/docs/pricing","date":"2026-10-28","change":"introductory until 30 November 2026: $2.00 input, $10.00 output and $0.10 cached input per 1M tokens","applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":4.0,"cached_input_tokens":0.2,"cache_write_input_tokens":4.0,"cache_write_1h_input_tokens":4.0,"output_tokens":20.0},"text":"$4 input, $0.20 cached input, $20 output per 1M tokens"},{"provider":"glide","model":"glide-v1.2","perplexity_mode":null,"basis":"Fastino's list price for the same model","source":"https://docs.fastino.ai/pricing","date":"2026-10-22","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.04,"cached_input_tokens":0.04,"cache_write_input_tokens":0.04,"cache_write_1h_input_tokens":0.04,"output_tokens":0.0},"text":"$0.04 input, output free per 1M tokens"},{"provider":"gliner","model":"gliner2.5-multi-decide","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"gliner","model":"gliner2.5-multi-v1","perplexity_mode":null,"basis":"Fastino's list price for the same model","source":"https://docs.fastino.ai/pricing","date":"2026-11-05","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.03,"cached_input_tokens":0.03,"cache_write_input_tokens":0.03,"cache_write_1h_input_tokens":0.03,"output_tokens":0.0},"text":"$0.03 input, output free per 1M tokens"},{"provider":"jev","model":"jev-1.13.0","perplexity_mode":null,"basis":"TypeSafe's System One price","source":"https://typesafe.ai/blog/introducing-system-one-models-and-jev","date":"2026-10-08","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.042,"cached_input_tokens":0.042,"cache_write_input_tokens":0.042,"cache_write_1h_input_tokens":0.042,"output_tokens":0.0},"text":"$0.042 input, output free per 1M tokens"},{"provider":"jev","model":"jev-1.14.0","perplexity_mode":null,"basis":"TypeSafe's System One price","source":"https://docs.typesafe.ai/models","date":"2026-10-16","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.042,"cached_input_tokens":0.042,"cache_write_input_tokens":0.042,"cache_write_1h_input_tokens":0.042,"output_tokens":0.0},"text":"$0.042 input, output free per 1M tokens"},{"provider":"karpo","model":"karpo-decision","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"knowline","model":"PelaAI/KnowLine-12B-Gen1","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"knowline","model":"PelaAI/KnowLine-31B-Gen2","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"knowline","model":"PelaAI/KnowLine-9B-Gen1","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"knowline","model":"PelaAI/KnowLine-9B-Gen2","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"laya","model":"laya","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"laya","model":"laya-multilingual","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"liquid","model":"d1-ultra","perplexity_mode":null,"basis":"Liquid AI's d1-ultra price","source":"https://www.liquid.ai/blog/d1-ultra","date":"2026-10-16","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.042,"cached_input_tokens":0.042,"cache_write_input_tokens":0.042,"cache_write_1h_input_tokens":0.042,"output_tokens":0.0},"text":"$0.042 input, output free per 1M tokens"},{"provider":"liquid","model":"d1.1","perplexity_mode":null,"basis":"Liquid AI's price for d1.1, as its API bills it","source":"https://www.liquid.ai/pricing","date":"2026-10-24","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.02,"cached_input_tokens":0.02,"cache_write_input_tokens":0.02,"cache_write_1h_input_tokens":0.02,"output_tokens":0.0},"text":"$0.02 input, output free per 1M tokens"},{"provider":"liquid","model":"d1.2","perplexity_mode":null,"basis":"Liquid AI's d1.2 price","source":"https://www.liquid.ai/blog/d1-2","date":"2026-11-16","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.02,"cached_input_tokens":0.02,"cache_write_input_tokens":0.02,"cache_write_1h_input_tokens":0.02,"output_tokens":0.0},"text":"$0.02 input, output free per 1M tokens"},{"provider":"liquid","model":"d1.2-ultra","perplexity_mode":null,"basis":"Liquid AI's d1.2-ultra price","source":"https://www.liquid.ai/blog/d1-2","date":"2026-11-16","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.042,"cached_input_tokens":0.042,"cache_write_input_tokens":0.042,"cache_write_1h_input_tokens":0.042,"output_tokens":0.0},"text":"$0.042 input, output free per 1M tokens"},{"provider":"liquid","model":"d1:free","perplexity_mode":null,"basis":"Liquid's list price for the same model","source":"https://www.liquid.ai/pricing","date":"2026-10-18","change":null,"applies_to":"free tier","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.02,"cached_input_tokens":0.02,"cache_write_input_tokens":0.02,"cache_write_1h_input_tokens":0.02,"output_tokens":0.0},"text":"$0.02 input, output free per 1M tokens"},{"provider":"meragpt","model":"decider-1","perplexity_mode":null,"basis":"meraGPT's Decider 1 price","source":"https://meragpt.com/docs/models","date":"2026-10-19","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.04,"cached_input_tokens":0.04,"cache_write_input_tokens":0.04,"cache_write_1h_input_tokens":0.04,"output_tokens":0.0},"text":"$0.04 input, output free per 1M tokens"},{"provider":"metask","model":"metask-jev-4b-policy-mix","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"nace","model":"drex-max-v1.1","perplexity_mode":null,"basis":"Nace.AI's list price (launch price $0.03 per 1M input tokens, 50% off, is what Nace bills)","source":"https://nace.ai/drex","date":"2026-10-30","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.06,"cached_input_tokens":0.06,"cache_write_input_tokens":0.06,"cache_write_1h_input_tokens":0.06,"output_tokens":0.0},"text":"$0.06 input, output free per 1M tokens"},{"provider":"nace","model":"drex-v2.0","perplexity_mode":null,"basis":"Nace.AI's list price","source":"https://console.nace.ai/docs/drex/reference/pricing","date":"2026-10-30","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.06,"cached_input_tokens":0.06,"cache_write_input_tokens":0.06,"cache_write_1h_input_tokens":0.06,"output_tokens":0.0},"text":"$0.06 input, output free per 1M tokens"},{"provider":"onejev","model":"OmniJev/OneJev-27B","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"open-jev","model":"open-jev-27b-v1.1","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"openai","model":"gpt-6-astra","perplexity_mode":null,"basis":"OpenAI's API list price","source":"https://developers.openai.com/api/docs/pricing","date":"2026-10-08","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":10.0,"cached_input_tokens":1.0,"cache_write_input_tokens":12.5,"cache_write_1h_input_tokens":10.0,"output_tokens":50.0},"text":"$10 input, $1 cached input, $12.50 cache writes, $50 output per 1M tokens"},{"provider":"openai","model":"gpt-6-luna","perplexity_mode":null,"basis":"OpenAI's API list price","source":"https://developers.openai.com/api/docs/pricing","date":"2026-10-08","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.1,"cached_input_tokens":0.01,"cache_write_input_tokens":0.125,"cache_write_1h_input_tokens":0.1,"output_tokens":0.5},"text":"$0.10 input, $0.01 cached input, $0.125 cache writes, $0.50 output per 1M tokens"},{"provider":"openai","model":"gpt-6.1-sol","perplexity_mode":null,"basis":"OpenAI's API list price","source":"https://developers.openai.com/api/docs/pricing","date":"2026-10-08","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":2.0,"cached_input_tokens":0.1,"cache_write_input_tokens":2.5,"cache_write_1h_input_tokens":2.0,"output_tokens":10.0},"text":"$2 input, $0.10 cached input, $2.50 cache writes, $10 output per 1M tokens"},{"provider":"openai","model":"gpt-6.1-terra","perplexity_mode":null,"basis":"OpenAI's API list price","source":"https://developers.openai.com/api/docs/pricing","date":"2026-10-28","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":1.0,"cached_input_tokens":0.05,"cache_write_input_tokens":1.25,"cache_write_1h_input_tokens":1.0,"output_tokens":5.0},"text":"$1 input, $0.05 cached input, $1.25 cache writes, $5 output per 1M tokens"},{"provider":"openai-decisions","model":"gpt-6-luna","perplexity_mode":null,"basis":"OpenAI's Decisions API price","source":"https://developers.openai.com/api/docs/guides/decisions#pricing-and-availability","date":"2026-10-08","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.1,"cached_input_tokens":0.1,"cache_write_input_tokens":0.1,"cache_write_1h_input_tokens":0.1,"output_tokens":0.0},"text":"$0.10 input, output free per 1M tokens"},{"provider":"openrouter","model":"google/gemini-decision-1","perplexity_mode":null,"basis":"OpenRouter's Gemini Decision 1 price (Google AI Studio endpoint)","source":"https://openrouter.ai/google/gemini-decision-1","date":"2026-11-11","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.1,"cached_input_tokens":0.1,"cache_write_input_tokens":0.1,"cache_write_1h_input_tokens":0.1,"output_tokens":0.0},"text":"$0.10 input, output free per 1M tokens"},{"provider":"openrouter","model":"inception/mercury-decide","perplexity_mode":null,"basis":"OpenRouter's Mercury Decide price","source":"https://openrouter.ai/inception/mercury-decide","date":"2026-10-17","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.02,"cached_input_tokens":0.02,"cache_write_input_tokens":0.02,"cache_write_1h_input_tokens":0.02,"output_tokens":0.0},"text":"$0.02 input, output free per 1M tokens"},{"provider":"openrouter","model":"microsoft/microsoft-decision-1","perplexity_mode":null,"basis":"OpenRouter's Microsoft-Decision-1 price","source":"https://openrouter.ai/microsoft/microsoft-decision-1","date":"2026-10-17","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.042,"cached_input_tokens":0.042,"cache_write_input_tokens":0.042,"cache_write_1h_input_tokens":0.042,"output_tokens":0.0},"text":"$0.042 input, output free per 1M tokens"},{"provider":"openrouter-chat","model":"anthropic/claude-haiku-5.5","perplexity_mode":null,"basis":"OpenRouter's price for Anthropic's own endpoint","source":"https://openrouter.ai/anthropic/claude-haiku-5.5","date":"2026-11-10","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.1,"cached_input_tokens":0.01,"cache_write_input_tokens":0.1,"cache_write_1h_input_tokens":0.1,"output_tokens":0.5},"text":"$0.10 input, $0.01 cached input, $0.50 output per 1M tokens"},{"provider":"openrouter-chat","model":"meta/muse-ember","perplexity_mode":null,"basis":"OpenRouter's price for Meta's own endpoint","source":"https://openrouter.ai/meta/muse-ember","date":"2026-11-10","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.1,"cached_input_tokens":0.004,"cache_write_input_tokens":0.1,"cache_write_1h_input_tokens":0.1,"output_tokens":0.5},"text":"$0.10 input, $0.004 cached input, $0.50 output per 1M tokens"},{"provider":"openrouter-chat","model":"minimax/minimax-m3.1-flash","perplexity_mode":null,"basis":"OpenRouter's price for MiniMax's own endpoint","source":"https://openrouter.ai/minimax/minimax-m3.1-flash","date":"2026-11-08","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.15,"cached_input_tokens":0.015,"cache_write_input_tokens":0.15,"cache_write_1h_input_tokens":0.15,"output_tokens":0.6},"text":"$0.15 input, $0.015 cached input, $0.60 output per 1M tokens"},{"provider":"openrouter-chat","model":"moonshotai/kimi-k3.1-pro","perplexity_mode":null,"basis":"OpenRouter's price for Moonshot AI's endpoint (mxfp4)","source":"https://openrouter.ai/moonshotai/kimi-k3.1-pro","date":"2026-10-28","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":2.0,"cached_input_tokens":0.2,"cache_write_input_tokens":2.0,"cache_write_1h_input_tokens":2.0,"output_tokens":10.0},"text":"$2 input, $0.20 cached input, $10 output per 1M tokens"},{"provider":"openrouter-chat","model":"qwen/qwen3.5-397b-a17b","perplexity_mode":null,"basis":"OpenRouter's price for DeepInfra's endpoint (fp8)","source":"https://openrouter.ai/qwen/qwen3.5-397b-a17b","date":"2026-11-08","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.45,"cached_input_tokens":0.22,"cache_write_input_tokens":0.45,"cache_write_1h_input_tokens":0.45,"output_tokens":3.0},"text":"$0.45 input, $0.22 cached input, $3 output per 1M tokens"},{"provider":"openrouter-chat","model":"qwen/qwen3.8-flash","perplexity_mode":null,"basis":"OpenRouter's price for Alibaba's endpoint","source":"https://openrouter.ai/qwen/qwen3.8-flash","date":"2026-11-11","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.15,"cached_input_tokens":0.016,"cache_write_input_tokens":0.15,"cache_write_1h_input_tokens":0.15,"output_tokens":0.47},"text":"$0.15 input, $0.016 cached input, $0.47 output per 1M tokens"},{"provider":"openrouter-chat","model":"qwen/qwen4-max","perplexity_mode":null,"basis":"OpenRouter's price for Alibaba's endpoint (the maker's)","source":"https://openrouter.ai/qwen/qwen4-max","date":"2026-11-16","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":1.5,"cached_input_tokens":1.5,"cache_write_input_tokens":1.5,"cache_write_1h_input_tokens":1.5,"output_tokens":4.5},"text":"$1.50 input, $4.50 output per 1M tokens"},{"provider":"openrouter-chat","model":"thinkingmachines/inkling-small","perplexity_mode":null,"basis":"OpenRouter's price for DeepInfra's endpoint (fp8)","source":"https://openrouter.ai/thinkingmachines/inkling-small","date":"2026-11-08","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.45,"cached_input_tokens":0.1,"cache_write_input_tokens":0.45,"cache_write_1h_input_tokens":0.45,"output_tokens":1.2},"text":"$0.45 input, $0.10 cached input, $1.20 output per 1M tokens"},{"provider":"openrouter-chat","model":"xiaomi/mimo-v2.6-flash","perplexity_mode":null,"basis":"OpenRouter's price for Xiaomi's own endpoint (fp8)","source":"https://openrouter.ai/xiaomi/mimo-v2.6-flash","date":"2026-11-08","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.14,"cached_input_tokens":0.0028,"cache_write_input_tokens":0.14,"cache_write_1h_input_tokens":0.14,"output_tokens":0.28},"text":"$0.14 input, $0.0028 cached input, $0.28 output per 1M tokens"},{"provider":"openrouter-chat","model":"z-ai/glm-5.3","perplexity_mode":null,"basis":"OpenRouter's price for Fireworks' spot endpoint, which served nearly every request","source":"https://openrouter.ai/z-ai/glm-5.3","date":"2026-10-28","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.98,"cached_input_tokens":0.182,"cache_write_input_tokens":0.98,"cache_write_1h_input_tokens":0.98,"output_tokens":3.08},"text":"$0.98 input, $0.182 cached input, $3.08 output per 1M tokens"},{"provider":"perplexity","model":"pplx-decider-v1-27b","perplexity_mode":null,"basis":"Perplexity's Decisions API price","source":"https://docs.perplexity.ai/docs/getting-started/pricing","date":"2026-10-04","change":"$0.02 per 1M input tokens since, as listed on 8 October 2026","applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.04,"cached_input_tokens":0.04,"cache_write_input_tokens":0.04,"cache_write_1h_input_tokens":0.04,"output_tokens":0.0},"text":"$0.04 input, output free per 1M tokens"},{"provider":"perplexity","model":"pplx-decider-v1.1-27b","perplexity_mode":null,"basis":"Perplexity's Decisions API price","source":"https://docs.perplexity.ai/docs/getting-started/pricing","date":"2026-10-09","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.02,"cached_input_tokens":0.02,"cache_write_input_tokens":0.02,"cache_write_1h_input_tokens":0.02,"output_tokens":0.0},"text":"$0.02 input, output free per 1M tokens"},{"provider":"perplexity","model":"pplx-decider-v1.2","perplexity_mode":null,"basis":"Perplexity's Decisions API price","source":"https://docs.perplexity.ai/docs/getting-started/pricing","date":"2026-11-05","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.02,"cached_input_tokens":0.02,"cache_write_input_tokens":0.02,"cache_write_1h_input_tokens":0.02,"output_tokens":0.0},"text":"$0.02 input, output free per 1M tokens"},{"provider":"perplexity","model":"pplx-decider-v1.2","perplexity_mode":"web_grounding","basis":"Perplexity's Decisions API price with web grounding","source":"https://docs.perplexity.ai/docs/getting-started/pricing","date":"2026-11-05","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.001,"rates":{"input_tokens":0.02,"cached_input_tokens":0.02,"cache_write_input_tokens":0.02,"cache_write_1h_input_tokens":0.02,"output_tokens":0.0},"text":"$0.02 input, output free per 1M tokens, plus $0.001 per request (one web search)"},{"provider":"pplx-decider","model":"jev-latest","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"pplx-decider","model":"pplx-decider-flash-v1.2","perplexity_mode":null,"basis":"Perplexity's list price for the same model","source":"https://docs.perplexity.ai/docs/getting-started/pricing","date":"2026-10-18","change":null,"applies_to":"maker","extra_input_per_request":3,"per_request":0.0,"rates":{"input_tokens":0.01,"cached_input_tokens":0.01,"cache_write_input_tokens":0.01,"cache_write_1h_input_tokens":0.01,"output_tokens":0.0},"text":"$0.01 input, output free per 1M tokens, plus 3 billed input tokens per request"},{"provider":"pplx-decider","model":"pplx-decider-flash-v1.3","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"pplx-decider","model":"pplx-decider-v1.2","perplexity_mode":null,"basis":"Perplexity's list price for the same model","source":"https://docs.perplexity.ai/docs/getting-started/pricing","date":"2026-10-18","change":null,"applies_to":"maker","extra_input_per_request":3,"per_request":0.0,"rates":{"input_tokens":0.02,"cached_input_tokens":0.02,"cache_write_input_tokens":0.02,"cache_write_1h_input_tokens":0.02,"output_tokens":0.0},"text":"$0.02 input, output free per 1M tokens, plus 3 billed input tokens per request"},{"provider":"pplx-decider","model":"pplx-decider-v1.3","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"reax","model":"reax13","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"strands","model":"strands-decider-2b","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"subq","model":"subq-snap-1.1","perplexity_mode":null,"basis":"SubQ's list price","source":"https://docs.subq.ai/using-the-api/models/","date":"2026-11-01","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.04,"cached_input_tokens":0.04,"cache_write_input_tokens":0.04,"cache_write_1h_input_tokens":0.04,"output_tokens":0.0},"text":"$0.04 input, output free per 1M tokens"},{"provider":"subq","model":"subq-snap-preview","perplexity_mode":null,"basis":"SubQ's Decisions API price","source":"https://docs.subq.ai/using-the-api/models/","date":"2026-10-13","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.05,"cached_input_tokens":0.05,"cache_write_input_tokens":0.05,"cache_write_1h_input_tokens":0.05,"output_tokens":0.0},"text":"$0.05 input, output free per 1M tokens"},{"provider":"valen","model":"valen","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"vllm","model":"deepseek-v4.1-flash","perplexity_mode":null,"basis":"Together's list price for the same weights (precision not stated; ours the published mixed precision)","source":"https://openrouter.ai/deepseek/deepseek-v4.1-flash-20260910","date":"2026-11-05","change":null,"applies_to":"weights","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.3,"cached_input_tokens":0.006,"cache_write_input_tokens":0.3,"cache_write_1h_input_tokens":0.3,"output_tokens":1.2},"text":"$0.30 input, $0.006 cached input, $1.20 output per 1M tokens"},{"provider":"vllm","model":"deepseek-v4.1-flash-1017","perplexity_mode":null,"basis":"DeepSeek's list price for the same model","source":"https://api-docs.deepseek.com/quick_start/pricing","date":"2026-11-05","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.3,"cached_input_tokens":0.006,"cache_write_input_tokens":0.3,"cache_write_1h_input_tokens":0.3,"output_tokens":1.2},"text":"$0.30 input, $0.006 cached input, $1.20 output per 1M tokens"},{"provider":"vllm","model":"erk-14b","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"vllm","model":"gemma-4-31B-it","perplexity_mode":null,"basis":"DeepInfra's list price for the same weights (FP8, as ours)","source":"https://openrouter.ai/google/gemma-4-31b-it","date":"2026-11-05","change":null,"applies_to":"weights","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.2,"cached_input_tokens":0.2,"cache_write_input_tokens":0.2,"cache_write_1h_input_tokens":0.2,"output_tokens":0.4},"text":"$0.20 input, $0.40 output per 1M tokens"},{"provider":"vllm","model":"qwen3-14b","perplexity_mode":null,"basis":"DeepInfra's list price for the same weights (FP8; ours BF16)","source":"https://openrouter.ai/qwen/qwen3-14b","date":"2026-11-05","change":null,"applies_to":"weights","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.12,"cached_input_tokens":0.12,"cache_write_input_tokens":0.12,"cache_write_1h_input_tokens":0.12,"output_tokens":0.24},"text":"$0.12 input, $0.24 output per 1M tokens"},{"provider":"vllm","model":"qwen38-27b-bf16","perplexity_mode":null,"basis":"Alibaba's list price for the same model","source":"https://www.alibabacloud.com/help/en/model-studio/qwen3-8-27b","date":"2026-10-18","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.5,"cached_input_tokens":0.1,"cache_write_input_tokens":0.5,"cache_write_1h_input_tokens":0.5,"output_tokens":3.0},"text":"$0.50 input, $0.10 cached input, $3 output per 1M tokens"},{"provider":"vllm-plain","model":"gemma-4-e4b-it","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"vllm-plain","model":"lfm2.5-2.6b","perplexity_mode":null,"basis":"Liquid AI's list price for the same model (FP8 at its endpoint)","source":"https://openrouter.ai/liquid/lfm-2.5-2.6b","date":"2026-11-12","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.35,"cached_input_tokens":0.35,"cache_write_input_tokens":0.35,"cache_write_1h_input_tokens":0.35,"output_tokens":0.5},"text":"$0.35 input, $0.50 output per 1M tokens"},{"provider":"vllm-plain","model":"lfm2.5-8b-a1b","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"vllm-plain","model":"lfm3-1.7b","perplexity_mode":null,"basis":"Liquid AI's list price for the same model (FP8 at its endpoint)","source":"https://openrouter.ai/liquid/lfm-3-1.7b","date":"2026-11-09","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.08,"cached_input_tokens":0.08,"cache_write_input_tokens":0.08,"cache_write_1h_input_tokens":0.08,"output_tokens":0.2},"text":"$0.08 input, $0.20 output per 1M tokens"},{"provider":"vllm-plain","model":"lfm3-1.7b-vllm-nothink","perplexity_mode":null,"basis":"Liquid AI's list price for the same model (FP8 at its endpoint)","source":"https://openrouter.ai/liquid/lfm-3-1.7b","date":"2026-11-09","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.08,"cached_input_tokens":0.08,"cache_write_input_tokens":0.08,"cache_write_1h_input_tokens":0.08,"output_tokens":0.2},"text":"$0.08 input, $0.20 output per 1M tokens"},{"provider":"vllm-plain","model":"minicpm5-2b-nothink","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"vllm-plain","model":"qwen3.5-2b-nothink","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"vllm-plain","model":"qwen3.5-4b","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"vllm-plain","model":"qwen3.5-4b-nothink","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"vllm-sr","model":"vllm-sr/d3","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"vllm-stated","model":"glm-5.3-flash","perplexity_mode":null,"basis":"Z.ai's list price for the same model (FP8; ours EXL3 4-bit experts with FP8 dense layers)","source":"https://openrouter.ai/z-ai/glm-5.3-flash","date":"2026-11-05","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.15,"cached_input_tokens":0.03,"cache_write_input_tokens":0.15,"cache_write_1h_input_tokens":0.15,"output_tokens":0.5},"text":"$0.15 input, $0.03 cached input, $0.50 output per 1M tokens"},{"provider":"vllm-verbal","model":"qwen38-27b-bf16","perplexity_mode":null,"basis":"Alibaba's list price for the same model","source":"https://www.alibabacloud.com/help/en/model-studio/qwen3-8-27b","date":"2026-10-18","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.5,"cached_input_tokens":0.1,"cache_write_input_tokens":0.5,"cache_write_1h_input_tokens":0.5,"output_tokens":3.0},"text":"$0.50 input, $0.10 cached input, $3 output per 1M tokens"},{"provider":"vllm-vote","model":"qwen38-27b-bf16","perplexity_mode":null,"basis":"Alibaba's list price for the same model","source":"https://www.alibabacloud.com/help/en/model-studio/qwen3-8-27b","date":"2026-10-18","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.5,"cached_input_tokens":0.1,"cache_write_input_tokens":0.5,"cache_write_1h_input_tokens":0.5,"output_tokens":3.0},"text":"$0.50 input, $0.10 cached input, $3 output per 1M tokens"},{"provider":"vllm-yes-no","model":"qwen38-27b-bf16","perplexity_mode":null,"basis":"Alibaba's list price for the same model","source":"https://www.alibabacloud.com/help/en/model-studio/qwen3-8-27b","date":"2026-10-18","change":null,"applies_to":"maker","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":0.5,"cached_input_tokens":0.1,"cache_write_input_tokens":0.5,"cache_write_1h_input_tokens":0.5,"output_tokens":3.0},"text":"$0.50 input, $0.10 cached input, $3 output per 1M tokens"},{"provider":"winnow","model":"winnow-12b","perplexity_mode":null,"basis":"Runpod's on-demand price for an RTX PRO 6000 (96 GB) on Secure Cloud","source":"https://www.runpod.io/pricing","date":"2026-11-05","change":null,"applies_to":"gpu","gpu":"RTX PRO 6000 Blackwell (96 GB)","gpus":1,"usd_per_gpu_hour":2.49,"text":"$2.49 per GPU-hour, 1 GPU, for the run's request time"},{"provider":"xai","model":"grok-4.7","perplexity_mode":null,"basis":"xAI's API list price","source":"https://docs.x.ai/docs/models/grok-4.7","date":"2026-10-28","change":null,"applies_to":"billed","extra_input_per_request":0,"per_request":0.0,"rates":{"input_tokens":2.0,"cached_input_tokens":0.5,"cache_write_input_tokens":2.0,"cache_write_1h_input_tokens":2.0,"output_tokens":6.0},"text":"$2 input, $0.50 cached input, $6 output per 1M tokens"}],"dont_know":{"option":"Bilmiyorum","option_en":"I don't know","sentence":"If you do not know the answer, choose Bilmiyorum.","instructions":"Which option is the correct answer to this Turkish multiple-choice question? If you do not know the answer, choose Bilmiyorum.","prompt_version":"1cdd542b597f","chat":{"sentence":"Cevabı bilmiyorsanız Bilmiyorum seçeneğini seçin.","confidence_system":"You are taking a Turkish multiple-choice benchmark. Select exactly one supplied option per question and state your confidence in each answer. Do not explain.","confidence_instruction":"Aşağıdaki çoktan seçmeli soruların her birini yanıtlayın. Açıklama yapmadan, her soru için doğru seçeneğin harfini ve cevabınızın doğru olduğundan ne kadar emin olduğunuzu gösteren 0 ile 1 arasında bir güven puanını belirtilen JSON biçiminde birlikte döndürün. Cevabı bilmiyorsanız Bilmiyorum seçeneğini seçin.","prompt_version":"d3c64e8c862e"},"plain":{"instruction":"Aşağıdaki çoktan seçmeli soruyu çöz. Yanıtının son satırına yalnızca 'CEVAP: X' yaz (X = {letters}). Cevabı bilmiyorsanız Bilmiyorum seçeneğini seçin.","sentence":"Cevabı bilmiyorsanız Bilmiyorum seçeneğini seçin.","prompt_version":"711479895d19"},"vs_confidence_share":0.01,"questions":2198,"models":[{"name":"Gemini 4 Carbon","model":"gemini-4-carbon","provider":"agbench","perplexity_mode":null,"group":"chat","run":1121,"runs":[1121],"right":2159,"wrong":32,"dont_know":7,"invalid":0,"invalid_replies":{"invalid_without_rule":64,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":4,"asked_again_with_answers":4,"answers_after_asking_again":64,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.9822565969062784,"attempt_rate":0.9968152866242038,"non_hallucination":0.1794871794871795,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1108,"original_accuracy":0.9845313921747043,"derived":{"cut_off":0.72,"attempt_rate":0.997270245677889,"non_hallucination":0.16216216216216217,"right":2161,"wrong":31,"abstained":6},"vs_confidence":null},{"name":"Gemini 3.8 Flash","model":"gemini-3.8-flash","provider":"gemini","perplexity_mode":null,"group":"chat","run":681,"runs":[681],"right":2154,"wrong":42,"dont_know":2,"invalid":0,"invalid_replies":null,"accuracy":0.9799818016378526,"attempt_rate":0.9990900818926297,"non_hallucination":0.045454545454545456,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":140,"original_accuracy":0.9808917197452229,"derived":{"cut_off":0.85,"attempt_rate":0.9990900818926297,"non_hallucination":0.047619047619047616,"right":2156,"wrong":40,"abstained":2},"vs_confidence":null},{"name":"GPT-6 Astra","model":"gpt-6-astra","provider":"openai","perplexity_mode":null,"group":"chat","run":818,"runs":[818],"right":2149,"wrong":42,"dont_know":7,"invalid":0,"invalid_replies":null,"accuracy":0.9777070063694268,"attempt_rate":0.9968152866242038,"non_hallucination":0.14285714285714285,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":277,"original_accuracy":0.9795268425841674,"derived":{"cut_off":0.53,"attempt_rate":0.9968152866242038,"non_hallucination":0.14285714285714285,"right":2149,"wrong":42,"abstained":7},"vs_confidence":null},{"name":"GPT-6.1 Sol","model":"gpt-6.1-sol","provider":"openai","perplexity_mode":null,"group":"chat","run":802,"runs":[802],"right":2146,"wrong":44,"dont_know":8,"invalid":0,"invalid_replies":null,"accuracy":0.9763421292083713,"attempt_rate":0.9963603275705186,"non_hallucination":0.15384615384615385,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":276,"original_accuracy":0.9777070063694268,"derived":{"cut_off":0.65,"attempt_rate":0.9959053685168335,"non_hallucination":0.16666666666666666,"right":2144,"wrong":45,"abstained":9},"vs_confidence":null},{"name":"GPT-6 Astra","model":"gpt-6-astra","provider":"openai","perplexity_mode":null,"group":"chat","run":817,"runs":[817],"right":2143,"wrong":48,"dont_know":7,"invalid":0,"invalid_replies":null,"accuracy":0.9749772520473158,"attempt_rate":0.9968152866242038,"non_hallucination":0.12727272727272726,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":610,"original_accuracy":0.9749772520473158,"derived":{"cut_off":0.52,"attempt_rate":0.9968152866242038,"non_hallucination":0.11864406779661017,"right":2139,"wrong":52,"abstained":7},"vs_confidence":null},{"name":"Gemini 4 Hydrogen","model":"gemini-4-hydrogen","provider":"agbench","perplexity_mode":null,"group":"chat","run":1117,"runs":[1117],"right":2141,"wrong":38,"dont_know":8,"invalid":11,"invalid_replies":{"invalid_without_rule":16,"read_from_broken_replies":5,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"unreadable":11}},"accuracy":0.9740673339399454,"attempt_rate":0.9913557779799818,"non_hallucination":0.17391304347826086,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1114,"original_accuracy":0.9808917197452229,"derived":{"cut_off":0.72,"attempt_rate":0.991810737033667,"non_hallucination":0.375,"right":2150,"wrong":30,"abstained":18},"vs_confidence":null},{"name":"GPT-6 Astra","model":"gpt-6-astra","provider":"openai","perplexity_mode":null,"group":"chat","run":816,"runs":[816],"right":2141,"wrong":51,"dont_know":6,"invalid":0,"invalid_replies":null,"accuracy":0.9740673339399454,"attempt_rate":0.997270245677889,"non_hallucination":0.10526315789473684,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":48,"original_accuracy":0.9758871701546861,"derived":{"cut_off":0.5,"attempt_rate":0.997270245677889,"non_hallucination":0.10909090909090909,"right":2143,"wrong":49,"abstained":6},"vs_confidence":null},{"name":"GPT-6 Astra","model":"gpt-6-astra","provider":"openai","perplexity_mode":null,"group":"chat","run":819,"runs":[819],"right":2139,"wrong":52,"dont_know":7,"invalid":0,"invalid_replies":null,"accuracy":0.9731574158325751,"attempt_rate":0.9968152866242038,"non_hallucination":0.11864406779661017,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":191,"original_accuracy":0.9740673339399454,"derived":{"cut_off":0.62,"attempt_rate":0.9968152866242038,"non_hallucination":0.11666666666666667,"right":2138,"wrong":53,"abstained":7},"vs_confidence":null},{"name":"Gemini 3.8 Flash","model":"gemini-3.8-flash","provider":"gemini","perplexity_mode":null,"group":"chat","run":691,"runs":[691],"right":2134,"wrong":60,"dont_know":4,"invalid":0,"invalid_replies":null,"accuracy":0.9708826205641492,"attempt_rate":0.9981801637852593,"non_hallucination":0.0625,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":188,"original_accuracy":0.9713375796178344,"derived":{"cut_off":0.55,"attempt_rate":0.997270245677889,"non_hallucination":0.09230769230769231,"right":2133,"wrong":59,"abstained":6},"vs_confidence":null},{"name":"GPT-6.1 Sol","model":"gpt-6.1-sol","provider":"openai","perplexity_mode":null,"group":"chat","run":799,"runs":[799],"right":2133,"wrong":59,"dont_know":6,"invalid":0,"invalid_replies":null,"accuracy":0.9704276615104641,"attempt_rate":0.997270245677889,"non_hallucination":0.09230769230769231,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":124,"original_accuracy":0.9667879890809827,"derived":{"cut_off":0.42,"attempt_rate":0.997270245677889,"non_hallucination":0.07894736842105263,"right":2122,"wrong":70,"abstained":6},"vs_confidence":null},{"name":"GPT-6.1 Sol","model":"gpt-6.1-sol","provider":"openai","perplexity_mode":null,"group":"chat","run":810,"runs":[810],"right":2129,"wrong":61,"dont_know":8,"invalid":0,"invalid_replies":null,"accuracy":0.9686078252957234,"attempt_rate":0.9963603275705186,"non_hallucination":0.11594202898550725,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":190,"original_accuracy":0.9672429481346679,"derived":{"cut_off":0.56,"attempt_rate":0.9963603275705186,"non_hallucination":0.10526315789473684,"right":2122,"wrong":68,"abstained":8},"vs_confidence":null},{"name":"Claude Fable 5.1","model":"claude-fable-5-1","provider":"claude","perplexity_mode":null,"group":"chat","run":695,"runs":[695],"right":2115,"wrong":78,"dont_know":5,"invalid":0,"invalid_replies":null,"accuracy":0.9622383985441311,"attempt_rate":0.9977252047315741,"non_hallucination":0.060240963855421686,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":310,"original_accuracy":0.9613284804367607,"derived":{"cut_off":0.35,"attempt_rate":0.9968152866242038,"non_hallucination":0.08045977011494253,"right":2111,"wrong":80,"abstained":7},"vs_confidence":null},{"name":"Gemini 4 Argon","model":"gemini-4-argon","provider":"gemini","perplexity_mode":null,"group":"chat","run":679,"runs":[679],"right":2112,"wrong":80,"dont_know":6,"invalid":0,"invalid_replies":null,"accuracy":0.9608735213830755,"attempt_rate":0.997270245677889,"non_hallucination":0.06976744186046512,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":557,"original_accuracy":0.9636032757051866,"derived":{"cut_off":0.75,"attempt_rate":0.997270245677889,"non_hallucination":0.07407407407407407,"right":2117,"wrong":75,"abstained":6},"vs_confidence":null},{"name":"Claude Opus 5.5","model":"claude-opus-5-5","provider":"claude","perplexity_mode":null,"group":"chat","run":773,"runs":[773],"right":2087,"wrong":108,"dont_know":3,"invalid":0,"invalid_replies":null,"accuracy":0.9494995450409464,"attempt_rate":0.9986351228389445,"non_hallucination":0.02702702702702703,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":278,"original_accuracy":0.9499545040946314,"derived":{"cut_off":0.25,"attempt_rate":0.9995450409463148,"non_hallucination":0.009009009009009009,"right":2087,"wrong":110,"abstained":1},"vs_confidence":null},{"name":"GPT-6.1 Terra","model":"gpt-6.1-terra","provider":"openai","perplexity_mode":null,"group":"chat","run":807,"runs":[807],"right":2074,"wrong":115,"dont_know":9,"invalid":0,"invalid_replies":null,"accuracy":0.9435850773430391,"attempt_rate":0.9959053685168335,"non_hallucination":0.07258064516129033,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":275,"original_accuracy":0.9426751592356688,"derived":{"cut_off":0.43,"attempt_rate":0.9959053685168335,"non_hallucination":0.06870229007633588,"right":2067,"wrong":122,"abstained":9},"vs_confidence":null},{"name":"Claude Opus 5.5","model":"claude-opus-5-5","provider":"claude","perplexity_mode":null,"group":"chat","run":803,"runs":[803],"right":2060,"wrong":137,"dont_know":1,"invalid":0,"invalid_replies":null,"accuracy":0.9372156505914467,"attempt_rate":0.9995450409463148,"non_hallucination":0.007246376811594203,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":192,"original_accuracy":0.9331210191082803,"derived":{"cut_off":0.3,"attempt_rate":0.9995450409463148,"non_hallucination":0.006802721088435374,"right":2051,"wrong":146,"abstained":1},"vs_confidence":null},{"name":"DeepSeek-V4.1-Flash 1017","model":"deepseek-v4.1-flash-1017","provider":"vllm","perplexity_mode":null,"group":"chat","run":811,"runs":[811],"right":2042,"wrong":154,"dont_know":2,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.9290263876251137,"attempt_rate":0.9990900818926297,"non_hallucination":0.01282051282051282,"mean_dont_know_probability":0.0009122733580191731,"spread":null,"answers_changed":null,"original_run":585,"original_accuracy":0.9176524112829846,"derived":{"cut_off":0.8354417683647574,"attempt_rate":0.9854413102820746,"non_hallucination":0.17679558011049723,"right":2017,"wrong":149,"abstained":32},"vs_confidence":null},{"name":"Qwen4-Max","model":"qwen/qwen4-max","provider":"openrouter-chat","perplexity_mode":null,"group":"chat","run":1339,"runs":[1339],"right":2042,"wrong":139,"dont_know":1,"invalid":16,"invalid_replies":{"invalid_without_rule":16,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":16}},"accuracy":0.9290263876251137,"attempt_rate":0.9922656960873522,"non_hallucination":0.007142857142857143,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1336,"original_accuracy":0.9171974522292994,"derived":{"cut_off":0.25,"attempt_rate":0.9854413102820746,"non_hallucination":0.17582417582417584,"right":2016,"wrong":150,"abstained":32},"vs_confidence":null},{"name":"Gemini 3.8 Flash","model":"gemini-3.8-flash","provider":"gemini","perplexity_mode":null,"group":"chat","run":676,"runs":[676],"right":2038,"wrong":157,"dont_know":3,"invalid":0,"invalid_replies":null,"accuracy":0.9272065514103731,"attempt_rate":0.9986351228389445,"non_hallucination":0.01875,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":123,"original_accuracy":0.9326660600545951,"derived":{"cut_off":0.8,"attempt_rate":0.9981801637852593,"non_hallucination":0.026845637583892617,"right":2049,"wrong":145,"abstained":4},"vs_confidence":null},{"name":"Claude Opus 5.5","model":"claude-opus-5-5","provider":"claude","perplexity_mode":null,"group":"chat","run":683,"runs":[683],"right":2027,"wrong":168,"dont_know":3,"invalid":0,"invalid_replies":null,"accuracy":0.9222020018198362,"attempt_rate":0.9986351228389445,"non_hallucination":0.017543859649122806,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":47,"original_accuracy":0.9044585987261147,"derived":{"cut_off":0.25,"attempt_rate":0.9990900818926297,"non_hallucination":0.009478672985781991,"right":1987,"wrong":209,"abstained":2},"vs_confidence":null},{"name":"Grok 4.7","model":"grok-4.7","provider":"xai","perplexity_mode":null,"group":"chat","run":675,"runs":[675],"right":2011,"wrong":131,"dont_know":56,"invalid":0,"invalid_replies":null,"accuracy":0.9149226569608735,"attempt_rate":0.9745222929936306,"non_hallucination":0.2994652406417112,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":311,"original_accuracy":0.9272065514103731,"derived":{"cut_off":0.42,"attempt_rate":0.9763421292083713,"non_hallucination":0.2694300518134715,"right":2005,"wrong":141,"abstained":52},"vs_confidence":null},{"name":"GPT-6.1 Terra","model":"gpt-6.1-terra","provider":"openai","perplexity_mode":null,"group":"chat","run":801,"runs":[801],"right":1993,"wrong":199,"dont_know":6,"invalid":0,"invalid_replies":null,"accuracy":0.9067333939945404,"attempt_rate":0.997270245677889,"non_hallucination":0.02926829268292683,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":274,"original_accuracy":0.9085532302092811,"derived":{"cut_off":0.55,"attempt_rate":0.997270245677889,"non_hallucination":0.029411764705882353,"right":1994,"wrong":198,"abstained":6},"vs_confidence":null},{"name":"Claude Sonnet 5.5","model":"claude-sonnet-5-5","provider":"claude","perplexity_mode":null,"group":"chat","run":812,"runs":[812],"right":1952,"wrong":244,"dont_know":2,"invalid":0,"invalid_replies":null,"accuracy":0.8880800727934486,"attempt_rate":0.9990900818926297,"non_hallucination":0.008130081300813009,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":193,"original_accuracy":0.89171974522293,"derived":{"cut_off":0.25,"attempt_rate":0.9986351228389445,"non_hallucination":0.012552301255230125,"right":1959,"wrong":236,"abstained":3},"vs_confidence":null},{"name":"Claude Sonnet 5.5","model":"claude-sonnet-5-5","provider":"claude","perplexity_mode":null,"group":"chat","run":800,"runs":[800],"right":1914,"wrong":282,"dont_know":2,"invalid":0,"invalid_replies":null,"accuracy":0.8707916287534122,"attempt_rate":0.9990900818926297,"non_hallucination":0.007042253521126761,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":55,"original_accuracy":0.8503184713375797,"derived":{"cut_off":0.15,"attempt_rate":0.9995450409463148,"non_hallucination":0.00303951367781155,"right":1869,"wrong":328,"abstained":1},"vs_confidence":null},{"name":"Muse Ember","model":"meta/muse-ember","provider":"openrouter-chat","perplexity_mode":null,"group":"chat","run":940,"runs":[940],"right":1863,"wrong":281,"dont_know":6,"invalid":48,"invalid_replies":{"invalid_without_rule":48,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":48}},"accuracy":0.8475887170154686,"attempt_rate":0.9754322111010009,"non_hallucination":0.020905923344947737,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":939,"original_accuracy":0.8562329390354868,"derived":{"cut_off":0.3,"attempt_rate":0.9754322111010009,"non_hallucination":0.17034700315457413,"right":1881,"wrong":263,"abstained":54},"vs_confidence":null},{"name":"Kimi K3.1 Pro","model":"moonshotai/kimi-k3.1-pro","provider":"openrouter-chat","perplexity_mode":null,"group":"chat","run":686,"runs":[686],"right":1848,"wrong":343,"dont_know":7,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.8407643312101911,"attempt_rate":0.9968152866242038,"non_hallucination":0.02,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":320,"original_accuracy":0.8275705186533212,"derived":{"cut_off":0.2,"attempt_rate":0.9986351228389445,"non_hallucination":0.007894736842105263,"right":1818,"wrong":377,"abstained":3},"vs_confidence":null},{"name":"Qwen 3.5 397B-A17B","model":"qwen/qwen3.5-397b-a17b","provider":"openrouter-chat","perplexity_mode":null,"group":"chat","run":872,"runs":[872],"right":1802,"wrong":390,"dont_know":6,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.8198362147406734,"attempt_rate":0.997270245677889,"non_hallucination":0.015151515151515152,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":870,"original_accuracy":0.827115559599636,"derived":{"cut_off":0.5,"attempt_rate":0.9977252047315741,"non_hallucination":0.013089005235602094,"right":1816,"wrong":377,"abstained":5},"vs_confidence":null},{"name":"Copernicus-31B","model":"Copernicus-31B","provider":"copernicus","perplexity_mode":null,"group":"decision","run":1172,"runs":[1172],"right":1791,"wrong":402,"dont_know":5,"invalid":0,"invalid_replies":null,"accuracy":0.8148316651501365,"attempt_rate":0.9977252047315741,"non_hallucination":0.012285012285012284,"mean_dont_know_probability":0.019526210281249037,"spread":null,"answers_changed":null,"original_run":1165,"original_accuracy":0.8180163785259327,"derived":{"cut_off":0.24991556038099777,"attempt_rate":0.9977252047315741,"non_hallucination":0.012437810945273632,"right":1796,"wrong":397,"abstained":5},"vs_confidence":{"option_only_wrong":50,"confidence_only_wrong":45,"p":0.6817425059181855,"p_holm":null}},{"name":"KnowLine-31B-Gen2","model":"PelaAI/KnowLine-31B-Gen2","provider":"knowline","perplexity_mode":null,"group":"decision","run":1331,"runs":[1331],"right":1790,"wrong":408,"dont_know":0,"invalid":0,"invalid_replies":null,"accuracy":0.8143767060964513,"attempt_rate":1.0,"non_hallucination":0.0,"mean_dont_know_probability":0.0022092818187403685,"spread":null,"answers_changed":null,"original_run":1314,"original_accuracy":0.8121019108280255,"derived":{"cut_off":0.22588837395978484,"attempt_rate":1.0,"non_hallucination":0.0,"right":1785,"wrong":413,"abstained":0},"vs_confidence":{"option_only_wrong":41,"confidence_only_wrong":46,"p":0.6682849967188074,"p_holm":null}},{"name":"KnowLine 31B Gen1.5","model":"PelaAI/KnowLine-31B-Gen1.5","provider":"knowline","perplexity_mode":null,"group":"decision","run":959,"runs":[959],"right":1786,"wrong":411,"dont_know":1,"invalid":0,"invalid_replies":null,"accuracy":0.8125568698817106,"attempt_rate":0.9995450409463148,"non_hallucination":0.0024271844660194173,"mean_dont_know_probability":0.006071740257811419,"spread":null,"answers_changed":null,"original_run":957,"original_accuracy":0.8180163785259327,"derived":{"cut_off":0.2517340544081843,"attempt_rate":0.9995450409463148,"non_hallucination":0.0024937655860349127,"right":1797,"wrong":400,"abstained":1},"vs_confidence":{"option_only_wrong":52,"confidence_only_wrong":41,"p":0.2997316968970305,"p_holm":null}},{"name":"Turion Jev 31B","model":"turion-ai/turion-jev-31b","provider":"turion-jev","perplexity_mode":null,"group":"decision","run":1285,"runs":[1285],"right":1747,"wrong":436,"dont_know":15,"invalid":0,"invalid_replies":null,"accuracy":0.794813466787989,"attempt_rate":0.9931756141947224,"non_hallucination":0.03325942350332594,"mean_dont_know_probability":0.031282787534121934,"spread":null,"answers_changed":null,"original_run":1273,"original_accuracy":0.8011828935395814,"derived":{"cut_off":0.246776,"attempt_rate":0.9931756141947224,"non_hallucination":0.033860045146726865,"right":1755,"wrong":428,"abstained":15},"vs_confidence":{"option_only_wrong":67,"confidence_only_wrong":59,"p":0.5330527502675572,"p_holm":null}},{"name":"REAX 1.3","model":"reax13","provider":"reax","perplexity_mode":null,"group":"decision","run":833,"runs":[833],"right":1721,"wrong":475,"dont_know":2,"invalid":0,"invalid_replies":null,"accuracy":0.7829845313921747,"attempt_rate":0.9990900818926297,"non_hallucination":0.0041928721174004195,"mean_dont_know_probability":0.01906359613829194,"spread":null,"answers_changed":null,"original_run":827,"original_accuracy":0.7793448589626933,"derived":{"cut_off":0.24619559693933474,"attempt_rate":0.9990900818926297,"non_hallucination":0.00411522633744856,"right":1712,"wrong":484,"abstained":2},"vs_confidence":{"option_only_wrong":35,"confidence_only_wrong":44,"p":0.3681876338306308,"p_holm":null}},{"name":"karpo-decision-31B","model":"karpo-decision","provider":"karpo","perplexity_mode":null,"group":"decision","run":1163,"runs":[1163],"right":1720,"wrong":475,"dont_know":3,"invalid":0,"invalid_replies":null,"accuracy":0.7825295723384895,"attempt_rate":0.9986351228389445,"non_hallucination":0.006276150627615063,"mean_dont_know_probability":0.009773009528902825,"spread":null,"answers_changed":null,"original_run":1156,"original_accuracy":0.7779799818016379,"derived":{"cut_off":0.26217391450767946,"attempt_rate":0.9986351228389445,"non_hallucination":0.006147540983606557,"right":1710,"wrong":485,"abstained":3},"vs_confidence":{"option_only_wrong":53,"confidence_only_wrong":63,"p":0.40346969965531165,"p_holm":null}},{"name":"Inkling Small","model":"thinkingmachines/inkling-small","provider":"openrouter-chat","perplexity_mode":null,"group":"chat","run":868,"runs":[868],"right":1715,"wrong":468,"dont_know":15,"invalid":0,"invalid_replies":{"invalid_without_rule":112,"read_from_broken_replies":80,"read_without_confidence":0,"empty_replies_asked_again":2,"asked_again_with_answers":2,"answers_after_asking_again":32,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.7802547770700637,"attempt_rate":0.9931756141947224,"non_hallucination":0.031055900621118012,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":866,"original_accuracy":0.7998180163785259,"derived":{"cut_off":0.62,"attempt_rate":0.9904458598726115,"non_hallucination":0.0472972972972973,"right":1754,"wrong":423,"abstained":21},"vs_confidence":null},{"name":"GLM-5.3-Flash","model":"glm-5.3-flash","provider":"vllm-stated","perplexity_mode":null,"group":"chat","run":825,"runs":[825],"right":1684,"wrong":336,"dont_know":2,"invalid":176,"invalid_replies":{"invalid_without_rule":176,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":176}},"accuracy":0.7661510464058234,"attempt_rate":0.9190172884440401,"non_hallucination":0.005917159763313609,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":160,"original_accuracy":0.7879890809827116,"derived":{"cut_off":0.45,"attempt_rate":0.9185623293903549,"non_hallucination":0.37060041407867494,"right":1715,"wrong":304,"abstained":179},"vs_confidence":null},{"name":"Gemini Decision 1","model":"google/gemini-decision-1","provider":"openrouter","perplexity_mode":null,"group":"decision","run":1023,"runs":[1023],"right":1683,"wrong":505,"dont_know":10,"invalid":0,"invalid_replies":null,"accuracy":0.7656960873521383,"attempt_rate":0.9954504094631483,"non_hallucination":0.019417475728155338,"mean_dont_know_probability":0.054572286062466606,"spread":null,"answers_changed":null,"original_run":1021,"original_accuracy":0.7752502274795269,"derived":{"cut_off":0.2612920612254481,"attempt_rate":0.9954504094631483,"non_hallucination":0.020161290322580645,"right":1702,"wrong":486,"abstained":10},"vs_confidence":{"option_only_wrong":78,"confidence_only_wrong":59,"p":0.12377482507747656,"p_holm":null}},{"name":"Qwen3.8-27B","model":"qwen38-27b-bf16","provider":"vllm","perplexity_mode":null,"group":"chat","run":666,"runs":[666],"right":1658,"wrong":493,"dont_know":31,"invalid":16,"invalid_replies":{"invalid_without_rule":176,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":11,"asked_again_with_answers":10,"answers_after_asking_again":160,"error_replies_asked_again":0,"invalid_by_kind":{"empty after retry":16}},"accuracy":0.7543221110100091,"attempt_rate":0.978616924476797,"non_hallucination":0.05916030534351145,"mean_dont_know_probability":0.01426252084081356,"spread":null,"answers_changed":null,"original_run":318,"original_accuracy":0.7465878070973613,"derived":{"cut_off":0.2018019629088712,"attempt_rate":0.978161965423112,"non_hallucination":0.08617594254937164,"right":1641,"wrong":509,"abstained":48},"vs_confidence":null},{"name":"GLM-5.3","model":"z-ai/glm-5.3","provider":"openrouter-chat","perplexity_mode":null,"group":"chat","run":656,"runs":[656],"right":1657,"wrong":519,"dont_know":6,"invalid":16,"invalid_replies":{"invalid_without_rule":16,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":16}},"accuracy":0.7538671519563239,"attempt_rate":0.9899909008189263,"non_hallucination":0.011428571428571429,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":314,"original_accuracy":0.7452229299363057,"derived":{"cut_off":0.35,"attempt_rate":0.991810737033667,"non_hallucination":0.031914893617021274,"right":1634,"wrong":546,"abstained":18},"vs_confidence":null},{"name":"Logits Sigma-1 Duo","model":"Logits-Sigma-1-Duo","provider":"di-serve","perplexity_mode":null,"group":"decision","run":1233,"runs":[1233],"right":1653,"wrong":543,"dont_know":2,"invalid":0,"invalid_replies":null,"accuracy":0.7520473157415832,"attempt_rate":0.9990900818926297,"non_hallucination":0.003669724770642202,"mean_dont_know_probability":0.019360682522399652,"spread":null,"answers_changed":null,"original_run":1221,"original_accuracy":0.7502274795268425,"derived":{"cut_off":0.24599711308067707,"attempt_rate":0.9990900818926297,"non_hallucination":0.0036429872495446266,"right":1649,"wrong":547,"abstained":2},"vs_confidence":{"option_only_wrong":46,"confidence_only_wrong":50,"p":0.7596492883236297,"p_holm":null}},{"name":"Sieve 31B v3","model":"sthanika-ai/Sieve-31B-v3","provider":"sieve","perplexity_mode":null,"group":"decision","run":1253,"runs":[1253],"right":1645,"wrong":553,"dont_know":0,"invalid":0,"invalid_replies":null,"accuracy":0.7484076433121019,"attempt_rate":1.0,"non_hallucination":0.0,"mean_dont_know_probability":0.00899909008189263,"spread":null,"answers_changed":null,"original_run":1241,"original_accuracy":0.7415832575068244,"derived":{"cut_off":0.2191,"attempt_rate":1.0,"non_hallucination":0.0,"right":1630,"wrong":568,"abstained":0},"vs_confidence":{"option_only_wrong":40,"confidence_only_wrong":55,"p":0.1505180638902296,"p_holm":null}},{"name":"Gemma 4 31B","model":"gemma-4-31B-it","provider":"vllm","perplexity_mode":null,"group":"chat","run":776,"runs":[776],"right":1606,"wrong":568,"dont_know":24,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.7306642402183804,"attempt_rate":0.989080982711556,"non_hallucination":0.04054054054054054,"mean_dont_know_probability":0.010368261142203771,"spread":null,"answers_changed":null,"original_run":119,"original_accuracy":0.7311191992720655,"derived":{"cut_off":0.5582807977404076,"attempt_rate":0.989080982711556,"non_hallucination":0.03986710963455149,"right":1596,"wrong":578,"abstained":24},"vs_confidence":null},{"name":"Kimi K3.1 Pro","model":"moonshotai/kimi-k3.1-pro","provider":"openrouter-chat","perplexity_mode":null,"group":"chat","run":653,"runs":[653],"right":1574,"wrong":621,"dont_know":3,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.716105550500455,"attempt_rate":0.9986351228389445,"non_hallucination":0.004807692307692308,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":315,"original_accuracy":0.727024567788899,"derived":{"cut_off":0.55,"attempt_rate":0.9986351228389445,"non_hallucination":0.005,"right":1598,"wrong":597,"abstained":3},"vs_confidence":null},{"name":"MiMo V2.6 Flash","model":"xiaomi/mimo-v2.6-flash","provider":"openrouter-chat","perplexity_mode":null,"group":"chat","run":879,"runs":[879],"right":1568,"wrong":557,"dont_know":20,"invalid":53,"invalid_replies":{"invalid_without_rule":58,"read_from_broken_replies":1,"read_without_confidence":0,"empty_replies_asked_again":4,"asked_again_with_answers":4,"answers_after_asking_again":4,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":53}},"accuracy":0.7133757961783439,"attempt_rate":0.9667879890809827,"non_hallucination":0.03466204506065858,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":877,"original_accuracy":0.7211101000909919,"derived":{"cut_off":0.4,"attempt_rate":0.9672429481346679,"non_hallucination":0.11707317073170732,"right":1583,"wrong":543,"abstained":72},"vs_confidence":null},{"name":"Drex Max v1","model":"drex-max-v1","provider":"drex-max","perplexity_mode":null,"group":"decision","run":367,"runs":[367],"right":1485,"wrong":711,"dont_know":2,"invalid":0,"invalid_replies":null,"accuracy":0.675614194722475,"attempt_rate":0.9990900818926297,"non_hallucination":0.002805049088359046,"mean_dont_know_probability":0.015052588128069306,"spread":null,"answers_changed":null,"original_run":322,"original_accuracy":0.6774340309372157,"derived":{"cut_off":0.235659473972748,"attempt_rate":0.9990900818926297,"non_hallucination":0.0028208744710860366,"right":1489,"wrong":707,"abstained":2},"vs_confidence":{"option_only_wrong":45,"confidence_only_wrong":41,"p":0.7465343822890573,"p_holm":null}},{"name":"vLLM-SR d3-ultra","model":"vllm-sr/d3-ultra","provider":"d3-server","perplexity_mode":null,"group":"decision","run":363,"runs":[363],"right":1484,"wrong":704,"dont_know":10,"invalid":0,"invalid_replies":null,"accuracy":0.6751592356687898,"attempt_rate":0.9954504094631483,"non_hallucination":0.014005602240896359,"mean_dont_know_probability":0.02463415869057922,"spread":null,"answers_changed":null,"original_run":269,"original_accuracy":0.6778889899909009,"derived":{"cut_off":0.2875844645345384,"attempt_rate":0.9954504094631483,"non_hallucination":0.014044943820224719,"right":1486,"wrong":702,"abstained":10},"vs_confidence":{"option_only_wrong":54,"confidence_only_wrong":52,"p":0.9226851594412759,"p_holm":null}},{"name":"vLLM-SR d3","model":"vllm-sr/d3","provider":"vllm-sr","perplexity_mode":null,"group":"decision","run":358,"runs":[358],"right":1483,"wrong":715,"dont_know":0,"invalid":0,"invalid_replies":null,"accuracy":0.6747042766151047,"attempt_rate":1.0,"non_hallucination":0.0,"mean_dont_know_probability":0.029138429914467894,"spread":null,"answers_changed":null,"original_run":259,"original_accuracy":0.6778889899909009,"derived":{"cut_off":0.22253464583592555,"attempt_rate":1.0,"non_hallucination":0.0,"right":1490,"wrong":708,"abstained":0},"vs_confidence":{"option_only_wrong":40,"confidence_only_wrong":33,"p":0.4827885568440213,"p_holm":null}},{"name":"Perplexity Decider 27B v1.3","model":"pplx-decider-v1.3","provider":"pplx-decider","perplexity_mode":null,"group":"decision","run":1141,"runs":[1141],"right":1482,"wrong":711,"dont_know":5,"invalid":0,"invalid_replies":null,"accuracy":0.6742493175614195,"attempt_rate":0.9977252047315741,"non_hallucination":0.006983240223463687,"mean_dont_know_probability":0.03247916743828544,"spread":null,"answers_changed":null,"original_run":1134,"original_accuracy":0.6842584167424932,"derived":{"cut_off":0.24242664331734232,"attempt_rate":0.9977252047315741,"non_hallucination":0.007204610951008645,"right":1504,"wrong":689,"abstained":5},"vs_confidence":{"option_only_wrong":52,"confidence_only_wrong":30,"p":0.019826915845841595,"p_holm":null}},{"name":"Perplexity Decider 27B v1.2 (local GPU)","model":"pplx-decider-v1.2","provider":"pplx-decider","perplexity_mode":null,"group":"decision","run":372,"runs":[372],"right":1479,"wrong":717,"dont_know":2,"invalid":0,"invalid_replies":null,"accuracy":0.672884440400364,"attempt_rate":0.9990900818926297,"non_hallucination":0.0027816411682892906,"mean_dont_know_probability":0.023578840422335552,"spread":null,"answers_changed":null,"original_run":217,"original_accuracy":0.6706096451319381,"derived":{"cut_off":0.22050537826136882,"attempt_rate":0.9990900818926297,"non_hallucination":0.0027624309392265192,"right":1474,"wrong":722,"abstained":2},"vs_confidence":{"option_only_wrong":45,"confidence_only_wrong":50,"p":0.6817425059181855,"p_holm":null}},{"name":"Perplexity Decider 27B v1.2 (API, with web)","model":"pplx-decider-v1.2","provider":"perplexity","perplexity_mode":"web_grounding","group":"decision","run":642,"runs":[642],"right":1473,"wrong":702,"dont_know":23,"invalid":0,"invalid_replies":null,"accuracy":0.6701546860782529,"attempt_rate":0.9895359417652412,"non_hallucination":0.031724137931034485,"mean_dont_know_probability":0.05571093717080597,"spread":null,"answers_changed":null,"original_run":640,"original_accuracy":0.6874431301182894,"derived":{"cut_off":0.2555645964735906,"attempt_rate":0.9895359417652412,"non_hallucination":0.03318903318903319,"right":1505,"wrong":670,"abstained":23},"vs_confidence":{"option_only_wrong":127,"confidence_only_wrong":95,"p":0.03723311121185057,"p_holm":1.0}},{"name":"DeepSeek-V4.1-Flash 1017","model":"deepseek-v4.1-flash-1017","provider":"vllm","perplexity_mode":null,"group":"chat","run":806,"runs":[806],"right":1469,"wrong":729,"dont_know":0,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.6683348498635123,"attempt_rate":1.0,"non_hallucination":0.0,"mean_dont_know_probability":0.003290966335332929,"spread":null,"answers_changed":null,"original_run":558,"original_accuracy":0.6601455868971793,"derived":{"cut_off":0.25283698110305447,"attempt_rate":1.0,"non_hallucination":0.0,"right":1451,"wrong":747,"abstained":0},"vs_confidence":null},{"name":"Perplexity Decider 27B v1.2 (API)","model":"pplx-decider-v1.2","provider":"perplexity","perplexity_mode":null,"group":"decision","run":641,"runs":[641],"right":1469,"wrong":726,"dont_know":3,"invalid":0,"invalid_replies":null,"accuracy":0.6683348498635123,"attempt_rate":0.9986351228389445,"non_hallucination":0.00411522633744856,"mean_dont_know_probability":0.026511651421818022,"spread":null,"answers_changed":null,"original_run":633,"original_accuracy":0.6633303002729755,"derived":{"cut_off":0.22523471992013844,"attempt_rate":0.9986351228389445,"non_hallucination":0.004048582995951417,"right":1457,"wrong":738,"abstained":3},"vs_confidence":{"option_only_wrong":48,"confidence_only_wrong":60,"p":0.28979821540506145,"p_holm":null}},{"name":"GLiDE 27B (open)","model":"glide-27b","provider":"glide","perplexity_mode":null,"group":"decision","run":905,"runs":[905],"right":1466,"wrong":441,"dont_know":291,"invalid":0,"invalid_replies":null,"accuracy":0.6669699727024567,"attempt_rate":0.867606915377616,"non_hallucination":0.3975409836065574,"mean_dont_know_probability":0.16841898100063576,"spread":null,"answers_changed":null,"original_run":899,"original_accuracy":0.7111010009099181,"derived":{"cut_off":0.4133139740374993,"attempt_rate":0.867606915377616,"non_hallucination":0.3991769547325103,"right":1469,"wrong":438,"abstained":291},"vs_confidence":{"option_only_wrong":63,"confidence_only_wrong":60,"p":0.8569843282497003,"p_holm":1.0}},{"name":"GPT-6 Luna","model":"gpt-6-luna","provider":"openai","perplexity_mode":null,"group":"chat","run":796,"runs":[796],"right":1463,"wrong":728,"dont_know":7,"invalid":0,"invalid_replies":null,"accuracy":0.6656050955414012,"attempt_rate":0.9968152866242038,"non_hallucination":0.009523809523809525,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":133,"original_accuracy":0.6828935395814377,"derived":{"cut_off":0.57,"attempt_rate":0.9968152866242038,"non_hallucination":0.010014306151645207,"right":1499,"wrong":692,"abstained":7},"vs_confidence":null},{"name":"Gemma 4 31B","model":"gemma-4-31B-it","provider":"vllm","perplexity_mode":null,"group":"chat","run":789,"runs":[789],"right":1453,"wrong":731,"dont_know":14,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.6610555050045496,"attempt_rate":0.9936305732484076,"non_hallucination":0.01879194630872483,"mean_dont_know_probability":0.006594931065642496,"spread":null,"answers_changed":null,"original_run":189,"original_accuracy":0.6678798908098271,"derived":{"cut_off":0.5605625459010407,"attempt_rate":0.9936305732484076,"non_hallucination":0.01907356948228883,"right":1464,"wrong":720,"abstained":14},"vs_confidence":null},{"name":"MiniMax-M3.1-Flash","model":"minimax/minimax-m3.1-flash","provider":"openrouter-chat","perplexity_mode":null,"group":"chat","run":864,"runs":[864],"right":1441,"wrong":704,"dont_know":41,"invalid":12,"invalid_replies":{"invalid_without_rule":28,"read_from_broken_replies":16,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"letter not allowed":12}},"accuracy":0.6555959963603276,"attempt_rate":0.9758871701546861,"non_hallucination":0.05503355704697987,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":862,"original_accuracy":0.664695177434031,"derived":{"cut_off":0.45,"attempt_rate":0.978616924476797,"non_hallucination":0.06283422459893048,"right":1450,"wrong":701,"abstained":47},"vs_confidence":null},{"name":"Claude Sonnet 5.5","model":"claude-sonnet-5-5","provider":"claude","perplexity_mode":null,"group":"chat","run":690,"runs":[690],"right":1424,"wrong":769,"dont_know":5,"invalid":0,"invalid_replies":null,"accuracy":0.6478616924476797,"attempt_rate":0.9977252047315741,"non_hallucination":0.006459948320413436,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":54,"original_accuracy":0.6137397634212921,"derived":{"cut_off":0.3,"attempt_rate":0.9986351228389445,"non_hallucination":0.0035294117647058825,"right":1348,"wrong":847,"abstained":3},"vs_confidence":null},{"name":"Qwen3.8 Flash","model":"qwen/qwen3.8-flash","provider":"openrouter-chat","perplexity_mode":null,"group":"chat","run":987,"runs":[987],"right":1396,"wrong":764,"dont_know":38,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.635122838944495,"attempt_rate":0.9827115559599636,"non_hallucination":0.04738154613466334,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":985,"original_accuracy":0.6305732484076433,"derived":{"cut_off":0.75,"attempt_rate":0.9899909008189263,"non_hallucination":0.026862026862026864,"right":1379,"wrong":797,"abstained":22},"vs_confidence":null},{"name":"KnowLine-12B-Gen1","model":"PelaAI/KnowLine-12B-Gen1","provider":"knowline","perplexity_mode":null,"group":"decision","run":1318,"runs":[1318],"right":1393,"wrong":804,"dont_know":1,"invalid":0,"invalid_replies":null,"accuracy":0.6337579617834395,"attempt_rate":0.9995450409463148,"non_hallucination":0.0012422360248447205,"mean_dont_know_probability":0.010573759597688444,"spread":null,"answers_changed":null,"original_run":1309,"original_accuracy":0.6246587807097361,"derived":{"cut_off":0.21474891443477306,"attempt_rate":0.9995450409463148,"non_hallucination":0.0012121212121212121,"right":1373,"wrong":824,"abstained":1},"vs_confidence":{"option_only_wrong":68,"confidence_only_wrong":88,"p":0.12793767570364561,"p_holm":null}},{"name":"Fastino GLiDE v1.2","model":"glide-v1.2","provider":"glide","perplexity_mode":null,"group":"decision","run":369,"runs":[369],"right":1386,"wrong":491,"dont_know":321,"invalid":0,"invalid_replies":null,"accuracy":0.6305732484076433,"attempt_rate":0.8539581437670609,"non_hallucination":0.39532019704433496,"mean_dont_know_probability":0.18583695105631345,"spread":null,"answers_changed":null,"original_run":248,"original_accuracy":0.6847133757961783,"derived":{"cut_off":0.4134294302863545,"attempt_rate":0.8539581437670609,"non_hallucination":0.40428211586901763,"right":1404,"wrong":473,"abstained":321},"vs_confidence":{"option_only_wrong":77,"confidence_only_wrong":59,"p":0.14463646515752107,"p_holm":1.0}},{"name":"Liquid d1-ultra","model":"d1-ultra","provider":"liquid","perplexity_mode":null,"group":"decision","run":350,"runs":[350,354,360,365,370],"right":1376,"wrong":205,"dont_know":617,"invalid":0,"invalid_replies":null,"accuracy":0.6260236578707916,"attempt_rate":0.7192902638762512,"non_hallucination":0.7506082725060828,"mean_dont_know_probability":0.30688419671830985,"spread":{"accuracy":[0.6255686988171064,0.6346678798908099],"attempt_rate":[0.7192902638762512,0.7397634212920837],"non_hallucination":[0.7123287671232876,0.7506082725060828],"mean_dont_know_probability":[0.3060557578090563,0.3076800813405489]},"answers_changed":518,"original_run":199,"original_accuracy":0.7707006369426752,"derived":{"cut_off":0.5632390296623447,"attempt_rate":0.7192902638762512,"non_hallucination":0.7869897959183674,"right":1414,"wrong":167,"abstained":617},"vs_confidence":{"option_only_wrong":93,"confidence_only_wrong":55,"p":0.002244365752749707,"p_holm":0.08528589860448887}},{"name":"Drex Max 1.1","model":"drex-max-v1.1","provider":"nace","perplexity_mode":null,"group":"decision","run":352,"runs":[352],"right":1361,"wrong":517,"dont_know":320,"invalid":0,"invalid_replies":null,"accuracy":0.6191992720655141,"attempt_rate":0.8544131028207461,"non_hallucination":0.3823178016726404,"mean_dont_know_probability":0.16972642800384946,"spread":null,"answers_changed":null,"original_run":326,"original_accuracy":0.6737943585077343,"derived":{"cut_off":0.38063265738985963,"attempt_rate":0.8544131028207461,"non_hallucination":0.39263803680981596,"right":1383,"wrong":495,"abstained":320},"vs_confidence":{"option_only_wrong":125,"confidence_only_wrong":103,"p":0.16415489186828677,"p_holm":1.0}},{"name":"Claude Haiku 5.5","model":"anthropic/claude-haiku-5.5","provider":"openrouter-chat","perplexity_mode":null,"group":"chat","run":950,"runs":[950],"right":1349,"wrong":138,"dont_know":39,"invalid":672,"invalid_replies":{"invalid_without_rule":672,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":672}},"accuracy":0.6137397634212921,"attempt_rate":0.6765241128298453,"non_hallucination":0.22033898305084745,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":948,"original_accuracy":0.5705186533212011,"derived":{"cut_off":0.15,"attempt_rate":0.643312101910828,"non_hallucination":0.8305084745762712,"right":1254,"wrong":160,"abstained":784},"vs_confidence":null},{"name":"Fastino GLiDE","model":"fastino/GLiDE","provider":"fastino","perplexity_mode":null,"group":"decision","run":351,"runs":[351,353,357,361,364],"right":1321,"wrong":356,"dont_know":521,"invalid":0,"invalid_replies":null,"accuracy":0.6010009099181074,"attempt_rate":0.7629663330300273,"non_hallucination":0.5940706955530216,"mean_dont_know_probability":0.2201435400931123,"spread":{"accuracy":[0.5905368516833485,0.6010009099181074],"attempt_rate":[0.7552320291173794,0.7629663330300273],"non_hallucination":[0.5940706955530216,0.5991091314031181],"mean_dont_know_probability":[0.21628588574582433,0.22014875465701816]},"answers_changed":189,"original_run":129,"original_accuracy":0.6787989080982711,"derived":{"cut_off":0.5206454205284788,"attempt_rate":0.7629663330300273,"non_hallucination":0.5801781737193764,"right":1300,"wrong":377,"abstained":521},"vs_confidence":{"option_only_wrong":192,"confidence_only_wrong":213,"p":0.32031790636017266,"p_holm":1.0}},{"name":"OneJev 27B","model":"OmniJev/OneJev-27B","provider":"onejev","perplexity_mode":null,"group":"decision","run":782,"runs":[782],"right":1305,"wrong":892,"dont_know":1,"invalid":0,"invalid_replies":null,"accuracy":0.5937215650591446,"attempt_rate":0.9995450409463148,"non_hallucination":0.0011198208286674132,"mean_dont_know_probability":0.020540149226569607,"spread":null,"answers_changed":null,"original_run":264,"original_accuracy":0.5959963603275705,"derived":{"cut_off":0.209385,"attempt_rate":0.9995450409463148,"non_hallucination":0.0011261261261261261,"right":1310,"wrong":887,"abstained":1},"vs_confidence":{"option_only_wrong":47,"confidence_only_wrong":42,"p":0.6718086491746106,"p_holm":null}},{"name":"GLM-5.3-Flash","model":"glm-5.3-flash","provider":"vllm-stated","perplexity_mode":null,"group":"chat","run":822,"runs":[822],"right":1295,"wrong":900,"dont_know":3,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.589171974522293,"attempt_rate":0.9986351228389445,"non_hallucination":0.0033222591362126247,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":159,"original_accuracy":0.5777979981801638,"derived":{"cut_off":0.4,"attempt_rate":0.9995450409463148,"non_hallucination":0.0010775862068965517,"right":1270,"wrong":927,"abstained":1},"vs_confidence":null},{"name":"Qwen3.5-4B","model":"qwen3.5-4b","provider":"vllm-plain","perplexity_mode":null,"group":"chat","run":1150,"runs":[1150],"right":1282,"wrong":881,"dont_know":30,"invalid":5,"invalid_replies":{"invalid_without_rule":5,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":2,"unreadable":3}},"accuracy":0.5832575068243858,"attempt_rate":0.9840764331210191,"non_hallucination":0.03293084522502744,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1149,"original_accuracy":0.589171974522293,"derived":null,"vs_confidence":null},{"name":"SubQ Snap 1.1","model":"subq-snap-1.1","provider":"subq","perplexity_mode":null,"group":"decision","run":392,"runs":[392],"right":1271,"wrong":490,"dont_know":437,"invalid":0,"invalid_replies":null,"accuracy":0.578252957233849,"attempt_rate":0.8011828935395814,"non_hallucination":0.4714131607335491,"mean_dont_know_probability":0.20102079041315762,"spread":null,"answers_changed":null,"original_run":346,"original_accuracy":0.6387625113739763,"derived":{"cut_off":0.512372612953186,"attempt_rate":0.8011828935395814,"non_hallucination":0.4760348583877996,"right":1280,"wrong":481,"abstained":437},"vs_confidence":{"option_only_wrong":148,"confidence_only_wrong":139,"p":0.6368426841662184,"p_holm":1.0}},{"name":"Jeffan-9B-1211","model":"Jeffan-9B-1211","provider":"di-serve","perplexity_mode":null,"group":"decision","run":1199,"runs":[1199],"right":1240,"wrong":917,"dont_know":41,"invalid":0,"invalid_replies":null,"accuracy":0.5641492265696088,"attempt_rate":0.9813466787989081,"non_hallucination":0.04279749478079332,"mean_dont_know_probability":0.05756002237403715,"spread":null,"answers_changed":null,"original_run":1192,"original_accuracy":0.5700636942675159,"derived":{"cut_off":0.24139061570167542,"attempt_rate":0.9813466787989081,"non_hallucination":0.0431578947368421,"right":1248,"wrong":909,"abstained":41},"vs_confidence":{"option_only_wrong":82,"confidence_only_wrong":74,"p":0.5753154836552089,"p_holm":1.0}},{"name":"Clef","model":"clef","provider":"clef","perplexity_mode":null,"group":"decision","run":788,"runs":[788],"right":1233,"wrong":894,"dont_know":71,"invalid":0,"invalid_replies":null,"accuracy":0.5609645131938126,"attempt_rate":0.967697907188353,"non_hallucination":0.07357512953367876,"mean_dont_know_probability":0.1010700636942675,"spread":null,"answers_changed":null,"original_run":125,"original_accuracy":0.5727934485896269,"derived":{"cut_off":0.3189,"attempt_rate":0.967697907188353,"non_hallucination":0.07395833333333333,"right":1238,"wrong":889,"abstained":71},"vs_confidence":{"option_only_wrong":83,"confidence_only_wrong":78,"p":0.7526889571542655,"p_holm":1.0}},{"name":"Open-Jev 27B v1.1","model":"open-jev-27b-v1.1","provider":"open-jev","perplexity_mode":null,"group":"decision","run":768,"runs":[768],"right":1219,"wrong":965,"dont_know":14,"invalid":0,"invalid_replies":null,"accuracy":0.5545950864422202,"attempt_rate":0.9936305732484076,"non_hallucination":0.014300306435137897,"mean_dont_know_probability":0.126699128010983,"spread":null,"answers_changed":null,"original_run":45,"original_accuracy":0.556869881710646,"derived":{"cut_off":0.20771854591552474,"attempt_rate":0.9936305732484076,"non_hallucination":0.014344262295081968,"right":1222,"wrong":962,"abstained":14},"vs_confidence":{"option_only_wrong":76,"confidence_only_wrong":73,"p":0.8699229710286416,"p_holm":null}},{"name":"Cygnet","model":"cygnet","provider":"cygnet","perplexity_mode":null,"group":"decision","run":757,"runs":[757],"right":1209,"wrong":957,"dont_know":32,"invalid":0,"invalid_replies":null,"accuracy":0.5500454959053686,"attempt_rate":0.9854413102820746,"non_hallucination":0.032355915065722954,"mean_dont_know_probability":0.050572584843940824,"spread":null,"answers_changed":null,"original_run":127,"original_accuracy":0.556869881710646,"derived":{"cut_off":0.2954889232646751,"attempt_rate":0.9854413102820746,"non_hallucination":0.0326197757390418,"right":1217,"wrong":949,"abstained":32},"vs_confidence":{"option_only_wrong":84,"confidence_only_wrong":76,"p":0.5801292063047091,"p_holm":1.0}},{"name":"Cerno 9B v1.2","model":"apertixai/cerno-9b-v1.2","provider":"cerno","perplexity_mode":null,"group":"decision","run":1204,"runs":[1204],"right":1198,"wrong":998,"dont_know":2,"invalid":0,"invalid_replies":null,"accuracy":0.5450409463148317,"attempt_rate":0.9990900818926297,"non_hallucination":0.002,"mean_dont_know_probability":0.028281392174704278,"spread":null,"answers_changed":null,"original_run":1201,"original_accuracy":0.5445859872611465,"derived":{"cut_off":0.2371,"attempt_rate":0.9990900818926297,"non_hallucination":0.001998001998001998,"right":1197,"wrong":999,"abstained":2},"vs_confidence":{"option_only_wrong":57,"confidence_only_wrong":58,"p":1.0,"p_holm":null}},{"name":"KnowLine-9B-Gen1","model":"PelaAI/KnowLine-9B-Gen1","provider":"knowline","perplexity_mode":null,"group":"decision","run":1181,"runs":[1181],"right":1197,"wrong":996,"dont_know":5,"invalid":0,"invalid_replies":null,"accuracy":0.5445859872611465,"attempt_rate":0.9977252047315741,"non_hallucination":0.004995004995004995,"mean_dont_know_probability":0.02125456542720374,"spread":null,"answers_changed":null,"original_run":1174,"original_accuracy":0.5454959053685169,"derived":{"cut_off":0.22812972977283574,"attempt_rate":0.9977252047315741,"non_hallucination":0.005,"right":1198,"wrong":995,"abstained":5},"vs_confidence":{"option_only_wrong":61,"confidence_only_wrong":60,"p":1.0,"p_holm":null}},{"name":"Mammoth-1","model":"baselabs/mammoth-1","provider":"baselabs","perplexity_mode":null,"group":"decision","run":386,"runs":[386],"right":1194,"wrong":314,"dont_know":690,"invalid":0,"invalid_replies":null,"accuracy":0.543221110100091,"attempt_rate":0.6860782529572339,"non_hallucination":0.6872509960159362,"mean_dont_know_probability":0.2445804940855323,"spread":null,"answers_changed":null,"original_run":374,"original_accuracy":0.6710646041856233,"derived":{"cut_off":0.569673,"attempt_rate":0.6860782529572339,"non_hallucination":0.7091469681397738,"right":1225,"wrong":283,"abstained":690},"vs_confidence":{"option_only_wrong":66,"confidence_only_wrong":35,"p":0.0026537860572285053,"p_holm":0.0981900841174547}},{"name":"Qwen3.8-27B","model":"qwen38-27b-bf16","provider":"vllm-verbal","perplexity_mode":null,"group":"chat","run":674,"runs":[674],"right":1192,"wrong":1000,"dont_know":6,"invalid":0,"invalid_replies":null,"accuracy":0.5423111919927207,"attempt_rate":0.997270245677889,"non_hallucination":0.005964214711729622,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":53,"original_accuracy":0.5418562329390355,"derived":{"cut_off":0.95,"attempt_rate":0.9981801637852593,"non_hallucination":0.003968253968253968,"right":1190,"wrong":1004,"abstained":4},"vs_confidence":null},{"name":"Winnow-12B","model":"winnow-12b","provider":"winnow","perplexity_mode":null,"group":"decision","run":750,"runs":[750],"right":1190,"wrong":965,"dont_know":43,"invalid":0,"invalid_replies":null,"accuracy":0.5414012738853503,"attempt_rate":0.9804367606915377,"non_hallucination":0.04265873015873016,"mean_dont_know_probability":0.034594174977333296,"spread":null,"answers_changed":null,"original_run":128,"original_accuracy":0.5545950864422202,"derived":{"cut_off":0.2922759781157272,"attempt_rate":0.9804367606915377,"non_hallucination":0.043478260869565216,"right":1209,"wrong":946,"abstained":43},"vs_confidence":{"option_only_wrong":94,"confidence_only_wrong":75,"p":0.16597930888290438,"p_holm":1.0}},{"name":"Qwen3.8-27B","model":"qwen38-27b-bf16","provider":"vllm-vote","perplexity_mode":null,"group":"chat","run":682,"runs":[682],"right":1188,"wrong":997,"dont_know":13,"invalid":0,"invalid_replies":null,"accuracy":0.54049135577798,"attempt_rate":0.9940855323020928,"non_hallucination":0.01287128712871287,"mean_dont_know_probability":0.2619047619047619,"spread":null,"answers_changed":null,"original_run":83,"original_accuracy":0.543221110100091,"derived":{"cut_off":0.4,"attempt_rate":0.9886260236578708,"non_hallucination":0.024752475247524754,"right":1188,"wrong":985,"abstained":25},"vs_confidence":null},{"name":"Adjudex 9B v5","model":"jev-latest","provider":"pplx-decider","perplexity_mode":null,"group":"decision","run":1190,"runs":[1190],"right":1169,"wrong":1021,"dont_know":8,"invalid":0,"invalid_replies":null,"accuracy":0.5318471337579618,"attempt_rate":0.9963603275705186,"non_hallucination":0.007774538386783284,"mean_dont_know_probability":0.015668016953251228,"spread":null,"answers_changed":null,"original_run":1183,"original_accuracy":0.5313921747042766,"derived":{"cut_off":0.23392014109612352,"attempt_rate":0.9963603275705186,"non_hallucination":0.007759456838021339,"right":1167,"wrong":1023,"abstained":8},"vs_confidence":{"option_only_wrong":55,"confidence_only_wrong":57,"p":0.9247750936572269,"p_holm":null}},{"name":"Gemma 4 E4B","model":"gemma-4-e4b-it","provider":"vllm-plain","perplexity_mode":null,"group":"chat","run":1049,"runs":[1049],"right":1164,"wrong":727,"dont_know":307,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.5295723384895359,"attempt_rate":0.8603275705186533,"non_hallucination":0.29690522243713735,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1040,"original_accuracy":0.575523202911738,"derived":null,"vs_confidence":null},{"name":"KnowLine-9B-Gen2","model":"PelaAI/KnowLine-9B-Gen2","provider":"knowline","perplexity_mode":null,"group":"decision","run":1325,"runs":[1325],"right":1141,"wrong":1030,"dont_know":27,"invalid":0,"invalid_replies":null,"accuracy":0.5191082802547771,"attempt_rate":0.9877161055505005,"non_hallucination":0.02554399243140965,"mean_dont_know_probability":0.06242743380597365,"spread":null,"answers_changed":null,"original_run":1306,"original_accuracy":0.5327570518653321,"derived":{"cut_off":0.237101379803763,"attempt_rate":0.9877161055505005,"non_hallucination":0.026061776061776062,"right":1162,"wrong":1009,"abstained":27},"vs_confidence":{"option_only_wrong":68,"confidence_only_wrong":47,"p":0.061708608239224014,"p_holm":1.0}},{"name":"Dexy-Decision-9B-V2","model":"Dexy-Decision-9B-V2","provider":"di-serve","perplexity_mode":null,"group":"decision","run":1262,"runs":[1262],"right":1115,"wrong":1046,"dont_know":37,"invalid":0,"invalid_replies":null,"accuracy":0.5072793448589626,"attempt_rate":0.9831665150136488,"non_hallucination":0.034164358264081256,"mean_dont_know_probability":0.02661423754551973,"spread":null,"answers_changed":null,"original_run":1255,"original_accuracy":0.5159235668789809,"derived":{"cut_off":0.2578863367050562,"attempt_rate":0.9831665150136488,"non_hallucination":0.03438661710037175,"right":1122,"wrong":1039,"abstained":37},"vs_confidence":{"option_only_wrong":81,"confidence_only_wrong":74,"p":0.6299975938841796,"p_holm":1.0}},{"name":"StartLux-Decision v1.1 9B","model":"startlux-models/StartLux-Decision-v1.1-9B-Instant","provider":"startlux","perplexity_mode":null,"group":"decision","run":1297,"runs":[1297],"right":1105,"wrong":1069,"dont_know":24,"invalid":0,"invalid_replies":null,"accuracy":0.502729754322111,"attempt_rate":0.989080982711556,"non_hallucination":0.021957913998170174,"mean_dont_know_probability":0.042620762647842106,"spread":null,"answers_changed":null,"original_run":1290,"original_accuracy":0.5122838944494995,"derived":{"cut_off":0.24310274026398151,"attempt_rate":0.989080982711556,"non_hallucination":0.0223463687150838,"right":1124,"wrong":1050,"abstained":24},"vs_confidence":{"option_only_wrong":67,"confidence_only_wrong":48,"p":0.09281592098218937,"p_holm":1.0}},{"name":"Liquid d1.2-ultra","model":"d1.2-ultra","provider":"liquid","perplexity_mode":null,"group":"decision","run":1367,"runs":[1367],"right":1057,"wrong":145,"dont_know":996,"invalid":0,"invalid_replies":null,"accuracy":0.48089171974522293,"attempt_rate":0.5468607825295724,"non_hallucination":0.8729184925503944,"mean_dont_know_probability":0.34625953468473547,"spread":null,"answers_changed":null,"original_run":1347,"original_accuracy":0.681073703366697,"derived":{"cut_off":0.5946971599608282,"attempt_rate":0.5468607825295724,"non_hallucination":0.8783068783068783,"right":1064,"wrong":138,"abstained":996},"vs_confidence":{"option_only_wrong":44,"confidence_only_wrong":37,"p":0.5052364407669359,"p_holm":1.0}},{"name":"Perplexity Decider Flash v1.3","model":"pplx-decider-flash-v1.3","provider":"pplx-decider","perplexity_mode":null,"group":"decision","run":1130,"runs":[1130],"right":1052,"wrong":1071,"dont_know":75,"invalid":0,"invalid_replies":null,"accuracy":0.4786169244767971,"attempt_rate":0.9658780709736123,"non_hallucination":0.06544502617801047,"mean_dont_know_probability":0.09078852754738051,"spread":null,"answers_changed":null,"original_run":1123,"original_accuracy":0.49727024567788897,"derived":{"cut_off":0.27685842124267235,"attempt_rate":0.9658780709736123,"non_hallucination":0.06672597864768683,"right":1074,"wrong":1049,"abstained":75},"vs_confidence":{"option_only_wrong":102,"confidence_only_wrong":80,"p":0.11931902228044168,"p_holm":1.0}},{"name":"GLiDE 9B (open)","model":"glide-9b","provider":"glide","perplexity_mode":null,"group":"decision","run":913,"runs":[913],"right":1049,"wrong":872,"dont_know":277,"invalid":0,"invalid_replies":null,"accuracy":0.47725204731574156,"attempt_rate":0.8739763421292084,"non_hallucination":0.2410791993037424,"mean_dont_know_probability":0.1522099190149912,"spread":null,"answers_changed":null,"original_run":907,"original_accuracy":0.5204731574158326,"derived":{"cut_off":0.33911940129407786,"attempt_rate":0.8739763421292084,"non_hallucination":0.24798567591763654,"right":1081,"wrong":840,"abstained":277},"vs_confidence":{"option_only_wrong":143,"confidence_only_wrong":111,"p":0.05154715491832388,"p_holm":1.0}},{"name":"Valen-Decision-1.5-9B","model":"valen","provider":"valen","perplexity_mode":null,"group":"decision","run":1282,"runs":[1282],"right":1040,"wrong":1157,"dont_know":1,"invalid":0,"invalid_replies":null,"accuracy":0.4731574158325751,"attempt_rate":0.9995450409463148,"non_hallucination":0.0008635578583765112,"mean_dont_know_probability":0.04754877442136828,"spread":null,"answers_changed":null,"original_run":1275,"original_accuracy":0.4695177434030937,"derived":{"cut_off":0.2162306667779448,"attempt_rate":0.9995450409463148,"non_hallucination":0.000856898029134533,"right":1031,"wrong":1166,"abstained":1},"vs_confidence":{"option_only_wrong":45,"confidence_only_wrong":54,"p":0.42152334361577004,"p_holm":null}},{"name":"Elephant-1","model":"baselabs/elephant-1","provider":"baselabs","perplexity_mode":null,"group":"decision","run":382,"runs":[382],"right":1035,"wrong":758,"dont_know":405,"invalid":0,"invalid_replies":null,"accuracy":0.4708826205641492,"attempt_rate":0.8157415832575068,"non_hallucination":0.3482373172828891,"mean_dont_know_probability":0.16036186169244768,"spread":null,"answers_changed":null,"original_run":373,"original_accuracy":0.5236578707916287,"derived":{"cut_off":0.388276,"attempt_rate":0.8157415832575068,"non_hallucination":0.35526315789473684,"right":1058,"wrong":735,"abstained":405},"vs_confidence":{"option_only_wrong":125,"confidence_only_wrong":102,"p":0.14407103116280764,"p_holm":1.0}},{"name":"StartLux-Decision v1.1 4B","model":"startlux-models/StartLux-Decision-v1.1-4B-Instant","provider":"startlux","perplexity_mode":null,"group":"decision","run":1239,"runs":[1239],"right":1030,"wrong":1147,"dont_know":21,"invalid":0,"invalid_replies":null,"accuracy":0.46860782529572337,"attempt_rate":0.9904458598726115,"non_hallucination":0.01797945205479452,"mean_dont_know_probability":0.05491166609075705,"spread":null,"answers_changed":null,"original_run":1236,"original_accuracy":0.4786169244767971,"derived":{"cut_off":0.24027224748590784,"attempt_rate":0.9904458598726115,"non_hallucination":0.018260869565217393,"right":1048,"wrong":1129,"abstained":21},"vs_confidence":{"option_only_wrong":89,"confidence_only_wrong":71,"p":0.1787772680491752,"p_holm":null}},{"name":"Verdin-4B-v1.1","model":"Verdin-4B-v1.1","provider":"di-serve","perplexity_mode":null,"group":"decision","run":1251,"runs":[1251],"right":1024,"wrong":1169,"dont_know":5,"invalid":0,"invalid_replies":null,"accuracy":0.4658780709736124,"attempt_rate":0.9977252047315741,"non_hallucination":0.004258943781942078,"mean_dont_know_probability":0.031943778878256,"spread":null,"answers_changed":null,"original_run":1244,"original_accuracy":0.467697907188353,"derived":{"cut_off":0.22685948317111995,"attempt_rate":0.9977252047315741,"non_hallucination":0.004266211604095563,"right":1026,"wrong":1167,"abstained":5},"vs_confidence":{"option_only_wrong":61,"confidence_only_wrong":59,"p":0.9273150210898833,"p_holm":null}},{"name":"KnowLine 4B Gen5.5","model":"PelaAI/KnowLine-4B-Gen5.5","provider":"knowline","perplexity_mode":null,"group":"decision","run":967,"runs":[967],"right":1022,"wrong":1121,"dont_know":55,"invalid":0,"invalid_replies":null,"accuracy":0.46496815286624205,"attempt_rate":0.9749772520473158,"non_hallucination":0.0467687074829932,"mean_dont_know_probability":0.02796991702493258,"spread":null,"answers_changed":null,"original_run":965,"original_accuracy":0.48498635122838946,"derived":{"cut_off":0.2688471818273842,"attempt_rate":0.9749772520473158,"non_hallucination":0.04795117698343505,"right":1051,"wrong":1092,"abstained":55},"vs_confidence":{"option_only_wrong":103,"confidence_only_wrong":74,"p":0.03502815419120742,"p_holm":1.0}},{"name":"vera-medium-omni","model":"vera-medium-omni","provider":"di-serve","perplexity_mode":null,"group":"decision","run":1271,"runs":[1271],"right":1018,"wrong":1075,"dont_know":105,"invalid":0,"invalid_replies":null,"accuracy":0.46314831665150136,"attempt_rate":0.9522292993630573,"non_hallucination":0.08898305084745763,"mean_dont_know_probability":0.0909099636032757,"spread":null,"answers_changed":null,"original_run":1264,"original_accuracy":0.4909008189262966,"derived":{"cut_off":0.2717,"attempt_rate":0.9522292993630573,"non_hallucination":0.09186351706036745,"right":1055,"wrong":1038,"abstained":105},"vs_confidence":{"option_only_wrong":125,"confidence_only_wrong":88,"p":0.013449713099605099,"p_holm":0.4841896715857836}},{"name":"Qwen3-14B","model":"qwen3-14b","provider":"vllm","perplexity_mode":null,"group":"chat","run":772,"runs":[772],"right":1004,"wrong":1171,"dont_know":23,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.456778889899909,"attempt_rate":0.9895359417652412,"non_hallucination":0.019262981574539362,"mean_dont_know_probability":0.010908756126070056,"spread":null,"answers_changed":null,"original_run":131,"original_accuracy":0.4572338489535942,"derived":{"cut_off":0.37737345041135073,"attempt_rate":0.9895359417652412,"non_hallucination":0.01921470342522974,"right":1001,"wrong":1174,"abstained":23},"vs_confidence":null},{"name":"Erk-14B","model":"erk-14b","provider":"vllm","perplexity_mode":null,"group":"chat","run":765,"runs":[765],"right":981,"wrong":1213,"dont_know":4,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.4463148316651501,"attempt_rate":0.9981801637852593,"non_hallucination":0.003286770747740345,"mean_dont_know_probability":0.009843504671441834,"spread":null,"answers_changed":null,"original_run":132,"original_accuracy":0.4499545040946315,"derived":{"cut_off":0.22875315191151124,"attempt_rate":0.9981801637852593,"non_hallucination":0.003305785123966942,"right":988,"wrong":1206,"abstained":4},"vs_confidence":null},{"name":"Chinese Decision Pointer 4B","model":"Chinese-Decision-Pointer-4B","provider":"di-serve","perplexity_mode":null,"group":"decision","run":1213,"runs":[1213],"right":980,"wrong":1082,"dont_know":136,"invalid":0,"invalid_replies":null,"accuracy":0.445859872611465,"attempt_rate":0.9381255686988171,"non_hallucination":0.1116584564860427,"mean_dont_know_probability":0.06849751419474755,"spread":null,"answers_changed":null,"original_run":1206,"original_accuracy":0.46678798908098273,"derived":{"cut_off":0.30477939446517205,"attempt_rate":0.9381255686988171,"non_hallucination":0.11295681063122924,"right":994,"wrong":1068,"abstained":136},"vs_confidence":{"option_only_wrong":146,"confidence_only_wrong":132,"p":0.435629098495047,"p_holm":1.0}},{"name":"Clef-flash","model":"clef-flash","provider":"clef","perplexity_mode":null,"group":"decision","run":664,"runs":[664],"right":980,"wrong":1108,"dont_know":110,"invalid":0,"invalid_replies":null,"accuracy":0.445859872611465,"attempt_rate":0.9499545040946314,"non_hallucination":0.090311986863711,"mean_dont_know_probability":0.09108621474067334,"spread":null,"answers_changed":null,"original_run":139,"original_accuracy":0.4572338489535942,"derived":{"cut_off":0.2929,"attempt_rate":0.9499545040946314,"non_hallucination":0.09038619556285948,"right":981,"wrong":1107,"abstained":110},"vs_confidence":{"option_only_wrong":109,"confidence_only_wrong":108,"p":1.0,"p_holm":1.0}},{"name":"Perplexity Decider Flash v1.2","model":"pplx-decider-flash-v1.2","provider":"pplx-decider","perplexity_mode":null,"group":"decision","run":791,"runs":[791],"right":974,"wrong":1044,"dont_know":180,"invalid":0,"invalid_replies":null,"accuracy":0.44313011828935395,"attempt_rate":0.9181073703366697,"non_hallucination":0.14705882352941177,"mean_dont_know_probability":0.12538404351080876,"spread":null,"answers_changed":null,"original_run":223,"original_accuracy":0.47770700636942676,"derived":{"cut_off":0.288310949330195,"attempt_rate":0.9181073703366697,"non_hallucination":0.15202702702702703,"right":1014,"wrong":1004,"abstained":180},"vs_confidence":{"option_only_wrong":162,"confidence_only_wrong":122,"p":0.02049282422875534,"p_holm":0.717248848006437}},{"name":"Liquid d1.2","model":"d1.2","provider":"liquid","perplexity_mode":null,"group":"decision","run":1365,"runs":[1365],"right":972,"wrong":253,"dont_know":973,"invalid":0,"invalid_replies":null,"accuracy":0.4422202001819836,"attempt_rate":0.5573248407643312,"non_hallucination":0.7936378466557912,"mean_dont_know_probability":0.3149403826078706,"spread":null,"answers_changed":null,"original_run":1346,"original_accuracy":0.6173794358507735,"derived":{"cut_off":0.49363874755897114,"attempt_rate":0.5573248407643312,"non_hallucination":0.8128654970760234,"right":1001,"wrong":224,"abstained":973},"vs_confidence":{"option_only_wrong":99,"confidence_only_wrong":70,"p":0.030947734779096257,"p_holm":1.0}},{"name":"Qwen3.5-4B","model":"qwen3.5-4b-nothink","provider":"vllm-plain","perplexity_mode":null,"group":"chat","run":1147,"runs":[1147],"right":964,"wrong":1132,"dont_know":84,"invalid":18,"invalid_replies":{"invalid_without_rule":18,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":8,"unreadable":10}},"accuracy":0.4385805277525023,"attempt_rate":0.9535941765241128,"non_hallucination":0.06907894736842106,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1146,"original_accuracy":0.46178343949044587,"derived":null,"vs_confidence":null},{"name":"Metask-Jev 4B","model":"metask-jev-4b-policy-mix","provider":"metask","perplexity_mode":null,"group":"decision","run":743,"runs":[743],"right":873,"wrong":1315,"dont_know":10,"invalid":0,"invalid_replies":null,"accuracy":0.39717925386715197,"attempt_rate":0.9954504094631483,"non_hallucination":0.007547169811320755,"mean_dont_know_probability":0.06338637977269893,"spread":null,"answers_changed":null,"original_run":126,"original_accuracy":0.40764331210191085,"derived":{"cut_off":0.21323101222515106,"attempt_rate":0.9954504094631483,"non_hallucination":0.007656967840735069,"right":892,"wrong":1296,"abstained":10},"vs_confidence":{"option_only_wrong":88,"confidence_only_wrong":69,"p":0.1506141469346891,"p_holm":null}},{"name":"LFM2.5-2.6B","model":"lfm2.5-2.6b","provider":"vllm-plain","perplexity_mode":null,"group":"chat","run":1038,"runs":[1038],"right":659,"wrong":1134,"dont_know":372,"invalid":33,"invalid_replies":{"invalid_without_rule":33,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":14,"unreadable":19}},"accuracy":0.29981801637852595,"attempt_rate":0.8157415832575068,"non_hallucination":0.24701195219123506,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1037,"original_accuracy":0.33894449499545043,"derived":null,"vs_confidence":null},{"name":"StartLux-Decision v1.1 2B","model":"startlux-models/StartLux-Decision-v1.1-2B-Instant","provider":"startlux","perplexity_mode":null,"group":"decision","run":979,"runs":[979],"right":652,"wrong":1213,"dont_know":333,"invalid":0,"invalid_replies":null,"accuracy":0.2966333030027298,"attempt_rate":0.848498635122839,"non_hallucination":0.21539456662354464,"mean_dont_know_probability":0.1319662286811646,"spread":null,"answers_changed":null,"original_run":977,"original_accuracy":0.34258416742493175,"derived":{"cut_off":0.2682035000421515,"attempt_rate":0.848498635122839,"non_hallucination":0.21778940483976456,"right":669,"wrong":1196,"abstained":333},"vs_confidence":{"option_only_wrong":248,"confidence_only_wrong":231,"p":0.4647797246010723,"p_holm":1.0}},{"name":"GLiDE 2B (open)","model":"glide-2b","provider":"glide","perplexity_mode":null,"group":"decision","run":921,"runs":[921],"right":608,"wrong":978,"dont_know":612,"invalid":0,"invalid_replies":null,"accuracy":0.27661510464058237,"attempt_rate":0.721565059144677,"non_hallucination":0.3849056603773585,"mean_dont_know_probability":0.2349647435973824,"spread":null,"answers_changed":null,"original_run":915,"original_accuracy":0.35304822565969063,"derived":{"cut_off":0.4077811881706783,"attempt_rate":0.721565059144677,"non_hallucination":0.38734177215189874,"right":618,"wrong":968,"abstained":612},"vs_confidence":{"option_only_wrong":211,"confidence_only_wrong":201,"p":0.6575300924895306,"p_holm":1.0}},{"name":"StartLux-Decision-v1.1-0.8B-Instant","model":"startlux-models/StartLux-Decision-v1.1-0.8B-Instant","provider":"startlux","perplexity_mode":null,"group":"decision","run":1017,"runs":[1017],"right":590,"wrong":1409,"dont_know":199,"invalid":0,"invalid_replies":null,"accuracy":0.2684258416742493,"attempt_rate":0.9094631483166515,"non_hallucination":0.12375621890547264,"mean_dont_know_probability":0.09414603931301203,"spread":null,"answers_changed":null,"original_run":1015,"original_accuracy":0.2857142857142857,"derived":{"cut_off":0.23181607612419805,"attempt_rate":0.9094631483166515,"non_hallucination":0.12337259764414135,"right":585,"wrong":1414,"abstained":199},"vs_confidence":{"option_only_wrong":173,"confidence_only_wrong":178,"p":0.8309716020852688,"p_holm":1.0}},{"name":"KnowLine 0.8B Gen1","model":"PelaAI/KnowLine-0.8B-Gen1","provider":"knowline","perplexity_mode":null,"group":"decision","run":1007,"runs":[1007],"right":579,"wrong":1515,"dont_know":104,"invalid":0,"invalid_replies":null,"accuracy":0.26342129208371245,"attempt_rate":0.9526842584167425,"non_hallucination":0.06423718344657196,"mean_dont_know_probability":0.08106536098781156,"spread":null,"answers_changed":null,"original_run":1005,"original_accuracy":0.27115559599636035,"derived":{"cut_off":0.23481813439629043,"attempt_rate":0.9526842584167425,"non_hallucination":0.06419753086419754,"right":578,"wrong":1516,"abstained":104},"vs_confidence":{"option_only_wrong":122,"confidence_only_wrong":123,"p":1.0,"p_holm":1.0}},{"name":"Strands Decider 2B","model":"strands-decider-2b","provider":"strands","perplexity_mode":null,"group":"decision","run":706,"runs":[706],"right":565,"wrong":1483,"dont_know":150,"invalid":0,"invalid_replies":null,"accuracy":0.2570518653321201,"attempt_rate":0.9317561419472248,"non_hallucination":0.09185548071034905,"mean_dont_know_probability":0.08797256596906278,"spread":null,"answers_changed":null,"original_run":141,"original_accuracy":0.2747952684258417,"derived":{"cut_off":0.2554,"attempt_rate":0.9317561419472248,"non_hallucination":0.09236453201970443,"right":574,"wrong":1474,"abstained":150},"vs_confidence":{"option_only_wrong":148,"confidence_only_wrong":139,"p":0.6368426841662184,"p_holm":1.0}},{"name":"GLiDE 0.8B (open)","model":"glide-0.8b","provider":"glide","perplexity_mode":null,"group":"decision","run":929,"runs":[929],"right":555,"wrong":1163,"dont_know":480,"invalid":0,"invalid_replies":null,"accuracy":0.25250227479526843,"attempt_rate":0.7816196542311192,"non_hallucination":0.29214850882531956,"mean_dont_know_probability":0.19867183788609985,"spread":null,"answers_changed":null,"original_run":923,"original_accuracy":0.29253867151956325,"derived":{"cut_off":0.3649873367151997,"attempt_rate":0.7816196542311192,"non_hallucination":0.2888086642599278,"right":536,"wrong":1182,"abstained":480},"vs_confidence":{"option_only_wrong":196,"confidence_only_wrong":215,"p":0.3746290784869244,"p_holm":1.0}},{"name":"Qwen3.5-2B","model":"qwen3.5-2b-nothink","provider":"vllm-plain","perplexity_mode":null,"group":"chat","run":1219,"runs":[1219],"right":460,"wrong":1202,"dont_know":315,"invalid":221,"invalid_replies":{"invalid_without_rule":221,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":5,"unreadable":216}},"accuracy":0.20928116469517744,"attempt_rate":0.7561419472247498,"non_hallucination":0.20764667106130522,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1218,"original_accuracy":0.2538671519563239,"derived":null,"vs_confidence":null},{"name":"OneJev1.6-0.8B","model":"OmniJev/OneJev1.6-0.8B","provider":"onejev","perplexity_mode":null,"group":"decision","run":1031,"runs":[1031],"right":459,"wrong":985,"dont_know":754,"invalid":0,"invalid_replies":null,"accuracy":0.20882620564149226,"attempt_rate":0.6569608735213831,"non_hallucination":0.4335825186889017,"mean_dont_know_probability":0.22170616378525934,"spread":null,"answers_changed":null,"original_run":1029,"original_accuracy":0.2911737943585077,"derived":{"cut_off":0.297159,"attempt_rate":0.6569608735213831,"non_hallucination":0.4345821325648415,"right":463,"wrong":981,"abstained":754},"vs_confidence":{"option_only_wrong":314,"confidence_only_wrong":310,"p":0.9044195603456318,"p_holm":1.0}},{"name":"GLiNER2.5-multi-Decide","model":"gliner2.5-multi-decide","provider":"gliner","perplexity_mode":null,"group":"decision","run":718,"runs":[718],"right":418,"wrong":1666,"dont_know":114,"invalid":0,"invalid_replies":null,"accuracy":0.19017288444040037,"attempt_rate":0.9481346678798908,"non_hallucination":0.06404494382022471,"mean_dont_know_probability":0.10655747927577333,"spread":null,"answers_changed":null,"original_run":135,"original_accuracy":0.20200181983621474,"derived":{"cut_off":0.2552577257156372,"attempt_rate":0.9481346678798908,"non_hallucination":0.06433408577878104,"right":426,"wrong":1658,"abstained":114},"vs_confidence":{"option_only_wrong":212,"confidence_only_wrong":204,"p":0.731491376282839,"p_holm":1.0}},{"name":"GLiClass v4 Edge","model":"knowledgator/gliclass-v4-edge","provider":"gliclass","perplexity_mode":null,"group":"decision","run":1048,"runs":[1048],"right":414,"wrong":1386,"dont_know":398,"invalid":0,"invalid_replies":null,"accuracy":0.18835304822565968,"attempt_rate":0.818926296633303,"non_hallucination":0.22309417040358745,"mean_dont_know_probability":0.14478884716355722,"spread":null,"answers_changed":null,"original_run":1046,"original_accuracy":0.23339399454049137,"derived":{"cut_off":0.2422740542762547,"attempt_rate":0.818926296633303,"non_hallucination":0.2257515598411798,"right":435,"wrong":1365,"abstained":398},"vs_confidence":{"option_only_wrong":261,"confidence_only_wrong":240,"p":0.371587475373936,"p_holm":1.0}},{"name":"LFM3-1.7B","model":"lfm3-1.7b","provider":"vllm-plain","perplexity_mode":null,"group":"chat","run":1090,"runs":[1090],"right":412,"wrong":589,"dont_know":1197,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"accuracy":0.18744313011828936,"attempt_rate":0.4554140127388535,"non_hallucination":0.6702127659574468,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1089,"original_accuracy":0.32393084622383983,"derived":null,"vs_confidence":null},{"name":"GLiNER2.5 Multi (base)","model":"gliner2.5-multi-v1","provider":"gliner","perplexity_mode":null,"group":"chat","run":724,"runs":[724],"right":363,"wrong":1719,"dont_know":116,"invalid":0,"invalid_replies":null,"accuracy":0.1651501364877161,"attempt_rate":0.9472247497725205,"non_hallucination":0.06321525885558583,"mean_dont_know_probability":0.12038671058683983,"spread":null,"answers_changed":null,"original_run":122,"original_accuracy":0.17470427661510465,"derived":{"cut_off":0.3186262547969818,"attempt_rate":0.9472247497725205,"non_hallucination":0.06318082788671024,"right":362,"wrong":1720,"abstained":116},"vs_confidence":null},{"name":"Liquid d1-2B (open weights)","model":"LiquidAI/d1-2B","provider":"d1-server","perplexity_mode":null,"group":"decision","run":712,"runs":[712],"right":352,"wrong":637,"dont_know":1209,"invalid":0,"invalid_replies":null,"accuracy":0.16014558689717925,"attempt_rate":0.4499545040946315,"non_hallucination":0.6549295774647887,"mean_dont_know_probability":0.2633142167217206,"spread":null,"answers_changed":null,"original_run":340,"original_accuracy":0.29390354868061874,"derived":{"cut_off":0.3129104235056768,"attempt_rate":0.4499545040946315,"non_hallucination":0.6653824986241057,"right":381,"wrong":608,"abstained":1209},"vs_confidence":{"option_only_wrong":194,"confidence_only_wrong":165,"p":0.13935710194324272,"p_holm":1.0}},{"name":"Laya multilingual","model":"laya-multilingual","provider":"laya","perplexity_mode":null,"group":"decision","run":737,"runs":[737],"right":310,"wrong":1608,"dont_know":280,"invalid":0,"invalid_replies":null,"accuracy":0.1410373066424022,"attempt_rate":0.8726114649681529,"non_hallucination":0.1483050847457627,"mean_dont_know_probability":0.14585250227479526,"spread":null,"answers_changed":null,"original_run":46,"original_accuracy":0.17879890809827115,"derived":{"cut_off":0.2794,"attempt_rate":0.8726114649681529,"non_hallucination":0.14997321906802358,"right":331,"wrong":1587,"abstained":280},"vs_confidence":{"option_only_wrong":261,"confidence_only_wrong":240,"p":0.371587475373936,"p_holm":1.0}},{"name":"Laya","model":"laya","provider":"laya","perplexity_mode":null,"group":"decision","run":730,"runs":[730],"right":307,"wrong":1355,"dont_know":536,"invalid":0,"invalid_replies":null,"accuracy":0.13967242948134667,"attempt_rate":0.7561419472247498,"non_hallucination":0.2834479111581174,"mean_dont_know_probability":0.18576005459508643,"spread":null,"answers_changed":null,"original_run":51,"original_accuracy":0.19017288444040037,"derived":{"cut_off":0.2527,"attempt_rate":0.7556869881710646,"non_hallucination":0.28533475026567484,"right":316,"wrong":1345,"abstained":537},"vs_confidence":{"option_only_wrong":404,"confidence_only_wrong":394,"p":0.7500551894762573,"p_holm":1.0}},{"name":"LFM3-1.7B","model":"lfm3-1.7b-vllm-nothink","provider":"vllm-plain","perplexity_mode":null,"group":"chat","run":1234,"runs":[1234],"right":270,"wrong":448,"dont_know":1433,"invalid":47,"invalid_replies":{"invalid_without_rule":47,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":21,"unreadable":26}},"accuracy":0.12283894449499545,"attempt_rate":0.32666060054595086,"non_hallucination":0.7618288144603934,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1227,"original_accuracy":0.28343949044585987,"derived":null,"vs_confidence":null},{"name":"LFM2.5-8B-A1B","model":"lfm2.5-8b-a1b","provider":"vllm-plain","perplexity_mode":null,"group":"chat","run":1298,"runs":[1298],"right":242,"wrong":690,"dont_know":878,"invalid":388,"invalid_replies":{"invalid_without_rule":388,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":4,"unreadable":384}},"accuracy":0.11010009099181074,"attempt_rate":0.4240218380345769,"non_hallucination":0.5599489795918368,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1288,"original_accuracy":0.2097361237488626,"derived":null,"vs_confidence":null},{"name":"MiniCPM5-2B","model":"minicpm5-2b-nothink","provider":"vllm-plain","perplexity_mode":null,"group":"chat","run":1304,"runs":[1304],"right":109,"wrong":332,"dont_know":113,"invalid":1644,"invalid_replies":{"invalid_without_rule":1644,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":5,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"empty after retry":3,"cut off":687,"unreadable":954}},"accuracy":0.049590536851683346,"attempt_rate":0.20063694267515925,"non_hallucination":0.2539325842696629,"mean_dont_know_probability":null,"spread":null,"answers_changed":null,"original_run":1299,"original_accuracy":0.05595996360327571,"derived":null,"vs_confidence":null},{"name":"GLiDE 0.4B (open)","model":"glide-0.4b","provider":"glide","perplexity_mode":null,"group":"decision","run":937,"runs":[937],"right":27,"wrong":101,"dont_know":2070,"invalid":0,"invalid_replies":null,"accuracy":0.012283894449499545,"attempt_rate":0.05823475887170155,"non_hallucination":0.9534776600644864,"mean_dont_know_probability":0.39109773835725975,"spread":null,"answers_changed":null,"original_run":931,"original_accuracy":0.22247497725204732,"derived":{"cut_off":0.4442926636124225,"attempt_rate":0.05823475887170155,"non_hallucination":0.9543568464730291,"right":29,"wrong":99,"abstained":2070},"vs_confidence":{"option_only_wrong":53,"confidence_only_wrong":51,"p":0.9219488276264309,"p_holm":1.0}}]},"pipelines":[{"decision":44,"frontier":48,"seed":1,"tolerance_points":0.5,"rule":"half","batch_size":16,"threshold":0.9156852342507505,"held_out":1099,"routed":194,"simulated":{"accuracy":0.9736123748862603,"frontier_accuracy":0.9736123748862603},"measured":{"pipeline_run":70,"alone_run":71,"accuracy":0.9708826205641492,"frontier_accuracy":0.9745222929936306,"difference":-0.003639672429481347,"low":-0.008002885959281333,"high":0.0007235411003186395,"p":0.21875,"within_tolerance":true,"decision_cost_usd":0.01707551607239819,"cost_usd":4.714345516072398,"frontier_cost_usd":5.67298,"request_seconds":1127.6062055643858,"frontier_request_seconds":992.3243447878631,"pipeline_tokens":{"input_tokens":466347,"cached_input_tokens":112000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":20836},"alone_tokens":{"input_tokens":582668,"cached_input_tokens":144000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":22846},"alternatives":[{"run":137,"accuracy":0.9681528662420382,"cost_usd":1.3225,"request_seconds":1362.2349697217578,"versus_pipeline":{"difference":-0.00272975432211101,"low":-0.01008129238170055,"high":0.0046217837374785305,"p":0.629058837890625},"versus_frontier":{"difference":-0.006369426751592357,"low":-0.012788672867968776,"high":4.981936478406189e-05,"p":0.09228515625},"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582670,"cached_input_tokens":57600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":26660}},{"run":138,"accuracy":0.910828025477707,"cost_usd":0.29709675,"request_seconds":300.7470318475971,"versus_pipeline":{"difference":-0.06005459508644222,"low":-0.07580918063887844,"high":-0.044300009534005994,"p":1.647020623262653e-14},"versus_frontier":{"difference":-0.06369426751592357,"low":-0.07939880684293842,"high":-0.04798972818890872,"p":1.5686991122453906e-16},"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"tokens":{"input_tokens":244424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30341}},{"run":165,"accuracy":0.8935395814376706,"cost_usd":5.387050599999999,"request_seconds":929.1633372180659,"versus_pipeline":{"difference":-0.07734303912647862,"low":-0.09591358971040771,"high":-0.05877248854254953,"p":1.3522957587316645e-16},"versus_frontier":{"difference":-0.08098271155595996,"low":-0.09915214754244522,"high":-0.06281327556947469,"p":4.0853725805661456e-19},"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":628800,"cached_input_tokens":171673,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":456987,"output_tokens":84813}},{"run":68,"accuracy":0.8489535941765242,"cost_usd":4.1268408,"request_seconds":1799.8465952617116,"versus_pipeline":{"difference":-0.12192902638762511,"low":-0.1425477397529321,"high":-0.1013103130223181,"p":7.799950339037261e-33},"versus_frontier":{"difference":-0.12556869881710647,"low":-0.14611091193863107,"high":-0.10502648569558187,"p":2.08925524731209e-35},"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":609112,"cached_input_tokens":148308,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":460666,"output_tokens":226907}}]}},{"decision":44,"frontier":55,"seed":1,"tolerance_points":0.5,"rule":"half","batch_size":16,"threshold":0.7190257589471052,"held_out":1099,"routed":343,"simulated":{"accuracy":0.8525932666060054,"frontier_accuracy":0.8462238398544131},"measured":{"pipeline_run":62,"alone_run":68,"accuracy":0.8480436760691538,"frontier_accuracy":0.8489535941765242,"difference":-0.0009099181073703367,"low":-0.01952949531030512,"high":0.017709659095564444,"p":1.0,"within_tolerance":true,"decision_cost_usd":0.01707551607239819,"cost_usd":3.2248042160723984,"frontier_cost_usd":4.1268408,"request_seconds":1845.901194359744,"frontier_request_seconds":1799.8465952617116,"pipeline_tokens":{"input_tokens":383982,"cached_input_tokens":102507,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":281379,"output_tokens":207177},"alone_tokens":{"input_tokens":609112,"cached_input_tokens":148308,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":460666,"output_tokens":226907},"alternatives":[{"run":71,"accuracy":0.9745222929936306,"cost_usd":5.67298,"request_seconds":992.3243447878631,"versus_pipeline":{"difference":0.1264786169244768,"low":0.10618973142600935,"high":0.14676750242294423,"p":2.1525608033566037e-37},"versus_frontier":{"difference":0.12556869881710647,"low":0.10502648569558187,"high":0.14611091193863107,"p":2.08925524731209e-35},"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582668,"cached_input_tokens":144000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":22846}},{"run":137,"accuracy":0.9681528662420382,"cost_usd":1.3225,"request_seconds":1362.2349697217578,"versus_pipeline":{"difference":0.12010919017288443,"low":0.0996077339118915,"high":0.14061064643387738,"p":2.797312157195774e-32},"versus_frontier":{"difference":0.1191992720655141,"low":0.09875690893398828,"high":0.13964163519703993,"p":5.294448923596039e-32},"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582670,"cached_input_tokens":57600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":26660}},{"run":138,"accuracy":0.910828025477707,"cost_usd":0.29709675,"request_seconds":300.7470318475971,"versus_pipeline":{"difference":0.06278434940855324,"low":0.04046156514295063,"high":0.08510713367415584,"p":5.283208926292809e-08},"versus_frontier":{"difference":0.06187443130118289,"low":0.03877269078891368,"high":0.0849761718134521,"p":2.2849587651185474e-07},"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"tokens":{"input_tokens":244424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30341}},{"run":165,"accuracy":0.8935395814376706,"cost_usd":5.387050599999999,"request_seconds":929.1633372180659,"versus_pipeline":{"difference":0.04549590536851683,"low":0.02324038085043073,"high":0.06775142988660293,"p":8.548342922070617e-05},"versus_frontier":{"difference":0.044585987261146494,"low":0.022829831897380495,"high":0.06634214262491249,"p":8.212852870228109e-05},"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":628800,"cached_input_tokens":171673,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":456987,"output_tokens":84813}}]}},{"decision":145,"frontier":48,"seed":1,"tolerance_points":0.5,"rule":"cutoff","batch_size":16,"threshold":0.99,"held_out":1099,"routed":176,"simulated":{"accuracy":0.9736123748862603,"frontier_accuracy":0.9736123748862603},"measured":{"pipeline_run":176,"alone_run":71,"accuracy":0.9754322111010009,"frontier_accuracy":0.9745222929936306,"difference":0.0009099181073703367,"low":-0.003808312575386305,"high":0.005628148790126978,"p":1.0,"within_tolerance":true,"decision_cost_usd":0.008533460000000001,"cost_usd":4.83464346,"frontier_cost_usd":5.67298,"request_seconds":1244.687020608093,"frontier_request_seconds":992.3243447878631,"pipeline_tokens":{"input_tokens":479281,"cached_input_tokens":115200,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":21402},"alone_tokens":{"input_tokens":582668,"cached_input_tokens":144000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":22846},"alternatives":[{"run":137,"accuracy":0.9681528662420382,"cost_usd":1.3225,"request_seconds":1362.2349697217578,"versus_pipeline":{"difference":-0.007279344858962694,"low":-0.014400108715310153,"high":-0.0001585810026152332,"p":0.076812744140625},"versus_frontier":{"difference":-0.006369426751592357,"low":-0.012788672867968776,"high":4.981936478406189e-05,"p":0.09228515625},"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582670,"cached_input_tokens":57600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":26660}},{"run":138,"accuracy":0.910828025477707,"cost_usd":0.29709675,"request_seconds":300.7470318475971,"versus_pipeline":{"difference":-0.0646041856232939,"low":-0.08059688556495011,"high":-0.04861148568163769,"p":2.7959374761947534e-16},"versus_frontier":{"difference":-0.06369426751592357,"low":-0.07939880684293842,"high":-0.04798972818890872,"p":1.5686991122453906e-16},"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"tokens":{"input_tokens":244424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30341}},{"run":165,"accuracy":0.8935395814376706,"cost_usd":5.387050599999999,"request_seconds":929.1633372180659,"versus_pipeline":{"difference":-0.0818926296633303,"low":-0.10013519006432656,"high":-0.06365006926233405,"p":2.262519224355612e-19},"versus_frontier":{"difference":-0.08098271155595996,"low":-0.09915214754244522,"high":-0.06281327556947469,"p":4.0853725805661456e-19},"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":628800,"cached_input_tokens":171673,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":456987,"output_tokens":84813}},{"run":68,"accuracy":0.8489535941765242,"cost_usd":4.1268408,"request_seconds":1799.8465952617116,"versus_pipeline":{"difference":-0.1264786169244768,"low":-0.14707865334275164,"high":-0.10587858050620194,"p":1.0875369892924252e-35},"versus_frontier":{"difference":-0.12556869881710647,"low":-0.14611091193863107,"high":-0.10502648569558187,"p":2.08925524731209e-35},"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":609112,"cached_input_tokens":148308,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":460666,"output_tokens":226907}}]}},{"decision":145,"frontier":48,"seed":1,"tolerance_points":0.5,"rule":"half","batch_size":1,"threshold":0.9825243141806727,"held_out":1099,"routed":219,"simulated":{"accuracy":0.9727024567788899,"frontier_accuracy":0.9736123748862603},"measured":{"pipeline_run":169,"alone_run":163,"accuracy":0.9727024567788899,"frontier_accuracy":0.9754322111010009,"difference":-0.00272975432211101,"low":-0.007445530788481704,"high":0.001986022144259684,"p":0.453125,"within_tolerance":true,"decision_cost_usd":0.008533460000000001,"cost_usd":23.68529946,"frontier_cost_usd":31.648706,"request_seconds":11274.370967360068,"frontier_request_seconds":7964.2585057840915,"waits":{"pipeline":{"questions":1099,"median_seconds":6.631,"p90_seconds":10.136600000000001,"mean_seconds":10.271715195632392,"max_seconds":394.978,"total_seconds":11288.615},"kept":{"questions":219,"median_seconds":0.231,"p90_seconds":0.2834,"mean_seconds":0.2428812785388128,"max_seconds":0.546,"total_seconds":53.191},"passed":{"questions":880,"median_seconds":7.1995,"p90_seconds":10.9358,"mean_seconds":12.767527272727273,"max_seconds":394.978,"total_seconds":11235.424},"alone":{"questions":1099,"median_seconds":6.665,"p90_seconds":9.7262,"mean_seconds":7.246808917197453,"max_seconds":28.965,"total_seconds":7964.243}},"pipeline_tokens":{"input_tokens":4074897,"cached_input_tokens":2221056,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":58346},"alone_tokens":{"input_tokens":5103512,"cached_input_tokens":2509696,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":64017},"alternatives":[{"run":166,"accuracy":0.9272065514103731,"cost_usd":12.8286324,"request_seconds":4694.714676301097,"versus_pipeline":{"difference":-0.04549590536851683,"low":-0.059507567242236284,"high":-0.03148424349479738,"p":7.638141345540223e-11},"versus_frontier":{"difference":-0.048225659690627844,"low":-0.06209114647456982,"high":-0.03436017290668587,"p":1.6623508125590547e-12},"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":1746017,"cached_input_tokens":701622,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":1042197,"output_tokens":217097}},{"run":167,"accuracy":0.8935395814376706,"cost_usd":8.6273988,"request_seconds":6064.148409693618,"versus_pipeline":{"difference":-0.0791628753412193,"low":-0.09664720801122059,"high":-0.061678542671217994,"p":5.109797372820077e-20},"versus_frontier":{"difference":-0.0818926296633303,"low":-0.09942390909910859,"high":-0.06436135022755202,"p":2.253920747846914e-21},"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":1648174,"cached_input_tokens":599508,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":1046468,"output_tokens":437718}}]}},{"decision":145,"frontier":48,"seed":1,"tolerance_points":0.5,"rule":"share","batch_size":16,"threshold":0.9642039538871846,"held_out":1099,"routed":275,"simulated":{"accuracy":0.9727024567788899,"frontier_accuracy":0.9736123748862603},"share":0.25,"measured":{"pipeline_run":175,"alone_run":71,"accuracy":0.9708826205641492,"frontier_accuracy":0.9745222929936306,"difference":-0.003639672429481347,"low":-0.00981393925260459,"high":0.0025345943936418975,"p":0.3876953125,"within_tolerance":true,"decision_cost_usd":0.008533460000000001,"cost_usd":4.67707346,"frontier_cost_usd":5.67298,"request_seconds":1113.8878569669614,"frontier_request_seconds":992.3243447878631,"pipeline_tokens":{"input_tokens":424904,"cached_input_tokens":60800,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":19334},"alone_tokens":{"input_tokens":582668,"cached_input_tokens":144000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":22846},"alternatives":[{"run":137,"accuracy":0.9681528662420382,"cost_usd":1.3225,"request_seconds":1362.2349697217578,"versus_pipeline":{"difference":-0.00272975432211101,"low":-0.010900907086924596,"high":0.005441398442702574,"p":0.6636238098144531},"versus_frontier":{"difference":-0.006369426751592357,"low":-0.012788672867968776,"high":4.981936478406189e-05,"p":0.09228515625},"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582670,"cached_input_tokens":57600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":26660}},{"run":138,"accuracy":0.910828025477707,"cost_usd":0.29709675,"request_seconds":300.7470318475971,"versus_pipeline":{"difference":-0.06005459508644222,"low":-0.07600979106647317,"high":-0.044099399106411266,"p":4.308584029113881e-14},"versus_frontier":{"difference":-0.06369426751592357,"low":-0.07939880684293842,"high":-0.04798972818890872,"p":1.5686991122453906e-16},"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"tokens":{"input_tokens":244424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30341}},{"run":165,"accuracy":0.8935395814376706,"cost_usd":5.387050599999999,"request_seconds":929.1633372180659,"versus_pipeline":{"difference":-0.07734303912647862,"low":-0.09608408130194028,"high":-0.05860199695101696,"p":2.8669598238307665e-16},"versus_frontier":{"difference":-0.08098271155595996,"low":-0.09915214754244522,"high":-0.06281327556947469,"p":4.0853725805661456e-19},"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":628800,"cached_input_tokens":171673,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":456987,"output_tokens":84813}},{"run":68,"accuracy":0.8489535941765242,"cost_usd":4.1268408,"request_seconds":1799.8465952617116,"versus_pipeline":{"difference":-0.12192902638762511,"low":-0.1425477397529321,"high":-0.1013103130223181,"p":7.799950339037261e-33},"versus_frontier":{"difference":-0.12556869881710647,"low":-0.14611091193863107,"high":-0.10502648569558187,"p":2.08925524731209e-35},"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":609112,"cached_input_tokens":148308,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":460666,"output_tokens":226907}}]}},{"decision":145,"frontier":48,"seed":1,"tolerance_points":0.5,"rule":"half","batch_size":16,"threshold":0.9825243141806727,"held_out":1099,"routed":219,"simulated":{"accuracy":0.9727024567788899,"frontier_accuracy":0.9736123748862603},"measured":{"pipeline_run":161,"alone_run":71,"accuracy":0.9745222929936306,"frontier_accuracy":0.9745222929936306,"difference":0.0,"low":-0.0050443286301205936,"high":0.0050443286301205936,"p":1.0,"within_tolerance":true,"decision_cost_usd":0.008533460000000001,"cost_usd":4.9963634599999995,"frontier_cost_usd":5.67298,"request_seconds":1189.094736152052,"frontier_request_seconds":992.3243447878631,"pipeline_tokens":{"input_tokens":453928,"cached_input_tokens":64000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":20491},"alone_tokens":{"input_tokens":582668,"cached_input_tokens":144000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":22846},"alternatives":[{"run":137,"accuracy":0.9681528662420382,"cost_usd":1.3225,"request_seconds":1362.2349697217578,"versus_pipeline":{"difference":-0.006369426751592357,"low":-0.014533492652310458,"high":0.0017946391491257437,"p":0.18924713134765625},"versus_frontier":{"difference":-0.006369426751592357,"low":-0.012788672867968776,"high":4.981936478406189e-05,"p":0.09228515625},"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582670,"cached_input_tokens":57600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":26660}},{"run":138,"accuracy":0.910828025477707,"cost_usd":0.29709675,"request_seconds":300.7470318475971,"versus_pipeline":{"difference":-0.06369426751592357,"low":-0.07960004848719057,"high":-0.04778848654465656,"p":5.137564699362503e-16},"versus_frontier":{"difference":-0.06369426751592357,"low":-0.07939880684293842,"high":-0.04798972818890872,"p":1.5686991122453906e-16},"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"tokens":{"input_tokens":244424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30341}},{"run":165,"accuracy":0.8935395814376706,"cost_usd":5.387050599999999,"request_seconds":929.1633372180659,"versus_pipeline":{"difference":-0.08098271155595996,"low":-0.09949894847741117,"high":-0.062466474634508745,"p":2.687543585380812e-18},"versus_frontier":{"difference":-0.08098271155595996,"low":-0.09915214754244522,"high":-0.06281327556947469,"p":4.0853725805661456e-19},"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":628800,"cached_input_tokens":171673,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":456987,"output_tokens":84813}},{"run":68,"accuracy":0.8489535941765242,"cost_usd":4.1268408,"request_seconds":1799.8465952617116,"versus_pipeline":{"difference":-0.12556869881710647,"low":-0.14611091193863107,"high":-0.10502648569558187,"p":2.08925524731209e-35},"versus_frontier":{"difference":-0.12556869881710647,"low":-0.14611091193863107,"high":-0.10502648569558187,"p":2.08925524731209e-35},"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":609112,"cached_input_tokens":148308,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":460666,"output_tokens":226907}}]}},{"decision":145,"frontier":47,"seed":1,"tolerance_points":0.5,"rule":"cutoff","batch_size":16,"threshold":0.99,"held_out":1099,"routed":176,"simulated":{"accuracy":0.905368516833485,"frontier_accuracy":0.905368516833485},"measured":{"pipeline_run":172,"alone_run":165,"accuracy":0.8844404003639672,"frontier_accuracy":0.8935395814376706,"difference":-0.009099181073703366,"low":-0.027796314332103306,"high":0.009597952184696572,"p":0.39092745669950485,"within_tolerance":false,"decision_cost_usd":0.008533460000000001,"cost_usd":4.4161930599999994,"frontier_cost_usd":5.387050599999999,"request_seconds":1078.980863467048,"frontier_request_seconds":929.1633372180659,"pipeline_tokens":{"input_tokens":495388,"cached_input_tokens":129618,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":365654,"output_tokens":72802},"alone_tokens":{"input_tokens":628800,"cached_input_tokens":171673,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":456987,"output_tokens":84813},"alternatives":[{"run":71,"accuracy":0.9745222929936306,"cost_usd":5.67298,"request_seconds":992.3243447878631,"versus_pipeline":{"difference":0.09008189262966333,"low":0.07206280755970734,"high":0.10810097769961932,"p":1.8478334088755076e-24},"versus_frontier":{"difference":0.08098271155595996,"low":0.06281327556947469,"high":0.09915214754244522,"p":4.0853725805661456e-19},"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582668,"cached_input_tokens":144000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":22846}},{"run":137,"accuracy":0.9681528662420382,"cost_usd":1.3225,"request_seconds":1362.2349697217578,"versus_pipeline":{"difference":0.08371246587807098,"low":0.06498223676411861,"high":0.10244269499202334,"p":4.764406071312224e-19},"versus_frontier":{"difference":0.0746132848043676,"low":0.05591847354625709,"high":0.09330809606247811,"p":2.9566626383126704e-15},"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582670,"cached_input_tokens":57600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":26660}},{"run":138,"accuracy":0.910828025477707,"cost_usd":0.29709675,"request_seconds":300.7470318475971,"versus_pipeline":{"difference":0.026387625113739762,"low":0.0061918159303973284,"high":0.0465834342970822,"p":0.013380437039567964},"versus_frontier":{"difference":0.017288444040036398,"low":-0.0034080250492140966,"high":0.037984913129286896,"p":0.12101372565511773},"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"tokens":{"input_tokens":244424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30341}},{"run":68,"accuracy":0.8489535941765242,"cost_usd":4.1268408,"request_seconds":1799.8465952617116,"versus_pipeline":{"difference":-0.03548680618744313,"low":-0.05843822530069169,"high":-0.012535387074194576,"p":0.00315764881359094},"versus_frontier":{"difference":-0.044585987261146494,"low":-0.06634214262491249,"high":-0.022829831897380495,"p":8.212852870228109e-05},"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":609112,"cached_input_tokens":148308,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":460666,"output_tokens":226907}}]}},{"decision":145,"frontier":47,"seed":1,"tolerance_points":0.5,"rule":"half","batch_size":1,"threshold":0.9825243141806727,"held_out":1099,"routed":219,"simulated":{"accuracy":0.9044585987261147,"frontier_accuracy":0.905368516833485},"measured":{"pipeline_run":168,"alone_run":166,"accuracy":0.9335759781619655,"frontier_accuracy":0.9272065514103731,"difference":0.006369426751592357,"low":-0.002889942045888322,"high":0.015628795549073038,"p":0.24778856337070465,"within_tolerance":true,"decision_cost_usd":0.008533460000000001,"cost_usd":10.56694966,"frontier_cost_usd":12.8286324,"request_seconds":4184.495180073995,"frontier_request_seconds":4694.714676301097,"waits":{"pipeline":{"questions":1099,"median_seconds":3.924,"p90_seconds":6.1256,"mean_seconds":3.82051228389445,"max_seconds":34.648,"total_seconds":4198.743},"kept":{"questions":219,"median_seconds":0.231,"p90_seconds":0.2834,"mean_seconds":0.2428812785388128,"max_seconds":0.546,"total_seconds":53.191},"passed":{"questions":880,"median_seconds":4.334,"p90_seconds":6.609100000000001,"mean_seconds":4.710854545454545,"max_seconds":34.648,"total_seconds":4145.552},"alone":{"questions":1099,"median_seconds":3.77,"p90_seconds":6.1594,"mean_seconds":4.271797088262057,"max_seconds":22.185,"total_seconds":4694.705}},"pipeline_tokens":{"input_tokens":1373985,"cached_input_tokens":568721,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":803504,"output_tokens":200480},"alone_tokens":{"input_tokens":1746017,"cached_input_tokens":701622,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":1042197,"output_tokens":217097},"alternatives":[{"run":163,"accuracy":0.9754322111010009,"cost_usd":31.648706,"request_seconds":7964.2585057840915,"versus_pipeline":{"difference":0.04185623293903549,"low":0.02874163169875927,"high":0.05497083417931171,"p":1.170292474039769e-10},"versus_frontier":{"difference":0.048225659690627844,"low":0.03436017290668587,"high":0.06209114647456982,"p":1.6623508125590547e-12},"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":5103512,"cached_input_tokens":2509696,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":64017}},{"run":167,"accuracy":0.8935395814376706,"cost_usd":8.6273988,"request_seconds":6064.148409693618,"versus_pipeline":{"difference":-0.040036396724294813,"low":-0.05771300818223522,"high":-0.0223597852663544,"p":1.2579150016678854e-05},"versus_frontier":{"difference":-0.03366696997270246,"low":-0.051657121097316086,"high":-0.015676818848088834,"p":0.00034099780128697693},"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":1648174,"cached_input_tokens":599508,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":1046468,"output_tokens":437718}}]}},{"decision":145,"frontier":47,"seed":1,"tolerance_points":0.5,"rule":"share","batch_size":16,"threshold":0.9642039538871846,"held_out":1099,"routed":275,"simulated":{"accuracy":0.9044585987261147,"frontier_accuracy":0.905368516833485},"share":0.25,"measured":{"pipeline_run":171,"alone_run":165,"accuracy":0.8871701546860783,"frontier_accuracy":0.8935395814376706,"difference":-0.006369426751592357,"low":-0.0242888152823936,"high":0.011549961779208884,"p":0.5507089135982718,"within_tolerance":false,"decision_cost_usd":0.008533460000000001,"cost_usd":4.13014146,"frontier_cost_usd":5.387050599999999,"request_seconds":1111.0155438639922,"frontier_request_seconds":929.1633372180659,"pipeline_tokens":{"input_tokens":434443,"cached_input_tokens":113700,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":320639,"output_tokens":76667},"alone_tokens":{"input_tokens":628800,"cached_input_tokens":171673,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":456987,"output_tokens":84813},"alternatives":[{"run":71,"accuracy":0.9745222929936306,"cost_usd":5.67298,"request_seconds":992.3243447878631,"versus_pipeline":{"difference":0.08735213830755233,"low":0.06973177466928716,"high":0.10497250194581749,"p":2.6256041585124165e-24},"versus_frontier":{"difference":0.08098271155595996,"low":0.06281327556947469,"high":0.09915214754244522,"p":4.0853725805661456e-19},"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582668,"cached_input_tokens":144000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":22846}},{"run":137,"accuracy":0.9681528662420382,"cost_usd":1.3225,"request_seconds":1362.2349697217578,"versus_pipeline":{"difference":0.08098271155595996,"low":0.06298918240699025,"high":0.09897624070492966,"p":1.457836211253665e-19},"versus_frontier":{"difference":0.0746132848043676,"low":0.05591847354625709,"high":0.09330809606247811,"p":2.9566626383126704e-15},"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582670,"cached_input_tokens":57600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":26660}},{"run":138,"accuracy":0.910828025477707,"cost_usd":0.29709675,"request_seconds":300.7470318475971,"versus_pipeline":{"difference":0.023657870791628753,"low":0.0036877362927157445,"high":0.04362800529054176,"p":0.02554336402978297},"versus_frontier":{"difference":0.017288444040036398,"low":-0.0034080250492140966,"high":0.037984913129286896,"p":0.12101372565511773},"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"tokens":{"input_tokens":244424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30341}},{"run":68,"accuracy":0.8489535941765242,"cost_usd":4.1268408,"request_seconds":1799.8465952617116,"versus_pipeline":{"difference":-0.03821656050955414,"low":-0.059197185877242015,"high":-0.01723593514186626,"p":0.00048561896725776954},"versus_frontier":{"difference":-0.044585987261146494,"low":-0.06634214262491249,"high":-0.022829831897380495,"p":8.212852870228109e-05},"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":609112,"cached_input_tokens":148308,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":460666,"output_tokens":226907}}]}},{"decision":145,"frontier":47,"seed":1,"tolerance_points":0.5,"rule":"half","batch_size":16,"threshold":0.9825243141806727,"held_out":1099,"routed":219,"simulated":{"accuracy":0.9044585987261147,"frontier_accuracy":0.905368516833485},"measured":{"pipeline_run":164,"alone_run":165,"accuracy":0.8926296633303002,"frontier_accuracy":0.8935395814376706,"difference":-0.0009099181073703367,"low":-0.01847467766231635,"high":0.016654841447575675,"p":1.0,"within_tolerance":true,"decision_cost_usd":0.008533460000000001,"cost_usd":4.38446586,"frontier_cost_usd":5.387050599999999,"request_seconds":1131.248071975031,"frontier_request_seconds":929.1633372180659,"pipeline_tokens":{"input_tokens":478223,"cached_input_tokens":130962,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":347149,"output_tokens":78605},"alone_tokens":{"input_tokens":628800,"cached_input_tokens":171673,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":456987,"output_tokens":84813},"alternatives":[{"run":71,"accuracy":0.9745222929936306,"cost_usd":5.67298,"request_seconds":992.3243447878631,"versus_pipeline":{"difference":0.0818926296633303,"low":0.06454372636014774,"high":0.09924153296651286,"p":5.657185283317709e-22},"versus_frontier":{"difference":0.08098271155595996,"low":0.06281327556947469,"high":0.09915214754244522,"p":4.0853725805661456e-19},"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582668,"cached_input_tokens":144000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":22846}},{"run":137,"accuracy":0.9681528662420382,"cost_usd":1.3225,"request_seconds":1362.2349697217578,"versus_pipeline":{"difference":0.07552320291173795,"low":0.05780227700995354,"high":0.09324412881352236,"p":1.3869287779064768e-17},"versus_frontier":{"difference":0.0746132848043676,"low":0.05591847354625709,"high":0.09330809606247811,"p":2.9566626383126704e-15},"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582670,"cached_input_tokens":57600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":26660}},{"run":138,"accuracy":0.910828025477707,"cost_usd":0.29709675,"request_seconds":300.7470318475971,"versus_pipeline":{"difference":0.018198362147406732,"low":-0.001308588285279768,"high":0.03770531258009323,"p":0.08240747932752655},"versus_frontier":{"difference":0.017288444040036398,"low":-0.0034080250492140966,"high":0.037984913129286896,"p":0.12101372565511773},"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"tokens":{"input_tokens":244424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30341}},{"run":68,"accuracy":0.8489535941765242,"cost_usd":4.1268408,"request_seconds":1799.8465952617116,"versus_pipeline":{"difference":-0.04367606915377616,"low":-0.0649209846060858,"high":-0.022431153701466536,"p":7.799113896848694e-05},"versus_frontier":{"difference":-0.044585987261146494,"low":-0.06634214262491249,"high":-0.022829831897380495,"p":8.212852870228109e-05},"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":609112,"cached_input_tokens":148308,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":460666,"output_tokens":226907}}]}},{"decision":145,"frontier":55,"seed":1,"tolerance_points":0.5,"rule":"cutoff","batch_size":16,"threshold":0.99,"held_out":1099,"routed":176,"simulated":{"accuracy":0.8498635122838945,"frontier_accuracy":0.8462238398544131},"measured":{"pipeline_run":174,"alone_run":68,"accuracy":0.8571428571428571,"frontier_accuracy":0.8489535941765242,"difference":0.00818926296633303,"low":-0.009186810653492545,"high":0.025565336586158605,"p":0.41191007368097654,"within_tolerance":true,"decision_cost_usd":0.008533460000000001,"cost_usd":3.68969116,"frontier_cost_usd":4.1268408,"request_seconds":2001.7496087580075,"frontier_request_seconds":1799.8465952617116,"pipeline_tokens":{"input_tokens":490249,"cached_input_tokens":124317,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":365816,"output_tokens":220523},"alone_tokens":{"input_tokens":609112,"cached_input_tokens":148308,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":460666,"output_tokens":226907},"alternatives":[{"run":71,"accuracy":0.9745222929936306,"cost_usd":5.67298,"request_seconds":992.3243447878631,"versus_pipeline":{"difference":0.11737943585077343,"low":0.09737165678674037,"high":0.1373872149148065,"p":7.352026653297796e-33},"versus_frontier":{"difference":0.12556869881710647,"low":0.10502648569558187,"high":0.14611091193863107,"p":2.08925524731209e-35},"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582668,"cached_input_tokens":144000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":22846}},{"run":137,"accuracy":0.9681528662420382,"cost_usd":1.3225,"request_seconds":1362.2349697217578,"versus_pipeline":{"difference":0.11101000909918107,"low":0.09079672440902348,"high":0.13122329378933867,"p":2.5747366898777346e-28},"versus_frontier":{"difference":0.1191992720655141,"low":0.09875690893398828,"high":0.13964163519703993,"p":5.294448923596039e-32},"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582670,"cached_input_tokens":57600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":26660}},{"run":138,"accuracy":0.910828025477707,"cost_usd":0.29709675,"request_seconds":300.7470318475971,"versus_pipeline":{"difference":0.05368516833484986,"low":0.03099742234231355,"high":0.07637291432738617,"p":5.076173568329694e-06},"versus_frontier":{"difference":0.06187443130118289,"low":0.03877269078891368,"high":0.0849761718134521,"p":2.2849587651185474e-07},"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"tokens":{"input_tokens":244424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30341}},{"run":165,"accuracy":0.8935395814376706,"cost_usd":5.387050599999999,"request_seconds":929.1633372180659,"versus_pipeline":{"difference":0.036396724294813464,"low":0.01436968479581385,"high":0.058423763793813074,"p":0.0015876769917063935},"versus_frontier":{"difference":0.044585987261146494,"low":0.022829831897380495,"high":0.06634214262491249,"p":8.212852870228109e-05},"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":628800,"cached_input_tokens":171673,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":456987,"output_tokens":84813}}]}},{"decision":145,"frontier":55,"seed":1,"tolerance_points":0.5,"rule":"half","batch_size":1,"threshold":0.7171164205094459,"held_out":1099,"routed":558,"simulated":{"accuracy":0.8343949044585988,"frontier_accuracy":0.8462238398544131},"measured":{"pipeline_run":170,"alone_run":167,"accuracy":0.8462238398544131,"frontier_accuracy":0.8935395814376706,"difference":-0.047315741583257506,"low":-0.0647478551295116,"high":-0.029883628037003413,"p":1.3172242120654852e-07,"within_tolerance":false,"decision_cost_usd":0.008533460000000001,"cost_usd":5.21685236,"frontier_cost_usd":8.6273988,"request_seconds":3980.5925824250007,"frontier_request_seconds":6064.148409693618,"waits":{"pipeline":{"questions":1099,"median_seconds":0.4,"p90_seconds":8.8948,"mean_seconds":3.634992720655141,"max_seconds":32.635999999999996,"total_seconds":3994.857},"kept":{"questions":558,"median_seconds":0.232,"p90_seconds":0.28300000000000003,"mean_seconds":0.29999283154121864,"max_seconds":31.381,"total_seconds":167.396},"passed":{"questions":541,"median_seconds":5.985,"p90_seconds":11.602,"mean_seconds":7.074789279112753,"max_seconds":32.635999999999996,"total_seconds":3827.461},"alone":{"questions":1099,"median_seconds":4.774,"p90_seconds":8.731800000000002,"mean_seconds":5.517877161055505,"max_seconds":21.935,"total_seconds":6064.147}},"pipeline_tokens":{"input_tokens":785837,"cached_input_tokens":298909,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":485846,"output_tokens":323288},"alone_tokens":{"input_tokens":1648174,"cached_input_tokens":599508,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":1046468,"output_tokens":437718},"alternatives":[{"run":163,"accuracy":0.9754322111010009,"cost_usd":31.648706,"request_seconds":7964.2585057840915,"versus_pipeline":{"difference":0.1292083712465878,"low":0.10828410243890053,"high":0.15013264005427504,"p":8.935182693751113e-36},"versus_frontier":{"difference":0.0818926296633303,"low":0.06436135022755202,"high":0.09942390909910859,"p":2.253920747846914e-21},"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":5103512,"cached_input_tokens":2509696,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":64017}},{"run":166,"accuracy":0.9272065514103731,"cost_usd":12.8286324,"request_seconds":4694.714676301097,"versus_pipeline":{"difference":0.08098271155595996,"low":0.05989641185577714,"high":0.10206901125614277,"p":6.155096365873754e-14},"versus_frontier":{"difference":0.03366696997270246,"low":0.015676818848088834,"high":0.051657121097316086,"p":0.00034099780128697693},"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":1746017,"cached_input_tokens":701622,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":1042197,"output_tokens":217097}}]}},{"decision":145,"frontier":55,"seed":1,"tolerance_points":0.5,"rule":"share","batch_size":16,"threshold":0.9642039538871846,"held_out":1099,"routed":275,"simulated":{"accuracy":0.8553230209281165,"frontier_accuracy":0.8462238398544131},"share":0.25,"measured":{"pipeline_run":173,"alone_run":68,"accuracy":0.8607825295723385,"frontier_accuracy":0.8489535941765242,"difference":0.011828935395814377,"low":-0.006777580673092972,"high":0.030435451464721725,"p":0.25028200469192174,"within_tolerance":true,"decision_cost_usd":0.008533460000000001,"cost_usd":3.37300646,"frontier_cost_usd":4.1268408,"request_seconds":1886.570894726974,"frontier_request_seconds":1799.8465952617116,"pipeline_tokens":{"input_tokens":429829,"cached_input_tokens":109050,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":320675,"output_tokens":207066},"alone_tokens":{"input_tokens":609112,"cached_input_tokens":148308,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":460666,"output_tokens":226907},"alternatives":[{"run":71,"accuracy":0.9745222929936306,"cost_usd":5.67298,"request_seconds":992.3243447878631,"versus_pipeline":{"difference":0.11373976342129208,"low":0.09381765627121447,"high":0.1336618705713697,"p":5.166045778807018e-31},"versus_frontier":{"difference":0.12556869881710647,"low":0.10502648569558187,"high":0.14611091193863107,"p":2.08925524731209e-35},"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582668,"cached_input_tokens":144000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":22846}},{"run":137,"accuracy":0.9681528662420382,"cost_usd":1.3225,"request_seconds":1362.2349697217578,"versus_pipeline":{"difference":0.10737033666969972,"low":0.08724587552101651,"high":0.12749479781838294,"p":1.0618810807448801e-26},"versus_frontier":{"difference":0.1191992720655141,"low":0.09875690893398828,"high":0.13964163519703993,"p":5.294448923596039e-32},"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582670,"cached_input_tokens":57600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":26660}},{"run":138,"accuracy":0.910828025477707,"cost_usd":0.29709675,"request_seconds":300.7470318475971,"versus_pipeline":{"difference":0.05004549590536852,"low":0.02718909948193704,"high":0.0729018923288,"p":2.505115608706493e-05},"versus_frontier":{"difference":0.06187443130118289,"low":0.03877269078891368,"high":0.0849761718134521,"p":2.2849587651185474e-07},"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"tokens":{"input_tokens":244424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30341}},{"run":165,"accuracy":0.8935395814376706,"cost_usd":5.387050599999999,"request_seconds":929.1633372180659,"versus_pipeline":{"difference":0.03275705186533212,"low":0.011443588455153298,"high":0.05407051527551095,"p":0.003393884063573226},"versus_frontier":{"difference":0.044585987261146494,"low":0.022829831897380495,"high":0.06634214262491249,"p":8.212852870228109e-05},"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":628800,"cached_input_tokens":171673,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":456987,"output_tokens":84813}}]}},{"decision":145,"frontier":55,"seed":1,"tolerance_points":0.5,"rule":"half","batch_size":16,"threshold":0.7171164205094459,"held_out":1099,"routed":558,"simulated":{"accuracy":0.8343949044585988,"frontier_accuracy":0.8462238398544131},"measured":{"pipeline_run":162,"alone_run":68,"accuracy":0.8425841674249318,"frontier_accuracy":0.8489535941765242,"difference":-0.006369426751592357,"low":-0.028281462396915237,"high":0.015542608893730525,"p":0.6255036238897121,"within_tolerance":false,"decision_cost_usd":0.008533460000000001,"cost_usd":2.5804246600000003,"frontier_cost_usd":4.1268408,"request_seconds":1677.1019263799535,"frontier_request_seconds":1799.8465952617116,"pipeline_tokens":{"input_tokens":274389,"cached_input_tokens":69792,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":204529,"output_tokens":174666},"alone_tokens":{"input_tokens":609112,"cached_input_tokens":148308,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":460666,"output_tokens":226907},"alternatives":[{"run":71,"accuracy":0.9745222929936306,"cost_usd":5.67298,"request_seconds":992.3243447878631,"versus_pipeline":{"difference":0.13193812556869883,"low":0.11069579859223826,"high":0.15318045254515938,"p":6.769507367528149e-36},"versus_frontier":{"difference":0.12556869881710647,"low":0.10502648569558187,"high":0.14611091193863107,"p":2.08925524731209e-35},"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582668,"cached_input_tokens":144000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":22846}},{"run":137,"accuracy":0.9681528662420382,"cost_usd":1.3225,"request_seconds":1362.2349697217578,"versus_pipeline":{"difference":0.12556869881710647,"low":0.10426637302645733,"high":0.1468710246077556,"p":4.576636824126176e-32},"versus_frontier":{"difference":0.1191992720655141,"low":0.09875690893398828,"high":0.13964163519703993,"p":5.294448923596039e-32},"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"tokens":{"input_tokens":582670,"cached_input_tokens":57600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":26660}},{"run":138,"accuracy":0.910828025477707,"cost_usd":0.29709675,"request_seconds":300.7470318475971,"versus_pipeline":{"difference":0.06824385805277525,"low":0.0448623126424854,"high":0.0916254034630651,"p":1.6420148695755048e-08},"versus_frontier":{"difference":0.06187443130118289,"low":0.03877269078891368,"high":0.0849761718134521,"p":2.2849587651185474e-07},"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"tokens":{"input_tokens":244424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30341}},{"run":165,"accuracy":0.8935395814376706,"cost_usd":5.387050599999999,"request_seconds":929.1633372180659,"versus_pipeline":{"difference":0.050955414012738856,"low":0.027760654036543823,"high":0.07415017398893389,"p":2.350182078373937e-05},"versus_frontier":{"difference":0.044585987261146494,"low":0.022829831897380495,"high":0.06634214262491249,"p":8.212852870228109e-05},"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"tokens":{"input_tokens":628800,"cached_input_tokens":171673,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":456987,"output_tokens":84813}}]}}],"repeats":[{"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"high","provider":"openai","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[281,292,295,299,302],"questions":200,"accuracy":[0.985,0.985,0.98,0.98,0.98],"changed_answer":1,"wrong_any":4,"wrong_every_run_same":3,"scored":200,"score_spread_median":0.010000000000000009,"score_identical":88,"sure_wrong":[1,0,1,1,0],"sure_wrong_every_run":0,"latency":{"runs":[281,292,295,299,302],"concurrency":1,"requests":65,"median_request_seconds":25.495603712974116,"p90_request_seconds":60.84645333419903}},{"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"high","provider":"openai","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[282,297,300,301,303],"questions":200,"accuracy":[0.985,0.98,0.975,0.98,0.985],"changed_answer":3,"wrong_any":5,"wrong_every_run_same":2,"scored":200,"score_spread_median":0.0,"score_identical":105,"sure_wrong":[2,0,1,0,0],"sure_wrong_every_run":0,"latency":{"runs":[282,297,300,301,303],"concurrency":1,"requests":65,"median_request_seconds":23.566122614982305,"p90_request_seconds":42.08493875039858}},{"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[69,78,82,87,91],"questions":200,"accuracy":[0.975,0.98,0.98,0.98,0.98],"changed_answer":1,"wrong_any":5,"wrong_every_run_same":4,"scored":200,"score_spread_median":0.0,"score_identical":105,"sure_wrong":[1,0,1,1,1],"sure_wrong_every_run":0,"latency":{"runs":[69,78,82,87,91],"concurrency":1,"requests":65,"median_request_seconds":13.043534992961213,"p90_request_seconds":16.78220434060786}},{"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[283,289,294,296,298],"questions":200,"accuracy":[0.985,0.975,0.975,0.97,0.98],"changed_answer":5,"wrong_any":8,"wrong_every_run_same":3,"scored":200,"score_spread_median":0.050000000000000044,"score_identical":15,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[283,289,294,296,298],"concurrency":1,"requests":65,"median_request_seconds":19.52282273999299,"p90_request_seconds":35.27809036319377}},{"model":"gpt-6.1-terra","name":"GPT-6.1 Terra","variant":null,"reasoning":"high","provider":"openai","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[280,285,288,291,293],"questions":200,"accuracy":[0.965,0.97,0.95,0.965,0.955],"changed_answer":11,"wrong_any":16,"wrong_every_run_same":5,"scored":200,"score_spread_median":0.010000000000000009,"score_identical":83,"sure_wrong":[1,0,2,1,1],"sure_wrong_every_run":0,"latency":{"runs":[280,285,288,291,293],"concurrency":1,"requests":65,"median_request_seconds":8.852608644025167,"p90_request_seconds":14.96516182362102}},{"model":"deepseek-v4.1-flash-1017","name":"DeepSeek-V4.1-Flash 1017","variant":null,"reasoning":"on","provider":"vllm","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[606,607,608,609,611],"questions":200,"accuracy":[0.965,0.945,0.97,0.95,0.96],"changed_answer":15,"wrong_any":16,"wrong_every_run_same":1,"scored":200,"score_spread_median":1.3869512834663311e-05,"score_identical":0,"sure_wrong":[7,11,6,10,8],"sure_wrong_every_run":2,"latency":{"runs":[606,607,608,609,611],"concurrency":8,"requests":65,"median_request_seconds":360.5134032180067,"p90_request_seconds":642.3510239871684}},{"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[67,77,81,86,90],"questions":200,"accuracy":[0.94,0.95,0.95,0.96,0.975],"changed_answer":20,"wrong_any":22,"wrong_every_run_same":2,"scored":200,"score_spread_median":0.050000000000000044,"score_identical":23,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[67,77,81,86,90],"concurrency":1,"requests":65,"median_request_seconds":13.402853598003276,"p90_request_seconds":19.79555194299901}},{"model":"gpt-6.1-terra","name":"GPT-6.1 Terra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[279,284,286,287,290],"questions":200,"accuracy":[0.93,0.96,0.96,0.955,0.96],"changed_answer":14,"wrong_any":18,"wrong_every_run_same":4,"scored":200,"score_spread_median":0.010000000000000009,"score_identical":87,"sure_wrong":[2,0,1,1,1],"sure_wrong_every_run":0,"latency":{"runs":[279,284,286,287,290],"concurrency":1,"requests":65,"median_request_seconds":5.419412001996534,"p90_request_seconds":6.897614485403755}},{"model":"deepseek-v4.1-flash","name":"DeepSeek-V4.1-Flash, first release","variant":null,"reasoning":"on","provider":"vllm","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[114,115,116,117,118],"questions":200,"accuracy":[0.95,0.97,0.94,0.935,0.95],"changed_answer":15,"wrong_any":20,"wrong_every_run_same":5,"scored":0,"score_spread_median":null,"score_identical":0,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[114,115,116,117,118],"concurrency":8,"requests":65,"median_request_seconds":274.6319791179849,"p90_request_seconds":443.7561121556093}},{"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[63,80,88,93,98],"questions":200,"accuracy":[0.895,0.91,0.895,0.87,0.87],"changed_answer":31,"wrong_any":40,"wrong_every_run_same":9,"scored":200,"score_spread_median":0.09999999999999998,"score_identical":14,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[63,80,88,93,98],"concurrency":1,"requests":65,"median_request_seconds":20.488734460028354,"p90_request_seconds":37.048832100233994}},{"model":"gemma-4-31B-it","name":"Gemma 4 31B","variant":null,"reasoning":"off","provider":"vllm","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[61,72,73,76,79],"questions":200,"accuracy":[0.705,0.705,0.705,0.705,0.705],"changed_answer":0,"wrong_any":59,"wrong_every_run_same":59,"scored":0,"score_spread_median":null,"score_identical":0,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[61,72,73,76,79],"concurrency":1,"requests":65,"median_request_seconds":3.5819392279954627,"p90_request_seconds":3.9002420175937003}},{"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[229,230,231,232,233],"questions":200,"accuracy":[0.7,0.655,0.68,0.68,0.685],"changed_answer":72,"wrong_any":92,"wrong_every_run_same":20,"scored":200,"score_spread_median":0.09999999999999998,"score_identical":40,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[229,230,231,232,233],"concurrency":1,"requests":65,"median_request_seconds":3.873064635001356,"p90_request_seconds":5.84579606440384}},{"model":"deepseek-v4.1-flash","name":"DeepSeek-V4.1-Flash, first release","variant":null,"reasoning":"off","provider":"vllm","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[109,110,111,112,113],"questions":200,"accuracy":[0.645,0.64,0.645,0.64,0.645],"changed_answer":12,"wrong_any":75,"wrong_every_run_same":63,"scored":0,"score_spread_median":null,"score_identical":0,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[109,110,111,112,113],"concurrency":1,"requests":65,"median_request_seconds":0.5750969999935478,"p90_request_seconds":5.717052210203837}},{"model":"deepseek-v4.1-flash-1017","name":"DeepSeek-V4.1-Flash 1017","variant":null,"reasoning":"off","provider":"vllm","perplexity_mode":null,"questions_per_request":16,"whole_bank":false,"runs":[601,602,603,604,605],"questions":200,"accuracy":[0.65,0.63,0.635,0.625,0.635],"changed_answer":10,"wrong_any":78,"wrong_every_run_same":68,"scored":200,"score_spread_median":0.020494121816564292,"score_identical":0,"sure_wrong":[2,1,2,1,1],"sure_wrong_every_run":0,"latency":{"runs":[601,602,603,604,605],"concurrency":1,"requests":65,"median_request_seconds":0.7039570200140588,"p90_request_seconds":2.666380101174582}},{"model":"d1-ultra","name":"Liquid d1-ultra","variant":null,"reasoning":null,"provider":"liquid","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[199,200,201,202,203],"questions":2198,"accuracy":[0.7707006369426752,0.7670609645131938,0.759326660600546,0.7666060054595086,0.7597816196542311],"changed_answer":375,"wrong_any":669,"wrong_every_run_same":294,"scored":2198,"score_spread_median":0.07732904799822926,"score_identical":0,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[199,200,201,202,203],"concurrency":1,"requests":10990,"median_request_seconds":0.22276320450328058,"p90_request_seconds":0.3530015264128451}},{"model":"glide-v1.2","name":"Fastino GLiDE v1.2","variant":"local","reasoning":null,"provider":"glide","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[248,250,251,252,253],"questions":2198,"accuracy":[0.6847133757961783,0.6847133757961783,0.6847133757961783,0.6847133757961783,0.6847133757961783],"changed_answer":0,"wrong_any":693,"wrong_every_run_same":693,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[248,250,251,252,253],"concurrency":1,"requests":10990,"median_request_seconds":0.4119799835025333,"p90_request_seconds":0.5165845516894478}},{"model":"d1.2-ultra","name":"Liquid d1.2-ultra","variant":null,"reasoning":null,"provider":"liquid","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[1347,1353,1368,1371,1373],"questions":2198,"accuracy":[0.681073703366697,0.681073703366697,0.681073703366697,0.681073703366697,0.681073703366697],"changed_answer":1,"wrong_any":701,"wrong_every_run_same":700,"scored":2198,"score_spread_median":0.0,"score_identical":2176,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[1347,1353,1368,1371,1373],"concurrency":1,"requests":10990,"median_request_seconds":0.2142794329993194,"p90_request_seconds":0.3174185156000021}},{"model":"vllm-sr/d3","name":"vLLM-SR d3","variant":null,"reasoning":null,"provider":"vllm-sr","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[259,260,261,262,263],"questions":2198,"accuracy":[0.6778889899909009,0.6778889899909009,0.6778889899909009,0.6778889899909009,0.6778889899909009],"changed_answer":0,"wrong_any":708,"wrong_every_run_same":708,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[259,260,261,262,263],"concurrency":1,"requests":10990,"median_request_seconds":0.08950294999522157,"p90_request_seconds":0.13542871531099082}},{"model":"vllm-sr/d3-ultra","name":"vLLM-SR d3-ultra","variant":"transformers, rev 52ba9e7","reasoning":null,"provider":"d3-server","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[269,270,271,272,273],"questions":2198,"accuracy":[0.6778889899909009,0.6778889899909009,0.6778889899909009,0.6778889899909009,0.6778889899909009],"changed_answer":0,"wrong_any":708,"wrong_every_run_same":708,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[1,1,1,1,1],"sure_wrong_every_run":1,"latency":{"runs":[269,270,271,272,273],"concurrency":1,"requests":10990,"median_request_seconds":0.27599614550126716,"p90_request_seconds":0.43350873780436816}},{"model":"drex-max-v1","name":"Drex Max v1","variant":null,"reasoning":null,"provider":"drex-max","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[322,323,324,327,328],"questions":2198,"accuracy":[0.6774340309372157,0.6774340309372157,0.6774340309372157,0.6774340309372157,0.6774340309372157],"changed_answer":0,"wrong_any":709,"wrong_every_run_same":709,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[1,1,1,1,1],"sure_wrong_every_run":1,"latency":{"runs":[322,323,324,327,328],"concurrency":1,"requests":10990,"median_request_seconds":0.0814295545060304,"p90_request_seconds":0.10906726221001009}},{"model":"drex-max-v1.1","name":"Drex Max 1.1","variant":null,"reasoning":null,"provider":"nace","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[326,329,330,331,332],"questions":2198,"accuracy":[0.6737943585077343,0.6737943585077343,0.6737943585077343,0.6737943585077343,0.6737943585077343],"changed_answer":0,"wrong_any":717,"wrong_every_run_same":717,"scored":2198,"score_spread_median":0.0,"score_identical":2194,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[326,329,330,331,332],"concurrency":1,"requests":10990,"median_request_seconds":0.3142536500017741,"p90_request_seconds":0.4515718497015769}},{"model":"baselabs/mammoth-1","name":"Mammoth-1","variant":null,"reasoning":null,"provider":"baselabs","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[374,377,380,383,384],"questions":2198,"accuracy":[0.6710646041856233,0.6710646041856233,0.6710646041856233,0.6710646041856233,0.6710646041856233],"changed_answer":0,"wrong_any":723,"wrong_every_run_same":723,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[374,377,380,383,384],"concurrency":1,"requests":10990,"median_request_seconds":0.16727188749064226,"p90_request_seconds":0.1937068673025351}},{"model":"pplx-decider-v1.2","name":"Perplexity Decider 27B v1.2 (local GPU)","variant":"local","reasoning":null,"provider":"pplx-decider","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[217,218,219,220,221],"questions":2198,"accuracy":[0.6706096451319381,0.6706096451319381,0.6706096451319381,0.6706096451319381,0.6706096451319381],"changed_answer":0,"wrong_any":724,"wrong_every_run_same":724,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[217,218,219,220,221],"concurrency":1,"requests":10990,"median_request_seconds":0.5787490529983188,"p90_request_seconds":0.7847130408059456}},{"model":"pplx-decider-v1.1-27b","name":"Perplexity Decider 27B v1.1","variant":null,"reasoning":null,"provider":"perplexity","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[145,146,147,148,149],"questions":2198,"accuracy":[0.6687898089171974,0.6687898089171974,0.6687898089171974,0.6687898089171974,0.6687898089171974],"changed_answer":0,"wrong_any":728,"wrong_every_run_same":728,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[145,146,147,148,149],"concurrency":1,"requests":10990,"median_request_seconds":0.230697883060202,"p90_request_seconds":0.28727554152719675}},{"model":"subq-snap-preview","name":"SubQ Snap Preview","variant":null,"reasoning":null,"provider":"subq","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[177,178,179,180,181],"questions":2198,"accuracy":[0.6524112829845314,0.6519563239308462,0.6524112829845314,0.651501364877161,0.6528662420382165],"changed_answer":9,"wrong_any":768,"wrong_every_run_same":759,"scored":2198,"score_spread_median":0.0,"score_identical":2069,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[177,178,179,180,181],"concurrency":1,"requests":10990,"median_request_seconds":0.3907852455158718,"p90_request_seconds":0.44049788439588156}},{"model":"subq-snap-1.1","name":"SubQ Snap 1.1","variant":null,"reasoning":null,"provider":"subq","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[346,387,388,389,390],"questions":2198,"accuracy":[0.6387625113739763,0.6387625113739763,0.6387625113739763,0.6387625113739763,0.6387625113739763],"changed_answer":0,"wrong_any":794,"wrong_every_run_same":794,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[1,1,1,1,1],"sure_wrong_every_run":1,"latency":{"runs":[346,387,388,389,390],"concurrency":1,"requests":10990,"median_request_seconds":0.37816790348733775,"p90_request_seconds":0.3870654027094133}},{"model":"d1.1","name":"Liquid d1.1","variant":null,"reasoning":null,"provider":"liquid","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[254,255,256,257,258],"questions":2198,"accuracy":[0.6151046405823476,0.6141947224749773,0.6205641492265697,0.6128298453139217,0.6223839854413102],"changed_answer":659,"wrong_any":1071,"wrong_every_run_same":412,"scored":2198,"score_spread_median":0.10694811785745692,"score_identical":0,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[254,255,256,257,258],"concurrency":1,"requests":10990,"median_request_seconds":0.19625577199963118,"p90_request_seconds":0.2373385658997904}},{"model":"d1.2","name":"Liquid d1.2","variant":null,"reasoning":null,"provider":"liquid","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[1346,1352,1369,1370,1372],"questions":2198,"accuracy":[0.6173794358507735,0.6087352138307552,0.6169244767970883,0.6132848043676069,0.6064604185623294],"changed_answer":706,"wrong_any":1091,"wrong_every_run_same":385,"scored":2198,"score_spread_median":0.10732118624082554,"score_identical":0,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[1346,1352,1369,1370,1372],"concurrency":1,"requests":10990,"median_request_seconds":0.20885935650039755,"p90_request_seconds":0.30600086439881125}},{"model":"pplx-decider-v1-27b","name":"Perplexity Decider 27B v1","variant":null,"reasoning":null,"provider":"perplexity","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[44,64,84,96,100],"questions":2198,"accuracy":[0.6037306642402184,0.6037306642402184,0.6037306642402184,0.6037306642402184,0.6041856232939036],"changed_answer":2,"wrong_any":871,"wrong_every_run_same":869,"scored":2198,"score_spread_median":0.0,"score_identical":2169,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[64,84,96,100],"concurrency":1,"requests":8792,"median_request_seconds":0.21641260248725303,"p90_request_seconds":0.2719852286914829}},{"model":"celeris-1-decision","name":"Celeris-1 Decision","variant":null,"reasoning":null,"provider":"celeris","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[150,151,156,157,158],"questions":2198,"accuracy":[0.6046405823475887,0.6028207461328481,0.6010009099181074,0.600090991810737,0.5982711555959963],"changed_answer":206,"wrong_any":944,"wrong_every_run_same":738,"scored":2198,"score_spread_median":0.030000000000000027,"score_identical":192,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[150,151,156,157,158],"concurrency":1,"requests":10990,"median_request_seconds":0.40173603700168314,"p90_request_seconds":0.41774079340102616}},{"model":"OmniJev/OneJev-27B","name":"OneJev 27B","variant":null,"reasoning":null,"provider":"onejev","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[264,265,266,267,268],"questions":2198,"accuracy":[0.5959963603275705,0.5959963603275705,0.5959963603275705,0.5959963603275705,0.5959963603275705],"changed_answer":0,"wrong_any":888,"wrong_every_run_same":888,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[264,265,266,267,268],"concurrency":1,"requests":10990,"median_request_seconds":0.1524845355161233,"p90_request_seconds":0.2812980904011056}},{"model":"inception/mercury-decide","name":"Inception Mercury Decide","variant":"via OpenRouter","reasoning":null,"provider":"openrouter","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[204,210,212,214,216],"questions":2198,"accuracy":[0.5828025477707006,0.5905368516833485,0.5859872611464968,0.5887170154686078,0.5878070973612375],"changed_answer":191,"wrong_any":969,"wrong_every_run_same":778,"scored":2198,"score_spread_median":0.029266423876022768,"score_identical":7,"sure_wrong":[13,14,11,13,13],"sure_wrong_every_run":11,"latency":{"runs":[204,210,212,214,216],"concurrency":1,"requests":10990,"median_request_seconds":0.30808473999786656,"p90_request_seconds":0.39868138850142715}},{"model":"jev-1.14.0","name":"Jev 1.14.0","variant":null,"reasoning":null,"provider":"jev","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[194,195,196,197,198],"questions":2198,"accuracy":[0.591901728844404,0.5818926296633303,0.5882620564149227,0.578252957233849,0.5828025477707006],"changed_answer":291,"wrong_any":1010,"wrong_every_run_same":719,"scored":2198,"score_spread_median":0.04999999999999999,"score_identical":227,"sure_wrong":[1,1,1,1,0],"sure_wrong_every_run":0,"latency":{"runs":[194,195,196,197,198],"concurrency":1,"requests":10990,"median_request_seconds":0.2106848010007525,"p90_request_seconds":0.2690009778045351}},{"model":"jev-1.13.0","name":"Jev 1.13.0","variant":null,"reasoning":null,"provider":"jev","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[49,65,85,97,101],"questions":2198,"accuracy":[0.5750682438580528,0.5714285714285714,0.5759781619654231,0.5727934485896269,0.5800727934485896],"changed_answer":314,"wrong_any":1035,"wrong_every_run_same":721,"scored":2198,"score_spread_median":0.04999999999999999,"score_identical":271,"sure_wrong":[1,0,0,1,0],"sure_wrong_every_run":0,"latency":{"runs":[65,85,97,101],"concurrency":1,"requests":8792,"median_request_seconds":0.24339387452346273,"p90_request_seconds":0.2807294782833196}},{"model":"d1:free","name":"Liquid d1","variant":null,"reasoning":null,"provider":"liquid","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[52,66,94,105,108],"questions":2198,"accuracy":[0.5700636942675159,0.5691537761601456,0.5677888989990901,0.5686988171064604,0.5673339399454049],"changed_answer":217,"wrong_any":1016,"wrong_every_run_same":799,"scored":2198,"score_spread_median":0.008535373288807185,"score_identical":824,"sure_wrong":[1,1,1,2,1],"sure_wrong_every_run":1,"latency":{"runs":[66,94,105,108],"concurrency":1,"requests":8792,"median_request_seconds":0.3054448555049021,"p90_request_seconds":0.78636241707718}},{"model":"drex-v2.0","name":"Drex 2.0","variant":null,"reasoning":null,"provider":"nace","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[334,335,336,337,338],"questions":2198,"accuracy":[0.5573248407643312,0.5582347588717016,0.5586897179253867,0.5577797998180164,0.5582347588717016],"changed_answer":19,"wrong_any":978,"wrong_every_run_same":959,"scored":2198,"score_spread_median":0.0,"score_identical":1887,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[334,335,336,337,338],"concurrency":1,"requests":10990,"median_request_seconds":0.3239434089991846,"p90_request_seconds":0.4030103706943919}},{"model":"open-jev-27b-v1.1","name":"Open-Jev 27B v1.1","variant":null,"reasoning":null,"provider":"open-jev","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[45,60,92,102,106],"questions":2198,"accuracy":[0.556869881710646,0.556869881710646,0.556869881710646,0.556869881710646,0.556869881710646],"changed_answer":0,"wrong_any":974,"wrong_every_run_same":974,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[60,92,102,106],"concurrency":1,"requests":8792,"median_request_seconds":0.386393806518754,"p90_request_seconds":0.5727336546988226}},{"model":"qwen38-27b-bf16","name":"Qwen3.8-27B","variant":"stated confidence","reasoning":"off","provider":"vllm-verbal","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[53,74,95,103,107],"questions":2198,"accuracy":[0.5418562329390355,0.5427661510464058,0.5427661510464058,0.5427661510464058,0.5427661510464058],"changed_answer":44,"wrong_any":1019,"wrong_every_run_same":975,"scored":2198,"score_spread_median":0.0,"score_identical":2193,"sure_wrong":[1,2,2,2,2],"sure_wrong_every_run":1,"latency":{"runs":[74,95,103,107],"concurrency":1,"requests":8792,"median_request_seconds":0.43298505601705983,"p90_request_seconds":0.4745744730113074}},{"model":"baselabs/elephant-1","name":"Elephant-1","variant":null,"reasoning":null,"provider":"baselabs","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[373,375,376,378,379],"questions":2198,"accuracy":[0.5236578707916287,0.5236578707916287,0.5236578707916287,0.5236578707916287,0.5236578707916287],"changed_answer":0,"wrong_any":1047,"wrong_every_run_same":1047,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[373,375,376,378,379],"concurrency":1,"requests":10990,"median_request_seconds":0.08010628948977683,"p90_request_seconds":0.09728465078223962}},{"model":"qwen38-27b-bf16","name":"Qwen3.8-27B","variant":"yes/no scoring","reasoning":null,"provider":"vllm-yes-no","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[50,75,89,99,104],"questions":2198,"accuracy":[0.5191082802547771,0.5168334849863512,0.5163785259326661,0.5141037306642402,0.5159235668789809],"changed_answer":160,"wrong_any":1109,"wrong_every_run_same":949,"scored":2198,"score_spread_median":0.016477080928385346,"score_identical":0,"sure_wrong":[2,2,2,2,2],"sure_wrong_every_run":2,"latency":{"runs":[75,89,99,104],"concurrency":1,"requests":8792,"median_request_seconds":0.2798689884948544,"p90_request_seconds":0.40955076351528985}},{"model":"pplx-decider-flash-v1.2","name":"Perplexity Decider Flash v1.2","variant":"local","reasoning":null,"provider":"pplx-decider","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[223,224,225,226,227],"questions":2198,"accuracy":[0.47770700636942676,0.47770700636942676,0.47770700636942676,0.47770700636942676,0.47770700636942676],"changed_answer":0,"wrong_any":1148,"wrong_every_run_same":1148,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[223,224,225,226,227],"concurrency":1,"requests":10990,"median_request_seconds":0.13157959599629976,"p90_request_seconds":0.1874813156988239}},{"model":"microsoft/microsoft-decision-1","name":"Microsoft-Decision-1","variant":"via OpenRouter","reasoning":null,"provider":"openrouter","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[205,209,211,213,215],"questions":2198,"accuracy":[0.46451319381255685,0.4672429481346679,0.4640582347588717,0.46360327570518656,0.4626933575978162],"changed_answer":279,"wrong_any":1256,"wrong_every_run_same":977,"scored":2198,"score_spread_median":0.0537542191919925,"score_identical":2,"sure_wrong":[3,3,3,2,2],"sure_wrong_every_run":2,"latency":{"runs":[205,209,211,213,215],"concurrency":1,"requests":10990,"median_request_seconds":0.28970416450465564,"p90_request_seconds":0.33072930260095745}},{"model":"LiquidAI/d1-2B","name":"Liquid d1-2B (open weights)","variant":null,"reasoning":null,"provider":"d1-server","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[340,341,342,343,344],"questions":2198,"accuracy":[0.29390354868061874,0.29390354868061874,0.29390354868061874,0.29390354868061874,0.29390354868061874],"changed_answer":0,"wrong_any":1552,"wrong_every_run_same":1552,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[340,341,342,343,344],"concurrency":1,"requests":10990,"median_request_seconds":0.015482019996852614,"p90_request_seconds":0.02136149551370181}},{"model":"decider-1","name":"meraGPT Decider 1","variant":null,"reasoning":null,"provider":"meragpt","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[243,244,245,246,247],"questions":2198,"accuracy":[0.27115559599636035,0.27115559599636035,0.27115559599636035,0.27115559599636035,0.27115559599636035],"changed_answer":0,"wrong_any":1602,"wrong_every_run_same":1602,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[243,244,245,246,247],"concurrency":1,"requests":10990,"median_request_seconds":0.3855947514966829,"p90_request_seconds":0.4280789560987614}},{"model":"laya","name":"Laya","variant":null,"reasoning":null,"provider":"laya","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[51,234,235,236,237],"questions":2198,"accuracy":[0.19017288444040037,0.19017288444040037,0.19017288444040037,0.19017288444040037,0.19017288444040037],"changed_answer":0,"wrong_any":1780,"wrong_every_run_same":1780,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[0,0,0,0,0],"sure_wrong_every_run":0,"latency":{"runs":[234,235,236,237],"concurrency":1,"requests":8792,"median_request_seconds":0.011380975003703497,"p90_request_seconds":0.012440272996900602}},{"model":"laya-multilingual","name":"Laya multilingual","variant":null,"reasoning":null,"provider":"laya","perplexity_mode":null,"questions_per_request":1,"whole_bank":true,"runs":[46,238,239,240,241],"questions":2198,"accuracy":[0.17879890809827115,0.17879890809827115,0.17879890809827115,0.17879890809827115,0.17879890809827115],"changed_answer":0,"wrong_any":1805,"wrong_every_run_same":1805,"scored":2198,"score_spread_median":0.0,"score_identical":2198,"sure_wrong":[1,1,1,1,1],"sure_wrong_every_run":1,"latency":{"runs":[238,239,240,241],"concurrency":1,"requests":8792,"median_request_seconds":0.009499495485215448,"p90_request_seconds":0.010441777898813598}}],"replay":{"decision":145,"questions":2198,"cutoffs":[{"cutoff":0.999,"kept":90,"wrong":0},{"cutoff":0.995,"kept":268,"wrong":0},{"cutoff":0.99,"kept":349,"wrong":0},{"cutoff":0.985,"kept":402,"wrong":0},{"cutoff":0.98,"kept":443,"wrong":3},{"cutoff":0.97,"kept":505,"wrong":4},{"cutoff":0.95,"kept":589,"wrong":6},{"cutoff":0.9,"kept":767,"wrong":25},{"cutoff":0.8,"kept":970,"wrong":62}],"share":{"share":0.25,"cut":0.9605055053763134,"kept":550,"wrong":4,"interval":[0.9814508624779456,0.9971682950573496]},"frontiers":[{"frontier":47,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":4.155468782438992},"share":{"kept":550,"first_only":1,"second_only":3,"difference":-0.0009099181073703367,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.0011096162611619414,"worse":0.72,"better":0.043333333333333335},"usd_per_1000":3.704582622046368}},{"frontier":55,"sure":{"kept":349,"first_only":6,"second_only":0,"difference":0.00272975432211101,"usd_per_1000":3.0818606839002136},"share":{"kept":550,"first_only":14,"second_only":1,"difference":0.005914467697907188,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":0.006266133896030463,"worse":0.0,"better":1.0},"usd_per_1000":2.747683678621865}},{"frontier":48,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":4.532032413918041},"share":{"kept":550,"first_only":1,"second_only":4,"difference":-0.001364877161055505,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.0015703993274813867,"worse":0.86,"better":0.02},"usd_per_1000":4.040210996669123}},{"frontier":124,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":0.9586563784431377},"share":{"kept":550,"first_only":1,"second_only":4,"difference":-0.001364877161055505,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.0015703993274813867,"worse":0.86,"better":0.02},"usd_per_1000":0.8552874128602311}},{"frontier":123,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":0.2337027411096696},"share":{"kept":550,"first_only":0,"second_only":4,"difference":-0.0018198362147406734,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.002019039082961255,"worse":0.9633333333333334,"better":0.0},"usd_per_1000":0.2091416073839979}},{"frontier":192,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":9.773955058730701},"share":{"kept":550,"first_only":1,"second_only":4,"difference":-0.001364877161055505,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.0015643304444155316,"worse":0.86,"better":0.02},"usd_per_1000":8.712297810434004}},{"frontier":193,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":6.525186903173618},"share":{"kept":550,"first_only":2,"second_only":1,"difference":0.00045495905368516835,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":0.0002546229359817252,"worse":0.21333333333333335,"better":0.43333333333333335},"usd_per_1000":5.816694824842833}},{"frontier":191,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":22.500030475666104},"share":{"kept":550,"first_only":0,"second_only":4,"difference":-0.0018198362147406734,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.002019039082961255,"worse":0.9633333333333334,"better":0.0},"usd_per_1000":20.054954536831808}},{"frontier":190,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":4.692110365316803},"share":{"kept":550,"first_only":1,"second_only":4,"difference":-0.001364877161055505,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.0015643304444155316,"worse":0.86,"better":0.02},"usd_per_1000":4.182887288667587}},{"frontier":188,"sure":{"kept":349,"first_only":1,"second_only":0,"difference":0.00045495905368516835,"usd_per_1000":0.8182289211778264},"share":{"kept":550,"first_only":1,"second_only":4,"difference":-0.001364877161055505,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.0015552587514272745,"worse":0.85,"better":0.013333333333333334},"usd_per_1000":0.7301254606843346}},{"frontier":278,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":5.48311498747724},"share":{"kept":550,"first_only":0,"second_only":4,"difference":-0.0018198362147406734,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.002019039082961255,"worse":0.9633333333333334,"better":0.0},"usd_per_1000":4.887903847521238}},{"frontier":277,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":5.1102266640282625},"share":{"kept":550,"first_only":1,"second_only":4,"difference":-0.001364877161055505,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.0015703993274813867,"worse":0.86,"better":0.02},"usd_per_1000":4.55555124771382}},{"frontier":276,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":1.3723824156794038},"share":{"kept":550,"first_only":2,"second_only":4,"difference":-0.0009099181073703367,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.0011671947860556657,"worse":0.7366666666666667,"better":0.08333333333333333},"usd_per_1000":1.2240383643828738}},{"frontier":275,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":0.44058317350291976},"share":{"kept":550,"first_only":1,"second_only":4,"difference":-0.001364877161055505,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.0015703993274813867,"worse":0.86,"better":0.02},"usd_per_1000":0.3935326039140553}},{"frontier":274,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":0.3663308364043414},"share":{"kept":550,"first_only":2,"second_only":3,"difference":-0.00045495905368516835,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.0005942213837353536,"worse":0.5533333333333333,"better":0.15333333333333332},"usd_per_1000":0.32735204602413803}},{"frontier":140,"sure":{"kept":349,"first_only":0,"second_only":0,"difference":0.0,"usd_per_1000":2.042760994756172},"share":{"kept":550,"first_only":0,"second_only":4,"difference":-0.0018198362147406734,"held_out":{"splits":300,"answered":0.2499574888999705,"difference":-0.002019039082961255,"worse":0.9633333333333334,"better":0.0},"usd_per_1000":1.82154182480392}}]},"baselines":{"options":5,"letter":"E","letter_share":0.22656960873521384},"runs":[{"run":1108,"model":"gemini-4-carbon","name":"Gemini 4 Carbon","variant":null,"reasoning":"high","provider":"agbench","perplexity_mode":null,"note":"One request of 16 questions came back empty twice during the subscription bridge's outage on 14 November 2026; it was asked a third time on 15 November 2026 at the owner's decision, with the same settings, and answered (15 of the 16 right). Both empty replies are kept in the run's record.","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://deepmind.google/models/model-cards/gemini-4-carbon/"],"parameters_read":"2026-11-11","access":"Google Antigravity through Can's agbench bridge (Google AI Ultra subscription), one request at a time, thinking high; each request a new Antigravity agent conversation, so Antigravity's own agent instructions and tool declarations (about 13,000 tokens) come before our prompt, with every tool call denied and, as the bridge's owner reports, no workspace files and no earlier conversation; the bridge applies no response format, so the prompt adds a paragraph stating the answer layout","questions_per_request":16,"concurrency":1,"correct":2164,"invalid":0,"invalid_replies":{"invalid_without_rule":64,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":4,"asked_again_with_answers":4,"answers_after_asking_again":64,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":640},"grammar":{"questions":1542,"correct":1524},"without_suspect":{"questions":2186,"correct":2161}},"tyt_mix":0.9778352472240677,"questions_per_minute":3.6149221258298287,"request_seconds":36482.113696910164,"median_request_seconds":217.69127955002477,"p90_request_seconds":392.07630887730045,"billing":"subscription","api_equivalent_usd":68.817728,"cost_basis":"subscription","cost_upper_bound":false,"requests":142,"hardware":null,"input_tokens":2337032,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2973480,"previous":null,"units":[126,153,68,94,103,96,139,65,91,71,78,67,143,73,84,218,76,132,115,172],"started":"2026-11-14T01:30:23Z","updated":"2026-11-14T11:29:07Z","format_of":null,"group":"a","latency":{"source":"run","runs":[1108],"concurrency":1,"median_request_seconds":217.69127955002477,"p90_request_seconds":392.07630887730045},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":1797,"correct":1792,"interval":[0.9935028474147877,0.9988109685254926]}}},{"run":980,"model":"claude-fable-5-2","name":"Claude Fable 5.2","variant":null,"reasoning":"max","provider":"claude","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.anthropic.com/claude-fable-and-mythos-5-2","https://platform.claude.com/docs/en/models/overview"],"parameters_read":"2026-11-11","access":"Claude Code CLI, Claude subscription","questions_per_request":16,"concurrency":1,"correct":2157,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":641},"grammar":{"questions":1542,"correct":1516},"without_suspect":{"questions":2186,"correct":2152}},"tyt_mix":0.9781849573724337,"questions_per_minute":5.614092227943244,"request_seconds":23490.885907357995,"median_request_seconds":81.08372651899117,"p90_request_seconds":393.04560667000476,"billing":"subscription","api_equivalent_usd":117.3311458,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1239756,"cached_input_tokens":311489,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":927989,"output_tokens":2560470,"previous":null,"units":[126,152,68,97,102,96,138,64,91,71,78,67,142,74,84,212,77,132,115,171],"started":"2026-11-11T13:43:08Z","updated":"2026-11-12T01:55:56Z","format_of":null,"group":"ab","latency":{"source":"run","runs":[980],"concurrency":1,"median_request_seconds":81.08372651899117,"p90_request_seconds":393.04560667000476},"single_question_latency":null,"tag":"max · chat","confidence":{"sure":{"questions":4,"correct":4,"interval":[0.5100999795960008,1.0]}}},{"run":140,"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"high","provider":"gemini","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://deepmind.google/models/model-cards/gemini-3-8-flash/"],"parameters_read":"2026-11-09","access":"Gemini API","questions_per_request":16,"concurrency":4,"correct":2156,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":637},"grammar":{"questions":1542,"correct":1519},"without_suspect":{"questions":2186,"correct":2151}},"tyt_mix":0.9734949364936257,"questions_per_minute":51.00400853213506,"request_seconds":2585.6791220029118,"median_request_seconds":53.66963235649746,"p90_request_seconds":134.23365150109166,"billing":"api","api_equivalent_usd":5.31718875,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":483830,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":1321151,"previous":null,"units":[127,153,66,96,99,96,138,65,91,71,78,67,142,73,84,217,77,133,115,168],"started":"2026-10-07T19:47:57Z","updated":"2026-10-07T21:20:08Z","format_of":null,"group":"abc","latency":{"source":"run","runs":[140],"concurrency":4,"median_request_seconds":53.66963235649746,"p90_request_seconds":134.23365150109166},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":1845,"correct":1835,"interval":[0.9900512791575007,0.9970532948633348]}}},{"run":1114,"model":"gemini-4-hydrogen","name":"Gemini 4 Hydrogen","variant":null,"reasoning":"high","provider":"agbench","perplexity_mode":null,"note":"Its English “I don't know” run (1109): one request of 16 questions came back empty twice during the subscription bridge's outage on 14 November 2026; it was asked a third time on 15 November 2026 at the owner's decision, with the same settings, and answered (15 of the 16 right). Both empty replies are kept.","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://deepmind.google/models/model-cards/gemini-4-hydrogen/"],"parameters_read":"2026-11-12","access":"Google Antigravity through Can's agbench bridge (Google AI Ultra subscription), one request at a time, thinking high; each request a new Antigravity agent conversation, so Antigravity's own agent instructions and tool declarations (about 13,000 tokens) come before our prompt, with every tool call denied and, as the bridge's owner reports, no workspace files and no earlier conversation; the bridge applies no response format, so the prompt adds a paragraph stating the answer layout","questions_per_request":16,"concurrency":1,"correct":2156,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":638},"grammar":{"questions":1542,"correct":1518},"without_suspect":{"questions":2186,"correct":2151}},"tyt_mix":0.9746390694694884,"questions_per_minute":24.682102882854153,"request_seconds":5343.142787546385,"median_request_seconds":31.690564248536248,"p90_request_seconds":71.33305833123158,"billing":"subscription","api_equivalent_usd":10.562949,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1402306,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":1127932,"previous":null,"units":[128,151,67,94,104,94,139,64,91,71,78,66,140,73,84,217,76,133,115,171],"started":"2026-11-14T03:02:27Z","updated":"2026-11-14T04:31:31Z","format_of":null,"group":"abc","latency":{"source":"run","runs":[1114],"concurrency":1,"median_request_seconds":31.690564248536248,"p90_request_seconds":71.33305833123158},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":1437,"correct":1432,"interval":[0.9918804802767308,0.9985129120295436]}}},{"run":277,"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"high","provider":"openai","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://developers.openai.com/api/docs/models/gpt-6-astra"],"parameters_read":"2026-11-09","access":"Codex CLI, ChatGPT subscription","questions_per_request":16,"concurrency":1,"correct":2153,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":637},"grammar":{"questions":1542,"correct":1516},"without_suspect":{"questions":2186,"correct":2149}},"tyt_mix":0.9731544695675556,"questions_per_minute":33.629793554323186,"request_seconds":3921.522735100065,"median_request_seconds":21.670585694999318,"p90_request_seconds":49.642695360598736,"billing":"subscription","api_equivalent_usd":13.33209,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1163564,"cached_input_tokens":265600,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":81737,"previous":null,"units":[128,152,65,95,101,96,139,64,91,71,78,66,142,73,84,214,76,132,114,172],"started":"2026-10-28T00:51:50Z","updated":"2026-10-28T01:57:12Z","format_of":null,"group":"ab","latency":{"source":"run","runs":[277],"concurrency":1,"median_request_seconds":21.670585694999318,"p90_request_seconds":49.642695360598736},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":1793,"correct":1786,"interval":[0.9919629649507112,0.9981076122220346]}}},{"run":276,"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"high","provider":"openai","perplexity_mode":null,"note":"It stopped after 1,792 answers when the service reported the model at capacity, and was resumed for the other 406 questions; the usage and cost of the failed request are unknown.","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://developers.openai.com/api/docs/models/gpt-6.1-sol"],"parameters_read":"2026-11-09","access":"Codex CLI, ChatGPT subscription","questions_per_request":16,"concurrency":1,"correct":2149,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":633},"grammar":{"questions":1542,"correct":1516},"without_suspect":{"questions":2186,"correct":2145}},"tyt_mix":0.9681239817626774,"questions_per_minute":27.420445672643766,"request_seconds":4809.549836440885,"median_request_seconds":24.741423804487567,"p90_request_seconds":70.32583790899079,"billing":"subscription","api_equivalent_usd":3.565574,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1163522,"cached_input_tokens":208000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":163373,"previous":null,"units":[128,151,64,94,101,95,138,64,91,71,78,66,142,73,84,217,76,132,112,172],"started":"2026-10-28T00:51:50Z","updated":"2026-10-28T02:12:27Z","format_of":null,"group":"abcd","latency":{"source":"run","runs":[276],"concurrency":1,"median_request_seconds":24.741423804487567,"p90_request_seconds":70.32583790899079},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":1826,"correct":1816,"interval":[0.9899480689473389,0.997022606547196]}}},{"run":48,"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://developers.openai.com/api/docs/models/gpt-6-astra"],"parameters_read":"2026-11-09","access":"Codex CLI, ChatGPT subscription","questions_per_request":16,"concurrency":1,"correct":2145,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":636},"grammar":{"questions":1542,"correct":1509},"without_suspect":{"questions":2186,"correct":2142}},"tyt_mix":0.9711024247888392,"questions_per_minute":61.21174024165934,"request_seconds":2166.2511060215766,"median_request_seconds":null,"p90_request_seconds":null,"billing":"subscription","api_equivalent_usd":11.88588,"cost_basis":"subscription","cost_upper_bound":false,"requests":139,"hardware":null,"input_tokens":1167698,"cached_input_tokens":240000,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":47378,"previous":{"run":37,"changed":{"questions":372,"older_correct":288,"newer_correct":361,"flipped":77,"changed_answer":79,"newer_wrong":11,"repeated_mistakes":7,"p":3.975760896174872e-20},"unchanged":{"questions":1814,"older_correct":1770,"newer_correct":1772,"flipped":24,"changed_answer":25,"newer_wrong":42,"repeated_mistakes":30,"p":0.8388197422027588},"added":{"questions":12,"newer_correct":12}},"units":[128,153,65,95,100,95,139,64,91,71,78,65,142,73,84,211,74,132,113,172],"started":"2026-10-04T08:31:16Z","updated":"2026-10-04T09:07:22Z","format_of":null,"group":"abcde","latency":{"source":"sample","runs":[69,78,82,87,91],"repeats":[1,2,3,4,5],"questions":200,"concurrency":1,"median_request_seconds":13.043534992961213,"p90_request_seconds":16.78220434060786},"single_question_latency":{"run":142,"questions":200,"median_request_seconds":6.135667477443349,"p90_request_seconds":8.567763022321742},"tag":"low · chat","confidence":{"sure":{"questions":1725,"correct":1720,"interval":[0.9932323994929521,0.9987613146795898]}}},{"run":610,"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"medium","provider":"openai","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://developers.openai.com/api/docs/models/gpt-6-astra"],"parameters_read":"2026-11-09","access":"Codex CLI, ChatGPT subscription","questions_per_request":16,"concurrency":1,"correct":2143,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":636},"grammar":{"questions":1542,"correct":1507},"without_suspect":{"questions":2186,"correct":2139}},"tyt_mix":0.970875446838126,"questions_per_minute":49.46029058589572,"request_seconds":2666.3814231129363,"median_request_seconds":15.334388918505283,"p90_request_seconds":30.63895340207382,"billing":"subscription","api_equivalent_usd":11.97292,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1163542,"cached_input_tokens":243200,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":50526,"previous":null,"units":[128,152,64,95,101,96,138,64,91,71,78,65,143,73,83,212,73,132,112,172],"started":"2026-11-05T07:18:02Z","updated":"2026-11-05T08:02:29Z","format_of":null,"group":"abcde","latency":{"source":"run","runs":[610],"concurrency":1,"median_request_seconds":15.334388918505283,"p90_request_seconds":30.63895340207382},"single_question_latency":null,"tag":"medium · chat","confidence":{"sure":{"questions":1749,"correct":1741,"interval":[0.9909998384331331,0.9976804849914138]}}},{"run":191,"model":"gpt-6-astra","name":"GPT-6 Astra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://developers.openai.com/api/docs/models/gpt-6-astra"],"parameters_read":"2026-11-09","access":"Codex CLI, ChatGPT subscription","questions_per_request":1,"concurrency":1,"correct":2141,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":638},"grammar":{"questions":1542,"correct":1503},"without_suspect":{"questions":2186,"correct":2137}},"tyt_mix":0.9729367348391383,"questions_per_minute":8.554164640275534,"request_seconds":15417.05187424965,"median_request_seconds":6.41418632400746,"p90_request_seconds":9.219829414092237,"billing":"subscription","api_equivalent_usd":58.769456,"cost_basis":"subscription","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":10201437,"cached_input_tokens":5527296,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":130015,"previous":null,"units":[128,152,66,96,101,95,137,63,91,70,78,66,141,73,84,211,75,133,109,172],"started":"2026-10-13T16:20:09Z","updated":"2026-10-13T20:37:11Z","format_of":48,"group":"abcde","latency":{"source":"run","runs":[191],"concurrency":1,"median_request_seconds":6.41418632400746,"p90_request_seconds":9.219829414092237},"single_question_latency":null,"tag":"low · chat · 1 per request","confidence":{"sure":{"questions":1839,"correct":1824,"interval":[0.9865853790384032,0.9950508071644697]}}},{"run":188,"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://deepmind.google/models/model-cards/gemini-3-8-flash/"],"parameters_read":"2026-11-09","access":"Gemini API","questions_per_request":1,"concurrency":1,"correct":2135,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":631},"grammar":{"questions":1542,"correct":1504},"without_suspect":{"questions":2186,"correct":2131}},"tyt_mix":0.9642468701559583,"questions_per_minute":19.128837453656914,"request_seconds":6894.302924550604,"median_request_seconds":2.717947673998424,"p90_request_seconds":4.674682790008956,"billing":"api","api_equivalent_usd":2.11764075,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":680631,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":428578,"previous":null,"units":[126,152,64,96,98,95,137,64,91,71,76,66,142,74,83,215,74,130,112,169],"started":"2026-10-13T16:20:09Z","updated":"2026-10-13T18:15:07Z","format_of":123,"group":"bcde","latency":{"source":"run","runs":[188],"concurrency":1,"median_request_seconds":2.717947673998424,"p90_request_seconds":4.674682790008956},"single_question_latency":null,"tag":"low · chat · 1 per request","confidence":{"sure":{"questions":1365,"correct":1359,"interval":[0.9904430302635897,0.9979839725393643]}}},{"run":190,"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://developers.openai.com/api/docs/models/gpt-6.1-sol"],"parameters_read":"2026-11-09","access":"Codex CLI, ChatGPT subscription","questions_per_request":1,"concurrency":1,"correct":2126,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":630},"grammar":{"questions":1542,"correct":1496},"without_suspect":{"questions":2186,"correct":2122}},"tyt_mix":0.9620813364018854,"questions_per_minute":8.103710483239363,"request_seconds":16274.026604573679,"median_request_seconds":6.613393928506412,"p90_request_seconds":9.831893868802581,"billing":"subscription","api_equivalent_usd":12.2396048,"cost_basis":"subscription","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":10202072,"cached_input_tokens":5136768,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":159532,"previous":null,"units":[127,150,64,94,99,96,137,64,89,71,78,65,141,73,84,209,72,131,110,172],"started":"2026-10-13T16:20:09Z","updated":"2026-10-13T20:51:28Z","format_of":124,"group":"cdef","latency":{"source":"run","runs":[190],"concurrency":1,"median_request_seconds":6.613393928506412,"p90_request_seconds":9.831893868802581},"single_question_latency":null,"tag":"low · chat · 1 per request","confidence":{"sure":{"questions":1741,"correct":1727,"interval":[0.9865470638705058,0.9952039443585508]}}},{"run":124,"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://developers.openai.com/api/docs/models/gpt-6.1-sol"],"parameters_read":"2026-11-09","access":"Codex CLI, ChatGPT subscription","questions_per_request":16,"concurrency":1,"correct":2125,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":631},"grammar":{"questions":1542,"correct":1494},"without_suspect":{"questions":2186,"correct":2121}},"tyt_mix":0.9631119804023915,"questions_per_minute":60.83084796503379,"request_seconds":2167.9789845409687,"median_request_seconds":13.26978919148678,"p90_request_seconds":24.609638098091818,"billing":"subscription","api_equivalent_usd":2.48456,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1159990,"cached_input_tokens":198400,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":54154,"previous":null,"units":[128,151,64,94,99,95,138,63,91,71,78,65,142,73,82,208,72,130,110,171],"started":"2026-10-05T20:36:36Z","updated":"2026-10-05T21:12:45Z","format_of":null,"group":"def","latency":{"source":"run","runs":[124],"concurrency":1,"median_request_seconds":13.26978919148678,"p90_request_seconds":24.609638098091818},"single_question_latency":null,"tag":"low · chat","confidence":{"sure":{"questions":1610,"correct":1597,"interval":[0.9862337881881241,0.995275177747566]}}},{"run":557,"model":"gemini-4-argon","name":"Gemini 4 Argon","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://deepmind.google/models/model-cards/gemini-4-argon/"],"parameters_read":"2026-11-09","access":"Gemini API","questions_per_request":16,"concurrency":1,"correct":2118,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":636},"grammar":{"questions":1542,"correct":1482},"without_suspect":{"questions":2186,"correct":2114}},"tyt_mix":0.968038222454209,"questions_per_minute":152.28388138219916,"request_seconds":866.0141756500816,"median_request_seconds":5.459406626003329,"p90_request_seconds":7.905744231294375,"billing":"api","api_equivalent_usd":3.925196,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":767554,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":42749,"previous":null,"units":[127,154,63,97,102,93,135,64,91,71,75,66,139,73,82,202,73,131,112,168],"started":"2026-11-03T18:31:52Z","updated":"2026-11-03T18:46:19Z","format_of":null,"group":"def","latency":{"source":"run","runs":[557],"concurrency":1,"median_request_seconds":5.459406626003329,"p90_request_seconds":7.905744231294375},"single_question_latency":null,"tag":"low · chat","confidence":{"sure":{"questions":1842,"correct":1819,"interval":[0.9813325235735316,0.9916653761379063]}}},{"run":310,"model":"claude-fable-5-1","name":"Claude Fable 5.1","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.anthropic.com/claude/fable","https://platform.claude.com/docs/en/models/overview"],"parameters_read":"2026-11-09","access":"Claude Code CLI, Claude subscription","questions_per_request":16,"concurrency":1,"correct":2113,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":627},"grammar":{"questions":1542,"correct":1486},"without_suspect":{"questions":2186,"correct":2110}},"tyt_mix":0.9571735807946601,"questions_per_minute":27.111672851118513,"request_seconds":4864.325441082445,"median_request_seconds":30.3313984014967,"p90_request_seconds":64.38088531556423,"billing":"subscription","api_equivalent_usd":37.4582275,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1239453,"cached_input_tokens":324710,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":914467,"output_tokens":381699,"previous":null,"units":[125,150,64,96,97,95,135,63,91,71,78,66,139,74,84,207,73,128,108,169],"started":"2026-10-28T14:33:55Z","updated":"2026-10-28T15:55:00Z","format_of":null,"group":"efg","latency":{"source":"run","runs":[310],"concurrency":1,"median_request_seconds":30.3313984014967,"p90_request_seconds":64.38088531556423},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":278,"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.anthropic.com/claude-opus-5-5","https://platform.claude.com/docs/en/models/overview"],"parameters_read":"2026-11-09","access":"Claude Code CLI, Claude subscription","questions_per_request":16,"concurrency":1,"correct":2088,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":627},"grammar":{"questions":1542,"correct":1461},"without_suspect":{"questions":2186,"correct":2086}},"tyt_mix":0.9543363564107432,"questions_per_minute":39.02604924802538,"request_seconds":3379.2813400570594,"median_request_seconds":22.122042668488575,"p90_request_seconds":44.978391354210906,"billing":"subscription","api_equivalent_usd":14.306400199999999,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1247691,"cached_input_tokens":318161,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":929252,"output_tokens":340382,"previous":null,"units":[126,151,63,97,96,94,136,64,89,71,78,65,141,71,81,207,70,120,104,164],"started":"2026-10-28T00:51:50Z","updated":"2026-10-28T01:48:10Z","format_of":null,"group":"fgh","latency":{"source":"run","runs":[278],"concurrency":1,"median_request_seconds":22.122042668488575,"p90_request_seconds":44.978391354210906},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":275,"model":"gpt-6.1-terra","name":"GPT-6.1 Terra","variant":null,"reasoning":"high","provider":"openai","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://developers.openai.com/api/docs/models/gpt-6.1-terra"],"parameters_read":"2026-11-09","access":"Codex CLI, ChatGPT subscription","questions_per_request":16,"concurrency":1,"correct":2072,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":631},"grammar":{"questions":1542,"correct":1441},"without_suspect":{"questions":2186,"correct":2070}},"tyt_mix":0.9570970647084875,"questions_per_minute":113.26624110162051,"request_seconds":1164.3363346160622,"median_request_seconds":7.102677927498007,"p90_request_seconds":12.638287538199801,"billing":"subscription","api_equivalent_usd":1.1309,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1163550,"cached_input_tokens":380800,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":65822,"previous":null,"units":[125,152,64,95,101,94,136,62,89,71,75,63,138,70,83,194,67,126,100,167],"started":"2026-10-28T00:51:50Z","updated":"2026-10-28T01:11:15Z","format_of":null,"group":"ghi","latency":{"source":"run","runs":[275],"concurrency":1,"median_request_seconds":7.102677927498007,"p90_request_seconds":12.638287538199801},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":1363,"correct":1344,"interval":[0.9783301784584166,0.9910579349850137]}}},{"run":192,"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.anthropic.com/claude-opus-5-5","https://platform.claude.com/docs/en/models/overview"],"parameters_read":"2026-11-09","access":"Claude Code CLI, Claude subscription","questions_per_request":1,"concurrency":1,"correct":2051,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":628},"grammar":{"questions":1542,"correct":1423},"without_suspect":{"questions":2186,"correct":2046}},"tyt_mix":0.9512813972984088,"questions_per_minute":14.516400274394256,"request_seconds":9084.896910195122,"median_request_seconds":3.7071781139966333,"p90_request_seconds":5.883561807710794,"billing":"subscription","api_equivalent_usd":25.5178246,"cost_basis":"subscription","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":3481382,"cached_input_tokens":1403883,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":2073103,"output_tokens":431732,"previous":null,"units":[128,149,60,95,100,96,134,63,89,71,77,64,139,70,78,199,66,113,96,164],"started":"2026-10-13T16:20:09Z","updated":"2026-10-13T18:51:39Z","format_of":47,"group":"hij","latency":{"source":"run","runs":[192],"concurrency":1,"median_request_seconds":3.7071781139966333,"p90_request_seconds":5.883561807710794},"single_question_latency":null,"tag":"low · chat · 1 per request","confidence":{"sure":{"questions":1,"correct":1,"interval":[0.20654329147389294,1.0]}}},{"run":123,"model":"gemini-3.8-flash","name":"Gemini 3.8 Flash","variant":null,"reasoning":"low","provider":"gemini","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://deepmind.google/models/model-cards/gemini-3-8-flash/"],"parameters_read":"2026-11-09","access":"Gemini API","questions_per_request":16,"concurrency":1,"correct":2050,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":624},"grammar":{"questions":1542,"correct":1426},"without_suspect":{"questions":2186,"correct":2047}},"tyt_mix":0.9465913764196008,"questions_per_minute":399.91697973147666,"request_seconds":329.7684436618583,"median_request_seconds":2.1872003605240025,"p90_request_seconds":2.6911692536843477,"billing":"api","api_equivalent_usd":0.5903475,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":483830,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":60660,"previous":null,"units":[126,149,62,96,96,95,133,62,91,68,72,64,130,73,79,190,66,128,104,166],"started":"2026-10-05T20:04:57Z","updated":"2026-10-05T20:10:28Z","format_of":null,"group":"hij","latency":{"source":"run","runs":[123],"concurrency":1,"median_request_seconds":2.1872003605240025,"p90_request_seconds":2.6911692536843477},"single_question_latency":null,"tag":"low · chat","confidence":{"sure":{"questions":976,"correct":961,"interval":[0.9747976246987011,0.9906645479626759]}}},{"run":311,"model":"grok-4.7","name":"Grok 4.7","variant":null,"reasoning":"high","provider":"xai","perplexity_mode":null,"note":"xAI did not answer 31 attempts within the time limit, and the runner sent each such request again. The run stopped twice and was resumed from the last answered question: at 1,696 answers, after one request timed out six times (the limit then went from 15 to 30 minutes), and at 2,144, when a retry sent an expired sign-in token. The usage and cost of the timed-out attempts are unknown.","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://media.x.ai/v1/website/4p7card-5eccc980.pdf","https://x.ai/news/grok-4-7","https://docs.x.ai/developers/models"],"parameters_read":"2026-11-09","access":"xAI Responses API, SuperGrok subscription","questions_per_request":16,"concurrency":4,"correct":2038,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":623},"grammar":{"questions":1542,"correct":1415},"without_suspect":{"questions":2186,"correct":2036}},"tyt_mix":0.9440853757394578,"questions_per_minute":6.595969744562386,"request_seconds":19994.027429965063,"median_request_seconds":401.9361946709978,"p90_request_seconds":1228.6228537980119,"billing":"subscription","api_equivalent_usd":31.414164,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":808614,"cached_input_tokens":215424,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":5020012,"previous":null,"units":[125,152,60,94,98,94,131,63,87,72,74,61,141,72,81,187,69,115,100,162],"started":"2026-10-28T14:33:55Z","updated":"2026-10-28T22:00:47Z","format_of":null,"group":"ijk","latency":{"source":"run","runs":[311],"concurrency":4,"median_request_seconds":401.9361946709978,"p90_request_seconds":1228.6228537980119},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":121,"model":"deepseek-v4.1-flash","name":"DeepSeek-V4.1-Flash","variant":null,"reasoning":"on","provider":"vllm","perplexity_mode":null,"note":null,"parameters":552000000000,"active_parameters":[8000000000,16000000000],"effective_parameters":null,"parameters_scope":"backbone, as DeepSeek states it (8B active in prefill, 16B in decode); the 196B Engram memory is separate","parameters_source":["https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash"],"parameters_read":"2026-11-09","access":"SGLang, self-hosted, published mixed precision; first-release weights (revision fb2764a), not the 1017 update","questions_per_request":16,"concurrency":8,"correct":2023,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":613},"grammar":{"questions":1542,"correct":1410},"without_suspect":{"questions":2186,"correct":2020}},"tyt_mix":0.9309417113504793,"questions_per_minute":19.260714724220154,"request_seconds":6847.09793422994,"median_request_seconds":365.14285134349484,"p90_request_seconds":686.8833388963016,"billing":"local","api_equivalent_usd":2.8553639999999993,"cost_basis":"weights","cost_upper_bound":false,"requests":138,"hardware":"4 × RTX PRO 6000 Blackwell (96 GB); FP4 experts, FP8 and BF16 layers","input_tokens":669316,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2212141,"previous":null,"units":[125,150,59,92,94,93,129,63,89,70,75,61,136,70,81,190,66,125,98,157],"started":"2026-10-05T19:02:51Z","updated":"2026-10-05T20:56:59Z","format_of":null,"group":"ijk","latency":{"source":"run","runs":[121],"concurrency":8,"median_request_seconds":365.14285134349484,"p90_request_seconds":686.8833388963016},"single_question_latency":null,"tag":"reasoning on · chat · first release","confidence":{"sure":{"questions":2198,"correct":2023,"interval":[0.9083178247126321,0.9309796070372266]}}},{"run":585,"model":"deepseek-v4.1-flash-1017","name":"DeepSeek-V4.1-Flash","variant":null,"reasoning":"on","provider":"vllm","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":"V4.1 Flash card (first release): 552B backbone + 196B conditional memory; 8B active in prefill, 16B in decode; the 1017 card states no size","parameters_source":["https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017","https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash"],"parameters_read":"2026-11-09","access":"SGLang, self-hosted, published mixed precision; 1017 weights (revision 1bae0e1)","questions_per_request":16,"concurrency":8,"correct":2017,"invalid":32,"invalid_replies":{"invalid_without_rule":32,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"empty, not retried":16,"cut off":16}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":617},"grammar":{"questions":1542,"correct":1400},"without_suspect":{"questions":2186,"correct":2015}},"tyt_mix":0.9348373094017906,"questions_per_minute":12.862473297108837,"request_seconds":10253.082510161039,"median_request_seconds":486.878976011998,"p90_request_seconds":1129.4343934595702,"billing":"local","api_equivalent_usd":2.8085495999999996,"cost_basis":"maker","cost_upper_bound":false,"requests":138,"hardware":"4 × RTX PRO 6000 Blackwell (96 GB); FP4 experts, FP8 and BF16 layers","input_tokens":669316,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2173129,"previous":null,"units":[123,152,63,93,93,93,134,64,89,71,76,61,138,69,83,165,67,122,103,158],"started":"2026-11-05T02:39:48Z","updated":"2026-11-05T05:30:41Z","format_of":null,"group":"jkl","latency":{"source":"run","runs":[585],"concurrency":8,"median_request_seconds":486.878976011998,"p90_request_seconds":1129.4343934595702},"single_question_latency":null,"tag":"reasoning on · chat · 1017","confidence":{"sure":{"questions":2164,"correct":2015,"interval":[0.9196956696967801,0.9410683270823205]}}},{"run":1336,"model":"qwen/qwen4-max","name":"Qwen4-Max","variant":null,"reasoning":"high","provider":"openrouter-chat","perplexity_mode":null,"note":"Alibaba's closed flagship chat model (OpenRouter id qwen/qwen4-max, listed 16 November 2026)","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://openrouter.ai/qwen/qwen4-max"],"parameters_read":"2026-11-16","access":"OpenRouter chat API, Alibaba's endpoint (the maker's), fallbacks off, reasoning effort high, output cap 32,768 tokens, reasoning included; the answer schema","questions_per_request":16,"concurrency":4,"correct":2016,"invalid":32,"invalid_replies":{"invalid_without_rule":32,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":32}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":621},"grammar":{"questions":1542,"correct":1395},"without_suspect":{"questions":2186,"correct":2014}},"tyt_mix":0.9393003523298852,"questions_per_minute":28.18997194535436,"request_seconds":4678.259356044997,"median_request_seconds":91.54818525950213,"p90_request_seconds":335.0108871188997,"billing":"api","api_equivalent_usd":5.567622,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":707233,"cached_input_tokens":104192,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":1001505,"previous":null,"units":[124,149,60,95,101,92,133,62,86,70,76,64,123,68,80,184,63,119,104,163],"started":"2026-11-16T16:09:57Z","updated":"2026-11-16T17:27:56Z","format_of":null,"group":"jkl","latency":{"source":"run","runs":[1336],"concurrency":4,"median_request_seconds":91.54818525950213,"p90_request_seconds":335.0108871188997},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":3,"correct":3,"interval":[0.43849391955098227,1.0]}}},{"run":274,"model":"gpt-6.1-terra","name":"GPT-6.1 Terra","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://developers.openai.com/api/docs/models/gpt-6.1-terra"],"parameters_read":"2026-11-09","access":"Codex CLI, ChatGPT subscription","questions_per_request":16,"concurrency":1,"correct":1997,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":629},"grammar":{"questions":1542,"correct":1368},"without_suspect":{"questions":2186,"correct":1993}},"tyt_mix":0.9462971256050109,"questions_per_minute":206.76412628602517,"request_seconds":637.8282459770853,"median_request_seconds":4.235071693503414,"p90_request_seconds":5.358385625202208,"billing":"subscription","api_equivalent_usd":0.936888,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1163508,"cached_input_tokens":396800,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":30068,"previous":null,"units":[127,150,64,94,98,96,131,58,87,69,73,54,130,67,80,180,55,120,98,166],"started":"2026-10-28T00:51:38Z","updated":"2026-10-28T01:02:17Z","format_of":null,"group":"jkl","latency":{"source":"run","runs":[274],"concurrency":1,"median_request_seconds":4.235071693503414,"p90_request_seconds":5.358385625202208},"single_question_latency":null,"tag":"low · chat","confidence":{"sure":{"questions":1306,"correct":1279,"interval":[0.970087348715641,0.9857534185954597]}}},{"run":47,"model":"claude-opus-5-5","name":"Claude Opus 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.anthropic.com/claude-opus-5-5","https://platform.claude.com/docs/en/models/overview"],"parameters_read":"2026-11-09","access":"Claude Code CLI, Claude subscription","questions_per_request":16,"concurrency":1,"correct":1988,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":622},"grammar":{"questions":1542,"correct":1366},"without_suspect":{"questions":2186,"correct":1987}},"tyt_mix":0.9372667939957611,"questions_per_minute":71.89577090051554,"request_seconds":1844.3365769522497,"median_request_seconds":null,"p90_request_seconds":null,"billing":"subscription","api_equivalent_usd":10.896595,"cost_basis":"subscription","cost_upper_bound":false,"requests":139,"hardware":null,"input_tokens":1248840,"cached_input_tokens":328215,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":920343,"output_tokens":173354,"previous":{"run":36,"changed":{"questions":372,"older_correct":224,"newer_correct":337,"flipped":137,"changed_answer":146,"newer_wrong":35,"repeated_mistakes":14,"p":7.0458837577401e-25},"unchanged":{"questions":1814,"older_correct":1621,"newer_correct":1639,"flipped":174,"changed_answer":189,"newer_wrong":175,"repeated_mistakes":82,"p":0.19733841594358562},"added":{"questions":12,"newer_correct":12}},"units":[125,149,60,96,101,91,137,64,86,70,75,58,139,71,79,163,54,111,99,160],"started":"2026-10-04T08:31:15Z","updated":"2026-10-04T09:02:00Z","format_of":null,"group":"kl","latency":{"source":"sample","runs":[67,77,81,86,90],"repeats":[1,2,3,4,5],"questions":200,"concurrency":1,"median_request_seconds":13.402853598003276,"p90_request_seconds":19.79555194299901},"single_question_latency":null,"tag":"low · chat","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":193,"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.anthropic.com/claude-sonnet-5-5","https://platform.claude.com/docs/en/models/overview"],"parameters_read":"2026-11-09","access":"Claude Code CLI, Claude subscription","questions_per_request":1,"concurrency":1,"correct":1960,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":618},"grammar":{"questions":1542,"correct":1342},"without_suspect":{"questions":2186,"correct":1956}},"tyt_mix":0.9295125707823226,"questions_per_minute":11.214821323849451,"request_seconds":11759.438353203528,"median_request_seconds":4.643365134485066,"p90_request_seconds":8.381210892091621,"billing":"subscription","api_equivalent_usd":17.0292028,"cost_basis":"subscription","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":3285774,"cached_input_tokens":1200108,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":2081270,"output_tokens":857532,"previous":null,"units":[124,148,58,94,101,93,131,61,86,67,75,61,130,65,80,184,57,102,89,154],"started":"2026-10-13T16:20:09Z","updated":"2026-10-13T19:36:13Z","format_of":55,"group":"l","latency":{"source":"run","runs":[193],"concurrency":1,"median_request_seconds":4.643365134485066,"p90_request_seconds":8.381210892091621},"single_question_latency":null,"tag":"high · chat · 1 per request","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":939,"model":"meta/muse-ember","name":"Muse Ember","variant":null,"reasoning":"max","provider":"openrouter-chat","perplexity_mode":null,"note":"Meta's closed chat model, released 9 November 2026, added as a chat-model comparison","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://research.meta.ai/blog/muse-ember","https://dev.meta.ai/models/muse-ember"],"parameters_read":"2026-11-10","access":"OpenRouter chat API, Meta's own endpoint, fallbacks off, output cap 65,536 tokens, reasoning max","questions_per_request":16,"concurrency":4,"correct":1882,"invalid":48,"invalid_replies":{"invalid_without_rule":48,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":48}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":592},"grammar":{"questions":1542,"correct":1290},"without_suspect":{"questions":2186,"correct":1880}},"tyt_mix":0.8909129733320679,"questions_per_minute":102.78761648802944,"request_seconds":1283.0339344950044,"median_request_seconds":30.90874948099372,"p90_request_seconds":64.57190969429503,"billing":"api","api_equivalent_usd":1.429666504,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":593823,"cached_input_tokens":10576,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2742599,"previous":null,"units":[121,149,55,93,84,90,134,62,88,68,74,62,132,68,80,177,22,93,90,140],"started":"2026-11-10T16:14:54Z","updated":"2026-11-10T16:36:24Z","format_of":null,"group":"m","latency":{"source":"run","runs":[939],"concurrency":4,"median_request_seconds":30.90874948099372,"p90_request_seconds":64.57190969429503},"single_question_latency":null,"tag":"max · chat","confidence":{"sure":{"questions":1,"correct":1,"interval":[0.20654329147389294,1.0]}}},{"run":55,"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"high","provider":"claude","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.anthropic.com/claude-sonnet-5-5","https://platform.claude.com/docs/en/models/overview"],"parameters_read":"2026-11-09","access":"Claude Code CLI, Claude subscription","questions_per_request":16,"concurrency":1,"correct":1869,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":559},"grammar":{"questions":1542,"correct":1310},"without_suspect":{"questions":2186,"correct":1868}},"tyt_mix":0.8516812284489577,"questions_per_minute":37.778240432131966,"request_seconds":3509.9570144939353,"median_request_seconds":null,"p90_request_seconds":null,"billing":"subscription","api_equivalent_usd":8.0760773,"cost_basis":"subscription","cost_upper_bound":false,"requests":139,"hardware":null,"input_tokens":1215038,"cached_input_tokens":301633,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":913127,"output_tokens":439285,"previous":{"run":39,"changed":{"questions":372,"older_correct":213,"newer_correct":323,"flipped":130,"changed_answer":148,"newer_wrong":49,"repeated_mistakes":21,"p":4.263429420007923e-25},"unchanged":{"questions":1814,"older_correct":1526,"newer_correct":1534,"flipped":194,"changed_answer":234,"newer_wrong":280,"repeated_mistakes":147,"p":0.615380740884383},"added":{"questions":12,"newer_correct":12}},"units":[105,138,48,91,87,90,131,59,79,68,75,58,130,68,77,170,61,101,94,139],"started":"2026-10-04T09:11:47Z","updated":"2026-10-04T10:10:17Z","format_of":null,"group":"mn","latency":{"source":"sample","runs":[63,80,88,93,98],"repeats":[1,2,3,4,5],"questions":200,"concurrency":1,"median_request_seconds":20.488734460028354,"p90_request_seconds":37.048832100233994},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":320,"model":"moonshotai/kimi-k3.1-pro","name":"Kimi K3.1 Pro","variant":null,"reasoning":"low","provider":"openrouter-chat","perplexity_mode":null,"note":null,"parameters":2800000000000,"active_parameters":[104000000000],"effective_parameters":null,"parameters_scope":"mixture of experts, as the model card states it","parameters_source":["https://huggingface.co/moonshotai/Kimi-K3.1-Pro"],"parameters_read":"2026-11-09","access":"OpenRouter chat API, Moonshot AI's endpoint (mxfp4), fallbacks off","questions_per_request":16,"concurrency":4,"correct":1819,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":615},"grammar":{"questions":1542,"correct":1204},"without_suspect":{"questions":2186,"correct":1817}},"tyt_mix":0.9100782263294424,"questions_per_minute":60.023205399428846,"request_seconds":2197.1502375188866,"median_request_seconds":44.8976706439862,"p90_request_seconds":144.5165325260954,"billing":"api","api_equivalent_usd":4.5005352,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":788026,"cached_input_tokens":11776,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":294568,"previous":null,"units":[120,152,57,96,96,94,116,55,79,67,64,51,116,62,69,148,40,103,84,150],"started":"2026-10-29T10:35:16Z","updated":"2026-10-29T11:11:54Z","format_of":null,"group":"mno","latency":{"source":"run","runs":[320],"concurrency":4,"median_request_seconds":44.8976706439862,"p90_request_seconds":144.5165325260954},"single_question_latency":null,"tag":"low · chat","confidence":{"sure":{"questions":1436,"correct":1322,"interval":[0.9054814839745762,0.9334996928437627]}}},{"run":870,"model":"qwen/qwen3.5-397b-a17b","name":"Qwen3.5-397B-A17B","variant":null,"reasoning":"high","provider":"openrouter-chat","perplexity_mode":null,"note":null,"parameters":397000000000,"active_parameters":[17000000000],"effective_parameters":null,"parameters_scope":"mixture of experts, as Qwen states it","parameters_source":["https://huggingface.co/Qwen/Qwen3.5-397B-A17B"],"parameters_read":"2026-11-09","access":"OpenRouter chat API, DeepInfra's endpoint (fp8), fallbacks off, output cap 65,536 tokens, reasoning included","questions_per_request":16,"concurrency":4,"correct":1818,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":597},"grammar":{"questions":1542,"correct":1221},"without_suspect":{"questions":2186,"correct":1817}},"tyt_mix":0.8893703437885545,"questions_per_minute":14.43452212749494,"request_seconds":9136.429930630984,"median_request_seconds":251.3234145815004,"p90_request_seconds":386.6777357280036,"billing":"api","api_equivalent_usd":5.29999845,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":521441,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":1688450,"previous":null,"units":[119,147,56,95,87,93,126,60,79,67,66,50,121,68,76,148,42,94,84,140],"started":"2026-11-08T15:09:52Z","updated":"2026-11-08T17:42:08Z","format_of":null,"group":"mno","latency":{"source":"run","runs":[870],"concurrency":4,"median_request_seconds":251.3234145815004,"p90_request_seconds":386.6777357280036},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":78,"correct":70,"interval":[0.8104536622780937,0.9471072847219651]}}},{"run":1165,"model":"Copernicus-31B","name":"Copernicus-31B","variant":null,"reasoning":null,"provider":"copernicus","perplexity_mode":null,"note":"Poindexter Labs' released checkpoint, a merge of KnowLine-31B-Gen1 and Sieve-31B-v3 (both on Gemma-4-31B-it), run on our hardware through its card's copernicus_server.py (Transformers, text only, the card's defaults).","parameters":31273088876,"active_parameters":null,"effective_parameters":null,"parameters_scope":"the whole checkpoint, its vision encoder included (text only here); the card says approximately 31 billion","parameters_source":["counted from the checkpoint's safetensors, Poindexter-Labs/Copernicus-31B@be71525","https://huggingface.co/Poindexter-Labs/Copernicus-31B"],"parameters_read":"2026-11-15","access":"Copernicus server (the card's copernicus_server.py, Transformers), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1798,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":611},"grammar":{"questions":1542,"correct":1187},"without_suspect":{"questions":2186,"correct":1794}},"tyt_mix":0.9031184259435007,"questions_per_minute":308.84129292016166,"request_seconds":427.01543810105795,"median_request_seconds":0.18386128850033856,"p90_request_seconds":0.25295338760188313,"billing":"local","api_equivalent_usd":0.29535234468656507,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":1619544,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[117,149,62,94,93,96,120,56,78,67,62,45,109,62,75,143,35,96,89,150],"started":"2026-11-15T05:44:13Z","updated":"2026-11-15T05:51:26Z","format_of":null,"group":"nop","latency":{"source":"run","runs":[1165],"concurrency":1,"median_request_seconds":0.18386128850033856,"p90_request_seconds":0.25295338760188313},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":107,"correct":107,"interval":[0.9653415324210404,1.0]}}},{"run":957,"model":"PelaAI/KnowLine-31B-Gen1.5","name":"KnowLine-31B-Gen1.5","variant":null,"reasoning":null,"provider":"knowline","perplexity_mode":null,"note":"PelaAI's released KnowLine-31B-Gen1.5 (PelaAI/KnowLine-31B-Gen1.5 at 8e56aa1), a merged LoRA fine-tune of Gemma-4-31B-it whose weights are those of Gen1; Gen1.5 changes only how long requests are served. It ran on our hardware (one DGX Spark), one request at a time, through the card's serve_knowline.sh (SGLang 0.5.21 in BF16 and the repository's knowline_server.py, which reads a request of up to 3,000 tokens twice), with vision on. Its card lists English and Chinese.","parameters":31273086512,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + 0.58B vision encoder","parameters_source":["counted from the checkpoint's safetensors, PelaAI/KnowLine-31B-Gen1.5@8e56aa1","https://huggingface.co/PelaAI/KnowLine-31B-Gen1.5"],"parameters_read":"2026-11-10","access":"KnowLine server (knowline_server.py over SGLang 0.5.21, PelaAI's release code), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1798,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":609},"grammar":{"questions":1542,"correct":1189},"without_suspect":{"questions":2186,"correct":1795}},"tyt_mix":0.9008301599917751,"questions_per_minute":53.71848351639353,"request_seconds":2455.0209046715463,"median_request_seconds":1.0509718134999275,"p90_request_seconds":1.4750827486888738,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":1619544,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[116,148,62,93,94,96,119,58,79,67,65,44,110,61,75,146,34,98,87,146],"started":"2026-11-10T17:34:15Z","updated":"2026-11-10T18:15:17Z","format_of":null,"group":"nop","latency":{"source":"run","runs":[957],"concurrency":1,"median_request_seconds":1.0509718134999275,"p90_request_seconds":1.4750827486888738},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":143,"correct":143,"interval":[0.9738384762900976,1.0]}}},{"run":1314,"model":"PelaAI/KnowLine-31B-Gen2","name":"KnowLine-31B-Gen2","variant":null,"reasoning":null,"provider":"knowline","perplexity_mode":null,"note":"PelaAI's released checkpoint (13 November 2026), a fine-tune of Gemma-4-31B-it, run on our hardware through its card's serve_knowline.sh (SGLang 0.5.23 in the lmsysorg/sglang image; the card's launcher names 0.5.21, then knowline_server.py), text only.","parameters":31273088876,"active_parameters":null,"effective_parameters":null,"parameters_scope":"the whole checkpoint, its vision encoder included (text only here); the card says 31B parameters","parameters_source":["counted from the checkpoint's safetensors, PelaAI/KnowLine-31B-Gen2@fd99a9e","https://huggingface.co/PelaAI/KnowLine-31B-Gen2"],"parameters_read":"2026-11-15","access":"KnowLine server (knowline_server.py over SGLang, PelaAI's release code), self-hosted","questions_per_request":1,"concurrency":1,"correct":1785,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":613},"grammar":{"questions":1542,"correct":1172},"without_suspect":{"questions":2186,"correct":1780}},"tyt_mix":0.9039313352155895,"questions_per_minute":346.3269401337005,"request_seconds":380.7962497779912,"median_request_seconds":0.16822417750000795,"p90_request_seconds":0.22240222820000782,"billing":"local","api_equivalent_usd":0.2633840727631106,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":1619544,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[119,146,63,94,97,94,117,54,78,69,64,39,109,63,71,140,41,90,90,147],"started":"2026-11-16T11:09:38Z","updated":"2026-11-16T11:16:06Z","format_of":null,"group":"nopq","latency":{"source":"run","runs":[1314],"concurrency":1,"median_request_seconds":0.16822417750000795,"p90_request_seconds":0.22240222820000782},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":81,"correct":81,"interval":[0.954720325877871,0.9999999999999999]}}},{"run":1273,"model":"turion-ai/turion-jev-31b","name":"Turion Jev 31B","variant":null,"reasoning":null,"provider":"turion-jev","perplexity_mode":null,"note":"Turion AI's released Turion Jev 31B (turion-ai/turion-jev-31b at 866c0fc), a merged LoRA fine-tune of Gemma-4-31B-it. It ran on our hardware (one DGX Spark), one request at a time, through the server its package ships (python -m turion_jev.serve on vLLM), with the BF16 weights quantised to FP8 at load and --no-recheck (no second reading of unsure questions), both as in the card's own benchmark run, text only. Probabilities rounded to 6 decimals by its own code; they sum to 1 only within that rounding.","parameters":31273086512,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + 0.58B vision encoder (not read by its server)","parameters_source":["counted from the checkpoint's safetensors, turion-ai/turion-jev-31b@866c0fc","https://huggingface.co/turion-ai/turion-jev-31b"],"parameters_read":"2026-11-10","access":"Turion Jev server (turion_jev.serve on vLLM, the maker's release code), self-hosted, FP8 at load","questions_per_request":1,"concurrency":1,"correct":1761,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":608},"grammar":{"questions":1542,"correct":1153},"without_suspect":{"questions":2186,"correct":1758}},"tyt_mix":0.895486934927715,"questions_per_minute":140.0938107119823,"request_seconds":941.3692106008239,"median_request_seconds":0.40546109349816106,"p90_request_seconds":0.5655567569992854,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); FP8 at load","input_tokens":1683286,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[118,149,61,95,90,95,116,49,75,63,64,36,112,65,71,131,39,101,87,144],"started":"2026-11-15T12:51:12Z","updated":"2026-11-15T13:07:00Z","format_of":null,"group":"opqr","latency":{"source":"run","runs":[1273],"concurrency":1,"median_request_seconds":0.40546109349816106,"p90_request_seconds":0.5655567569992854},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":42,"correct":42,"interval":[0.9161983874908382,1.0]}}},{"run":866,"model":"thinkingmachines/inkling-small","name":"Inkling-Small","variant":null,"reasoning":"high","provider":"openrouter-chat","perplexity_mode":null,"note":null,"parameters":265956439040,"active_parameters":[12000000000],"effective_parameters":null,"parameters_scope":"mixture of experts: language model + multi-token prediction layer + vision and audio encoders, counted; active per token as the card states it; card: 276B total, 12B active","parameters_source":["counted from the checkpoint's safetensors, thinkingmachines/Inkling-Small@8cc5877","https://thinkingmachines.ai/model-card/inkling-small/","https://huggingface.co/thinkingmachines/Inkling-Small"],"parameters_read":"2026-11-09","access":"OpenRouter chat API, DeepInfra's endpoint (fp8), fallbacks off, output cap 65,536 tokens, reasoning included; no response format (the endpoint has no structured outputs), so the prompt adds a paragraph stating the answer layout","questions_per_request":16,"concurrency":4,"correct":1758,"invalid":0,"invalid_replies":{"invalid_without_rule":64,"read_from_broken_replies":16,"read_without_confidence":0,"empty_replies_asked_again":3,"asked_again_with_answers":3,"answers_after_asking_again":48,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":572},"grammar":{"questions":1542,"correct":1186},"without_suspect":{"questions":2186,"correct":1756}},"tyt_mix":0.853957680870583,"questions_per_minute":31.780061296307256,"request_seconds":4149.771731728033,"median_request_seconds":97.66078765300335,"p90_request_seconds":173.41299188119154,"billing":"api","api_equivalent_usd":2.49945345,"cost_basis":"api","cost_upper_bound":false,"requests":141,"hardware":null,"input_tokens":544949,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":1878522,"previous":null,"units":[109,146,52,92,84,89,124,55,74,67,73,47,120,65,66,151,37,100,77,130],"started":"2026-11-08T15:09:44Z","updated":"2026-11-08T16:07:54Z","format_of":null,"group":"opqr","latency":{"source":"run","runs":[866],"concurrency":4,"median_request_seconds":97.66078765300335,"p90_request_seconds":173.41299188119154},"single_question_latency":null,"tag":"high · chat · Jul 2026","confidence":{"sure":{"questions":34,"correct":34,"interval":[0.8984820937803899,1.0]}}},{"run":160,"model":"glm-5.3-flash","name":"GLM-5.3-Flash","variant":null,"reasoning":"on","provider":"vllm-stated","perplexity_mode":null,"note":null,"parameters":320000000000,"active_parameters":[18000000000],"effective_parameters":null,"parameters_scope":"mixture of experts, as Z.ai states it","parameters_source":["https://huggingface.co/zai-org/GLM-5.3-Flash"],"parameters_read":"2026-11-09","access":"TensorFold, self-hosted, EXL3 4-bit experts","questions_per_request":16,"concurrency":8,"correct":1732,"invalid":128,"invalid_replies":{"invalid_without_rule":128,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":128}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":586},"grammar":{"questions":1542,"correct":1146},"without_suspect":{"questions":2186,"correct":1729}},"tyt_mix":0.8670248291733891,"questions_per_minute":34.1746540558756,"request_seconds":3859.0002925670015,"median_request_seconds":102.61677862299985,"p90_request_seconds":639.2771307083002,"billing":"local","api_equivalent_usd":1.19315485,"cost_basis":"maker","cost_upper_bound":false,"requests":138,"hardware":"4 × RTX PRO 6000 Blackwell (96 GB); EXL3 4-bit experts, FP8 dense layers and KV cache","input_tokens":557949,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2218925,"previous":null,"units":[116,142,55,94,87,92,117,55,79,65,70,53,122,66,72,138,0,86,82,141],"started":"2026-10-11T10:53:32Z","updated":"2026-10-11T11:57:51Z","format_of":null,"group":"pqrs","latency":{"source":"run","runs":[160],"concurrency":8,"median_request_seconds":102.61677862299985,"p90_request_seconds":639.2771307083002},"single_question_latency":null,"tag":"reasoning on · chat","confidence":{"sure":{"questions":9,"correct":9,"interval":[0.7008472464490407,1.0]}}},{"run":827,"model":"reax13","name":"REAX 1.3","variant":null,"reasoning":null,"provider":"reax","perplexity_mode":null,"note":"REAX's released REAX 1.3 weights (decisionmodels/REAX-1.3 at fa4bf7e), a decision model of two parts that both score every question; each option's probability is the 50/50 mean of the two parts'. Part G blends CharlieBT/gx31-decider (Gemma line, from tattrongvu/Jeffan-31b-1710) with PelaAI/KnowLine-31B-Gen1 (Gemma-4-31B); part Q blends gurujustin/sn92-gen-p5 (Qwen3.5-27B line, through baselabs/mammoth-1) with vllm-sr/d3-ultra. The model card calls REAX a decentralised lab on Bittensor subnet 92. It ran on our hardware through the maker's engine (the reax13 mode of reaxlabs/reax-engine at ff8f595, its own server, serve.py), both parts in FP8 (block-128) on one RTX PRO 6000 Blackwell, text only, one request at a time. Its card lists English only.","parameters":57359722272,"active_parameters":null,"effective_parameters":null,"parameters_scope":"two full models: part G (Gemma 4 31B line, with vision encoder) + part Q (Qwen3.8-27B line, with vision encoder and 255-way readout)","parameters_source":["counted from the checkpoint's safetensors, decisionmodels/REAX-1.3@fa4bf7e","https://huggingface.co/decisionmodels/REAX-1.3"],"parameters_read":"2026-11-09","access":"REAX Engine server (reax13 mode, the maker's release code), self-hosted, FP8","questions_per_request":1,"concurrency":1,"correct":1713,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":606},"grammar":{"questions":1542,"correct":1107},"without_suspect":{"questions":2186,"correct":1710}},"tyt_mix":0.8877511981588688,"questions_per_minute":224.7916085680708,"request_seconds":586.6767039929982,"median_request_seconds":0.25025921245105565,"p90_request_seconds":0.36129771744599565,"billing":"local","api_equivalent_usd":0.4057847202618238,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); FP8 (block-128)","input_tokens":0,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[117,146,61,94,96,92,105,48,69,62,60,39,103,61,71,133,30,97,88,141],"started":"2026-11-06T22:00:28Z","updated":"2026-11-06T22:10:22Z","format_of":null,"group":"rt","latency":{"source":"run","runs":[827],"concurrency":1,"median_request_seconds":0.25025921245105565,"p90_request_seconds":0.36129771744599565},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":240,"correct":240,"interval":[0.9842455102000642,1.0]}}},{"run":1156,"model":"karpo-decision","name":"Karpo Decision 31B","variant":null,"reasoning":null,"provider":"karpo","perplexity_mode":null,"note":"MachinePulse's released checkpoint, a fine-tune of Gemma-4-31B-it with a 255-way readout that also reads images, run on our hardware through the System One server of the package its card ships (karpo-decision at ac40d39). Its Reading runs shared the server with the Logic & Math site's runs from 05:00 UTC on 15 November 2026, so its latency and GPU-time cost were measured under that load.","parameters":31274459756,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model + vision encoder (575,743,536) + 255-way readout (1,370,880); the card says 31.3B in bf16 plus the readout","parameters_source":["counted from the checkpoint's safetensors, MachinePulseAI/karpo-decision-31B@d69cdd5","https://huggingface.co/MachinePulseAI/karpo-decision-31B"],"parameters_read":"2026-11-15","access":"karpo-decision server (the System One server of the card's karpo-decision package, machinepulse-ai/karpo-decision at ac40d39), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1710,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":613},"grammar":{"questions":1542,"correct":1097},"without_suspect":{"questions":2186,"correct":1708}},"tyt_mix":0.8954196620638385,"questions_per_minute":395.94888187628027,"request_seconds":333.07329818703147,"median_request_seconds":0.15888651099885465,"p90_request_seconds":0.20728088509858936,"billing":"local","api_equivalent_usd":0.2303756979126968,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":936157,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[117,149,63,94,93,97,110,51,75,62,55,39,101,60,67,111,35,99,87,145],"started":"2026-11-15T04:58:46Z","updated":"2026-11-15T05:04:27Z","format_of":null,"group":"rstu","latency":{"source":"run","runs":[1156],"concurrency":1,"median_request_seconds":0.15888651099885465,"p90_request_seconds":0.20728088509858936},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":505,"correct":505,"interval":[0.9924503028054309,0.9999999999999999]}}},{"run":1021,"model":"google/gemini-decision-1","name":"Gemini Decision 1","variant":"via OpenRouter","reasoning":null,"provider":"openrouter","perplexity_mode":null,"note":"Google's decision model, called through OpenRouter's System One route on its one endpoint, Google AI Studio. It returns a probability for each option and bills input tokens only.","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://aistudio.google.com/docs/models/gemini-decision-1","https://openrouter.ai/google/gemini-decision-1","https://blog.google/innovation-and-ai/technology/developers-tools/gemini-decisions-api/"],"parameters_read":"2026-11-11","access":"OpenRouter decisions API, Google AI Studio endpoint","questions_per_request":1,"concurrency":1,"correct":1704,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":604},"grammar":{"questions":1542,"correct":1100},"without_suspect":{"questions":2186,"correct":1702}},"tyt_mix":0.8844415314289329,"questions_per_minute":197.4865762738425,"request_seconds":667.7922241009946,"median_request_seconds":0.2272767955000745,"p90_request_seconds":0.353589776615263,"billing":"api","api_equivalent_usd":0.0817405,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":817405,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[118,147,62,93,92,92,109,52,75,61,61,40,96,61,64,124,35,109,74,139],"started":"2026-11-11T19:56:18Z","updated":"2026-11-11T20:07:33Z","format_of":null,"group":"qrstuv","latency":{"source":"run","runs":[1021],"concurrency":1,"median_request_seconds":0.2272767955000745,"p90_request_seconds":0.353589776615263},"single_question_latency":null,"tag":"20261111","confidence":{"sure":{"questions":318,"correct":318,"interval":[0.9880636934442285,1.0]}}},{"run":199,"model":"d1-ultra","name":"d1-ultra","variant":null,"reasoning":null,"provider":"liquid","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.liquid.ai/blog/d1-ultra"],"parameters_read":"2026-11-09","access":"Liquid AI decisions API","questions_per_request":1,"concurrency":1,"correct":1694,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":604},"grammar":{"questions":1542,"correct":1090},"without_suspect":{"questions":2186,"correct":1691}},"tyt_mix":0.8833066416753661,"questions_per_minute":241.99067818307225,"request_seconds":544.9796702508902,"median_request_seconds":0.22344711299228948,"p90_request_seconds":0.350752345994988,"billing":"api","api_equivalent_usd":0.031515876,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":750378,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[116,148,60,94,96,90,110,50,75,55,54,32,105,57,64,136,30,91,88,143],"started":"2026-10-16T13:30:45Z","updated":"2026-10-16T13:39:57Z","format_of":null,"group":"rstuvw","latency":{"source":"run","runs":[199],"concurrency":1,"median_request_seconds":0.22344711299228948,"p90_request_seconds":0.350752345994988},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":90,"correct":90,"interval":[0.9590629315783191,0.9999999999999999]}}},{"run":1221,"model":"Logits-Sigma-1-Duo","name":"Logits Sigma-1 Duo","variant":null,"reasoning":null,"provider":"di-serve","perplexity_mode":null,"note":"Logits' released Sigma-1 Duo (logitshq/Logits-Sigma-1-Duo at a0c1b6e, the weights and code of d9641cd), two decision models on one GPU whose per-option probabilities it averages 50/50: KnowLine-31B-Gen1 with Logits' LoRA merged, and vLLM-SR d3-ultra. It ran on our hardware in FP8 through its card's own engine and image (plus gcc, libc6-dev and python3-dev, which its Triton kernels need) behind compactor's di_serve.py, text only; the engine took its own Triton FP8 kernel on this GPU.","parameters":57358419036,"active_parameters":null,"effective_parameters":null,"parameters_scope":"both members: KnowLine-31B-Gen1 with Logits' LoRA merged, and d3-ultra","parameters_source":["the Jev Decision Index's served_params for logitshq/Logits-Sigma-1-Duo@a0c1b6e8","https://huggingface.co/logitshq/Logits-Sigma-1-Duo"],"parameters_read":"2026-11-15","access":"Decision Index engine (compactor's di_serve.py around the engine its card ships), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1649,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":603},"grammar":{"questions":1542,"correct":1046},"without_suspect":{"questions":2186,"correct":1646}},"tyt_mix":0.8770555048084526,"questions_per_minute":229.31760251200276,"request_seconds":575.0975876049342,"median_request_seconds":0.25003337149246363,"p90_request_seconds":0.3241096193960402,"billing":"local","api_equivalent_usd":0.39777583142674616,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); FP8","input_tokens":0,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[114,145,63,93,96,92,96,45,65,60,56,35,101,60,66,129,31,89,79,134],"started":"2026-11-15T09:09:11Z","updated":"2026-11-15T09:18:53Z","format_of":null,"group":"suvwx","latency":{"source":"run","runs":[1221],"concurrency":1,"median_request_seconds":0.25003337149246363,"p90_request_seconds":0.3241096193960402},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":42,"correct":42,"interval":[0.9161983874908382,1.0]}}},{"run":318,"model":"qwen38-27b-bf16","name":"Qwen3.8-27B","variant":null,"reasoning":"low","provider":"vllm","perplexity_mode":null,"note":"Open-Jev's base model Reasoning on at the chat template's low effort. Five of the 138 replies never left the thinking part, so their 80 questions have no answer and count as wrong. The GPU was shared with another program during the run, so its request times are longer than on a free GPU.","parameters":27781427952,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head + 0.42B multi-token prediction layer + 0.46B vision encoder","parameters_source":["counted from the checkpoint's safetensors, Qwen/Qwen3.8-27B@1d4bf0f","https://huggingface.co/Qwen/Qwen3.8-27B"],"parameters_read":"2026-11-09","access":"vLLM, self-hosted","questions_per_request":16,"concurrency":4,"correct":1641,"invalid":48,"invalid_replies":{"invalid_without_rule":80,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":5,"asked_again_with_answers":2,"answers_after_asking_again":32,"error_replies_asked_again":0,"invalid_by_kind":{"empty after retry":48}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":586},"grammar":{"questions":1542,"correct":1055},"without_suspect":{"questions":2186,"correct":1639}},"tyt_mix":0.8566973324159312,"questions_per_minute":4.987759002835143,"request_seconds":26440.732185543995,"median_request_seconds":648.3749564884638,"p90_request_seconds":1357.999595651008,"billing":"local","api_equivalent_usd":6.803051,"cost_basis":"maker","cost_upper_bound":false,"requests":143,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":541600,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2177417,"previous":null,"units":[113,148,55,94,86,90,109,52,73,61,63,37,115,59,66,101,20,87,77,135],"started":"2026-10-28T22:04:39Z","updated":"2026-10-29T04:45:15Z","format_of":null,"group":"tuvwxy","latency":{"source":"run","runs":[318],"concurrency":4,"median_request_seconds":648.3749564884638,"p90_request_seconds":1357.999595651008},"single_question_latency":null,"tag":"low · chat","confidence":{"sure":{"questions":2145,"correct":1639,"interval":[0.7456730955128594,0.7815877319022464]}}},{"run":314,"model":"z-ai/glm-5.3","name":"GLM-5.3","variant":null,"reasoning":"low","provider":"openrouter-chat","perplexity_mode":null,"note":"The Fireworks pin admits all of Fireworks' endpoints: OpenRouter billed 135 of the 138 requests at the spot endpoint's rates, 2 at the standard rates and 1 at the fast endpoint's rates ($1.82 in all); the cost shown uses the spot rates.","parameters":753329940480,"active_parameters":null,"effective_parameters":null,"parameters_scope":"mixture of experts: language model with its LM head + one multi-token prediction layer (text only); Z.ai states no active count","parameters_source":["counted from the checkpoint's safetensors, zai-org/GLM-5.3@ba32f8c","https://huggingface.co/zai-org/GLM-5.3","https://docs.z.ai/guides/llm/glm-5.3"],"parameters_read":"2026-11-09","access":"OpenRouter chat API, Fireworks endpoint, fallbacks off","questions_per_request":16,"concurrency":4,"correct":1638,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":575},"grammar":{"questions":1542,"correct":1063},"without_suspect":{"questions":2186,"correct":1636}},"tyt_mix":0.84377140275537,"questions_per_minute":143.52350440961675,"request_seconds":918.8738844030304,"median_request_seconds":17.073907982499804,"p90_request_seconds":54.499747568386375,"billing":"api","api_equivalent_usd":1.7894617860000002,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":557949,"cached_input_tokens":25563,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":410088,"previous":null,"units":[111,144,52,94,86,88,109,42,69,65,62,38,106,52,64,138,26,89,73,130],"started":"2026-10-28T14:52:06Z","updated":"2026-10-28T15:07:25Z","format_of":null,"group":"tuvwxy","latency":{"source":"run","runs":[314],"concurrency":4,"median_request_seconds":17.073907982499804,"p90_request_seconds":54.499747568386375},"single_question_latency":null,"tag":"low · chat","confidence":{"sure":{"questions":1,"correct":1,"interval":[0.20654329147389294,1.0]}}},{"run":1241,"model":"sthanika-ai/Sieve-31B-v3","name":"Sieve 31B v3","variant":null,"reasoning":null,"provider":"sieve","perplexity_mode":null,"note":"sthanika-ai's released Sieve-31B-v3 (sthanika-ai/Sieve-31B-v3 at 8fcee85), a full fine-tune of Gemma-4-31B-it with bidirectional attention in its global layers, read in two option orders. It ran on our hardware (one DGX Spark), one request at a time, through the server its card names (sieve-serve from the Sieve repository's main at 1e64657, on torch 2.14 where the card tested torch 2.8), text only. Probabilities rounded to 4 decimals by its own code; they sum to 1 only within that rounding. Its server reports no token counts. Its card lists English only.","parameters":31273086512,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + 0.58B vision encoder (not read by its server)","parameters_source":["counted from the checkpoint's safetensors, sthanika-ai/Sieve-31B-v3@8fcee85","https://huggingface.co/sthanika-ai/Sieve-31B-v3"],"parameters_read":"2026-11-10","access":"Sieve server (sieve-serve, the maker's release code), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1630,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":602},"grammar":{"questions":1542,"correct":1028},"without_suspect":{"questions":2186,"correct":1628}},"tyt_mix":0.873755081300813,"questions_per_minute":56.41885388222909,"request_seconds":2337.5164670181257,"median_request_seconds":1.0042131584996241,"p90_request_seconds":1.3671181583995349,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":0,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[115,149,60,94,89,95,98,45,70,61,56,40,88,59,62,87,36,102,84,140],"started":"2026-11-15T10:45:17Z","updated":"2026-11-15T11:24:23Z","format_of":null,"group":"vwxy","latency":{"source":"run","runs":[1241],"concurrency":1,"median_request_seconds":1.0042131584996241,"p90_request_seconds":1.3671181583995349},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":329,"correct":328,"interval":[0.9829860458610046,0.9994632633599582]}}},{"run":119,"model":"gemma-4-31B-it","name":"Gemma 4 31B","variant":null,"reasoning":"off","provider":"vllm","perplexity_mode":null,"note":null,"parameters":31273086512,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + 0.58B vision encoder","parameters_source":["counted from the checkpoint's safetensors, google/gemma-4-31B-it@842da37","https://huggingface.co/google/gemma-4-31B-it"],"parameters_read":"2026-11-09","access":"vLLM, self-hosted, FP8","questions_per_request":16,"concurrency":1,"correct":1607,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":596},"grammar":{"questions":1542,"correct":1011},"without_suspect":{"questions":2186,"correct":1604}},"tyt_mix":0.8642800370124324,"questions_per_minute":245.32171674964297,"request_seconds":537.5798023400712,"median_request_seconds":3.972206312988419,"p90_request_seconds":4.256375251116697,"billing":"local","api_equivalent_usd":0.10598339999999999,"cost_basis":"weights","cost_upper_bound":false,"requests":138,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); FP8 weights and KV cache","input_tokens":489825,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":20046,"previous":null,"units":[113,144,62,95,89,93,94,42,69,57,54,38,93,57,58,109,38,98,82,122],"started":"2026-10-05T18:23:31Z","updated":"2026-10-05T18:32:29Z","format_of":null,"group":"wxy","latency":{"source":"run","runs":[119],"concurrency":1,"median_request_seconds":3.972206312988419,"p90_request_seconds":4.256375251116697},"single_question_latency":{"run":136,"questions":200,"median_request_seconds":0.29488859651610255,"p90_request_seconds":0.30137023626593873},"tag":"reasoning off · chat","confidence":{"sure":{"questions":1782,"correct":1450,"interval":[0.7949466515577485,0.831088714624784]}}},{"run":315,"model":"moonshotai/kimi-k3.1-pro","name":"Kimi K3.1 Pro","variant":null,"reasoning":"none","provider":"openrouter-chat","perplexity_mode":null,"note":null,"parameters":2800000000000,"active_parameters":[104000000000],"effective_parameters":null,"parameters_scope":"mixture of experts, as the model card states it","parameters_source":["https://huggingface.co/moonshotai/Kimi-K3.1-Pro"],"parameters_read":"2026-11-09","access":"OpenRouter chat API, Moonshot AI's endpoint (mxfp4), fallbacks off","questions_per_request":16,"concurrency":4,"correct":1598,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":607},"grammar":{"questions":1542,"correct":991},"without_suspect":{"questions":2186,"correct":1593}},"tyt_mix":0.8758440989687134,"questions_per_minute":625.5590601469394,"request_seconds":210.8194228200009,"median_request_seconds":5.745670074480586,"p90_request_seconds":7.645184530690313,"billing":"api","api_equivalent_usd":1.875844,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":778642,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":31856,"previous":null,"units":[119,148,58,96,91,95,94,36,69,51,51,35,89,61,58,111,24,97,76,139],"started":"2026-10-28T14:52:06Z","updated":"2026-10-28T14:55:37Z","format_of":null,"group":"xyz","latency":{"source":"run","runs":[315],"concurrency":4,"median_request_seconds":5.745670074480586,"p90_request_seconds":7.645184530690313},"single_question_latency":null,"tag":"no reasoning · chat","confidence":{"sure":{"questions":442,"correct":406,"interval":[0.8893010060494675,0.9405901481182908]}}},{"run":877,"model":"xiaomi/mimo-v2.6-flash","name":"MiMo-V2.6-Flash","variant":null,"reasoning":"high","provider":"openrouter-chat","perplexity_mode":null,"note":null,"parameters":309000000000,"active_parameters":[15000000000],"effective_parameters":null,"parameters_scope":"mixture-of-experts backbone, as Xiaomi states it","parameters_source":["https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL"],"parameters_read":"2026-11-09","access":"OpenRouter chat API, Xiaomi's own endpoint (fp8), fallbacks off, output cap 65,536 tokens, reasoning included","questions_per_request":1,"concurrency":32,"correct":1585,"invalid":64,"invalid_replies":{"invalid_without_rule":71,"read_from_broken_replies":1,"read_without_confidence":0,"empty_replies_asked_again":12,"asked_again_with_answers":6,"answers_after_asking_again":6,"error_replies_asked_again":0,"invalid_by_kind":{"empty after retry":3,"cut off":61}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":568},"grammar":{"questions":1542,"correct":1017},"without_suspect":{"questions":2186,"correct":1584}},"tyt_mix":0.8297475562304261,"questions_per_minute":7.745055487188042,"request_seconds":17027.637854648994,"median_request_seconds":21.394753752996166,"p90_request_seconds":533.6048188495998,"billing":"api","api_equivalent_usd":4.877592333599999,"cost_basis":"api","cost_upper_bound":false,"requests":2210,"hardware":null,"input_tokens":812421,"cached_input_tokens":198912,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":17111229,"previous":null,"units":[121,137,46,91,84,89,104,52,64,55,66,32,98,58,61,119,24,83,73,128],"started":"2026-11-08T16:39:16Z","updated":"2026-11-08T23:24:00Z","format_of":null,"group":"xyzAB","latency":{"source":"run","runs":[877],"concurrency":32,"median_request_seconds":21.394753752996166,"p90_request_seconds":533.6048188495998},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":18,"correct":17,"interval":[0.7424220019799247,0.9901250641690756]}}},{"run":899,"model":"glide-27b","name":"GLiDE 27B","variant":"local","reasoning":null,"provider":"glide","perplexity_mode":null,"note":"Fastino's released GLiDE 27B (fastino/GLiDE-27B at 4d32743, 5 November 2026), a modified Qwen3.8-27B whose layers 24 to 47 run twice (recurrent depth, its decision_config.json), run on our hardware through the server it ships. The same repository's earlier checkpoint, kept at its tag glide-v1.2, is the Fastino GLiDE v1.2 row.","parameters":26086635760,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 0.46B vision encoder + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, fastino/GLiDE-27B@4d32743","https://huggingface.co/fastino/GLiDE-27B"],"parameters_read":"2026-11-10","access":"GLiDE server (Fastino's release code), self-hosted","questions_per_request":1,"concurrency":1,"correct":1563,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":596},"grammar":{"questions":1542,"correct":967},"without_suspect":{"questions":2186,"correct":1560}},"tyt_mix":0.8592865220967385,"questions_per_minute":106.11907521616403,"request_seconds":1242.7548933248909,"median_request_seconds":0.5415343810018385,"p90_request_seconds":0.6761139269103296,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":789604,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[120,146,60,91,87,92,97,42,62,57,49,32,92,56,53,108,25,85,75,134],"started":"2026-11-10T09:50:02Z","updated":"2026-11-10T10:10:52Z","format_of":null,"group":"yzAC","latency":{"source":"run","runs":[899],"concurrency":1,"median_request_seconds":0.5415343810018385,"p90_request_seconds":0.6761139269103296},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":247,"correct":247,"interval":[0.984685155891208,1.0]}}},{"run":640,"model":"pplx-decider-v1.2","name":"Perplexity Decider 27B v1.2","variant":"with web","reasoning":null,"provider":"perplexity","perplexity_mode":"web_grounding","note":"Perplexity's hosted Decider v1.2 through its Decisions API with web grounding on: a web search per question, its results added as evidence. The search sends the question out, and for some questions it returned pages of the bank's source book (the Web grounding page). Its cost is an upper-bound estimate: one $0.001 search billed per request, while Perplexity bills only completed searches.","parameters":26086635760,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 0.46B vision encoder + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, perplexity-ai/pplx-decider-v1.2@225c913 (the same weights as cce0f3f)","https://huggingface.co/perplexity-ai/pplx-decider-v1.2"],"parameters_read":"2026-11-09","access":"Perplexity Decisions API","questions_per_request":1,"concurrency":1,"correct":1511,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":583},"grammar":{"questions":1542,"correct":928},"without_suspect":{"questions":2186,"correct":1507}},"tyt_mix":0.8385113666919743,"questions_per_minute":106.750165133643,"request_seconds":1235.407924989122,"median_request_seconds":0.4823899704788346,"p90_request_seconds":0.6839786746073514,"billing":"api","api_equivalent_usd":2.27317148,"cost_basis":"api","cost_upper_bound":true,"requests":2198,"hardware":null,"input_tokens":3758574,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[112,144,57,93,86,91,93,40,56,52,49,29,82,52,56,94,33,83,83,126],"started":"2026-11-05T13:23:43Z","updated":"2026-11-05T13:44:26Z","format_of":null,"group":"zABCDE","latency":{"source":"run","runs":[640],"concurrency":1,"median_request_seconds":0.4823899704788346,"p90_request_seconds":0.6839786746073514},"single_question_latency":null,"tag":"API · with web","confidence":{"sure":{"questions":351,"correct":350,"interval":[0.9840405263583258,0.9994969168275918]}}},{"run":248,"model":"glide-v1.2","name":"GLiDE","variant":"local","reasoning":null,"provider":"glide","perplexity_mode":null,"note":"Fastino's released checkpoint, a modified Qwen3.8-27B, run on our hardware through the server it ships. Fastino's hosted GLiDE gave the same answers on 20 sample questions on 22 October 2026; the other GLiDE row is the hosted model as it answered on 6 October 2026, before a change on Fastino's side.","parameters":26086635760,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 0.46B vision encoder + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, fastino/GLiDE@a7e4589","https://fastino.space/blog/glide-decision-model-family"],"parameters_read":"2026-11-09","access":"GLiDE server (Fastino's release code), self-hosted","questions_per_request":1,"concurrency":1,"correct":1505,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":588},"grammar":{"questions":1542,"correct":917},"without_suspect":{"questions":2186,"correct":1501}},"tyt_mix":0.8435510977191484,"questions_per_minute":139.71066492704924,"request_seconds":943.9508434725576,"median_request_seconds":0.41326581648900174,"p90_request_seconds":0.5194234423222952,"billing":"local","api_equivalent_usd":0.03158416,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":789604,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[117,144,60,91,83,93,90,39,56,48,52,28,84,52,53,104,27,80,76,128],"started":"2026-10-22T18:39:36Z","updated":"2026-10-22T18:55:25Z","format_of":null,"group":"zABCD","latency":{"source":"run","runs":[248],"concurrency":1,"median_request_seconds":0.41326581648900174,"p90_request_seconds":0.5194234423222952},"single_question_latency":null,"tag":"v1.2","confidence":{"sure":{"questions":240,"correct":240,"interval":[0.9842455102000642,1.0]}}},{"run":1134,"model":"pplx-decider-v1.3","name":"Perplexity Decider 27B v1.3","variant":"local","reasoning":null,"provider":"pplx-decider","perplexity_mode":null,"note":"Perplexity's released v1.3 checkpoint, the next version of Decider 27B v1.2 on Qwen3.8-27B, run on our hardware through the package it ships.","parameters":26086635760,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + vision encoder + 255-way readout; its locate-span adapter and head (0.47B), used only by its locate task, not counted","parameters_source":["counted from the checkpoint's safetensors, perplexity-ai/pplx-decider-v1.3@9c2ebb2","https://huggingface.co/perplexity-ai/pplx-decider-v1.3"],"parameters_read":"2026-11-09","access":"Perplexity Decider server, self-hosted","questions_per_request":1,"concurrency":1,"correct":1504,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":584},"grammar":{"questions":1542,"correct":920},"without_suspect":{"questions":2186,"correct":1501}},"tyt_mix":0.8388610768403405,"questions_per_minute":511.1864367874537,"request_seconds":257.9880656239602,"median_request_seconds":0.10971509299906756,"p90_request_seconds":0.14523346810037766,"billing":"local","api_equivalent_usd":0.17844174538990581,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":818178,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[113,143,57,93,87,91,88,43,59,50,47,30,88,50,53,111,23,82,72,124],"started":"2026-11-14T16:32:49Z","updated":"2026-11-14T16:37:11Z","format_of":null,"group":"zABCDE","latency":{"source":"run","runs":[1134],"concurrency":1,"median_request_seconds":0.10971509299906756,"p90_request_seconds":0.14523346810037766},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":276,"correct":276,"interval":[0.9862722340066665,1.0]}}},{"run":133,"model":"gpt-6-luna","name":"GPT-6 Luna","variant":null,"reasoning":"low","provider":"openai","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://developers.openai.com/api/docs/models/gpt-6-luna"],"parameters_read":"2026-11-09","access":"Codex CLI, ChatGPT subscription","questions_per_request":16,"concurrency":1,"correct":1501,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":573},"grammar":{"questions":1542,"correct":928},"without_suspect":{"questions":2186,"correct":1499}},"tyt_mix":0.8259351471797792,"questions_per_minute":106.27926346966372,"request_seconds":1240.8817646505777,"median_request_seconds":7.705671253963374,"p90_request_seconds":11.16688626020914,"billing":"subscription","api_equivalent_usd":0.10553408,"cost_basis":"subscription","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1197871,"cached_input_tokens":316928,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":28541,"previous":null,"units":[114,144,52,93,79,91,77,31,64,49,51,36,78,55,55,91,29,109,76,127],"started":"2026-10-06T12:36:07Z","updated":"2026-10-06T13:02:06Z","format_of":null,"group":"ABCDEF","latency":{"source":"run","runs":[133],"concurrency":1,"median_request_seconds":7.705671253963374,"p90_request_seconds":11.16688626020914},"single_question_latency":null,"tag":"low · chat","confidence":{"sure":{"questions":830,"correct":693,"interval":[0.8081516217664547,0.8586416876582609]}}},{"run":1347,"model":"d1.2-ultra","name":"d1.2-ultra","variant":null,"reasoning":null,"provider":"liquid","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.liquid.ai/blog/d1-2"],"parameters_read":"2026-11-16","access":"Liquid AI decisions API","questions_per_request":1,"concurrency":1,"correct":1497,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":589},"grammar":{"questions":1542,"correct":908},"without_suspect":{"questions":2186,"correct":1494}},"tyt_mix":0.8437873188921579,"questions_per_minute":236.35157619949678,"request_seconds":557.9823165159869,"median_request_seconds":0.21963629050151212,"p90_request_seconds":0.32078671869894604,"billing":"api","api_equivalent_usd":0.031515876,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":750378,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[114,142,60,95,87,91,87,36,61,50,50,28,82,55,52,102,29,82,73,121],"started":"2026-11-16T18:51:19Z","updated":"2026-11-16T19:00:41Z","format_of":null,"group":"ABCDEG","latency":{"source":"run","runs":[1347],"concurrency":1,"median_request_seconds":0.21963629050151212,"p90_request_seconds":0.32078671869894604},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":11,"correct":11,"interval":[0.7411599827511859,1.0]}}},{"run":129,"model":"fastino/GLiDE","name":"GLiDE","variant":null,"reasoning":null,"provider":"fastino","perplexity_mode":null,"note":"Hosted decision model; Fastino describes it as reasoning on uncertain decisions","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://docs.fastino.ai/inference/systemone","https://fastino.ai/blog/introducing-glide-the-first-thinking-decision-model"],"parameters_read":"2026-11-09","access":"Fastino System One API","questions_per_request":1,"concurrency":4,"correct":1492,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":576},"grammar":{"questions":1542,"correct":916},"without_suspect":{"questions":2186,"correct":1489}},"tyt_mix":0.8283461453291576,"questions_per_minute":152.131021033152,"request_seconds":866.8843415654264,"median_request_seconds":2.088375358027406,"p90_request_seconds":2.2788841759203935,"billing":"api","api_equivalent_usd":0.35185395,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":2345693,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":558873,"previous":null,"units":[108,144,58,89,88,89,87,40,59,50,49,31,98,51,54,100,25,80,64,128],"started":"2026-10-06T10:53:16Z","updated":"2026-10-06T11:07:43Z","format_of":null,"group":"ABCDEFGH","latency":{"source":"run","runs":[129],"concurrency":4,"median_request_seconds":2.088375358027406,"p90_request_seconds":2.2788841759203935},"single_question_latency":null,"tag":"API","confidence":{"sure":{"questions":121,"correct":121,"interval":[0.969228205982621,0.9999999999999999]}}},{"run":259,"model":"vllm-sr/d3","name":"d3","variant":null,"reasoning":null,"provider":"vllm-sr","perplexity_mode":null,"note":"The vLLM Semantic Router's released Decision 3.0 checkpoint (vllm-sr/d3 at 20f37b9), fine-tuned from Qwen3.8-27B (Qwen3.5 architecture), served on our hardware through vLLM's System One endpoint by a nightly build (0.32.1rc1.dev613+g8977b13a1) with --max-num-seqs 256: with the default, vLLM's startup memory check did not fit the GPU. One request at a time, as for the other setups.","parameters":26086635760,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 0.46B vision encoder + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, vllm-sr/d3@20f37b9","https://huggingface.co/vllm-sr/d3"],"parameters_read":"2026-11-09","access":"vLLM System One endpoint (nightly), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1490,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":585},"grammar":{"questions":1542,"correct":905},"without_suspect":{"questions":2186,"correct":1486}},"tyt_mix":0.8384163641612098,"questions_per_minute":591.0559151650041,"request_seconds":223.12609791441355,"median_request_seconds":0.0924945070000831,"p90_request_seconds":0.14396354870405048,"billing":"local","api_equivalent_usd":0.15432888439080272,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":864336,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[112,142,57,94,89,91,92,39,57,51,49,29,85,50,53,100,24,79,72,125],"started":"2026-10-27T09:16:30Z","updated":"2026-10-27T09:20:19Z","format_of":null,"group":"BDEG","latency":{"source":"run","runs":[259],"concurrency":1,"median_request_seconds":0.0924945070000831,"p90_request_seconds":0.14396354870405048},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":250,"correct":250,"interval":[0.9848661527503766,1.0]}}},{"run":269,"model":"vllm-sr/d3-ultra","name":"d3-ultra","variant":"transformers, rev 52ba9e7","reasoning":null,"provider":"d3-server","perplexity_mode":null,"note":"The vLLM Semantic Router's released d3-ultra checkpoint (vllm-sr/d3-ultra at 52ba9e7), fine-tuned from Qwen3.8-27B (Qwen3.5 architecture), served on our hardware through the server its repository ships (d3_server.py, the package's transformers runtime as in the model card's Quickstart; transformers 5.17.0). vLLM cannot serve it yet: it rejects the checkpoint's attention mode (prefix_bidir). One request at a time, as for the other setups.","parameters":26086635760,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 0.46B vision encoder + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, vllm-sr/d3-ultra@52ba9e7","https://huggingface.co/vllm-sr/d3-ultra"],"parameters_read":"2026-11-09","access":"d3 package server (transformers, the release code), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1490,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":589},"grammar":{"questions":1542,"correct":901},"without_suspect":{"questions":2186,"correct":1488}},"tyt_mix":0.842992896064661,"questions_per_minute":192.26901396874433,"request_seconds":685.9139560649055,"median_request_seconds":0.3334329740027897,"p90_request_seconds":0.4403526291134767,"billing":"local","api_equivalent_usd":0.47442381961155966,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":1126925,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[112,143,58,93,91,92,91,35,55,49,49,27,85,56,59,98,21,81,72,123],"started":"2026-10-27T14:45:17Z","updated":"2026-10-27T14:56:49Z","format_of":null,"group":"BDEG","latency":{"source":"run","runs":[269],"concurrency":1,"median_request_seconds":0.3334329740027897,"p90_request_seconds":0.4403526291134767},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":384,"correct":383,"interval":[0.9853980164858338,0.999540164462265]}}},{"run":322,"model":"drex-max-v1","name":"Drex Max","variant":null,"reasoning":null,"provider":"drex-max","perplexity_mode":null,"note":"Nace.AI's released Drex Max weights (nace-ai/drex-max-v1 at cac65e1), the original Drex Max, a fine-tune of Qwen3.8-27B that also reads images. It ran on our hardware through Nace's own loader (drex_max, from nace-ai/drex-decision-models at e206b27), which makes 16 of the 64 layers attend non-causally, behind our HTTP server (servers/drex_max_server.py). One request at a time, as for the other setups.","parameters":26086635760,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 0.46B vision encoder + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, nace-ai/drex-max-v1@cac65e1","https://huggingface.co/nace-ai/drex-max-v1"],"parameters_read":"2026-11-09","access":"Drex Max server (Nace's loader, drex_max), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1489,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":588},"grammar":{"questions":1542,"correct":901},"without_suspect":{"questions":2186,"correct":1485}},"tyt_mix":0.8417352741134415,"questions_per_minute":673.5294239576491,"request_seconds":195.804363267569,"median_request_seconds":0.08219127899792511,"p90_request_seconds":0.11037726640497567,"billing":"local","api_equivalent_usd":0.05120076,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":853346,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[110,143,57,92,93,93,93,39,58,50,51,28,75,50,55,102,25,79,72,124],"started":"2026-10-31T15:17:59Z","updated":"2026-10-31T15:21:22Z","format_of":null,"group":"BDEG","latency":{"source":"run","runs":[322],"concurrency":1,"median_request_seconds":0.08219127899792511,"p90_request_seconds":0.11037726640497567},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":443,"correct":442,"interval":[0.9873254872500569,0.9996014237627048]}}},{"run":326,"model":"drex-max-v1.1","name":"Drex Max v1.1","variant":null,"reasoning":null,"provider":"nace","perplexity_mode":null,"note":"Nace.AI's hosted Drex Max v1.1 (drex-max-v1.1, released 28 October 2026), a newer checkpoint than the open Drex Max v1 weights; Nace publishes no weights for it. Nace bills $0.03 per million input tokens during its launch, half the $0.06 list price the cost here uses.","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://console.nace.ai/docs/drex/reference/models","https://console.nace.ai/docs/drex/guides/drex-max"],"parameters_read":"2026-11-09","access":"Nace.AI System One API","questions_per_request":1,"concurrency":1,"correct":1481,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":581},"grammar":{"questions":1542,"correct":900},"without_suspect":{"questions":2186,"correct":1478}},"tyt_mix":0.8328184314795483,"questions_per_minute":118.9534211632787,"request_seconds":1108.6692480998754,"median_request_seconds":0.3127196345012635,"p90_request_seconds":0.448678748808743,"billing":"api","api_equivalent_usd":0.050805119999999995,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":846752,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":357765,"previous":null,"units":[111,144,59,92,86,89,92,37,57,49,51,30,78,45,56,102,26,82,72,123],"started":"2026-10-31T15:41:43Z","updated":"2026-10-31T16:00:19Z","format_of":null,"group":"DEGI","latency":{"source":"run","runs":[326],"concurrency":1,"median_request_seconds":0.3127196345012635,"p90_request_seconds":0.448678748808743},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":220,"correct":220,"interval":[0.9828378639180564,1.0]}}},{"run":374,"model":"baselabs/mammoth-1","name":"Mammoth-1","variant":null,"reasoning":null,"provider":"baselabs","perplexity_mode":null,"note":"Base Labs' released mammoth-1 weights (baselabs/mammoth-1 at 61cec44), a decision model post-trained from Qwen3.5-27B that also reads images. It ran on our hardware through its model card's loader (qev's DecisionEngine from OneJev at 81ce62f with the checkpoint's serve.patch, as the card's setup.sh installs it) behind our HTTP server (servers/baselabs_server.py), one request at a time. That runtime reads images only through an engine the patch leaves without the attention the model was trained with, so it is not on the Page images tab. The card mentions a trained \"I don't know\" option, but its answers have no such output, and our questions offer five options. Probabilities rounded to 6 decimals by the model's own loader. They sum to 1 only within that rounding, and the check before its runs allowed for it. Its card lists English only.","parameters":27356728560,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head + 0.46B vision encoder","parameters_source":["counted from the checkpoint's safetensors, baselabs/mammoth-1@61cec44","https://huggingface.co/baselabs/mammoth-1"],"parameters_read":"2026-11-09","access":"Base Labs server (qev, the model card's loader), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1475,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":593},"grammar":{"questions":1542,"correct":882},"without_suspect":{"questions":2186,"correct":1472}},"tyt_mix":0.8458670933377622,"questions_per_minute":355.75443930545117,"request_seconds":370.70514216905576,"median_request_seconds":0.1691462635062635,"p90_request_seconds":0.19061791689891833,"billing":"local","api_equivalent_usd":0.087093,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":870930,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[111,142,58,94,92,96,97,38,53,42,51,34,76,52,44,88,30,77,75,125],"started":"2026-11-01T18:14:08Z","updated":"2026-11-01T18:20:26Z","format_of":null,"group":"DEFGHIJ","latency":{"source":"run","runs":[374],"concurrency":1,"median_request_seconds":0.1691462635062635,"p90_request_seconds":0.19061791689891833},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":285,"correct":285,"interval":[0.9866999767346533,1.0]}}},{"run":217,"model":"pplx-decider-v1.2","name":"Perplexity Decider 27B v1.2","variant":"local","reasoning":null,"provider":"pplx-decider","perplexity_mode":null,"note":"Perplexity's released checkpoint, run on our hardware. Its run through Perplexity's Decisions API is a setup of its own.","parameters":26086635760,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 0.46B vision encoder + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, perplexity-ai/pplx-decider-v1.2@225c913 (the same weights as cce0f3f)","https://huggingface.co/perplexity-ai/pplx-decider-v1.2"],"parameters_read":"2026-11-09","access":"Perplexity Decider server, self-hosted","questions_per_request":1,"concurrency":1,"correct":1474,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":585},"grammar":{"questions":1542,"correct":889},"without_suspect":{"questions":2186,"correct":1471}},"tyt_mix":0.8366005405555029,"questions_per_minute":97.394776360673,"request_seconds":1354.0767269860662,"median_request_seconds":0.5795541804982349,"p90_request_seconds":0.7866766697028652,"billing":"local","api_equivalent_usd":0.017066920000000003,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":846752,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[109,144,59,92,90,91,93,38,55,47,49,28,79,49,51,97,21,81,74,127],"started":"2026-10-17T18:04:03Z","updated":"2026-10-17T18:26:45Z","format_of":null,"group":"DEGI","latency":{"source":"run","runs":[217],"concurrency":1,"median_request_seconds":0.5795541804982349,"p90_request_seconds":0.7866766697028652},"single_question_latency":null,"tag":"DGX Spark","confidence":{"sure":{"questions":360,"correct":360,"interval":[0.9894415591839967,1.0]}}},{"run":145,"model":"pplx-decider-v1.1-27b","name":"Perplexity Decider 27B v1.1","variant":null,"reasoning":null,"provider":"perplexity","perplexity_mode":null,"note":null,"parameters":26086635760,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 0.46B vision encoder + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, perplexity-ai/pplx-decider-v1.1-27b@5cd25e3","https://huggingface.co/perplexity-ai/pplx-decider-v1.1-27b"],"parameters_read":"2026-11-09","access":"Perplexity Decisions API","questions_per_request":1,"concurrency":1,"correct":1470,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":584},"grammar":{"questions":1542,"correct":886},"without_suspect":{"questions":2186,"correct":1467}},"tyt_mix":0.8350024516782133,"questions_per_minute":230.1414509049513,"request_seconds":573.0388831800083,"median_request_seconds":0.23294744099257514,"p90_request_seconds":0.29029369752388445,"billing":"api","api_equivalent_usd":0.017066920000000003,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":853346,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[110,143,59,93,88,91,89,38,59,47,50,27,77,53,52,93,19,83,73,126],"started":"2026-10-09T08:15:06Z","updated":"2026-10-09T08:24:44Z","format_of":null,"group":"DEFGHIJ","latency":{"source":"run","runs":[145],"concurrency":1,"median_request_seconds":0.23294744099257514,"p90_request_seconds":0.29029369752388445},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":349,"correct":349,"interval":[0.9891123949103505,0.9999999999999999]}}},{"run":189,"model":"gemma-4-31B-it","name":"Gemma 4 31B","variant":null,"reasoning":"off","provider":"vllm","perplexity_mode":null,"note":null,"parameters":31273086512,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + 0.58B vision encoder","parameters_source":["counted from the checkpoint's safetensors, google/gemma-4-31B-it@842da37","https://huggingface.co/google/gemma-4-31B-it"],"parameters_read":"2026-11-09","access":"vLLM, self-hosted, FP8","questions_per_request":1,"concurrency":1,"correct":1468,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":584},"grammar":{"questions":1542,"correct":884},"without_suspect":{"questions":2186,"correct":1464}},"tyt_mix":0.8347754737275,"questions_per_minute":195.4762313352229,"request_seconds":674.6600295042444,"median_request_seconds":0.29770804748113733,"p90_request_seconds":0.3055964823171962,"billing":"local","api_equivalent_usd":0.14987080000000003,"cost_basis":"weights","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); FP8 weights and KV cache","input_tokens":698986,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":25184,"previous":null,"units":[113,141,61,94,84,91,91,34,60,45,48,33,70,48,43,88,32,93,74,125],"started":"2026-10-13T16:20:09Z","updated":"2026-10-13T19:45:13Z","format_of":119,"group":"CDEFGHIJK","latency":{"source":"run","runs":[189],"concurrency":1,"median_request_seconds":0.29770804748113733,"p90_request_seconds":0.3055964823171962},"single_question_latency":null,"tag":"reasoning off · chat · 1 per request","confidence":{"sure":{"questions":1826,"correct":1339,"interval":[0.7125381376763197,0.7530759351483446]}}},{"run":862,"model":"minimax/minimax-m3.1-flash","name":"MiniMax-M3.1-Flash","variant":null,"reasoning":"high","provider":"openrouter-chat","perplexity_mode":null,"note":null,"parameters":428000000000,"active_parameters":[23000000000],"effective_parameters":null,"parameters_scope":"mixture of experts, as MiniMax states it","parameters_source":["https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash"],"parameters_read":"2026-11-09","access":"OpenRouter chat API, MiniMax's own endpoint, fallbacks off, output cap 65,536 tokens, reasoning included; JSON mode without the answer schema (the endpoint has no structured outputs), so the prompt adds a paragraph stating the answer layout","questions_per_request":16,"concurrency":4,"correct":1461,"invalid":2,"invalid_replies":{"invalid_without_rule":2,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"letter not allowed":2}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":544},"grammar":{"questions":1542,"correct":917},"without_suspect":{"questions":2186,"correct":1461}},"tyt_mix":0.7882157318654899,"questions_per_minute":26.683641866992506,"request_seconds":4942.353845752019,"median_request_seconds":112.64711910600454,"p90_request_seconds":279.2127233066043,"billing":"api","api_equivalent_usd":1.788733575,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":704202,"cached_input_tokens":22995,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2810346,"previous":null,"units":[96,140,50,90,78,90,98,44,63,56,56,35,79,58,62,106,20,71,66,103],"started":"2026-11-08T15:09:36Z","updated":"2026-11-08T16:31:59Z","format_of":null,"group":"CDEFGHIJKL","latency":{"source":"run","runs":[862],"concurrency":4,"median_request_seconds":112.64711910600454,"p90_request_seconds":279.2127233066043},"single_question_latency":null,"tag":"high · chat","confidence":{"sure":{"questions":375,"correct":321,"interval":[0.8168516425679543,0.8879284027069047]}}},{"run":633,"model":"pplx-decider-v1.2","name":"Perplexity Decider 27B v1.2","variant":null,"reasoning":null,"provider":"perplexity","perplexity_mode":null,"note":"Perplexity's hosted Decider v1.2 through its Decisions API, web grounding off. The other Decider v1.2 rows are Perplexity's released checkpoint on our hardware and this model with web grounding on.","parameters":26086635760,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 0.46B vision encoder + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, perplexity-ai/pplx-decider-v1.2@225c913 (the same weights as cce0f3f)","https://huggingface.co/perplexity-ai/pplx-decider-v1.2"],"parameters_read":"2026-11-09","access":"Perplexity Decisions API","questions_per_request":1,"concurrency":1,"correct":1458,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":584},"grammar":{"questions":1542,"correct":874},"without_suspect":{"questions":2186,"correct":1455}},"tyt_mix":0.8336405839739331,"questions_per_minute":176.46305964009935,"request_seconds":747.3518835555296,"median_request_seconds":0.2536797354987357,"p90_request_seconds":0.4693139127048198,"billing":"api","api_equivalent_usd":0.016935040000000002,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":846752,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[111,144,59,92,87,91,93,37,57,46,46,30,74,46,53,94,22,80,74,122],"started":"2026-11-05T11:55:44Z","updated":"2026-11-05T12:08:19Z","format_of":null,"group":"DEFGHIJK","latency":{"source":"run","runs":[633],"concurrency":1,"median_request_seconds":0.2536797354987357,"p90_request_seconds":0.4693139127048198},"single_question_latency":null,"tag":"API","confidence":{"sure":{"questions":327,"correct":327,"interval":[0.9883884009749682,1.0]}}},{"run":558,"model":"deepseek-v4.1-flash-1017","name":"DeepSeek-V4.1-Flash","variant":null,"reasoning":"off","provider":"vllm","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":"V4.1 Flash card (first release): 552B backbone + 196B conditional memory; 8B active in prefill, 16B in decode; the 1017 card states no size","parameters_source":["https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017","https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash"],"parameters_read":"2026-11-09","access":"SGLang, self-hosted, published mixed precision; 1017 weights (revision 1bae0e1)","questions_per_request":16,"concurrency":1,"correct":1451,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":579},"grammar":{"questions":1542,"correct":872},"without_suspect":{"questions":2186,"correct":1449}},"tyt_mix":0.8271254962671222,"questions_per_minute":147.28194712194326,"request_seconds":895.4254243448377,"median_request_seconds":5.080297439504648,"p90_request_seconds":11.498785002605292,"billing":"local","api_equivalent_usd":0.21643859999999998,"cost_basis":"maker","cost_upper_bound":false,"requests":138,"hardware":"4 × RTX PRO 6000 Blackwell (96 GB); FP4 experts, FP8 and BF16 layers","input_tokens":665866,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":13899,"previous":null,"units":[111,141,55,96,86,90,86,35,60,46,40,30,71,53,47,77,32,101,68,126],"started":"2026-11-05T02:19:23Z","updated":"2026-11-05T02:34:20Z","format_of":null,"group":"DEFGHIJKLM","latency":{"source":"run","runs":[558],"concurrency":1,"median_request_seconds":5.080297439504648,"p90_request_seconds":11.498785002605292},"single_question_latency":null,"tag":"reasoning off · chat · 1017","confidence":{"sure":{"questions":696,"correct":680,"interval":[0.9629849624237918,0.9858011628937087]}}},{"run":120,"model":"deepseek-v4.1-flash","name":"DeepSeek-V4.1-Flash","variant":null,"reasoning":"off","provider":"vllm","perplexity_mode":null,"note":null,"parameters":552000000000,"active_parameters":[8000000000,16000000000],"effective_parameters":null,"parameters_scope":"backbone, as DeepSeek states it (8B active in prefill, 16B in decode); the 196B Engram memory is separate","parameters_source":["https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash"],"parameters_read":"2026-11-09","access":"SGLang, self-hosted, published mixed precision; first-release weights (revision fb2764a), not the 1017 update","questions_per_request":16,"concurrency":1,"correct":1449,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":576},"grammar":{"questions":1542,"correct":873},"without_suspect":{"questions":2186,"correct":1447}},"tyt_mix":0.8234661193888204,"questions_per_minute":177.54848553740285,"request_seconds":742.783018401009,"median_request_seconds":4.361823945015203,"p90_request_seconds":7.860654652415542,"billing":"local","api_equivalent_usd":0.2164134,"cost_basis":"weights","cost_upper_bound":false,"requests":138,"hardware":"4 × RTX PRO 6000 Blackwell (96 GB); FP4 experts, FP8 and BF16 layers","input_tokens":665866,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":13878,"previous":null,"units":[111,140,56,94,85,90,84,34,60,46,46,28,69,53,46,82,31,102,65,127],"started":"2026-10-05T18:49:31Z","updated":"2026-10-05T19:01:55Z","format_of":null,"group":"DEFGHIJKLM","latency":{"source":"run","runs":[120],"concurrency":1,"median_request_seconds":4.361823945015203,"p90_request_seconds":7.860654652415542},"single_question_latency":null,"tag":"reasoning off · chat · first release","confidence":{"sure":{"questions":576,"correct":571,"interval":[0.9798419495618019,0.9962867035048287]}}},{"run":177,"model":"subq-snap-preview","name":"SubQ Snap Preview","variant":null,"reasoning":null,"provider":"subq","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://docs.subq.ai/using-the-api/models/"],"parameters_read":"2026-11-09","access":"SubQ Decisions API (public preview)","questions_per_request":1,"concurrency":1,"correct":1434,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":575},"grammar":{"questions":1542,"correct":859},"without_suspect":{"questions":2186,"correct":1430}},"tyt_mix":0.8206196517826074,"questions_per_minute":150.8851709604276,"request_seconds":874.042155107396,"median_request_seconds":0.38925252700573765,"p90_request_seconds":0.4138357652031118,"billing":"api","api_equivalent_usd":0.042667300000000005,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":853346,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[106,146,53,90,87,93,86,35,52,49,48,25,79,50,48,95,20,74,71,127],"started":"2026-10-13T06:41:20Z","updated":"2026-10-13T06:55:58Z","format_of":null,"group":"EFGHIJKLMN","latency":{"source":"run","runs":[177],"concurrency":1,"median_request_seconds":0.38925252700573765,"p90_request_seconds":0.4138357652031118},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":346,"model":"subq-snap-1.1","name":"SubQ Snap 1.1","variant":null,"reasoning":null,"provider":"subq","perplexity_mode":null,"note":"SubQ's subq-snap-1.1, in public preview, through its Decisions API, which also takes images. Its option probabilities arrive as 32-bit floats, so they sum to 1 only within 32-bit rounding. Its probability check used an experimental 32-bit rule of ours. For this model only, the check before its runs accepted a sum within one 32-bit epsilon (2^-23) per option when every value is an exact 32-bit float, instead of 7 decimal places; this is our tolerance, not an error bound SubQ states. The values are stored as SubQ returned them. It was paused twice while the approval of its probability check was confirmed, for 12 minutes and then for 3 hours 24 minutes, and resumed from its stored answers. The two requests in flight count those pauses in their times (706 and 12,236 seconds; the first after one retry, when SubQ had closed the connection), so the run's total request time and questions per minute include them. Without them, its median request time stays 0.378 s and its 90th percentile moves from 0.388 s to 0.387 s.","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://docs.subq.ai/using-the-api/models/"],"parameters_read":"2026-11-09","access":"SubQ Decisions API (public preview)","questions_per_request":1,"concurrency":1,"correct":1404,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":571},"grammar":{"questions":1542,"correct":833},"without_suspect":{"questions":2186,"correct":1401}},"tyt_mix":0.8126384506184555,"questions_per_minute":9.571646632418748,"request_seconds":13778.193561108725,"median_request_seconds":0.378216398996301,"p90_request_seconds":0.38751173970231323,"billing":"api","api_equivalent_usd":0.033870080000000004,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":846752,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[108,142,55,90,85,91,90,36,50,46,47,24,73,44,48,90,15,79,70,121],"started":"2026-11-01T15:44:28Z","updated":"2026-11-01T19:34:16Z","format_of":null,"group":"FHJKLMNO","latency":{"source":"run","runs":[346],"concurrency":1,"median_request_seconds":0.378216398996301,"p90_request_seconds":0.38751173970231323},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":299,"correct":298,"interval":[0.9813013202671312,0.9994093879050621]}}},{"run":143,"model":"gpt-6-luna","name":"GPT-6 Luna","variant":"Decisions API, beta","reasoning":null,"provider":"openai-decisions","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://developers.openai.com/api/docs/models/gpt-6-luna"],"parameters_read":"2026-11-09","access":"OpenAI Decisions API (public beta)","questions_per_request":1,"concurrency":4,"correct":1398,"invalid":4,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":561},"grammar":{"questions":1542,"correct":837},"without_suspect":{"questions":2186,"correct":1396}},"tyt_mix":0.8005161870076872,"questions_per_minute":1214.8216473277553,"request_seconds":108.55914552567992,"median_request_seconds":0.19129549904027954,"p90_request_seconds":0.24994003138272092,"billing":"api","api_equivalent_usd":0.08887830000000001,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":888783,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[114,136,51,93,79,88,70,27,60,45,38,31,74,48,49,80,33,101,62,119],"started":"2026-10-08T12:06:17Z","updated":"2026-10-08T12:08:06Z","format_of":null,"group":"GHIJKLMNOP","latency":{"source":"run","runs":[143],"concurrency":4,"median_request_seconds":0.19129549904027954,"p90_request_seconds":0.24994003138272092},"single_question_latency":null,"tag":"Decisions API","confidence":{"sure":{"questions":368,"correct":365,"interval":[0.9763090368954989,0.9972237792283561]}}},{"run":985,"model":"qwen/qwen3.8-flash","name":"Qwen3.8 Flash","variant":null,"reasoning":"none","provider":"openrouter-chat","perplexity_mode":null,"note":"Alibaba's closed chat model (API id qwen3.8-flash), added on the reading bench as on backgammon and Logic & Math","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.alibabacloud.com/help/en/model-studio/model-pricing","https://openrouter.ai/qwen/qwen3.8-flash"],"parameters_read":"2026-11-11","access":"OpenRouter chat API, Alibaba's endpoint, fallbacks off, reasoning off","questions_per_request":16,"concurrency":4,"correct":1386,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":572},"grammar":{"questions":1542,"correct":814},"without_suspect":{"questions":2186,"correct":1384}},"tyt_mix":0.8117397820378981,"questions_per_minute":384.642712069552,"request_seconds":342.8636390650063,"median_request_seconds":10.045079395000357,"p90_request_seconds":12.072642541196547,"billing":"api","api_equivalent_usd":0.10358409999999998,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":521993,"cached_input_tokens":3840,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":54893,"previous":null,"units":[112,139,58,92,79,92,73,29,61,44,38,24,71,50,55,77,25,76,68,123],"started":"2026-11-11T15:06:48Z","updated":"2026-11-11T15:12:31Z","format_of":null,"group":"IJKLMNOP","latency":{"source":"run","runs":[985],"concurrency":4,"median_request_seconds":10.045079395000357,"p90_request_seconds":12.072642541196547},"single_question_latency":null,"tag":"no reasoning · chat","confidence":{"sure":{"questions":55,"correct":46,"interval":[0.717360036126882,0.9114467944147754]}}},{"run":1309,"model":"PelaAI/KnowLine-12B-Gen1","name":"KnowLine-12B-Gen1","variant":null,"reasoning":null,"provider":"knowline","perplexity_mode":null,"note":"PelaAI's released checkpoint (13 November 2026), a fine-tune of Gemma-4-12B-it, run on our hardware through its card's serve_knowline.sh (SGLang 0.5.23 in the lmsysorg/sglang image; the card's launcher names 0.5.21, then knowline_server.py), text only.","parameters":11959730224,"active_parameters":null,"effective_parameters":null,"parameters_scope":"the whole checkpoint, its vision encoder included (text only here); the card says 12B parameters","parameters_source":["counted from the checkpoint's safetensors, PelaAI/KnowLine-12B-Gen1@f26fbc2","https://huggingface.co/PelaAI/KnowLine-12B-Gen1"],"parameters_read":"2026-11-15","access":"KnowLine server (knowline_server.py over SGLang, PelaAI's release code), self-hosted","questions_per_request":1,"concurrency":1,"correct":1373,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":564},"grammar":{"questions":1542,"correct":809},"without_suspect":{"questions":2186,"correct":1372}},"tyt_mix":0.8011113615513586,"questions_per_minute":793.7723429939449,"request_seconds":166.14335478428984,"median_request_seconds":0.07107090100180358,"p90_request_seconds":0.09740792851662264,"billing":"local","api_equivalent_usd":0.11491582039246714,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":1619544,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[107,145,56,92,77,87,75,34,65,47,43,25,72,45,43,89,30,72,58,111],"started":"2026-11-16T10:34:53Z","updated":"2026-11-16T11:07:18Z","format_of":null,"group":"KLMNOPQ","latency":{"source":"run","runs":[1309],"concurrency":1,"median_request_seconds":0.07107090100180358,"p90_request_seconds":0.09740792851662264},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1346,"model":"d1.2","name":"d1.2","variant":null,"reasoning":null,"provider":"liquid","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.liquid.ai/blog/d1-2"],"parameters_read":"2026-11-16","access":"Liquid AI decisions API","questions_per_request":1,"concurrency":1,"correct":1357,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":564},"grammar":{"questions":1542,"correct":793},"without_suspect":{"questions":2186,"correct":1357}},"tyt_mix":0.7992955379456518,"questions_per_minute":242.46155535738333,"request_seconds":543.9212818939959,"median_request_seconds":0.21261778850021074,"p90_request_seconds":0.3196910808004759,"billing":"api","api_equivalent_usd":0.01500756,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":750378,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[104,143,52,92,85,88,72,33,51,45,45,24,81,41,37,86,22,68,73,115],"started":"2026-11-16T18:51:18Z","updated":"2026-11-16T19:00:26Z","format_of":null,"group":"LMNOPQR","latency":{"source":"run","runs":[1346],"concurrency":1,"median_request_seconds":0.21261778850021074,"p90_request_seconds":0.3196910808004759},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":33,"correct":33,"interval":[0.8957265699643882,1.0]}}},{"run":254,"model":"d1.1","name":"d1.1","variant":null,"reasoning":null,"provider":"liquid","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://docs.liquid.ai/lfm/models/decision-models"],"parameters_read":"2026-11-09","access":"Liquid AI decisions API","questions_per_request":1,"concurrency":1,"correct":1352,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":555},"grammar":{"questions":1542,"correct":797},"without_suspect":{"questions":2186,"correct":1351}},"tyt_mix":0.7884308962861029,"questions_per_minute":274.85478875697214,"request_seconds":479.81699935600864,"median_request_seconds":0.19458774350005115,"p90_request_seconds":0.2346541320999677,"billing":"api","api_equivalent_usd":0.01500756,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":750378,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[101,141,53,93,80,87,70,31,53,46,41,23,83,43,43,90,25,67,70,112],"started":"2026-10-24T07:34:52Z","updated":"2026-10-24T07:42:55Z","format_of":null,"group":"MNOPQRS","latency":{"source":"run","runs":[254],"concurrency":1,"median_request_seconds":0.19458774350005115,"p90_request_seconds":0.2346541320999677},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":87,"correct":87,"interval":[0.957711004649852,1.0]}}},{"run":54,"model":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","variant":null,"reasoning":"low","provider":"claude","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.anthropic.com/claude-sonnet-5-5","https://platform.claude.com/docs/en/models/overview"],"parameters_read":"2026-11-09","access":"Claude Code CLI, Claude subscription","questions_per_request":16,"concurrency":1,"correct":1349,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":544},"grammar":{"questions":1542,"correct":805},"without_suspect":{"questions":2186,"correct":1346}},"tyt_mix":0.7755049666255418,"questions_per_minute":230.28394322199546,"request_seconds":575.8108800150803,"median_request_seconds":null,"p90_request_seconds":null,"billing":"subscription","api_equivalent_usd":4.1900397,"cost_basis":"subscription","cost_upper_bound":false,"requests":139,"hardware":null,"input_tokens":1215056,"cached_input_tokens":298797,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":915981,"output_tokens":49568,"previous":{"run":38,"changed":{"questions":372,"older_correct":155,"newer_correct":229,"flipped":134,"changed_answer":205,"newer_wrong":143,"repeated_mistakes":42,"p":9.319307580879302e-11},"unchanged":{"questions":1814,"older_correct":1095,"newer_correct":1113,"flipped":276,"changed_answer":424,"newer_wrong":701,"repeated_mistakes":424,"p":0.3061693933230553},"added":{"questions":12,"newer_correct":7}},"units":[107,133,50,91,76,87,75,31,61,44,42,28,64,53,41,68,27,93,62,116],"started":"2026-10-04T09:02:05Z","updated":"2026-10-04T09:11:41Z","format_of":null,"group":"NOPQRST","latency":{"source":"sample","runs":[229,230,231,232,233],"repeats":[1,2,3,4,5],"questions":200,"concurrency":1,"median_request_seconds":3.873064635001356,"p90_request_seconds":5.84579606440384},"single_question_latency":null,"tag":"low · chat","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":150,"model":"celeris-1-decision","name":"celeris-1-decision","variant":null,"reasoning":null,"provider":"celeris","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://docs.celeris.ai/decisions","https://celeris.ai/decisions"],"parameters_read":"2026-11-09","access":"Celeris System One API","questions_per_request":1,"concurrency":1,"correct":1329,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":559},"grammar":{"questions":1542,"correct":770},"without_suspect":{"questions":2186,"correct":1326}},"tyt_mix":0.7903971817563507,"questions_per_minute":148.99183210646953,"request_seconds":885.1491933179168,"median_request_seconds":0.4010543754975515,"p90_request_seconds":0.4175183980994916,"billing":"api","api_equivalent_usd":0.033577800000000005,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":839445,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[104,140,57,87,83,88,77,34,56,36,41,26,66,41,48,78,32,60,60,115],"started":"2026-10-10T18:37:11Z","updated":"2026-10-10T18:52:00Z","format_of":null,"group":"OPQRST","latency":{"source":"run","runs":[150],"concurrency":1,"median_request_seconds":0.4010543754975515,"p90_request_seconds":0.4175183980994916},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":44,"model":"pplx-decider-v1-27b","name":"Perplexity Decider 27B v1","variant":null,"reasoning":null,"provider":"perplexity","perplexity_mode":null,"note":null,"parameters":26086635760,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 0.46B vision encoder + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, perplexity-ai/pplx-decider-v1-27b@5117a6c","https://huggingface.co/perplexity-ai/pplx-decider-v1-27b"],"parameters_read":"2026-11-09","access":"Perplexity Decisions API","questions_per_request":1,"concurrency":1,"correct":1327,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":564},"grammar":{"questions":1542,"correct":763},"without_suspect":{"questions":2186,"correct":1324}},"tyt_mix":0.7958908686849514,"questions_per_minute":284.5404418373233,"request_seconds":466.01459934405284,"median_request_seconds":null,"p90_request_seconds":null,"billing":"api","api_equivalent_usd":0.034337480000000004,"cost_basis":"api","cost_upper_bound":false,"requests":2210,"hardware":null,"input_tokens":858437,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2210,"previous":{"run":43,"changed":{"questions":372,"older_correct":147,"newer_correct":192,"flipped":111,"changed_answer":183,"newer_wrong":180,"repeated_mistakes":75,"p":2.32626705010725e-05},"unchanged":{"questions":1814,"older_correct":1127,"newer_correct":1127,"flipped":0,"changed_answer":0,"newer_wrong":687,"repeated_mistakes":687,"p":1.0},"added":{"questions":12,"newer_correct":8}},"units":[103,137,56,92,83,93,76,33,55,33,45,20,64,49,49,80,24,67,59,109],"started":"2026-10-04T08:31:15Z","updated":"2026-10-04T08:39:05Z","format_of":null,"group":"OPQRST","latency":{"source":"repeats","runs":[64,84,96,100],"repeats":[2,3,4,5],"questions":2198,"concurrency":1,"median_request_seconds":0.21641260248725303,"p90_request_seconds":0.2719852286914829},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":21,"correct":21,"interval":[0.8453561767357979,0.9999999999999999]}}},{"run":264,"model":"OmniJev/OneJev-27B","name":"OneJev-27B","variant":null,"reasoning":null,"provider":"onejev","perplexity_mode":null,"note":"OmniJev's released OneJev checkpoint (OmniJev/OneJev-27B at a5e5f55), a full fine-tune of Qwen3.8-27B that also reads images, served on our hardware through the server its package ships (qev 0.1.4, qev serve) with its defaults. One request at a time, as for the other setups. The five runs are identical because decoding is deterministic: each run sent all 2,198 questions again as separate requests.","parameters":27356728560,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head + 0.46B vision encoder","parameters_source":["counted from the checkpoint's safetensors, OmniJev/OneJev-27B@a5e5f55","https://huggingface.co/OmniJev/OneJev-27B"],"parameters_read":"2026-11-09","access":"OneJev server (qev, OmniJev's release code), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1310,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":559},"grammar":{"questions":1542,"correct":751},"without_suspect":{"questions":2186,"correct":1306}},"tyt_mix":0.7882408912245737,"questions_per_minute":418.8018794259747,"request_seconds":314.8983003150788,"median_request_seconds":0.1249308105034288,"p90_request_seconds":0.1888540582935093,"billing":"local","api_equivalent_usd":0.21780465771792953,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":870930,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[106,133,56,91,83,90,71,31,53,37,47,23,64,43,44,77,25,66,61,109],"started":"2026-10-27T13:44:13Z","updated":"2026-10-27T13:49:37Z","format_of":null,"group":"PQRSTU","latency":{"source":"run","runs":[264],"concurrency":1,"median_request_seconds":0.1249308105034288,"p90_request_seconds":0.1888540582935093},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":2,"correct":2,"interval":[0.34237195288961925,1.0]}}},{"run":194,"model":"jev-1.14.0","name":"Jev 1.14","variant":null,"reasoning":null,"provider":"jev","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://docs.typesafe.ai/models","https://typesafe.ai/blog/jev-1-14"],"parameters_read":"2026-11-09","access":"TypeSafe System One API","questions_per_request":1,"concurrency":1,"correct":1301,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":558},"grammar":{"questions":1542,"correct":743},"without_suspect":{"questions":2186,"correct":1301}},"tyt_mix":0.7860753574705008,"questions_per_minute":265.6886711846118,"request_seconds":496.3704301428952,"median_request_seconds":0.2105813504967955,"p90_request_seconds":0.2774909528983699,"billing":"api","api_equivalent_usd":0.063384972,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":1509166,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":114296,"previous":null,"units":[104,138,52,93,82,89,78,31,53,38,38,24,66,43,43,81,26,55,60,107],"started":"2026-10-16T09:52:25Z","updated":"2026-10-16T10:06:22Z","format_of":null,"group":"PQRSTU","latency":{"source":"run","runs":[194],"concurrency":1,"median_request_seconds":0.2105813504967955,"p90_request_seconds":0.2774909528983699},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":260,"correct":259,"interval":[0.978538689004996,0.9993207539334944]}}},{"run":1149,"model":"qwen3.5-4b","name":"Qwen3.5-4B","variant":null,"reasoning":"on","provider":"vllm-plain","perplexity_mode":null,"note":null,"parameters":4659865088,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + multi-token prediction layer + 0.33B vision encoder","parameters_source":["counted from the checkpoint's safetensors, Qwen/Qwen3.5-4B@851bf6e","https://huggingface.co/Qwen/Qwen3.5-4B"],"parameters_read":"2026-11-09","access":"our GPU (logic's server, vLLM v0.30.0 BF16, revision Qwen/Qwen3.5-4B@851bf6e), one question per request, plain answer line CEVAP: X (no answer schema); sampling min_p 0.0, presence_penalty 1.5, repetition_penalty 1.0, seed 0, temperature 1.0, top_k 20, top_p 0.95, thinking on; output cap 32,768","questions_per_request":1,"concurrency":32,"correct":1295,"invalid":5,"invalid_replies":{"invalid_without_rule":5,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":1,"unreadable":4}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":519},"grammar":{"questions":1542,"correct":776},"without_suspect":{"questions":2186,"correct":1291}},"tyt_mix":0.7407732375597103,"questions_per_minute":13.843333986155718,"request_seconds":9526.606822597001,"median_request_seconds":140.2851593714986,"p90_request_seconds":204.2336049738995,"billing":"local","api_equivalent_usd":6.589236385629593,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":579767,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":16259552,"previous":null,"units":[88,135,50,90,77,79,79,33,53,47,37,31,78,50,46,81,20,54,60,107],"started":"2026-11-14T23:28:01Z","updated":"2026-11-15T02:06:48Z","format_of":null,"group":"PQRSTUVW","latency":{"source":"run","runs":[1149],"concurrency":32,"median_request_seconds":140.2851593714986,"p90_request_seconds":204.2336049738995},"single_question_latency":null,"tag":"reasoning on · chat"},{"run":204,"model":"inception/mercury-decide","name":"Mercury Decide","variant":"via OpenRouter","reasoning":null,"provider":"openrouter","perplexity_mode":null,"note":"Inception's decision model, called through OpenRouter's System One route (paid listing). It returns a probability for each option.","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.inceptionlabs.ai/"],"parameters_read":"2026-11-09","access":"OpenRouter System One API","questions_per_request":1,"concurrency":1,"correct":1281,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":554},"grammar":{"questions":1542,"correct":727},"without_suspect":{"questions":2186,"correct":1275}},"tyt_mix":0.7792290460599158,"questions_per_minute":140.8778626326503,"request_seconds":936.1300458105834,"median_request_seconds":0.31287493899435503,"p90_request_seconds":0.7718596629958483,"billing":"api","api_equivalent_usd":0.01694982,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":847491,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[103,137,52,92,81,89,79,30,46,31,41,22,56,44,39,76,27,68,59,109],"started":"2026-10-17T15:08:47Z","updated":"2026-10-17T15:24:29Z","format_of":null,"group":"QRSTUV","latency":{"source":"run","runs":[204],"concurrency":1,"median_request_seconds":0.31287493899435503,"p90_request_seconds":0.7718596629958483},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":451,"correct":438,"interval":[0.9513124663540422,0.9830787590747665]}}},{"run":159,"model":"glm-5.3-flash","name":"GLM-5.3-Flash","variant":null,"reasoning":"off","provider":"vllm-stated","perplexity_mode":null,"note":null,"parameters":320000000000,"active_parameters":[18000000000],"effective_parameters":null,"parameters_scope":"mixture of experts, as Z.ai states it","parameters_source":["https://huggingface.co/zai-org/GLM-5.3-Flash"],"parameters_read":"2026-11-09","access":"TensorFold, self-hosted, EXL3 4-bit experts","questions_per_request":16,"concurrency":1,"correct":1270,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":539},"grammar":{"questions":1542,"correct":731},"without_suspect":{"questions":2186,"correct":1269}},"tyt_mix":0.7608186726930499,"questions_per_minute":879.5015970459278,"request_seconds":149.94856227999912,"median_request_seconds":1.065357404999986,"p90_request_seconds":1.2150850257999992,"billing":"local","api_equivalent_usd":0.10391914999999999,"cost_basis":"maker","cost_upper_bound":false,"requests":138,"hardware":"4 × RTX PRO 6000 Blackwell (96 GB); EXL3 4-bit experts, FP8 dense layers and KV cache","input_tokens":557121,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":40702,"previous":null,"units":[100,130,44,91,83,91,73,27,49,35,38,21,56,44,38,75,28,76,58,113],"started":"2026-10-11T10:20:58Z","updated":"2026-10-11T10:23:29Z","format_of":null,"group":"QRSTUVWX","latency":{"source":"run","runs":[159],"concurrency":1,"median_request_seconds":1.065357404999986,"p90_request_seconds":1.2150850257999992},"single_question_latency":null,"tag":"reasoning off · chat","confidence":{"sure":{"questions":4,"correct":4,"interval":[0.5100999795960008,1.0]}}},{"run":1040,"model":"gemma-4-e4b-it","name":"Gemma 4 E4B","variant":null,"reasoning":"default","provider":"vllm-plain","perplexity_mode":null,"note":null,"parameters":7996156448,"active_parameters":null,"effective_parameters":4500000000,"parameters_scope":"language model with per-layer embeddings + vision and audio encoders; effective size as Google states it","parameters_source":["counted from the checkpoint's safetensors, google/gemma-4-E4B-it@4ecc2ba","https://huggingface.co/google/gemma-4-E4B-it"],"parameters_read":"2026-11-09","access":"our GPU (logic's server, vLLM v0.30.0 BF16, revision google/gemma-4-E4B-it@4ecc2ba), one question per request, plain answer line CEVAP: X (no answer schema); sampling temperature 0.0; output cap 8,192","questions_per_request":1,"concurrency":32,"correct":1265,"invalid":3,"invalid_replies":{"invalid_without_rule":3,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"unreadable":3}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":509},"grammar":{"questions":1542,"correct":756},"without_suspect":{"questions":2186,"correct":1263}},"tyt_mix":0.7259272385403814,"questions_per_minute":135.0329976513523,"request_seconds":976.6501691720332,"median_request_seconds":13.376780516031431,"p90_request_seconds":20.376569115376334,"billing":"local","api_equivalent_usd":0.6755163670106563,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":545549,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":1782365,"previous":null,"units":[90,134,54,86,64,81,81,36,57,44,36,22,75,40,47,104,15,53,53,93],"started":"2026-11-12T08:32:06Z","updated":"2026-11-12T08:48:22Z","format_of":null,"group":"QRSTUVWXY","latency":{"source":"run","runs":[1040],"concurrency":32,"median_request_seconds":13.376780516031431,"p90_request_seconds":20.376569115376334},"single_question_latency":null,"tag":"default reasoning · chat"},{"run":49,"model":"jev-1.13.0","name":"Jev 1.13","variant":null,"reasoning":null,"provider":"jev","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://docs.typesafe.ai/models","https://typesafe.ai/blog/jev-1-14"],"parameters_read":"2026-11-09","access":"TypeSafe System One API","questions_per_request":1,"concurrency":1,"correct":1264,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":549},"grammar":{"questions":1542,"correct":715},"without_suspect":{"questions":2186,"correct":1263}},"tyt_mix":0.7715790685995382,"questions_per_minute":251.69948821856477,"request_seconds":526.8187112277956,"median_request_seconds":null,"p90_request_seconds":null,"billing":"api","api_equivalent_usd":0.06375499200000001,"cost_basis":"api","cost_upper_bound":false,"requests":2210,"hardware":null,"input_tokens":1517976,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":114920,"previous":{"run":29,"changed":{"questions":372,"older_correct":135,"newer_correct":186,"flipped":121,"changed_answer":201,"newer_wrong":186,"repeated_mistakes":71,"p":4.004113903707686e-06},"unchanged":{"questions":1814,"older_correct":1079,"newer_correct":1071,"flipped":70,"changed_answer":114,"newer_wrong":743,"repeated_mistakes":660,"p":0.40296307504607976},"added":{"questions":12,"newer_correct":7}},"units":[105,137,50,93,78,86,76,32,51,41,39,24,65,44,41,68,21,52,55,106],"started":"2026-10-04T08:39:08Z","updated":"2026-10-04T08:47:59Z","format_of":null,"group":"RSTUVWX","latency":{"source":"repeats","runs":[65,85,97,101],"repeats":[2,3,4,5],"questions":2198,"concurrency":1,"median_request_seconds":0.24339387452346273,"p90_request_seconds":0.2807294782833196},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":312,"correct":311,"interval":[0.9820706464658654,0.9994340064994186]}}},{"run":125,"model":"clef","name":"Clef","variant":null,"reasoning":null,"provider":"clef","perplexity_mode":null,"note":"Cloudflare's decision model, post-trained from Qwen3.8-27B","parameters":27484784884,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head + 0.46B vision encoder + joint schema head","parameters_source":["counted from the checkpoint's safetensors, Cloudflare/clef@2f3de3d","https://huggingface.co/Cloudflare/clef"],"parameters_read":"2026-11-09","access":"Clef server, self-hosted","questions_per_request":1,"concurrency":1,"correct":1259,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":558},"grammar":{"questions":1542,"correct":701},"without_suspect":{"questions":2186,"correct":1256}},"tyt_mix":0.7813088205055203,"questions_per_minute":285.0197317975998,"request_seconds":462.7048070259625,"median_request_seconds":0.20457697298843414,"p90_request_seconds":0.24825825718580746,"billing":"local","api_equivalent_usd":0.24108839999999998,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":1004535,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[101,142,54,93,80,88,71,27,47,30,41,25,50,38,42,75,24,70,55,106],"started":"2026-10-05T20:59:17Z","updated":"2026-10-05T21:07:04Z","format_of":null,"group":"RSTUVWXY","latency":{"source":"run","runs":[125],"concurrency":1,"median_request_seconds":0.20457697298843414,"p90_request_seconds":0.24825825718580746},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":948,"model":"anthropic/claude-haiku-5.5","name":"Claude Haiku 5.5","variant":null,"reasoning":"max","provider":"openrouter-chat","perplexity_mode":null,"note":"Anthropic's small chat model, added on the reading bench as a chat model","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.anthropic.com/claude-haiku-5-5","https://platform.claude.com/docs/en/models/overview"],"parameters_read":"2026-11-10","access":"OpenRouter chat API, Anthropic's own endpoint, fallbacks off, output cap 128,000 tokens, reasoning max","questions_per_request":16,"concurrency":4,"correct":1254,"invalid":784,"invalid_replies":{"invalid_without_rule":784,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":3,"invalid_by_kind":{"cut off":784}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":570},"grammar":{"questions":1542,"correct":684},"without_suspect":{"questions":2186,"correct":1252}},"tyt_mix":0.7944709713390907,"questions_per_minute":8.572462237069244,"request_seconds":15384.144759450952,"median_request_seconds":417.0346134795109,"p90_request_seconds":551.1150343503017,"billing":"api","api_equivalent_usd":6.5160982,"cost_basis":"api","cost_upper_bound":false,"requests":138,"hardware":null,"input_tokens":1190192,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":12794158,"previous":null,"units":[103,147,57,95,78,90,113,31,58,62,60,3,58,34,41,0,0,62,46,116],"started":"2026-11-10T16:59:07Z","updated":"2026-11-10T20:44:00Z","format_of":null,"group":"RSTUVWXYZ","latency":{"source":"run","runs":[948],"concurrency":4,"median_request_seconds":417.0346134795109,"p90_request_seconds":551.1150343503017},"single_question_latency":null,"tag":"max · chat","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1192,"model":"Jeffan-9B-1211","name":"Jeffan-9B-1211","variant":null,"reasoning":null,"provider":"di-serve","perplexity_mode":null,"note":"tattrongvu's released Jeffan-9B-1211 (tattrongvu/Jeffan-9B-1211 at 862d1a7), built on Qwen3.5-9B, which reads every option's probability from one forward pass and generates no text. Its card ships only a Decision Index engine (jeffan_engine:DecisionEngine), so it ran on our hardware behind compactor's di_serve.py (transformers 5.17.0, flash-linear-attention 0.5.2, without causal-conv1d, so PyTorch's reference convolution), text only.","parameters":8953803264,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model, text only; the card says bf16, 8.95B parameters","parameters_source":["counted from the checkpoint's safetensors, tattrongvu/Jeffan-9B-1211@862d1a7","https://huggingface.co/tattrongvu/Jeffan-9B-1211"],"parameters_read":"2026-11-15","access":"Decision Index engine (compactor's di_serve.py around the engine its card ships), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1253,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":544},"grammar":{"questions":1542,"correct":709},"without_suspect":{"questions":2186,"correct":1252}},"tyt_mix":0.7646100249913006,"questions_per_minute":936.9973726480706,"request_seconds":140.7474597578548,"median_request_seconds":0.06390045749867568,"p90_request_seconds":0.07293606689709123,"billing":"local","api_equivalent_usd":0.09735032633251625,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":864336,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[98,139,51,91,79,86,72,29,43,32,36,21,73,40,42,83,23,51,57,107],"started":"2026-11-15T07:20:04Z","updated":"2026-11-15T07:22:32Z","format_of":null,"group":"STUVWXY","latency":{"source":"run","runs":[1192],"concurrency":1,"median_request_seconds":0.06390045749867568,"p90_request_seconds":0.07293606689709123},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":52,"model":"d1:free","name":"d1","variant":null,"reasoning":null,"provider":"liquid","perplexity_mode":null,"note":null,"parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://www.liquid.ai/blog/d1-decision-model"],"parameters_read":"2026-11-09","access":"Liquid AI decisions API","questions_per_request":1,"concurrency":1,"correct":1253,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":543},"grammar":{"questions":1542,"correct":710},"without_suspect":{"questions":2186,"correct":1251}},"tyt_mix":0.7634658920154377,"questions_per_minute":150.12319443415166,"request_seconds":883.2745699277148,"median_request_seconds":null,"p90_request_seconds":null,"billing":"free","api_equivalent_usd":0.016426200000000002,"cost_basis":"free tier","cost_upper_bound":false,"requests":2210,"hardware":null,"input_tokens":821310,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":{"run":40,"changed":{"questions":372,"older_correct":138,"newer_correct":201,"flipped":115,"changed_answer":193,"newer_wrong":171,"repeated_mistakes":67,"p":2.952378329881734e-09},"unchanged":{"questions":1814,"older_correct":1041,"newer_correct":1045,"flipped":58,"changed_answer":91,"newer_wrong":769,"repeated_mistakes":709,"p":0.6940040940761316},"added":{"questions":12,"newer_correct":7}},"units":[103,133,53,93,76,85,69,33,46,40,39,24,66,40,40,63,22,61,57,110],"started":"2026-10-04T08:48:02Z","updated":"2026-10-04T09:02:49Z","format_of":null,"group":"TUVWXY","latency":{"source":"repeats","runs":[66,94,105,108],"repeats":[2,3,4,5],"questions":2198,"concurrency":1,"median_request_seconds":0.3054448555049021,"p90_request_seconds":0.78636241707718},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":274,"correct":273,"interval":[0.9796195950792701,0.9993554756084886]}}},{"run":334,"model":"drex-v2.0","name":"Drex 2.0","variant":null,"reasoning":null,"provider":"nace","perplexity_mode":null,"note":"Nace.AI's hosted Drex 2.0 (drex-v2.0, released 6 October 2026), built on Qwen3.6-35B-A3B. It reads text only, so it is not on the Page images tab. Nace's API gives its option probabilities rounded to 4 decimal places, so they sum to 1 only within that rounding; the check before its runs allowed for it.","parameters":35107181936,"active_parameters":[3000000000],"effective_parameters":null,"parameters_scope":"mixture of experts: language model with its LM head + 0.45B vision tower (not loaded for text, per the card); about 3B active per token, as Nace states it","parameters_source":["counted from the checkpoint's safetensors, nace-ai/drex-v2.0@e646674","https://huggingface.co/nace-ai/drex-v2.0","https://console.nace.ai/docs/drex/reference/models"],"parameters_read":"2026-11-09","access":"Nace.AI System One API","questions_per_request":1,"concurrency":1,"correct":1225,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":534},"grammar":{"questions":1542,"correct":691},"without_suspect":{"questions":2186,"correct":1224}},"tyt_mix":0.7499910039226851,"questions_per_minute":120.51272554278829,"request_seconds":1094.324266636686,"median_request_seconds":0.32470757849660004,"p90_request_seconds":0.39495088000403483,"billing":"api","api_equivalent_usd":0.04210104,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":701684,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":185394,"previous":null,"units":[96,130,53,92,79,84,68,26,52,35,33,25,61,45,36,73,20,60,58,99],"started":"2026-10-31T21:26:44Z","updated":"2026-10-31T21:45:07Z","format_of":null,"group":"UVWXYZα","latency":{"source":"run","runs":[334],"concurrency":1,"median_request_seconds":0.32470757849660004,"p90_request_seconds":0.39495088000403483},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":33,"correct":33,"interval":[0.8957265699643882,1.0]}}},{"run":127,"model":"cygnet","name":"Cygnet","variant":null,"reasoning":null,"provider":"cygnet","perplexity_mode":null,"note":"Gemma 4 12B IT, unchanged, read out by option letter","parameters":11959730176,"active_parameters":null,"effective_parameters":null,"parameters_scope":"Gemma 4 12B IT: language model (tied embeddings) + vision and audio embedders","parameters_source":["counted from the checkpoint's safetensors, google/gemma-4-12B-it@707f0a3","https://huggingface.co/google/gemma-4-12B-it"],"parameters_read":"2026-11-09","access":"Cygnet decision server on vLLM, self-hosted","questions_per_request":1,"concurrency":1,"correct":1224,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":543},"grammar":{"questions":1542,"correct":681},"without_suspect":{"questions":2186,"correct":1222}},"tyt_mix":0.760174711730094,"questions_per_minute":1619.6114236317753,"request_seconds":81.4269386321539,"median_request_seconds":0.0359221679973416,"p90_request_seconds":0.04518266418017447,"billing":"local","api_equivalent_usd":0.08658210000000001,"cost_basis":"weights","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":859227,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[100,140,48,94,76,85,70,27,54,37,39,25,54,38,34,63,24,61,54,101],"started":"2026-10-06T10:05:32Z","updated":"2026-10-06T10:06:56Z","format_of":null,"group":"UVWXYZα","latency":{"source":"run","runs":[127],"concurrency":1,"median_request_seconds":0.0359221679973416,"p90_request_seconds":0.04518266418017447},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":2,"correct":2,"interval":[0.34237195288961925,1.0]}}},{"run":45,"model":"open-jev-27b-v1.1","name":"Open-Jev-27B-v1.1","variant":null,"reasoning":null,"provider":"open-jev","perplexity_mode":null,"note":"Fine-tune of Qwen3.8-27B (LoRA and a scoring head)","parameters":27796899569,"active_parameters":null,"effective_parameters":null,"parameters_scope":"Qwen3.8-27B (with its vision encoder) + LoRA adapter and scalar decision head (15.5M); the repo holds only the adapter and head","parameters_source":["counted from the checkpoints' safetensors, ZefanCai/Open-Jev-27B-v1.1@28cf730 on Qwen/Qwen3.8-27B@1d4bf0f","https://huggingface.co/ZefanCai/Open-Jev-27B-v1.1"],"parameters_read":"2026-11-09","access":"Open-Jev server, self-hosted","questions_per_request":1,"concurrency":1,"correct":1224,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":544},"grammar":{"questions":1542,"correct":680},"without_suspect":{"questions":2186,"correct":1221}},"tyt_mix":0.7613188447059568,"questions_per_minute":153.40666379539002,"request_seconds":864.3692309015896,"median_request_seconds":null,"p90_request_seconds":null,"billing":"local","api_equivalent_usd":0.5978553847069329,"cost_basis":"gpu","cost_upper_bound":false,"requests":2210,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":3043367,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":{"run":31,"changed":{"questions":372,"older_correct":137,"newer_correct":179,"flipped":108,"changed_answer":192,"newer_wrong":193,"repeated_mistakes":76,"p":6.550370792480044e-05},"unchanged":{"questions":1814,"older_correct":1037,"newer_correct":1037,"flipped":0,"changed_answer":0,"newer_wrong":777,"repeated_mistakes":777,"p":1.0},"added":{"questions":12,"newer_correct":8}},"units":[101,131,50,92,80,90,71,34,52,33,47,19,58,40,35,40,18,69,55,109],"started":"2026-10-04T08:31:15Z","updated":"2026-10-04T08:45:43Z","format_of":null,"group":"UVWXYZα","latency":{"source":"repeats","runs":[60,92,102,106],"repeats":[2,3,4,5],"questions":2198,"concurrency":1,"median_request_seconds":0.386393806518754,"p90_request_seconds":0.5727336546988226},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":128,"model":"winnow-12b","name":"Winnow-12B","variant":null,"reasoning":null,"provider":"winnow","perplexity_mode":null,"note":"Fine-tune of Gemma 4 12B IT (Q8_0 GGUF)","parameters":11907350272,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model only (Gemma 4 12B architecture, LoRA merged; text only)","parameters_source":["counted from the GGUF's logical tensor shapes, EldanRing/Winnow-12B@b2b1421 (Winnow-12B-Q8_0.gguf)","https://huggingface.co/EldanRing/Winnow-12B"],"parameters_read":"2026-11-09","access":"Winnow server (llama.cpp), self-hosted","questions_per_request":1,"concurrency":1,"correct":1219,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":538},"grammar":{"questions":1542,"correct":681},"without_suspect":{"questions":2186,"correct":1217}},"tyt_mix":0.7538866019739964,"questions_per_minute":1002.408676880479,"request_seconds":131.56310698587913,"median_request_seconds":0.05853745195781812,"p90_request_seconds":0.0698976724059321,"billing":"local","api_equivalent_usd":0.09099781566523309,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); Q8_0","input_tokens":887137,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[99,138,47,91,79,84,64,28,46,31,38,26,58,32,33,74,26,63,57,105],"started":"2026-10-06T10:25:55Z","updated":"2026-10-06T10:28:09Z","format_of":null,"group":"UVWXYZα","latency":{"source":"run","runs":[128],"concurrency":1,"median_request_seconds":0.05853745195781812,"p90_request_seconds":0.0698976724059321},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":163,"correct":162,"interval":[0.9660709164826251,0.9989162330052723]}}},{"run":1174,"model":"PelaAI/KnowLine-9B-Gen1","name":"KnowLine-9B-Gen1","variant":null,"reasoning":null,"provider":"knowline","perplexity_mode":null,"note":"PelaAI's released checkpoint, a merged LoRA fine-tune of Qwen3.5-9B, run on our hardware through its card's serve_knowline.sh (SGLang 0.5.23 in the lmsysorg/sglang image; the card's launcher names 0.5.21, then knowline_server.py), text only.","parameters":9653104368,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model + the base's vision components (model-extra-from-base.safetensors; text only here)","parameters_source":["counted from the checkpoint's safetensors, PelaAI/KnowLine-9B-Gen1@325378d","https://huggingface.co/PelaAI/KnowLine-9B-Gen1"],"parameters_read":"2026-11-15","access":"KnowLine server (knowline_server.py over SGLang, PelaAI's release code), self-hosted","questions_per_request":1,"concurrency":1,"correct":1199,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":524},"grammar":{"questions":1542,"correct":675},"without_suspect":{"questions":2186,"correct":1198}},"tyt_mix":0.7355989608047832,"questions_per_minute":1019.8903047506243,"request_seconds":129.30802399601816,"median_request_seconds":0.056655265998415416,"p90_request_seconds":0.07768903699980001,"billing":"local","api_equivalent_usd":0.08943804993057924,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":1689108,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[97,131,49,88,75,84,68,24,48,38,40,22,60,43,40,72,21,48,53,98],"started":"2026-11-15T06:29:01Z","updated":"2026-11-15T06:31:17Z","format_of":null,"group":"VWXYZα","latency":{"source":"run","runs":[1174],"concurrency":1,"median_request_seconds":0.056655265998415416,"p90_request_seconds":0.07768903699980001},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":62,"correct":62,"interval":[0.9416539087749995,1.0]}}},{"run":1201,"model":"apertixai/cerno-9b-v1.2","name":"Cerno 9B v1.2","variant":null,"reasoning":null,"provider":"cerno","perplexity_mode":null,"note":"Apertix AI's released Cerno 9B v1.2 (apertixai/cerno-9b-v1.2 at f02a632), a full fine-tune of Qwen3.5-9B with block-bidirectional attention. Its card ships no server, only an in-process engine (CernoEngine, transformers, BF16); it ran on our hardware, one request at a time, through a thin System One wrapper around that engine, text only: its English runs on one DGX Spark (logic's wrapper), its Turkish runs on one RTX PRO 6000 (compactor's di_serve.py, 15 November 2026). Input tokens are counted only where the engine reports them. Probabilities rounded to 4 decimals by its own code; they sum to 1 only within that rounding.","parameters":8953803264,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head (text only)","parameters_source":["counted from the checkpoint's safetensors, apertixai/cerno-9b-v1.2@f02a632","https://huggingface.co/apertixai/cerno-9b-v1.2"],"parameters_read":"2026-11-10","access":"Cerno server (a thin System One server around the card's CernoEngine, transformers: logic's for its English runs, compactor's di_serve.py for its Turkish runs), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1197,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":523},"grammar":{"questions":1542,"correct":674},"without_suspect":{"questions":2186,"correct":1195}},"tyt_mix":0.7342278498782069,"questions_per_minute":1152.189103799277,"request_seconds":114.46037769766554,"median_request_seconds":0.051093491492792964,"p90_request_seconds":0.06626715850579785,"billing":"local","api_equivalent_usd":0.079168427907552,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":0,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[95,132,42,90,78,86,72,24,49,33,36,26,68,40,36,80,16,53,49,92],"started":"2026-11-15T07:37:59Z","updated":"2026-11-15T07:40:02Z","format_of":null,"group":"VWXYZαβ","latency":{"source":"run","runs":[1201],"concurrency":1,"median_request_seconds":0.051093491492792964,"p90_request_seconds":0.06626715850579785},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":83,"model":"qwen38-27b-bf16","name":"Qwen3.8-27B","variant":"vote share","reasoning":"off","provider":"vllm-vote","perplexity_mode":null,"note":"Open-Jev's base model","parameters":27781427952,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head + 0.42B multi-token prediction layer + 0.46B vision encoder","parameters_source":["counted from the checkpoint's safetensors, Qwen/Qwen3.8-27B@1d4bf0f","https://huggingface.co/Qwen/Qwen3.8-27B"],"parameters_read":"2026-11-09","access":"vLLM, self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1194,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":551},"grammar":{"questions":1542,"correct":643},"without_suspect":{"questions":2186,"correct":1190}},"tyt_mix":0.7659231062762962,"questions_per_minute":36.42730498097688,"request_seconds":3620.3611567990156,"median_request_seconds":1.6220232204941567,"p90_request_seconds":1.8360200999944936,"billing":"local","api_equivalent_usd":1.814203,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":737600,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":481801,"previous":null,"units":[106,133,52,91,82,87,59,22,42,27,33,25,59,33,37,52,24,67,59,104],"started":"2026-10-04T20:32:46Z","updated":"2026-10-04T21:33:38Z","format_of":null,"group":"VWXYZαβ","latency":{"source":"run","runs":[83],"concurrency":1,"median_request_seconds":1.6220232204941567,"p90_request_seconds":1.8360200999944936},"single_question_latency":null,"tag":"vote share · reasoning off · chat","confidence":{"sure":{"questions":875,"correct":746,"interval":[0.8275396883011108,0.8745208422882469]}}},{"run":53,"model":"qwen38-27b-bf16","name":"Qwen3.8-27B","variant":"stated confidence","reasoning":"off","provider":"vllm-verbal","perplexity_mode":null,"note":"Open-Jev's base model","parameters":27781427952,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head + 0.42B multi-token prediction layer + 0.46B vision encoder","parameters_source":["counted from the checkpoint's safetensors, Qwen/Qwen3.8-27B@1d4bf0f","https://huggingface.co/Qwen/Qwen3.8-27B"],"parameters_read":"2026-11-09","access":"vLLM, self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1191,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":547},"grammar":{"questions":1542,"correct":644},"without_suspect":{"questions":2186,"correct":1187}},"tyt_mix":0.7610061074467749,"questions_per_minute":147.0429555938642,"request_seconds":901.7773035400896,"median_request_seconds":null,"p90_request_seconds":null,"billing":"local","api_equivalent_usd":0.516798,"cost_basis":"maker","cost_upper_bound":false,"requests":2210,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":741942,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":48609,"previous":{"run":35,"changed":{"questions":372,"older_correct":135,"newer_correct":181,"flipped":112,"changed_answer":181,"newer_wrong":191,"repeated_mistakes":89,"p":1.6356858141740275e-05},"unchanged":{"questions":1814,"older_correct":996,"newer_correct":1000,"flipped":18,"changed_answer":31,"newer_wrong":814,"repeated_mistakes":794,"p":0.480682373046875},"added":{"questions":12,"newer_correct":10}},"units":[102,133,52,91,79,90,55,22,41,28,39,19,62,34,35,52,29,64,59,105],"started":"2026-10-04T08:54:01Z","updated":"2026-10-04T09:09:07Z","format_of":null,"group":"WXYZαβ","latency":{"source":"repeats","runs":[74,95,103,107],"repeats":[2,3,4,5],"questions":2198,"concurrency":1,"median_request_seconds":0.43298505601705983,"p90_request_seconds":0.4745744730113074},"single_question_latency":null,"tag":"stated confidence · reasoning off · chat","confidence":{"sure":{"questions":8,"correct":7,"interval":[0.5291051942301386,0.9775830911367038]}}},{"run":1306,"model":"PelaAI/KnowLine-9B-Gen2","name":"KnowLine-9B-Gen2","variant":null,"reasoning":null,"provider":"knowline","perplexity_mode":null,"note":"PelaAI's released checkpoint (13 November 2026), a fine-tune of Qwen3.5-9B, run on our hardware through its card's serve_knowline.sh (SGLang 0.5.23 in the lmsysorg/sglang image; the card's launcher names 0.5.21, then knowline_server.py), text only.","parameters":9653104368,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model + the base's vision components (model-extra-from-base.safetensors; text only here)","parameters_source":["counted from the checkpoint's safetensors, PelaAI/KnowLine-9B-Gen2@8ec8eb6","https://huggingface.co/PelaAI/KnowLine-9B-Gen2"],"parameters_read":"2026-11-15","access":"KnowLine server (knowline_server.py over SGLang, PelaAI's release code), self-hosted","questions_per_request":1,"concurrency":1,"correct":1171,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":522},"grammar":{"questions":1542,"correct":649},"without_suspect":{"questions":2186,"correct":1170}},"tyt_mix":0.7301330035430704,"questions_per_minute":1172.4712697598188,"request_seconds":112.48036809210316,"median_request_seconds":0.04812320999917574,"p90_request_seconds":0.06635775659815409,"billing":"local","api_equivalent_usd":0.07779892126370469,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":1689108,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[92,132,45,89,78,86,67,24,39,33,37,20,62,40,33,73,25,41,57,98],"started":"2026-11-16T10:30:36Z","updated":"2026-11-16T11:17:24Z","format_of":null,"group":"XYZαβγ","latency":{"source":"run","runs":[1306],"concurrency":1,"median_request_seconds":0.04812320999917574,"p90_request_seconds":0.06635775659815409},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1183,"model":"jev-latest","name":"Adjudex 9B v5","variant":"local","reasoning":null,"provider":"pplx-decider","perplexity_mode":null,"note":"newsbang's released Adjudex 9B v5 (newsbang/adjudex-9b-v5 at ffdcb82), a fine-tune of Qwen3.5-9B with a 255-way decision head, run on our hardware through its card's autojev server (Perplexity Decider v1.1's source, unchanged), which answers only to the model name 'jev-latest', text only.","parameters":8392695024,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model + 255-way decision head (readout.safetensors)","parameters_source":["counted from the checkpoint's safetensors, newsbang/adjudex-9b-v5@ffdcb82","https://huggingface.co/newsbang/adjudex-9b-v5"],"parameters_read":"2026-11-15","access":"Perplexity Decider server, self-hosted","questions_per_request":1,"concurrency":1,"correct":1168,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":530},"grammar":{"questions":1542,"correct":638},"without_suspect":{"questions":2186,"correct":1166}},"tyt_mix":0.738945600423903,"questions_per_minute":1577.7057341287439,"request_seconds":83.58973232282005,"median_request_seconds":0.03787083600036567,"p90_request_seconds":0.04507773509976687,"billing":"local","api_equivalent_usd":0.05781623152328388,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":853346,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[97,137,45,94,77,80,63,28,40,32,33,25,59,38,36,72,19,44,51,98],"started":"2026-11-15T06:57:25Z","updated":"2026-11-15T06:58:56Z","format_of":null,"group":"YZαβγ","latency":{"source":"run","runs":[1183],"concurrency":1,"median_request_seconds":0.03787083600036567,"p90_request_seconds":0.04507773509976687},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":132,"correct":130,"interval":[0.9464388294934194,0.9958351130985406]}}},{"run":373,"model":"baselabs/elephant-1","name":"Elephant-1","variant":null,"reasoning":null,"provider":"baselabs","perplexity_mode":null,"note":"Base Labs' released elephant-1 weights (baselabs/elephant-1 at b3f8a93), a decision model post-trained from Qwen3.5-9B that also reads images. It ran on our hardware through its model card's loader (qev's DecisionEngine from OneJev at 81ce62f with the checkpoint's serve.patch, as the card's setup.sh installs it) behind our HTTP server (servers/baselabs_server.py), one request at a time. That runtime reads images only through an engine the patch leaves without the attention the model was trained with, so it is not on the Page images tab. The card mentions a trained \"I don't know\" option, but its answers have no such output, and our questions offer five options. Probabilities rounded to 6 decimals by the model's own loader. They sum to 1 only within that rounding, and the check before its runs allowed for it. Its card lists English only.","parameters":9409813744,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head + 0.46B vision encoder","parameters_source":["counted from the checkpoint's safetensors, baselabs/elephant-1@b3f8a93","https://huggingface.co/baselabs/elephant-1"],"parameters_read":"2026-11-09","access":"Base Labs server (qev, the model card's loader), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1151,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":528},"grammar":{"questions":1542,"correct":623},"without_suspect":{"questions":2186,"correct":1149}},"tyt_mix":0.7347280218911139,"questions_per_minute":783.5925546206023,"request_seconds":168.30175225931453,"median_request_seconds":0.07427877650479786,"p90_request_seconds":0.08908395380713045,"billing":"local","api_equivalent_usd":0.030482550000000004,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":870930,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[96,134,48,92,75,83,65,25,33,28,34,20,57,43,27,65,16,51,55,104],"started":"2026-11-01T18:13:53Z","updated":"2026-11-01T18:16:49Z","format_of":null,"group":"Zαβγδ","latency":{"source":"run","runs":[373],"concurrency":1,"median_request_seconds":0.07427877650479786,"p90_request_seconds":0.08908395380713045},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":155,"correct":155,"interval":[0.9758148998750957,1.0]}}},{"run":907,"model":"glide-9b","name":"GLiDE 9B","variant":"local","reasoning":null,"provider":"glide","perplexity_mode":null,"note":"Fastino's released GLiDE 9B (fastino/GLiDE-9B at ab54df5), on Qwen3.5-9B, text only, whose layers 12 to 23 run twice (recurrent depth, its decision_config.json), run on our hardware through the server it ships.","parameters":7937729024,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, fastino/GLiDE-9B@ab54df5","https://huggingface.co/fastino/GLiDE-9B"],"parameters_read":"2026-11-10","access":"GLiDE server (Fastino's release code), self-hosted","questions_per_request":1,"concurrency":1,"correct":1144,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":526},"grammar":{"questions":1542,"correct":618},"without_suspect":{"questions":2186,"correct":1141}},"tyt_mix":0.7316453331118915,"questions_per_minute":281.0036021823433,"request_seconds":469.31782715875306,"median_request_seconds":0.20652708249690477,"p90_request_seconds":0.2530394938919926,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":789604,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[92,134,45,90,78,87,62,22,40,32,40,20,55,38,26,66,17,48,52,100],"started":"2026-11-10T12:09:18Z","updated":"2026-11-10T12:17:14Z","format_of":null,"group":"Zαβγδζ","latency":{"source":"run","runs":[907],"concurrency":1,"median_request_seconds":0.20652708249690477,"p90_request_seconds":0.2530394938919926},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":27,"correct":27,"interval":[0.8754409628553641,1.0]}}},{"run":50,"model":"qwen38-27b-bf16","name":"Qwen3.8-27B","variant":"yes/no scoring","reasoning":null,"provider":"vllm-yes-no","perplexity_mode":null,"note":"Open-Jev's base model","parameters":27781427952,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head + 0.42B multi-token prediction layer + 0.46B vision encoder","parameters_source":["counted from the checkpoint's safetensors, Qwen/Qwen3.8-27B@1d4bf0f","https://huggingface.co/Qwen/Qwen3.8-27B"],"parameters_read":"2026-11-09","access":"vLLM, self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1141,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":528},"grammar":{"questions":1542,"correct":613},"without_suspect":{"questions":2186,"correct":1135}},"tyt_mix":0.7335931321375471,"questions_per_minute":193.206449833369,"request_seconds":686.3124917121604,"median_request_seconds":null,"p90_request_seconds":null,"billing":"local","api_equivalent_usd":1.5548335,"cost_basis":"maker","cost_upper_bound":false,"requests":2210,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":3043367,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":11050,"previous":{"run":34,"changed":{"questions":372,"older_correct":123,"newer_correct":166,"flipped":103,"changed_answer":174,"newer_wrong":206,"repeated_mistakes":105,"p":2.717641312570181e-05},"unchanged":{"questions":1814,"older_correct":967,"newer_correct":969,"flipped":68,"changed_answer":115,"newer_wrong":845,"repeated_mistakes":765,"p":0.9035973456835125},"added":{"questions":12,"newer_correct":6}},"units":[91,135,51,89,79,83,59,19,41,27,29,21,55,33,27,66,25,62,47,102],"started":"2026-10-04T08:42:28Z","updated":"2026-10-04T08:53:57Z","format_of":null,"group":"Zαβγδζ","latency":{"source":"repeats","runs":[75,89,99,104],"repeats":[2,3,4,5],"questions":2198,"concurrency":1,"median_request_seconds":0.2798689884948544,"p90_request_seconds":0.40955076351528985},"single_question_latency":null,"tag":"yes/no scoring","confidence":{"sure":{"questions":289,"correct":287,"interval":[0.9751222274562773,0.99810015624877]}}},{"run":1255,"model":"Dexy-Decision-9B-V2","name":"Dexy-Decision-9B-V2","variant":null,"reasoning":null,"provider":"di-serve","perplexity_mode":null,"note":"Patrick-18's released Dexy-Decision-9B-V2 (Patrick-18/Dexy-Decision-9B-V2 at 4f434e2), a fine-tune of vLLM-SR's d3-flash (8.394B parameters with its 0.456B vision encoder, as its card says). It ran on our hardware through its card's own server, text only.","parameters":8392695024,"active_parameters":null,"effective_parameters":null,"parameters_scope":"the whole checkpoint with its 0.456B vision encoder (text only here); its card says 8.394B","parameters_source":["counted from the checkpoint's safetensors, Patrick-18/Dexy-Decision-9B-V2@4f434e2","https://huggingface.co/Patrick-18/Dexy-Decision-9B-V2"],"parameters_read":"2026-11-15","access":"Dexy server (the card's Transformers d3_server.py, in its requirements.txt pins: torch 2.14.0, transformers 5.19.0), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1134,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":522},"grammar":{"questions":1542,"correct":612},"without_suspect":{"questions":2186,"correct":1130}},"tyt_mix":0.7259339114548733,"questions_per_minute":1802.3725341144059,"request_seconds":73.17022286116844,"median_request_seconds":0.03119600799982436,"p90_request_seconds":0.0399598092015367,"billing":"local","api_equivalent_usd":0.05060940414564151,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":864336,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[93,133,48,90,79,79,71,27,42,31,34,21,56,35,32,56,22,51,45,89],"started":"2026-11-15T12:11:04Z","updated":"2026-11-15T12:12:23Z","format_of":null,"group":"αβγδζ","latency":{"source":"run","runs":[1255],"concurrency":1,"median_request_seconds":0.03119600799982436,"p90_request_seconds":0.0399598092015367},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":138,"correct":137,"interval":[0.9600963092363003,0.9987197221677447]}}},{"run":1290,"model":"startlux-models/StartLux-Decision-v1.1-9B-Instant","name":"StartLux-Decision-v1.1-9B-Instant","variant":null,"reasoning":null,"provider":"startlux","perplexity_mode":null,"note":"StartLux Labs' released StartLux-Decision-v1.1-9B-Instant (at 18a17f7), a fine-tune of Qwen3.5-9B that reads the prompt bidirectionally and also reads images. It ran on our hardware (one DGX Spark), one request at a time, through the server of its package (startlux-decide serve --launch-vllm, StartLux-Decision at a99977d) on vLLM 0.31.0, with vision on, as its 4B sibling. Its weights are licensed for non-commercial use only (CC BY-NC 4.0); we use them non-commercially, for evaluation. Its card lists English only.","parameters":9653104368,"active_parameters":null,"effective_parameters":null,"parameters_scope":"the whole checkpoint, its vision encoder included","parameters_source":["counted from the checkpoint's safetensors, startlux-models/StartLux-Decision-v1.1-9B-Instant@18a17f7","https://huggingface.co/startlux-models/StartLux-Decision-v1.1-9B-Instant"],"parameters_read":"2026-11-10","access":"StartLux-Decision server (startlux-decide on vLLM 0.31.0, the maker's release code), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1126,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":517},"grammar":{"questions":1542,"correct":609},"without_suspect":{"questions":2186,"correct":1125}},"tyt_mix":0.7193053347727056,"questions_per_minute":382.95695596014866,"request_seconds":344.3729065303196,"median_request_seconds":0.15422002549894387,"p90_request_seconds":0.16897474359429906,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":798396,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[89,133,50,91,75,79,67,21,29,30,38,22,53,39,31,73,18,44,51,93],"started":"2026-11-15T14:18:32Z","updated":"2026-11-15T14:24:24Z","format_of":null,"group":"αβγδζη","latency":{"source":"run","runs":[1290],"concurrency":1,"median_request_seconds":0.15422002549894387,"p90_request_seconds":0.16897474359429906},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":20,"correct":20,"interval":[0.8388698745050667,1.0]}}},{"run":1123,"model":"pplx-decider-flash-v1.3","name":"Perplexity Decider Flash v1.3","variant":"local","reasoning":null,"provider":"pplx-decider","perplexity_mode":null,"note":"Perplexity's released Flash v1.3 checkpoint, distilled from Decider 27B v1.3 on Qwen3.5-4B, run on our hardware through the package it ships.","parameters":4539918336,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + vision encoder + 255-way readout; its locate-span adapter and head (0.13B), used only by its locate task, not counted","parameters_source":["counted from the checkpoint's safetensors, perplexity-ai/pplx-decider-flash-v1.3@d06fb3f","https://huggingface.co/perplexity-ai/pplx-decider-flash-v1.3"],"parameters_read":"2026-11-09","access":"Perplexity Decider server, self-hosted","questions_per_request":1,"concurrency":1,"correct":1093,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":488},"grammar":{"questions":1542,"correct":605},"without_suspect":{"questions":2186,"correct":1090}},"tyt_mix":0.682380342285913,"questions_per_minute":1680.9738363878473,"request_seconds":78.45452269702764,"median_request_seconds":0.03307015349946596,"p90_request_seconds":0.049414389199955625,"billing":"local","api_equivalent_usd":0.054264378198777455,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":818178,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[83,126,50,84,69,76,64,28,32,32,35,20,57,31,30,75,19,39,50,93],"started":"2026-11-14T15:54:43Z","updated":"2026-11-14T15:56:08Z","format_of":null,"group":"βγδζηθ","latency":{"source":"run","runs":[1123],"concurrency":1,"median_request_seconds":0.03307015349946596,"p90_request_seconds":0.049414389199955625},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":29,"correct":29,"interval":[0.8830264055344442,1.0]}}},{"run":1264,"model":"vera-medium-omni","name":"vera-medium-omni","variant":null,"reasoning":null,"provider":"di-serve","perplexity_mode":null,"note":"scar-ai's released vera-medium-omni (scar-ai/vera-medium-omni at 1f23a6c), a decision model on Qwen3.5-4B (about 4.2B parameters for text, plus the stock 0.33B vision tower, unused here) that reads each question in one forward pass. It ran on our hardware through its card's own server, text only; that server rounds each option's probability to 4 decimals (vera/schema.py line 110), so they sum to one only within rounding.","parameters":4207714822,"active_parameters":null,"effective_parameters":null,"parameters_scope":"the whole checkpoint, its vision tower included (text only here)","parameters_source":["counted from the checkpoint's safetensors, scar-ai/vera-medium-omni@1f23a6c","https://huggingface.co/scar-ai/vera-medium-omni"],"parameters_read":"2026-11-15","access":"vera server (the card's own vera-serve from vera-s1 0.1.8, PyTorch engine), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1079,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":495},"grammar":{"questions":1542,"correct":584},"without_suspect":{"questions":2186,"correct":1076}},"tyt_mix":0.6888004274619595,"questions_per_minute":2003.2340784002738,"request_seconds":65.83354457773385,"median_request_seconds":0.02944849800405791,"p90_request_seconds":0.033116961903579066,"billing":"local","api_equivalent_usd":0.04553486833293258,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":864336,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":74709,"previous":null,"units":[86,130,46,88,68,77,63,23,38,28,31,23,50,30,35,76,18,40,40,89],"started":"2026-11-15T12:28:13Z","updated":"2026-11-15T12:29:25Z","format_of":null,"group":"γδζηθ","latency":{"source":"run","runs":[1264],"concurrency":1,"median_request_seconds":0.02944849800405791,"p90_request_seconds":0.033116961903579066},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":4,"correct":4,"interval":[0.5100999795960008,1.0]}}},{"run":965,"model":"PelaAI/KnowLine-4B-Gen5.5","name":"KnowLine 4B Gen5.5","variant":null,"reasoning":null,"provider":"knowline","perplexity_mode":null,"note":"PelaAI's released KnowLine-4B-Gen5.5 (PelaAI/KnowLine-4B-Gen5.5 at 779944d), a merged LoRA fine-tune of Qwen3.5-4B (the average of two) whose weights are those of Gen5. It ran on our hardware (one DGX Spark), one request at a time, through the card's serve_knowline.sh (SGLang 0.5.21 with the weights quantised to FP8 at load, and the repository's knowline_server.py, which reads a request of up to 3,000 tokens twice), with vision on. Its card lists English and Chinese.","parameters":4659865088,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + multi-token prediction layer + 0.33B vision encoder","parameters_source":["counted from the checkpoint's safetensors, PelaAI/KnowLine-4B-Gen5.5@779944d","https://huggingface.co/PelaAI/KnowLine-4B-Gen5.5"],"parameters_read":"2026-11-10","access":"KnowLine server (knowline_server.py over SGLang 0.5.21, PelaAI's release code), self-hosted, FP8 at load","questions_per_request":1,"concurrency":1,"correct":1066,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":491},"grammar":{"questions":1542,"correct":575},"without_suspect":{"questions":2186,"correct":1063}},"tyt_mix":0.6827485388788712,"questions_per_minute":571.3564798985019,"request_seconds":230.81912018120056,"median_request_seconds":0.0994676639966201,"p90_request_seconds":0.1450828845059732,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); FP8 at load","input_tokens":1689108,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[85,125,47,88,70,76,60,27,32,31,29,22,48,28,29,65,24,44,41,95],"started":"2026-11-11T11:17:56Z","updated":"2026-11-11T11:21:52Z","format_of":null,"group":"δζηθ","latency":{"source":"run","runs":[965],"concurrency":1,"median_request_seconds":0.0994676639966201,"p90_request_seconds":0.1450828845059732},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":39,"correct":39,"interval":[0.9103301463997611,1.0]}}},{"run":1236,"model":"startlux-models/StartLux-Decision-v1.1-4B-Instant","name":"StartLux-Decision-v1.1-4B-Instant","variant":null,"reasoning":null,"provider":"startlux","perplexity_mode":null,"note":"StartLux Labs' released StartLux-Decision-v1.1-4B-Instant (at 416088f), a fine-tune of Qwen3.5-4B that reads the prompt bidirectionally and also reads images. It ran on our hardware (one DGX Spark), one request at a time, through the server of its package (startlux-decide serve --launch-vllm, StartLux-Decision at a99977d) on vLLM 0.31.0, with vision on: the card's scores were measured on vLLM 0.19.1, which has no build for the DGX Spark's processor (aarch64). Its weights are licensed for non-commercial use only (CC BY-NC 4.0); we use them non-commercially, for evaluation. Its card lists English only.","parameters":4659865088,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + multi-token prediction layer + 0.33B vision encoder","parameters_source":["counted from the checkpoint's safetensors, startlux-models/StartLux-Decision-v1.1-4B-Instant@416088f","https://huggingface.co/startlux-models/StartLux-Decision-v1.1-4B-Instant"],"parameters_read":"2026-11-10","access":"StartLux-Decision server (startlux-decide on vLLM 0.31.0, the maker's release code), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1052,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":486},"grammar":{"questions":1542,"correct":566},"without_suspect":{"questions":2186,"correct":1051}},"tyt_mix":0.6754390283445637,"questions_per_minute":773.4997300704841,"request_seconds":170.49779705544643,"median_request_seconds":0.07378342649462866,"p90_request_seconds":0.08942243150086142,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":798396,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[81,123,51,88,65,78,70,21,39,36,30,20,46,27,26,66,14,42,38,91],"started":"2026-11-15T10:32:43Z","updated":"2026-11-15T10:35:41Z","format_of":null,"group":"ζηθ","latency":{"source":"run","runs":[1236],"concurrency":1,"median_request_seconds":0.07378342649462866,"p90_request_seconds":0.08942243150086142},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":9,"correct":9,"interval":[0.7008472464490407,1.0]}}},{"run":223,"model":"pplx-decider-flash-v1.2","name":"Perplexity Decider Flash v1.2","variant":"local","reasoning":null,"provider":"pplx-decider","perplexity_mode":null,"note":"Perplexity's released Flash checkpoint, based on Qwen3.5-4B, run on our hardware. An API run is pending; this is an additional local setup.","parameters":4539918336,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + 0.33B vision encoder + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, perplexity-ai/pplx-decider-flash-v1.2@6d771ad","https://huggingface.co/perplexity-ai/pplx-decider-flash-v1.2"],"parameters_read":"2026-11-09","access":"Perplexity Decider server, self-hosted","questions_per_request":1,"concurrency":1,"correct":1050,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":477},"grammar":{"questions":1542,"correct":573},"without_suspect":{"questions":2186,"correct":1048}},"tyt_mix":0.6649148536110847,"questions_per_minute":419.92277077942566,"request_seconds":314.05774865510466,"median_request_seconds":0.13165274250059156,"p90_request_seconds":0.186934632298653,"billing":"local","api_equivalent_usd":0.008533460000000001,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":846752,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[81,123,49,85,61,78,63,25,37,38,33,18,48,28,31,60,12,42,46,92],"started":"2026-10-17T20:36:02Z","updated":"2026-10-17T20:41:22Z","format_of":null,"group":"ζηθ","latency":{"source":"run","runs":[223],"concurrency":1,"median_request_seconds":0.13165274250059156,"p90_request_seconds":0.186934632298653},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1275,"model":"valen","name":"Valen-Decision-1.5-9B","variant":null,"reasoning":null,"provider":"valen","perplexity_mode":null,"note":"Valen-Team's released Valen-Decision-1.5-9B (Valen-Team/Valen-Decision-1.5-9B at 5a51211), a decision model on Qwen3.5-9B. It ran on our hardware through its card's vLLM deployment, the valen-vllm plugin installed without its dependencies into the official vLLM v0.33.0 image, which answers only to the model name 'valen', the key its runs carry; text only.","parameters":8404269597,"active_parameters":null,"effective_parameters":null,"parameters_scope":"the whole checkpoint","parameters_source":["counted from the checkpoint's safetensors, Valen-Team/Valen-Decision-1.5-9B@5a51211","https://huggingface.co/Valen-Team/Valen-Decision-1.5-9B"],"parameters_read":"2026-11-15","access":"Valen server (the card's valen-vllm plugin, Valen-Team/Valen at 773d3a6, in vllm/vllm-openai v0.33.0), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1032,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":492},"grammar":{"questions":1542,"correct":540},"without_suspect":{"questions":2186,"correct":1030}},"tyt_mix":0.6800340466926069,"questions_per_minute":1025.0459396896204,"request_seconds":128.6576483000681,"median_request_seconds":0.05891781899117632,"p90_request_seconds":0.06550232789595611,"billing":"local","api_equivalent_usd":0.08898820674088045,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":756634,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[82,125,43,90,70,82,54,27,39,26,35,19,48,36,27,43,15,39,46,86],"started":"2026-11-15T12:56:56Z","updated":"2026-11-15T12:59:11Z","format_of":null,"group":"θ","latency":{"source":"run","runs":[1275],"concurrency":1,"median_request_seconds":0.05891781899117632,"p90_request_seconds":0.06550232789595611},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":7,"correct":7,"interval":[0.6456611570247934,1.0]}}},{"run":1244,"model":"Verdin-4B-v1.1","name":"Verdin-4B-v1.1","variant":null,"reasoning":null,"provider":"di-serve","perplexity_mode":null,"note":"victorbennett's released Verdin-4B-v1.1 (victorbennett/Verdin-4B-v1.1 at 45830d2): Qwen3.5-4B with a LoRA fine-tune merged into its weights (the mean of three seeds' LoRA runs) and a pointer head that reads the option-boundary tokens, 4.21B parameters served. It ran on our hardware through its card's own server, without causal-conv1d (its torch convolution path), text only. Weights CC-BY-NC-4.0 (research use).","parameters":4205751296,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with the merged LoRA and its pointer head; its card says 4.21B served","parameters_source":["counted from the checkpoint's safetensors, victorbennett/Verdin-4B-v1.1@45830d2","https://huggingface.co/victorbennett/Verdin-4B-v1.1"],"parameters_read":"2026-11-15","access":"Verdin server (the card's own python -m verdin.server, in the package versions its card lists), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1028,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":467},"grammar":{"questions":1542,"correct":561},"without_suspect":{"questions":2186,"correct":1027}},"tyt_mix":0.6509767663946094,"questions_per_minute":1212.9794610414576,"request_seconds":108.7240173768223,"median_request_seconds":0.04870661650056718,"p90_request_seconds":0.05645271801040508,"billing":"local","api_equivalent_usd":0.07520077868563543,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":957679,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[69,125,45,86,67,75,56,29,31,31,30,19,52,27,29,67,13,42,43,92],"started":"2026-11-15T11:35:37Z","updated":"2026-11-15T11:37:32Z","format_of":null,"group":"θ","latency":{"source":"run","runs":[1244],"concurrency":1,"median_request_seconds":0.04870661650056718,"p90_request_seconds":0.05645271801040508},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":27,"correct":27,"interval":[0.8754409628553641,1.0]}}},{"run":1206,"model":"Chinese-Decision-Pointer-4B","name":"Chinese Decision Pointer 4B","variant":null,"reasoning":null,"provider":"di-serve","perplexity_mode":null,"note":"quincyqiang's released Chinese Decision Pointer 4B (quincyqiang/chinese-decision-pointer-4b at 889f23e), a merged LoRA fine-tune of Qwen3.5-4B that reads every candidate's probability in one forward pass. Its card ships only a Decision Index engine (ChineseDecisionEngine from yanqiangmiffy/chinese-decision at c16450b), so it ran on our hardware behind compactor's di_serve.py, with the card's debias 'reverse' (the original and reversed option order averaged), without causal-conv1d (PyTorch's reference convolution), text only.","parameters":4207067905,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its pointer readout, text only","parameters_source":["counted from the checkpoint's safetensors, quincyqiang/chinese-decision-pointer-4b@889f23e","https://huggingface.co/quincyqiang/chinese-decision-pointer-4b"],"parameters_read":"2026-11-15","access":"Decision Index engine (compactor's di_serve.py around the engine its card ships), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":1026,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":465},"grammar":{"questions":1542,"correct":561},"without_suspect":{"questions":2186,"correct":1023}},"tyt_mix":0.6484615224921704,"questions_per_minute":1688.4407887720897,"request_seconds":78.10756579501322,"median_request_seconds":0.03479494549537776,"p90_request_seconds":0.04292104289488634,"billing":"local","api_equivalent_usd":0.05402439967488415,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":1601188,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[75,118,47,86,66,73,51,29,35,27,32,31,45,24,28,54,28,39,43,95],"started":"2026-11-15T07:56:57Z","updated":"2026-11-15T07:58:21Z","format_of":null,"group":"ηθ","latency":{"source":"run","runs":[1206],"concurrency":1,"median_request_seconds":0.03479494549537776,"p90_request_seconds":0.04292104289488634},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":51,"correct":50,"interval":[0.8969519232441061,0.9965303967465577]}}},{"run":205,"model":"microsoft/microsoft-decision-1","name":"Microsoft-Decision-1","variant":"via OpenRouter","reasoning":null,"provider":"openrouter","perplexity_mode":null,"note":"Microsoft's decision model, called through OpenRouter's System One route.","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://commandline.microsoft.com/microsoft-decision-1-model-foundry/"],"parameters_read":"2026-11-09","access":"OpenRouter System One API","questions_per_request":1,"concurrency":1,"correct":1021,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":482},"grammar":{"questions":1542,"correct":539},"without_suspect":{"questions":2186,"correct":1021}},"tyt_mix":0.6673443382050552,"questions_per_minute":202.12929603415938,"request_seconds":652.4536649932852,"median_request_seconds":0.28612651149887824,"p90_request_seconds":0.3269303315973957,"billing":"api","api_equivalent_usd":0.030368352,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":723056,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[86,121,49,86,67,73,61,19,40,24,29,21,47,30,25,56,18,44,42,83],"started":"2026-10-17T15:08:47Z","updated":"2026-10-17T15:19:46Z","format_of":null,"group":"θ","latency":{"source":"run","runs":[205],"concurrency":1,"median_request_seconds":0.28612651149887824,"p90_request_seconds":0.3269303315973957},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":61,"correct":58,"interval":[0.8650865720629857,0.9831343231647693]}}},{"run":1146,"model":"qwen3.5-4b-nothink","name":"Qwen3.5-4B","variant":null,"reasoning":"off","provider":"vllm-plain","perplexity_mode":null,"note":null,"parameters":4659865088,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + multi-token prediction layer + 0.33B vision encoder","parameters_source":["counted from the checkpoint's safetensors, Qwen/Qwen3.5-4B@851bf6e","https://huggingface.co/Qwen/Qwen3.5-4B"],"parameters_read":"2026-11-09","setup_of":"qwen3.5-4b","access":"our GPU (logic's server, vLLM v0.30.0 BF16, revision Qwen/Qwen3.5-4B@851bf6e), one question per request, plain answer line CEVAP: X (no answer schema); sampling min_p 0.0, presence_penalty 1.5, repetition_penalty 1.0, seed 0, temperature 1.0, top_k 20, top_p 0.95, thinking off; output cap 32,768","questions_per_request":1,"concurrency":32,"correct":1015,"invalid":21,"invalid_replies":{"invalid_without_rule":21,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":12,"unreadable":9}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":449},"grammar":{"questions":1542,"correct":566},"without_suspect":{"questions":2186,"correct":1012}},"tyt_mix":0.6289070161494417,"questions_per_minute":46.43560919781533,"request_seconds":2840.0618033930004,"median_request_seconds":32.18796883500181,"p90_request_seconds":74.77572861049848,"billing":"local","api_equivalent_usd":1.964376080680159,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":584163,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":4804243,"previous":null,"units":[74,116,41,81,59,78,60,22,36,34,26,27,38,33,33,69,21,45,37,85],"started":"2026-11-14T21:56:16Z","updated":"2026-11-14T22:43:36Z","format_of":null,"group":"θ","latency":{"source":"run","runs":[1146],"concurrency":32,"median_request_seconds":32.18796883500181,"p90_request_seconds":74.77572861049848},"single_question_latency":null,"tag":"reasoning off · chat"},{"run":139,"model":"clef-flash","name":"Clef-Flash","variant":null,"reasoning":null,"provider":"clef","perplexity_mode":null,"note":"Cloudflare's 9B decision model, post-trained from Qwen3.5-9B","parameters":9531576564,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head + 0.46B vision encoder + joint schema head","parameters_source":["counted from the checkpoint's safetensors, Cloudflare/clef-flash@fde727a","https://huggingface.co/Cloudflare/clef-flash"],"parameters_read":"2026-11-09","access":"Clef server, self-hosted","questions_per_request":1,"concurrency":1,"correct":1005,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":479},"grammar":{"questions":1542,"correct":526},"without_suspect":{"questions":2186,"correct":1001}},"tyt_mix":0.6620961156717599,"questions_per_minute":795.294880781204,"request_seconds":165.8252846673131,"median_request_seconds":0.07536213693674654,"p90_request_seconds":0.08099431772716344,"billing":"local","api_equivalent_usd":0.038172330000000004,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":1004535,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[77,132,42,86,65,77,52,25,33,25,30,17,47,26,26,57,22,40,45,81],"started":"2026-10-07T19:44:42Z","updated":"2026-10-07T19:47:57Z","format_of":null,"group":"θ","latency":{"source":"run","runs":[139],"concurrency":1,"median_request_seconds":0.07536213693674654,"p90_request_seconds":0.08099431772716344},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":131,"model":"qwen3-14b","name":"Qwen3-14B","variant":null,"reasoning":"off","provider":"vllm","perplexity_mode":null,"note":"Erk-14B's base model","parameters":14768307200,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head (text only)","parameters_source":["counted from the checkpoint's safetensors, Qwen/Qwen3-14B@40c0698","https://huggingface.co/Qwen/Qwen3-14B"],"parameters_read":"2026-11-09","access":"vLLM, self-hosted, BF16","questions_per_request":16,"concurrency":1,"correct":1005,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":473},"grammar":{"questions":1542,"correct":532},"without_suspect":{"questions":2186,"correct":1003}},"tyt_mix":0.6552313178165827,"questions_per_minute":331.1300910910813,"request_seconds":398.27247220405843,"median_request_seconds":2.8455167649663053,"p90_request_seconds":3.1145364760770464,"billing":"local","api_equivalent_usd":0.07102272,"cost_basis":"weights","cost_upper_bound":false,"requests":138,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":556424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":17716,"previous":null,"units":[80,115,46,84,72,76,45,19,27,24,24,25,37,25,27,61,24,50,53,91],"started":"2026-10-06T12:25:34Z","updated":"2026-10-06T12:32:15Z","format_of":null,"group":"θ","latency":{"source":"run","runs":[131],"concurrency":1,"median_request_seconds":2.8455167649663053,"p90_request_seconds":3.1145364760770464},"single_question_latency":null,"tag":"reasoning off · chat","confidence":{"sure":{"questions":975,"correct":633,"interval":[0.6187435577014109,0.6785466269413343]}}},{"run":132,"model":"erk-14b","name":"Erk-14B","variant":null,"reasoning":"off","provider":"vllm","perplexity_mode":null,"note":"Qwen3-14B with continued Turkish training (eCloud)","parameters":14768307200,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head (text only)","parameters_source":["counted from the checkpoint's safetensors, ecloudtech/Erk-14B@8584b14","https://huggingface.co/ecloudtech/Erk-14B"],"parameters_read":"2026-11-09","access":"vLLM, self-hosted, BF16","questions_per_request":16,"concurrency":1,"correct":989,"invalid":0,"invalid_replies":{"invalid_without_rule":0,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":474},"grammar":{"questions":1542,"correct":515},"without_suspect":{"questions":2186,"correct":988}},"tyt_mix":0.6545596271867388,"questions_per_minute":392.70250832565097,"request_seconds":335.82673195109237,"median_request_seconds":2.3896605125046335,"p90_request_seconds":2.662659723334946,"billing":"local","api_equivalent_usd":0.23228015626617224,"cost_basis":"gpu","cost_upper_bound":false,"requests":138,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":556424,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":14720,"previous":null,"units":[84,114,46,89,64,77,45,20,27,28,30,16,38,21,25,60,20,48,40,97],"started":"2026-10-06T12:25:34Z","updated":"2026-10-06T12:31:11Z","format_of":null,"group":"θλ","latency":{"source":"run","runs":[132],"concurrency":1,"median_request_seconds":2.3896605125046335,"p90_request_seconds":2.662659723334946},"single_question_latency":null,"tag":"reasoning off · chat","confidence":{"sure":{"questions":101,"correct":96,"interval":[0.8893042659293906,0.9786718046380176]}}},{"run":126,"model":"metask-jev-4b-policy-mix","name":"Metask-Jev-4B","variant":null,"reasoning":null,"provider":"metask","perplexity_mode":null,"note":"Fine-tune of Qwen3.5-4B (merged LoRA)","parameters":4539265536,"active_parameters":null,"effective_parameters":null,"parameters_scope":"Qwen3.5-4B with its LoRA merged: language model (tied embeddings) + 0.33B vision encoder","parameters_source":["counted from the checkpoint's safetensors, wayfind/metask-jev-4b-policy-mix@ea20fe8","https://huggingface.co/wayfind/metask-jev-4b-policy-mix"],"parameters_read":"2026-11-09","access":"Metask-Jev server, self-hosted","questions_per_request":1,"concurrency":1,"correct":896,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":412},"grammar":{"questions":1542,"correct":484},"without_suspect":{"questions":2186,"correct":895}},"tyt_mix":0.5730689079750719,"questions_per_minute":660.6373826690542,"request_seconds":199.62539732037112,"median_request_seconds":0.08993917948100716,"p90_request_seconds":0.09815078696701676,"billing":"local","api_equivalent_usd":0.13807423314659004,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":1137590,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[66,106,38,84,52,66,47,22,36,31,26,21,41,19,21,55,26,36,32,71],"started":"2026-10-06T09:45:14Z","updated":"2026-10-06T09:48:37Z","format_of":null,"group":"λμ","latency":{"source":"run","runs":[126],"concurrency":1,"median_request_seconds":0.08993917948100716,"p90_request_seconds":0.09815078696701676},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1079,"model":"PelaAI/KnowLine-2B-Gen1","name":"KnowLine-2B-Gen1","variant":null,"reasoning":null,"provider":"knowline","perplexity_mode":null,"note":"PelaAI's released KnowLine-2B-Gen1 (PelaAI/KnowLine-2B-Gen1 at 53245d4), a merged LoRA fine-tune of Qwen3.5-2B with its readout temperature (T = 1.32) folded into the weights. It ran on our hardware (one DGX Spark), one request at a time, through the card's serve_knowline.sh (SGLang 0.5.21 in BF16 and the repository's knowline_server.py, which reads a request of up to 3,000 tokens twice), with vision on: its vision tower is the base model's, not trained on images. Its card lists English and Chinese.","parameters":2274069824,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + multi-token prediction layer + 0.33B vision encoder","parameters_source":["counted from the checkpoint's safetensors, PelaAI/KnowLine-2B-Gen1@53245d4","https://huggingface.co/PelaAI/KnowLine-2B-Gen1"],"parameters_read":"2026-11-11","access":"KnowLine server (knowline_server.py over SGLang 0.5.21, PelaAI's release code), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":809,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":356},"grammar":{"questions":1542,"correct":453},"without_suspect":{"questions":2186,"correct":809}},"tyt_mix":0.4991239204707222,"questions_per_minute":914.2484364980592,"request_seconds":144.24963142967317,"median_request_seconds":0.062147766002453864,"p90_request_seconds":0.08494557262747549,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":1689108,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[53,95,33,71,43,61,41,19,33,24,24,14,41,23,23,54,17,38,43,59],"started":"2026-11-13T15:54:23Z","updated":"2026-11-13T15:56:53Z","format_of":null,"group":"μξ","latency":{"source":"run","runs":[1079],"concurrency":1,"median_request_seconds":0.062147766002453864,"p90_request_seconds":0.08494557262747549},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":915,"model":"glide-2b","name":"GLiDE 2B","variant":"local","reasoning":null,"provider":"glide","perplexity_mode":null,"note":"Fastino's released GLiDE 2B (fastino/GLiDE-2B at 00f122c), on Qwen3.5-2B, run on our hardware through the server it ships. Its checkpoint holds no vision encoder, so its server takes text only.","parameters":1882347328,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, fastino/GLiDE-2B@00f122c","https://huggingface.co/fastino/GLiDE-2B"],"parameters_read":"2026-11-10","access":"GLiDE server (Fastino's release code), self-hosted","questions_per_request":1,"concurrency":1,"correct":776,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":336},"grammar":{"questions":1542,"correct":440},"without_suspect":{"questions":2186,"correct":773}},"tyt_mix":0.4724961247666951,"questions_per_minute":1671.052410550383,"request_seconds":78.92032539934735,"median_request_seconds":0.03431983401242178,"p90_request_seconds":0.04401278798759449,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":789604,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[58,89,29,67,47,46,46,22,30,23,20,16,34,22,19,53,18,35,39,63],"started":"2026-11-10T12:45:53Z","updated":"2026-11-10T12:47:18Z","format_of":null,"group":"ξ","latency":{"source":"run","runs":[915],"concurrency":1,"median_request_seconds":0.03431983401242178,"p90_request_seconds":0.04401278798759449},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":19,"correct":19,"interval":[0.8318156346315496,1.0]}}},{"run":977,"model":"startlux-models/StartLux-Decision-v1.1-2B-Instant","name":"StartLux-Decision-v1.1-2B-Instant","variant":null,"reasoning":null,"provider":"startlux","perplexity_mode":null,"note":"StartLux Labs' released StartLux-Decision-v1.1-2B-Instant (at f1e6cd1), a fine-tune of Qwen3.5-2B that reads the prompt bidirectionally and also reads images. It ran on our hardware (one DGX Spark), one request at a time, through the server of its package (startlux-decide serve --launch-vllm, StartLux-Decision at a99977d) on vLLM 0.31.0, with vision on: the card's scores were measured on vLLM 0.19.1, which has no build for the DGX Spark's processor (aarch64). Its weights are licensed for non-commercial use only (CC BY-NC 4.0); we use them non-commercially, for evaluation. Its card lists English only.","parameters":2274069824,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + multi-token prediction layer + 0.33B vision encoder","parameters_source":["counted from the checkpoint's safetensors, startlux-models/StartLux-Decision-v1.1-2B-Instant@f1e6cd1","https://huggingface.co/startlux-models/StartLux-Decision-v1.1-2B-Instant"],"parameters_read":"2026-11-11","access":"StartLux-Decision server (startlux-decide on vLLM 0.31.0, the maker's release code), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":753,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":347},"grammar":{"questions":1542,"correct":406},"without_suspect":{"questions":2186,"correct":752}},"tyt_mix":0.48247134106798273,"questions_per_minute":1461.2210408642334,"request_seconds":90.25328565074597,"median_request_seconds":0.03998677650815807,"p90_request_seconds":0.04615934189932887,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":798396,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[55,93,33,72,44,50,43,21,29,22,25,17,35,20,22,54,15,34,21,48],"started":"2026-11-11T12:31:25Z","updated":"2026-11-11T12:32:58Z","format_of":null,"group":"ξπ","latency":{"source":"run","runs":[977],"concurrency":1,"median_request_seconds":0.03998677650815807,"p90_request_seconds":0.04615934189932887},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1037,"model":"lfm2.5-2.6b","name":"LFM2.5-2.6B","variant":null,"reasoning":"default","provider":"vllm-plain","perplexity_mode":null,"note":null,"parameters":2697198592,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings, text only)","parameters_source":["counted from the checkpoint's safetensors, LiquidAI/LFM2.5-2.6B@7141b56","https://huggingface.co/LiquidAI/LFM2.5-2.6B"],"parameters_read":"2026-11-12","access":"our GPU (logic's server, vLLM v0.30.0 BF16, revision LiquidAI/LFM2.5-2.6B@7141b56), one question per request, plain answer line CEVAP: X (no answer schema); sampling repetition_penalty 1.1, seed 0, temperature 0.1, top_k 50; output cap 32,768","questions_per_request":1,"concurrency":32,"correct":745,"invalid":40,"invalid_replies":{"invalid_without_rule":40,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":9,"unreadable":31}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":298},"grammar":{"questions":1542,"correct":447},"without_suspect":{"questions":2186,"correct":743}},"tyt_mix":0.4255009134478504,"questions_per_minute":33.153948729613106,"request_seconds":3977.806718455977,"median_request_seconds":51.20819945249241,"p90_request_seconds":96.91006861727801,"billing":"local","api_equivalent_usd":6.0685373,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":600368,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":11716817,"previous":null,"units":[44,76,35,59,38,46,45,12,33,20,17,19,54,21,24,51,20,29,42,60],"started":"2026-11-12T04:09:47Z","updated":"2026-11-12T05:16:04Z","format_of":null,"group":"ξπσ","latency":{"source":"run","runs":[1037],"concurrency":32,"median_request_seconds":51.20819945249241,"p90_request_seconds":96.91006861727801},"single_question_latency":null,"tag":"default reasoning · chat"},{"run":1089,"model":"lfm3-1.7b","name":"LFM3-1.7B","variant":null,"reasoning":"on","provider":"vllm-plain","perplexity_mode":null,"note":null,"parameters":1728801536,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings, text only)","parameters_source":["counted from the checkpoint's safetensors, LiquidAI/LFM3-1.7B@0dc4268","https://huggingface.co/LiquidAI/LFM3-1.7B"],"parameters_read":"2026-11-09","access":"our GPU (logic's server, vLLM nightly 0.34.0rc1.dev302 (commit be47189f; no released vLLM serves LFM3) BF16, revision LiquidAI/LFM3-1.7B@0dc4268), one question per request, plain answer line CEVAP: X (no answer schema); sampling min_p 0.0, repetition_penalty 1.1, seed 0, temperature 0.1, top_k 50, top_p 1.0, thinking on; output cap 32,768","questions_per_request":1,"concurrency":32,"correct":712,"invalid":3,"invalid_replies":{"invalid_without_rule":3,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":3}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":290},"grammar":{"questions":1542,"correct":422},"without_suspect":{"questions":2186,"correct":708}},"tyt_mix":0.4126027134541773,"questions_per_minute":59.95805846780944,"request_seconds":2199.5375329040107,"median_request_seconds":24.186758014024235,"p90_request_seconds":65.84055170980282,"billing":"local","api_equivalent_usd":2.80680084,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":600368,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":13793857,"previous":null,"units":[44,89,26,51,38,42,50,15,25,15,17,20,39,24,19,48,15,36,43,56],"started":"2026-11-13T18:21:39Z","updated":"2026-11-13T18:58:18Z","format_of":null,"group":"ξπσφ","latency":{"source":"run","runs":[1089],"concurrency":32,"median_request_seconds":24.186758014024235,"p90_request_seconds":65.84055170980282},"single_question_latency":null,"tag":"reasoning on · chat"},{"run":340,"model":"LiquidAI/d1-2B","name":"d1-2B","variant":null,"reasoning":null,"provider":"d1-server","perplexity_mode":null,"note":"Liquid AI's released d1-2B weights (LiquidAI/d1-2B at 10e7671), a decision model built on LFM3-VL-2B that also reads images. It ran on our hardware through its model card's own call (model.system_one, Liquid's model code in Transformers) behind our HTTP server (servers/d1_server.py), one request at a time. Liquid lists 16 languages for it; Turkish is not among them. Its request times come from one GPU with no other known work while its runs took place, though the GPU was not monitored throughout. Only this model's server showed on it at 11:55 UTC on 1 November 2026, before the runs, and nothing at 12:35 UTC, after them; the one other project with a claim on that GPU confirmed it ran nothing there in between.","parameters":1918181888,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + 0.16B vision encoder + projector","parameters_source":["counted from the checkpoint's safetensors, LiquidAI/d1-2B@10e7671","https://huggingface.co/LiquidAI/d1-2B"],"parameters_read":"2026-11-09","access":"d1 server (Liquid's model code in Transformers, the model card's call), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":646,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":268},"grammar":{"questions":1542,"correct":378},"without_suspect":{"questions":2186,"correct":643}},"tyt_mix":0.3799415156116542,"questions_per_minute":3150.5652888015998,"request_seconds":41.85915475827642,"median_request_seconds":0.019204194002668373,"p90_request_seconds":0.021930550708202646,"billing":"local","api_equivalent_usd":0.02895258204114119,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":745982,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[50,67,27,54,30,40,28,15,25,15,20,15,39,25,22,46,19,27,26,56],"started":"2026-11-01T12:00:08Z","updated":"2026-11-01T12:00:56Z","format_of":null,"group":"πσφψ","latency":{"source":"run","runs":[340],"concurrency":1,"median_request_seconds":0.019204194002668373,"p90_request_seconds":0.021930550708202646},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":923,"model":"glide-0.8b","name":"GLiDE 0.8B","variant":"local","reasoning":null,"provider":"glide","perplexity_mode":null,"note":"Fastino's released GLiDE 0.8B (fastino/GLiDE-0.8B at 9eacdee), on Qwen3.5-0.8B, run on our hardware through the server it ships. Its checkpoint holds no vision encoder, so its server takes text only.","parameters":752654144,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (no LM head) + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, fastino/GLiDE-0.8B@9eacdee","https://huggingface.co/fastino/GLiDE-0.8B"],"parameters_read":"2026-11-10","access":"GLiDE server (Fastino's release code), self-hosted","questions_per_request":1,"concurrency":1,"correct":643,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":278},"grammar":{"questions":1542,"correct":365},"without_suspect":{"questions":2186,"correct":640}},"tyt_mix":0.3910423784442124,"questions_per_minute":2538.085007878885,"request_seconds":51.960434575914405,"median_request_seconds":0.022486086003482342,"p90_request_seconds":0.02896486108948011,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":789604,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[33,80,26,59,39,41,35,14,19,15,27,23,31,12,16,41,23,27,30,52],"started":"2026-11-10T12:56:04Z","updated":"2026-11-10T12:57:01Z","format_of":null,"group":"σφψ","latency":{"source":"run","runs":[923],"concurrency":1,"median_request_seconds":0.022486086003482342,"p90_request_seconds":0.02896486108948011},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":18,"correct":17,"interval":[0.7424220019799247,0.9901250641690756]}}},{"run":1029,"model":"OmniJev/OneJev1.6-0.8B","name":"OneJev1.6-0.8B","variant":null,"reasoning":null,"provider":"onejev","perplexity_mode":null,"note":"OmniJev's released OneJev1.6-0.8B (OmniJev/OneJev1.6-0.8B at bcd88a3, whose files but its README are those of c89933c), an average of three full fine-tunes of Qwen3.5-0.8B that also reads images. It ran on our hardware (one DGX Spark), one request at a time, through the server its card names (Flide 0.1.11, flide serve), with vision on (torchvision 0.29.1). Probabilities rounded to 6 decimals by Flide; they sum to 1 only within that rounding.","parameters":852985920,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + 0.10B vision encoder","parameters_source":["counted from the checkpoint's safetensors, OmniJev/OneJev1.6-0.8B@bcd88a3","https://huggingface.co/OmniJev/OneJev1.6-0.8B"],"parameters_read":"2026-11-11","access":"Flide server (flide 0.1.11, the card's server for OneJev 1.6), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":640,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":260},"grammar":{"questions":1542,"correct":380},"without_suspect":{"questions":2186,"correct":638}},"tyt_mix":0.37010751795261143,"questions_per_minute":3363.7371481648565,"request_seconds":39.20639282767661,"median_request_seconds":0.01743363050627522,"p90_request_seconds":0.021503209782531486,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":870930,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[41,69,25,48,33,44,40,14,19,14,21,11,34,20,19,51,17,26,33,61],"started":"2026-11-11T21:05:47Z","updated":"2026-11-11T21:06:32Z","format_of":null,"group":"σφψ","latency":{"source":"run","runs":[1029],"concurrency":1,"median_request_seconds":0.01743363050627522,"p90_request_seconds":0.021503209782531486},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1015,"model":"startlux-models/StartLux-Decision-v1.1-0.8B-Instant","name":"StartLux-Decision-v1.1-0.8B-Instant","variant":null,"reasoning":null,"provider":"startlux","perplexity_mode":null,"note":"StartLux Labs' released StartLux-Decision-v1.1-0.8B-Instant (at 6496ae6), a fine-tune of Qwen3.5-0.8B that reads the prompt bidirectionally and also reads images. It ran on our hardware (one DGX Spark), one request at a time, through the server of its package (startlux-decide serve --launch-vllm, StartLux-Decision at a99977d) on vLLM 0.31.0, with vision on: the card's scores were measured on vLLM 0.19.1, which has no build for the DGX Spark's processor (aarch64). Its weights are licensed for non-commercial use only (CC BY-NC 4.0); we use them non-commercially, for evaluation. Its card lists English only.","parameters":873438784,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + multi-token prediction layer + 0.10B vision encoder","parameters_source":["counted from the checkpoint's safetensors, startlux-models/StartLux-Decision-v1.1-0.8B-Instant@6496ae6","https://huggingface.co/startlux-models/StartLux-Decision-v1.1-0.8B-Instant"],"parameters_read":"2026-11-11","access":"StartLux-Decision server (startlux-decide on vLLM 0.31.0, the maker's release code), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":628,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":230},"grammar":{"questions":1542,"correct":398},"without_suspect":{"questions":2186,"correct":625}},"tyt_mix":0.3344216609724463,"questions_per_minute":2129.034310282692,"request_seconds":61.94357665494317,"median_request_seconds":0.02780695651017595,"p90_request_seconds":0.0317301560920896,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":798396,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[39,52,25,47,31,36,44,16,23,10,19,13,44,17,18,51,20,23,41,59],"started":"2026-11-11T19:42:08Z","updated":"2026-11-11T19:43:17Z","format_of":null,"group":"φψ","latency":{"source":"run","runs":[1015],"concurrency":1,"median_request_seconds":0.02780695651017595,"p90_request_seconds":0.0317301560920896},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1227,"model":"lfm3-1.7b-vllm-nothink","name":"LFM3-1.7B","variant":null,"reasoning":"off","provider":"vllm-plain","perplexity_mode":null,"note":null,"parameters":1728801536,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings, text only)","parameters_source":["counted from the checkpoint's safetensors, LiquidAI/LFM3-1.7B@0dc4268","https://huggingface.co/LiquidAI/LFM3-1.7B"],"parameters_read":"2026-11-09","setup_of":"lfm3-1.7b","access":"our GPU (logic's server, vLLM nightly 0.34.0rc1.dev302 (commit be47189f; no released vLLM serves LFM3) BF16, revision LiquidAI/LFM3-1.7B@0dc4268), one question per request, plain answer line CEVAP: X (no answer schema); sampling min_p 0.0, repetition_penalty 1.1, seed 0, temperature 0.1, top_k 50, top_p 1.0, thinking off; output cap 32,768","questions_per_request":1,"concurrency":32,"correct":623,"invalid":21,"invalid_replies":{"invalid_without_rule":21,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":17,"unreadable":4}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":244},"grammar":{"questions":1542,"correct":379},"without_suspect":{"questions":2186,"correct":618}},"tyt_mix":0.34987207775774254,"questions_per_minute":116.18522106538138,"request_seconds":1135.084125078065,"median_request_seconds":4.901049380001496,"p90_request_seconds":43.6483753145003,"billing":"local","api_equivalent_usd":1.41993308,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":602566,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":6858639,"previous":null,"units":[34,73,30,48,37,22,38,22,28,21,10,12,30,18,19,54,11,33,34,49],"started":"2026-11-15T09:25:01Z","updated":"2026-11-15T09:43:56Z","format_of":null,"group":"φψΓ","latency":{"source":"run","runs":[1227],"concurrency":32,"median_request_seconds":4.901049380001496,"p90_request_seconds":43.6483753145003},"single_question_latency":null,"tag":"reasoning off · chat"},{"run":141,"model":"strands-decider-2b","name":"Strands Decider 2B","variant":null,"reasoning":null,"provider":"strands","perplexity_mode":null,"note":"Amazon's decision model: a LoRA adapter and a decision head on Qwen3.5-2B-Base","parameters":2291942208,"active_parameters":null,"effective_parameters":null,"parameters_scope":"Qwen3.5-2B-Base (with its vision encoder) + LoRA adapter + readout head; the repo holds only the adapter and head","parameters_source":["counted from the checkpoints' safetensors, StrandsAgents/strands-decider-2B-hobson-v21@2b52a62 on Qwen/Qwen3.5-2B-Base@b1485b2","https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v21"],"parameters_read":"2026-11-09","access":"Strands Decider server, self-hosted","questions_per_request":1,"concurrency":1,"correct":604,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":242},"grammar":{"questions":1542,"correct":362},"without_suspect":{"questions":2186,"correct":602}},"tyt_mix":0.34542752127424,"questions_per_minute":2062.4513111246683,"request_seconds":63.94332767452579,"median_request_seconds":0.026102936477400362,"p90_request_seconds":0.0304201258928515,"billing":"local","api_equivalent_usd":0.044227468308213676,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":789516,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[42,71,23,59,35,12,35,15,29,17,19,10,26,15,17,49,15,35,29,51],"started":"2026-10-07T19:49:08Z","updated":"2026-10-07T19:50:14Z","format_of":null,"group":"φψΓ","latency":{"source":"run","runs":[141],"concurrency":1,"median_request_seconds":0.026102936477400362,"p90_request_seconds":0.0304201258928515},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1005,"model":"PelaAI/KnowLine-0.8B-Gen1","name":"KnowLine-0.8B-Gen1","variant":null,"reasoning":null,"provider":"knowline","perplexity_mode":null,"note":"PelaAI's released KnowLine-0.8B-Gen1 (PelaAI/KnowLine-0.8B-Gen1 at 6b27831), a merged LoRA fine-tune of Qwen3.5-0.8B with its readout temperature (T = 1.54) folded into the weights. It ran on our hardware (one DGX Spark), one request at a time, through the card's serve_knowline.sh (SGLang 0.5.21 in BF16 and the repository's knowline_server.py, which reads a request of up to 3,000 tokens twice), with vision on: its vision tower is the base model's, not trained on images. Its card lists English and Chinese.","parameters":873438784,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + multi-token prediction layer + 0.10B vision encoder","parameters_source":["counted from the checkpoint's safetensors, PelaAI/KnowLine-0.8B-Gen1@6b27831","https://huggingface.co/PelaAI/KnowLine-0.8B-Gen1"],"parameters_read":"2026-11-11","access":"KnowLine server (knowline_server.py over SGLang 0.5.21, PelaAI's release code), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":596,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":251},"grammar":{"questions":1542,"correct":345},"without_suspect":{"questions":2186,"correct":593}},"tyt_mix":0.354816806254152,"questions_per_minute":1702.1939497451015,"request_seconds":77.47648264156305,"median_request_seconds":0.03282420900359284,"p90_request_seconds":0.04675902890157886,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":1689108,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2198,"previous":null,"units":[41,64,25,53,32,36,32,9,22,8,17,19,33,17,19,35,16,29,39,50],"started":"2026-11-11T18:18:21Z","updated":"2026-11-11T18:19:45Z","format_of":null,"group":"ψΓΔ","latency":{"source":"run","runs":[1005],"concurrency":1,"median_request_seconds":0.03282420900359284,"p90_request_seconds":0.04675902890157886},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":243,"model":"decider-1","name":"Decider 1","variant":null,"reasoning":null,"provider":"meragpt","perplexity_mode":null,"note":"Hosted decision model; meraGPT serves its current version under this name","parameters":null,"active_parameters":null,"effective_parameters":null,"parameters_scope":null,"parameters_source":["https://meragpt.com/models/decider-1","https://meragpt.com/blog/introducing-decider-1"],"parameters_read":"2026-11-09","access":"meraGPT System One API","questions_per_request":1,"concurrency":1,"correct":596,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":227},"grammar":{"questions":1542,"correct":369},"without_suspect":{"questions":2186,"correct":595}},"tyt_mix":0.32735761483344405,"questions_per_minute":153.69089032800073,"request_seconds":858.085991424392,"median_request_seconds":0.38244215249142144,"p90_request_seconds":0.4169523003976792,"billing":"api","api_equivalent_usd":0.032449,"cost_basis":"api","cost_upper_bound":false,"requests":2198,"hardware":null,"input_tokens":811225,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[35,46,24,53,33,36,42,17,23,14,23,14,36,16,13,38,17,28,30,58],"started":"2026-10-19T10:31:37Z","updated":"2026-10-19T10:46:02Z","format_of":null,"group":"ψΓΔ","latency":{"source":"run","runs":[243],"concurrency":1,"median_request_seconds":0.38244215249142144,"p90_request_seconds":0.4169523003976792},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1218,"model":"qwen3.5-2b-nothink","name":"Qwen3.5-2B","variant":null,"reasoning":"off","provider":"vllm-plain","perplexity_mode":null,"note":null,"parameters":2274069824,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model (tied embeddings) + multi-token prediction layer + 0.33B vision encoder","parameters_source":["counted from the checkpoint's safetensors, Qwen/Qwen3.5-2B@15852e8","https://huggingface.co/Qwen/Qwen3.5-2B"],"parameters_read":"2026-11-15","setup_of":"qwen3.5-2b","access":"our GPU (logic's server, vLLM v0.30.0 BF16, revision Qwen/Qwen3.5-2B@15852e8), one question per request, plain answer line CEVAP: X (no answer schema); sampling min_p 0.0, presence_penalty 1.5, repetition_penalty 1.0, seed 0, temperature 1.0, top_k 20, top_p 0.95, thinking off; output cap 32,768","questions_per_request":1,"concurrency":32,"correct":558,"invalid":154,"invalid_replies":{"invalid_without_rule":154,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":8,"unreadable":146}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":225},"grammar":{"questions":1542,"correct":333},"without_suspect":{"questions":2186,"correct":556}},"tyt_mix":0.32075676781816453,"questions_per_minute":155.11066376833074,"request_seconds":850.2316784419963,"median_request_seconds":8.840266972998506,"p90_request_seconds":20.075167110607435,"billing":"local","api_equivalent_usd":0.5880769109223809,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":584163,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":2808965,"previous":null,"units":[39,55,18,47,33,33,29,12,17,20,16,13,34,16,16,48,13,24,21,54],"started":"2026-11-15T08:49:35Z","updated":"2026-11-15T09:03:46Z","format_of":null,"group":"ψΓΔΘ","latency":{"source":"run","runs":[1218],"concurrency":32,"median_request_seconds":8.840266972998506,"p90_request_seconds":20.075167110607435},"single_question_latency":null,"tag":"reasoning off · chat"},{"run":1046,"model":"knowledgator/gliclass-v4-edge","name":"GLiClass v4 Edge","variant":null,"reasoning":null,"provider":"gliclass","perplexity_mode":null,"note":"Knowledgator's released GLiClass v4 Edge (knowledgator/gliclass-v4-edge at 885af1a), a zero-shot classifier fine-tuned from DeBERTa-v3-large (supervised training, reinforcement learning, then a calibrator). Its card serves it as a library pipeline (gliclass) and maps a decision request onto it: the passage is the text, the instructions the prompt, each option a label, and a softmax over the calibrated label scores. It ran on our hardware (one DGX Spark), one request at a time, through our HTTP server (servers/gliclass_server.py) with that mapping and gliclass 0.1.22, in BF16, text only; a request over 16,384 tokens (the card's limit) is refused, never cut. Its card lists English only.","parameters":439790596,"active_parameters":null,"effective_parameters":null,"parameters_scope":"DeBERTa-v3-large encoder + label projectors and scorer + calibrator (text only)","parameters_source":["counted from the checkpoint's safetensors, knowledgator/gliclass-v4-edge@885af1a","https://huggingface.co/knowledgator/gliclass-v4-edge"],"parameters_read":"2026-11-11","access":"GLiClass server (our servers/gliclass_server.py: the card's gliclass pipeline and its System One mapping), self-hosted, BF16","questions_per_request":1,"concurrency":1,"correct":513,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":170},"grammar":{"questions":1542,"correct":343},"without_suspect":{"questions":2186,"correct":510}},"tyt_mix":0.2527224502546582,"questions_per_minute":2104.6484652435274,"request_seconds":62.661295783065725,"median_request_seconds":0.02523164649028331,"p90_request_seconds":0.0421514681132976,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":870118,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[32,44,15,42,22,15,27,13,16,16,20,18,40,18,21,51,16,27,24,36],"started":"2026-11-12T08:38:26Z","updated":"2026-11-12T08:39:34Z","format_of":null,"group":"ΓΔΘΛ","latency":{"source":"run","runs":[1046],"concurrency":1,"median_request_seconds":0.02523164649028331,"p90_request_seconds":0.0421514681132976},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":931,"model":"glide-0.4b","name":"GLiDE 0.4B","variant":"local","reasoning":null,"provider":"glide","perplexity_mode":null,"note":"Fastino's released GLiDE 0.4B (fastino/GLiDE-0.4B at 6bda7ab), an encoder (ModernBERT architecture, from Ettin-encoder-400m) that reads at most 7,999 tokens, run on our hardware through the server it ships.","parameters":395042816,"active_parameters":null,"effective_parameters":null,"parameters_scope":"encoder (no MLM head) + 255-way readout","parameters_source":["counted from the checkpoint's safetensors, fastino/GLiDE-0.4B@6bda7ab","https://huggingface.co/fastino/GLiDE-0.4B"],"parameters_read":"2026-11-10","access":"GLiDE server (Fastino's release code), self-hosted","questions_per_request":1,"concurrency":1,"correct":489,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":161},"grammar":{"questions":1542,"correct":328},"without_suspect":{"questions":2186,"correct":486}},"tyt_mix":0.2397015180633324,"questions_per_minute":3608.4674550193217,"request_seconds":36.54737132700393,"median_request_seconds":0.015583213011268526,"p90_request_seconds":0.02118301391310524,"billing":"local","api_equivalent_usd":null,"cost_basis":null,"cost_upper_bound":false,"requests":2198,"hardware":"1 × DGX Spark, GB10 (128 GB unified memory); BF16","input_tokens":1156776,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[29,44,11,31,25,21,35,9,20,14,18,14,32,15,15,37,12,31,31,45],"started":"2026-11-10T13:03:56Z","updated":"2026-11-10T13:04:39Z","format_of":null,"group":"ΔΘΛΞ","latency":{"source":"run","runs":[931],"concurrency":1,"median_request_seconds":0.015583213011268526,"p90_request_seconds":0.02118301391310524},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1288,"model":"lfm2.5-8b-a1b","name":"LFM2.5-8B-A1B","variant":null,"reasoning":"default","provider":"vllm-plain","perplexity_mode":null,"note":null,"parameters":8467856832,"active_parameters":[1685667264],"effective_parameters":null,"parameters_scope":"mixture of experts: language model (tied embeddings, text only), counted; the card says 8.3B total, 1.5B active","parameters_source":["counted from the checkpoint's safetensors, LiquidAI/LFM2.5-8B-A1B@dd83f5c","https://huggingface.co/LiquidAI/LFM2.5-8B-A1B"],"parameters_read":"2026-11-09","access":"our GPU (logic's server, vLLM v0.30.0 BF16, revision LiquidAI/LFM2.5-8B-A1B@dd83f5c), one question per request, plain answer line CEVAP: X (no answer schema); sampling repetition_penalty 1.05, seed 0, temperature 0.2, top_k 80; output cap 32,768","questions_per_request":1,"concurrency":32,"correct":461,"invalid":346,"invalid_replies":{"invalid_without_rule":346,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":0,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":6,"unreadable":340}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":168},"grammar":{"questions":1542,"correct":293},"without_suspect":{"questions":2186,"correct":460}},"tyt_mix":0.2445327575843852,"questions_per_minute":39.532904544131426,"request_seconds":3335.955238320006,"median_request_seconds":39.22178941198945,"p90_request_seconds":77.80463325590244,"billing":"local","api_equivalent_usd":2.3073690398380045,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":598170,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":5576817,"previous":null,"units":[30,43,16,34,22,23,15,15,12,17,13,11,29,12,15,40,14,25,24,51],"started":"2026-11-15T14:05:44Z","updated":"2026-11-15T15:01:20Z","format_of":null,"group":"ΘΛΞΠ","latency":{"source":"run","runs":[1288],"concurrency":32,"median_request_seconds":39.22178941198945,"p90_request_seconds":77.80463325590244},"single_question_latency":null,"tag":"default reasoning · chat"},{"run":135,"model":"gliner2.5-multi-decide","name":"GLiNER2.5-multi-Decide","variant":null,"reasoning":null,"provider":"gliner","perplexity_mode":null,"note":"Fastino's decision fine-tune of GLiNER2.5 Multi","parameters":287355159,"active_parameters":null,"effective_parameters":null,"parameters_scope":"mDeBERTa-v3-base encoder + extraction and classification heads","parameters_source":["counted from the checkpoint's safetensors, fastino/GLiNER2.5-multi-Decide@a35a0cd","https://huggingface.co/fastino/GLiNER2.5-multi-Decide"],"parameters_read":"2026-11-09","access":"GLiNER2 server, self-hosted","questions_per_request":1,"concurrency":1,"correct":444,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":138},"grammar":{"questions":1542,"correct":306},"without_suspect":{"questions":2186,"correct":440}},"tyt_mix":0.20827945572743664,"questions_per_minute":6615.931772680665,"request_seconds":19.933700124383904,"median_request_seconds":0.008824052987620234,"p90_request_seconds":0.010334555187728255,"billing":"local","api_equivalent_usd":0.013787475919365536,"cost_basis":"gpu","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); FP16","input_tokens":431150,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[29,36,12,22,15,24,25,8,23,9,14,14,37,18,15,39,15,26,27,36],"started":"2026-10-07T17:25:44Z","updated":"2026-10-07T17:26:07Z","format_of":null,"group":"ΘΛΞΠ","latency":{"source":"run","runs":[135],"concurrency":1,"median_request_seconds":0.008824052987620234,"p90_request_seconds":0.010334555187728255},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":51,"model":"laya","name":"Laya","variant":null,"reasoning":null,"provider":"laya","perplexity_mode":null,"note":"Encoder with a decision head","parameters":421293827,"active_parameters":null,"effective_parameters":null,"parameters_scope":"ModernBERT-large encoder + decision head","parameters_source":["counted from the checkpoint's safetensors, convaiinnovations/laya@c5d7873","https://huggingface.co/convaiinnovations/laya"],"parameters_read":"2026-11-09","access":"Laya server, self-hosted","questions_per_request":1,"concurrency":1,"correct":418,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":137},"grammar":{"questions":1542,"correct":281},"without_suspect":{"questions":2186,"correct":413}},"tyt_mix":0.20418460939230015,"questions_per_minute":4612.493508939851,"request_seconds":28.74800793605391,"median_request_seconds":null,"p90_request_seconds":null,"billing":"local","api_equivalent_usd":0.01988403882243729,"cost_basis":"gpu","cost_upper_bound":false,"requests":2210,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB)","input_tokens":891078,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":{"run":41,"changed":{"questions":372,"older_correct":68,"newer_correct":60,"flipped":46,"changed_answer":99,"newer_wrong":312,"repeated_mistakes":232,"p":0.3019956131896606},"unchanged":{"questions":1814,"older_correct":356,"newer_correct":356,"flipped":0,"changed_answer":0,"newer_wrong":1458,"repeated_mistakes":1458,"p":1.0},"added":{"questions":12,"newer_correct":2}},"units":[33,35,8,18,28,15,18,10,15,10,13,10,33,12,23,47,17,22,17,34],"started":"2026-10-04T08:46:03Z","updated":"2026-10-04T08:46:34Z","format_of":null,"group":"ΛΞΠ","latency":{"source":"repeats","runs":[234,235,236,237],"repeats":[2,3,4,5],"questions":2198,"concurrency":1,"median_request_seconds":0.011380975003703497,"p90_request_seconds":0.012440272996900602},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":46,"model":"laya-multilingual","name":"Laya Multilingual","variant":null,"reasoning":null,"provider":"laya","perplexity_mode":null,"note":"Encoder with a decision head","parameters":321908995,"active_parameters":null,"effective_parameters":null,"parameters_scope":"mmBERT-base encoder + decision head","parameters_source":["counted from the checkpoint's safetensors, convaiinnovations/laya@c5d7873 (multilingual/)","https://huggingface.co/convaiinnovations/laya"],"parameters_read":"2026-11-09","access":"Laya server, self-hosted","questions_per_request":1,"concurrency":1,"correct":393,"invalid":0,"invalid_replies":null,"answered_questions":2210,"groups":{"reading":{"questions":656,"correct":112},"grammar":{"questions":1542,"correct":281},"without_suspect":{"questions":2186,"correct":390}},"tyt_mix":0.17274406061181233,"questions_per_minute":5364.397759879482,"request_seconds":24.71852497436339,"median_request_seconds":null,"p90_request_seconds":null,"billing":"local","api_equivalent_usd":0.01709697977393468,"cost_basis":"gpu","cost_upper_bound":false,"requests":2210,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB)","input_tokens":676255,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":{"run":42,"changed":{"questions":372,"older_correct":71,"newer_correct":69,"flipped":40,"changed_answer":91,"newer_wrong":303,"repeated_mistakes":231,"p":0.8746293123804207},"unchanged":{"questions":1814,"older_correct":320,"newer_correct":320,"flipped":0,"changed_answer":0,"newer_wrong":1494,"repeated_mistakes":1494,"p":1.0},"added":{"questions":12,"newer_correct":4}},"units":[25,19,11,21,20,16,26,8,14,12,16,11,24,11,17,42,13,29,20,38],"started":"2026-10-04T08:31:15Z","updated":"2026-10-04T08:31:44Z","format_of":null,"group":"ΞΠ","latency":{"source":"repeats","runs":[238,239,240,241],"repeats":[2,3,4,5],"questions":2198,"concurrency":1,"median_request_seconds":0.009499495485215448,"p90_request_seconds":0.010441777898813598},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":1,"correct":0,"interval":[0.0,0.7934567085261071]}}},{"run":122,"model":"gliner2.5-multi-v1","name":"GLiNER2.5 Multi","variant":null,"reasoning":null,"provider":"gliner","perplexity_mode":null,"note":"Multilingual extraction and classification model","parameters":287355159,"active_parameters":null,"effective_parameters":null,"parameters_scope":"mDeBERTa-v3-base encoder + extraction and classification heads","parameters_source":["counted from the checkpoint's safetensors, fastino/gliner2.5-multi-v1@2ca71aa","https://huggingface.co/fastino/gliner2.5-multi-v1"],"parameters_read":"2026-11-09","access":"GLiNER2 server, self-hosted","questions_per_request":1,"concurrency":1,"correct":384,"invalid":0,"invalid_replies":null,"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":113},"grammar":{"questions":1542,"correct":271},"without_suspect":{"questions":2186,"correct":382}},"tyt_mix":0.17286679280946507,"questions_per_minute":1922.5854452978283,"request_seconds":68.59513075090945,"median_request_seconds":0.03178766701603308,"p90_request_seconds":0.03576318579725921,"billing":"local","api_equivalent_usd":0.0129345,"cost_basis":"maker","cost_upper_bound":false,"requests":2198,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); FP16","input_tokens":431150,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":0,"previous":null,"units":[24,20,12,21,22,14,23,9,16,11,17,9,27,15,19,32,13,18,17,45],"started":"2026-10-05T19:34:39Z","updated":"2026-10-05T19:35:51Z","format_of":null,"group":"Π","latency":{"source":"run","runs":[122],"concurrency":1,"median_request_seconds":0.03178766701603308,"p90_request_seconds":0.03576318579725921},"single_question_latency":null,"tag":"","confidence":{"sure":{"questions":0,"correct":0,"interval":null}}},{"run":1299,"model":"minicpm5-2b-nothink","name":"MiniCPM5-2B","variant":null,"reasoning":"off","provider":"vllm-plain","perplexity_mode":null,"note":null,"parameters":2516756480,"active_parameters":null,"effective_parameters":null,"parameters_scope":"language model with its LM head, text only","parameters_source":["counted from the checkpoint's safetensors, openbmb/MiniCPM5-2B@42433e9","https://huggingface.co/openbmb/MiniCPM5-2B"],"parameters_read":"2026-11-12","setup_of":"minicpm5-2b","access":"our GPU (logic's server, vLLM v0.30.0 BF16, revision openbmb/MiniCPM5-2B@42433e9), one question per request, plain answer line CEVAP: X (no answer schema); sampling seed 0, temperature 1.0, top_p 0.95, thinking off; output cap 32,768","questions_per_request":1,"concurrency":32,"correct":123,"invalid":1654,"invalid_replies":{"invalid_without_rule":1654,"read_from_broken_replies":0,"read_without_confidence":0,"empty_replies_asked_again":1,"asked_again_with_answers":0,"answers_after_asking_again":0,"error_replies_asked_again":0,"invalid_by_kind":{"cut off":674,"unreadable":980}},"answered_questions":2198,"groups":{"reading":{"questions":656,"correct":43},"grammar":{"questions":1542,"correct":80},"without_suspect":{"questions":2186,"correct":122}},"tyt_mix":0.06315686193097339,"questions_per_minute":7.595782859539219,"request_seconds":17362.265672770984,"median_request_seconds":172.0228919145011,"p90_request_seconds":546.8574326845046,"billing":"local","api_equivalent_usd":12.0089004236666,"cost_basis":"gpu","cost_upper_bound":false,"requests":2199,"hardware":"1 × RTX PRO 6000 Blackwell (96 GB); BF16","input_tokens":844339,"cached_input_tokens":0,"cache_write_input_tokens":0,"cache_write_1h_input_tokens":0,"output_tokens":23351180,"previous":null,"units":[5,13,3,15,1,6,5,2,7,2,4,3,11,4,5,7,0,9,11,10],"started":"2026-11-15T15:41:37Z","updated":"2026-11-15T22:49:04Z","format_of":null,"group":"Σ","latency":{"source":"run","runs":[1299],"concurrency":32,"median_request_seconds":172.0228919145011,"p90_request_seconds":546.8574326845046},"single_question_latency":null,"tag":"reasoning off · chat"}]}
